POISAXReader
h2:first-child, body>h1:first-child, body>h1:first-child+h2, body>h3:first-child, body>h4:first-child, body>h5:first-child, body>h6:first-child {
margin-top: 0;
padding-top: 0;
}
a:first-child h1, a:first-child h2, a:first-child h3, a:first-child h4, a:first-child h5, a:first-child h6 {
margin-top: 0;
padding-top: 0;
}
h1+p, h2+p, h3+p, h4+p, h5+p, h6+p {
margin-top: 10px;
}
/* LINKS
=============================================================================*/
a {
color: #4183C4;
text-decoration: none;
}
a:hover {
text-decoration: underline;
}
/* LISTS
=============================================================================*/
ul, ol {
padding-left: 30px;
}
ul li > :first-child,
ol li > :first-child,
ul li ul:first-of-type,
ol li ol:first-of-type,
ul li ol:first-of-type,
ol li ul:first-of-type {
margin-top: 0px;
}
ul ul, ul ol, ol ol, ol ul {
margin-bottom: 0;
}
dl {
padding: 0;
}
dl dt {
font-size: 14px;
font-weight: bold;
font-style: italic;
padding: 0;
margin: 15px 0 5px;
}
dl dt:first-child {
padding: 0;
}
dl dt>:first-child {
margin-top: 0px;
}
dl dt>:last-child {
margin-bottom: 0px;
}
dl dd {
margin: 0 0 15px;
padding: 0 15px;
}
dl dd>:first-child {
margin-top: 0px;
}
dl dd>:last-child {
margin-bottom: 0px;
}
/* CODE
=============================================================================*/
pre, code, tt {
font-size: 12px;
font-family: Consolas, "Liberation Mono", Courier, monospace;
}
code, tt {
margin: 0 0px;
padding: 0px 0px;
white-space: nowrap;
border: 1px solid #eaeaea;
background-color: #f8f8f8;
border-radius: 3px;
}
pre>code {
margin: 0;
padding: 0;
white-space: pre;
border: none;
background: transparent;
}
pre {
background-color: #f8f8f8;
border: 1px solid #ccc;
font-size: 13px;
line-height: 19px;
overflow: auto;
padding: 6px 10px;
border-radius: 3px;
}
pre code, pre tt {
background-color: transparent;
border: none;
}
kbd {
-moz-border-bottom-colors: none;
-moz-border-left-colors: none;
-moz-border-right-colors: none;
-moz-border-top-colors: none;
background-color: #DDDDDD;
background-image: linear-gradient(#F1F1F1, #DDDDDD);
background-repeat: repeat-x;
border-color: #DDDDDD #CCCCCC #CCCCCC #DDDDDD;
border-image: none;
border-radius: 2px 2px 2px 2px;
border-style: solid;
border-width: 1px;
font-family: "Helvetica Neue",Helvetica,Arial,sans-serif;
line-height: 10px;
padding: 1px 4px;
}
/* QUOTES
=============================================================================*/
blockquote {
border-left: 4px solid #DDD;
padding: 0 15px;
color: #777;
}
blockquote>:first-child {
margin-top: 0px;
}
blockquote>:last-child {
margin-bottom: 0px;
}
/* HORIZONTAL RULES
=============================================================================*/
hr {
clear: both;
margin: 15px 0;
height: 0px;
overflow: hidden;
border: none;
background: transparent;
border-bottom: 4px solid #ddd;
padding: 0;
}
/* TABLES
=============================================================================*/
table th {
font-weight: bold;
}
table th, table td {
border: 1px solid #ccc;
padding: 6px 13px;
}
table tr {
border-top: 1px solid #ccc;
background-color: #fff;
}
table tr:nth-child(2n) {
background-color: #f8f8f8;
}
/* IMAGES
=============================================================================*/
img {
max-width: 100%
}
-->
解决POI读取XLSX文件内存占用过过多
poi处理excel分别提供比较友好的用户模式以及比较底层的事件模式。其中，用户模式提供良好的封装，同时兼容2003以及2007以上的格式，使用相当方便。不过，代价是花费巨大的内存。只要超过6w条以后，基本是就是内存溢出了。
好在POI团队也提供了更底层的的流处理模式eventMode,对于大数据的Xlsx文件的写入，poi 3.8 提供SXSSF，采用缓存方式写如文件。对于文件的读取采用sax的方式直接读取每个sheet对应的xml文件。
POI SheetContentsHandler 接口
在POI中已经对SAX当时读取对应的Sheet的xml文件已经做了基本的封装，所以我们仅仅需要实现接口SheetContentsHandler,就可以完成SAX的方式读取。这个接口中需要是实现三个方法
- public void startRow(int rowNum) 读取某行开始
- public void endRow(int rowNum) 读取某行结束
- public void cell(String cellReference, String formattedValue,XSSFComment comment) 读取某行中的单元格
- public void headerFooter(String text, boolean isHeader, String tagName) 暂时不清楚
POI SheetContentsHandler实现
这里我主要参照poi XLSX2CSV.java实现方式，需要提供对应的xlsx文件最大列数。其次，我在此基础上做了扩展，在 endRow 提供了一个事件，当前处理的的行数据,让这个解析功能更加独立。
实现思路，在startRow方法中构造一个List对象，在cell函数中添加每个单元内容，在endRow函数中判断当前列是否等于最大列数，如果不等循环补齐，并出发添加行事件
SheetSaxHandler详细代码
protected class SheetSaxHandler implements SheetContentsHandler {
private int currentRow = -1;
private int currentCol = -1;
private int minColumns;
public void setMinColumns(int minColumns) {
this.minColumns = minColumns;
}
public SheetSaxHandler(int minColumns) {
super();
this.minColumns = minColumns;
}
public SheetSaxHandler() {
}
private List<SheetRowListener> listeners = new ArrayList<SheetRowListener>();
private List<String> lRows = new ArrayList<String>(); // 处理一行信息
public void rowAdded(SheetRowListener add) {
listeners.add(add);
}
private void postRowAdded(List<String> row, int rowNum)
throws SQLException {
for (SheetRowListener hl : listeners)
hl.addRow(row, rowNum);
}
@Override
public void startRow(int rowNum) {
currentRow = rowNum;
currentCol = -1;
lRows.clear();
}
@Override
public void endRow(int rowNum) {
// 添加数据
for (int i = currentCol; i < minColumns; i++) {
lRows.add("");
}
try {
postRowAdded(lRows, rowNum);
} catch (SQLException e) {
e.printStackTrace();
}
}
@Override
public void cell(String cellReference, String formattedValue,
XSSFComment comment) {
if (cellReference == null) {
cellReference = new CellAddress(currentRow, currentCol)
.formatAsString();
}
int thisCol = (new CellReference(cellReference)).getCol();
int missedCols = thisCol - currentCol - 1;//处理数据中间存在空白
for (int i = 0; i < missedCols; i++) {
this.lRows.add("");
}
currentCol = thisCol;
// TODO 数据类型处理
try {
Double.parseDouble(formattedValue);
this.lRows.add(formattedValue);
} catch (NumberFormatException e) {
this.lRows.add(formattedValue);
}
}
@Override
public void headerFooter(String text, boolean isHeader, String tagName) {
System.out.println(text + "==" + isHeader + "==" + tagName);
}
}
事件接口
interface SheetRowListener {
void addRow(List<String> row, int rowNum);
}
调用方式
- 打开文件
- 找到对应sheet的xml文件
- 使用上边的方法依次处理每一个sheet
处理文件
@Override
public int saveToOracle(String filePath, String pcId)
throws FileNotFoundException, EncryptedDocumentException,
InvalidFormatException, IOException, ClassNotFoundException,
SQLException, OpenXML4JException, SAXException,
ParserConfigurationException {
File f = new File(filePath);
OPCPackage p = null;
int num = 0;
Connection conn = null;
if (f.exists()) {
try {
JSONArray sheetCfgs = this.cfgJson.getJSONArray("sheets");
dataBuferRows = this.cfgJson.getInteger("dataBuferRows");
dataBuferRows = dataBuferRows == null ? 1000 : dataBuferRows;
conn = ca.getConnection(ca.getSqlCfg(serverPath));
String importTime = new SimpleDateFormat(
"yyyy-MM-dd HH:mm:ss.SSS").format(new Date());
p = OPCPackage.open(f, PackageAccess.READ);
ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(
p);
XSSFReader xssfReader = new XSSFReader(p);
StylesTable styles = xssfReader.getStylesTable();
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader
.getSheetsData();
HashMap<Integer, JSONObject> hSheetCfg = new HashMap<Integer, JSONObject>();
for (int i = 0; i < sheetCfgs.size(); i++) {
JSONObject sheetCfg = sheetCfgs.getJSONObject(i);
hSheetCfg.put(sheetCfg.getInteger("sheetIndex"), sheetCfg);
}
int index = 1;
while (iter.hasNext()) {
InputStream sheetStream = iter.next();
if (hSheetCfg.containsKey(index)) {
processSheet(styles, strings, new SheetSaxHandler(),
sheetStream, hSheetCfg.get(index), conn, pcId,
this.fileName, importTime);
}
index++;
}
p.close();
f = null;
conn.close();
} catch (SQLException e) {
conn.close();
conn = null;
throw e;
}
}
return num;
}
处理Sheet
public void processSheet(StylesTable styles,
ReadOnlySharedStringsTable strings, SheetSaxHandler sheetHandler,
InputStream sheetInputStream, final JSONObject sheetCfg,
final Connection conn, String PcID, String fileName,
String importTime) throws IOException,
ParserConfigurationException, SAXException, SQLException {
final PreparedStatement ps = conn.prepareStatement(ca.buildInsertSql(
sheetCfg, PcID, fileName, importTime));
final int dataStartNum = sheetCfg.getIntValue("dataStartNum");
sheetHandler.setMinColumns(sheetCfg.getJSONArray("fieldReference")
.size());
sheetHandler.rowAdded(new SheetRowListener() {
@Override
public void addRow(List<String> row, int rowNum) {
if (rowNum < dataStartNum - 1)
return;
try {
ca.setParamter(ps, sheetCfg, row, rowNum - dataStartNum);
if (rowNum % dataBuferRows == 0) {
ps.executeBatch();
ps.clearBatch();
}
} catch (SQLException e) {
try {
ps.close();
conn.close();
throw e;
} catch (SQLException e1) {
e1.printStackTrace();
}
e.printStackTrace();
}
}
});
XMLReader sheetParser = SAXHelper.newXMLReader();
DataFormatter formatter = new DataFormatter();
InputSource sheetSource = new InputSource(sheetInputStream);
ContentHandler handler = new XSSFSheetXMLHandler(styles, null, strings,
sheetHandler, formatter, false);
sheetParser.setContentHandler(handler);
sheetParser.parse(sheetSource);
// 处理剩下的数据
ps.executeBatch();
ps.clearBatch();
// 关闭当前ps
ps.close();
}
`
总结
在最初使用poi的用户模式，很快的就完成一个excel文件的解析，很方便。随着项目的逐渐深入，处理的excel文件越来越大，用户模式已经不能胜任。于是开始查找资料，在官网上看到了转csv的实例。
这段代码的主要功能将excel文件中的数据导入到oracle数据库对应的表中，在实现功能方面，我主要遇到了以下问题
- 解决excel文件解析内存泄露（2007以后文件采用sax方式基本解决）
- 对应大量数据的保存，速度一直很慢，尽管我这里采用了批量提交的方式（目前这问题我依然没找到很好的方案，如果有同行看到的，还希望多多指教）