今天帮同学处理数据, 主要是从1w多条记录中随机获取8k条, 然后再从8k条记录中随机获取2k条记录. 最后将2k条记录中随机分成10组,使得每组的记录都不重复.
下面将我的代码都贴上来, 好以后处理csv文件.
- 首先使用第三方的jar文件 javcsv.jar : 链接: http://pan.baidu.com/s/1qW5b3u0 密码: qjmx
- 虽然该类库可以相对方便提供操作, 但是为了方便处理, 我将处理的字段都放在配置文件中, 然后将每一条记录都封装为Map<String, String>对象,我将读写的基础类封装为 CSVBasic:
package spt.csv; import java.io.Serializable; import java.nio.charset.Charset; import spt.util.PropertyConfig; /** * CSV文件操作基础类. */ abstract public class CSVBasic implements Serializable { private Charset charset; //编码. private char delimiter; //分隔符. private String fileName; /** * 默认编码. * * @return */ public static Charset getDefaultCharset() { return Charset.forName(PropertyConfig.getProperty("charset")); } /** * 默认分割符. * * @return */ public static char getDefaultDelimiter() { return PropertyConfig.getProperty("delimiter").charAt(0); } public String getFileName() { return fileName; } public void setFileName(String fileName) { this.fileName = fileName; } public Charset getCharset() { return charset; } public void setCharset(Charset charset) { this.charset = charset; } public void setDelimiter(char delimiter) { this.delimiter = delimiter; } public char getDelimiter() { return delimiter; } public CSVBasic() {} /**使用默认的分隔符和编码. * @param fileName */ public CSVBasic(String fileName) { this(fileName, getDefaultDelimiter(), getDefaultCharset()); } public CSVBasic(String fileName, char delimiter, Charset charset) { setFileName(fileName); setDelimiter(delimiter); setCharset(charset); } /** * */ private static final long serialVersionUID = 7916808982930771124L; }3.读取csv文件,并映射记录为List<Map<String, String>> 对象:
package spt.csv; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import spt.util.PropertyConfig; import com.csvreader.CsvReader; /** * 读取csv文件的类. */ public class Reader extends CSVBasic { private CsvReader reader; public CsvReader getReader() { return reader; } public void setReader(CsvReader reader) { this.reader = reader; } public Reader(String fileName) throws FileNotFoundException { this(fileName, getDefaultDelimiter(), getDefaultCharset()); } public Reader(String fileName, char delimiter, Charset charset) throws FileNotFoundException { // set before getting. super(fileName, delimiter, charset); setReader(new CsvReader(fileName, delimiter, charset)); } /**根据字段列表,见每条记录映射为一个Map对象的列表. * @param fieldNames * 指定配置文件中字段名的‘键‘的列表. * @return */ public List<Map<String, String>> getResult(List<String> fieldNames) { // 每行中的每一个项是一个Map<String, String>的键值对. List<Map<String, String>> lines = new ArrayList<Map<String, String>>(); CsvReader r = null; try { r = getReader(); r.readHeaders(); // 读取表头. Map<String, String> itemMap = null; // 每一条记录是一个Map<String, String>. while (r.readRecord()) { itemMap = new HashMap<String, String>(); String k = null; // 每一条记录添加键值对. for (String fieldName : fieldNames) { // 字段名. k = PropertyConfig.getProperty(fieldName); itemMap.put(k, r.get(k)); } lines.add(itemMap); } return lines; } catch (IOException e) { e.printStackTrace(); return null; } finally { if(r != null) r.close(); } } @Override public String toString() { return getFileName(); } /** * */ private static final long serialVersionUID = -1712774594374451546L; }4.将List<Map<String, String>>输出为csv文件的类:
package spt.csv; import java.io.IOException; import java.nio.charset.Charset; import java.util.List; import java.util.Map; import com.csvreader.CsvWriter; /** * csv文件写入类. */ public class Writer extends CSVBasic { private CsvWriter writer = null; public boolean write(List<String> fieldNames, List<Map<String, String>> mapList) { CsvWriter writer = null; try { writer = getWriter(); // 写入表头. writer.writeRecord((String[]) fieldNames .toArray(new String[fieldNames.size()])); for (Map<String, String> map : mapList) { // 存储每行记录. String[] records = new String[fieldNames.size()]; for (int i = 0; i < fieldNames.size(); i++) records[i] = map.get(fieldNames.get(i)); // 写入每行记录. writer.writeRecord(records); } return true; } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); return false; } finally { if (writer != null) writer.close(); } } public Writer() { this(null, getDefaultDelimiter(), getDefaultCharset()); } public Writer(String fileName) { this(fileName, getDefaultDelimiter(), getDefaultCharset()); } public Writer(String fileName, char delimiter, Charset charset) { super(fileName, delimiter, charset); writer = new CsvWriter(fileName, delimiter, charset); } public CsvWriter getWriter() { return writer; } public void setWriter(CsvWriter writer) { this.writer = writer; } /** * */ private static final long serialVersionUID = -9141083858975437622L; }5.表中有一个字段NYR, 表示时间, 由于需要将结果按照时间的先后顺序排序, 所以定义一个比较器:
package spt.csv; import java.text.ParseException; import java.util.Comparator; import java.util.Map; import spt.util.DateService; import spt.util.PropertyConfig; /** *每条记录是一个Map对象,按照每条记录中的‘时间‘的列进行排序. */ public class RecordDateComparator implements Comparator<Map<String, String>> { @Override public int compare(Map<String, String> m1, Map<String, String> m2) { try { long l01 = DateService.getDate(m1.get(PropertyConfig.getProperty("NYR"))).getTime(); long l02 = DateService.getDate(m2.get(PropertyConfig.getProperty("NYR"))).getTime(); //long的范围和int的范围不同. long diff = l01 - l02; if(diff < 0) return -1; else if(diff > 0) return 1; return 0; } catch (ParseException e) { e.printStackTrace(); return 0; } } }6,在main类中:
package spt.csv; import java.io.File; import java.io.FileNotFoundException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Random; import spt.util.PropertyConfig; /** * 从1w多条记录中先选出8k条,然后在8k条记录中选出2k条,最后将2k条记录分成10组. */ public class ReadWriteDemo { /** * @param args */ public static void main(String[] args) { // if (args.length < 1) // throw new NullPointerException("请指定文件路径"); System.out.println("执行中...执行过程请不要关闭此窗口!"); final int first_size = Integer.parseInt(PropertyConfig .getProperty("first_size")); // 初次提取长度(8k). final int second_size = Integer.parseInt(PropertyConfig .getProperty("second_size")); // 初次提取(2k). final int groupCount = Integer.parseInt(PropertyConfig .getProperty("groupCount")); // 分组个数(10). String file = PropertyConfig.getProperty("input_file"); // 源文件路径. List<String> fieldNames = null; try { fieldNames = initFields(); Reader csv = new Reader(file); // 总记录. List<Map<String, String>> totalList = csv.getResult(fieldNames); // 初次提取的值(8k). List<Map<String, String>> firstTaken = random(totalList, first_size); // 再次提取的值(2k). List<Map<String, String>> secondTaken = random(firstTaken, second_size); // 每组记录数(2百). List<Map<String, String>> tmpTaken = secondTaken; for (int i = 0; i < groupCount; i++) { List<Map<String, String>> AGroupTaken = random(tmpTaken, second_size / groupCount); // 除去上次已经使用的元素. tmpTaken.removeAll(AGroupTaken); // 在当前目录上输出(并验证是否存在). String outputFile = null; // 如果文件已存在,则自动命名. int fileCount = 0; do { outputFile = "result" + fileCount++ + ".csv"; } while (new File(outputFile).exists()); Writer writer = new Writer(outputFile); // (集合)排序. Collections.sort(AGroupTaken, new RecordDateComparator()); writer.write(fieldNames, AGroupTaken); } System.out.println("done!"); } catch (FileNotFoundException e) { System.out.println("请指定正确的文件路径!"); // TODO Auto-generated catch block e.printStackTrace(); } } /** * 随机产生新的列表(长度比原来小). * * @param originalList * 输入列表. * @param new_size * 新列表的长度. */ public static List<Map<String, String>> random( List<Map<String, String>> originalList, int new_size) { if (new_size <= 0 || new_size > originalList.size()) throw new IndexOutOfBoundsException("新列表的长度错误!"); List<Map<String, String>> newList = new ArrayList<Map<String, String>>( new_size); // 标识是否已被提取. boolean[] taken = new boolean[originalList.size()]; Random r = new Random(); Map<String, String> map = null; // 即将获取的元素. int rIdx = 0; for (int i = 0; i < new_size; i++) { do { rIdx = r.nextInt(new_size); map = originalList.get(rIdx); } while (taken[rIdx]); // 如果发现已经提取,则重复操作. taken[rIdx] = true; // 标识已被提取. newList.add(map); } return newList; } private static List<String> initFields() { // 所有字段. List<String> fieldNames = new ArrayList<String>(14); fieldNames.add("id"); fieldNames.add("AJMC"); fieldNames.add("JYAQ"); fieldNames.add("AJLB"); fieldNames.add("AJFAB"); fieldNames.add("AJZT"); fieldNames.add("BASJ"); fieldNames.add("FXSJ"); fieldNames.add("FASJSX"); fieldNames.add("FASJXX"); fieldNames.add("AJBH"); fieldNames.add("ZBX"); fieldNames.add("ZBY"); fieldNames.add("NYR"); return fieldNames; } }7,用到的自定义工具类为:
package spt.util; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; /** * 2015-2-27 提供日期转换的工具类. */ public class DateService { // 定义称线程共享,而不是没调用一次就创建一个对象. private static DateFormat formater = new SimpleDateFormat(PropertyConfig.getProperty("date_format")); /** * 将字符串类型的日期转换为Date. * * @param strDate * @return * @throws ParseException */ public static Date getDate(String strDate) throws ParseException { // 如果输入为空,则返回null. if (Str.isEmpty(strDate)) return null; return formater.parse(strDate); } /** * 将java.util.Date转换为java.sql.Date;用于诸如‘PreparedStatement.setDate‘方法. * * @param utilDate * @return */ public static java.sql.Date getSQLDate(java.util.Date utilDate) { if (utilDate == null) return null; return new java.sql.Date(utilDate.getTime()); } /** * 将指定的日期转换为 * * @param date * @return */ public static String getDateStr(java.util.Date date) { if (date == null) return null; return formater.format(date); } /** * 计算指定日期与今天的间隔,判断是否是需要日期. disDay表示与今天相隔天数,0:等于今天;1:明天;-1:昨天. * * @param anotherDate * @param disDay * @return */ public static boolean isSpecifiedDay(Date anotherDate, int disDay) { if (anotherDate == null) return false; Calendar cNow = Calendar.getInstance(); cNow.setTime(new Date()); // 每调用一次,都是与当前时间做比较. cNow.add(Calendar.DAY_OF_MONTH, disDay); Calendar cAnotherDate = Calendar.getInstance(); cAnotherDate.setTime(anotherDate); return cNow.get(Calendar.YEAR) == cAnotherDate.get(Calendar.YEAR) && cNow.get(Calendar.MONTH) == cAnotherDate.get(Calendar.MONTH) && cNow.get(Calendar.DAY_OF_MONTH) == cAnotherDate.get(Calendar.DAY_OF_MONTH); } }package spt.util; import java.io.IOException; import java.net.URL; import java.util.Properties; /** * 2015-2-27 */ public class PropertyConfig { /** * @param key * @return */ public static String getProperty(String key) { Properties properties = getProperties(); return properties.getProperty(key); } /** * @param resources * @return */ public static Properties getProperties() { final String configFilePath = "raw/properties.properties"; URL url = PropertyConfig.class.getClassLoader().getResource(configFilePath); Properties props = new Properties(); try { props.load(url.openStream()); } catch (IOException e) { e.printStackTrace(); return null; } return props; } }package spt.util; /** *字符串工具类. */ public class Str { /** * 判断一个字符串是否有内容. * * @param str * @return 如果不不为空,则返回true,否则返回false. */ public static boolean hasLength(String str) { return !isEmpty(str); } /**判断字符串是否为空. * @param str * @return */ public static boolean isEmpty(String str) { return str == null || str.isEmpty(); } }其中,配置文件"raw/properties.properties"是放置在src目录下.
时间： 2024-10-13 20:58:14