package org.phoenix.cases.kafka;import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; import org.junit.Test; /** * 内存映射文件方式处理文件的实例 * @author mengfeiyang * */ public class MappedBytes { String filePath = "E:\\工作目录\\新项目\\凤舞一期\\backup\\new_show_style_json.txt"; /** * 内存映射文件方式处理文件的实例 * @throws FileNotFoundException * @throws IOException */ @SuppressWarnings("resource") @Test public void testMappedByte() throws FileNotFoundException, IOException { long start = System.currentTimeMillis(); File file = new File(filePath); long fileLength = file.length(); final int BUFFER_SIZE=0x500000;//5M MappedByteBuffer inputBuffer = new RandomAccessFile(file,"rw").getChannel().map(FileChannel.MapMode.READ_WRITE, 0, fileLength); byte[] dst = new byte[BUFFER_SIZE]; int count=0; for(int offset=0;offset<fileLength;offset += BUFFER_SIZE){ if (fileLength - offset >= BUFFER_SIZE) { for (int i = 0; i < BUFFER_SIZE; i++) dst[i] = inputBuffer.get(offset + i); } else { for (int i = 0; i < fileLength - offset; i++) dst[i] = inputBuffer.get(offset + i); } String bs = new String(dst,"UTF-8");//将buffer中的字节转成字符串 String[] ns = bs.split("\n"); for(String s : ns){ if(s.contains("-1-")){ count++; System.out.println(s.split("-1-")[0]); } } System.out.println(); //String s = IOUtils.toString(new ByteArrayInputStream(dst)); //System.out.println(s); } System.out.println("总处理条数:"+count); long end = System.currentTimeMillis(); System.out.println((end - start)/1000);//处理809M的文件,90000条数据,只用了6秒 } } }
时间: 2024-11-03 13:11:03