package com.wecash.bi.util; import java.io.File; import java.io.FileOutputStream; import java.net.URI; import java.util.List; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class HdfsUtils { public static Configuration conf = new Configuration(); static{ conf.addResource(new Path(Config.instace().getProperty("hadoop") + "core-site.xml")); conf.addResource(new Path(Config.instace().getProperty("hadoop") + "hdfs-site.xml")); conf.reloadConfiguration(); } public static void write(List data, String path) { Configuration conf = new Configuration(); conf.addResource(new Path(Config.instace().getProperty("hadoop") + "core-site.xml")); conf.addResource(new Path(Config.instace().getProperty("hadoop") + "hdfs-site.xml")); conf.reloadConfiguration(); FileSystem fs = null; FSDataOutputStream f = null; try { fs = FileSystem.get(conf); if (fs.exists(new Path(path))) { f = fs.append(new Path(new URI(path))); } else { f = fs.create(new Path(new URI(path)), false); } for (Object object : data) { f.write(object.toString().getBytes()); } f.close(); fs.close(); } catch (Exception e) { throw new RuntimeException(e); } finally { IOUtils.closeQuietly(f); IOUtils.closeQuietly(fs); } } public static void write(String data, String path) { FileSystem fs = null; FSDataOutputStream f = null; try { fs = FileSystem.get(conf); if (fs.exists(new Path(path))) { f = fs.append(new Path(new URI(path))); } else { f = fs.create(new Path(new URI(path)), false); } f.write(data.getBytes()); f.hsync(); } catch (Exception e) { e.printStackTrace(); } finally { IOUtils.closeQuietly(f); IOUtils.closeQuietly(fs); } } public static void writeLocal(String data, String path) { FileOutputStream fo = null; try { fo = new FileOutputStream(new File(path),true); fo.write(data.getBytes()); } catch (Exception e) { e.printStackTrace(); } finally { IOUtils.closeQuietly(fo); } } }
时间: 2024-10-05 19:29:56