Oracle快速导出数据文件

数据文件生成方式有多种，但是大多效率低下，对于大型数据仓库来说，高效导出文件是迫切需求：

这里通过shell+java的方式集成老熊写的数据文件生成工具，效率比普通的导出方法高70%：

shell文件生成脚本：

Java调用脚本：这里implement Callable接口，实现了多线程的调用方式：

/**
 * @Title: Db2FileSync.java
 * @Package etl.etlUtils
 * @Description: TODO
 * @author Lubin Su
 * @date 2015年7月15日 下午12:43:09
 * @version V1.0
 */
package etl.etlUtils;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.Callable;

import etl.dao.impl.GenericDaoImpl;

/**
 *
 * @author Lubin Su
 * @date 2015年7月15日 下午12:43:09
 * @version V1.0
 */
public class Db2FileSync implements Callable<Object> {

	private String procId;
	private String srcDb;
	private String destDir;
	private String srcSql;
	private String fileNm;
	private String statCycleId;

	/**
	 * <p>Title: 构造函数</p>
	 * <p>Description: </p>
	 * @param procId
	 * @param srcDb
	 * @param destDir
	 * @param srcSql
	 * @param fileNm
	 * @param statCycleId
	 */
	public Db2FileSync(String procId, String srcDb, String destDir,
			String srcSql, String fileNm, String statCycleId) {
		super();
		this.procId = procId;
		this.srcDb = srcDb;
		this.destDir = destDir;
		this.srcSql = srcSql;
		this.fileNm = fileNm;
		this.statCycleId = statCycleId;
	}
	/**
	 * 数据表以文件的方式下发
	 * @author lubinsu
	 * @date 2015年7月12日
	 * @param procId
	 * @throws UnsupportedEncodingException
	 */
	public void db2FileExtractByProcInfo(String srcDb, String descDir, String srcSql, String fileNm) throws UnsupportedEncodingException {
		//获取当前时间
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		Date vStartTime = new Date();
		System.out.println("Start sending ... " + sdf.format(vStartTime));
		String[] cmd = new String[5];
		//环境变量
		String[] env = {"lubinsu_dir=/int_file/lubinsu","AUTHSTATE=compat","TERM=vt100","SHELL=/usr/bin/bash","NLS_LANG=SIMPLIFIED CHINESE_CHINA.ZHS16GBK","SSH_CLIENT=132.228.78.73 49796 22","SSH_TTY=/dev/pts/4","LOCPATH=/usr/lib/nls/loc","USER=css_int","ODMDIR=/etc/objrepos","ORACLE_BASE=/oracle/app/oracle","TMOUT=0","MAIL=/usr/spool/mail/css_int","PATH=:/usr/java6_64/bin:/opt/freeware/bin:/usr/local/bin:/bin:/usr/bin:/usr/sbin:/oracle/app/oracle/product/11.2.0/dbhome_2/bin:/home/css_app/bin:/oracle/app/oracle/product/11.2.0/dbhome_2/bin:/urs/ccs/bin:/bin:/usr/bin:/usr/bin:/etc:/usr/sbin:/usr/ucb:/usr/bin/X11:/sbin:/usr/java5/jre/bin:/usr/java5/bin:.","css_int_HOME=/int_file","LOGIN=css_int","PWD=/int_file/lubinsu/scripts/etl_dispatch/css_etl","JAVA_HOME=:/usr/java6_64","EDITOR=vi","JAVA_COMPILER=NONE","LANG=AMERICAN_AMERICA.ZHS16GBK","etl_dir=/int_file/etl_dir","TZ=Asia/Shanghai","LIB_PATH=/oracle/app/oracle/product/11.2.0/dbhome_2/lib:/oracle/app/oracle/product/11.2.0/dbhome_2/rdbms/lib:/usr/lib:/lib","SHLVL=1","HOME=/int_file","etl_dispatch=/int_file/lubinsu/scripts/etl_dispatch/css_etl","LC__FASTMSG=true","MAILMSG=[YOU HAVE NEW MAIL]","LOGNAME=css_int","CLASSPATH=::/usr/java6_64/lib","SSH_CONNECTION=132.228.78.73 49796 132.228.27.132 22","CLCMD_PASSTHRU=1","SHLIB_PATH=/oracle/app/oracle/product/11.2.0/dbhome_2/lib:/oracle/app/oracle/product/11.2.0/dbhome_2/rdbms/lib","ODS_HOME=/int_file/ODS_ETL/","ORACLE_HOME=/oracle/app/oracle/product/11.2.0/dbhome_2","_=/bin/env","OLDPWD=/int_file/lubinsu/etl_data","NLSPATH=/usr/lib/nls/msg/%L/%N:/usr/lib/nls/msg/%L/%N.cat","LD_LIBRARY_PATH=/oracle/app/oracle/product/11.2.0/dbhome_2/bin:/oracle/app/oracle/product/11.2.0/dbhome_2/lib:/oracle/app/oracle/product/11.2.0/dbhome_2/rdbms/lib:/usr/lib:/lib"};
		Process prc = null;
		int exitVal = 0;
		cmd[0] = "/int_file/lubinsu/scripts/etl_dispatch/css_etl/java_etl/db2File.sh";
		cmd[1] = srcDb;
		cmd[2] = descDir;
		cmd[3] = srcSql;
		cmd[4] = fileNm;
		try {
			prc = Runtime.getRuntime().exec(cmd, env);
		} catch (IOException e) {
			e.printStackTrace();
		}

		InputStream stderr = prc.getErrorStream();
		InputStreamReader isr = new InputStreamReader(stderr);
		BufferedReader br = new BufferedReader(isr);
		String line = null;
		//打印返回信息
		try {
			while ((line = br.readLine()) != null)
				System.out.println(line);
		} catch (IOException e) {
			e.printStackTrace();
		}
		try {
			exitVal = prc.waitFor();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		//打印返回号
		if (exitVal > 0) {
			System.out.println("Process exitValue: " + exitVal);
		}
		//获取结束时间
		Date vEndTime = new Date();
		System.out.println("End sending ... " + sdf.format(vEndTime) + ",elapsed time: " + (vEndTime.getTime() - vStartTime.getTime())/1000 + " seconds.");
	}
	/**
	 *
	 * @return
	 * @throws Exception
	 * @see java.util.concurrent.Callable#call()
	 */
	@Override
	public Object call() throws Exception {
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
		Date vStartDt = new Date();
		GenericDaoImpl.update("INSERT INTO shell_proc_db_2_db_log(proc_id, run_msg, status, stat_cycle_id) VALUES(?, ?, ?, ?)", new String[]{procId, sdf.format(vStartDt) + " 文件开始导出", "2", statCycleId});
		db2FileExtractByProcInfo(srcDb, destDir, srcSql, fileNm.toUpperCase() + "_" + statCycleId + ".DEL");

		GenericDaoImpl.update("UPDATE shell_proc_db_2_db_log o SET o.status = 0, modify_dt = SYSDATE, run_msg = o.run_msg || CHR(13) || to_char(SYSDATE, 'yyyy/mm/dd hh24:mi:ss') || ' 文件生成结束' WHERE proc_id = ? and stat_cycle_id = ?", new String[] { procId, statCycleId});
		GenericDaoImpl.update("UPDATE shell_proc_db_2_db_cfg o SET o.last_succ_cycle = ? WHERE o.proc_id = ?", new String[] { statCycleId, procId});
		return null;
	}

}

获取配置的流程信息：

public static List<Map<String, Object>> getProcIdForDb2File() {
		//获取当前时间
		SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HH:mm:ss");
		Date vNowDate = new Date();
		String vNowDateStr = sdf.format(vNowDate);
		ResultSetHandler rsh = new BeanListHandler();
		String vRunSql = "SELECT proc_id, src_db, src_sql, dest_dir, file_nm, stat_cycle_id\n" +
						"  FROM (SELECT o.run_type,\n" +
						"               o.proc_id,\n" +
						"               o.src_db,\n" +
						"               o.src_sql,\n" +
						"               o.dest_dir,\n" +
						"               o.file_nm,\n" +
						"               o.last_succ_cycle,\n" +
						"               CASE\n" +
						"                 WHEN o.run_type = 2 THEN\n" +
						"                  to_char(to_date('" + vNowDateStr + "', 'yyyymmdd hh24:mi:ss') - 1, 'yyyymmdd')\n" +
						"                 WHEN o.run_type = 3 THEN\n" +
						"                  to_char(to_date('" + vNowDateStr + "', 'yyyymmdd hh24:mi:ss') - 1, 'yyyymmdd')\n" +
						"                 WHEN o.run_type = 4 THEN\n" +
						"                  to_char(to_date('" + vNowDateStr + "', 'yyyymmdd hh24:mi:ss') - o.run_hour, 'yyyymmdd')\n" +
						"                 WHEN o.run_type = 1 THEN\n" +
						"                  to_char(add_months(to_date('" + vNowDateStr + "', 'yyyymmdd hh24:mi:ss'), -1), 'yyyymm')\n" +
						"               END stat_cycle_id\n" +
						"          FROM shell_proc_db_2_db_cfg o\n" +
						"         WHERE o.etl_type = 'Db2File'\n" +
						"           AND o.state = 'A'\n" +
						"           AND CASE\n" +
						"                 WHEN o.run_type = 2 THEN\n" +
						"                  to_date(to_char(to_date('" + vNowDateStr + "', 'yyyymmdd hh24:mi:ss'), 'yyyymmdd') || to_char(o.run_hour || o.run_minute), 'yyyymmddhh24mi')\n" +
						"                 WHEN o.run_type = 3 THEN\n" +
						"                  to_date(to_char(to_date('" + vNowDateStr + "', 'yyyymmdd hh24:mi:ss'), 'yyyymmdd') || to_char(o.run_hour || o.run_minute), 'yyyymmddhh24mi')\n" +
						"                 WHEN o.run_type = 4 THEN\n" +
						"                  to_date(to_char(to_date('" + vNowDateStr + "', 'yyyymmdd hh24:mi:ss'), 'yyyymmdd') || to_char(o.run_hour || o.run_minute), 'yyyymmddhh24mi')\n" +
						"                 WHEN o.run_type = 1 THEN\n" +
						"                  to_date(to_char(to_date('" + vNowDateStr + "', 'yyyymmdd hh24:mi:ss'), 'yyyymm') || o.run_day || o.run_hour || o.run_minute, 'yyyymmddhh24mi')\n" +
						"               END <= SYSDATE) v\n" +
						" WHERE v.last_succ_cycle < v.stat_cycle_id\n" +
						"   AND NOT EXISTS (SELECT 1\n" +
						"          FROM shell_proc_db_2_db_log i\n" +
						"         WHERE i.proc_id = v.proc_id\n" +
						"           AND i.stat_cycle_id = v.stat_cycle_id)";

		List<Map<String, Object>> arr = null;
		try {
			arr = (List<Map<String, Object>>) GenericDaoImpl.query(vRunSql, null, rsh);
		} catch (SQLException e) {
			e.printStackTrace();
		}
		return arr;
	}

调用测试：

public static void main(String[] args) throws IOException, InterruptedException {
		int taskSize = 13;
		// 创建一个线程池
		ExecutorService pool = Executors.newFixedThreadPool(taskSize);
		// 创建多个有返回值的任务
		//List<Future> list = new ArrayList<Future>();

		/*List<Map<String, Object>> procs = DataSynchronizer.getProcIdForDb2Db();
		for (Map<String, Object> map : procs) {
			Callable<Object> db2Db = new Db2DbSync(map.get("PROC_ID").toString(), map.get("PROC_NAME").toString());
			pool.submit(db2Db);
		}*/

		List<Map<String, Object>> db2Files = DataSynchronizer.getProcIdForDb2File();
		for (Map<String, Object> map : db2Files) {
			Callable<Object> db2File = new Db2FileSync(map.get("PROC_ID").toString(), map.get("SRC_DB").toString(), map.get("DEST_DIR").toString(), map.get("SRC_SQL").toString(), map.get("FILE_NM").toString(), map.get("STAT_CYCLE_ID").toString());
			pool.submit(db2File);
		}
		pool.shutdown();
	}

版权声明：本文为博主原创文章，未经博主允许不得转载。

时间： 2024-08-10 19:18:17

Oracle快速导出数据文件的相关文章

SqlServer导出数据文件(图)

1.对数据库右键->Tasks->Generate Scripts 2.点击Next 3.选择是导出整个数据库开始单个数据表 4.设置成保存为文件,并设置存储位置,然后点击高级按钮 5.根据选择是导出模式.模式加数据.还是仅数据,点击ok 6.会有让你review的一个窗口,点击next,再点击finish 7.以下是我只导出数据的文件结果 转载请注明:康瑞的部落 ? SqlServer导出数据文件

关于oracle 11g导出数据时 报 ORA 1455错误的处理

由于导出的该用户的表可能存在空数据表,那么可能就会出现此其异常. 首先: 查看: SQL>show parameter deferred_segment_creation; 如果为TRUE,则将该参数改为FALSE: 在sqlplus中,执行如下命令: SQL>alter system set deferred_segment_creation=false; 然后: 可以针对数据表.索引.物化视图等手工分配Extent SQL>Select 'alter table '||table_n

oracle 11g dbf数据文件从C盘迁移到D盘

服务器系统为 windows 2008 R2 64位,由于C盘空间将满,要将C盘的oracle的DBF数据文件迁移到D盘下,步骤如下: 1.输入cmd,启动 cmd.exe窗口 2.输入 sqlplus /nolog 3.输入 conn sys/[email protected] as sysdba 4.输入sql:select name from v$datafile; 查看相关data文件存放路径 5. shutdown immediate 关闭 6. startup mount; 7. a

oracle的dmp数据文件的导出和导入以及创建用户

关于dmp文件我们用的还是比较多的,dmp文件它是作为oracle导入和导出表使用的文件格式,今天就将dmp文件导出和导入进行学习. dmp文件导出 dmp文件导出用的比较多的一般是三种,他们分别是:导出整个数据库实例下的所有数据.导出指定用户的所有表.导出指定表. 这里已我的数据库为例,进行介绍,我的Oracle数据库实例为"ORACLE",查看自己数据库实例可以从"任务管理器-->服务"中进行查看,如下图 打开命令行: 1,将数据库ORACLE完全导出,用

Oracle 导入导出数据 imp/exp impdp/expdp

IMPDP/EXPDP 一.创建逻辑目录,该命令不会在操作系统创建真正的目录,最好以system等管理员创建.         create directory dpdata as '/opt'; 二.查看管理理员目录(同时查看操作系统是否存在,因为Oracle并不关心该目录是否存在,如果不存在,则出错)         select * from dba_directories; 三.给scott用户赋予在指定目录的操作权限,最好以system等管理员赋予.         grant rea

oracle 导入导出数据

数据导出: 1 将数据库TEST完全导出,用户名system 密码manager 导出到D:/daochu.dmp中      exp system/manager@TEST file=d:/daochu.dmp full=y (远程:exp jeesite/[email protected]:1521/ORCL file=d:/daochu.dmp owner=(jeesite))2 将数据库中system用户与sys用户的表导出   exp system/manager@TEST file=

ORACLE导入导出数据dmp

imp testwms3/[email protected] FILE=C:\ZKGL_201407012334.dmp ignore=y fromuser=GMMCZKGL touser=testwms tables=(GMMCZKGL.t_kdttrcktz)imp testapar/[email protected] FILE=C:\CWYSYF_201511292330.dmp ignore=y fromuser=GMMCCWYSYF touser=testapar tables=(GM

Oracle—— 导入/导出 数据：exp，imp 命令

exp,imp 命令是需要在windows  的 cmd  命令中执行的命令,主要用于数据的导入和导出工作,方便高效. 远程地址--  @ip:port/orcl   注:该远程地址不写,就意味着执行本地的库. Oracle的数据导出 (1)导出数据的提示模式: exp  jl_bhps/[email protected]:1521/orcl (2)表方式:将指定表的数据导出. exp  jl_bhps/[email protected]:1521/orcl  file=d:temp/1.dmp

sketch 快速导出psd文件

也许是世界上最早讲这个事情的教程.sketch快速导出为psd文件.知乎,豆瓣,fb,quroa...都暂时没有 故事情节是这样的.最近优狐哥用sketch迅速的做了一些app案子--大概几周就是几套200p的工作量.问题来了,在对接某些开发交接的时候,他 们  要 psd !!!! 这是每个sketcher都会苦恼的.优狐哥本着"为设计人民服务"的社会主义理想,悬梁刺骨,在历经2个小时,借助Googl+蝌蚪文,终于破解此世界难题. 此方法,乃优狐哥sketch秘籍第一大招,绝不外传.