其他分享
首页 > 其他分享> > 使用反射机制编写数据与文件映射工具

使用反射机制编写数据与文件映射工具

作者:互联网

背景

 

思路

 

实现

 <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
<!DOCTYPE context [
   <!ELEMENT context (fileMapper | fileTemplate)*>
   <!ELEMENT fileMapper (mapper+)>
   <!ELEMENT mapper EMPTY>
   <!ATTLIST fileMapper id ID #REQUIRED>
   <!ATTLIST fileMapper isFileHead (true|false) "true">
   <!ATTLIST fileMapper separator CDATA #REQUIRED>
   <!ATTLIST fileMapper charEncod CDATA "UTF-8">
   <!ATTLIST fileMapper fileTemplate CDATA #IMPLIED >
   <!ATTLIST mapper entityAttribute CDATA #REQUIRED>
   <!ATTLIST mapper fileAttribute CDATA #REQUIRED>
]>

<context>
    <!--  
        一, fileMapper属性注释: 
            id = "配置的唯一值, 根据ID可从容器中获取该条配置"
            isFileHead = "是否存在文件头 true存在, false不存在"
            separator = "每一列数据的分隔符"
            charEncod = "文件编码"
        二, mapper属性注释
            entityAttribute = "Java类的属性名"
            fileAttribute = "文件的列名"
    -->
    <fileMapper id="userInfo_MIIT" isFileHead="false" separator="," charEncod="UTF-8" >
        <mapper entityAttribute="name" fileAttribute="fullName" />
        <mapper entityAttribute="age" fileAttribute="age" />
        <mapper entityAttribute="sex" fileAttribute="gender" />
        <mapper entityAttribute="phone" fileAttribute="phoneNumber" />
        <mapper entityAttribute="company" fileAttribute="organize" />
        <mapper entityAttribute="department" fileAttribute="department" />
        <mapper entityAttribute="location" fileAttribute="province" />
    </fileMapper>
    
</context>
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.stereotype.Component;

import com.ctid.core.exception.ServiceException;
import com.ctid.mainlib.system.util.datehandle.DateHandle;
import com.ctid.mainlib.system.util.datehandle.PathHandler;
import com.ctid.mainlib.system.util.datehandle.StringHandle;
import com.ctid.mainlib.system.util.file.FileMapping.Context.FileMapper;

/**
 * @功能描述 文件映射工具类
 * @author 索睿君
 * @date 2019年7月5日14:55:35
 */
public class FileMapping {
    private static final Log Log = LogFactory.getLog(FileMapping.class);
    @Component public static class FileMapperContext{
        private final static String CLASSPATH = "com/config/fileMapper/fileMapper.xml";
        private final static Log Log = LogFactory.getLog(FileMapperContext.class);
        public final static Map<String, Context.FileMapper> MAPPER_CONTEXT = new HashMap<>();
        public final static Map<String,BufferedWriter> BufferedWriter_CONTEXT = new HashMap<>();
        static{
            try {
                File file = new File(Context.class.getClassLoader().getResource(CLASSPATH).getFile());
                JAXBContext jaxbContext = JAXBContext.newInstance(Context.class);
                Context context = (Context) jaxbContext.createUnmarshaller().unmarshal(file);
                context.fileMapperList.forEach(fileMapper -> {
                    fileMapper.mapperList.forEach(mapper -> {
                        fileMapper.mapper.put(mapper.entityAttribute, mapper.fileAttribute);
                    });
                    MAPPER_CONTEXT.put(fileMapper.id, fileMapper);
                    //创建文件写入流
                    if(fileMapper.fileTemplate != null && fileMapper.fileTemplate.trim().length() > 0){
                        try {
                            String collect = fileMapper.mapper.values().stream().collect(Collectors.joining(fileMapper.separator));
                            fileMapper.filePath = initFile(fileMapper.fileTemplate);
                            if(!new File(fileMapper.filePath).isFile()){
                                PathHandler.initPath(fileMapper.filePath);
                                BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(fileMapper.filePath,true));
                                BufferedWriter_CONTEXT.put(fileMapper.id, bufferedWriter);
                                if(fileMapper.isFileHead){
                                    bufferedWriter.write(collect + System.getProperty("line.separator"));        //生成文件头
                                    bufferedWriter.flush();
                                }
                            }else{
                                PathHandler.initPath(fileMapper.filePath);
                                BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(fileMapper.filePath,true));
                                BufferedWriter_CONTEXT.put(fileMapper.id, bufferedWriter);
                            }
                        } catch (Exception e) {
                            Log.error(e);
                        }
                    }
                    Log.info("=============文件映射配置[" + fileMapper.id + "]加载完毕!~");
                });
            } catch (Exception e) {
                Log.error("文件映射配置加载异常, 嵌套异常: \n" + e.toString());
            }
        }
        public static String initFile(String fileTemplate) throws ServiceException {
            try {
                if(fileTemplate == null || fileTemplate.trim().length() < 1){
                    return null;
                }
                return StringHandle.dateFormatPath(fileTemplate, new Date()) + ".MQ";
            } catch (Exception e) {
                throw new ServiceException("静态缓冲输出流加载异常, 嵌套异常: \n" + e.toString());
            }
        }
    }
    
    /**
     * @功能描述 读取多个路径
     * @param mapperKey
     * @param filePaths
     * @param clazz
     * @return
     * @throws ServiceException
     */
    public static <T> List<T> readFiles(String mapperKey, String[] filePaths , Class<T> clazz) throws ServiceException {
        if(filePaths  == null || filePaths.length < 1){
            throw new ServiceException("文件读取数据异常, 异常信息: 无效路径!~");
        }
        ArrayList<T> arrayList = new ArrayList<>();
        for (String filePath : filePaths) {
            List<T> readFile = readFile(mapperKey,filePath, clazz);
            if(readFile != null && readFile.size() > 0){
                arrayList.addAll(readFile);
            }
        }
        return arrayList;
    }
    
    /**
     * @功能描述 将一行数据写入文件
     * @param mapperKey
     * @param filePath
     * @param datas
     * @throws ServiceException
     */
    public static void writeFile(String mapperKey, String filePath, String[] datas) throws ServiceException {
        //参数验证
        if (mapperKey == null || mapperKey.trim().length() < 1 || filePath == null || !PathHandler.initPath(filePath).isFile()) {
            throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
        }
        //获取分隔符, 字符编码
        FileMapper fileMapper = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
        String separator = mapperKey;
        Charset charEncod = Charset.forName("UTF-8");
        if(fileMapper != null){
            separator = fileMapper.separator;
            charEncod = Charset.forName(fileMapper.charEncod);
        }
        String fileLine = Stream.of(datas).collect(Collectors.joining(separator));
        try(BufferedWriter writer = Files.newBufferedWriter(Paths.get(filePath), charEncod)){
            writer.write(fileLine);
        } catch (IOException e) {
            throw new ServiceException("数据写入文件IO异常, 嵌套异常: " + e.toString());
        }
    }
    /**
     * @param <T>
     * @功能描述 文件读取数据
     * @param mapperKey
     * @param filePath
     * @return
     * @throws ServiceException
     */
    public static < T> List<T> readFile(String mapperKey,String filePath,Class<T> javaClass) throws ServiceException{
        //参数验证
        if(mapperKey == null || mapperKey.trim().length() < 1 || !new File(filePath).isFile()){
            throw new ServiceException("文件读取数据异常, 异常信息: 非法参数!~");
        }
        long startTime = System.currentTimeMillis();
        Log.info("开始读取文件["+PathHandler.getFileName(filePath)+"]");
        try {
            //获取映射
            Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
            if(fileMapperObject == null){
                throw new ServiceException("文件读取数据异常, 异常信息: 无效的文件映射键!~");
            }
            Charset charEncod = Charset.forName(fileMapperObject.charEncod);            //获取字符编码
            Map<String, String> fileMapperMap = fileMapperObject.mapper;                        //获取文件对象映射
            String[]  variableArray = fileMapperMap.keySet().toArray(new String[]{});    //获取属性数组
            String separator = fileMapperObject.separator;                                                            //获取文件列分隔符
            boolean isFileHead = fileMapperObject.isFileHead;                                                    //文件是否存文件头
            //读取文件
            List<String> fileContext = Files.readAllLines(Paths.get(filePath),charEncod);    
            if(fileContext == null || fileContext.size() < 1) return null;
            List<T> dataList = new LinkedList<>();
            //开始解析
            for (int lineIndex = isFileHead ? 1 : 0; lineIndex < fileContext.size(); lineIndex++) {
                //获取行内容
                String fileLine =  fileContext.get(lineIndex);
                T newInstance = javaClass.newInstance();
                if(fileLine == null || fileLine.trim().length() < 1) continue;
                String[] fileColumn = fileLine.split("["+separator+"]");
                for (int columnIndex = 0; columnIndex < fileColumn.length; columnIndex++) {
                    Field declaredField = javaClass.getDeclaredField(variableArray[columnIndex]);
                    declaredField.setAccessible(true);
                    declaredField.set(newInstance, baseClass(fileColumn[columnIndex], declaredField.getType()) );
                    declaredField.setAccessible(false);
                }
                dataList.add(newInstance);
            }
            fileContext.clear();
            long endTime = System.currentTimeMillis();
            Log.info("文件["+PathHandler.getFileName(filePath)+"]读取完毕, 耗时: " + (endTime - startTime) + "ms");
            return dataList;
        } catch (IOException e) {
            throw new ServiceException("文件读取数据IO读取异常, 异常信息: " + e.toString());
        } catch (InstantiationException | IllegalAccessException e) {
            throw new ServiceException("文件读取数据反射对象异常, 异常信息: " + e.toString());
        }  catch (NoSuchFieldException | SecurityException e) {
            throw new ServiceException("文件读取数据反射属性异常, 异常信息: " + e.toString());
        } 
    }
    /**
     * @功能描述 数据写入文件
     * @param mapperKey
     * @param filePath
     * @param dataList
     * @throws ServiceException
     */
    public static <T> void writeFile(String mapperKey, String filePath, List<T> dataList) throws ServiceException {
        // 参数验证
        if (mapperKey == null || mapperKey.trim().length() < 1 || filePath == null || !PathHandler.initPath(filePath).isFile()) {
            throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
        }
        long startTime = System.currentTimeMillis();
        Log.info("开始写入文件["+PathHandler.getFileName(filePath)+"]");
        // 获取映射
        Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
        if (fileMapperObject == null) {
            throw new ServiceException("数据写入文件异常, 异常信息: 无效的文件映射键!~");
        }
        Charset charEncod = Charset.forName(fileMapperObject.charEncod);                //字符编码
        boolean isFileHead = fileMapperObject.isFileHead;                                                        //是否拥有文件头
        String separator = fileMapperObject.separator;                                                                //列分隔符
        Map<String, String> mapper = fileMapperObject.mapper;                                            //属性文件映射
        String fileHeads = mapper.values().stream().collect(Collectors.joining(separator));    //文件头
        List<String> variableName = mapper.keySet().stream().collect(Collectors.toList());    //属性名
        try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(filePath),charEncod)) {
            if(isFileHead) writer.write(fileHeads + System.getProperty("line.separator"));
            if(dataList == null || dataList.size() < 1) return;
            for (T data : dataList) {
                ArrayList<String> variableValueList = new ArrayList<>();
                Class<? extends Object> clazz = data.getClass();
                for (String variable : variableName) {
                    Field declaredField = clazz.getDeclaredField(variable);
                    declaredField.setAccessible(true);
                    Object object = declaredField.get(data);
                    if(object == null){
                        variableValueList.add("");
                    }else if(declaredField.getType().getName().equals(Date.class.getName())){
                        String format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(object);
                        variableValueList.add(format);
                    }else{
                        variableValueList.add(String.valueOf(object));
                    }
                }
                String fileLine = variableValueList.stream().collect(Collectors.joining(separator));
                writer.write(fileLine + System.getProperty("line.separator"));
            }
            Long endTime = System.currentTimeMillis();
            Log.info("文件["+PathHandler.getFileName(filePath)+"]写入结束, 耗时: " + (endTime - startTime) + "ms");
        } catch (IOException e) {
            throw new ServiceException("数据写入文件IO异常, 嵌套异常: " + e.toString());
        } catch (NoSuchFieldException | SecurityException e) {
            throw new ServiceException("数据写入文件反射属性异常, 嵌套异常: " + e.toString());
        } catch (IllegalArgumentException | IllegalAccessException e) {
            throw new ServiceException("数据写入文件读取属性异常, 嵌套异常: " + e.toString());
            
        } 
    }
    
    /**
     * @功能描述 将单条记录写入文件(续写)
     * @param mapperKey
     * @param filePath
     * @param data
     * @throws ServiceException
     * @throws  
     */
    public static <T> String writeFile(String mapperKey, T data) throws Exception {
        // 参数验证
        if (mapperKey == null || mapperKey.trim().length() < 1 || data == null) {
            throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
        }
        // 获取映射
        Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
        BufferedWriter bufferedWriter = FileMapperContext.BufferedWriter_CONTEXT.get(mapperKey);
        if (fileMapperObject == null) {
            throw new ServiceException("数据写入文件异常, 异常信息: 无效的文件映射键!~");
        }
        String separator = fileMapperObject.separator;                                                                            //列分隔符
        Map<String, String> mapper = fileMapperObject.mapper;                                        //属性文件映射
        List<String> variableName = mapper.keySet().stream().collect(Collectors.toList());    //属性名
        Class<? extends Object> clazz = data.getClass();
        try {
            List<String> fileLine = new LinkedList<>();
            for (String variable : variableName) {
                Field declaredField = clazz.getDeclaredField(variable);
                declaredField.setAccessible(true);
                Object variableValue = declaredField.get(data);
                if(variableValue == null){
                    fileLine.add("");
                }else if(declaredField.getType().getName().equals(Date.class)){
                    fileLine.add(new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(variableValue));
                }else{
                    fileLine.add(variableValue.toString());
                }
                declaredField.setAccessible(false);
            }
            String filePath = fileMapperObject.filePath;
            Date charExtractDate = StringHandle.charExtractDate(filePath);
            if(!DateHandle.dateEquals(charExtractDate,new Date())){
                bufferedWriter.close();
                String collect = mapper.values().stream().collect(Collectors.joining(separator));
                fileMapperObject.filePath = FileMapperContext.initFile(fileMapperObject.fileTemplate);
                PathHandler.initPath(fileMapperObject.filePath);
                bufferedWriter = new BufferedWriter(new FileWriter(filePath,true));
                if(fileMapperObject.isFileHead){
                    bufferedWriter.write(collect + System.getProperty("line.separator"));        //生成文件头
                }
                FileMapperContext.BufferedWriter_CONTEXT.put(mapperKey, bufferedWriter);
            }
            bufferedWriter.write(fileLine.stream().collect(Collectors.joining(separator)) + System.getProperty("line.separator"));
            return  fileMapperObject.filePath;
        }catch (Exception e) {
            throw new ServiceException("数据文件续写执行异常, 嵌套异常: \n" + e.toString());
        }finally {
            bufferedWriter.flush();
        }
    }
    /**
     * @功能描述 文件映射上下文
     * @author 索睿君
     * @date 2019年7月5日10:01:21
     */
    @XmlAccessorType(XmlAccessType.FIELD)
    @XmlRootElement(name = "context")
    public static class Context {
        @XmlElement(name = "fileMapper")
        public List<FileMapper> fileMapperList;

        @XmlAccessorType(XmlAccessType.FIELD)
        @XmlRootElement(name = "fileMapper")
        public static class FileMapper {
            /** 唯一标识 */
            @XmlAttribute
            private String id;
            /** 是否拥有文件头 (true是false否) */
            @XmlAttribute
            private boolean isFileHead;
            /** 每列的分隔符 */
            @XmlAttribute
            private String separator;
            /** 字符编码 */
            @XmlAttribute
            private String charEncod = "UTF-8";
            /** 文件路径 */
            @XmlAttribute
            private String fileTemplate;
            private String filePath;
            /** Java属性名与文件头名称映射 */
            @XmlElement(name = "mapper")
            private List<Mapper> mapperList;
            private Map<String, String> mapper = new LinkedHashMap<>();
            public String getFileTemplate() {
                return fileTemplate;
            }
            @XmlAccessorType(XmlAccessType.FIELD)
            @XmlRootElement(name = "mapper")
            public static class Mapper {
                @XmlAttribute
                private String entityAttribute;
                @XmlAttribute
                private String fileAttribute;
            }
            @Override
            public String toString() {
                return "FileMapper [id=" + id + ", isFileHead=" + isFileHead + ", separator=" + separator
                        + ", charEncod=" + charEncod + ", fileTemplate=" + fileTemplate + ", filePath=" + filePath
                        + ", mapperList=" + mapperList + ", mapper=" + mapper + "]";
            }
        }
    }
    
    @SuppressWarnings("unchecked")
    public static <T> T baseClass(Object value,Class<?> classType) {
        try {
            if(String.class.equals(classType)){
                return (T) new String(value.toString());
            }
            if(Character.class.equals(classType) || "char".equals(classType.getName())){
                return (T) new Character(value.toString().charAt(0));
            }
            if(Integer.class.equals(classType) || "int".equals(classType.getName())){
                return (T) new Integer(value.toString());
            }
            if(Long.class.equals(classType) || "long".equals(classType.getName())){
                return (T) new Long(value.toString());
            }
            if(Short.class.equals(classType) || "short".equals(classType.getName())){
                return (T) new Short(value.toString());
            }
            if(Double.class.equals(classType) || "double".equals(classType.getName())){
                return (T) new Double(value.toString());
            }
            if(Float.class.equals(classType) || "float".equals(classType.getName())){
                return (T) new Float(value.toString());
            }
            if(Boolean.class.equals(classType) || "boolean".equals(classType.getName())){
                return (T) new Boolean(value.toString());
            }
            if(Date.class.equals(classType)){
                return (T) DateHandle.format(value.toString());
            }
            throw new ClassCastException(value+"转换为" + classType+"类型出现异常,未收录的类型!");
        } catch (Exception e) {
            throw new ClassCastException(value+"转换为" + classType+"类型出现异常," + e.toString());
        }
    }
}

 

使用

//生成文件
FileMapping.writeFile('配置ID','文件路径', new List<数据模型>list);
//读取文件
List<数据模型> list = FileMapping.readFile('配置ID','文件路径',数据模型.class);

 

总结

生成500万数据耗时13113ms, 文件大小352MB

读取500万数据待测试

标签:反射,ServiceException,String,映射,filePath,import,文件,new,编写
来源: https://www.cnblogs.com/suoruijun/p/suoruijun.html