switch(type) {
            case NODES:
                return writableNodeIndices;
            case RELATIONSHIPS:
                return writableRelationshipIndices;
            default:
                throw new IllegalArgumentException("No such fulltext index type: " + type);
        }

/*
 *
 * This file is part of ONgDB Enterprise Edition. The included source
 */
/*
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://example.com>.
 */
package org.neo4j.kernel.api.impl.fulltext;

import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
import java.util.function.Function;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Resource;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.helpers.collection.Iterables;
import org.neo4j.internal.kernel.api.InternalIndexState;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.AvailabilityGuard;
import org.neo4j.kernel.api.exceptions.InvalidArgumentsException;
import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
import org.neo4j.kernel.impl.transaction.state.NeoStoreFileListing;
import org.neo4j.kernel.impl.util.MultiResource;
import org.neo4j.logging.Log;
import org.neo4j.scheduler.JobScheduler;
import org.neo4j.storageengine.api.StoreFileMetadata;
import static org.neo4j.kernel.impl.transaction.state.NeoStoreFileListing.getSnapshotFilesMetadata;

/**
 * Provider class that manages and provides fulltext indices. This is the main entry point for the fulltext addon.
 */
public class FulltextProviderImpl implements FulltextProvider {

    private final GraphDatabaseService db;

    private final Log log;

    private final TransactionIdStore transactionIdStore;

    private final FulltextTransactionEventUpdater fulltextTransactionEventUpdater;

    private final Set<String> nodeProperties;

    private final Set<String> relationshipProperties;

    private final Map<String, WritableFulltext> writableNodeIndices;

    private final Map<String, WritableFulltext> writableRelationshipIndices;

    private final FulltextUpdateApplier applier;

    private final FulltextFactory factory;

    private final ReadWriteLock configurationLock;

    /**
     * Creates a provider of fulltext indices for the given database. This is the entry point for all fulltext index
     * operations.
     * @param db Database that this provider should work with.
     * @param log For logging errors.
     * @param availabilityGuard Used for waiting with populating the index until the database is available.
     * @param scheduler For background work.
     * @param transactionIdStore Used for checking if the store has had transactions applied to it, while the fulltext
     * @param fileSystem The filesystem to use.
     * @param storeDir Store directory of the database.
     * @param analyzerClassName The Lucene analyzer to use for the {@link LuceneFulltext} created by this factory.
     */
    public FulltextProviderImpl(GraphDatabaseService db, Log log, AvailabilityGuard availabilityGuard, JobScheduler scheduler, TransactionIdStore transactionIdStore, FileSystemAbstraction fileSystem, File storeDir, String analyzerClassName) {
        this.db = db;
        this.log = log;
        this.transactionIdStore = transactionIdStore;
        applier = new FulltextUpdateApplier(log, availabilityGuard, scheduler);
        applier.start();
        factory = new FulltextFactory(fileSystem, storeDir, analyzerClassName);
        fulltextTransactionEventUpdater = new FulltextTransactionEventUpdater(this, applier);
        nodeProperties = ConcurrentHashMap.newKeySet();
        relationshipProperties = ConcurrentHashMap.newKeySet();
        writableNodeIndices = new ConcurrentHashMap<>();
        writableRelationshipIndices = new ConcurrentHashMap<>();
        configurationLock = new ReentrantReadWriteLock(true);
    }

    @Override
    public void registerTransactionEventHandler() {
        db.registerTransactionEventHandler(fulltextTransactionEventUpdater);
    }

    private boolean matchesConfiguration(WritableFulltext index) throws IOException {
        long txId = transactionIdStore.getLastCommittedTransactionId();
        FulltextIndexConfiguration currentConfig = new FulltextIndexConfiguration(index.getAnalyzerName(), index.getProperties(), txId);
        FulltextIndexConfiguration storedConfig;
        try (ReadOnlyFulltext indexReader = index.getIndexReader()) {
            storedConfig = indexReader.getConfigurationDocument();
        }
        return storedConfig == null && index.getProperties().isEmpty() || storedConfig != null && storedConfig.equals(currentConfig);
    }

    @Override
    public void awaitPopulation() {
        try {
            applier.writeBarrier().awaitCompletion();
        } catch (ExecutionException e) {
            throw new AssertionError("The writeBarrier operation should never throw an exception", e);
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    @Override
    public void openIndex(String identifier, FulltextIndexType type) throws IOException {
        LuceneFulltext index = factory.openFulltextIndex(identifier, type);
        register(index);
    }

    @Override
    public void createIndex(String identifier, FulltextIndexType type, List<String> properties) throws IOException {
        LuceneFulltext index = factory.createFulltextIndex(identifier, type, properties);
        register(index);
    }

    private void register(LuceneFulltext fulltextIndex) throws IOException {
        configurationLock.writeLock().lock();
        try {
            WritableFulltext writableFulltext = new WritableFulltext(fulltextIndex);
            writableFulltext.open();
            if (fulltextIndex.getType() == FulltextIndexType.NODES) {
                if (!matchesConfiguration(writableFulltext)) {
                    writableFulltext.drop();
                    writableFulltext.open();
                    if (!writableFulltext.getProperties().isEmpty()) {
                        applier.populateNodes(writableFulltext, db);
                    }
                }
                writableNodeIndices.put(fulltextIndex.getIdentifier(), writableFulltext);
                nodeProperties.addAll(fulltextIndex.getProperties());
            } else {
                if (!matchesConfiguration(writableFulltext)) {
                    writableFulltext.drop();
                    writableFulltext.open();
                    if (!writableFulltext.getProperties().isEmpty()) {
                        applier.populateRelationships(writableFulltext, db);
                    }
                }
                writableRelationshipIndices.put(fulltextIndex.getIdentifier(), writableFulltext);
                relationshipProperties.addAll(fulltextIndex.getProperties());
            }
        } finally {
            configurationLock.writeLock().unlock();
        }
    }

    String[] getNodeProperties() {
        return nodeProperties.toArray(new String[0]);
    }

    String[] getRelationshipProperties() {
        return relationshipProperties.toArray(new String[0]);
    }

    Collection<WritableFulltext> writableNodeIndices() {
        return Collections.unmodifiableCollection(writableNodeIndices.values());
    }

    Collection<WritableFulltext> writableRelationshipIndices() {
        return Collections.unmodifiableCollection(writableRelationshipIndices.values());
    }

    @Override
    public ReadOnlyFulltext getReader(String identifier, FulltextIndexType type) throws IOException {
        WritableFulltext writableFulltext = getIndexMap(type).get(identifier);
        if (writableFulltext == null) {
            throw new IllegalArgumentException("No such " + type + " index '" + identifier + "'.");
        }
        return writableFulltext.getIndexReader();
    }

    private Map<String, WritableFulltext> getIndexMap(FulltextIndexType type) {

    }

    @Override
    public Set<String> getProperties(String identifier, FulltextIndexType type) {
        return applyToMatchingIndex(identifier, type, WritableFulltext::getProperties);
    }

    private <E> E applyToMatchingIndex(String identifier, FulltextIndexType type, Function<WritableFulltext, E> function) {
        if (type == FulltextIndexType.NODES) {
            return function.apply(writableNodeIndices.get(identifier));
        } else {
            return function.apply(writableRelationshipIndices.get(identifier));
        }
    }

    @Override
    public InternalIndexState getState(String identifier, FulltextIndexType type) {
        return applyToMatchingIndex(identifier, type, WritableFulltext::getState);
    }

    void drop(String identifier, FulltextIndexType type) throws IOException {
        configurationLock.writeLock().lock();
        try {
            // Wait for the queue of updates to drain, before deleting an index.
            awaitPopulation();
            if (type == FulltextIndexType.NODES) {
                writableNodeIndices.remove(identifier).drop();
            } else {
                writableRelationshipIndices.remove(identifier).drop();
            }
            rebuildProperties();
        } finally {
            configurationLock.writeLock().unlock();
        }
    }

    private void rebuildProperties() {
        nodeProperties.clear();
        relationshipProperties.clear();
        writableNodeIndices.forEach((s, index) -> nodeProperties.addAll(index.getProperties()));
        writableRelationshipIndices.forEach((s, index) -> relationshipProperties.addAll(index.getProperties()));
    }

    Lock readLockIndexConfiguration() {
        Lock lock = configurationLock.readLock();
        lock.lock();
        return lock;
    }

    @Override
    public void changeIndexedProperties(String identifier, FulltextIndexType type, List<String> propertyKeys) throws IOException, InvalidArgumentsException {
        configurationLock.writeLock().lock();
        try {
            if (propertyKeys.stream().anyMatch(s -> s.startsWith(FulltextProvider.LUCENE_FULLTEXT_ADDON_PREFIX))) {
                throw new InvalidArgumentsException("It is not possible to index property keys starting with " + FulltextProvider.LUCENE_FULLTEXT_ADDON_PREFIX);
            }
            Set<String> currentProperties = getProperties(identifier, type);
            if (!currentProperties.containsAll(propertyKeys) || !propertyKeys.containsAll(currentProperties)) {
                drop(identifier, type);
                createIndex(identifier, type, propertyKeys);
            }
        } finally {
            configurationLock.writeLock().unlock();
        }
    }

    @Override
    public void registerFileListing(NeoStoreFileListing fileListing) {
        fileListing.registerStoreFileProvider(this::snapshotStoreFiles);
    }

    private Resource snapshotStoreFiles(Collection<StoreFileMetadata> files) throws IOException {
        final Collection<ResourceIterator<File>> snapshots = new ArrayList<>();
        for (WritableFulltext index : Iterables.concat(writableNodeIndices.values(), writableRelationshipIndices.values())) {
            // Save the last committed transaction, then drain the update queue to make sure that we have applied _at least_ the commits the config claims.
            index.saveConfiguration(transactionIdStore.getLastCommittedTransactionId());
            try {
                applier.writeBarrier().awaitCompletion();
            } catch (ExecutionException e) {
                throw new IOException("Unable to prepare index for snapshot.");
            }
            index.flush();
            ResourceIterator<File> snapshot = index.snapshot();
            snapshots.add(snapshot);
            files.addAll(getSnapshotFilesMetadata(snapshot));
        }
        // Intentionally don't close the snapshots here, return them for closing by the consumer of the targetFiles list.
        return new MultiResource(snapshots);
    }

    /**
     * Closes the provider and all associated resources.
     */
    @Override
    public void close() {
        db.unregisterTransactionEventHandler(fulltextTransactionEventUpdater);
        applier.stop();
        Consumer<WritableFulltext> fulltextCloser = luceneFulltextIndex -> {
            try {
                luceneFulltextIndex.saveConfiguration(transactionIdStore.getLastCommittedTransactionId());
                luceneFulltextIndex.close();
            } catch (IOException e) {
                log.error("Unable to close fulltext index.", e);
            }
        };
        writableNodeIndices.values().forEach(fulltextCloser);
        writableRelationshipIndices.values().forEach(fulltextCloser);
    }
}


switch(c) {
                        case 0:
                            iter_dict = idfs_dict_neg.entrySet().iterator();
                            break;
                        case 1:
                            iter_dict = idfs_dict_pos.entrySet().iterator();
                            break;
                    }

package TIDF;

import java.io.*;
import java.util.*;
import org.wltea.analyzer.lucene.IKAnalyzer;

/**
 * 分词-TFIDF-信息增益
 * @author LJ
 *
 * @datetime 2015-6-15
 */
public class TestTfIdf {

    // 加载停用词库
    public static final String stopWordTable = "C:/Users/zzw/Desktop/sc_ot-tingyongzhongwen_hc/stopWordTable.txt";

    // 文件列表
    private static ArrayList<String> FileList = new ArrayList<String>();

    // 递归读取该路径下文件返回文件列表
    public static List<String> readDirs(String filepath) throws FileNotFoundException, IOException {
        try {
            File file = new File(filepath);
            if (!file.isDirectory()) {
                System.out.println("输入的[]");
                System.out.println("filepath:" + file.getAbsolutePath());
            } else {
                String[] flist = file.list();
                for (int i = 0; i < flist.length; i++) {
                    File newfile = new File(filepath + "\\" + flist[i]);
                    if (!newfile.isDirectory()) {
                        FileList.add(newfile.getAbsolutePath());
                    } else if (newfile.isDirectory()) {
                        readDirs(filepath + "\\" + flist[i]);
                    }
                }
            }
        } catch (FileNotFoundException e) {
            System.out.println(e.getMessage());
        }
        return FileList;
    }

    // 读入文件
    public static String readFile(String file) throws FileNotFoundException, IOException {
        StringBuffer strSb = new StringBuffer();
        InputStreamReader inStrR = new InputStreamReader(new FileInputStream(file), "gbk");
        BufferedReader br = new BufferedReader(inStrR);
        String line = br.readLine();
        while (line != null) {
            strSb.append(line).append("\r\n");
            line = br.readLine();
        }
        return strSb.toString();
    }

    // 分词处理
    public static ArrayList<String> cutWords(String file) throws IOException {
        ArrayList<String> fenci = new ArrayList<String>();
        ArrayList<String> words = new ArrayList<String>();
        String text = TestTfIdf.readFile(file);
        IKAnalyzer analyzer = new IKAnalyzer();
        // 分词处理
        fenci = analyzer.split(text);
        BufferedReader StopWordFileBr = new BufferedReader(new InputStreamReader(new FileInputStream(new File(stopWordTable))));
        // 用来存放停用词的集合
        Set<String> stopWordSet = new HashSet<String>();
        // 初如化停用词集
        String stopWord = null;
        for (; (stopWord = StopWordFileBr.readLine()) != null; ) {
            stopWordSet.add(stopWord);
        }
        for (String word : fenci) {
            if (stopWordSet.contains(word)) {
                continue;
            }
            words.add(word);
        }
        System.out.println(words);
        return words;
    }

    // 统计一个文件中每个词出现的次数
    public static HashMap<String, Integer> normalTF(ArrayList<String> cutwords) {
        HashMap<String, Integer> resTF = new HashMap<String, Integer>();
        for (String word : cutwords) {
            if (resTF.get(word) == null) {
                resTF.put(word, 1);
                System.out.println(word);
            } else {
                resTF.put(word, resTF.get(word) + 1);
                System.out.println(word.toString());
            }
        }
        System.out.println(resTF);
        return resTF;
    }

    // 计算一个文件每个词tf值
    @SuppressWarnings("unchecked")
    public static HashMap<String, Float> tf(ArrayList<String> cutwords) {
        HashMap<String, Float> resTF = new HashMap<String, Float>();
        int wordLen = cutwords.size();
        HashMap<String, Integer> intTF = TestTfIdf.normalTF(cutwords);
        Iterator iter = intTF.entrySet().iterator();
        while (iter.hasNext()) {
            Map.Entry entry = (Map.Entry) iter.next();
            resTF.put(entry.getKey().toString(), Float.parseFloat(entry.getValue().toString()) / wordLen);
            System.out.println(entry.getKey().toString() + " = " + Float.parseFloat(entry.getValue().toString()) / wordLen);
        }
        return resTF;
    }

    // tf times for file 。。。。。。。
    public static HashMap<String, HashMap<String, Integer>> normalTFAllFiles(String dirc) throws IOException {
        FileList.clear();
        HashMap<String, HashMap<String, Integer>> allNormalTF = new HashMap<String, HashMap<String, Integer>>();
        List<String> filelist = TestTfIdf.readDirs(dirc);
        for (String file : filelist) {
            HashMap<String, Integer> dict = new HashMap<String, Integer>();
            ArrayList<String> cutwords = TestTfIdf.cutWords(file);
            dict = TestTfIdf.normalTF(cutwords);
            allNormalTF.put(file, dict);
        }
        return allNormalTF;
    }

    // 返回所有文件tf值
    public static HashMap<String, HashMap<String, Float>> tfAllFiles(String dirc) throws IOException {
        FileList.clear();
        HashMap<String, HashMap<String, Float>> allTF = new HashMap<String, HashMap<String, Float>>();
        List<String> filelist = TestTfIdf.readDirs(dirc);
        for (String file : filelist) {
            HashMap<String, Float> dict = new HashMap<String, Float>();
            ArrayList<String> cutwords = TestTfIdf.cutWords(file);
            dict = TestTfIdf.tf(cutwords);
            allTF.put(file, dict);
        }
        return allTF;
    }

    // 计算该目录下所有词的idf
    @SuppressWarnings("unchecked")
    public static HashMap<String, Float> idf(HashMap<String, HashMap<String, Float>> all_tf, String file) throws IOException {
        FileList.clear();
        HashMap<String, Float> resIdf = new HashMap<String, Float>();
        HashMap<String, Integer> dict = new HashMap<String, Integer>();
        int docNum = readDirs(file).size();
        for (int i = 0; i < docNum; i++) {
            HashMap<String, Float> temp = all_tf.get(FileList.get(i));
            Iterator iter = temp.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String word = entry.getKey().toString();
                if (dict.get(word) == null) {
                    dict.put(word, 1);
                } else {
                    dict.put(word, dict.get(word) + 1);
                }
            }
        }
        // 生成文件记录所有词和包含该词的文件数
        StringBuilder sb1 = new StringBuilder();
        Iterator iter1 = dict.entrySet().iterator();
        while (iter1.hasNext()) {
            Map.Entry entry = (Map.Entry) iter1.next();
            if (entry.getKey().toString() != null) {
                sb1.append(entry.getKey().toString() + " " + dict.get(entry.getKey()) + "\r\n");
            }
        }
        File filewriter = new File("E:/allCount.txt");
        FileWriter fw = new FileWriter(filewriter.getAbsoluteFile());
        BufferedWriter bb = new BufferedWriter(fw);
        bb.write(sb1.toString());
        bb.close();
        System.out.println(dict);
        // 计算idf
        System.out.println("IDF for every word is:");
        Iterator iter_dict = dict.entrySet().iterator();
        while (iter_dict.hasNext()) {
            Map.Entry entry = (Map.Entry) iter_dict.next();
            float value = (float) Math.log(docNum / Float.parseFloat(entry.getValue().toString()));
            resIdf.put(entry.getKey().toString(), value);
            System.out.println(entry.getKey().toString() + " = " + value);
        }
        return resIdf;
    }

    // 返回该目录下所有词以及包含词的文件数
    @SuppressWarnings("unchecked")
    public static HashMap<String, Integer> idf_dict(HashMap<String, HashMap<String, Float>> all_tf, String file) throws IOException {
        FileList.clear();
        HashMap<String, Integer> dict = new HashMap<String, Integer>();
        List<String> filelist = readDirs(file);
        int docNum = filelist.size();
        for (int i = 0; i < docNum; i++) {
            HashMap<String, Float> temp = all_tf.get(filelist.get(i));
            Iterator iter = temp.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String word = entry.getKey().toString();
                if (dict.get(word) == null) {
                    dict.put(word, 1);
                } else {
                    dict.put(word, dict.get(word) + 1);
                }
            }
        }
        System.out.println(dict);
        return dict;
    }

    // 计算TFIDF值
    @SuppressWarnings("unchecked")
    public static void tf_idf(HashMap<String, HashMap<String, Float>> all_tf, HashMap<String, Float> idfs, String file) throws IOException {
        HashMap<String, HashMap<String, Float>> resTfIdf = new HashMap<String, HashMap<String, Float>>();
        FileList.clear();
        int docNum = readDirs(file).size();
        for (int i = 0; i < docNum; i++) {
            String filepath = FileList.get(i);
            HashMap<String, Float> tfidf = new HashMap<String, Float>();
            HashMap<String, Float> temp = all_tf.get(filepath);
            Iterator iter = temp.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String word = entry.getKey().toString();
                Float value = (float) Float.parseFloat(entry.getValue().toString()) * idfs.get(word);
                tfidf.put(word, value);
            }
            resTfIdf.put(filepath, tfidf);
        }
        System.out.println("TF-IDF for Every file is :");
        // 显示TFIDF
        DisTfIdf(resTfIdf);
    }

    // 返回计算的TFIDF值
    @SuppressWarnings("unchecked")
    public static HashMap<String, HashMap<String, Float>> tf_idf_return(HashMap<String, HashMap<String, Float>> all_tf, HashMap<String, Float> idfs, String file) throws IOException {
        FileList.clear();
        HashMap<String, HashMap<String, Float>> resTfIdf = new HashMap<String, HashMap<String, Float>>();
        int docNum = readDirs(file).size();
        for (int i = 0; i < docNum; i++) {
            @SuppressWarnings("unused")
            HashMap<String, Float> tfidf_reduce = new HashMap<String, Float>();
            String filepath = FileList.get(i);
            HashMap<String, Float> tfidf = new HashMap<String, Float>();
            HashMap<String, Float> temp = all_tf.get(filepath);
            Iterator iter = temp.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String word = entry.getKey().toString();
                Float value = (float) Float.parseFloat(entry.getValue().toString()) * idfs.get(word);
                tfidf.put(word, value);
            }
            resTfIdf.put(filepath, tfidf);
        }
        return resTfIdf;
    }

    // TFIDF显示输出 并建立文件存储该信息
    @SuppressWarnings("unchecked")
    public static void DisTfIdf(HashMap<String, HashMap<String, Float>> tfidf) throws IOException {
        StringBuilder stall = new StringBuilder();
        Iterator iter1 = tfidf.entrySet().iterator();
        while (iter1.hasNext()) {
            Map.Entry entrys = (Map.Entry) iter1.next();
            System.out.println("FileName: " + entrys.getKey().toString());
            System.out.print("{");
            HashMap<String, Float> temp = (HashMap<String, Float>) entrys.getValue();
            Iterator iter2 = temp.entrySet().iterator();
            while (iter2.hasNext()) {
                Map.Entry entry = (Map.Entry) iter2.next();
                System.out.print(entry.getKey().toString() + " = " + entry.getValue().toString() + ", ");
                stall.append(entrys.getKey().toString() + " " + entry.getKey().toString() + " " + entry.getValue().toString() + "\r\n");
            }
            System.out.println("}");
        }
        File filewriter = new File("E:/allTFIDF.txt");
        FileWriter fw = new FileWriter(filewriter.getAbsoluteFile());
        BufferedWriter bz = new BufferedWriter(fw);
        bz.write(stall.toString());
        bz.close();
    }

    // 单属性熵
    public static double Entropy(double[] p, double tot) {
        double entropy = 0.0;
        for (int i = 0; i < p.length; i++) {
            if (p[i] > 0.0) {
                entropy += -p[i] / tot * Math.log(p[i] / tot) / Math.log(2.0);
            }
        }
        return entropy;
    }

    // 信息增益特征降维
    @SuppressWarnings("unchecked")
    private static void Total(int N, HashMap<String, HashMap<String, Float>> result, HashMap<String, Integer> idfs_dict_neg, HashMap<String, Integer> idfs_dict_pos, String file) throws IOException {
        FileList.clear();
        // 类别数组
        double[] classCnt = new double[N];
        // 总文件数
        double totalCnt = 0.0;
        for (int c = 0; c < N; c++) {
            // 每个类别的文件数目
            classCnt[c] = 125;
            totalCnt += classCnt[c];
        }
        int docNum = readDirs(file).size();
        // 词f的编号
        int num = 0;
        // 词f的编号
        int numb = 0;
        // 总的熵
        double totalEntroy = Entropy(classCnt, totalCnt);
        // 存储词及其编号
        HashMap<String, Integer> count = new HashMap<String, Integer>();
        // 存储特征降维后word和其编号
        HashMap<String, Integer> countG = new HashMap<String, Integer>();
        // 存储特征降维后word和其信息增益
        HashMap<String, Double> countG1 = new HashMap<String, Double>();
        // 存储词和该词的信息增益
        HashMap<String, Double> infogains = new HashMap<String, Double>();
        // 缓存文件名,词,信息增益,TFIDF
        StringBuilder st = new StringBuilder();
        // 缓存未特征处理的类别,单词的编号,单词的TFIDF值
        StringBuilder ss = new StringBuilder();
        // 缓存经过特征处理后的类别,单词的编号,单词的TFIDF值
        StringBuilder sr = new StringBuilder();
        for (int i = 0; i < docNum; i++) {
            String filepath = FileList.get(i);
            HashMap<String, Float> temp = result.get(filepath);
            Iterator iter = temp.entrySet().iterator();
            if (filepath.contains("dubo")) {
                ss.append(1 + "  ");
            } else if (filepath.contains("fangdong")) {
                ss.append(2 + "  ");
            }
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String f = entry.getKey().toString();
                // 包括词F的分布（类别1,2分别包含该词的文件数）
                double[] featureCntWithF = new double[N];
                // 不包括词F的分布
                double[] featureCntWithoutF = new double[N];
                // 所有类别中包括词F的文件数
                double totalCntWithF = 0.0;
                // 所有类别中不包括词F的文件数
                double totalCntWithoutF = 0.0;
                for (int c = 0; c < N; c++) {
                    Iterator iter_dict = null;

                    while (iter_dict.hasNext()) {
                        Map.Entry entry_neg = (Map.Entry) iter_dict.next();
                        if (f.equals(entry_neg.getKey().toString())) {
                            // 该词在该类别中出现
                            featureCntWithF[c] = Double.parseDouble(entry_neg.getValue().// 将该出现该词的文件数赋值给数组
                            toString());
                            break;
                        } else {
                            featureCntWithF[c] = 0.0;
                        }
                    }
                    // 不包括词F的文件数等于该类别总数减去包含该词的文件数
                    featureCntWithoutF[c] = classCnt[c] - featureCntWithF[c];
                    totalCntWithF += featureCntWithF[c];
                    totalCntWithoutF += featureCntWithoutF[c];
                }
                double entropyWithF = Entropy(featureCntWithF, totalCntWithF);
                double entropyWithoutF = Entropy(featureCntWithoutF, totalCntWithoutF);
                double wf = totalCntWithF / totalCnt;
                double infoGain = totalEntroy - wf * entropyWithF - // 信息增益的公式
                (1.0 - wf) * entropyWithoutF;
                infogains.put(f, infoGain);
                st.append(filepath + " " + f + " " + "信息增益" + "=" + // 缓存格式
                infoGain + " " + "tfidf" + "=" + entry.getValue().toString() + "\r\n");
                // 方式一：直接用阈值选取特征值可以省去下面再次遍历的过程
                // 给词f编号
                if (count.get(f) == null) {
                    num++;
                    count.put(f, num);
                }
                // 缓存格式
                ss.append(count.get(f) + ":" + entry.getValue() + " ");
            }
            ss.append("\r\n");
        }
        File fileprepare = new File("E:/test.txt");
        FileWriter fz = new FileWriter(fileprepare.getAbsoluteFile());
        BufferedWriter bz = new BufferedWriter(fz);
        bz.write(ss.toString());
        bz.close();
        File filewriter = new File("E:/jieguo.txt");
        FileWriter fw = new FileWriter(filewriter.getAbsoluteFile());
        BufferedWriter bw = new BufferedWriter(fw);
        bw.write(st.toString());
        bw.close();
        // 方式二：将信息增益从大到小排列,选取前特定数的词为特征词
        // 对信息增益排序（从大到小）
        ArrayList<Map.Entry<String, Double>> infoIds = new ArrayList<Map.Entry<String, Double>>(infogains.entrySet());
        Collections.sort(infoIds, new Comparator<Map.Entry<String, Double>>() {

            public int compare(Map.Entry<String, Double> o1, Map.Entry<String, Double> o2) {
                if (o2.getValue() - o1.getValue() > 0) {
                    // 降序排列
                    return 1;
                } else {
                    return -1;
                }
            }
        });
        // 选取信息增益为前2000的词做特征词
        for (int c = 0; c < 2000; c++) {
            // 将处理后的数据存储到countG1中
            countG1.put(infoIds.get(c).getKey(), infoIds.get(c).getValue());
        }
        // 再次遍历
        for (int i = 0; i < docNum; i++) {
            String filepath = FileList.get(i);
            HashMap<String, Float> temp = result.get(filepath);
            Iterator iter = temp.entrySet().iterator();
            if (filepath.contains("dubo")) {
                sr.append(1 + "  ");
            } else if (filepath.contains("fangdong")) {
                sr.append(2 + "  ");
            }
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String f = entry.getKey().toString();
                // 判断该词是特征降维后的那些词
                if (countG1.get(f) != null) {
                    // 给该词编号
                    if (countG.get(f) == null) {
                        numb++;
                        countG.put(f, numb);
                    }
                    sr.append(countG.get(f) + ":" + entry.getValue() + " ");
                }
            }
            sr.append("\r\n");
        }
        File fileprepare1 = new File("E:/testt.txt");
        FileWriter fr = new FileWriter(fileprepare1.getAbsoluteFile());
        BufferedWriter br = new BufferedWriter(fr);
        br.write(sr.toString());
        br.close();
    }

    public static void main(String[] args) throws IOException {
        // TODO Auto-generated method stub
        // 总的数据路径
        String file = "C:/Users/zzw/Desktop/项目管理/语料/test";
        // 类1数据路径
        String file1 = "C:/Users/zzw/Desktop/项目管理/语料/test/赌博";
        // 类2数据路径
        String file2 = "C:/Users/zzw/Desktop/项目管理/语料/test/反动";
        HashMap<String, HashMap<String, Float>> all_tf = tfAllFiles(file);
        // file1文件的tf值和路径
        HashMap<String, HashMap<String, Float>> all_tf_neg = tfAllFiles(file1);
        // file2文件的tf值和路径
        HashMap<String, HashMap<String, Float>> all_tf_pos = tfAllFiles(file2);
        System.out.println();
        // 返回file1下所有词以及包含词的文件数
        HashMap<String, Integer> idfs_dict_neg = idf_dict(all_tf_neg, file1);
        // 返回file2下所有词以及包含词的文件数
        HashMap<String, Integer> idfs_dict_pos = idf_dict(all_tf_pos, file2);
        HashMap<String, Float> idfs = idf(all_tf, file);
        System.out.println();
        tf_idf(all_tf, idfs, file);
        HashMap<String, HashMap<String, Float>> result = tf_idf_return(all_tf, idfs, file);
        // 输入类别数
        int N = 2;
        // 按信息增益进行特征降维
        Total(N, result, idfs_dict_neg, idfs_dict_pos, file);
    }
}


package TIDF;

import java.io.*;
import java.util.*;
import org.wltea.analyzer.lucene.IKAnalyzer;

/**
 * 分词-TFIDF-信息增益
 * @author LJ
 *
 * @datetime 2015-6-15
 */
public class TestTfIdf {

    // 加载停用词库
    public static final String stopWordTable = "C:/Users/zzw/Desktop/sc_ot-tingyongzhongwen_hc/stopWordTable.txt";

    // 文件列表
    private static ArrayList<String> FileList = new ArrayList<String>();

    // 递归读取该路径下文件返回文件列表
    public static List<String> readDirs(String filepath) throws FileNotFoundException, IOException {
        try {
            File file = new File(filepath);
            if (!file.isDirectory()) {
                System.out.println("输入的[]");
                System.out.println("filepath:" + file.getAbsolutePath());
            } else {
                String[] flist = file.list();
                for (int i = 0; i < flist.length; i++) {
                    File newfile = new File(filepath + "\\" + flist[i]);
                    if (!newfile.isDirectory()) {
                        FileList.add(newfile.getAbsolutePath());
                    } else if (newfile.isDirectory()) {
                        readDirs(filepath + "\\" + flist[i]);
                    }
                }
            }
        } catch (FileNotFoundException e) {
            System.out.println(e.getMessage());
        }
        return FileList;
    }

    // 读入文件
    public static String readFile(String file) throws FileNotFoundException, IOException {
        StringBuffer strSb = new StringBuffer();
        InputStreamReader inStrR = new InputStreamReader(new FileInputStream(file), "gbk");
        BufferedReader br = new BufferedReader(inStrR);
        String line = br.readLine();
        while (line != null) {
            strSb.append(line).append("\r\n");
            line = br.readLine();
        }
        return strSb.toString();
    }

    // 分词处理
    public static ArrayList<String> cutWords(String file) throws IOException {
        ArrayList<String> fenci = new ArrayList<String>();
        ArrayList<String> words = new ArrayList<String>();
        String text = TestTfIdf.readFile(file);
        IKAnalyzer analyzer = new IKAnalyzer();
        // 分词处理
        fenci = analyzer.split(text);
        BufferedReader StopWordFileBr = new BufferedReader(new InputStreamReader(new FileInputStream(new File(stopWordTable))));
        // 用来存放停用词的集合
        Set<String> stopWordSet = new HashSet<String>();
        // 初如化停用词集
        String stopWord = null;
        for (; (stopWord = StopWordFileBr.readLine()) != null; ) {
            stopWordSet.add(stopWord);
        }
        for (String word : fenci) {
            if (stopWordSet.contains(word)) {
                continue;
            }
            words.add(word);
        }
        System.out.println(words);
        return words;
    }

    // 统计一个文件中每个词出现的次数
    public static HashMap<String, Integer> normalTF(ArrayList<String> cutwords) {
        HashMap<String, Integer> resTF = new HashMap<String, Integer>();
        for (String word : cutwords) {
            if (resTF.get(word) == null) {
                resTF.put(word, 1);
                System.out.println(word);
            } else {
                resTF.put(word, resTF.get(word) + 1);
                System.out.println(word.toString());
            }
        }
        System.out.println(resTF);
        return resTF;
    }

    // 计算一个文件每个词tf值
    @SuppressWarnings("unchecked")
    public static HashMap<String, Float> tf(ArrayList<String> cutwords) {
        HashMap<String, Float> resTF = new HashMap<String, Float>();
        int wordLen = cutwords.size();
        HashMap<String, Integer> intTF = TestTfIdf.normalTF(cutwords);
        Iterator iter = intTF.entrySet().iterator();
        while (iter.hasNext()) {
            Map.Entry entry = (Map.Entry) iter.next();
            resTF.put(entry.getKey().toString(), Float.parseFloat(entry.getValue().toString()) / wordLen);
            System.out.println(entry.getKey().toString() + " = " + Float.parseFloat(entry.getValue().toString()) / wordLen);
        }
        return resTF;
    }

    // tf times for file 。。。。。。。
    public static HashMap<String, HashMap<String, Integer>> normalTFAllFiles(String dirc) throws IOException {
        FileList.clear();
        HashMap<String, HashMap<String, Integer>> allNormalTF = new HashMap<String, HashMap<String, Integer>>();
        List<String> filelist = TestTfIdf.readDirs(dirc);
        for (String file : filelist) {
            HashMap<String, Integer> dict = new HashMap<String, Integer>();
            ArrayList<String> cutwords = TestTfIdf.cutWords(file);
            dict = TestTfIdf.normalTF(cutwords);
            allNormalTF.put(file, dict);
        }
        return allNormalTF;
    }

    // 返回所有文件tf值
    public static HashMap<String, HashMap<String, Float>> tfAllFiles(String dirc) throws IOException {
        FileList.clear();
        HashMap<String, HashMap<String, Float>> allTF = new HashMap<String, HashMap<String, Float>>();
        List<String> filelist = TestTfIdf.readDirs(dirc);
        for (String file : filelist) {
            HashMap<String, Float> dict = new HashMap<String, Float>();
            ArrayList<String> cutwords = TestTfIdf.cutWords(file);
            dict = TestTfIdf.tf(cutwords);
            allTF.put(file, dict);
        }
        return allTF;
    }

    // 计算该目录下所有词的idf
    @SuppressWarnings("unchecked")
    public static HashMap<String, Float> idf(HashMap<String, HashMap<String, Float>> all_tf, String file) throws IOException {
        FileList.clear();
        HashMap<String, Float> resIdf = new HashMap<String, Float>();
        HashMap<String, Integer> dict = new HashMap<String, Integer>();
        int docNum = readDirs(file).size();
        for (int i = 0; i < docNum; i++) {
            HashMap<String, Float> temp = all_tf.get(FileList.get(i));
            Iterator iter = temp.entrySet().iterator();

        }
        // 生成文件记录所有词和包含该词的文件数
        StringBuilder sb1 = new StringBuilder();
        Iterator iter1 = dict.entrySet().iterator();
        while (iter1.hasNext()) {
            Map.Entry entry = (Map.Entry) iter1.next();
            if (entry.getKey().toString() != null) {
                sb1.append(entry.getKey().toString() + " " + dict.get(entry.getKey()) + "\r\n");
            }
        }
        File filewriter = new File("E:/allCount.txt");
        FileWriter fw = new FileWriter(filewriter.getAbsoluteFile());
        BufferedWriter bb = new BufferedWriter(fw);
        bb.write(sb1.toString());
        bb.close();
        System.out.println(dict);
        // 计算idf
        System.out.println("IDF for every word is:");
        Iterator iter_dict = dict.entrySet().iterator();
        while (iter_dict.hasNext()) {
            Map.Entry entry = (Map.Entry) iter_dict.next();
            float value = (float) Math.log(docNum / Float.parseFloat(entry.getValue().toString()));
            resIdf.put(entry.getKey().toString(), value);
            System.out.println(entry.getKey().toString() + " = " + value);
        }
        return resIdf;
    }

    // 返回该目录下所有词以及包含词的文件数
    @SuppressWarnings("unchecked")
    public static HashMap<String, Integer> idf_dict(HashMap<String, HashMap<String, Float>> all_tf, String file) throws IOException {
        FileList.clear();
        HashMap<String, Integer> dict = new HashMap<String, Integer>();
        List<String> filelist = readDirs(file);
        int docNum = filelist.size();
        for (int i = 0; i < docNum; i++) {
            HashMap<String, Float> temp = all_tf.get(filelist.get(i));
            Iterator iter = temp.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String word = entry.getKey().toString();
                if (dict.get(word) == null) {
                    dict.put(word, 1);
                } else {
                    dict.put(word, dict.get(word) + 1);
                }
            }
        }
        System.out.println(dict);
        return dict;
    }

    // 计算TFIDF值
    @SuppressWarnings("unchecked")
    public static void tf_idf(HashMap<String, HashMap<String, Float>> all_tf, HashMap<String, Float> idfs, String file) throws IOException {
        HashMap<String, HashMap<String, Float>> resTfIdf = new HashMap<String, HashMap<String, Float>>();
        FileList.clear();
        int docNum = readDirs(file).size();
        for (int i = 0; i < docNum; i++) {
            String filepath = FileList.get(i);
            HashMap<String, Float> tfidf = new HashMap<String, Float>();
            HashMap<String, Float> temp = all_tf.get(filepath);
            Iterator iter = temp.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String word = entry.getKey().toString();
                Float value = (float) Float.parseFloat(entry.getValue().toString()) * idfs.get(word);
                tfidf.put(word, value);
            }
            resTfIdf.put(filepath, tfidf);
        }
        System.out.println("TF-IDF for Every file is :");
        // 显示TFIDF
        DisTfIdf(resTfIdf);
    }

    // 返回计算的TFIDF值
    @SuppressWarnings("unchecked")
    public static HashMap<String, HashMap<String, Float>> tf_idf_return(HashMap<String, HashMap<String, Float>> all_tf, HashMap<String, Float> idfs, String file) throws IOException {
        FileList.clear();
        HashMap<String, HashMap<String, Float>> resTfIdf = new HashMap<String, HashMap<String, Float>>();
        int docNum = readDirs(file).size();
        for (int i = 0; i < docNum; i++) {
            @SuppressWarnings("unused")
            HashMap<String, Float> tfidf_reduce = new HashMap<String, Float>();
            String filepath = FileList.get(i);
            HashMap<String, Float> tfidf = new HashMap<String, Float>();
            HashMap<String, Float> temp = all_tf.get(filepath);
            Iterator iter = temp.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String word = entry.getKey().toString();
                Float value = (float) Float.parseFloat(entry.getValue().toString()) * idfs.get(word);
                tfidf.put(word, value);
            }
            resTfIdf.put(filepath, tfidf);
        }
        return resTfIdf;
    }

    // TFIDF显示输出 并建立文件存储该信息
    @SuppressWarnings("unchecked")
    public static void DisTfIdf(HashMap<String, HashMap<String, Float>> tfidf) throws IOException {
        StringBuilder stall = new StringBuilder();
        Iterator iter1 = tfidf.entrySet().iterator();
        while (iter1.hasNext()) {
            Map.Entry entrys = (Map.Entry) iter1.next();
            System.out.println("FileName: " + entrys.getKey().toString());
            System.out.print("{");
            HashMap<String, Float> temp = (HashMap<String, Float>) entrys.getValue();
            Iterator iter2 = temp.entrySet().iterator();
            while (iter2.hasNext()) {
                Map.Entry entry = (Map.Entry) iter2.next();
                System.out.print(entry.getKey().toString() + " = " + entry.getValue().toString() + ", ");
                stall.append(entrys.getKey().toString() + " " + entry.getKey().toString() + " " + entry.getValue().toString() + "\r\n");
            }
            System.out.println("}");
        }
        File filewriter = new File("E:/allTFIDF.txt");
        FileWriter fw = new FileWriter(filewriter.getAbsoluteFile());
        BufferedWriter bz = new BufferedWriter(fw);
        bz.write(stall.toString());
        bz.close();
    }

    // 单属性熵
    public static double Entropy(double[] p, double tot) {
        double entropy = 0.0;
        for (int i = 0; i < p.length; i++) {
            if (p[i] > 0.0) {
                entropy += -p[i] / tot * Math.log(p[i] / tot) / Math.log(2.0);
            }
        }
        return entropy;
    }

    // 信息增益特征降维
    @SuppressWarnings("unchecked")
    private static void Total(int N, HashMap<String, HashMap<String, Float>> result, HashMap<String, Integer> idfs_dict_neg, HashMap<String, Integer> idfs_dict_pos, String file) throws IOException {
        FileList.clear();
        // 类别数组
        double[] classCnt = new double[N];
        // 总文件数
        double totalCnt = 0.0;
        for (int c = 0; c < N; c++) {
            // 每个类别的文件数目
            classCnt[c] = 125;
            totalCnt += classCnt[c];
        }
        int docNum = readDirs(file).size();
        // 词f的编号
        int num = 0;
        // 词f的编号
        int numb = 0;
        // 总的熵
        double totalEntroy = Entropy(classCnt, totalCnt);
        // 存储词及其编号
        HashMap<String, Integer> count = new HashMap<String, Integer>();
        // 存储特征降维后word和其编号
        HashMap<String, Integer> countG = new HashMap<String, Integer>();
        // 存储特征降维后word和其信息增益
        HashMap<String, Double> countG1 = new HashMap<String, Double>();
        // 存储词和该词的信息增益
        HashMap<String, Double> infogains = new HashMap<String, Double>();
        // 缓存文件名,词,信息增益,TFIDF
        StringBuilder st = new StringBuilder();
        // 缓存未特征处理的类别,单词的编号,单词的TFIDF值
        StringBuilder ss = new StringBuilder();
        // 缓存经过特征处理后的类别,单词的编号,单词的TFIDF值
        StringBuilder sr = new StringBuilder();
        for (int i = 0; i < docNum; i++) {
            String filepath = FileList.get(i);
            HashMap<String, Float> temp = result.get(filepath);
            Iterator iter = temp.entrySet().iterator();
            if (filepath.contains("dubo")) {
                ss.append(1 + "  ");
            } else if (filepath.contains("fangdong")) {
                ss.append(2 + "  ");
            }
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String f = entry.getKey().toString();
                // 包括词F的分布（类别1,2分别包含该词的文件数）
                double[] featureCntWithF = new double[N];
                // 不包括词F的分布
                double[] featureCntWithoutF = new double[N];
                // 所有类别中包括词F的文件数
                double totalCntWithF = 0.0;
                // 所有类别中不包括词F的文件数
                double totalCntWithoutF = 0.0;
                for (int c = 0; c < N; c++) {
                    Iterator iter_dict = null;
                    switch(c) {
                        case 0:
                            iter_dict = idfs_dict_neg.entrySet().iterator();
                            break;
                        case 1:
                            iter_dict = idfs_dict_pos.entrySet().iterator();
                            break;
                    }
                    while (iter_dict.hasNext()) {
                        Map.Entry entry_neg = (Map.Entry) iter_dict.next();
                        if (f.equals(entry_neg.getKey().toString())) {
                            // 该词在该类别中出现
                            featureCntWithF[c] = Double.parseDouble(entry_neg.getValue().// 将该出现该词的文件数赋值给数组
                            toString());
                            break;
                        } else {
                            featureCntWithF[c] = 0.0;
                        }
                    }
                    // 不包括词F的文件数等于该类别总数减去包含该词的文件数
                    featureCntWithoutF[c] = classCnt[c] - featureCntWithF[c];
                    totalCntWithF += featureCntWithF[c];
                    totalCntWithoutF += featureCntWithoutF[c];
                }
                double entropyWithF = Entropy(featureCntWithF, totalCntWithF);
                double entropyWithoutF = Entropy(featureCntWithoutF, totalCntWithoutF);
                double wf = totalCntWithF / totalCnt;
                double infoGain = totalEntroy - wf * entropyWithF - // 信息增益的公式
                (1.0 - wf) * entropyWithoutF;
                infogains.put(f, infoGain);
                st.append(filepath + " " + f + " " + "信息增益" + "=" + // 缓存格式
                infoGain + " " + "tfidf" + "=" + entry.getValue().toString() + "\r\n");
                // 方式一：直接用阈值选取特征值可以省去下面再次遍历的过程
                // 给词f编号
                if (count.get(f) == null) {
                    num++;
                    count.put(f, num);
                }
                // 缓存格式
                ss.append(count.get(f) + ":" + entry.getValue() + " ");
            }
            ss.append("\r\n");
        }
        File fileprepare = new File("E:/test.txt");
        FileWriter fz = new FileWriter(fileprepare.getAbsoluteFile());
        BufferedWriter bz = new BufferedWriter(fz);
        bz.write(ss.toString());
        bz.close();
        File filewriter = new File("E:/jieguo.txt");
        FileWriter fw = new FileWriter(filewriter.getAbsoluteFile());
        BufferedWriter bw = new BufferedWriter(fw);
        bw.write(st.toString());
        bw.close();
        // 方式二：将信息增益从大到小排列,选取前特定数的词为特征词
        // 对信息增益排序（从大到小）
        ArrayList<Map.Entry<String, Double>> infoIds = new ArrayList<Map.Entry<String, Double>>(infogains.entrySet());
        Collections.sort(infoIds, new Comparator<Map.Entry<String, Double>>() {

            public int compare(Map.Entry<String, Double> o1, Map.Entry<String, Double> o2) {
                if (o2.getValue() - o1.getValue() > 0) {
                    // 降序排列
                    return 1;
                } else {
                    return -1;
                }
            }
        });
        // 选取信息增益为前2000的词做特征词
        for (int c = 0; c < 2000; c++) {
            // 将处理后的数据存储到countG1中
            countG1.put(infoIds.get(c).getKey(), infoIds.get(c).getValue());
        }
        // 再次遍历
        for (int i = 0; i < docNum; i++) {
            String filepath = FileList.get(i);
            HashMap<String, Float> temp = result.get(filepath);
            Iterator iter = temp.entrySet().iterator();
            if (filepath.contains("dubo")) {
                sr.append(1 + "  ");
            } else if (filepath.contains("fangdong")) {
                sr.append(2 + "  ");
            }
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                String f = entry.getKey().toString();
                // 判断该词是特征降维后的那些词
                if (countG1.get(f) != null) {
                    // 给该词编号
                    if (countG.get(f) == null) {
                        numb++;
                        countG.put(f, numb);
                    }
                    sr.append(countG.get(f) + ":" + entry.getValue() + " ");
                }
            }
            sr.append("\r\n");
        }
        File fileprepare1 = new File("E:/testt.txt");
        FileWriter fr = new FileWriter(fileprepare1.getAbsoluteFile());
        BufferedWriter br = new BufferedWriter(fr);
        br.write(sr.toString());
        br.close();
    }

    public static void main(String[] args) throws IOException {
        // TODO Auto-generated method stub
        // 总的数据路径
        String file = "C:/Users/zzw/Desktop/项目管理/语料/test";
        // 类1数据路径
        String file1 = "C:/Users/zzw/Desktop/项目管理/语料/test/赌博";
        // 类2数据路径
        String file2 = "C:/Users/zzw/Desktop/项目管理/语料/test/反动";
        HashMap<String, HashMap<String, Float>> all_tf = tfAllFiles(file);
        // file1文件的tf值和路径
        HashMap<String, HashMap<String, Float>> all_tf_neg = tfAllFiles(file1);
        // file2文件的tf值和路径
        HashMap<String, HashMap<String, Float>> all_tf_pos = tfAllFiles(file2);
        System.out.println();
        // 返回file1下所有词以及包含词的文件数
        HashMap<String, Integer> idfs_dict_neg = idf_dict(all_tf_neg, file1);
        // 返回file2下所有词以及包含词的文件数
        HashMap<String, Integer> idfs_dict_pos = idf_dict(all_tf_pos, file2);
        HashMap<String, Float> idfs = idf(all_tf, file);
        System.out.println();
        tf_idf(all_tf, idfs, file);
        HashMap<String, HashMap<String, Float>> result = tf_idf_return(all_tf, idfs, file);
        // 输入类别数
        int N = 2;
        // 按信息增益进行特征降维
        Total(N, result, idfs_dict_neg, idfs_dict_pos, file);
    }
}


switch(type) {
            case 0:
                if (reMeasure.matcher(s).matches())
                    return 0;
                break;
        }

/**
 * All Rights Reserved.  Use is subject to license terms.
 *
 * This file is part of MARY TTS.
 */
package marytts.language.de.preprocess;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import marytts.util.MaryUtils;
import marytts.util.dom.MaryDomUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

/**
 * An expansion pattern implementation for measure patterns.
 *
 * @author Marc Schröder
 */
public class MeasureEP extends ExpansionPattern {

    private final String[] _knownTypes = { "measure" };

    /**
     * Every subclass has its own list knownTypes, an internal string representation of known types. These are possible values of
     * the type attribute to the say-as element, as defined in MaryXML.dtd. If there is more than one
     * known type, the first type (knownTypes[0]) is expected to be the most general one, of which the others are
     * specializations.
     */
    private final List<String> knownTypes = Arrays.asList(_knownTypes);

    public List<String> knownTypes() {
        return knownTypes;
    }

    // number-dependent-feminine measure-symbol-names
    private final String[] _nuDeFeMeasureSymbolNames = { "s", "Sekunde", "sec", "Sekunde", "ms", "Millisekunde", "msec", "Millisekunde", "min", "Minute", "kcal", "Kilokalorie", "oz.", "Unze", "oz", "Unze" };

    // (number-independent) masculine (or neuter) measure-symbol-names
    private final String[] _maMeasureSymbolNames = { "km", "Kilometer", "dm", "Dezimeter", "cm", "Zentimeter", "mm", "Millimeter", "g", "Gramm", "kg", "Kilogramm", "mg", "Milligramm", "A", "Ampere[am-'pe:6]", "V", "Volt", "K", "Kelvin['kEl-vi:n]", Character.valueOf((char) 176).toString() + "C", "Grad Celsius['tsEl-zi:-Us]", Character.valueOf((char) 730).toString() + "C", "Grad Celsius['tsEl-zi:-Us]", "\u2103", "Grad Celsius['tsEl-zi:-Us]", Character.valueOf((char) 176).toString() + "F", "Grad Fahrenheit", Character.valueOf((char) 730).toString() + "F", "Grad Fahrenheit", "\u2109", "Grad Fahrenheit", "Hz", "Hertz", "kHz", "Kilohertz", "MHz", "Megahertz", "GHz", "GigaHertz", "N", "Newton['nju:-t@n]", "Pa", "Pascal", "J", "Joule['dZu:l]", "kJ", "Kilojoule['ki:-lo:-dZu:l]", "W", "Watt", "kW", "Kilowatt", "MW", "Megawatt", "GW", "Gigawatt", "mW", "Milliwatt", "l", "Liter", "dl", "Deziliter", "cl", "Zentiliter", "ml", "Milliliter", "Bq", "Becquerel[bE-k@-'rEl]", "EL", "Esslöffel", "TL", "Teelöffel", "qm", "Quadratmeter", "m" + Character.valueOf((char) 178).toString(), "Quadratmeter", "m" + Character.valueOf((char) 179).toString(), "Kubikmeter", "ccm", "Kubikzentimeter", "m", "Meter", "%", "Prozent" };

    private final Map<String, String> nuDeFeMeasureSymbolNames = MaryUtils.arrayToMap(_nuDeFeMeasureSymbolNames);

    private final Map<String, String> maMeasureSymbolNames = MaryUtils.arrayToMap(_maMeasureSymbolNames);

    // Domain-specific primitives:
    protected final String sMeasureSymbol = getMeasureSymbols();

    // We don't use sMatchingChars here, but override isCandidate().
    // Now the actual match patterns:
    protected final Pattern reMeasureSymbol = Pattern.compile("(" + sMeasureSymbol + ")");

    protected final Pattern reMeasure = Pattern.compile("(" + NumberEP.sInteger + "|" + NumberEP.sFloat + ")" + "(" + sMeasureSymbol + ")");

    private final Pattern reMatchingChars = null;

    public Pattern reMatchingChars() {
        return reMatchingChars;
    }

    /**
     * Every subclass has its own logger. The important point is that if several threads are accessing the variable at the same
     * time, the logger needs to be thread-safe or it will produce rubbish.
     *
     * @return _sMeasureSymbol.toString()
     */
    // Only used to initialize sMeasureSymbol from _measureSymbolNames[]:
    private String getMeasureSymbols() {
        StringBuilder _sMeasureSymbol = new StringBuilder("(?:");
        if (_nuDeFeMeasureSymbolNames.length > 0)
            _sMeasureSymbol.append(_nuDeFeMeasureSymbolNames[0]);
        for (int i = 2; i < _nuDeFeMeasureSymbolNames.length; i += 2) {
            _sMeasureSymbol.append("|" + _nuDeFeMeasureSymbolNames[i]);
        }
        if (_nuDeFeMeasureSymbolNames.length > 0 && _maMeasureSymbolNames.length > 0)
            _sMeasureSymbol.append("|");
        if (_maMeasureSymbolNames.length > 0)
            _sMeasureSymbol.append(_maMeasureSymbolNames[0]);
        for (int i = 2; i < _maMeasureSymbolNames.length; i += 2) {
            _sMeasureSymbol.append("|" + _maMeasureSymbolNames[i]);
        }
        _sMeasureSymbol.append(")");
        return _sMeasureSymbol.toString();
    }

    public MeasureEP() {
        super();
    }

    protected boolean isCandidate(Element t) {
        String s = MaryDomUtils.tokenText(t);
        return (reMeasureSymbol.matcher(s).matches() || number.isCandidate(t) || reMeasure.matcher(s).matches());
    }

    protected int canDealWith(String s, int type) {
        return match(s, type);
    }

    protected int match(String s, int type) {

        return -1;
    }

    protected List<Element> expand(List<Element> tokens, String s, int type) {
        if (tokens == null)
            throw new NullPointerException("Received null argument");
        if (tokens.isEmpty())
            throw new IllegalArgumentException("Received empty list");
        Document doc = ((Element) tokens.get(0)).getOwnerDocument();
        // we expect type to be one of the return values of match():
        List<Element> expanded = null;
        switch(type) {
            case 0:
                expanded = expandMeasure(doc, s);
                break;
        }
        replaceTokens(tokens, expanded);
        return expanded;
    }

    protected List<Element> expandMeasure(Document doc, String s) {
        ArrayList<Element> exp = new ArrayList<Element>();
        StringBuilder sb = new StringBuilder();
        String measure = null;
        String amount = null;
        Matcher reMatcher = reMeasure.matcher(s);
        if (!reMatcher.find())
            return null;
        amount = reMatcher.group(1);
        measure = reMatcher.group(2);
        boolean measureIsMasculine = false;
        String measureName = (String) nuDeFeMeasureSymbolNames.get(measure);
        if (measureName == null) {
            measureName = (String) maMeasureSymbolNames.get(measure);
            measureIsMasculine = true;
            if (amount.equals("1"))
                sb.append("ein");
            else
                sb.append(number.expandFloat(amount));
        } else {
            if (amount.equals("1"))
                sb.append("eine");
            else
                sb.append(number.expandFloat(amount));
        }
        sb.append(" ");
        sb.append(measureName);
        if (!measureIsMasculine && !amount.equals("1"))
            sb.append("n");
        exp.addAll(makeNewTokens(doc, sb.toString(), true, s));
        return exp;
    }
}


while (enumeration.hasMoreElements()) {
                    ZipEntry ze = enumeration.nextElement();
                    if (ze.getName().equals("AndroidManifest.xml") || ze.getName().equals("classes.dex") || ze.getName().startsWith("META-INF/"))
                        continue;
                    zos.copyZipEntry(ze, zipFile);
                }

package com.mcal.apksignkill.utils;

import android.content.Context;
import com.android.dex.Dex;
import com.android.dx.command.dexer.DxContext;
import com.android.dx.merge.CollisionPolicy;
import com.android.dx.merge.DexMerger;
import com.mcal.apksignkill.R;
import org.jetbrains.annotations.NotNull;
import org.jf.dexlib2.Opcodes;
import org.jf.dexlib2.dexbacked.DexBackedClassDef;
import org.jf.dexlib2.dexbacked.DexBackedDexFile;
import org.jf.dexlib2.iface.ClassDef;
import org.jf.dexlib2.writer.builder.DexBuilder;
import org.jf.dexlib2.writer.io.MemoryDataStore;
import org.jf.smali.Smali;
import org.jf.smali.SmaliOptions;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import bin.util.StreamUtil;
import bin.xml.decode.AXmlDecoder;
import bin.xml.decode.AXmlResourceParser;
import bin.xml.decode.XmlPullParser;
import bin.zip.ZipEntry;
import bin.zip.ZipFile;
import bin.zip.ZipOutputStream;

public class SuperSignatureTool {

    private Context mContext;

    private boolean customApplication = false;

    private String customApplicationName;

    private String packageName;

    private String outApk;

    private String srcApk;

    private String tempApk;

    public SuperSignatureTool(Context context) {
        mContext = context;
    }

    public void setPath(String input, String output) {
        srcApk = input;
        outApk = output;
        tempApk = new File(srcApk).getParentFile().toString() + "/.temp";
    }

    public void Kill() {
        new File(outApk).delete();
        System.out.println("Чтение APK:" + srcApk);
        try (ZipFile zipFile = new ZipFile(srcApk)) {
            byte[] apk = StreamUtil.readBytes(new FileInputStream(srcApk));
            System.out.println("  -- Обработка AndroidManifest.xml");
            ZipEntry manifestEntry = zipFile.getEntry("AndroidManifest.xml");
            byte[] manifestData = parseManifest(zipFile.getInputStream(manifestEntry));
            System.out.println("\nОптимизация APK:" + outApk);
            ZipOutputStream zipOutputStream = new ZipOutputStream(new File(tempApk));
            zipOutputStream.setLevel(1);
            Enumeration<ZipEntry> entries = zipFile.getEntries();
            while (entries.hasMoreElements()) {
                ZipEntry ze = entries.nextElement();
                String name = ze.getName();
                if ((name.startsWith("classes") && name.endsWith("dex")) || name.startsWith("./")) {
                    zipOutputStream.copyZipEntry(ze, zipFile);
                }
            }
            zipOutputStream.close();
            System.out.println("\nЗапись в APK:" + outApk);
            try (ZipOutputStream zos = new ZipOutputStream(new File(outApk))) {
                zos.putNextEntry("AndroidManifest.xml");
                zos.write(manifestData);
                zos.closeEntry();
                zos.putNextEntry("classes.dex");
                zos.write(processDex());
                zos.closeEntry();
                zos.putNextEntry("assets/ysh/hook.apk");
                zos.write(apk);
                zos.closeEntry();
                Enumeration<ZipEntry> enumeration = zipFile.getEntries();

                new File(tempApk).delete();
                zipFile.close();
            } catch (Throwable th) {
                th.printStackTrace();
            }
        } catch (Throwable th2) {
            th2.printStackTrace();
        }
    }

    private byte @NotNull [] processDex() throws Exception {
        DexBackedDexFile dex = DexBackedDexFile.fromInputStream(Opcodes.getDefault(), new BufferedInputStream(dexMerge()));
        DexBuilder dexBuilder = new DexBuilder(Opcodes.getDefault());
        try (InputStream fis = mContext.getResources().openRawResource(R.raw.super_hook_app)) {
            String src = new String(StreamUtil.readBytes(fis), StandardCharsets.UTF_8);
            if (customApplication) {
                if (customApplicationName.startsWith(".")) {
                    if (packageName == null)
                        throw new NullPointerException("Package name is null.");
                    customApplicationName = packageName + customApplicationName;
                }
                customApplicationName = "L" + customApplicationName.replace('.', '/') + ";";
                src = src.replace("Landroid/app/Application;", customApplicationName);
            }
            ClassDef classDef = Smali.assembleSmaliFile(src, dexBuilder, new SmaliOptions());
            if (classDef == null)
                throw new Exception("Parse smali failed");
            for (DexBackedClassDef dexBackedClassDef : dex.getClasses()) {
                dexBuilder.internClassDef(dexBackedClassDef);
            }
        }
        MemoryDataStore store = new MemoryDataStore();
        dexBuilder.writeTo(store);
        return Arrays.copyOf(store.getBufferData(), store.getSize());
    }

    private byte @NotNull [] parseManifest(InputStream is) throws IOException {
        AXmlDecoder axml = AXmlDecoder.decode(is);
        AXmlResourceParser parser = new AXmlResourceParser();
        parser.open(new ByteArrayInputStream(axml.getData()), axml.mTableStrings);
        boolean success = false;
        int type;
        while ((type = parser.next()) != XmlPullParser.END_DOCUMENT) {
            if (type != XmlPullParser.START_TAG)
                continue;
            if (parser.getName().equals("manifest")) {
                int size = parser.getAttributeCount();
                for (int i = 0; i < size; ++i) {
                    if (parser.getAttributeName(i).equals("package")) {
                        packageName = parser.getAttributeValue(i);
                    }
                }
            } else if (parser.getName().equals("application")) {
                int size = parser.getAttributeCount();
                for (int i = 0; i < size; ++i) {
                    if (parser.getAttributeNameResource(i) == 0x01010003) {
                        customApplication = true;
                        customApplicationName = parser.getAttributeValue(i);
                        int index = axml.mTableStrings.getSize();
                        byte[] data = axml.getData();
                        int off = parser.currentAttributeStart + 20 * i;
                        off += 8;
                        FileUtils.writeInt(data, off, index);
                        off += 8;
                        FileUtils.writeInt(data, off, index);
                    }
                }
                if (!customApplication) {
                    int off = parser.currentAttributeStart;
                    byte[] data = axml.getData();
                    byte[] newData = new byte[data.length + 20];
                    System.arraycopy(data, 0, newData, 0, off);
                    System.arraycopy(data, off, newData, off + 20, data.length - off);
                    // chunkSize
                    int chunkSize = FileUtils.readInt(newData, off - 32);
                    FileUtils.writeInt(newData, off - 32, chunkSize + 20);
                    // attributeCount
                    FileUtils.writeInt(newData, off - 8, size + 1);
                    int idIndex = parser.findResourceID(0x01010003);
                    if (idIndex == -1)
                        throw new IOException("idIndex == -1");
                    boolean isMax = true;
                    for (int i = 0; i < size; ++i) {
                        int id = parser.getAttributeNameResource(i);
                        if (id > 0x01010003) {
                            isMax = false;
                            if (i != 0) {
                                System.arraycopy(newData, off + 20, newData, off, 20 * i);
                                off += 20 * i;
                            }
                            break;
                        }
                    }
                    if (isMax) {
                        System.arraycopy(newData, off + 20, newData, off, 20 * size);
                        off += 20 * size;
                    }
                    FileUtils.writeInt(newData, off, axml.mTableStrings.find("http://schemas.android.com/apk/res/android"));
                    FileUtils.writeInt(newData, off + 4, idIndex);
                    FileUtils.writeInt(newData, off + 8, axml.mTableStrings.getSize());
                    FileUtils.writeInt(newData, off + 12, 0x03000008);
                    FileUtils.writeInt(newData, off + 16, axml.mTableStrings.getSize());
                    axml.setData(newData);
                }
                success = true;
                break;
            }
        }
        if (!success)
            throw new IOException();
        ArrayList<String> list = new ArrayList<>(axml.mTableStrings.getSize());
        axml.mTableStrings.getStrings(list);
        list.add("com.ysh.hook.App");
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        axml.write(list, baos);
        return baos.toByteArray();
    }

    @NotNull
    private FileInputStream dexMerge() throws Exception {
        FileUtils.extractFile("classes.dex", srcApk, mContext.getFilesDir().getAbsolutePath() + "/classesO.dex");
        File outputMergeDex = null;
        try {
            FileInputStream origDex = new FileInputStream(mContext.getFilesDir().getAbsolutePath() + "/classesO.dex");
            InputStream hookDex = mContext.getResources().openRawResource(R.raw.super_hook);
            outputMergeDex = new File(mContext.getFilesDir().getAbsolutePath() + "/classes.dex");
            Dex[] toBeMerge = { new Dex(origDex), new Dex(hookDex) };
            DexMerger dexMerger = new DexMerger(toBeMerge, CollisionPolicy.FAIL, new DxContext());
            Dex merged = dexMerger.merge();
            merged.writeTo(outputMergeDex);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return new FileInputStream(outputMergeDex);
    }
}


while (true) {
                if (!iterator.hasNext()) {
                    return;
                }
                block = iterator.next();
                if (block.getMaterial().isPlacementObstacle()) {
                    return;
                }
                if (block.getMaterial() instanceof Water && VanillaMaterials.WATER.isSource(block)) {
                    break;
                }
            }

/*
 * This file is part of Vanilla.
 *
 *
 * License and see <http://example.com> for the full license, including
 */
package org.spout.vanilla.material.item.potion;

import org.spout.api.entity.Entity;
import org.spout.api.event.player.Action;
import org.spout.api.geo.cuboid.Block;
import org.spout.api.inventory.ItemStack;
import org.spout.api.inventory.Slot;
import org.spout.api.util.BlockIterator;
import org.spout.vanilla.component.entity.inventory.PlayerInventory;
import org.spout.vanilla.component.entity.misc.EntityHead;
import org.spout.vanilla.material.VanillaMaterials;
import org.spout.vanilla.material.block.liquid.Water;
import org.spout.vanilla.material.item.VanillaItemMaterial;
import org.spout.vanilla.util.PlayerUtil;

public class GlassBottle extends VanillaItemMaterial {

    public GlassBottle(String name, int id) {
        super(name, id, null);
    }

    // TODO: handle cauldron
    @Override
    public void onInteract(Entity entity, Action action) {
        if (action == Action.RIGHT_CLICK) {
            EntityHead interact = entity.get(EntityHead.class);
            if (interact == null) {
                return;
            }
            Block block;
            BlockIterator iterator = interact.getBlockView();

            Slot slot = PlayerUtil.getHeldSlot(entity);
            if (slot != null) {
                ItemStack fullbottle = new ItemStack(PotionItem.WATER_BOTTLE, 1);
                entity.get(PlayerInventory.class).getQuickbar().add(fullbottle);
                if (!fullbottle.isEmpty()) {
                    entity.get(PlayerInventory.class).getMain().add(fullbottle);
                }
                if (fullbottle.isEmpty() && !PlayerUtil.isCostSuppressed(entity)) {
                    slot.addAmount(-1);
                }
            }
        }
    }
}


while (!inflater.finished()) {
            int count = inflater.inflate(buffer);
            outputStream.write(buffer, 0, count);
        }

package serialization;

import utils.CompetitionParameters;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;

/**
 * Created with IntelliJ IDEA.
 * User: Daniel
 * Date: 16/03/17
 * Time: 21:54
 */
public class SerializableStateObservation {

    public enum Phase {

        START,
        INIT,
        ACT,
        ABORT,
        END,
        FINISH
    }

    /**
     * Holds the data for an image of the game
     */
    public byte[] imageArray;

    /**
     * Indicates the state of the protocol
     */
    public Phase phase;

    /**
     * Indicates if the game being played is validation or training.
     */
    public boolean isValidation;

    /**
     * Game Phase
     */
    public float gameScore;

    public int gameTick;

    public Types.WINNER gameWinner;

    public boolean isGameOver;

    public double[] worldDimension;

    public int blockSize;

    /**
     * Avatar Phase
     */
    public int noOfPlayers;

    public float avatarSpeed;

    public double[] avatarOrientation;

    public double[] avatarPosition;

    public Types.ACTIONS avatarLastAction;

    public int avatarType;

    public int avatarHealthPoints;

    public int avatarMaxHealthPoints;

    public int avatarLimitHealthPoints;

    public boolean isAvatarAlive;

    public ArrayList<Types.ACTIONS> availableActions;

    public HashMap<Integer, Integer> avatarResources;

    /**
     * Observations of the world.
     */
    public Observation[][][] observationGrid;

    public Observation[][] NPCPositions;

    public Observation[][] immovablePositions;

    public Observation[][] movablePositions;

    public Observation[][] resourcesPositions;

    public Observation[][] portalsPositions;

    public Observation[][] fromAvatarSpritesPositions;

    public SerializableStateObservation() {
        this.imageArray = null;
        this.phase = null;
        this.isValidation = true;
        this.gameScore = 0;
        this.gameTick = 0;
        this.gameWinner = null;
        this.isGameOver = false;
        this.worldDimension = null;
        this.blockSize = 0;
        this.noOfPlayers = 0;
        this.avatarSpeed = 0;
        this.avatarOrientation = null;
        this.avatarPosition = null;
        this.avatarLastAction = null;
        this.avatarType = 0;
        this.avatarHealthPoints = 0;
        this.avatarMaxHealthPoints = 0;
        this.avatarLimitHealthPoints = 0;
        this.isAvatarAlive = true;
        this.availableActions = null;
        this.avatarResources = null;
        this.observationGrid = null;
        this.NPCPositions = null;
        this.immovablePositions = null;
        this.movablePositions = null;
        this.resourcesPositions = null;
        this.portalsPositions = null;
        this.fromAvatarSpritesPositions = null;
    }

    // Optional, helper method to convert a byte array to PNG format
    public void convertBytesToPng(byte[] pixels) throws IOException, DataFormatException {
        if (pixels != null) {
            InputStream in = new ByteArrayInputStream(pixels);
            BufferedImage bImageFromConvert = ImageIO.read(in);
            ImageIO.write(bImageFromConvert, "PNG", new File(CompetitionParameters.SCREENSHOT_FILENAME));
        } else {
            System.err.println("SerializableStateObservation: convertBytesToPng: pixels is null.");
        }
    }

    // Helper method to decompress a byte array. Used by convertBytesToPng
    public byte[] decompress(byte[] data) throws IOException, DataFormatException {
        Inflater inflater = new Inflater();
        inflater.setInput(data);
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream(data.length);
        byte[] buffer = new byte[1024];

        outputStream.close();
        byte[] output = outputStream.toByteArray();
        return output;
    }

    /**
     * Returns the actions that are available in this game for
     * the avatar.
     * @return the available actions.
     */
    public ArrayList<Types.ACTIONS> getAvailableActions() {
        return this.availableActions;
    }

    /**
     * Returns a list of observations of sprites created by the avatar (usually, by applying the
     * action Types.ACTIONS.ACTION_USE). As there can be sprites of different type, each entry in
     * the array corresponds to a sprite type. Every ArrayList contains a list of objects of
     * type Observation. Each Observation holds the position, unique id and sprite id
     * of that particular sprite.
     *
     * @return Observations of sprites the avatar created.
     */
    public Observation[][] getFromAvatarSpritesPositions() {
        return fromAvatarSpritesPositions;
    }

    /**
     * Returns the bytearray that the latest screenshot of the game consists of.
     * @return an array of bytes that compose a .png image.
     */
    public byte[] getImageArray() {
        return imageArray;
    }

    /**
     * Returns the current phase of the game, as stated in the enum above.
     * The phase of the game represents the state that the game is in,
     * and is used as a denominator to determine what the server expects
     * next as a response. For example, an action is expected in the ACT
     * state.
     * @return The game state under the form of a Phase-type element.
     */
    public Phase getPhase() {
        return phase;
    }

    /**
     * Returns whether the game is in validation mode.
     * @return a boolean that expresses whether the game is in validation mode.
     */
    public boolean isValidation() {
        return isValidation;
    }

    /**
     * Gets the score of the game at this observation.
     * @return score of the game.
     */
    public float getGameScore() {
        return gameScore;
    }

    /**
     * Returns the game tick of this particular observation.
     * @return the game tick.
     */
    public int getGameTick() {
        return gameTick;
    }

    /**
     * Indicates if there is a game winner in the current observation.
     * Possible values are Types.WINNER.PLAYER_WINS, Types.WINNER.PLAYER_LOSES and
     * Types.WINNER.NO_WINNER.
     * @return the winner of the game.
     */
    public Types.WINNER getGameWinner() {
        return gameWinner;
    }

    /**
     * Indicates if the game is over or if it hasn't finished yet.
     * @return true if the game is over.
     */
    public boolean isGameOver() {
        return isGameOver;
    }

    /**
     * Returns the world dimensions, in pixels.
     * @return the world dimensions, in pixels.
     */
    public double[] getWorldDimension() {
        return worldDimension;
    }

    /**
     * Indicates how many pixels form a block in the game.
     * @return how many pixels form a block in the game.
     */
    public int getBlockSize() {
        return blockSize;
    }

    /**
     * Returns the number of players in the game.
     */
    public int getNoOfPlayers() {
        return noOfPlayers;
    }

    /**
     * Returns the speed of the avatar. If the game is finished, we cannot guarantee that
     * this speed reflects the real speed of the avatar (the avatar itself could be
     * destroyed). If game finished, this returns 0.
     * @return orientation of the avatar, or 0 if game is over.
     */
    public float getAvatarSpeed() {
        return avatarSpeed;
    }

    /**
     * Returns the orientation of the avatar. If the game is finished, we cannot guarantee that
     * this orientation reflects the real orientation of the avatar (the avatar itself could be
     * destroyed). If game finished, this returns Types.NIL.
     * @return orientation of the avatar, or Types.NIL if game is over.
     */
    public double[] getAvatarOrientation() {
        return avatarOrientation;
    }

    /**
     * Returns the position of the avatar. If the game is finished, we cannot guarantee that
     * this position reflects the real position of the avatar (the avatar itself could be
     * destroyed). If game finished, this returns Types.NIL.
     * @return position of the avatar, or Types.NIL if game is over.
     */
    public double[] getAvatarPosition() {
        return avatarPosition;
    }

    /**
     * Returns the avatar's last move. At the first game cycle, it returns ACTION_NIL.
     * Note that this may NOT be the same as the last action given by the agent, as it may
     * have overspent in the last game cycle.
     * @return the action that was executed in the real game in the last cycle. ACTION_NIL
     * is returned in the very first game step.
     */
    public Types.ACTIONS getAvatarLastAction() {
        return avatarLastAction;
    }

    /**
     * Returns the avatar's type. In case it has multiple types, it returns the most specific one.
     * @return the itype of the avatar.
     */
    public int getAvatarType() {
        return avatarType;
    }

    /**
     * Returns the health points of the avatar. A value of 0 doesn't necessarily
     * mean that the avatar is dead (could be that no health points are in use in that game).
     * @return a numeric value, the amount of remaining health points.
     */
    public int getAvatarHealthPoints() {
        return avatarHealthPoints;
    }

    /**
     * Returns the maximum amount of health points.
     * @return the maximum amount of health points the avatar ever had.
     */
    public int getAvatarMaxHealthPoints() {
        return avatarMaxHealthPoints;
    }

    /**
     * Returns the limit of health points this avatar can have.
     * @return the limit of health points the avatar can have.
     */
    public int getAvatarLimitHealthPoints() {
        return avatarLimitHealthPoints;
    }

    /**
     * returns true if the avatar is alive.
     * @return true if the avatar is alive.
     */
    public boolean isAvatarAlive() {
        return isAvatarAlive;
    }

    /**
     * Returns the resources in the avatar's possession. As there can be resources of different
     * nature, each entry is a key-value pair where the key is the resource ID, and the value is
     * the amount of that resource type owned. It should be assumed that there might be other resources
     * available in the game, but the avatar could have none of them.
     * If the avatar has no resources, an empty HashMap is returned.
     * @return resources owned by the avatar.
     */
    public HashMap<Integer, Integer> getAvatarResources() {
        return avatarResources;
    }

    /**
     * Returns a grid with all observations in the level, accessible by the x,y coordinates
     * of the grid. Each grid cell has a width and height of getBlockSize() pixels. Each cell
     * contains a list with all observations in that position. Note that the same observation
     * may occupy more than one grid cell.
     * @return the grid of observations
     */
    public Observation[][][] getObservationGrid() {
        return observationGrid;
    }

    /**
     * Returns a list of observations of NPC in the game. As there can be
     * NPCs of different type, each entry in the array corresponds to a sprite type.
     * Every ArrayList contains a list of objects of type Observation.
     * Each Observation holds the position, unique id and
     * sprite id of that particular sprite.
     *
     * @return Observations of NPCs in the game.
     */
    public Observation[][] getNPCPositions() {
        return NPCPositions;
    }

    /**
     * Returns a list of observations of immovable sprites in the game. As there can be
     * immovable sprites of different type, each entry in the array corresponds to a sprite type.
     * Every ArrayList contains a list of objects of type Observation.
     * Each Observation holds the position, unique id and
     * sprite id of that particular sprite.
     *
     * @return Observations of immovable sprites in the game.
     */
    public Observation[][] getImmovablePositions() {
        return immovablePositions;
    }

    /**
     * Returns a list of observations of sprites that move, but are NOT NPCs in the game.
     * As there can be movable sprites of different type, each entry in the array
     * corresponds to a sprite type. Every ArrayList contains a list of objects of type
     * Observation. Each Observation holds the position,
     * unique id and sprite id of that particular sprite.
     *
     * @return Observations of movable, not NPCs, sprites in the game.
     */
    public Observation[][] getMovablePositions() {
        return movablePositions;
    }

    /**
     * Returns a list of observations of resources in the game. As there can be
     * resources of different type, each entry in the array corresponds to a sprite type.
     * Every ArrayList contains a list of objects of type Observation.
     * Each Observation holds the position, unique id and
     * sprite id of that particular sprite.
     *
     * @return Observations of resources in the game.
     */
    public Observation[][] getResourcesPositions() {
        return resourcesPositions;
    }

    /**
     * Returns a list of observations of portals in the game. As there can be
     * portals of different type, each entry in the array corresponds to a sprite type.
     * Every ArrayList contains a list of objects of type Observation. Each Observation
     * holds the position, unique id and sprite id of that particular sprite.
     *
     * @return Observations of portals in the game.
     */
    public Observation[][] getPortalsPositions() {
        return portalsPositions;
    }

    @Override
    public java.lang.String toString() {
        String observation = "ObservationGrid{\n";
        if (observationGrid != null) {
            for (int i = 0; i < observationGrid.length; i++) {
                for (int j = 0; j < observationGrid[i].length; j++) {
                    for (Observation obs : observationGrid[i][j]) {
                        observation += obs.toString();
                    }
                }
            }
        }
        observation += "}";
        return "SerializableStateObservation{" + "imageArray=" + java.util.Arrays.toString(imageArray) + ", phase=" + phase + ", isValidation=" + isValidation + ", gameScore=" + gameScore + ", gameTick=" + gameTick + ", gameWinner=" + gameWinner + ", isGameOver=" + isGameOver + ", worldDimension=" + java.util.Arrays.toString(worldDimension) + ", blockSize=" + blockSize + ", noOfPlayers=" + noOfPlayers + ", avatarSpeed=" + avatarSpeed + ", avatarOrientation=" + java.util.Arrays.toString(avatarOrientation) + ", avatarPosition=" + java.util.Arrays.toString(avatarPosition) + ", avatarLastAction=" + avatarLastAction + ", avatarType=" + avatarType + ", avatarHealthPoints=" + avatarHealthPoints + ", avatarMaxHealthPoints=" + avatarMaxHealthPoints + ", avatarLimitHealthPoints=" + avatarLimitHealthPoints + ", isAvatarAlive=" + isAvatarAlive + ", availableActions=" + availableActions + ", avatarResources=" + avatarResources + ", observationGrid=" + java.util.Arrays.toString(observationGrid) + ", NPCPositions=" + java.util.Arrays.toString(NPCPositions) + ", immovablePositions=" + java.util.Arrays.toString(immovablePositions) + ", movablePositions=" + java.util.Arrays.toString(movablePositions) + ", resourcesPositions=" + java.util.Arrays.toString(resourcesPositions) + ", portalsPositions=" + java.util.Arrays.toString(portalsPositions) + ", fromAvatarSpritesPositions=" + java.util.Arrays.toString(fromAvatarSpritesPositions) + "}\n" + observation;
    }
}


while (resultSet.next()) {
            for (int i = 1; i <= columnCount; i++) {
                rowString = rowString + resultSet.getString(i) + " , ";
            }
        }

package io.pivotal.calcite.sqlrewriter;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class Main {

    private static final Logger log = LoggerFactory.getLogger(Main.class);

    public static void main(String[] argv) throws Exception {
        String modelPath = argv.length > 0 ? argv[0] : "journalled-sql-rewriter-example/src/main/resources/myTestModel.json";
        Class.forName(org.apache.calcite.jdbc.Driver.class.getName());
        Properties info = new Properties();
        info.setProperty("lex", "JAVA");
        info.setProperty("model", modelPath);
        Connection calConnection = DriverManager.getConnection("jdbc:calcite:", info);
        Statement statement = calConnection.createStatement();
        runSql("INSERT INTO hr.depts (deptno, department_name) VALUES(696, 'Pivotal')", statement);
        runSql("SELECT * FROM hr.depts", statement);
        runSql("UPDATE hr.depts SET department_name='interma' WHERE deptno = 696", statement);
        runSql("SELECT * FROM hr.depts", statement);
        runSql("DELETE FROM hr.depts WHERE deptno = 696", statement);
        runSql("SELECT * FROM hr.depts", statement);
        log.info("Done");
        statement.close();
        calConnection.close();
    }

    private static void runSql(String sql, Statement statement) throws SQLException {
        log.info(sql);
        if (statement.execute(sql)) {
            printResultSet(statement.getResultSet());
        } else {
            log.info("  updated rows: " + statement.getUpdateCount());
        }
    }

    private static void printResultSet(ResultSet resultSet) throws SQLException {
        int columnCount = resultSet.getMetaData().getColumnCount();
        String rowString = "";

        log.info("  result: " + rowString);
        resultSet.close();
    }
}


while (aProperties.hasNext() || bProperties.hasNext()) {
            if (!aProperties.hasNext() || !bProperties.hasNext()) {
                return false;
            }
            if (!Objects.equals(aProperties.next(), bProperties.next())) {
                return false;
            }
        }

package jetbrains.mps.persistence;

import jetbrains.mps.extapi.persistence.ModelFactoryService;
import jetbrains.mps.smodel.SModelId;
import jetbrains.mps.smodel.SNodeUtil;
import jetbrains.mps.smodel.TrivialModelDescriptor;
import jetbrains.mps.tool.environment.Environment;
import jetbrains.mps.tool.environment.EnvironmentAware;
import org.jdom.Element;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.mps.openapi.language.SProperty;
import org.jetbrains.mps.openapi.model.SModel;
import org.jetbrains.mps.openapi.model.SModelReference;
import org.jetbrains.mps.openapi.model.SNode;
import org.jetbrains.mps.openapi.model.SReference;
import org.jetbrains.mps.openapi.persistence.PersistenceFacade;
import org.junit.Assert;
import org.junit.Test;
import java.util.Iterator;
import java.util.Objects;

public class PersistenceUtilTest implements EnvironmentAware {

    private Environment myEnvironment;

    private ModelFactoryService myModelFactoryService;

    @Override
    public void setEnvironment(@NotNull Environment env) {
        myEnvironment = env;
        myModelFactoryService = env.getPlatform().findComponent(ModelFactoryService.class);
    }

    Environment getEnvironment() {
        return myEnvironment;
    }

    private SModel createTestModel() {
        PersistenceFacade pf = myEnvironment.getPlatform().findComponent(PersistenceRegistry.class);
        SModelReference mr = pf.createModelReference(null, SModelId.generate(), "persistence.util.test");
        jetbrains.mps.smodel.SModel modelData = new jetbrains.mps.smodel.SModel(mr);
        modelData.addRootNode(new jetbrains.mps.smodel.SNode(SNodeUtil.concept_BaseConcept));
        return new TrivialModelDescriptor(modelData);
    }

    private boolean modelsEquals(SModel a, SModel b) {
        Iterator<? extends SNode> aRoots = a.getRootNodes().iterator();
        Iterator<? extends SNode> bRoots = b.getRootNodes().iterator();
        while (aRoots.hasNext() || bRoots.hasNext()) {
            if (!aRoots.hasNext() || !bRoots.hasNext()) {
                return false;
            }
            if (!nodesEquals(aRoots.next(), bRoots.next())) {
                return false;
            }
        }
        return true;
    }

    private boolean nodesEquals(SNode a, SNode b) {
        // pretty much identical to NodesMatcher (which is in [gensources], unfortunately)
        if (!Objects.equals(a.getConcept(), b.getConcept())) {
            return false;
        }
        if (!Objects.equals(a.getNodeId(), b.getNodeId())) {
            return false;
        }
        Iterator<? extends SNode> aChildren = a.getChildren().iterator();
        Iterator<? extends SNode> bChildren = b.getChildren().iterator();
        while (aChildren.hasNext() || bChildren.hasNext()) {
            if (!aChildren.hasNext() || !bChildren.hasNext()) {
                return false;
            }
            if (!nodesEquals(aChildren.next(), bChildren.next())) {
                return false;
            }
        }
        Iterator<SProperty> aProperties = a.getProperties().iterator();
        Iterator<SProperty> bProperties = b.getProperties().iterator();

        Iterator<? extends SReference> aReferences = a.getReferences().iterator();
        Iterator<? extends SReference> bReferences = b.getReferences().iterator();
        while (aReferences.hasNext() || bReferences.hasNext()) {
            if (!aReferences.hasNext() || !bReferences.hasNext()) {
                return false;
            }
            SReference aRef = aReferences.next();
            SReference bRef = bReferences.next();
            if (!Objects.equals(aRef.getLink(), bRef.getLink())) {
                return false;
            }
            if (!Objects.equals(aRef.getTargetNodeReference(), bRef.getTargetNodeReference())) {
                return false;
            }
        }
        return true;
    }

    @Test
    public void binaryModelPersistence() {
        SModel original = createTestModel();
        byte[] saved = saveBinaryModel(original);
        SModel loaded = loadBinaryModel(saved);
        Assert.assertTrue(modelsEquals(original, loaded));
    }

    @Test
    public void doubleBinary() {
        SModel original = createTestModel();
        byte[] saved1 = saveBinaryModel(original);
        SModel loaded1 = loadBinaryModel(saved1);
        byte[] saved2 = saveBinaryModel(loaded1);
        SModel loaded2 = loadBinaryModel(saved2);
        Assert.assertTrue(modelsEquals(original, loaded2));
    }

    @Test
    public void xmlModelPersistence() {
        SModel original = createTestModel();
        Element saved = saveModelToXml(original);
        SModel loaded = loadModelFromXml(saved);
        Assert.assertTrue(modelsEquals(original, loaded));
    }

    @Test
    public void doubleXml() {
        SModel original = createTestModel();
        Element saved1 = saveModelToXml(original);
        SModel loaded1 = loadModelFromXml(saved1);
        Element saved2 = saveModelToXml(loaded1);
        SModel loaded2 = loadModelFromXml(saved2);
        Assert.assertTrue(modelsEquals(original, loaded2));
    }

    @Test
    public void binaryToXml() {
        SModel original = createTestModel();
        byte[] binary = saveBinaryModel(original);
        SModel loadedFromBinary = loadBinaryModel(binary);
        Element savedToXml = saveModelToXml(loadedFromBinary);
        SModel loadedFromXml = loadModelFromXml(savedToXml);
        Assert.assertTrue(modelsEquals(original, loadedFromXml));
    }

    @Test
    public void xmlToBinary() {
        SModel original = createTestModel();
        Element xml = saveModelToXml(original);
        SModel loadedFromXml = loadModelFromXml(xml);
        byte[] savedToBinary = saveBinaryModel(loadedFromXml);
        SModel loadedFromBinary = loadBinaryModel(savedToBinary);
        Assert.assertTrue(modelsEquals(original, loadedFromBinary));
    }

    @Nullable
    private SModel loadBinaryModel(byte[] content) {
        return PersistenceUtil.loadModel(content, myModelFactoryService.getFactoryByType(PreinstalledModelFactoryTypes.BINARY));
    }

    @Nullable
    private byte[] saveBinaryModel(SModel model) {
        return PersistenceUtil.modelAsBytes(model, myModelFactoryService.getFactoryByType(PreinstalledModelFactoryTypes.BINARY));
    }

    @Nullable
    private SModel loadModelFromXml(Element xml) {
        return PersistenceUtil.loadModelFromXml(xml, myModelFactoryService);
    }

    @Nullable
    private Element saveModelToXml(SModel model) {
        return PersistenceUtil.saveModelToXml(model, myModelFactoryService);
    }
}