SCViewer/plugins/com.minres.scviewer.databas.../src/com/minres/scviewer/database/text/TextDbLoader.java

550 lines
17 KiB
Java
Raw Normal View History

/*******************************************************************************
* Copyright (c) 2012 IT Just working.
* Copyright (c) 2020 MINRES Technologies GmbH
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* IT Just working - initial API and implementation
*******************************************************************************/
package com.minres.scviewer.database.text;
2021-01-09 20:10:58 +01:00
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
2021-01-08 20:50:24 +01:00
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
2021-01-08 15:04:30 +01:00
import java.util.List;
import java.util.Map;
2021-01-14 23:13:11 +01:00
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
2021-01-08 20:31:24 +01:00
import org.eclipse.collections.impl.map.mutable.UnifiedMap;
2021-01-08 15:04:30 +01:00
import org.mapdb.DB;
import org.mapdb.DBMaker;
2021-01-12 11:46:21 +01:00
import org.mapdb.HTreeMap;
2021-01-08 15:04:30 +01:00
import org.mapdb.Serializer;
import com.google.common.collect.HashMultimap;
import com.minres.scviewer.database.AssociationType;
import com.minres.scviewer.database.DataType;
import com.minres.scviewer.database.EventKind;
import com.minres.scviewer.database.IWaveform;
import com.minres.scviewer.database.IWaveformDb;
import com.minres.scviewer.database.IWaveformDbLoader;
import com.minres.scviewer.database.InputFormatException;
import com.minres.scviewer.database.RelationType;
2021-01-04 17:39:11 +01:00
import com.minres.scviewer.database.RelationTypeFactory;
import com.minres.scviewer.database.tx.ITx;
/**
* The Class TextDbLoader.
*/
public class TextDbLoader implements IWaveformDbLoader {
2021-01-12 11:30:37 +01:00
/** the file size limit of a zipped txlog where the loader starts to use a file mapped database */
private static final long MEMMAP_LIMIT=256l*1024l*1024l;
private static final long MAPDB_INITIAL_ALLOC = 512l*1024l*1024l;
private static final long MAPDB_INCREMENTAL_ALLOC = 128l*1024l*1024l;
/** The max time. */
private Long maxTime = 0L;
/** The map db. */
DB mapDb = null;
2021-01-08 15:04:30 +01:00
/** The attr values. */
final List<String> attrValues = new ArrayList<>();
/** The relation types. */
2021-01-08 20:31:24 +01:00
final Map<String, RelationType> relationTypes = UnifiedMap.newMap();
2021-01-04 17:39:11 +01:00
/** The tx streams. */
final Map<Long, TxStream> txStreams = UnifiedMap.newMap();
/** The tx generators. */
final Map<Long, TxGenerator> txGenerators = UnifiedMap.newMap();
/** The transactions. */
Map<Long, ScvTx> transactions = null;
/** The attribute types. */
final Map<String, TxAttributeType> attributeTypes = UnifiedMap.newMap();
/** The relations in. */
2021-01-08 15:04:30 +01:00
final HashMultimap<Long, ScvRelation> relationsIn = HashMultimap.create();
/** The relations out. */
2021-01-08 15:04:30 +01:00
final HashMultimap<Long, ScvRelation> relationsOut = HashMultimap.create();
/** The tx cache. */
HashMap<Long, Tx> txCache = new HashMap<>();
2021-01-08 15:04:30 +01:00
/** The threads. */
List<Thread> threads = new ArrayList<>();
2021-01-09 20:10:58 +01:00
/** The pcs. */
protected PropertyChangeSupport pcs = new PropertyChangeSupport(this);
/** The Constant x. */
static final byte[] x = "scv_tr_stream".getBytes();
2021-01-14 23:13:11 +01:00
/**
* Adds the property change listener.
*
* @param l the l
*/
@Override
public void addPropertyChangeListener(PropertyChangeListener l) {
pcs.addPropertyChangeListener(l);
}
/**
* Removes the property change listener.
*
* @param l the l
*/
@Override
public void removePropertyChangeListener(PropertyChangeListener l) {
pcs.removePropertyChangeListener(l);
}
/**
* Gets the max time.
*
* @return the max time
*/
@Override
2021-02-27 14:59:00 +01:00
public long getMaxTime() {
return maxTime;
}
2021-01-14 23:13:11 +01:00
/**
* Gets the transaction.
*
* @param txId the tx id
* @return the transaction
*/
public ITx getTransaction(long txId) {
if (txCache.containsKey(txId))
return txCache.get(txId);
if(transactions.containsKey(txId)) {
2021-01-14 23:36:07 +01:00
Tx tx = new Tx(this, transactions.get(txId));
2021-01-14 23:13:11 +01:00
txCache.put(txId, tx);
return tx;
} else {
throw new IllegalArgumentException();
}
}
public ScvTx getScvTx(long id) {
2021-01-14 23:13:11 +01:00
if(transactions.containsKey(id))
return transactions.get(id);
else
throw new IllegalArgumentException();
}
/**
* Gets the all waves.
*
* @return the all waves
*/
@Override
public Collection<IWaveform> getAllWaves() {
return new ArrayList<>(txStreams.values());
}
2021-01-14 23:13:11 +01:00
/**
* Gets the all relation types.
*
* @return the all relation types
*/
public Collection<RelationType> getAllRelationTypes() {
return relationTypes.values();
}
2021-01-09 20:10:58 +01:00
/**
* Can load.
*
* @param inputFile the input file
* @return true, if successful
*/
@Override
public boolean canLoad(File inputFile) {
if (!inputFile.isDirectory() && inputFile.exists()) {
boolean gzipped = isGzipped(inputFile);
try(InputStream stream = gzipped ? new GZIPInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){
byte[] buffer = new byte[x.length];
int readCnt = stream.read(buffer, 0, x.length);
if (readCnt == x.length) {
for (int i = 0; i < x.length; i++)
if (buffer[i] != x[i])
return false;
}
return true;
} catch (Exception e) {
return false;
}
}
return false;
}
2021-01-14 23:13:11 +01:00
/**
* Checks if is gzipped.
*
* @param f the f
* @return true, if is gzipped
*/
private static boolean isGzipped(File f) {
try (InputStream is = new FileInputStream(f)) {
byte[] signature = new byte[2];
int nread = is.read(signature); // read the gzip signature
return nread == 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b;
} catch (IOException e) {
return false;
}
}
/**
* Load.
*
* @param db the db
* @param file the file
* @return true, if successful
* @throws InputFormatException the input format exception
*/
@SuppressWarnings("unchecked")
@Override
2021-01-09 20:10:58 +01:00
public void load(IWaveformDb db, File file) throws InputFormatException {
2021-01-08 15:04:30 +01:00
dispose();
2021-01-02 18:04:48 +01:00
boolean gzipped = isGzipped(file);
2021-01-12 11:30:37 +01:00
if (file.length() < MEMMAP_LIMIT * (gzipped ? 1 : 10)
|| "memory".equals(System.getProperty("ScvBackingDB", "file")))
2021-01-12 11:30:37 +01:00
mapDb = DBMaker.memoryDirectDB().make();
2021-01-08 15:04:30 +01:00
else {
File mapDbFile;
try {
mapDbFile = File.createTempFile("." + file.getName(), ".mapdb", null /* file.parentFile */);
2021-01-08 20:50:24 +01:00
Files.delete(Paths.get(mapDbFile.getPath()));
2021-01-09 23:24:00 +01:00
} catch (IOException e) {
throw new InputFormatException(e.toString());
2021-01-08 15:04:30 +01:00
}
mapDb = DBMaker.fileDB(mapDbFile).fileMmapEnable() // Always enable mmap
2021-01-12 11:30:37 +01:00
.fileMmapPreclearDisable().allocateStartSize(MAPDB_INITIAL_ALLOC)
.allocateIncrement(MAPDB_INCREMENTAL_ALLOC).cleanerHackEnable().make();
2021-01-08 15:04:30 +01:00
mapDbFile.deleteOnExit();
}
2021-01-09 20:10:58 +01:00
TextDbParser parser = new TextDbParser(this);
2021-01-02 18:04:48 +01:00
try {
2021-01-12 11:46:21 +01:00
parser.txSink = mapDb.hashMap("transactions", Serializer.LONG, Serializer.JAVA).create();
parser.parseInput(gzipped ? new GZIPInputStream(new FileInputStream(file)) : new FileInputStream(file));
2021-01-12 11:46:21 +01:00
transactions = parser.txSink;
} catch (IllegalArgumentException | ArrayIndexOutOfBoundsException e) {
} catch (Exception e) {
2021-01-09 23:24:00 +01:00
throw new InputFormatException(e.toString());
}
2021-01-14 23:13:11 +01:00
txStreams.values().parallelStream().forEach(TxStream::calculateConcurrency);
}
/**
* Dispose.
*/
2021-01-08 20:50:24 +01:00
@Override
2021-01-08 15:04:30 +01:00
public void dispose() {
2021-01-08 20:31:24 +01:00
attrValues.clear();
2021-01-08 15:04:30 +01:00
relationTypes.clear();
txStreams.clear();
txGenerators.clear();
transactions = null;
attributeTypes.clear();
2021-01-08 15:04:30 +01:00
relationsIn.clear();
relationsOut.clear();
if (mapDb != null) {
2021-01-08 15:04:30 +01:00
mapDb.close();
mapDb = null;
2021-01-08 15:04:30 +01:00
}
}
/**
* The Class TextDbParser.
*/
2021-01-02 18:04:48 +01:00
static class TextDbParser {
/** The Constant scv_tr_stream. */
static final Pattern scv_tr_stream = Pattern
.compile("^scv_tr_stream\\s+\\(ID (\\d+),\\s+name\\s+\"([^\"]+)\",\\s+kind\\s+\"([^\"]+)\"\\)$");
/** The Constant scv_tr_generator. */
static final Pattern scv_tr_generator = Pattern
.compile("^scv_tr_generator\\s+\\(ID\\s+(\\d+),\\s+name\\s+\"([^\"]+)\",\\s+scv_tr_stream\\s+(\\d+),$");
/** The Constant begin_attribute. */
static final Pattern begin_attribute = Pattern
.compile("^begin_attribute \\(ID (\\d+), name \"([^\"]+)\", type \"([^\"]+)\"\\)$");
/** The Constant end_attribute. */
static final Pattern end_attribute = Pattern
.compile("^end_attribute \\(ID (\\d+), name \"([^\"]+)\", type \"([^\"]+)\"\\)$");
/** The loader. */
2021-01-02 18:04:48 +01:00
final TextDbLoader loader;
/** The transaction by id. */
2021-01-08 15:04:30 +01:00
HashMap<Long, ScvTx> transactionById = new HashMap<>();
/** The tx sink. */
2021-01-12 11:46:21 +01:00
HTreeMap<Long, ScvTx> txSink;
/** The reader. */
BufferedReader reader = null;
/** The generator. */
TxGenerator generator = null;
/** The attr value lut. */
2021-01-08 20:31:24 +01:00
Map<String, Integer> attrValueLut = new HashMap<>();
/**
* Instantiates a new text db parser.
*
* @param loader the loader
*/
2021-01-02 18:04:48 +01:00
public TextDbParser(TextDbLoader loader) {
super();
this.loader = loader;
}
/**
* Parses the input.
*
* @param inputStream the input stream
* @throws IOException Signals that an I/O exception has occurred.
2021-01-09 23:24:00 +01:00
* @throws InputFormatException Signals that the input format is wrong
*/
2021-01-09 23:24:00 +01:00
void parseInput(InputStream inputStream) throws IOException, InputFormatException {
2021-01-08 20:50:24 +01:00
reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
2021-01-02 18:04:48 +01:00
String curLine = reader.readLine();
String nextLine = null;
while ((nextLine = reader.readLine()) != null && curLine != null) {
curLine = parseLine(curLine, nextLine);
}
if (curLine != null)
2021-01-02 18:04:48 +01:00
parseLine(curLine, nextLine);
2021-01-14 23:13:11 +01:00
for(Entry<Long, ScvTx> e: transactionById.entrySet()) {
ScvTx scvTx = e.getValue();
scvTx.endTime=loader.maxTime;
txSink.put(e.getKey(), scvTx);
}
2021-01-02 18:04:48 +01:00
}
/**
* Gets the attr type.
*
* @param name the name
* @param dataType the data type
* @param type the type
* @return the attr type
*/
private TxAttributeType getAttrType(String name, DataType dataType, AssociationType type) {
String key = name + "-" + dataType.toString();
TxAttributeType res;
if (loader.attributeTypes.containsKey(key)) {
res = loader.attributeTypes.get(key);
} else {
res = new TxAttributeType(name, dataType, type);
loader.attributeTypes.put(key, res);
}
return res;
}
/**
* Parses the line.
*
* @param curLine the cur line
* @param nextLine the next line
* @return the string
* @throws IOException Signals that an I/O exception has occurred.
2021-01-09 23:24:00 +01:00
* @throws InputFormatException Signals that the input format is wrong
*/
2021-01-09 23:24:00 +01:00
private String parseLine(String curLine, String nextLine) throws IOException, InputFormatException {
2021-01-02 18:04:48 +01:00
String[] tokens = curLine.split("\\s+");
if ("tx_record_attribute".equals(tokens[0])) {
2021-01-02 18:04:48 +01:00
Long id = Long.parseLong(tokens[1]);
2021-01-12 11:30:37 +01:00
String name = tokens[2].substring(1, tokens[2].length()-1);
2021-01-02 18:04:48 +01:00
DataType type = DataType.valueOf(tokens[3]);
2021-01-12 11:30:37 +01:00
String remaining = tokens.length > 5 ? String.join(" ", Arrays.copyOfRange(tokens, 5, tokens.length)) : "";
TxAttributeType attrType = getAttrType(name, type, AssociationType.RECORD);
2021-01-08 20:31:24 +01:00
transactionById.get(id).attributes.add(new TxAttribute(attrType, getAttrString(attrType, remaining)));
} else if ("tx_begin".equals(tokens[0])) {
2021-01-02 18:04:48 +01:00
Long id = Long.parseLong(tokens[1]);
Long genId = Long.parseLong(tokens[2]);
TxGenerator gen = loader.txGenerators.get(genId);
ScvTx scvTx = new ScvTx(id, gen.stream.getId(), genId,
Long.parseLong(tokens[3]) * stringToScale(tokens[4]));
loader.maxTime = loader.maxTime > scvTx.beginTime ? loader.maxTime : scvTx.beginTime;
if (nextLine != null && nextLine.charAt(0) == 'a') {
int idx = 0;
while (nextLine != null && nextLine.charAt(0) == 'a') {
String[] attrTokens = nextLine.split("\\s+");
2021-01-08 20:31:24 +01:00
TxAttributeType attrType = gen.beginAttrs.get(idx);
TxAttribute attr = new TxAttribute(attrType, getAttrString(attrType, attrTokens[1]));
2021-01-08 15:04:30 +01:00
scvTx.attributes.add(attr);
2021-01-02 18:04:48 +01:00
idx++;
nextLine = reader.readLine();
2021-01-02 18:04:48 +01:00
}
}
2021-01-08 15:04:30 +01:00
transactionById.put(id, scvTx);
} else if ("tx_end".equals(tokens[0])) {
2021-01-02 18:04:48 +01:00
Long id = Long.parseLong(tokens[1]);
2021-01-08 15:04:30 +01:00
ScvTx scvTx = transactionById.get(id);
assert Long.parseLong(tokens[2]) == scvTx.generatorId;
scvTx.endTime = Long.parseLong(tokens[3]) * stringToScale(tokens[4]);
loader.maxTime = loader.maxTime > scvTx.endTime ? loader.maxTime : scvTx.endTime;
2021-01-08 15:04:30 +01:00
TxGenerator gen = loader.txGenerators.get(scvTx.generatorId);
TxStream stream = loader.txStreams.get(gen.stream.getId());
if (scvTx.beginTime == scvTx.endTime) {
2021-01-15 08:53:06 +01:00
stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime));
gen.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime));
} else {
2021-01-15 08:53:06 +01:00
stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime));
gen.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime));
stream.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime));
gen.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime));
}
if (nextLine != null && nextLine.charAt(0) == 'a') {
int idx = 0;
while (nextLine != null && nextLine.charAt(0) == 'a') {
String[] attrTokens = nextLine.split("\\s+");
2021-01-08 20:31:24 +01:00
TxAttributeType attrType = gen.endAttrs.get(idx);
TxAttribute attr = new TxAttribute(attrType, getAttrString(attrType, attrTokens[1]));
2021-01-08 15:04:30 +01:00
scvTx.attributes.add(attr);
2021-01-02 18:04:48 +01:00
idx++;
nextLine = reader.readLine();
2021-01-02 18:04:48 +01:00
}
}
2021-01-12 11:46:21 +01:00
txSink.put(scvTx.getId(), scvTx);
transactionById.remove(id);
} else if ("tx_relation".equals(tokens[0])) {
Long tr2 = Long.parseLong(tokens[2]);
Long tr1 = Long.parseLong(tokens[3]);
String relType = tokens[1].substring(1, tokens[1].length() - 1);
if (!loader.relationTypes.containsKey(relType))
2021-01-04 17:39:11 +01:00
loader.relationTypes.put(relType, RelationTypeFactory.create(relType));
ScvRelation rel = new ScvRelation(loader.relationTypes.get(relType), tr1, tr2);
loader.relationsOut.put(tr1, rel);
loader.relationsIn.put(tr2, rel);
} else if ("scv_tr_stream".equals(tokens[0])) {
2021-01-02 18:04:48 +01:00
Matcher matcher = scv_tr_stream.matcher(curLine);
if (matcher.matches()) {
Long id = Long.parseLong(matcher.group(1));
TxStream stream = new TxStream(loader, id, matcher.group(2), matcher.group(3));
2021-01-09 20:10:58 +01:00
add(id, stream);
2021-01-02 18:04:48 +01:00
}
} else if ("scv_tr_generator".equals(tokens[0])) {
2021-01-02 18:04:48 +01:00
Matcher matcher = scv_tr_generator.matcher(curLine);
if ((matcher.matches())) {
Long id = Long.parseLong(matcher.group(1));
TxStream stream = loader.txStreams.get(Long.parseLong(matcher.group(3)));
generator = new TxGenerator(loader, id, matcher.group(2), stream);
2021-01-09 20:10:58 +01:00
add(id, generator);
2021-01-02 18:04:48 +01:00
}
} else if ("begin_attribute".equals(tokens[0])) {
2021-01-02 18:04:48 +01:00
Matcher matcher = begin_attribute.matcher(curLine);
if ((matcher.matches())) {
TxAttributeType attrType = getAttrType(matcher.group(2), DataType.valueOf(matcher.group(3)),
AssociationType.BEGIN);
generator.beginAttrs.add(attrType);
2021-01-02 18:04:48 +01:00
}
} else if ("end_attribute".equals(tokens[0])) {
2021-01-02 18:04:48 +01:00
Matcher matcher = end_attribute.matcher(curLine);
if ((matcher.matches())) {
TxAttributeType attrType = getAttrType(matcher.group(2), DataType.valueOf(matcher.group(3)),
AssociationType.END);
generator.endAttrs.add(attrType);
2021-01-02 18:04:48 +01:00
}
} else if (")".equals(tokens[0])) {
generator = null;
2021-01-02 18:04:48 +01:00
} else
2021-01-09 23:24:00 +01:00
throw new InputFormatException("Don't know what to do with: '" + curLine + "'");
2021-01-02 18:04:48 +01:00
return nextLine;
}
/**
* Gets the attr string.
*
* @param attrType the attr type
* @param string the string
* @return the attr string
*/
2021-01-08 20:31:24 +01:00
private String getAttrString(TxAttributeType attrType, String string) {
String value;
switch (attrType.getDataType()) {
2021-01-08 20:31:24 +01:00
case STRING:
case ENUMERATION:
value = string.substring(1, string.length() - 1);
2021-01-08 20:31:24 +01:00
break;
default:
value = string;
2021-01-08 20:31:24 +01:00
}
if (attrValueLut.containsKey(value)) {
2021-01-08 20:31:24 +01:00
return loader.attrValues.get(attrValueLut.get(value));
} else {
attrValueLut.put(value, loader.attrValues.size());
loader.attrValues.add(value);
return value;
}
}
/**
* String to scale.
*
* @param scale the scale
* @return the long
*/
private long stringToScale(String scale) {
2021-01-02 18:04:48 +01:00
String cmp = scale.trim();
if ("fs".equals(cmp))
return 1L;
if ("ps".equals(cmp))
return 1000L;
if ("ns".equals(cmp))
return 1000000L;
if ("us".equals(cmp))
return 1000000000L;
if ("ms".equals(cmp))
return 1000000000000L;
if ("s".equals(cmp))
return 1000000000000000L;
2021-01-02 18:04:48 +01:00
return 1L;
}
2021-01-09 20:10:58 +01:00
private void add(Long id, TxStream stream) {
loader.txStreams.put(id, stream);
loader.pcs.firePropertyChange(IWaveformDbLoader.STREAM_ADDED, null, stream);
}
private void add(Long id, TxGenerator generator) {
loader.txGenerators.put(id, generator);
loader.pcs.firePropertyChange(IWaveformDbLoader.GENERATOR_ADDED, null, generator);
}
}
}