Tools/SCViewer
Tools/SCViewer
10
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add LZ4 (de)compressor to target platform and add as input format

Browse Source
This commit is contained in:
Eyck Jentzsch 2022-05-16 20:34:23 +02:00
parent 8522627081
commit e629bdc5bc
4 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
plugins/com.minres.scviewer.database.text/META-INF/MANIFEST.MF
View File

@ -9,7 +9,8 @@ Import-Package: org.osgi.framework;version="1.3.0"
Require-Bundle: com.minres.scviewer.database,
org.eclipse.osgi.services;bundle-version="3.4.0",
com.google.guava;bundle-version="15.0.0",
org.eclipse.collections;bundle-version="10.4.0"
org.eclipse.collections;bundle-version="10.4.0",
org.apache.commons.compress;bundle-version="1.20.0"
Service-Component: OSGI-INF/component.xml
Bundle-ActivationPolicy: lazy
Automatic-Module-Name: com.minres.scviewer.database.text

29
plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java
View File

@ -13,6 +13,7 @@ package com.minres.scviewer.database.text;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@ -33,6 +34,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
import org.eclipse.collections.impl.map.mutable.UnifiedMap;
import org.mapdb.DB;
import org.mapdb.DBMaker;
@ -56,6 +58,8 @@ import com.minres.scviewer.database.tx.ITx;
*/
public class TextDbLoader implements IWaveformDbLoader {
enum FileType { NONE, PLAIN, GZIP, LZ4};
/** the file size limit of a zipped txlog where the loader starts to use a file mapped database */
private static final long MEMMAP_LIMIT=256l*1024l*1024l;
@ -190,8 +194,9 @@ public class TextDbLoader implements IWaveformDbLoader {
@Override
public boolean canLoad(File inputFile) {
if (!inputFile.isDirectory() && inputFile.exists()) {
boolean gzipped = isGzipped(inputFile);
try(InputStream stream = gzipped ? new GZIPInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){
FileType fType = getFileType(inputFile);
try(InputStream stream = fType==FileType.GZIP ? new GZIPInputStream(new FileInputStream(inputFile)) :
fType==FileType.LZ4? new FramedLZ4CompressorInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){
byte[] buffer = new byte[x.length];
int readCnt = stream.read(buffer, 0, x.length);
if (readCnt == x.length) {
@ -213,13 +218,18 @@ public class TextDbLoader implements IWaveformDbLoader {
* @param f the f
* @return true, if is gzipped
*/
private static boolean isGzipped(File f) {
private static FileType getFileType(File f) {
try (InputStream is = new FileInputStream(f)) {
byte[] signature = new byte[2];
byte[] signature = new byte[4];
int nread = is.read(signature); // read the gzip signature
return nread == 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b;
if(nread > 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b)
return FileType.GZIP;
else if(nread>4 && signature[0] == (byte) 0x04 && signature[1] == (byte) 0x22 && signature[2] == (byte) 0x4d && signature[3] == (byte) 0x18)
return FileType.LZ4;
else
return FileType.PLAIN;
} catch (IOException e) {
return false;
return FileType.NONE;
}
}
@ -235,8 +245,8 @@ public class TextDbLoader implements IWaveformDbLoader {
@Override
public void load(IWaveformDb db, File file) throws InputFormatException {
dispose();
boolean gzipped = isGzipped(file);
if (file.length() < MEMMAP_LIMIT * (gzipped ? 1 : 10)
FileType fType = getFileType(file);
if (file.length() < MEMMAP_LIMIT * (fType!=FileType.PLAIN ? 1 : 10)
|| "memory".equals(System.getProperty("ScvBackingDB", "file")))
mapDb = DBMaker.memoryDirectDB().make();
else {
@ -256,7 +266,8 @@ public class TextDbLoader implements IWaveformDbLoader {
try {
parser.txSink = mapDb.hashMap("transactions", Serializer.LONG, Serializer.JAVA).create();
parser.parseInput(gzipped ? new GZIPInputStream(new FileInputStream(file)) : new FileInputStream(file));
InputStream is = new BufferedInputStream(new FileInputStream(file));
parser.parseInput(fType==FileType.GZIP ? new GZIPInputStream(is) : fType==FileType.LZ4? new FramedLZ4CompressorInputStream(is) : is);
transactions = parser.txSink;
} catch (IllegalArgumentException | ArrayIndexOutOfBoundsException e) {
} catch (Exception e) {

2
releng/com.minres.scviewer.target/.settings/org.eclipse.core.resources.prefs
View File

@ -1,2 +0,0 @@
eclipse.preferences.version=1
encoding/<project>=UTF-8

5
releng/com.minres.scviewer.target/com.minres.scviewer.target.target
View File

@ -37,6 +37,11 @@
<repository location="https://download.eclipse.org/justj/jres/15/updates/release/latest"/>
<unit id="org.eclipse.justj.openjdk.hotspot.jre.minimal.stripped.feature.group" version="0.0.0"/>
</location>
<location includeAllPlatforms="false" includeConfigurePhase="true" includeMode="planner" includeSource="true" type="InstallableUnit">
<repository location="https://download.eclipse.org/tools/orbit/downloads/2021-09"/>
<unit id="org.apache.commons.compress" version="1.20.0.v20210713-1928"/>
<unit id="org.apache.commons.compress.source" version="1.20.0.v20210713-1928"/>
</location>
</locations>
<targetJRE path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-15"/>
</target>