add LZ4 (de)compressor to target platform and add as input format

This commit is contained in:
Eyck Jentzsch 2022-05-16 20:34:23 +02:00
parent 8522627081
commit e629bdc5bc
4 changed files with 27 additions and 12 deletions

View File

@ -9,7 +9,8 @@ Import-Package: org.osgi.framework;version="1.3.0"
Require-Bundle: com.minres.scviewer.database, Require-Bundle: com.minres.scviewer.database,
org.eclipse.osgi.services;bundle-version="3.4.0", org.eclipse.osgi.services;bundle-version="3.4.0",
com.google.guava;bundle-version="15.0.0", com.google.guava;bundle-version="15.0.0",
org.eclipse.collections;bundle-version="10.4.0" org.eclipse.collections;bundle-version="10.4.0",
org.apache.commons.compress;bundle-version="1.20.0"
Service-Component: OSGI-INF/component.xml Service-Component: OSGI-INF/component.xml
Bundle-ActivationPolicy: lazy Bundle-ActivationPolicy: lazy
Automatic-Module-Name: com.minres.scviewer.database.text Automatic-Module-Name: com.minres.scviewer.database.text

View File

@ -13,6 +13,7 @@ package com.minres.scviewer.database.text;
import java.beans.PropertyChangeListener; import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport; import java.beans.PropertyChangeSupport;
import java.io.BufferedInputStream;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
@ -33,6 +34,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
import org.eclipse.collections.impl.map.mutable.UnifiedMap; import org.eclipse.collections.impl.map.mutable.UnifiedMap;
import org.mapdb.DB; import org.mapdb.DB;
import org.mapdb.DBMaker; import org.mapdb.DBMaker;
@ -56,6 +58,8 @@ import com.minres.scviewer.database.tx.ITx;
*/ */
public class TextDbLoader implements IWaveformDbLoader { public class TextDbLoader implements IWaveformDbLoader {
enum FileType { NONE, PLAIN, GZIP, LZ4};
/** the file size limit of a zipped txlog where the loader starts to use a file mapped database */ /** the file size limit of a zipped txlog where the loader starts to use a file mapped database */
private static final long MEMMAP_LIMIT=256l*1024l*1024l; private static final long MEMMAP_LIMIT=256l*1024l*1024l;
@ -190,8 +194,9 @@ public class TextDbLoader implements IWaveformDbLoader {
@Override @Override
public boolean canLoad(File inputFile) { public boolean canLoad(File inputFile) {
if (!inputFile.isDirectory() && inputFile.exists()) { if (!inputFile.isDirectory() && inputFile.exists()) {
boolean gzipped = isGzipped(inputFile); FileType fType = getFileType(inputFile);
try(InputStream stream = gzipped ? new GZIPInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){ try(InputStream stream = fType==FileType.GZIP ? new GZIPInputStream(new FileInputStream(inputFile)) :
fType==FileType.LZ4? new FramedLZ4CompressorInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){
byte[] buffer = new byte[x.length]; byte[] buffer = new byte[x.length];
int readCnt = stream.read(buffer, 0, x.length); int readCnt = stream.read(buffer, 0, x.length);
if (readCnt == x.length) { if (readCnt == x.length) {
@ -213,13 +218,18 @@ public class TextDbLoader implements IWaveformDbLoader {
* @param f the f * @param f the f
* @return true, if is gzipped * @return true, if is gzipped
*/ */
private static boolean isGzipped(File f) { private static FileType getFileType(File f) {
try (InputStream is = new FileInputStream(f)) { try (InputStream is = new FileInputStream(f)) {
byte[] signature = new byte[2]; byte[] signature = new byte[4];
int nread = is.read(signature); // read the gzip signature int nread = is.read(signature); // read the gzip signature
return nread == 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b; if(nread > 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b)
return FileType.GZIP;
else if(nread>4 && signature[0] == (byte) 0x04 && signature[1] == (byte) 0x22 && signature[2] == (byte) 0x4d && signature[3] == (byte) 0x18)
return FileType.LZ4;
else
return FileType.PLAIN;
} catch (IOException e) { } catch (IOException e) {
return false; return FileType.NONE;
} }
} }
@ -235,8 +245,8 @@ public class TextDbLoader implements IWaveformDbLoader {
@Override @Override
public void load(IWaveformDb db, File file) throws InputFormatException { public void load(IWaveformDb db, File file) throws InputFormatException {
dispose(); dispose();
boolean gzipped = isGzipped(file); FileType fType = getFileType(file);
if (file.length() < MEMMAP_LIMIT * (gzipped ? 1 : 10) if (file.length() < MEMMAP_LIMIT * (fType!=FileType.PLAIN ? 1 : 10)
|| "memory".equals(System.getProperty("ScvBackingDB", "file"))) || "memory".equals(System.getProperty("ScvBackingDB", "file")))
mapDb = DBMaker.memoryDirectDB().make(); mapDb = DBMaker.memoryDirectDB().make();
else { else {
@ -256,7 +266,8 @@ public class TextDbLoader implements IWaveformDbLoader {
try { try {
parser.txSink = mapDb.hashMap("transactions", Serializer.LONG, Serializer.JAVA).create(); parser.txSink = mapDb.hashMap("transactions", Serializer.LONG, Serializer.JAVA).create();
parser.parseInput(gzipped ? new GZIPInputStream(new FileInputStream(file)) : new FileInputStream(file)); InputStream is = new BufferedInputStream(new FileInputStream(file));
parser.parseInput(fType==FileType.GZIP ? new GZIPInputStream(is) : fType==FileType.LZ4? new FramedLZ4CompressorInputStream(is) : is);
transactions = parser.txSink; transactions = parser.txSink;
} catch (IllegalArgumentException | ArrayIndexOutOfBoundsException e) { } catch (IllegalArgumentException | ArrayIndexOutOfBoundsException e) {
} catch (Exception e) { } catch (Exception e) {

View File

@ -1,2 +0,0 @@
eclipse.preferences.version=1
encoding/<project>=UTF-8

View File

@ -37,6 +37,11 @@
<repository location="https://download.eclipse.org/justj/jres/15/updates/release/latest"/> <repository location="https://download.eclipse.org/justj/jres/15/updates/release/latest"/>
<unit id="org.eclipse.justj.openjdk.hotspot.jre.minimal.stripped.feature.group" version="0.0.0"/> <unit id="org.eclipse.justj.openjdk.hotspot.jre.minimal.stripped.feature.group" version="0.0.0"/>
</location> </location>
<location includeAllPlatforms="false" includeConfigurePhase="true" includeMode="planner" includeSource="true" type="InstallableUnit">
<repository location="https://download.eclipse.org/tools/orbit/downloads/2021-09"/>
<unit id="org.apache.commons.compress" version="1.20.0.v20210713-1928"/>
<unit id="org.apache.commons.compress.source" version="1.20.0.v20210713-1928"/>
</location>
</locations> </locations>
<targetJRE path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-15"/> <targetJRE path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-15"/>
</target> </target>