From e629bdc5bc61311685cd5e1da9401170cbdcf904 Mon Sep 17 00:00:00 2001 From: Eyck Jentzsch Date: Mon, 16 May 2022 20:34:23 +0200 Subject: [PATCH] add LZ4 (de)compressor to target platform and add as input format --- .../META-INF/MANIFEST.MF | 3 +- .../scviewer/database/text/TextDbLoader.java | 29 +++++++++++++------ .../org.eclipse.core.resources.prefs | 2 -- .../com.minres.scviewer.target.target | 5 ++++ 4 files changed, 27 insertions(+), 12 deletions(-) delete mode 100644 releng/com.minres.scviewer.target/.settings/org.eclipse.core.resources.prefs diff --git a/plugins/com.minres.scviewer.database.text/META-INF/MANIFEST.MF b/plugins/com.minres.scviewer.database.text/META-INF/MANIFEST.MF index 6de81a4..a6887f6 100644 --- a/plugins/com.minres.scviewer.database.text/META-INF/MANIFEST.MF +++ b/plugins/com.minres.scviewer.database.text/META-INF/MANIFEST.MF @@ -9,7 +9,8 @@ Import-Package: org.osgi.framework;version="1.3.0" Require-Bundle: com.minres.scviewer.database, org.eclipse.osgi.services;bundle-version="3.4.0", com.google.guava;bundle-version="15.0.0", - org.eclipse.collections;bundle-version="10.4.0" + org.eclipse.collections;bundle-version="10.4.0", + org.apache.commons.compress;bundle-version="1.20.0" Service-Component: OSGI-INF/component.xml Bundle-ActivationPolicy: lazy Automatic-Module-Name: com.minres.scviewer.database.text diff --git a/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java b/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java index 16fcf6e..95e616e 100644 --- a/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java +++ b/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java @@ -13,6 +13,7 @@ package com.minres.scviewer.database.text; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; +import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -33,6 +34,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; +import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream; import org.eclipse.collections.impl.map.mutable.UnifiedMap; import org.mapdb.DB; import org.mapdb.DBMaker; @@ -56,6 +58,8 @@ import com.minres.scviewer.database.tx.ITx; */ public class TextDbLoader implements IWaveformDbLoader { + enum FileType { NONE, PLAIN, GZIP, LZ4}; + /** the file size limit of a zipped txlog where the loader starts to use a file mapped database */ private static final long MEMMAP_LIMIT=256l*1024l*1024l; @@ -190,8 +194,9 @@ public class TextDbLoader implements IWaveformDbLoader { @Override public boolean canLoad(File inputFile) { if (!inputFile.isDirectory() && inputFile.exists()) { - boolean gzipped = isGzipped(inputFile); - try(InputStream stream = gzipped ? new GZIPInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){ + FileType fType = getFileType(inputFile); + try(InputStream stream = fType==FileType.GZIP ? new GZIPInputStream(new FileInputStream(inputFile)) : + fType==FileType.LZ4? new FramedLZ4CompressorInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){ byte[] buffer = new byte[x.length]; int readCnt = stream.read(buffer, 0, x.length); if (readCnt == x.length) { @@ -213,13 +218,18 @@ public class TextDbLoader implements IWaveformDbLoader { * @param f the f * @return true, if is gzipped */ - private static boolean isGzipped(File f) { + private static FileType getFileType(File f) { try (InputStream is = new FileInputStream(f)) { - byte[] signature = new byte[2]; + byte[] signature = new byte[4]; int nread = is.read(signature); // read the gzip signature - return nread == 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b; + if(nread > 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b) + return FileType.GZIP; + else if(nread>4 && signature[0] == (byte) 0x04 && signature[1] == (byte) 0x22 && signature[2] == (byte) 0x4d && signature[3] == (byte) 0x18) + return FileType.LZ4; + else + return FileType.PLAIN; } catch (IOException e) { - return false; + return FileType.NONE; } } @@ -235,8 +245,8 @@ public class TextDbLoader implements IWaveformDbLoader { @Override public void load(IWaveformDb db, File file) throws InputFormatException { dispose(); - boolean gzipped = isGzipped(file); - if (file.length() < MEMMAP_LIMIT * (gzipped ? 1 : 10) + FileType fType = getFileType(file); + if (file.length() < MEMMAP_LIMIT * (fType!=FileType.PLAIN ? 1 : 10) || "memory".equals(System.getProperty("ScvBackingDB", "file"))) mapDb = DBMaker.memoryDirectDB().make(); else { @@ -256,7 +266,8 @@ public class TextDbLoader implements IWaveformDbLoader { try { parser.txSink = mapDb.hashMap("transactions", Serializer.LONG, Serializer.JAVA).create(); - parser.parseInput(gzipped ? new GZIPInputStream(new FileInputStream(file)) : new FileInputStream(file)); + InputStream is = new BufferedInputStream(new FileInputStream(file)); + parser.parseInput(fType==FileType.GZIP ? new GZIPInputStream(is) : fType==FileType.LZ4? new FramedLZ4CompressorInputStream(is) : is); transactions = parser.txSink; } catch (IllegalArgumentException | ArrayIndexOutOfBoundsException e) { } catch (Exception e) { diff --git a/releng/com.minres.scviewer.target/.settings/org.eclipse.core.resources.prefs b/releng/com.minres.scviewer.target/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 99f26c0..0000000 --- a/releng/com.minres.scviewer.target/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,2 +0,0 @@ -eclipse.preferences.version=1 -encoding/=UTF-8 diff --git a/releng/com.minres.scviewer.target/com.minres.scviewer.target.target b/releng/com.minres.scviewer.target/com.minres.scviewer.target.target index e4d2353..f16827f 100644 --- a/releng/com.minres.scviewer.target/com.minres.scviewer.target.target +++ b/releng/com.minres.scviewer.target/com.minres.scviewer.target.target @@ -37,6 +37,11 @@ + + + + + \ No newline at end of file