diff --git a/Ghidra/Features/Base/data/file_extension_icons.xml b/Ghidra/Features/Base/data/file_extension_icons.xml
index 52a3bdd107..50b56ac3ea 100644
--- a/Ghidra/Features/Base/data/file_extension_icons.xml
+++ b/Ghidra/Features/Base/data/file_extension_icons.xml
@@ -38,6 +38,7 @@
+
diff --git a/Ghidra/Features/Base/src/main/help/help/topics/FileSystemBrowserPlugin/FileSystemBrowserPlugin.html b/Ghidra/Features/Base/src/main/help/help/topics/FileSystemBrowserPlugin/FileSystemBrowserPlugin.html
index 3019e8dbe3..ca03dfbab1 100644
--- a/Ghidra/Features/Base/src/main/help/help/topics/FileSystemBrowserPlugin/FileSystemBrowserPlugin.html
+++ b/Ghidra/Features/Base/src/main/help/help/topics/FileSystemBrowserPlugin/FileSystemBrowserPlugin.html
@@ -123,6 +123,19 @@
of the file systems will display that file system's browser tree.
+
Clear Cached Passwords
+
+
+
Clears any cached passwords that previously entered when accessing a password
+ protected container file.
+
+
+
Refresh
+
+
+
Refreshes the status of the selected items.
+
+
Close
@@ -150,6 +163,20 @@
+
Password Dialog
+
+
+
This dialog is used to prompt for a password when opening a password protected container.
+
+
If the password isn't known, the Cancel button will stop Ghidra from re-prompting
+ for the password for the current file during the current operation (but the user may be
+ re-prompted again later depending on the logic of the operation), and Cancel All
+ will stop Ghidra from prompting for passwords for any file during the current operation.
+
+
Passwords can also be provided via a text file specified on the java command line, useful
+ for headless operations. See ghidra.formats.gfilesystem.crypto.CmdLinePasswordProvider
+
+
How To Handle Unsupported File Systems
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteArrayProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteArrayProvider.java
index 37abf30fa2..f12f16a5c3 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteArrayProvider.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteArrayProvider.java
@@ -20,7 +20,7 @@ import java.io.*;
import ghidra.formats.gfilesystem.FSRL;
/**
- * An implementation of {@link ByteProvider} where the underlying bytes are supplied by a static
+ * An implementation of {@link ByteProvider} where the underlying bytes are supplied by a
* byte array.
*
* NOTE: Use of this class is discouraged when the byte array could be large.
@@ -66,6 +66,16 @@ public class ByteArrayProvider implements ByteProvider {
// don't do anything for now
}
+ /**
+ * Releases the byte storage of this instance.
+ *
+ * This is separate from the normal close() to avoid changing existing
+ * behavior of this class.
+ */
+ public void hardClose() {
+ srcBytes = new byte[0];
+ }
+
@Override
public FSRL getFSRL() {
return fsrl;
@@ -78,12 +88,12 @@ public class ByteArrayProvider implements ByteProvider {
@Override
public String getName() {
- return name;
+ return fsrl != null ? fsrl.getName() : name;
}
@Override
public String getAbsolutePath() {
- return "";
+ return fsrl != null ? fsrl.getPath() : "";
}
@Override
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProvider.java
index c0c0a80ad4..b871e35dd9 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProvider.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProvider.java
@@ -25,6 +25,11 @@ import ghidra.formats.gfilesystem.FileSystemService;
*/
public interface ByteProvider extends Closeable {
+ /**
+ * A static re-usable empty {@link ByteProvider} instance.
+ */
+ public static final ByteProvider EMPTY_BYTEPROVIDER = new EmptyByteProvider();
+
/**
* Returns the {@link FSRL} of the underlying file for this byte provider,
* or null if this byte provider is not associated with a file.
@@ -109,9 +114,17 @@ public interface ByteProvider extends Closeable {
*
* The caller is responsible for closing the returned {@link InputStream} instance.
*
+ * If you need to override this default implementation, please document why your inputstream
+ * is needed.
+ *
* @param index where in the {@link ByteProvider} to start the {@link InputStream}
* @return the {@link InputStream}
* @throws IOException if an I/O error occurs
*/
- public InputStream getInputStream(long index) throws IOException;
+ default public InputStream getInputStream(long index) throws IOException {
+ if (index < 0 || index > length()) {
+ throw new IOException("Invalid start position: " + index);
+ }
+ return new ByteProviderInputStream(this, index);
+ }
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderInputStream.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderInputStream.java
index 4b0f420050..32b9f25bf1 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderInputStream.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderInputStream.java
@@ -18,25 +18,122 @@ package ghidra.app.util.bin;
import java.io.IOException;
import java.io.InputStream;
+/**
+ * An {@link InputStream} that reads from a {@link ByteProvider}.
+ *
+ * Does not close the underlying ByteProvider when closed itself.
+ *
+ */
public class ByteProviderInputStream extends InputStream {
- private ByteProvider provider;
- private long offset;
- private long length;
- private long nextOffset;
- public ByteProviderInputStream( ByteProvider provider, long offset, long length ) {
+ /**
+ * An {@link InputStream} that reads from a {@link ByteProvider}, and DOES
+ * {@link ByteProvider#close() close()} the underlying ByteProvider when
+ * closed itself.
+ *
+ */
+ public static class ClosingInputStream extends ByteProviderInputStream {
+ /**
+ * Creates an {@link InputStream} that reads from a {@link ByteProvider}, that
+ * DOES {@link ByteProvider#close() close()} the underlying ByteProvider when
+ * closed itself.
+ *
+ * @param provider the {@link ByteProvider} to read from (and close)
+ */
+ public ClosingInputStream(ByteProvider provider) {
+ super(provider);
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ if (provider != null) {
+ provider.close();
+ provider = null;
+ }
+ }
+ }
+
+ protected ByteProvider provider;
+ private long currentPosition;
+ private long markPosition;
+
+ /**
+ * Creates an InputStream that uses a ByteProvider as its source of bytes.
+ *
+ * @param provider the {@link ByteProvider} to wrap
+ */
+ public ByteProviderInputStream(ByteProvider provider) {
+ this(provider, 0);
+ }
+
+ /**
+ * Creates an InputStream that uses a ByteProvider as its source of bytes.
+ *
+ * @param provider the {@link ByteProvider} to wrap
+ * @param startPosition starting position in the provider
+ */
+ public ByteProviderInputStream(ByteProvider provider, long startPosition) {
this.provider = provider;
- this.offset = offset;
- this.length = length;
- this.nextOffset = offset;
+ this.markPosition = startPosition;
+ this.currentPosition = startPosition;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // nothing to do here
+ }
+
+ @Override
+ public int available() throws IOException {
+ return (int) Math.min(provider.length() - currentPosition, Integer.MAX_VALUE);
+ }
+
+ @Override
+ public boolean markSupported() {
+ return true;
+ }
+
+ @Override
+ public synchronized void mark(int readlimit) {
+ this.markPosition = currentPosition;
+ }
+
+ @Override
+ public synchronized void reset() throws IOException {
+ // synchronized because the base class's method is synchronized.
+ this.currentPosition = markPosition;
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (n <= 0) {
+ return 0;
+ }
+ long newPosition = Math.min(provider.length(), currentPosition + n);
+ long skipped = newPosition - currentPosition;
+ currentPosition = newPosition;
+ return skipped;
}
@Override
public int read() throws IOException {
- if ( nextOffset < offset + length ) {
- return provider.readByte( nextOffset++ ) & 0xff;
+ return (currentPosition < provider.length())
+ ? Byte.toUnsignedInt(provider.readByte(currentPosition++))
+ : -1;
+ }
+
+ @Override
+ public int read(byte[] b, int bufferOffset, int len) throws IOException {
+ long eof = provider.length();
+ if (currentPosition >= eof) {
+ return -1;
}
- return -1;
+ len = (int) Math.min(len, eof - currentPosition);
+ byte[] bytes = provider.readBytes(currentPosition, len);
+ System.arraycopy(bytes, 0, b, bufferOffset, len);
+ currentPosition += len;
+ return len;
}
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderWrapper.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderWrapper.java
index 0358c096b5..b608c8b7f4 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderWrapper.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderWrapper.java
@@ -15,12 +15,13 @@
*/
package ghidra.app.util.bin;
-import java.io.*;
+import java.io.File;
+import java.io.IOException;
import ghidra.formats.gfilesystem.FSRL;
/**
- * Creates a {@link ByteProvider} constrained to a sub-section of an existing {@link ByteProvider}.
+ * A {@link ByteProvider} constrained to a sub-section of an existing {@link ByteProvider}.
*/
public class ByteProviderWrapper implements ByteProvider {
private ByteProvider provider;
@@ -29,7 +30,21 @@ public class ByteProviderWrapper implements ByteProvider {
private FSRL fsrl;
/**
- * Constructs a {@link ByteProviderWrapper} around the specified {@link ByteProvider}
+ * Creates a wrapper around a {@link ByteProvider} that contains the same bytes as the specified
+ * provider, but with a new {@link FSRL} identity.
+ *
+ *
+ * @param provider {@link ByteProvider} to wrap
+ * @param fsrl {@link FSRL} identity for the instance
+ * @throws IOException if error
+ */
+ public ByteProviderWrapper(ByteProvider provider, FSRL fsrl) throws IOException {
+ this(provider, 0, provider.length(), fsrl);
+ }
+
+ /**
+ * Constructs a {@link ByteProviderWrapper} around the specified {@link ByteProvider},
+ * constrained to a subsection of the provider.
*
* @param provider the {@link ByteProvider} to wrap
* @param subOffset the offset in the {@link ByteProvider} of where to start the new
@@ -41,13 +56,14 @@ public class ByteProviderWrapper implements ByteProvider {
}
/**
- * Constructs a {@link ByteProviderWrapper} around the specified {@link ByteProvider}
+ * Constructs a {@link ByteProviderWrapper} around the specified {@link ByteProvider},
+ * constrained to a subsection of the provider.
*
* @param provider the {@link ByteProvider} to wrap
* @param subOffset the offset in the {@link ByteProvider} of where to start the new
* {@link ByteProviderWrapper}
* @param subLength the length of the new {@link ByteProviderWrapper}
- * @param fsrl FSRL identity of the file this ByteProvider represents
+ * @param fsrl {@link FSRL} identity of the file this ByteProvider represents
*/
public ByteProviderWrapper(ByteProvider provider, long subOffset, long subLength, FSRL fsrl) {
this.provider = provider;
@@ -57,8 +73,8 @@ public class ByteProviderWrapper implements ByteProvider {
}
@Override
- public void close() {
- // don't do anything for now
+ public void close() throws IOException {
+ // do not close the wrapped provider
}
@Override
@@ -68,24 +84,22 @@ public class ByteProviderWrapper implements ByteProvider {
@Override
public File getFile() {
- return provider.getFile();
- }
-
- @Override
- public InputStream getInputStream(long index) throws IOException {
- return new ByteProviderInputStream(this, index, subLength - index);
+ // there is no file that represents the actual contents of the subrange, so return null
+ return null;
}
@Override
public String getName() {
- return provider.getName() + "[0x" + Long.toHexString(subOffset) + ",0x" +
- Long.toHexString(subLength) + "]";
+ return (fsrl != null)
+ ? fsrl.getName()
+ : String.format("%s[0x%x,0x%x]", provider.getName(), subOffset, subLength);
}
@Override
public String getAbsolutePath() {
- return provider.getAbsolutePath() + "[0x" + Long.toHexString(subOffset) + ",0x" +
- Long.toHexString(subLength) + "]";
+ return (fsrl != null)
+ ? fsrl.getPath()
+ : String.format("%s[0x%x,0x%x]", provider.getAbsolutePath(), subOffset, subLength);
}
@Override
@@ -95,10 +109,7 @@ public class ByteProviderWrapper implements ByteProvider {
@Override
public boolean isValidIndex(long index) {
- if (provider.isValidIndex(index)) {
- return index >= subOffset && index < subLength;
- }
- return false;
+ return (0 <= index && index < subLength) && provider.isValidIndex(subOffset + index);
}
@Override
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/EmptyByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/EmptyByteProvider.java
new file mode 100644
index 0000000000..a45fbd58a5
--- /dev/null
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/EmptyByteProvider.java
@@ -0,0 +1,103 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.util.bin;
+
+import java.io.*;
+
+import ghidra.formats.gfilesystem.FSRL;
+
+/**
+ * A {@link ByteProvider} that has no contents.
+ *
+ */
+public class EmptyByteProvider implements ByteProvider {
+
+ private final FSRL fsrl;
+
+ /**
+ * Create an instance with a null identity
+ */
+ public EmptyByteProvider() {
+ this(null);
+ }
+
+ /**
+ * Create an instance with the specified {@link FSRL} identity.
+ *
+ * @param fsrl {@link FSRL} identity for this instance
+ */
+ public EmptyByteProvider(FSRL fsrl) {
+ this.fsrl = fsrl;
+ }
+
+ @Override
+ public FSRL getFSRL() {
+ return fsrl;
+ }
+
+ @Override
+ public File getFile() {
+ return null;
+ }
+
+ @Override
+ public String getName() {
+ return fsrl != null ? fsrl.getName() : null;
+ }
+
+ @Override
+ public String getAbsolutePath() {
+ return fsrl != null ? fsrl.getPath() : null;
+ }
+
+ @Override
+ public byte readByte(long index) throws IOException {
+ throw new IOException("Not supported");
+ }
+
+ @Override
+ public byte[] readBytes(long index, long length) throws IOException {
+ if (index != 0 || length != 0) {
+ throw new IOException("Not supported");
+ }
+ return new byte[0];
+ }
+
+ @Override
+ public long length() {
+ return 0;
+ }
+
+ @Override
+ public boolean isValidIndex(long index) {
+ return false;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // do nothing
+ }
+
+ @Override
+ public InputStream getInputStream(long index) throws IOException {
+ if (index != 0) {
+ throw new IOException("Invalid offset");
+ }
+ return InputStream.nullInputStream();
+ }
+
+
+}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/FileByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/FileByteProvider.java
new file mode 100644
index 0000000000..8462ce0b60
--- /dev/null
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/FileByteProvider.java
@@ -0,0 +1,323 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.util.bin;
+
+import java.io.*;
+import java.nio.file.AccessMode;
+
+import org.apache.commons.collections4.map.ReferenceMap;
+
+import ghidra.formats.gfilesystem.FSRL;
+import ghidra.util.Msg;
+import ghidra.util.datastruct.LRUMap;
+
+/**
+ * A {@link ByteProvider} that reads its bytes from a file.
+ *
+ */
+public class FileByteProvider implements ByteProvider, MutableByteProvider {
+
+ static final int BUFFER_SIZE = 64 * 1024;
+ private static final int BUFFERS_TO_PIN = 4;
+
+ private FSRL fsrl;
+ private File file;
+ private RandomAccessFile raf;
+ private ReferenceMap buffers = new ReferenceMap<>();
+ private LRUMap lruBuffers = new LRUMap<>(BUFFERS_TO_PIN); // only used to pin a small set of recently used buffers in memory. not used for lookup
+ private long currentLength;
+ private AccessMode accessMode; // probably wrong Enum class, but works for now
+
+ /**
+ * Creates a new instance.
+ *
+ * @param file {@link File} to open
+ * @param fsrl {@link FSRL} identity of the file
+ * @param accessMode {@link AccessMode#READ} or {@link AccessMode#WRITE}
+ * @throws IOException if error
+ */
+ public FileByteProvider(File file, FSRL fsrl, AccessMode accessMode)
+ throws IOException {
+ this.file = file;
+ this.fsrl = fsrl;
+ this.accessMode = accessMode;
+ this.raf = new RandomAccessFile(file, accessModeToString(accessMode));
+ this.currentLength = raf.length();
+ }
+
+ /**
+ * Returns the access mode the file was opened with.
+ *
+ * @return {@link AccessMode} used to open file
+ */
+ public AccessMode getAccessMode() {
+ return accessMode;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (raf != null) {
+ raf.close();
+ raf = null;
+ }
+ buffers.clear();
+ lruBuffers.clear();
+ }
+
+ @Override
+ public File getFile() {
+ return file;
+ }
+
+ @Override
+ public String getName() {
+ return fsrl != null ? fsrl.getName() : file.getName();
+ }
+
+ @Override
+ public String getAbsolutePath() {
+ return fsrl != null ? fsrl.getPath() : file.getAbsolutePath();
+ }
+
+ @Override
+ public FSRL getFSRL() {
+ return fsrl;
+ }
+
+ @Override
+ public long length() throws IOException {
+ return currentLength;
+ }
+
+ @Override
+ public boolean isValidIndex(long index) {
+ return 0 <= index && index < currentLength;
+ }
+
+ @Override
+ public byte readByte(long index) throws IOException {
+ ensureBounds(index, 1);
+ Buffer fileBuffer = getBufferFor(index);
+ int ofs = fileBuffer.getBufferOffset(index);
+
+ return fileBuffer.bytes[ofs];
+ }
+
+ @Override
+ public byte[] readBytes(long index, long length) throws IOException {
+ ensureBounds(index, length);
+ if (length > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException();
+ }
+ int len = (int) length;
+ byte[] result = new byte[len];
+ int bytesRead = readBytes(index, result, 0, len);
+ if (bytesRead != len) {
+ throw new IOException("Unable to read " + len + " bytes at " + index);
+ }
+ return result;
+ }
+
+ /**
+ * Read bytes at the specified index into the given byte array.
+ *
+ * See {@link InputStream#read(byte[], int, int)}.
+ *
+ *
+ * @param index file offset to start reading
+ * @param buffer byte array that will receive the bytes
+ * @param offset offset inside the byte array to place the bytes
+ * @param length number of bytes to read
+ * @return number of actual bytes read
+ * @throws IOException if error
+ */
+ public int readBytes(long index, byte[] buffer, int offset, int length) throws IOException {
+ ensureBounds(index, 0);
+ length = (int) Math.min(currentLength - index, length);
+
+ int totalBytesRead = 0;
+ while (length > 0) {
+ Buffer fileBuffer = getBufferFor(index);
+ int ofs = fileBuffer.getBufferOffset(index);
+ int bytesToReadFromThisBuffer = Math.min(fileBuffer.len - ofs, length);
+ System.arraycopy(fileBuffer.bytes, ofs, buffer, totalBytesRead,
+ bytesToReadFromThisBuffer);
+
+ length -= bytesToReadFromThisBuffer;
+ index += bytesToReadFromThisBuffer;
+ totalBytesRead += bytesToReadFromThisBuffer;
+ }
+ return totalBytesRead;
+ }
+
+ @Override
+ protected void finalize() {
+ if (raf != null) {
+ Msg.warn(this, "FAIL TO CLOSE " + file);
+ }
+ }
+
+ /**
+ * Writes bytes to the specified offset in the file.
+ *
+ * @param index the location in the file to starting writing
+ * @param buffer bytes to write
+ * @param offset offset in the buffer byte array to start
+ * @param length number of bytes to write
+ * @throws IOException if bad {@link AccessMode} or other io error
+ */
+ public synchronized void writeBytes(long index, byte[] buffer, int offset, int length)
+ throws IOException {
+ if (accessMode != AccessMode.WRITE) {
+ throw new IOException("Not write mode");
+ }
+
+ doWriteBytes(index, buffer, offset, length);
+ long writeEnd = index + length;
+ currentLength = Math.max(currentLength, writeEnd);
+
+ // after writing new bytes to the file, update
+ // any buffers that we can completely fill with the contents of
+ // this write buffer, and invalidate any buffers that we can't
+ // completely fill (they can be re-read in a normal fashion later when needed)
+ while (length > 0) {
+ long bufferPos = getBufferPos(index);
+ int bufferOfs = (int) (index - bufferPos);
+ int bytesAvailForThisBuffer = Math.min(length, BUFFER_SIZE - bufferOfs);
+
+ Buffer fileBuffer = buffers.get(bufferPos);
+ if (fileBuffer != null) {
+ if (bufferOfs == 0 && length >= BUFFER_SIZE) {
+ System.arraycopy(buffer, offset, fileBuffer.bytes, 0, BUFFER_SIZE);
+ fileBuffer.len = BUFFER_SIZE;
+ }
+ else {
+ buffers.remove(bufferPos);
+ lruBuffers.remove(bufferPos);
+ }
+ }
+ index += bytesAvailForThisBuffer;
+ offset += bytesAvailForThisBuffer;
+ length -= bytesAvailForThisBuffer;
+ }
+ }
+
+ @Override
+ public void writeByte(long index, byte value) throws IOException {
+ writeBytes(index, new byte[] { value }, 0, 1);
+ }
+
+ @Override
+ public void writeBytes(long index, byte[] values) throws IOException {
+ writeBytes(index, values, 0, values.length);
+ }
+
+ //------------------------------------------------------------------------------------
+ /**
+ * Reads bytes from the file.
+ *
+ * Protected by synchronized lock. (See {@link #getBufferFor(long)}).
+ *
+ * @param index file position of where to read
+ * @param buffer byte array that will receive bytes
+ * @return actual number of byte read
+ * @throws IOException if error
+ */
+ protected int doReadBytes(long index, byte[] buffer) throws IOException {
+ raf.seek(index);
+ return raf.read(buffer, 0, buffer.length);
+ }
+
+ /**
+ * Writes the specified bytes to the file.
+ *
+ * Protected by synchronized lock (See {@link #writeBytes(long, byte[], int, int)})
+ *
+ * @param index file position of where to write
+ * @param buffer byte array containing bytes to write
+ * @param offset offset inside of byte array to start
+ * @param length number of bytes from buffer to write
+ * @throws IOException if error
+ */
+ protected void doWriteBytes(long index, byte[] buffer, int offset, int length)
+ throws IOException {
+ raf.seek(index);
+ raf.write(buffer, offset, length);
+ }
+
+ //------------------------------------------------------------------------------------
+ private void ensureBounds(long index, long length) throws IOException {
+ if (index < 0 || index > currentLength) {
+ throw new IOException("Invalid index: " + index);
+ }
+ if (index + length > currentLength) {
+ throw new IOException("Unable to read past EOF: " + index + ", " + length);
+ }
+ }
+
+ private long getBufferPos(long index) {
+ return (index / BUFFER_SIZE) * BUFFER_SIZE;
+ }
+
+ private synchronized Buffer getBufferFor(long pos) throws IOException {
+ long bufferPos = getBufferPos(pos);
+ if (bufferPos >= currentLength) {
+ throw new EOFException();
+ }
+ Buffer buffer = buffers.get(bufferPos);
+ if (buffer == null) {
+ buffer = new Buffer(bufferPos, (int) Math.min(currentLength - bufferPos, BUFFER_SIZE));
+ int bytesRead = doReadBytes(bufferPos, buffer.bytes);
+ if (bytesRead != buffer.len) {
+ buffer.len = bytesRead;
+ // warn?
+ }
+ buffers.put(bufferPos, buffer);
+ }
+ lruBuffers.put(bufferPos, buffer);
+ return buffer;
+ }
+
+ private static class Buffer {
+ long pos; // absolute position in file of this buffer
+ int len; // number of valid bytes in buffer
+ byte[] bytes;
+
+ Buffer(long pos, int len) {
+ this.pos = pos;
+ this.len = len;
+ this.bytes = new byte[len];
+ }
+
+ int getBufferOffset(long filePos) throws EOFException {
+ int ofs = (int) (filePos - pos);
+ if (ofs >= len) {
+ throw new EOFException();
+ }
+ return ofs;
+ }
+ }
+
+ private String accessModeToString(AccessMode mode) {
+ switch (mode) {
+ default:
+ case READ:
+ return "r";
+ case WRITE:
+ return "rw";
+ }
+ }
+}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/MemBufferByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/MemBufferByteProvider.java
index c8f50af18e..497efbde89 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/MemBufferByteProvider.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/MemBufferByteProvider.java
@@ -105,53 +105,4 @@ public class MemBufferByteProvider implements ByteProvider {
return bytes;
}
- @Override
- public InputStream getInputStream(long index) throws IOException {
- if (index < 0 || index > Integer.MAX_VALUE) {
- throw new IOException("index out of range");
- }
- return new MemBufferProviderInputStream((int) index);
- }
-
- private class MemBufferProviderInputStream extends InputStream {
-
- private int initialOffset;
- private int offset;
-
- MemBufferProviderInputStream(int offset) {
- this.offset = offset;
- this.initialOffset = offset;
- }
-
- @Override
- public int read() throws IOException {
- byte b = readByte(offset++);
- return b & 0xff;
- }
-
- @Override
- public int read(byte[] b, int off, int len) {
- byte[] bytes = new byte[len];
- int count = buffer.getBytes(bytes, offset);
- System.arraycopy(bytes, 0, b, off, count);
- offset += count;
- return count;
- }
-
- @Override
- public int available() {
- return (int) length() - offset;
- }
-
- @Override
- public synchronized void reset() throws IOException {
- offset = initialOffset;
- }
-
- @Override
- public void close() throws IOException {
- // not applicable
- }
- }
-
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedFileByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedFileByteProvider.java
new file mode 100644
index 0000000000..45d5dc6f62
--- /dev/null
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedFileByteProvider.java
@@ -0,0 +1,97 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.util.bin;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.AccessMode;
+
+import ghidra.formats.gfilesystem.FSRL;
+
+/**
+ * A {@link ByteProvider} that reads from an on-disk file, but obfuscates / de-obfuscates the
+ * contents of the file when reading / writing.
+ */
+public class ObfuscatedFileByteProvider extends FileByteProvider {
+
+ // @formatter:off
+ // copied from ChainedBuffer
+ static final byte[] XOR_MASK_BYTES = new byte[] {
+ (byte)0x59, (byte)0xea, (byte)0x67, (byte)0x23, (byte)0xda, (byte)0xb8, (byte)0x00, (byte)0xb8,
+ (byte)0xc3, (byte)0x48, (byte)0xdd, (byte)0x8b, (byte)0x21, (byte)0xd6, (byte)0x94, (byte)0x78,
+ (byte)0x35, (byte)0xab, (byte)0x2b, (byte)0x7e, (byte)0xb2, (byte)0x4f, (byte)0x82, (byte)0x4e,
+ (byte)0x0e, (byte)0x16, (byte)0xc4, (byte)0x57, (byte)0x12, (byte)0x8e, (byte)0x7e, (byte)0xe6,
+ (byte)0xb6, (byte)0xbd, (byte)0x56, (byte)0x91, (byte)0x57, (byte)0x72, (byte)0xe6, (byte)0x91,
+ (byte)0xdc, (byte)0x52, (byte)0x2e, (byte)0xf2, (byte)0x1a, (byte)0xb7, (byte)0xd6, (byte)0x6f,
+ (byte)0xda, (byte)0xde, (byte)0xe8, (byte)0x48, (byte)0xb1, (byte)0xbb, (byte)0x50, (byte)0x6f,
+ (byte)0xf4, (byte)0xdd, (byte)0x11, (byte)0xee, (byte)0xf2, (byte)0x67, (byte)0xfe, (byte)0x48,
+ (byte)0x8d, (byte)0xae, (byte)0x69, (byte)0x1a, (byte)0xe0, (byte)0x26, (byte)0x8c, (byte)0x24,
+ (byte)0x8e, (byte)0x17, (byte)0x76, (byte)0x51, (byte)0xe2, (byte)0x60, (byte)0xd7, (byte)0xe6,
+ (byte)0x83, (byte)0x65, (byte)0xd5, (byte)0xf0, (byte)0x7f, (byte)0xf2, (byte)0xa0, (byte)0xd6,
+ (byte)0x4b, (byte)0xbd, (byte)0x24, (byte)0xd8, (byte)0xab, (byte)0xea, (byte)0x9e, (byte)0xa6,
+ (byte)0x48, (byte)0x94, (byte)0x3e, (byte)0x7b, (byte)0x2c, (byte)0xf4, (byte)0xce, (byte)0xdc,
+ (byte)0x69, (byte)0x11, (byte)0xf8, (byte)0x3c, (byte)0xa7, (byte)0x3f, (byte)0x5d, (byte)0x77,
+ (byte)0x94, (byte)0x3f, (byte)0xe4, (byte)0x8e, (byte)0x48, (byte)0x20, (byte)0xdb, (byte)0x56,
+ (byte)0x32, (byte)0xc1, (byte)0x87, (byte)0x01, (byte)0x2e, (byte)0xe3, (byte)0x7f, (byte)0x40,
+
+ };
+ // @formatter:on
+
+ /**
+ * Creates an instance of {@link ObfuscatedFileByteProvider}.
+ *
+ * @param file {@link File} to read from / write to
+ * @param fsrl {@link FSRL} identity of this file
+ * @param accessMode {@link AccessMode#READ} or {@link AccessMode#WRITE}
+ * @throws IOException if error
+ */
+ public ObfuscatedFileByteProvider(File file, FSRL fsrl, AccessMode accessMode)
+ throws IOException {
+ super(file, fsrl, accessMode);
+ }
+
+ @Override
+ public File getFile() {
+ // obfuscated file isn't readable, so force null
+ return null;
+ }
+
+ @Override
+ protected int doReadBytes(long index, byte[] buffer) throws IOException {
+ int bytesRead = super.doReadBytes(index, buffer);
+ for (int i = 0; i < bytesRead; i++) {
+ long byteIndex = index + i;
+ int xorMaskIndex = (int) (byteIndex % XOR_MASK_BYTES.length);
+ byte xorMask = XOR_MASK_BYTES[xorMaskIndex];
+ buffer[i] ^= xorMask;
+ }
+ return bytesRead;
+ }
+
+ @Override
+ protected void doWriteBytes(long index, byte[] buffer, int offset, int length)
+ throws IOException {
+ byte[] tmpBuffer = new byte[length];
+ for (int i = 0; i < length; i++) {
+ long byteIndex = index + i;
+ int xorMaskIndex = (int) (byteIndex % XOR_MASK_BYTES.length);
+ byte xorMask = XOR_MASK_BYTES[xorMaskIndex];
+ tmpBuffer[i] = (byte) (buffer[i + offset] ^ xorMask);
+ }
+ super.doWriteBytes(index, tmpBuffer, 0, length);
+ }
+
+}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedInputStream.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedInputStream.java
new file mode 100644
index 0000000000..76d75ac436
--- /dev/null
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedInputStream.java
@@ -0,0 +1,99 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.util.bin;
+
+import java.io.*;
+
+
+/**
+ * An {@link InputStream} wrapper that de-obfuscates the bytes being read from the underlying
+ * stream.
+ */
+public class ObfuscatedInputStream extends InputStream {
+
+ private InputStream delegate;
+ private long currentPosition;
+
+ /**
+ * Creates instance.
+ *
+ * @param delegate {@link InputStream} to wrap
+ */
+ public ObfuscatedInputStream(InputStream delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ super.close();
+ }
+
+ @Override
+ public int read() throws IOException {
+ byte[] buffer = new byte[1];
+ int bytesRead = read(buffer, 0, 1);
+ return bytesRead == 1 ? Byte.toUnsignedInt(buffer[0]) : -1;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int bytesRead = delegate.read(b, off, len);
+
+ for (int i = 0; i < bytesRead; i++, currentPosition++) {
+ int xorMaskIndex =
+ (int) (currentPosition % ObfuscatedFileByteProvider.XOR_MASK_BYTES.length);
+ byte xorMask = ObfuscatedFileByteProvider.XOR_MASK_BYTES[xorMaskIndex];
+ b[off + i] ^= xorMask;
+
+ }
+ return bytesRead;
+ }
+
+ /**
+ * Entry point to enable command line users to retrieve the contents of an obfuscated
+ * file.
+ *
+ * @param args either ["--help"], or [ "input_filename", "output_filename" ]
+ * @throws IOException if error
+ */
+ public static void main(String[] args) throws IOException {
+ if (args.length != 2 || (args.length > 1 && args[0].equals("--help"))) {
+ System.err.println("De-Obfuscator Usage:");
+ System.err.println("\t" + ObfuscatedInputStream.class.getName() +
+ " obfuscated_input_filename_path plain_dest_output_filename_path");
+ System.err.println("");
+ System.err.println("\tExample:");
+ System.err.println("\t\t" + ObfuscatedInputStream.class.getName() +
+ " /tmp/myuserid-Ghidra/fscache2/aa/bb/aabbccddeeff00112233445566778899 /tmp/aabbccddeeff00112233445566778899.plaintext");
+ System.err.println("");
+ return;
+ }
+ File obfuscatedInputFile = new File(args[0]);
+ File plainTextOutputFile = new File(args[1]);
+
+ try (InputStream is = new ObfuscatedInputStream(new FileInputStream(obfuscatedInputFile));
+ OutputStream os = new FileOutputStream(plainTextOutputFile)) {
+
+ byte buffer[] = new byte[4096];
+ int bytesRead;
+ while ((bytesRead = is.read(buffer)) > 0) {
+ os.write(buffer, 0, bytesRead);
+ }
+ }
+ }
+
+}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedOutputStream.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedOutputStream.java
new file mode 100644
index 0000000000..32103895d2
--- /dev/null
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedOutputStream.java
@@ -0,0 +1,69 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.util.bin;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * An {@link OutputStream} wrapper that obfuscates the bytes being written to the underlying
+ * stream.
+ */
+public class ObfuscatedOutputStream extends OutputStream {
+
+ private OutputStream delegate;
+ private long currentPosition;
+
+ /**
+ * Creates instance.
+ *
+ * @param delegate {@link OutputStream} to wrap
+ */
+ public ObfuscatedOutputStream(OutputStream delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ super.close();
+ }
+
+ @Override
+ public void flush() throws IOException {
+ delegate.flush();
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ byte[] tmpBuffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ long byteIndex = currentPosition + i;
+ int xorMaskIndex =
+ (int) (byteIndex % ObfuscatedFileByteProvider.XOR_MASK_BYTES.length);
+ byte xorMask = ObfuscatedFileByteProvider.XOR_MASK_BYTES[xorMaskIndex];
+ tmpBuffer[i] = (byte) (b[i + off] ^ xorMask);
+ }
+ delegate.write(tmpBuffer, 0, tmpBuffer.length);
+ currentPosition += len;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ write(new byte[] { (byte) b }, 0, 1);
+ }
+
+}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RandomAccessByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RandomAccessByteProvider.java
index 403b27d0ec..3160ec342c 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RandomAccessByteProvider.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RandomAccessByteProvider.java
@@ -29,7 +29,10 @@ import ghidra.util.Msg;
* {@link ArrayIndexOutOfBoundsException}s.
*
* See {@link SynchronizedByteProvider} as a solution.
+ *
+ * @deprecated See {@link FileByteProvider} as replacement ByteProvider.
*/
+@Deprecated(since = "10.1", forRemoval = true)
public class RandomAccessByteProvider implements ByteProvider {
protected File file;
protected GhidraRandomAccessFile randomAccessFile;
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RangeMappedByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RangeMappedByteProvider.java
index e30f49129c..c48f7915bc 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RangeMappedByteProvider.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RangeMappedByteProvider.java
@@ -220,11 +220,6 @@ public class RangeMappedByteProvider implements ByteProvider {
return totalBytesRead;
}
- @Override
- public InputStream getInputStream(long index) throws IOException {
- return new ByteProviderInputStream(this, 0, length);
- }
-
private void ensureBounds(long index, long count) throws IOException {
if (index < 0 || index > length) {
throw new IOException("Invalid index: " + index);
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/SynchronizedByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/SynchronizedByteProvider.java
index 3d36e431c4..8f67f7b198 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/SynchronizedByteProvider.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/SynchronizedByteProvider.java
@@ -78,9 +78,12 @@ public class SynchronizedByteProvider implements ByteProvider {
public synchronized byte[] readBytes(long index, long length) throws IOException {
return provider.readBytes(index, length);
}
-
+
@Override
public synchronized InputStream getInputStream(long index) throws IOException {
- return provider.getInputStream(index);
+ // Return a ByteProviderInputStream that reads its bytes via this wrapper so that it is completely
+ // synchronized. Returning the delegate provider's getInputStream() would subvert
+ // synchronization and allow direct access to the underlying delegate provider.
+ return ByteProvider.super.getInputStream(index);
}
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/CoffArchiveMemberHeader.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/CoffArchiveMemberHeader.java
index 08b4e3a7d1..5fc6063a00 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/CoffArchiveMemberHeader.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/CoffArchiveMemberHeader.java
@@ -15,14 +15,14 @@
*/
package ghidra.app.util.bin.format.coff.archive;
+import java.io.IOException;
+
import ghidra.app.util.bin.*;
import ghidra.program.model.data.*;
import ghidra.util.Msg;
import ghidra.util.StringUtilities;
import ghidra.util.exception.DuplicateNameException;
-import java.io.IOException;
-
public class CoffArchiveMemberHeader implements StructConverter {
public static final String SLASH = "/";
public static final String SLASH_SLASH = "//";
@@ -249,6 +249,24 @@ public class CoffArchiveMemberHeader implements StructConverter {
return groupId;
}
+ public int getUserIdInt() {
+ try {
+ return Integer.parseInt(userId);
+ }
+ catch (NumberFormatException nfe) {
+ return 0;
+ }
+ }
+
+ public int getGroupIdInt() {
+ try {
+ return Integer.parseInt(groupId);
+ }
+ catch (NumberFormatException nfe) {
+ return 0;
+ }
+ }
+
public String getMode() {
return mode;
}
@@ -274,6 +292,7 @@ public class CoffArchiveMemberHeader implements StructConverter {
!name.equals(CoffArchiveMemberHeader.SLASH_SLASH);
}
+ @Override
public DataType toDataType() throws DuplicateNameException, IOException {
String camh_name = StructConverterUtil.parseName(CoffArchiveMemberHeader.class);
Structure struct = new StructureDataType(camh_name, 0);
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/FirstLinkerMember.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/FirstLinkerMember.java
index b63623481a..38c32f0fa2 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/FirstLinkerMember.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/FirstLinkerMember.java
@@ -15,31 +15,30 @@
*/
package ghidra.app.util.bin.format.coff.archive;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import ghidra.app.util.bin.*;
import ghidra.program.model.data.*;
import ghidra.util.BigEndianDataConverter;
import ghidra.util.DataConverter;
import ghidra.util.exception.DuplicateNameException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
public final class FirstLinkerMember implements StructConverter {
private int numberOfSymbols;
private int [] offsets;
- private List stringTable = new ArrayList();
+ private List stringTable = new ArrayList<>();
private long _fileOffset;
- private List stringLengths = new ArrayList();
+ private List stringLengths = new ArrayList<>();
public FirstLinkerMember(BinaryReader reader, CoffArchiveMemberHeader header, boolean skip)
throws IOException {
_fileOffset = reader.getPointerIndex();
-
- boolean isLittleEndian = reader.isLittleEndian();
- reader.setLittleEndian(false);//this entire structure is stored as big-endian..
+ BinaryReader origReader = reader;
+ reader = reader.asBigEndian(); //this entire structure is stored as big-endian..
numberOfSymbols = readNumberOfSymbols(reader);
@@ -57,7 +56,7 @@ public final class FirstLinkerMember implements StructConverter {
}
}
else {
- stringTable = new ArrayList(numberOfSymbols);
+ stringTable = new ArrayList<>(numberOfSymbols);
for (int i = 0 ; i < numberOfSymbols ; ++i) {
String string = reader.readNextAsciiString();
stringTable.add( string );
@@ -65,8 +64,7 @@ public final class FirstLinkerMember implements StructConverter {
}
}
- reader.setLittleEndian(isLittleEndian);
- reader.setPointerIndex(_fileOffset + header.getSize());
+ origReader.setPointerIndex(_fileOffset + header.getSize());
}
/**
@@ -100,9 +98,10 @@ public final class FirstLinkerMember implements StructConverter {
if (stringTable.isEmpty()) {
throw new RuntimeException("FirstLinkerMember::getStringTable() has been skipped.");
}
- return new ArrayList(stringTable);
+ return new ArrayList<>(stringTable);
}
+ @Override
public DataType toDataType() throws DuplicateNameException, IOException {
String name = StructConverterUtil.parseName(FirstLinkerMember.class);
Structure struct = new StructureDataType(name + "_" + numberOfSymbols, 0);
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/sectionprovider/NullSectionProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/sectionprovider/NullSectionProvider.java
index fb40b19466..783d7cfebc 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/sectionprovider/NullSectionProvider.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/sectionprovider/NullSectionProvider.java
@@ -15,11 +15,10 @@
*/
package ghidra.app.util.bin.format.dwarf4.next.sectionprovider;
-import ghidra.app.util.bin.ByteArrayProvider;
-import ghidra.app.util.bin.ByteProvider;
-
import java.io.IOException;
+import ghidra.app.util.bin.ByteProvider;
+
public class NullSectionProvider implements DWARFSectionProvider {
public NullSectionProvider() {
@@ -28,7 +27,7 @@ public class NullSectionProvider implements DWARFSectionProvider {
@Override
public ByteProvider getSectionAsByteProvider(String sectionName) throws IOException {
- return new ByteArrayProvider(new byte[] {});
+ return ByteProvider.EMPTY_BYTEPROVIDER;
}
@Override
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java
index adf23d7161..fb76e3716e 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java
@@ -22,6 +22,7 @@ import java.util.List;
import ghidra.app.util.Option;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.importer.MessageLog;
+import ghidra.formats.gfilesystem.FSRL;
import ghidra.framework.model.DomainFolder;
import ghidra.framework.model.DomainObject;
import ghidra.program.model.listing.Program;
@@ -169,7 +170,9 @@ public interface Loader extends ExtensionPoint, Comparable {
* @return The preferred file name to use when loading.
*/
public default String getPreferredFileName(ByteProvider provider) {
- return provider.getName().replaceAll("[\\\\:|]+", "/");
+ FSRL fsrl = provider.getFSRL();
+ String name = (fsrl != null) ? fsrl.getName() : provider.getName();
+ return name.replaceAll("[\\\\:|]+", "/");
}
/**
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/AbstractFileExtractorTask.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/AbstractFileExtractorTask.java
index 6c45e5026d..b4dabb78c3 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/AbstractFileExtractorTask.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/AbstractFileExtractorTask.java
@@ -15,15 +15,10 @@
*/
package ghidra.formats.gfilesystem;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
-import ghidra.util.exception.CryptoException;
import ghidra.util.exception.IOCancelledException;
import ghidra.util.task.Task;
import ghidra.util.task.TaskMonitor;
@@ -190,7 +185,7 @@ public abstract class AbstractFileExtractorTask extends Task {
}
protected void extractFile(GFile srcFile, File outputFile, TaskMonitor monitor)
- throws CancelledException, CryptoException {
+ throws CancelledException {
monitor.setMessage(srcFile.getName());
try (InputStream in = getSourceFileInputStream(srcFile, monitor)) {
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/DerivedFileProducer.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/DerivedFileProducer.java
deleted file mode 100644
index 4f5cab2650..0000000000
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/DerivedFileProducer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/* ###
- * IP: GHIDRA
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package ghidra.formats.gfilesystem;
-
-import ghidra.util.exception.CancelledException;
-import ghidra.util.task.TaskMonitor;
-
-import java.io.*;
-
-/**
- * Used by {@link FileSystemService#getDerivedFile(FSRL, String, DerivedFileProducer, TaskMonitor)}
- * to produce a derived file from a source file.
- *
- * The {@link InputStream} returned from the method will be closed by the caller.
- */
-public interface DerivedFileProducer {
-
- /**
- * Callback method intended to be implemented by the caller to
- * {@link FileSystemService#getDerivedFile(FSRL, String, DerivedFileProducer, TaskMonitor)}.
- *
- * The implementation needs to return an {@link InputStream} that contains the bytes
- * of the derived file.
- *
- * @param srcFile {@link File} location of the source file (usually in the file cache)
- * @return a new {@link InputStream} that will produce all the bytes of the derived file.
- * @throws IOException if there is a problem while producing the InputStream.
- * @throws CancelledException if the user canceled.
- */
- public InputStream produceDerivedStream(File srcFile) throws IOException, CancelledException;
-}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/DerivedFilePushProducer.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/DerivedFilePushProducer.java
deleted file mode 100644
index e02095300d..0000000000
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/DerivedFilePushProducer.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/* ###
- * IP: GHIDRA
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package ghidra.formats.gfilesystem;
-
-import ghidra.util.exception.CancelledException;
-import ghidra.util.task.TaskMonitor;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * Used by {@link FileSystemService#getDerivedFilePush(FSRL, String, DerivedFilePushProducer, TaskMonitor)}
- * to produce a derived file from a source file.
- */
-public interface DerivedFilePushProducer {
- /**
- * Callback method intended to be implemented by the caller to
- * {@link FileSystemService#getDerivedFilePush(FSRL, String, DerivedFilePushProducer, TaskMonitor)}.
- *
- * The implementation needs to write bytes to the supplied {@link OutputStream}.
- *
- * @param os {@link OutputStream} that the implementor should write the bytes to. Do
- * not close the stream when done.
- * @throws IOException if there is a problem while writing to the OutputStream.
- * @throws CancelledException if the user canceled.
- */
- public void push(OutputStream os) throws IOException, CancelledException;
-}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSRL.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSRL.java
index a01458e1cd..74dc1726a1 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSRL.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSRL.java
@@ -96,6 +96,19 @@ public class FSRL {
return parent;
}
+ /**
+ * Ensures that a FSRL instance is a file type reference by converting any FSRLRoots
+ * into the container file that hosts the FSRLRoot.
+ *
+ * @param fsrl FSRL or FSRLRoot instance to possibly convert
+ * @return original FSRL if already a normal FSRL, or the container if it was a FSRLRoot
+ */
+ public static FSRL convertRootToContainer(FSRL fsrl) {
+ return (fsrl instanceof FSRLRoot && fsrl.getFS().hasContainer())
+ ? fsrl.getFS().getContainer()
+ : fsrl;
+ }
+
/**
* Creates a single {@link FSRL} from a FSRL-part string.
*
@@ -282,15 +295,29 @@ public class FSRL {
return md5;
}
+ /**
+ * Tests specified MD5 value against MD5 in this FSRL.
+ *
+ * @param otherMD5 md5 in a hex string
+ * @return boolean true if equal, or that both are null, false otherwise
+ */
+ public boolean isMD5Equal(String otherMD5) {
+ if (this.md5 == null) {
+ return otherMD5 == null;
+ }
+ return this.md5.equalsIgnoreCase(otherMD5);
+ }
+
/**
* Creates a new {@link FSRL} instance, using the same information as this instance,
* but with a new {@link #getMD5() MD5} value.
*
* @param newMD5 string md5
- * @return new {@link FSRL} instance with the same path and the specified md5 value.
+ * @return new {@link FSRL} instance with the same path and the specified md5 value,
+ * or if newMD5 is same as existing, returns this
*/
public FSRL withMD5(String newMD5) {
- return new FSRL(getFS(), path, newMD5);
+ return Objects.equals(md5, newMD5) ? this : new FSRL(getFS(), path, newMD5);
}
/**
@@ -312,7 +339,7 @@ public class FSRL {
*
* Used when re-root'ing a FSRL path onto another parent object (usually during intern()'ing)
*
- * @param copyPath
+ * @param copyPath another FSRL to copy path and md5 from
* @return new FSRL instance
*/
public FSRL withPath(FSRL copyPath) {
@@ -323,7 +350,7 @@ public class FSRL {
* Creates a new {@link FSRL} instance, using the same {@link FSRLRoot} as this instance,
* combining the current {@link #getPath() path} with the {@code relPath} value.
*
- * @param relPath
+ * @param relPath relative path string to append, '/'s will be automatically added
* @return new {@link FSRL} instance with additional path appended.
*/
public FSRL appendPath(String relPath) {
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java
index e29902b1cc..89850e5f00 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java
@@ -20,6 +20,7 @@ import java.io.*;
import java.net.MalformedURLException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.*;
@@ -27,9 +28,8 @@ import java.util.Map.Entry;
import org.apache.commons.io.FilenameUtils;
-import com.google.common.io.ByteStreams;
-
import docking.widgets.OptionDialog;
+import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.util.*;
import ghidra.util.exception.CancelledException;
@@ -45,16 +45,6 @@ public class FSUtilities {
private static final char DOT = '.';
private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
- public static class StreamCopyResult {
- public long bytesCopied;
- public byte[] md5;
-
- public StreamCopyResult(long bytesCopied, byte[] md5) {
- this.bytesCopied = bytesCopied;
- this.md5 = md5;
- }
- }
-
private static char[] hexdigit = "0123456789abcdef".toCharArray();
/**
@@ -62,25 +52,17 @@ public class FSUtilities {
* case-insensitive.
*/
public static final Comparator GFILE_NAME_TYPE_COMPARATOR = (o1, o2) -> {
- String n1 = o1.getName();
- String n2 = o2.getName();
- if (n1 == null) {
- return -1;
+ int result = Boolean.compare(!o1.isDirectory(), !o2.isDirectory());
+ if (result == 0) {
+ String n1 = Objects.requireNonNullElse(o1.getName(), "");
+ String n2 = Objects.requireNonNullElse(o2.getName(), "");
+ result = n1.compareToIgnoreCase(n2);
}
- if (o1.isDirectory()) {
- if (o2.isDirectory()) {
- return n1.compareToIgnoreCase(n2);
- }
- return -1;
- }
- else if (o2.isDirectory()) {
- return 1;
- }
- return n1.compareToIgnoreCase(n2);
+ return result;
};
/**
- * Converts a string -> string mapping into a "key: value" multi-line string.
+ * Converts a string-to-string mapping into a "key: value\n" multi-line string.
*
* @param info map of string key to string value.
* @return Multi-line string "key: value" string.
@@ -349,55 +331,61 @@ public class FSUtilities {
}
/**
- * Copies a stream and calculates the md5 at the same time.
- *
- * Does not close the passed-in InputStream or OutputStream.
- *
- * @param is {@link InputStream} to copy. NOTE: not closed by this method.
- * @param os {@link OutputStream} to write to. NOTE: not closed by this method.
- * @return {@link StreamCopyResult} with md5 and bytes copied count, never null.
+ * Copy the contents of a {@link ByteProvider} to a file.
+ *
+ * @param provider {@link ByteProvider} source of bytes
+ * @param destFile {@link File} destination file
+ * @param monitor {@link TaskMonitor} to update
+ * @return number of bytes copied
* @throws IOException if error
- * @throws CancelledException if canceled
+ * @throws CancelledException if cancelled
*/
- @SuppressWarnings("resource")
- public static StreamCopyResult streamCopy(InputStream is, OutputStream os, TaskMonitor monitor)
- throws IOException, CancelledException {
- HashingOutputStream hos;
- try {
- // This wrapping outputstream is not closed on purpose.
- hos = new HashingOutputStream(os, "MD5");
- }
- catch (NoSuchAlgorithmException e) {
- throw new IOException("Could not get MD5 hash algo", e);
+ public static long copyByteProviderToFile(ByteProvider provider, File destFile,
+ TaskMonitor monitor) throws IOException, CancelledException {
+ try (InputStream is = provider.getInputStream(0);
+ FileOutputStream fos = new FileOutputStream(destFile)) {
+ return streamCopy(is, fos, monitor);
}
+ }
- // TODO: use FileUtilities.copyStreamToStream()
+ /**
+ * Copy a stream while updating a TaskMonitor.
+ *
+ * @param is {@link InputStream} source of bytes
+ * @param os {@link OutputStream} destination of bytes
+ * @param monitor {@link TaskMonitor} to update
+ * @return number of bytes copied
+ * @throws IOException if error
+ * @throws CancelledException if cancelled
+ */
+ public static long streamCopy(InputStream is, OutputStream os, TaskMonitor monitor)
+ throws IOException, CancelledException {
byte buffer[] = new byte[FileUtilities.IO_BUFFER_SIZE];
int bytesRead;
long totalBytesCopied = 0;
while ((bytesRead = is.read(buffer)) > 0) {
- hos.write(buffer, 0, bytesRead);
+ os.write(buffer, 0, bytesRead);
totalBytesCopied += bytesRead;
monitor.setProgress(totalBytesCopied);
monitor.checkCanceled();
}
- hos.flush();
- return new StreamCopyResult(totalBytesCopied, hos.getDigest());
+ os.flush();
+ return totalBytesCopied;
}
/**
- * Calculate the MD5 of a stream.
- *
- * @param is {@link InputStream} to read
- * @param monitor {@link TaskMonitor} to watch for cancel
- * @return md5 as a hex encoded string, never null.
+ * Returns the text lines in the specified ByteProvider.
+ *
+ * See {@link FileUtilities#getLines(InputStream)}
+ *
+ * @param byteProvider {@link ByteProvider} to read
+ * @return list of text lines
* @throws IOException if error
- * @throws CancelledException if cancelled
*/
- public static String getStreamMD5(InputStream is, TaskMonitor monitor)
- throws IOException, CancelledException {
- StreamCopyResult results = streamCopy(is, ByteStreams.nullOutputStream(), monitor);
- return NumericUtilities.convertBytesToString(results.md5);
+ public static List getLines(ByteProvider byteProvider) throws IOException {
+ try (InputStream is = byteProvider.getInputStream(0)) {
+ return FileUtilities.getLines(is);
+ }
}
/**
@@ -412,7 +400,54 @@ public class FSUtilities {
public static String getFileMD5(File f, TaskMonitor monitor)
throws IOException, CancelledException {
try (FileInputStream fis = new FileInputStream(f)) {
- return getStreamMD5(fis, monitor);
+ monitor.initialize(f.length());
+ monitor.setMessage("Hashing file: " + f.getName());
+ return getMD5(fis, monitor);
+ }
+ }
+
+ /**
+ * Calculate the MD5 of a file.
+ *
+ * @param provider {@link ByteProvider}
+ * @param monitor {@link TaskMonitor} to watch for cancel
+ * @return md5 as a hex encoded string, never null.
+ * @throws IOException if error
+ * @throws CancelledException if cancelled
+ */
+ public static String getMD5(ByteProvider provider, TaskMonitor monitor)
+ throws IOException, CancelledException {
+ try (InputStream is = provider.getInputStream(0)) {
+ monitor.initialize(provider.length());
+ monitor.setMessage("Hashing file: " + provider.getName());
+ return getMD5(is, monitor);
+ }
+ }
+
+ /**
+ * Calculate the hash of an {@link InputStream}.
+ *
+ * @param is {@link InputStream}
+ * @param monitor {@link TaskMonitor} to update
+ * @return md5 as a hex encoded string, never null
+ * @throws IOException if error
+ * @throws CancelledException if cancelled
+ */
+ public static String getMD5(InputStream is, TaskMonitor monitor)
+ throws IOException, CancelledException {
+ try {
+ MessageDigest messageDigest = MessageDigest.getInstance(HashUtilities.MD5_ALGORITHM);
+ byte[] buf = new byte[16 * 1024];
+ int bytesRead;
+ while ((bytesRead = is.read(buf)) >= 0) {
+ messageDigest.update(buf, 0, bytesRead);
+ monitor.incrementProgress(bytesRead);
+ monitor.checkCanceled();
+ }
+ return NumericUtilities.convertBytesToString(messageDigest.digest());
+ }
+ catch (NoSuchAlgorithmException e) {
+ throw new IOException(e);
}
}
@@ -534,4 +569,22 @@ public class FSUtilities {
: "NA";
}
+ /**
+ * Helper method to invoke close() on a Closeable without having to catch
+ * an IOException.
+ *
+ * @param c {@link Closeable} to close
+ * @param msg optional msg to log if exception is thrown, null is okay
+ */
+ public static void uncheckedClose(Closeable c, String msg) {
+ try {
+ if (c != null) {
+ c.close();
+ }
+ }
+ catch (IOException e) {
+ Msg.warn(FSUtilities.class, Objects.requireNonNullElse(msg, "Problem closing object"),
+ e);
+ }
+ }
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileCache.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileCache.java
index c72c4ccccc..dc1951938b 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileCache.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileCache.java
@@ -16,12 +16,14 @@
package ghidra.formats.gfilesystem;
import java.io.*;
+import java.nio.file.*;
import java.security.NoSuchAlgorithmException;
-import java.util.Date;
-import java.util.UUID;
+import java.util.*;
import java.util.regex.Pattern;
-import ghidra.formats.gfilesystem.FSUtilities.StreamCopyResult;
+import org.apache.commons.collections4.map.ReferenceMap;
+
+import ghidra.app.util.bin.*;
import ghidra.util.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@@ -31,44 +33,58 @@ import utilities.util.FileUtilities;
* File caching implementation.
*
* Caches files based on a hash of the contents of the file.
- * Files are retrieved using the hash string.
- * Cached files are stored in a file with a name that is the hex encoded value of the hash.
+ * Files are retrieved using the hash string.
+ * Cached files are stored in a file with a name that is the hex encoded value of the hash.
+ * Cached files are obfuscated/de-obfuscated when written/read to/from disk. See
+ * {@link ObfuscatedFileByteProvider}, {@link ObfuscatedInputStream},
+ * {@link ObfuscatedOutputStream}.
* Cached files are organized into a nested directory structure to prevent
* overwhelming a single directory with thousands of files.
*
- * Nested directory structure is based on the file's name:
- * File: AABBCCDDEEFF...
- * Directory (2 level nesting): AA/BB/AABBCCDDEEFF...
+ * Nested directory structure is based on the file's name:
+ *
File: AABBCCDDEEFF... → AA/AABBCCDDEEFF...
*
* Cache size is not bounded.
*
- * Cache maint is done during startup if interval since last maint has been exceeded
+ * Cache maintenance is done during startup if interval since last maintenance has been exceeded.
*
- * No file data is maintained in memory.
- *
- * No file is moved or removed from the cache after being added (except during startup)
- * as there is no use count or reference tracking of the files.
+ * Files are not removed from the cache after being added, except during startup maintenance.
*
*/
public class FileCache {
-
+ /**
+ * Max size of a file that will be kept in {@link #memCache} (2Mb)
+ */
+ public static final int MAX_INMEM_FILESIZE = 2 * 1024 * 1024; // 2mb
+ private static final long FREESPACE_RESERVE_BYTES = 50 * 1024 * 1024; // 50mb
private static final Pattern NESTING_DIR_NAME_REGEX = Pattern.compile("[0-9a-fA-F][0-9a-fA-F]");
+ private static final Pattern FILENAME_REGEX = Pattern.compile("[0-9a-fA-F]{32}");
private static final int MD5_BYTE_LEN = 16;
public static final int MD5_HEXSTR_LEN = MD5_BYTE_LEN * 2;
- private static final int NESTING_LEVEL = 2;
private static final long MAX_FILE_AGE_MS = DateUtils.MS_PER_DAY;
private static final long MAINT_INTERVAL_MS = DateUtils.MS_PER_DAY * 2;
private final File cacheDir;
+ private final FileStore cacheDirFileStore;
private final File newDir;
- private final File lastMaintFile;
private FileCacheMaintenanceDaemon cleanDaemon;
+ private ReferenceMap memCache = new ReferenceMap<>();
- private int fileAddCount;
- private int fileReUseCount;
- private long storageEstimateBytes;
- private long lastMaintTS;
+ /**
+ * Backwards compatible with previous cache directories to age off the files located
+ * therein.
+ *
+ * @param oldCacheDir the old 2-level cache directory
+ * @deprecated Marked as deprecated to ensure this is removed in a few versions after most
+ * user's old-style cache dirs have been cleaned up.
+ */
+ @Deprecated(forRemoval = true, since = "10.1")
+ public static void performCacheMaintOnOldDirIfNeeded(File oldCacheDir) {
+ if (oldCacheDir.isDirectory()) {
+ performCacheMaintIfNeeded(oldCacheDir, 2 /* old nesting level */);
+ }
+ }
/**
* Creates a new {@link FileCache} instance where files are stored under the specified
@@ -81,12 +97,13 @@ public class FileCache {
public FileCache(File cacheDir) throws IOException {
this.cacheDir = cacheDir;
this.newDir = new File(cacheDir, "new");
- this.lastMaintFile = new File(cacheDir, ".lastmaint");
if ((!cacheDir.exists() && !cacheDir.mkdirs()) || (!newDir.exists() && !newDir.mkdirs())) {
throw new IOException("Unable to initialize cache dir " + cacheDir);
}
- performCacheMaintIfNeeded();
+
+ cacheDirFileStore = Files.getFileStore(cacheDir.toPath());
+ cleanDaemon = performCacheMaintIfNeeded(cacheDir, 1 /* current nesting level */);
}
/**
@@ -102,22 +119,26 @@ public class FileCache {
FileUtilities.deleteDir(f);
}
}
+ memCache.clear();
}
- /**
- * Adds a {@link File} to the cache, returning a {@link FileCacheEntry}.
- *
- * @param f {@link File} to add to cache.
- * @param monitor {@link TaskMonitor} to monitor for cancel and to update progress.
- * @return {@link FileCacheEntry} with new File and md5.
- * @throws IOException if error
- * @throws CancelledException if canceled
- */
- public FileCacheEntry addFile(File f, TaskMonitor monitor)
- throws IOException, CancelledException {
- try (FileInputStream fis = new FileInputStream(f)) {
- return addStream(fis, monitor);
+ synchronized boolean hasEntry(String md5) {
+ FileCacheEntry fce = memCache.get(md5);
+ if (fce == null) {
+ fce = getFileByMD5(md5);
}
+ return fce != null;
+ }
+
+ private void ensureAvailableSpace(long sizeHint) throws IOException {
+ if ( sizeHint > MAX_INMEM_FILESIZE ) {
+ long usableSpace = cacheDirFileStore.getUsableSpace();
+ if (usableSpace >= 0 && usableSpace < sizeHint + FREESPACE_RESERVE_BYTES) {
+ throw new IOException("Not enough storage available in " + cacheDir +
+ " to store file sized: " + sizeHint);
+ }
+ }
+
}
/**
@@ -130,12 +151,26 @@ public class FileCache {
* @return {@link FileCacheEntry} with a File and it's md5 string or {@code null} if no
* matching file exists in cache.
*/
- public FileCacheEntry getFile(String md5) {
- FileCacheEntry cfi = getFileByMD5(md5);
- if (cfi != null) {
- cfi.file.setLastModified(System.currentTimeMillis());
+ synchronized FileCacheEntry getFileCacheEntry(String md5) {
+ if (md5 == null) {
+ return null;
+ }
+ FileCacheEntry fce = memCache.get(md5);
+ if (fce == null) {
+ fce = getFileByMD5(md5);
+ if (fce != null) {
+ fce.file.setLastModified(System.currentTimeMillis());
+ }
+ }
+ return fce;
+ }
+
+ synchronized void releaseFileCacheEntry(String md5) {
+ FileCacheEntry fce = memCache.get(md5);
+ if (fce != null) {
+ memCache.remove(md5);
+ Msg.debug(this, "Releasing memCache entry: " + fce.md5 + ", " + fce.bytes.length);
}
- return cfi;
}
/**
@@ -150,165 +185,53 @@ public class FileCache {
}
/**
- * Prunes cache if interval since last maintenance exceeds {@link #MAINT_INTERVAL_MS}
- *
- * Only called during construction, and the only known multi-process conflict that can occur
- * is when re-writing the "lastMaint" timestamp file, which isn't a problem as its the
- * approximate timestamp of that file that is important, not the contents.
- *
- * @throws IOException if error when writing metadata file.
+ * Creates a randomly generated file name in the temp directory.
+ *
+ * @return randomly generated file name in the cache's temp directory
*/
- private void performCacheMaintIfNeeded() throws IOException {
- lastMaintTS = (lastMaintTS == 0) ? lastMaintFile.lastModified() : lastMaintTS;
- if (lastMaintTS + MAINT_INTERVAL_MS > System.currentTimeMillis()) {
- return;
- }
-
- cleanDaemon = new FileCacheMaintenanceDaemon();
- cleanDaemon.start();
+ private File createTempFile() {
+ return new File(newDir, UUID.randomUUID().toString());
}
/**
- * Prunes files in cache if they are old, calculates space used by cache.
- */
- private void performCacheMaint() {
- storageEstimateBytes = 0;
- Msg.info(this, "Starting cache cleanup: " + cacheDir);
- // TODO: add check for orphan files in ./new
- cacheMaintForDir(cacheDir, 0);
- Msg.info(this, "Finished cache cleanup, estimated storage used: " + storageEstimateBytes);
- }
-
- private void cacheMaintForDir(File dir, int nestingLevel) {
- if (nestingLevel < NESTING_LEVEL) {
- for (File f : dir.listFiles()) {
- String name = f.getName();
- if (f.isDirectory() && NESTING_DIR_NAME_REGEX.matcher(name).matches()) {
- cacheMaintForDir(f, nestingLevel + 1);
- }
- }
- }
- else if (nestingLevel == NESTING_LEVEL) {
- cacheMaintForLeafDir(dir);
- }
- }
-
- private void cacheMaintForLeafDir(File dir) {
- long cutoffMS = System.currentTimeMillis() - MAX_FILE_AGE_MS;
-
- for (File f : dir.listFiles()) {
- if (f.isFile() && isCacheFileName(f.getName())) {
- if (f.lastModified() < cutoffMS) {
- if (!f.delete()) {
- Msg.error(this, "Failed to delete cache file " + f);
- }
- else {
- Msg.info(this, "Expired cache file " + f);
- continue;
- }
- }
- storageEstimateBytes += f.length();
- }
- }
- }
-
- private boolean isCacheFileName(String s) {
- try {
- byte[] bytes = NumericUtilities.convertStringToBytes(s);
- return (bytes != null) && bytes.length == MD5_BYTE_LEN;
- }
- catch (IllegalArgumentException e) {
- return false;
- }
- }
-
- /**
- * Adds a contents of a stream to the cache, returning the md5 identifier of the stream.
- *
- * The stream is copied into a temp file in the cacheDir/new directory while its md5
- * is calculated. The temp file is then moved into its final location
- * based on the md5 of the stream: AA/BB/AABBCCDDEEFF....
- *
- * The monitor progress is updated with the number of bytes that are being copied. No
- * message or maximum is set.
- *
- * @param is {@link InputStream} to add to the cache. Not closed when done.
- * @param monitor {@link TaskMonitor} that will be checked for canceling and updating progress.
- * @return {@link FileCacheEntry} with file info and md5, never null.
+ * Creates a new {@link FileCacheEntryBuilder} that will accept bytes written to it
+ * (via its {@link OutputStream} methods). When finished writing, the {@link FileCacheEntryBuilder}
+ * will give the caller a {@link FileCacheEntry}.
+ *
+ * @param sizeHint a hint about the size of the file being added. Use -1 if unsure or unknown
+ * @return new {@link FileCacheEntryBuilder}
* @throws IOException if error
- * @throws CancelledException if canceled
*/
- public FileCacheEntry addStream(InputStream is, TaskMonitor monitor)
- throws IOException, CancelledException {
- File tmpFile = new File(newDir, UUID.randomUUID().toString());
- try (FileOutputStream fos = new FileOutputStream(tmpFile)) {
- StreamCopyResult copyResults = FSUtilities.streamCopy(is, fos, monitor);
-
- // Close the fos so the tmpFile can be moved even though
- // the try(){} will attempt to close it as well.
- fos.close();
-
- String md5 = NumericUtilities.convertBytesToString(copyResults.md5);
-
- return addTmpFileToCache(tmpFile, md5, copyResults.bytesCopied);
- }
- finally {
- if (tmpFile.exists()) {
- Msg.debug(this, "Removing left-over temp file " + tmpFile);
- tmpFile.delete();
- }
- }
+ FileCacheEntryBuilder createCacheEntryBuilder(long sizeHint) throws IOException {
+ ensureAvailableSpace(sizeHint);
+ return new FileCacheEntryBuilder(sizeHint);
}
+
/**
- * Adds a file to the cache, using a 'pusher' strategy where the producer is given a
- * {@link OutputStream} to write to.
+ * Adds a plaintext file to this cache, consuming it.
*
- * Unbeknownst to the producer, but knownst to us, the outputstream is really a
- * {@link HashingOutputStream} that will allow us to get the MD5 hash when the producer
- * is finished pushing.
- *
- * @param pusher functional callback that will accept an {@link OutputStream} and write
- * to it.
- *
(os) -> { os.write(.....); }
- * @param monitor {@link TaskMonitor} that will be checked for cancel and updated with
- * file io progress.
- * @return a new {@link FileCacheEntry} with the newly added cache file's File and MD5,
- * never null.
- * @throws IOException if an IO error
- * @throws CancelledException if the user cancels
+ * @param file plaintext file
+ * @param monitor {@link TaskMonitor}
+ * @return a {@link FileCacheEntry} that controls the contents of the newly added file
+ * @throws IOException if error
+ * @throws CancelledException if cancelled
*/
- public FileCacheEntry pushStream(DerivedFilePushProducer pusher, TaskMonitor monitor)
- throws IOException, CancelledException {
- File tmpFile = new File(newDir, UUID.randomUUID().toString());
- try (HashingOutputStream hos =
- new HashingOutputStream(new FileOutputStream(tmpFile), "MD5")) {
- pusher.push(hos);
- // early hos.close() so it can be renamed/moved on the filesystem
- hos.close();
-
- String md5 = NumericUtilities.convertBytesToString(hos.getDigest());
- long fileSize = tmpFile.length();
-
- return addTmpFileToCache(tmpFile, md5, fileSize);
- }
- catch (NoSuchAlgorithmException e) {
- throw new IOException("Error getting MD5 algo", e);
- }
- catch (Throwable th) {
- throw new IOException("Error while pushing stream into cache", th);
+ FileCacheEntry giveFile(File file, TaskMonitor monitor) throws IOException, CancelledException {
+ try (InputStream fis = new FileInputStream(file);
+ FileCacheEntryBuilder fceBuilder = createCacheEntryBuilder(file.length())) {
+ FSUtilities.streamCopy(fis, fceBuilder, monitor);
+ return fceBuilder.finish();
}
finally {
- if (tmpFile.exists()) {
- Msg.debug(this, "Removing left-over temp file " + tmpFile);
- tmpFile.delete();
+ if (!file.delete()) {
+ Msg.warn(this, "Failed to delete temporary file: " + file);
}
}
-
}
/**
- * Adds a File to this cache, consuming the file.
+ * Adds an already obfuscated File to this cache, consuming the file.
*
* This method makes some assumptions:
*
@@ -318,16 +241,14 @@ public class FileCache {
* existence and the attempt to place the file into the directory. Solution: no
* process may remove a nested directory after it has been created.
* 2) The source file is co-located with the cache directory to ensure its on the
- * same physical filesystem volume.
+ * same physical filesystem volume, and is already obfuscated.
*
* @param tmpFile the File to add to the cache
* @param md5 hex string md5 of the file
- * @param fileLen the length in bytes of the file being added
* @return a new {@link FileCacheEntry} with the File's location and its md5
* @throws IOException if an file error occurs
*/
- private FileCacheEntry addTmpFileToCache(File tmpFile, String md5, long fileLen)
- throws IOException {
+ private FileCacheEntry addTmpFileToCache(File tmpFile, String md5) throws IOException {
String relPath = getCacheRelPath(md5);
File destCacheFile = new File(cacheDir, relPath);
@@ -337,91 +258,28 @@ public class FileCache {
throw new IOException("Failed to create cache dir " + destCacheFileDir);
}
- boolean moved = false;
- boolean reused = false;
- if (destCacheFile.exists()) {
- reused = true;
+ try {
+ tmpFile.renameTo(destCacheFile);
}
- else {
- moved = tmpFile.renameTo(destCacheFile);
-
- // test again to see if another process was racing us if the rename failed
- reused = !moved && destCacheFile.exists();
- }
- if (!moved && reused) {
- //Msg.info(this, "File already exists in cache, reusing: " + destCacheFile);
+ finally {
tmpFile.delete();
- }
- else if (!moved) {
- throw new IOException("Failed to move " + tmpFile + " to " + destCacheFile);
- }
-
- synchronized (this) {
- fileAddCount++;
- if (reused) {
- fileReUseCount++;
- destCacheFile.setLastModified(System.currentTimeMillis());
- }
- else {
- storageEstimateBytes += fileLen;
+ if (!destCacheFile.exists()) {
+ throw new IOException("Failed to move " + tmpFile + " to " + destCacheFile);
}
}
-
+ destCacheFile.setLastModified(System.currentTimeMillis());
return new FileCacheEntry(destCacheFile, md5);
}
private String getCacheRelPath(String md5) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < NESTING_LEVEL; i++) {
- sb.append(md5.substring(i * 2, (i + 1) * 2));
- sb.append('/');
- }
- sb.append(md5);
- return sb.toString();
+ return String.format("%s/%s",
+ md5.substring(0, 2),
+ md5);
}
@Override
public String toString() {
- return "FileCache [cacheDir=" + cacheDir + ", fileAddCount=" + fileAddCount +
- ", storageEstimateBytes=" + storageEstimateBytes + ", lastMaintTS=" + lastMaintTS + "]";
- }
-
- /**
- * Number of files added to this cache.
- *
- * @return Number of files added to this cache
- */
- public int getFileAddCount() {
- return fileAddCount;
- }
-
- /**
- * Number of times a file-add was a no-op and the contents were already present
- * in the cache.
- *
- * @return Number of times a file-add was a no-op and the contents were already present
- * in the cache.
- */
- public int getFileReUseCount() {
- return fileReUseCount;
- }
-
- /**
- * Estimate of the number of bytes in the cache.
- *
- * @return estimate of the number of bytes in the cache - could be very wrong
- */
- public long getStorageEstimateBytes() {
- return storageEstimateBytes;
- }
-
- /**
- * How old (in milliseconds) files must be before being aged-off during cache maintenance.
- *
- * @return Max cache file age in milliseconds.
- */
- public long getMaxFileAgeMS() {
- return MAX_FILE_AGE_MS;
+ return "FileCache [cacheDir=" + cacheDir + "]";
}
/**
@@ -433,19 +291,53 @@ public class FileCache {
return cleanDaemon != null && cleanDaemon.isAlive();
}
- private class FileCacheMaintenanceDaemon extends Thread {
+ /**
+ * Prunes cache if interval since last maintenance exceeds {@link #MAINT_INTERVAL_MS}
+ *
+ * Only called during construction, and the only known multi-process conflict that can occur
+ * is when re-writing the "lastMaint" timestamp file, which isn't a problem as its the
+ * approximate timestamp of that file that is important, not the contents.
+ *
+ * @param cacheDir cache directory location
+ * @param nestingLevel the depth of directory nesting, 2 for old style, 1 for newer style
+ * @return {@link FileCacheMaintenanceDaemon} instance if started, null otherwise
+ */
+ private static FileCacheMaintenanceDaemon performCacheMaintIfNeeded(File cacheDir,
+ int nestingLevel) {
+ File lastMaintFile = new File(cacheDir, ".lastmaint");
+ long lastMaintTS = lastMaintFile.isFile() ? lastMaintFile.lastModified() : 0;
+ if (lastMaintTS + MAINT_INTERVAL_MS > System.currentTimeMillis()) {
+ return null;
+ }
- FileCacheMaintenanceDaemon() {
+ FileCacheMaintenanceDaemon cleanDaemon =
+ new FileCacheMaintenanceDaemon(cacheDir, lastMaintFile, nestingLevel);
+ cleanDaemon.start();
+ return cleanDaemon;
+ }
+
+ private static class FileCacheMaintenanceDaemon extends Thread {
+ private File lastMaintFile;
+ private File cacheDir;
+ private long storageEstimateBytes;
+ private int nestingLevel;
+
+ FileCacheMaintenanceDaemon(File cacheDir, File lastMaintFile, int nestingLevel) {
setDaemon(true);
+ setName("FileCacheMaintenanceDaemon for " + cacheDir.getName());
+ this.cacheDir = cacheDir;
+ this.lastMaintFile = lastMaintFile;
+ this.nestingLevel = nestingLevel;
}
@Override
public void run() {
-
- performCacheMaint();
+ Msg.info(this, "Starting cache cleanup: " + cacheDir);
+ cacheMaintForDir(cacheDir, 0);
+ Msg.info(this,
+ "Finished cache cleanup, estimated storage used: " + storageEstimateBytes);
// stamp the file after we finish, in case the VM stopped this daemon thread
- lastMaintTS = System.currentTimeMillis();
try {
FileUtilities.writeStringToFile(lastMaintFile, "Last maint run at " + (new Date()));
}
@@ -453,5 +345,255 @@ public class FileCache {
Msg.error(this, "Unable to write file cache maintenance file: " + lastMaintFile, e);
}
}
+
+ private void cacheMaintForDir(File dir, int dirLevel) {
+ if (dirLevel < nestingLevel) {
+ for (File f : dir.listFiles()) {
+ String name = f.getName();
+ if (f.isDirectory() && NESTING_DIR_NAME_REGEX.matcher(name).matches()) {
+ cacheMaintForDir(f, dirLevel + 1);
+ }
+ }
+ }
+ else if (dirLevel == nestingLevel) {
+ cacheMaintForLeafDir(dir);
+ }
+ }
+
+ private void cacheMaintForLeafDir(File dir) {
+ long cutoffMS = System.currentTimeMillis() - MAX_FILE_AGE_MS;
+
+ for (File f : dir.listFiles()) {
+ if (f.isFile() && isCacheFileName(f.getName())) {
+ if (f.lastModified() < cutoffMS) {
+ if (f.delete()) {
+ Msg.debug(this, "Expired cache file " + f);
+ continue;
+ }
+ Msg.error(this, "Failed to delete cache file " + f);
+ }
+ storageEstimateBytes += f.length();
+ }
+ }
+ }
+
+ private boolean isCacheFileName(String s) {
+ return FILENAME_REGEX.matcher(s).matches();
+ }
+
+ }
+
+ /**
+ * Helper class, keeps a FileCacheEntry pinned while the ByteProvider is alive. When
+ * the ByteProvider is closed, the FileCacheEntry is allowed to be garbage collected
+ * if there is enough memory pressure to also remove its entry from the {@link FileCache#memCache}
+ * map.
+ */
+ private static class RefPinningByteArrayProvider extends ByteArrayProvider {
+ @SuppressWarnings("unused")
+ private FileCacheEntry fce; // its just here to be pinned in memory
+
+ public RefPinningByteArrayProvider(FileCacheEntry fce, FSRL fsrl) {
+ super(fce.bytes, fsrl);
+
+ this.fce = fce;
+ }
+
+ @Override
+ public void close() {
+ fce = null;
+ super.hardClose();
+ }
+ }
+
+ /**
+ * Allows creating {@link FileCacheEntry file cache entries} at the caller's convenience.
+ *
+ */
+ public class FileCacheEntryBuilder extends OutputStream {
+
+ private OutputStream delegate;
+ private HashingOutputStream hos;
+ private FileCacheEntry fce;
+ private long delegateLength;
+ private File tmpFile;
+
+ private FileCacheEntryBuilder(long sizeHint) throws IOException {
+ sizeHint = sizeHint <= 0 ? 512 : sizeHint;
+ if (sizeHint < MAX_INMEM_FILESIZE) {
+ delegate = new ByteArrayOutputStream((int) sizeHint);
+ }
+ else {
+ tmpFile = createTempFile();
+ delegate = new ObfuscatedOutputStream(new FileOutputStream(tmpFile));
+ }
+ initHashingOutputStream();
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ if (hos != null) {
+ Msg.warn(this, "FAIL TO CLOSE FileCacheEntryBuilder, currentSize=" +
+ delegateLength + ", file=" + (tmpFile != null ? tmpFile : "not set"));
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ switchToTempFileIfNecessary(1);
+ hos.write(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ switchToTempFileIfNecessary(b.length);
+ hos.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ switchToTempFileIfNecessary(len);
+ hos.write(b, off, len);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ hos.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ finish();
+ }
+
+ private void initHashingOutputStream() throws IOException {
+ try {
+ hos = new HashingOutputStream(delegate, HashUtilities.MD5_ALGORITHM);
+ }
+ catch (NoSuchAlgorithmException e) {
+ throw new IOException("Error getting MD5 algo", e);
+ }
+ }
+
+ private void switchToTempFileIfNecessary(int bytesToAdd) throws IOException {
+ delegateLength += bytesToAdd;
+ if (tmpFile == null && delegateLength > MAX_INMEM_FILESIZE) {
+ tmpFile = createTempFile();
+ byte[] bytes = ((ByteArrayOutputStream) delegate).toByteArray();
+ delegate = new ObfuscatedOutputStream(new FileOutputStream(tmpFile));
+ initHashingOutputStream();
+ // send the old bytes through the new hasher and to the tmp file
+ hos.write(bytes);
+ }
+ }
+
+ /**
+ * Finalizes this builder, pushing the bytes that have been written to it into
+ * the FileCache.
+ *
+ * @return new {@link FileCacheEntry}
+ * @throws IOException if error
+ */
+ public FileCacheEntry finish() throws IOException {
+ if (hos != null) {
+ hos.close();
+ String md5 = NumericUtilities.convertBytesToString(hos.getDigest());
+ if (tmpFile != null) {
+ fce = addTmpFileToCache(tmpFile, md5);
+ }
+ else {
+ ByteArrayOutputStream baos = (ByteArrayOutputStream) delegate;
+ byte[] bytes = baos.toByteArray();
+ fce = new FileCacheEntry(bytes, md5);
+ synchronized (FileCache.this) {
+ memCache.put(md5, fce);
+ }
+ }
+ hos = null;
+ delegate = null;
+ }
+ return fce;
+ }
+
+ }
+
+ /**
+ * Represents a cached file. It may be an actual file if {@link FileCacheEntry#file file}
+ * is set, or if smaller than {@link FileCache#MAX_INMEM_FILESIZE 2Mb'ish} just an
+ * in-memory byte array that is weakly pinned in the {@link FileCache#memCache} map.
+ */
+ public static class FileCacheEntry {
+
+ final String md5;
+ final File file;
+ final byte[] bytes;
+
+ private FileCacheEntry(File file, String md5) {
+ this.file = file;
+ this.bytes = null;
+ this.md5 = md5;
+ }
+
+ private FileCacheEntry(byte[] bytes, String md5) {
+ this.file = null;
+ this.bytes = bytes;
+ this.md5 = md5;
+ }
+
+ /**
+ * Returns the contents of this cache entry as a {@link ByteProvider}, using the specified
+ * {@link FSRL}.
+ *
+ * @param fsrl {@link FSRL} that the returned {@link ByteProvider} should have as its
+ * identity
+ * @return new {@link ByteProvider} containing the contents of this cache entry, caller is
+ * responsible for {@link ByteProvider#close() closing}
+ * @throws IOException if error
+ */
+ public ByteProvider asByteProvider(FSRL fsrl) throws IOException {
+ if (fsrl.getMD5() == null) {
+ fsrl = fsrl.withMD5(md5);
+ }
+ if (file != null) {
+ file.setLastModified(System.currentTimeMillis());
+ }
+ return (bytes != null)
+ ? new RefPinningByteArrayProvider(this, fsrl)
+ : new ObfuscatedFileByteProvider(file, fsrl, AccessMode.READ);
+ }
+
+ /**
+ * Returns the MD5 of this cache entry.
+ *
+ * @return the MD5 (as a string) of this cache entry
+ */
+ public String getMD5() {
+ return md5;
+ }
+
+ public long length() {
+ return bytes != null ? bytes.length : file.length();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(md5);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ FileCacheEntry other = (FileCacheEntry) obj;
+ return Objects.equals(md5, other.md5);
+ }
+
}
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileFingerprintCache.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileFingerprintCache.java
deleted file mode 100644
index 64bed1ed41..0000000000
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileFingerprintCache.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/* ###
- * IP: GHIDRA
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package ghidra.formats.gfilesystem;
-
-import org.apache.commons.collections4.map.ReferenceMap;
-
-/**
- * A best-effort cache of MD5 values of local files based on their {name,timestamp,length} fingerprint.
- *
- * Used to quickly verify that a local file hasn't changed.
- *
- */
-public class FileFingerprintCache {
-
- private ReferenceMap fileFingerprintToMD5Map = new ReferenceMap<>();
-
- /**
- * Clears the cache.
- */
- public synchronized void clear() {
- fileFingerprintToMD5Map.clear();
- }
-
- /**
- * Add a file's fingerprint to the cache.
- *
- * @param path String path to the file
- * @param md5 hex-string md5 of the file
- * @param timestamp long last modified timestamp of the file
- * @param length long file size
- */
- public synchronized void add(String path, String md5, long timestamp, long length) {
- fileFingerprintToMD5Map.put(new FileFingerprintRec(path, timestamp, length), md5);
- }
-
- /**
- * Returns true if the specified file with the specified fingerprints (timestamp, length)
- * was previously added to the cache with the specified md5.
- *
- * @param path String path to the file
- * @param md5 hex-string md5 of the file
- * @param timestamp long last modified timestamp of the file
- * @param length long file size
- * @return true if the fingerprint has previously been added to the cache.
- */
- public synchronized boolean contains(String path, String md5, long timestamp, long length) {
- String prevMD5 =
- fileFingerprintToMD5Map.get(new FileFingerprintRec(path, timestamp, length));
- return prevMD5 != null && prevMD5.equals(md5);
- }
-
- /**
- * Retrieves the md5 for the specified file that has the specified fingerprint (timestamp, length).
- *
- * @param path String path to the file
- * @param timestamp long last modified timestamp of the file
- * @param length long file size
- * @return hex-string md5 or null if not present in the cache.
- */
- public synchronized String getMD5(String path, long timestamp, long length) {
- String prevMD5 =
- fileFingerprintToMD5Map.get(new FileFingerprintRec(path, timestamp, length));
- return prevMD5;
- }
-
- //-----------------------------------------------------------------------------------
-
- static class FileFingerprintRec {
- final String path;
- final long timestamp;
- final long length;
-
- FileFingerprintRec(String path, long timestamp, long length) {
- this.path = path;
- this.timestamp = timestamp;
- this.length = length;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (int) (length ^ (length >>> 32));
- result = prime * result + ((path == null) ? 0 : path.hashCode());
- result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof FileFingerprintRec)) {
- return false;
- }
- FileFingerprintRec other = (FileFingerprintRec) obj;
- if (length != other.length) {
- return false;
- }
- if (path == null) {
- if (other.path != null) {
- return false;
- }
- }
- else if (!path.equals(other.path)) {
- return false;
- }
- if (timestamp != other.timestamp) {
- return false;
- }
- return true;
- }
- }
-}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemIndexHelper.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemIndexHelper.java
index 81009360e1..848eecf0fe 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemIndexHelper.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemIndexHelper.java
@@ -15,14 +15,17 @@
*/
package ghidra.formats.gfilesystem;
+import java.io.IOException;
import java.util.*;
+import java.util.stream.Collectors;
+
+import ghidra.util.Msg;
/**
* A helper class used by GFilesystem implementors to track mappings between GFile
* instances and the underlying container filesystem's native file objects.
*
- * Threadsafe after initial use of {@link #storeFile(String, int, boolean, long, Object) storeFile()}
- * by the owning filesystem.
+ * Threadsafe (methods are synchronized).
*
* This class also provides filename 'unique-ifying' (per directory) where an auto-incrementing
* number will be added to a file's filename if it is not unique in the directory.
@@ -33,9 +36,16 @@ import java.util.*;
public class FileSystemIndexHelper {
private GFile rootDir;
+
+ static class FileData {
+ GFile file;
+ METADATATYPE metaData;
+ long fileIndex;
+ }
- protected Map fileToEntryMap = new HashMap<>();
- protected Map> directoryToListing = new HashMap<>();
+ protected Map> fileToEntryMap = new HashMap<>();
+ protected Map> fileIndexToEntryMap = new HashMap<>();
+ protected Map>> directoryToListing = new HashMap<>();
/**
* Creates a new {@link FileSystemIndexHelper} for the specified {@link GFileSystem}.
@@ -49,6 +59,17 @@ public class FileSystemIndexHelper {
*/
public FileSystemIndexHelper(GFileSystem fs, FSRLRoot fsFSRL) {
this.rootDir = GFileImpl.fromFSRL(fs, null, fsFSRL.withPath("/"), true, -1);
+ initRootDir(null);
+ }
+
+ private void initRootDir(METADATATYPE metadata) {
+ FileData fileData = new FileData<>();
+ fileData.file = rootDir;
+ fileData.fileIndex = -1;
+ fileData.metaData = metadata;
+
+ fileToEntryMap.put(rootDir, fileData);
+ directoryToListing.put(rootDir, new HashMap<>());
}
/**
@@ -63,7 +84,7 @@ public class FileSystemIndexHelper {
/**
* Removes all file info from this index.
*/
- public void clear() {
+ public synchronized void clear() {
fileToEntryMap.clear();
directoryToListing.clear();
}
@@ -71,56 +92,84 @@ public class FileSystemIndexHelper {
/**
* Number of files in this index.
*
- * @return number of file in this index.
+ * @return number of file in this index
*/
- public int getFileCount() {
+ public synchronized int getFileCount() {
return fileToEntryMap.size();
}
/**
* Gets the opaque filesystem specific blob that was associated with the specified file.
*
- * @param f {@link GFile} to look for.
- * @return Filesystem specific blob associated with the specified file, or null if not found.
+ * @param f {@link GFile} to look for
+ * @return Filesystem specific blob associated with the specified file, or null if not found
*/
- public METADATATYPE getMetadata(GFile f) {
- return fileToEntryMap.get(f);
+ public synchronized METADATATYPE getMetadata(GFile f) {
+ FileData fileData = fileToEntryMap.get(f);
+ return fileData != null ? fileData.metaData : null;
+ }
+
+ /**
+ * Sets the associated metadata blob for the specified file.
+ *
+ * @param f GFile to update
+ * @param metaData new metdata blob
+ * @throws IOException if unknown file
+ */
+ public synchronized void setMetadata(GFile f, METADATATYPE metaData) throws IOException {
+ FileData fileData = fileToEntryMap.get(f);
+ if ( fileData == null ) {
+ throw new IOException("Unknown file: " + f);
+ }
+ fileData.metaData = metaData;
+ }
+
+ /**
+ * Gets the GFile instance that was associated with the filesystem file index.
+ *
+ * @param fileIndex index of the file in its filesystem
+ * @return the associated GFile instance, or null if not found
+ */
+ public synchronized GFile getFileByIndex(long fileIndex) {
+ FileData fileData = fileIndexToEntryMap.get(fileIndex);
+ return (fileData != null) ? fileData.file : null;
}
/**
* Mirror's {@link GFileSystem#getListing(GFile)} interface.
*
* @param directory {@link GFile} directory to get the list of child files that have been
- * added to this index, null means root directory.
- * @return {@link List} of GFile files that are in the specified directory, never null.
+ * added to this index, null means root directory
+ * @return {@link List} of GFile files that are in the specified directory, never null
*/
- public List getListing(GFile directory) {
- Map dirListing = getDirectoryContents(directory, false);
- List results =
- (dirListing != null) ? new ArrayList<>(dirListing.values()) : Collections.emptyList();
- return results;
+ public synchronized List getListing(GFile directory) {
+ Map> dirListing = getDirectoryContents(directory, false);
+ if (dirListing == null) {
+ return List.of();
+ }
+ return dirListing.values()
+ .stream()
+ .map(fd -> fd.file)
+ .collect(Collectors.toList());
}
/**
* Mirror's {@link GFileSystem#lookup(String)} interface.
*
- * @param path path and filename of a file to find.
- * @return {@link GFile} instance or null if no file was added to the index at that path.
+ * @param path path and filename of a file to find
+ * @return {@link GFile} instance or null if no file was added to the index at that path
*/
- public GFile lookup(String path) {
+ public synchronized GFile lookup(String path) {
String[] nameparts = (path != null ? path : "").split("/");
GFile parent = lookupParent(nameparts);
- if (nameparts.length == 0) {
- return parent;
- }
-
- String name = nameparts[nameparts.length - 1];
+ String name = (nameparts.length > 0) ? nameparts[nameparts.length - 1] : null;
if (name == null || name.isEmpty()) {
return parent;
}
- Map dirListing = getDirectoryContents(parent, false);
- return (dirListing != null) ? dirListing.get(name) : null;
+ Map> dirListing = getDirectoryContents(parent, false);
+ FileData fileData = (dirListing != null) ? dirListing.get(name) : null;
+ return (fileData != null) ? fileData.file : null;
}
/**
@@ -133,38 +182,27 @@ public class FileSystemIndexHelper {
* suffix added to the resultant GFile name, where nnn is the file's
* order of occurrence in the container file.
*
+ *
* @param path string path and filename of the file being added to the index. Back
- * slashes are normalized to forward slashes.
+ * slashes are normalized to forward slashes
* @param fileIndex the filesystem specific unique index for this file, or -1
- * if not available.
+ * if not available
* @param isDirectory boolean true if the new file is a directory
- * @param length number of bytes in the file or -1 if not known or directory.
- * @param fileInfo opaque blob that will be stored and associated with the new
- * GFile instance.
- * @return new GFile instance.
+ * @param length number of bytes in the file or -1 if not known or directory
+ * @param metadata opaque blob that will be stored and associated with the new
+ * GFile instance
+ * @return new GFile instance
*/
- public GFileImpl storeFile(String path, int fileIndex, boolean isDirectory, long length,
- METADATATYPE fileInfo) {
+ public synchronized GFile storeFile(String path, long fileIndex, boolean isDirectory,
+ long length, METADATATYPE metadata) {
String[] nameparts = path.replaceAll("[\\\\]", "/").split("/");
GFile parent = lookupParent(nameparts);
- int fileNum = (fileIndex != -1) ? fileIndex : fileToEntryMap.size();
String lastpart = nameparts[nameparts.length - 1];
- Map dirContents = getDirectoryContents(parent, true);
- String uniqueName = dirContents.containsKey(lastpart) && !isDirectory
- ? lastpart + "[" + Integer.toString(fileNum) + "]"
- : lastpart;
-
- GFileImpl file = createNewFile(parent, uniqueName, isDirectory, length, fileInfo);
-
- dirContents.put(uniqueName, file);
- if (file.isDirectory()) {
- getDirectoryContents(file, true);
- }
-
- fileToEntryMap.put(file, fileInfo);
- return file;
+ FileData fileData =
+ doStoreFile(lastpart, parent, fileIndex, isDirectory, length, metadata);
+ return fileData.file;
}
/**
@@ -176,47 +214,81 @@ public class FileSystemIndexHelper {
* suffix added to the resultant GFile name, where nnn is the file's
* order of occurrence in the container file.
*
+ *
* @param filename the new file's name
* @param parent the new file's parent directory
* @param fileIndex the filesystem specific unique index for this file, or -1
- * if not available.
+ * if not available
* @param isDirectory boolean true if the new file is a directory
- * @param length number of bytes in the file or -1 if not known or directory.
- * @param fileInfo opaque blob that will be stored and associated with the new
- * GFile instance.
- * @return new GFile instance.
+ * @param length number of bytes in the file or -1 if not known or directory
+ * @param metadata opaque blob that will be stored and associated with the new
+ * GFile instance
+ * @return new GFile instance
*/
- public GFile storeFileWithParent(String filename, GFile parent, int fileIndex,
- boolean isDirectory, long length, METADATATYPE fileInfo) {
+ public synchronized GFile storeFileWithParent(String filename, GFile parent, long fileIndex,
+ boolean isDirectory, long length, METADATATYPE metadata) {
+ FileData fileData =
+ doStoreFile(filename, parent, fileIndex, isDirectory, length, metadata);
+ return fileData.file;
+ }
+
+ private FileData doStoreMissingDir(String filename, GFile parent) {
parent = (parent == null) ? rootDir : parent;
- int fileNum = (fileIndex != -1) ? fileIndex : fileToEntryMap.size();
- Map dirContents = getDirectoryContents(parent, true);
- String uniqueName = dirContents.containsKey(filename) && !isDirectory
- ? filename + "[" + Integer.toString(fileNum) + "]"
- : filename;
+ Map> dirContents = getDirectoryContents(parent, true);
+ GFile file = createNewFile(parent, filename, true, -1, null);
- GFile file = createNewFile(parent, uniqueName, isDirectory, length, fileInfo);
+ FileData fileData = new FileData<>();
+ fileData.file = file;
+ fileData.fileIndex = -1;
+ fileToEntryMap.put(file, fileData);
+ dirContents.put(filename, fileData);
+ getDirectoryContents(file, true);
- dirContents.put(uniqueName, file);
- if (file.isDirectory()) {
+ return fileData;
+ }
+
+ private FileData doStoreFile(String filename, GFile parent, long fileIndex,
+ boolean isDirectory, long length, METADATATYPE metadata) {
+ parent = (parent == null) ? rootDir : parent;
+ long fileNum = (fileIndex != -1) ? fileIndex : fileToEntryMap.size();
+ if (fileIndexToEntryMap.containsKey(fileNum)) {
+ Msg.warn(this, "Duplicate fileNum for file " + parent.getPath() + "/" + filename);
+ }
+
+ Map> dirContents = getDirectoryContents(parent, true);
+ String uniqueName = makeUniqueFilename(dirContents.containsKey(filename) && !isDirectory,
+ filename, fileNum);
+
+ GFile file = createNewFile(parent, uniqueName, isDirectory, length, metadata);
+
+ FileData fileData = new FileData<>();
+ fileData.file = file;
+ fileData.fileIndex = fileNum;
+ fileData.metaData = metadata;
+ fileToEntryMap.put(file, fileData);
+ fileIndexToEntryMap.put(fileNum, fileData);
+
+ dirContents.put(uniqueName, fileData);
+ if (isDirectory) {
+ // side-effect of get will eagerly create the directorylisting entry
getDirectoryContents(file, true);
}
- fileToEntryMap.put(file, fileInfo);
- return file;
+ return fileData;
}
- /**
- * Returns a string->GFile map that holds the contents of a single directory.
- * @param directoryFile
- * @return
- */
- protected Map getDirectoryContents(GFile directoryFile,
+ private String makeUniqueFilename(boolean wasNameCollision, String filename, long fileIndex) {
+ return wasNameCollision
+ ? filename + "[" + Long.toString(fileIndex) + "]"
+ : filename;
+ }
+
+ private Map> getDirectoryContents(GFile directoryFile,
boolean createIfMissing) {
directoryFile = (directoryFile != null) ? directoryFile : rootDir;
- Map dirContents = directoryToListing.get(directoryFile);
+ Map> dirContents = directoryToListing.get(directoryFile);
if (dirContents == null && createIfMissing) {
dirContents = new HashMap<>();
directoryToListing.put(directoryFile, dirContents);
@@ -242,23 +314,21 @@ public class FileSystemIndexHelper {
protected GFile lookupParent(String[] nameparts) {
GFile currentDir = rootDir;
- GFile currentFile = rootDir;
for (int i = 0; i < nameparts.length - 1; i++) {
- Map currentDirContents = getDirectoryContents(currentDir, true);
+ Map> currentDirContents =
+ getDirectoryContents(currentDir, true);
String name = nameparts[i];
if (name.isEmpty()) {
continue;
}
- currentFile = currentDirContents.get(name);
- if (currentFile == null) {
- currentFile = createNewFile(currentDir, name, true, -1, null);
- currentDirContents.put(name, currentFile);
- getDirectoryContents(currentFile, true);
+ FileData fileData = currentDirContents.get(name);
+ if (fileData == null) {
+ fileData = doStoreMissingDir(name, currentDir);
}
- currentDir = currentFile;
+ currentDir = fileData.file;
}
- return currentFile;
+ return currentDir;
}
/**
@@ -282,6 +352,38 @@ public class FileSystemIndexHelper {
size);
}
+ /**
+ * Updates the FSRL of a file already in the index.
+ *
+ * @param file current {@link GFile}
+ * @param newFSRL the new FSRL the new file will be given
+ */
+ public synchronized void updateFSRL(GFile file, FSRL newFSRL) {
+ GFileImpl newFile = GFileImpl.fromFSRL(rootDir.getFilesystem(), file.getParentFile(),
+ newFSRL, file.isDirectory(), file.getLength());
+
+ FileData fileData = fileToEntryMap.get(file);
+ if (fileData != null) {
+ fileToEntryMap.remove(file);
+ fileIndexToEntryMap.remove(fileData.fileIndex);
+
+ fileData.file = newFile;
+
+ fileToEntryMap.put(newFile, fileData);
+ if (fileData.fileIndex != -1) {
+ fileIndexToEntryMap.put(fileData.fileIndex, fileData);
+ }
+ }
+
+ Map> dirListing = directoryToListing.get(file);
+ if ( dirListing != null) {
+ // typically this shouldn't ever happen as directory entries don't have MD5s and won't need to be updated
+ // after the fact
+ directoryToListing.remove(file);
+ directoryToListing.put(newFile, dirListing);
+ }
+ }
+
@Override
public String toString() {
return "FileSystemIndexHelper for " + rootDir.getFilesystem();
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemCache.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemInstanceManager.java
similarity index 91%
rename from Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemCache.java
rename to Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemInstanceManager.java
index 0483c9a545..c922b4ab66 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemCache.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemInstanceManager.java
@@ -27,7 +27,7 @@ import ghidra.util.Msg;
* Any filesystems that are not referenced by outside users (via a {@link FileSystemRef}) will
* be closed and removed from the cache when the next {@link #cacheMaint()} is performed.
*/
-public class FileSystemCache implements FileSystemEventListener {
+class FileSystemInstanceManager implements FileSystemEventListener {
private static class FSCacheInfo {
FileSystemRef ref;
@@ -47,7 +47,7 @@ public class FileSystemCache implements FileSystemEventListener {
* @param rootFS reference to the global root file system, which is a special case
* file system that is not subject to eviction.
*/
- public FileSystemCache(GFileSystem rootFS) {
+ public FileSystemInstanceManager(GFileSystem rootFS) {
this.rootFS = rootFS;
this.rootFSRL = rootFS.getFSRL();
}
@@ -190,7 +190,7 @@ public class FileSystemCache implements FileSystemEventListener {
continue;
}
- if (fsContainer.equals(containerFSRL)) {
+ if (containerFSRL.isEquivalent(fsContainer)) {
return ref.dup();
}
}
@@ -267,4 +267,23 @@ public class FileSystemCache implements FileSystemEventListener {
Msg.error(this, "Error closing filesystem", e);
}
}
+
+ /**
+ * Closes the specified ref, and if no other refs to the file system remain, closes the file system.
+ *
+ * @param ref {@link FileSystemRef} to close
+ */
+ public synchronized void releaseImmediate(FileSystemRef ref) {
+ FSCacheInfo fsci = filesystems.get(ref.getFilesystem().getFSRL());
+ ref.close();
+ if (fsci == null) {
+ Msg.warn(this, "Unknown file system reference: " + ref.getFilesystem().getFSRL());
+ return;
+ }
+ FileSystemRefManager refManager = fsci.ref.getFilesystem().getRefManager();
+ if (refManager.canClose(fsci.ref)) {
+ release(fsci);
+ }
+
+ }
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemRefManager.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemRefManager.java
index cbe3a4f335..a07b782315 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemRefManager.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemRefManager.java
@@ -15,13 +15,13 @@
*/
package ghidra.formats.gfilesystem;
+import java.util.ArrayList;
+import java.util.List;
+
import ghidra.util.Msg;
import ghidra.util.datastruct.WeakDataStructureFactory;
import ghidra.util.datastruct.WeakSet;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* A threadsafe helper class that manages creating and releasing {@link FileSystemRef} instances
* and broadcasting events to {@link FileSystemEventListener} listeners.
@@ -166,7 +166,8 @@ public class FileSystemRefManager {
// where instances are created and thrown away without a close() to probe
// filesystem container files.
if (fs != null && !(fs instanceof GFileSystemBase)) {
- Msg.warn(this, "Unclosed FilesytemRefManager for filesystem: " + fs.getClass());
+ Msg.warn(this, "Unclosed FilesytemRefManager for filesystem: " + fs.getClass() + ", " +
+ fs.getName());
}
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemService.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemService.java
index 1ada2cecae..5a05e9bbf7 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemService.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemService.java
@@ -18,15 +18,14 @@ package ghidra.formats.gfilesystem;
import java.io.*;
import java.util.List;
-import org.apache.commons.io.FilenameUtils;
-
-import ghidra.app.util.bin.ByteProvider;
-import ghidra.app.util.bin.RandomAccessByteProvider;
+import ghidra.app.util.bin.*;
+import ghidra.formats.gfilesystem.FileCache.FileCacheEntry;
+import ghidra.formats.gfilesystem.FileCache.FileCacheEntryBuilder;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
+import ghidra.formats.gfilesystem.crypto.*;
import ghidra.formats.gfilesystem.factory.FileSystemFactoryMgr;
import ghidra.framework.Application;
import ghidra.util.Msg;
-import ghidra.util.datastruct.FixedSizeHashMap;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import ghidra.util.timer.GTimer;
@@ -48,40 +47,14 @@ import ghidra.util.timer.GTimer;
* If you are working with {@link GFile} instances, you should have a
* {@link FileSystemRef fs ref} that you are using to pin the filesystem.
*
+ * Files written to the {@code fscache} directory are obfuscated to prevent interference from
+ * virus scanners. See {@link ObfuscatedInputStream} or {@link ObfuscatedOutputStream} or
+ * {@link ObfuscatedFileByteProvider}.
+ *
* Thread-safe.
*
- *
- *
{@literal
- * TODO list:
- *
- * Refactor fileInfo -> needs dialog to show properties
- * Refactor GFile.getInfo() to return Map<> instead of String.
- * Persistent filesystem - when reopen tool, filesystems should auto-reopen.
- * Unify GhidraFileChooser with GFileSystem.
- * Add "Mounted Filesystems" button to show currently opened GFilesystems?
- * Dockable filesystem browser in FrontEnd.
- * Reorg filesystem browser right-click popup menu to be more Eclipse action-like
- * Show In -> Project tree
- * Tool [CodeBrowser name]
- * Import
- * Open With -> Text Viewer
- * Image Viewer
- * Export -> To Project dir
- * To Home dir
- * To Dir
- * To Eclipse Project
- * Decompiled source
- * ProgramMappingService - more robust, precache when open project.
- * Make BatchImportDialog modeless, drag-and-drop to src list
- *
- * Testing:
- *
- * More format tests
- * Large test binary support
- * }
*/
public class FileSystemService {
- private static int FSRL_INTERN_SIZE = 1000;
private static FileSystemService instance;
@@ -100,20 +73,82 @@ public class FileSystemService {
return instance != null;
}
- private final LocalFileSystem localFS = LocalFileSystem.makeGlobalRootFS();
- private final FSRLRoot localFSRL = localFS.getFSRL();
- private final FileSystemFactoryMgr fsFactoryMgr = FileSystemFactoryMgr.getInstance();
- private FileCache fileCache;
- private FileSystemCache filesystemCache = new FileSystemCache(localFS);
- private FileCacheNameIndex fileCacheNameIndex = new FileCacheNameIndex();
- private FileFingerprintCache fileFingerprintCache = new FileFingerprintCache();
- private long fsCacheMaintIntervalMS = 10 * 1000;
+ /**
+ * Used by {@link FileSystemService#getDerivedByteProvider(FSRL, FSRL, String, long, DerivedStreamProducer, TaskMonitor) getDerivedByteProvider()}
+ * to produce a derivative stream from a source file.
+ *
+ * The {@link InputStream} returned from the method needs to supply the bytes of the derived file
+ * and will be closed by the caller.
+ *
+ * See {@link #produceDerivedStream()}.
+ */
+ public interface DerivedStreamProducer {
+
+ /**
+ * Callback method intended to be implemented by the caller to
+ * {@link FileSystemService#getDerivedByteProvider(FSRL, FSRL, String, long, DerivedStreamProducer, TaskMonitor)}
+ *
+ * The implementation needs to return an {@link InputStream} that contains the bytes
+ * of the derived file.
+ *
+ * @return a new {@link InputStream} that will produce all the bytes of the derived file
+ * @throws IOException if there is a problem while producing the InputStream
+ * @throws CancelledException if the user canceled
+ */
+ InputStream produceDerivedStream() throws IOException, CancelledException;
+ }
/**
- * LRU hashmap, limited in size to FSRL_INTERN_SIZE.
+ * Used by {@link FileSystemService#getDerivedByteProviderPush(FSRL, FSRL, String, long, DerivedStreamPushProducer, TaskMonitor) getDerivedByteProviderPush()}
+ * to produce a derivative stream from a source file.
+ *
+ * The implementation needs to write bytes to the supplied {@link OutputStream}.
+ *
+ * See {@link #push(OutputStream)}.
+ *
*/
- private FixedSizeHashMap fsrlInternMap =
- new FixedSizeHashMap<>(FSRL_INTERN_SIZE, FSRL_INTERN_SIZE);
+ public interface DerivedStreamPushProducer {
+ /**
+ * Callback method intended to be implemented by the caller to
+ * {@link FileSystemService#getDerivedByteProviderPush(FSRL, FSRL, String, long, DerivedStreamPushProducer, TaskMonitor) getDerivedByteProviderPush()}
+ *
+ * @param os {@link OutputStream} that the implementor should write the bytes to. Do
+ * not close the stream when done
+ * @throws IOException if there is a problem while writing to the OutputStream
+ * @throws CancelledException if the user canceled
+ */
+ void push(OutputStream os) throws IOException, CancelledException;
+ }
+
+ private final LocalFileSystem localFS = LocalFileSystem.makeGlobalRootFS();
+ private final FileSystemFactoryMgr fsFactoryMgr = FileSystemFactoryMgr.getInstance();
+ private final FSRLRoot cacheFSRL = FSRLRoot.makeRoot("cache");
+ private final FileCache fileCache;
+ private final FileSystemInstanceManager fsInstanceManager =
+ new FileSystemInstanceManager(localFS);
+ private final FileCacheNameIndex fileCacheNameIndex = new FileCacheNameIndex();
+ private long fsCacheMaintIntervalMS = 10 * 1000;
+ private CryptoSession currentCryptoSession;
/**
* Creates a FilesystemService instance, using the {@link Application}'s default value
@@ -121,7 +156,11 @@ public class FileSystemService {
* cache directory.
*/
public FileSystemService() {
- this(new File(Application.getUserCacheDirectory(), "fscache"));
+ this(new File(Application.getUserCacheDirectory(), "fscache2"));
+
+ // age off files in old cache dir. Remove this after a few versions
+ FileCache.performCacheMaintOnOldDirIfNeeded(
+ new File(Application.getUserCacheDirectory(), "fscache"));
}
/**
@@ -134,7 +173,7 @@ public class FileSystemService {
try {
fileCache = new FileCache(fscacheDir);
GTimer.scheduleRepeatingRunnable(fsCacheMaintIntervalMS, fsCacheMaintIntervalMS,
- () -> filesystemCache.cacheMaint());
+ () -> fsInstanceManager.cacheMaint());
}
catch (IOException e) {
throw new RuntimeException("Failed to init global cache " + fscacheDir, e);
@@ -145,9 +184,8 @@ public class FileSystemService {
* Forcefully closes all open filesystems and clears caches.
*/
public void clear() {
- synchronized (filesystemCache) {
- filesystemCache.clear();
- fsrlInternMap.clear();
+ synchronized (fsInstanceManager) {
+ fsInstanceManager.clear();
fileCacheNameIndex.clear();
}
}
@@ -156,7 +194,19 @@ public class FileSystemService {
* Close unused filesystems.
*/
public void closeUnusedFileSystems() {
- filesystemCache.closeAllUnused();
+ fsInstanceManager.closeAllUnused();
+ }
+
+ /**
+ * Releases the specified {@link FileSystemRef}, and if no other references remain, removes
+ * it from the shared cache of file system instances.
+ *
+ * @param fsRef the ref to release
+ */
+ public void releaseFileSystemImmediate(FileSystemRef fsRef) {
+ if (fsRef != null && !fsRef.isClosed()) {
+ fsInstanceManager.releaseImmediate(fsRef);
+ }
}
/**
@@ -168,6 +218,27 @@ public class FileSystemService {
return localFS;
}
+ /**
+ * Returns true if the specified location is a path on the local computer's
+ * filesystem.
+ *
+ * @param fsrl {@link FSRL} path to query
+ * @return true if local, false if the path points to an embedded file in a container.
+ */
+ public boolean isLocal(FSRL fsrl) {
+ return fsrl.getFS().hasContainer() == false;
+ }
+
+ /**
+ * Builds a {@link FSRL} of a {@link File file} located on the local filesystem.
+ *
+ * @param f {@link File} on the local filesystem
+ * @return {@link FSRL} pointing to the same file, never null
+ */
+ public FSRL getLocalFSRL(File f) {
+ return localFS.getLocalFSRL(f);
+ }
+
/**
* Returns true of there is a {@link GFileSystem filesystem} mounted at the requested
* {@link FSRL} location.
@@ -176,7 +247,7 @@ public class FileSystemService {
* @return boolean true if filesystem mounted at location.
*/
public boolean isFilesystemMountedAt(FSRL fsrl) {
- return filesystemCache.isFilesystemMountedAt(fsrl);
+ return fsInstanceManager.isFilesystemMountedAt(fsrl);
}
/**
@@ -195,8 +266,7 @@ public class FileSystemService {
*/
public RefdFile getRefdFile(FSRL fsrl, TaskMonitor monitor)
throws CancelledException, IOException {
- FSRLRoot fsRoot = fsrl.getFS();
- FileSystemRef ref = getFilesystem(fsRoot, monitor);
+ FileSystemRef ref = getFilesystem(fsrl.getFS(), monitor);
try {
GFile gfile = ref.getFilesystem().lookup(fsrl.getPath());
if (gfile == null) {
@@ -214,83 +284,6 @@ public class FileSystemService {
}
}
- /**
- * Return a {@link FileCacheEntry} with information about the requested file specified
- * by the FSRL, forcing a read/cache add of the file is it is missing from the cache.
- *
- * Never returns NULL, instead throws IOException.
- *
- * @param fsrl {@link FSRL} of the desired file.
- * @param monitor {@link TaskMonitor} to watch and update with progress.
- * @return new {@link FileCacheEntry} with info about the cached file.
- * @throws IOException if IO error when getting file.
- * @throws CancelledException if user canceled.
- */
- private FileCacheEntry getCacheFile(FSRL fsrl, TaskMonitor monitor)
- throws IOException, CancelledException {
-
- if (fsrl.getPath() == null) {
- throw new IOException("Invalid FSRL specified: " + fsrl);
- }
- String md5 = fsrl.getMD5();
- if (md5 == null && fsrl.getNestingDepth() == 1) {
- // if this is a real file on the local file system, and the FSRL doesn't specify
- // its MD5, try to fetch the MD5 from the fingerprint cache based on its
- // size and lastmod time, which will help us locate the file in the cache
- File f = localFS.getLocalFile(fsrl);
- if (f.isFile()) {
- md5 = fileFingerprintCache.getMD5(f.getPath(), f.lastModified(), f.length());
- }
- }
- FSRLRoot fsRoot = fsrl.getFS();
-
- FileCacheEntry result = (md5 != null) ? fileCache.getFile(md5) : null;
- if (result == null) {
- try (FileSystemRef ref = getFilesystem(fsRoot, monitor)) {
- GFileSystem fs = ref.getFilesystem();
- GFile gfile = fs.lookup(fsrl.getPath());
- if (gfile == null) {
- throw new IOException(
- "File [" + fsrl + "] not found in filesystem [" + fs.getFSRL() + "]");
- }
-
- // Its possible the filesystem added the file to the cache when it was mounted,
- // or that we now have a better FSRL with a MD5 value that we can use to
- // search the file cache.
- if (gfile.getFSRL().getMD5() != null) {
- result = fileCache.getFile(gfile.getFSRL().getMD5());
- if (result != null) {
- return result;
- }
- }
-
- try (InputStream dataStream = fs.getInputStream(gfile, monitor)) {
- if (dataStream == null) {
- throw new IOException("Unable to get datastream for " + fsrl);
- }
- monitor.setMessage("Caching " + gfile.getName());
- monitor.initialize(gfile.getLength());
- result = fileCache.addStream(dataStream, monitor);
- if (md5 != null && !md5.equals(result.md5)) {
- throw new IOException("Error reading file, MD5 has changed: " + fsrl +
- ", md5 now " + result.md5);
- }
- }
- if (fsrl.getNestingDepth() == 1) {
- // if this is a real file on the local filesystem, now that we have its
- // MD5, save it in the fingerprint cache so it can be found later
- File f = localFS.getLocalFile(fsrl);
- if (f.isFile()) {
- fileFingerprintCache.add(f.getPath(), result.md5, f.lastModified(),
- f.length());
- }
- }
-
- }
- }
-
- return result;
- }
/**
* Returns a filesystem instance for the requested {@link FSRLRoot}, either from an already
@@ -311,148 +304,25 @@ public class FileSystemService {
*/
public FileSystemRef getFilesystem(FSRLRoot fsFSRL, TaskMonitor monitor)
throws IOException, CancelledException {
- synchronized (filesystemCache) {
- FileSystemRef ref = filesystemCache.getRef(fsFSRL);
+ synchronized (fsInstanceManager) {
+ FileSystemRef ref = fsInstanceManager.getRef(fsFSRL);
if (ref == null) {
if (!fsFSRL.hasContainer()) {
throw new IOException("Bad FSRL " + fsFSRL);
}
- fsFSRL = intern(fsFSRL);
- FSRL containerFSRL = fsFSRL.getContainer();
- FileCacheEntry cfi = getCacheFile(containerFSRL, monitor);
- if (containerFSRL.getMD5() == null) {
- containerFSRL = containerFSRL.withMD5(cfi.md5);
- }
- GFileSystem fs = FileSystemFactoryMgr.getInstance()
- .mountFileSystem(
- fsFSRL.getProtocol(), containerFSRL, cfi.file, this, monitor);
+ ByteProvider containerByteProvider = getByteProvider(fsFSRL.getContainer(), true, monitor);
+ GFileSystem fs =
+ fsFactoryMgr.mountFileSystem(fsFSRL.getProtocol(), containerByteProvider, this, monitor);
ref = fs.getRefManager().create();
- filesystemCache.add(fs);
+ fsInstanceManager.add(fs);
}
return ref;
}
}
/**
- * Adds a {@link GFile file}'s stream's contents to the file cache, returning its MD5 hash.
- *
- * @param file {@link GFile} not really used currently
- * @param is {@link InputStream} to add to the cache.
- * @param monitor {@link TaskMonitor} to monitor and update.
- * @return string with new file's md5.
- * @throws IOException if IO error
- * @throws CancelledException if user canceled.
- */
- public FileCacheEntry addFileToCache(GFile file, InputStream is, TaskMonitor monitor)
- throws IOException, CancelledException {
- FileCacheEntry fce = fileCache.addStream(is, monitor);
- return fce;
- }
-
- /**
- * Stores a stream in the file cache.
- *
- * @param is {@link InputStream} to store in the cache.
- * @param monitor {@link TaskMonitor} to watch and update.
- * @return {@link File} location of the new file.
- * @throws IOException if IO error
- * @throws CancelledException if the user cancels.
- */
- public FileCacheEntry addStreamToCache(InputStream is, TaskMonitor monitor)
- throws IOException, CancelledException {
- FileCacheEntry fce = fileCache.addStream(is, monitor);
- return fce;
- }
-
- /**
- * Returns a {@link File java.io.file} with the data from the requested FSRL.
- * Simple local files will be returned directly, and files nested in containers
- * will be located in the file cache directory and have a 'random' name.
- *
- * Never returns nulls, throws IOException if not found or error.
- *
- * @param fsrl {@link FSRL} of the desired file.
- * @param monitor {@link TaskMonitor} to watch and update.
- * @return {@link File} of the desired file in the cache, never null.
- * @throws CancelledException if user cancels.
- * @throws IOException if IO problem.
- */
- public File getFile(FSRL fsrl, TaskMonitor monitor) throws CancelledException, IOException {
- if (fsrl.getNestingDepth() == 1) {
- // If this is a real files on the local filesystem, verify any
- // MD5 embedded in the FSRL before returning the live local file
- // as the result.
- File f = localFS.getLocalFile(fsrl);
- if (f.isFile() && fsrl.getMD5() != null) {
- if (!fileFingerprintCache.contains(f.getPath(), fsrl.getMD5(), f.lastModified(),
- f.length())) {
- String fileMD5 = FSUtilities.getFileMD5(f, monitor);
- if (!fsrl.getMD5().equals(fileMD5)) {
- throw new IOException("Exact file no longer exists: " + f.getPath() +
- " contents have changed, old md5: " + fsrl.getMD5() + ", new md5: " +
- fileMD5);
- }
- fileFingerprintCache.add(f.getPath(), fileMD5, f.lastModified(), f.length());
- }
- }
- return f;
- }
- FileCacheEntry fce = getCacheFile(fsrl, monitor);
- return fce.file;
- }
-
- private String getMD5(FSRL fsrl, TaskMonitor monitor) throws CancelledException, IOException {
- if (fsrl.getNestingDepth() == 1) {
- File f = localFS.getLocalFile(fsrl);
- if (!f.isFile()) {
- return null;
- }
- String md5 = fileFingerprintCache.getMD5(f.getPath(), f.lastModified(), f.length());
- if (md5 == null) {
- md5 = FSUtilities.getFileMD5(f, monitor);
- fileFingerprintCache.add(f.getPath(), md5, f.lastModified(), f.length());
- }
- return md5;
- }
- FileCacheEntry fce = getCacheFile(fsrl, monitor);
- return fce.md5;
- }
-
- /**
- * Builds a {@link FSRL} of a {@link File file} located on the local filesystem.
- *
- * @param f {@link File} on the local filesystem
- * @return {@link FSRL} pointing to the same file, never null
- */
- public FSRL getLocalFSRL(File f) {
- return localFS.getFSRL()
- .withPath(
- FSUtilities.appendPath("/", FilenameUtils.separatorsToUnix(f.getPath())));
- }
-
- /**
- * Converts a java {@link File} instance into a GFilesystem {@link GFile} hosted on the
- * {@link #getLocalFS() local filesystem}.
- *
- * @param f {@link File} on the local filesystem
- * @return {@link GFile} representing the same file or {@code null} if there was a problem
- * with the file path.
- */
- public GFile getLocalGFile(File f) {
- try {
- return localFS.lookup(f.getPath());
- }
- catch (IOException e) {
- // the LocalFileSystem impl doesn't check the validity of the path so this
- // exception should never happen. If it does, fall thru and return null.
- }
- return null;
- }
-
- /**
- * Returns a {@link ByteProvider} with the contents of the requested {@link GFile file}
- * (in the Global file cache directory).
+ * Returns a {@link ByteProvider} with the contents of the requested {@link GFile file}.
*
* Never returns null, throws IOException if there was a problem.
*
@@ -460,81 +330,142 @@ public class FileSystemService {
* when finished.
*
* @param fsrl {@link FSRL} file to wrap
- * @param monitor {@link TaskMonitor} to watch and update.
+ * @param fullyQualifiedFSRL if true, the returned ByteProvider's FSRL will always have a MD5
+ * hash
+ * @param monitor {@link TaskMonitor} to watch and update
* @return new {@link ByteProvider}
* @throws CancelledException if user cancels
- * @throws IOException if IO problem.
+ * @throws IOException if IO problem
*/
- public ByteProvider getByteProvider(FSRL fsrl, TaskMonitor monitor)
- throws CancelledException, IOException {
- File file = getFile(fsrl, monitor);
- RandomAccessByteProvider rabp = new RandomAccessByteProvider(file, fsrl);
- return rabp;
- }
-
- /**
- * Returns a reference to a file in the FileCache that contains the
- * derived (ie. decompressed or decrypted) contents of a source file, as well as
- * its md5.
- *
- * If the file was not present in the cache, the {@link DerivedFileProducer producer}
- * lambda will be called and it will be responsible for returning an {@link InputStream}
- * which has the derived contents, which will be added to the file cache for next time.
- *
- * @param fsrl {@link FSRL} of the source (or container) file that this derived file is based on
- * @param derivedName a unique string identifying the derived file inside the source (or container) file
- * @param producer a {@link DerivedFileProducer callback or lambda} that returns an
- * {@link InputStream} that will be streamed into a file and placed into the file cache.
- * Example:{@code (file) -> { return new XYZDecryptorInputStream(file); }}
- * @param monitor {@link TaskMonitor} that will be monitor for cancel requests and updated
- * with file io progress
- * @return {@link FileCacheEntry} with file and md5 fields
- * @throws CancelledException if the user cancels
- * @throws IOException if there was an io error
- */
- public FileCacheEntry getDerivedFile(FSRL fsrl, String derivedName,
- DerivedFileProducer producer, TaskMonitor monitor)
+ public ByteProvider getByteProvider(FSRL fsrl, boolean fullyQualifiedFSRL, TaskMonitor monitor)
throws CancelledException, IOException {
- // fileCacheNameIndex is queried and updated in separate steps,
- // which could be a race issue with another thread, but in this
- // case should be okay as the only bad result will be extra
- // work being performed recreating the contents of the same derived file a second
- // time.
- FileCacheEntry cacheEntry = getCacheFile(fsrl, monitor);
- String derivedMD5 = fileCacheNameIndex.get(cacheEntry.md5, derivedName);
- FileCacheEntry derivedFile = (derivedMD5 != null) ? fileCache.getFile(derivedMD5) : null;
- if (derivedFile == null) {
- monitor.setMessage(derivedName + " " + fsrl.getName());
- try (InputStream is = producer.produceDerivedStream(cacheEntry.file)) {
- derivedFile = fileCache.addStream(is, monitor);
- fileCacheNameIndex.add(cacheEntry.md5, derivedName, derivedFile.md5);
+ if ( fsrl.getMD5() != null ) {
+ FileCacheEntry fce = fileCache.getFileCacheEntry(fsrl.getMD5());
+ if ( fce != null ) {
+ return fce.asByteProvider(fsrl);
}
}
- return derivedFile;
+
+ try ( FileSystemRef fsRef = getFilesystem(fsrl.getFS(), monitor) ) {
+ GFileSystem fs = fsRef.getFilesystem();
+ GFile file = fs.lookup(fsrl.getPath());
+ if (file == null) {
+ throw new IOException("File not found: " + fsrl);
+ }
+ if (file.getFSRL().getMD5() != null) {
+ fsrl = file.getFSRL();
+ // try again to fetch cached file if we now have a md5
+ FileCacheEntry fce = fileCache.getFileCacheEntry(fsrl.getMD5());
+ if (fce != null) {
+ return fce.asByteProvider(fsrl);
+ }
+ }
+ ByteProvider provider = fs.getByteProvider(file, monitor);
+ if (provider == null) {
+ throw new IOException("Unable to get ByteProvider for " + fsrl);
+ }
+
+ // use the returned provider's FSRL as it may have more info
+ FSRL resultFSRL = provider.getFSRL();
+ if (resultFSRL.getMD5() == null && (fsrl.getMD5() != null || fullyQualifiedFSRL)) {
+ String md5 = (fs instanceof GFileHashProvider)
+ ? ((GFileHashProvider) fs).getMD5Hash(file, true, monitor)
+ : FSUtilities.getMD5(provider, monitor);
+ resultFSRL = resultFSRL.withMD5(md5);
+ }
+ if (fsrl.getMD5() != null) {
+ if (!fsrl.isMD5Equal(resultFSRL.getMD5())) {
+ throw new IOException("Unable to retrieve requested file, hash has changed: " +
+ fsrl + ", new hash: " + resultFSRL.getMD5());
+ }
+ }
+ return new RefdByteProvider(fsRef.dup(), provider, resultFSRL);
+ }
}
/**
- * Returns a reference to a file in the FileCache that contains the
- * derived (ie. decompressed or decrypted) contents of a source file, as well as
- * its md5.
+ * Returns a {@link ByteProvider} that contains the
+ * derived (ie. decompressed or decrypted) contents of the requested file.
*
- * If the file was not present in the cache, the {@link DerivedFilePushProducer push producer}
- * lambda will be called and it will be responsible for producing and writing the derived
- * file's bytes to a {@link OutputStream}, which will be added to the file cache for next time.
+ * The resulting ByteProvider will be a cached file, either written to a
+ * temporary file, or a in-memory buffer if small enough (see {@link FileCache#MAX_INMEM_FILESIZE}).
+ *
+ * If the file was not present in the cache, the {@link DerivedStreamProducer producer}
+ * will be called and it will be responsible for returning an {@link InputStream}
+ * which has the derived contents, which will be added to the file cache for next time.
*
- * @param fsrl {@link FSRL} of the source (or container) file that this derived file is based on
+ * @param containerFSRL {@link FSRL} w/hash of the source (or container) file that this
+ * derived file is based on
+ * @param derivedFSRL (optional) {@link FSRL} to assign to the resulting ByteProvider
* @param derivedName a unique string identifying the derived file inside the source (or container) file
- * @param pusher a {@link DerivedFilePushProducer callback or lambda} that recieves a {@link OutputStream}.
- * Example:{@code (os) -> { ...write to outputstream os here...; }}
+ * @param sizeHint the expected size of the resulting ByteProvider, or -1 if unknown
+ * @param producer supplies an InputStream if needed. See {@link DerivedStreamProducer}
* @param monitor {@link TaskMonitor} that will be monitor for cancel requests and updated
* with file io progress
- * @return {@link FileCacheEntry} with file and md5 fields
+ * @return a {@link ByteProvider} containing the bytes of the requested file, that has the
+ * specified derivedFSRL, or a pseudo FSRL if not specified. Never null
* @throws CancelledException if the user cancels
* @throws IOException if there was an io error
*/
- public FileCacheEntry getDerivedFilePush(FSRL fsrl, String derivedName,
- DerivedFilePushProducer pusher, TaskMonitor monitor)
+ public ByteProvider getDerivedByteProvider(FSRL containerFSRL, FSRL derivedFSRL,
+ String derivedName, long sizeHint, DerivedStreamProducer producer,
+ TaskMonitor monitor) throws CancelledException, IOException {
+
+ // fileCacheNameIndex is queried and updated in separate steps,
+ // which could be a race issue with another thread, but in this
+ // case should be okay as the only bad result will be extra
+ // work being performed recreating the contents of the same derived file a second
+ // time.
+ assertFullyQualifiedFSRL(containerFSRL);
+ String containerMD5 = containerFSRL.getMD5();
+ String derivedMD5 = fileCacheNameIndex.get(containerMD5, derivedName);
+ FileCacheEntry derivedFile = fileCache.getFileCacheEntry(derivedMD5);
+ if (derivedFile == null) {
+ monitor.setMessage("Caching " + containerFSRL.getName() + " " + derivedName);
+ if (sizeHint > 0) {
+ monitor.initialize(sizeHint);
+ }
+ try (InputStream is = producer.produceDerivedStream();
+ FileCacheEntryBuilder fceBuilder =
+ fileCache.createCacheEntryBuilder(sizeHint)) {
+ FSUtilities.streamCopy(is, fceBuilder, monitor);
+ derivedFile = fceBuilder.finish();
+ fileCacheNameIndex.add(containerMD5, derivedName, derivedFile.getMD5());
+ }
+ }
+ derivedFSRL = (derivedFSRL != null)
+ ? derivedFSRL.withMD5(derivedFile.getMD5())
+ : createCachedFileFSRL(derivedFile.getMD5());
+ return derivedFile.asByteProvider(derivedFSRL);
+ }
+
+ /**
+ * Returns a {@link ByteProvider} that contains the
+ * derived (ie. decompressed or decrypted) contents of the requested file.
+ *
+ * The resulting ByteProvider will be a cached file, either written to a
+ * temporary file, or a in-memory buffer if small enough (see {@link FileCache#MAX_INMEM_FILESIZE}).
+ *
+ * If the file was not present in the cache, the {@link DerivedStreamPushProducer pusher}
+ * will be called and it will be responsible for producing and writing the derived
+ * file's bytes to a {@link OutputStream}, which will be added to the file cache for next time.
+ *
+ * @param containerFSRL {@link FSRL} w/hash of the source (or container) file that this
+ * derived file is based on
+ * @param derivedFSRL (optional) {@link FSRL} to assign to the resulting ByteProvider
+ * @param derivedName a unique string identifying the derived file inside the source (or container) file
+ * @param sizeHint the expected size of the resulting ByteProvider, or -1 if unknown
+ * @param pusher writes bytes to the supplied OutputStream. See {@link DerivedStreamPushProducer}
+ * @param monitor {@link TaskMonitor} that will be monitor for cancel requests and updated
+ * with file io progress
+ * @return a {@link ByteProvider} containing the bytes of the requested file, that has the
+ * specified derivedFSRL, or a pseudo FSRL if not specified. Never null
+ * @throws CancelledException if the user cancels
+ * @throws IOException if there was an io error
+ */
+ public ByteProvider getDerivedByteProviderPush(FSRL containerFSRL, FSRL derivedFSRL,
+ String derivedName, long sizeHint, DerivedStreamPushProducer pusher, TaskMonitor monitor)
throws CancelledException, IOException {
// fileCacheNameIndex is queried and updated in separate steps,
@@ -542,32 +473,100 @@ public class FileSystemService {
// case should be okay as the only bad result will be extra
// work being performed recreating the contents of the same derived file a second
// time.
- FileCacheEntry cacheEntry = getCacheFile(fsrl, monitor);
- String derivedMD5 = fileCacheNameIndex.get(cacheEntry.md5, derivedName);
- FileCacheEntry derivedFile = (derivedMD5 != null) ? fileCache.getFile(derivedMD5) : null;
+ assertFullyQualifiedFSRL(containerFSRL);
+ String containerMD5 = containerFSRL.getMD5();
+ String derivedMD5 = fileCacheNameIndex.get(containerMD5, derivedName);
+ FileCacheEntry derivedFile = fileCache.getFileCacheEntry(derivedMD5);
if (derivedFile == null) {
- monitor.setMessage("Caching " + fsrl.getName() + " " + derivedName);
- derivedFile = fileCache.pushStream(pusher, monitor);
- fileCacheNameIndex.add(cacheEntry.md5, derivedName, derivedFile.md5);
+ monitor.setMessage("Caching " + containerFSRL.getName() + " " + derivedName);
+ if (sizeHint > 0) {
+ monitor.initialize(sizeHint);
+ }
+ try (FileCacheEntryBuilder fceBuilder = fileCache.createCacheEntryBuilder(sizeHint)) {
+ pusher.push(fceBuilder);
+ derivedFile = fceBuilder.finish();
+ }
+ fileCacheNameIndex.add(containerMD5, derivedName, derivedFile.getMD5());
}
- return derivedFile;
+ derivedFSRL = (derivedFSRL != null)
+ ? derivedFSRL.withMD5(derivedFile.getMD5())
+ : createCachedFileFSRL(derivedFile.getMD5());
+ return derivedFile.asByteProvider(derivedFSRL);
+ }
+
+ private FSRL createCachedFileFSRL(String md5) {
+ return cacheFSRL.withPathMD5("/" + md5, md5);
+ }
+
+ /**
+ * Returns a {@link FileCacheEntryBuilder} that will allow the caller to
+ * write bytes to it.
+ *
+ * After calling {@link FileCacheEntryBuilder#finish() finish()},
+ * the caller will have a {@link FileCacheEntry} that can provide access to a
+ * {@link ByteProvider}.
+ *
+ * Temporary files that are written to disk are obfuscated to avoid interference from
+ * overzealous virus scanners. See {@link ObfuscatedInputStream} /
+ * {@link ObfuscatedOutputStream}.
+ *
+ * @param sizeHint the expected size of the file, or -1 if unknown
+ * @return {@link FileCacheEntryBuilder} that must be finalized by calling
+ * {@link FileCacheEntryBuilder#finish() finish()}
+ * @throws IOException if error
+ */
+ public FileCacheEntryBuilder createTempFile(long sizeHint) throws IOException {
+ return fileCache.createCacheEntryBuilder(sizeHint);
+ }
+
+ /**
+ * Allows the resources used by caching the specified file to be released.
+ *
+ * @param fsrl {@link FSRL} file to release cache resources for
+ */
+ public void releaseFileCache(FSRL fsrl) {
+ if (fsrl.getMD5() != null) {
+ fileCache.releaseFileCacheEntry(fsrl.getMD5());
+ }
+ }
+
+ /**
+ * Adds a plaintext (non-obfuscated) file to the cache, consuming it in the process, and returns
+ * a {@link ByteProvider} that contains the contents of the file.
+ *
+ * NOTE: only use this if you have no other choice and are forced to deal with already
+ * existing files in the local filesystem.
+ *
+ * @param file {@link File} to add
+ * @param fsrl {@link FSRL} of the file that is being added
+ * @param monitor {@link TaskMonitor}
+ * @return {@link ByteProvider} (hosted in the FileCache) that contains the bytes of the
+ * specified file
+ * @throws CancelledException if cancelled
+ * @throws IOException if error
+ */
+ public ByteProvider pushFileToCache(File file, FSRL fsrl, TaskMonitor monitor)
+ throws CancelledException, IOException {
+ FileCacheEntry fce = fileCache.giveFile(file, monitor);
+ return fce.asByteProvider(fsrl);
}
/**
* Returns true if the specified derived file exists in the file cache.
*
- * @param fsrl {@link FSRL} of the container
+ * @param containerFSRL {@link FSRL} w/hash of the container
* @param derivedName name of the derived file inside of the container
* @param monitor {@link TaskMonitor}
* @return boolean true if file exists at time of query, false if file is not in cache
* @throws CancelledException if user cancels
* @throws IOException if other IO error
*/
- public boolean hasDerivedFile(FSRL fsrl, String derivedName, TaskMonitor monitor)
+ public boolean hasDerivedFile(FSRL containerFSRL, String derivedName, TaskMonitor monitor)
throws CancelledException, IOException {
- FileCacheEntry cacheEntry = getCacheFile(fsrl, monitor);
- String derivedMD5 = fileCacheNameIndex.get(cacheEntry.md5, derivedName);
- return derivedMD5 != null;
+ assertFullyQualifiedFSRL(containerFSRL);
+ String containerMD5 = containerFSRL.getMD5();
+ String derivedMD5 = fileCacheNameIndex.get(containerMD5, derivedName);
+ return derivedMD5 != null && fileCache.hasEntry(derivedMD5);
}
/**
@@ -582,8 +581,9 @@ public class FileSystemService {
*/
public boolean isFileFilesystemContainer(FSRL containerFSRL, TaskMonitor monitor)
throws CancelledException, IOException {
- File containerFile = getFile(containerFSRL, monitor);
- return fsFactoryMgr.test(containerFSRL, containerFile, this, monitor);
+ try (ByteProvider byteProvider = getByteProvider(containerFSRL, false, monitor)) {
+ return fsFactoryMgr.test(byteProvider, this, monitor);
+ }
}
/**
@@ -637,43 +637,30 @@ public class FileSystemService {
FileSystemProbeConflictResolver conflictResolver, int priorityFilter)
throws CancelledException, IOException {
- // Fix up FSRL first before querying
- containerFSRL = getFullyQualifiedFSRL(containerFSRL, monitor);
-
- synchronized (filesystemCache) {
- containerFSRL = intern(containerFSRL);
- FileSystemRef ref = filesystemCache.getFilesystemRefMountedAt(containerFSRL);
+ synchronized (fsInstanceManager) {
+ FileSystemRef ref = fsInstanceManager.getFilesystemRefMountedAt(containerFSRL);
if (ref != null) {
return ref;
}
- // Special case when the container is really a local filesystem directory.
- // Return a LocalFilesystem subfs instance.
- if (localFS.isLocalSubdir(containerFSRL)) {
- try {
- File localDir = new File(containerFSRL.getPath());
- GFileSystem fs = new LocalFileSystemSub(localDir, getLocalFS());
- ref = fs.getRefManager().create();
- filesystemCache.add(fs);
- return ref;
- }
- catch (IOException e) {
- Msg.error(this, "Problem when probing for local directory: ", e);
- }
- return null;
+ GFileSystem subdirFS = probeForLocalSubDirFilesystem(containerFSRL);
+ if (subdirFS != null) {
+ ref = subdirFS.getRefManager().create();
+ fsInstanceManager.add(subdirFS);
+ return ref;
}
}
// Normal case, probe the container file and create a filesystem instance.
// Do this outside of the sync lock so if any swing stuff happens in the conflictResolver
// it doesn't deadlock us.
- File containerFile = getFile(containerFSRL, monitor);
try {
- GFileSystem fs = fsFactoryMgr.probe(containerFSRL, containerFile, this,
- conflictResolver, priorityFilter, monitor);
+ ByteProvider byteProvider = getByteProvider(containerFSRL, true, monitor);
+ GFileSystem fs =
+ fsFactoryMgr.probe(byteProvider, this, conflictResolver, priorityFilter, monitor);
if (fs != null) {
- synchronized (filesystemCache) {
- FileSystemRef fsRef = filesystemCache.getFilesystemRefMountedAt(fs.getFSRL());
+ synchronized (fsInstanceManager) {
+ FileSystemRef fsRef = fsInstanceManager.getFilesystemRefMountedAt(fs.getFSRL());
if (fsRef != null) {
// race condition between sync block at top of this func and here.
// Throw away our new FS instance and use instance already in
@@ -682,7 +669,7 @@ public class FileSystemService {
return fsRef;
}
- filesystemCache.add(fs);
+ fsInstanceManager.add(fs);
return fs.getRefManager().create();
}
}
@@ -694,6 +681,18 @@ public class FileSystemService {
return null;
}
+ private GFileSystem probeForLocalSubDirFilesystem(FSRL containerFSRL) {
+ if (localFS.isLocalSubdir(containerFSRL)) {
+ try {
+ return localFS.getSubFileSystem(containerFSRL);
+ }
+ catch (IOException e) {
+ Msg.error(this, "Problem when probing for local directory: ", e);
+ }
+ }
+ return null;
+ }
+
/**
* Mount a specific file system (by class) using a specified container file.
*
@@ -713,16 +712,15 @@ public class FileSystemService {
public FSTYPE mountSpecificFileSystem(FSRL containerFSRL,
Class fsClass, TaskMonitor monitor) throws CancelledException, IOException {
- containerFSRL = getFullyQualifiedFSRL(containerFSRL, monitor);
- File containerFile = getFile(containerFSRL, monitor);
String fsType = fsFactoryMgr.getFileSystemType(fsClass);
if (fsType == null) {
Msg.error(this, "Specific file system implemention " + fsClass.getName() +
" not registered correctly in file system factory.");
return null;
}
+ ByteProvider byteProvider = getByteProvider(containerFSRL, true, monitor);
GFileSystem fs =
- fsFactoryMgr.mountFileSystem(fsType, containerFSRL, containerFile, this, monitor);
+ fsFactoryMgr.mountFileSystem(fsType, byteProvider, this, monitor);
Class producedClass = fs.getClass();
if (!fsClass.isAssignableFrom(fs.getClass())) {
fs.close();
@@ -750,24 +748,20 @@ public class FileSystemService {
public GFileSystem openFileSystemContainer(FSRL containerFSRL, TaskMonitor monitor)
throws CancelledException, IOException {
- if (localFS.isLocalSubdir(containerFSRL)) {
- File localDir = localFS.getLocalFile(containerFSRL);
- return new LocalFileSystemSub(localDir, localFS);
+ GFileSystem subdirFS = probeForLocalSubDirFilesystem(containerFSRL);
+ if (subdirFS != null) {
+ return subdirFS;
}
- File containerFile = getFile(containerFSRL, monitor);
- return fsFactoryMgr.probe(containerFSRL, containerFile, this, null,
- FileSystemInfo.PRIORITY_LOWEST, monitor);
+ ByteProvider byteProvider = getByteProvider(containerFSRL, true, monitor);
+ return fsFactoryMgr.probe(byteProvider, this, null, FileSystemInfo.PRIORITY_LOWEST,
+ monitor);
}
/**
* Returns a cloned copy of the {@code FSRL} that should have MD5 values specified.
* (excluding GFile objects that don't have data streams)
*
- * Also implements a best-effort caching of non-root filesystem FSRL's MD5 values.
- * (ie. the md5 values of files inside of containers are cached. The md5 value of
- * files on the real OS filesystem are not cached)
- *
* @param fsrl {@link FSRL} of the file that should be forced to have a MD5
* @param monitor {@link TaskMonitor} to watch and update with progress.
* @return possibly new {@link FSRL} instance with a MD5 value.
@@ -776,83 +770,63 @@ public class FileSystemService {
*/
public FSRL getFullyQualifiedFSRL(FSRL fsrl, TaskMonitor monitor)
throws CancelledException, IOException {
- if (fsrl == null) {
- return null;
+ if (fsrl == null || fsrl.getMD5() != null) {
+ return fsrl;
}
-
- FSRL fqParentContainer = getFullyQualifiedFSRL(fsrl.getFS().getContainer(), monitor);
-
- FSRL resultFSRL = (fqParentContainer != fsrl.getFS().getContainer())
- ? FSRLRoot.nestedFS(fqParentContainer, fsrl.getFS()).withPath(fsrl)
- : fsrl;
-
- if (resultFSRL.getMD5() == null) {
- String md5 = null;
- if (fqParentContainer != null) {
- md5 = fileCacheNameIndex.get(fqParentContainer.getMD5(), resultFSRL.getPath());
- }
- if (md5 == null) {
- try {
- md5 = getMD5(resultFSRL, monitor);
- if (fqParentContainer != null) {
- fileCacheNameIndex.add(fqParentContainer.getMD5(), resultFSRL.getPath(),
- md5);
- }
- }
- catch (IOException ioe) {
- // ignore, default to no MD5 value
- }
- }
- if (md5 != null) {
- resultFSRL = resultFSRL.withMD5(md5);
- }
+ try (FileSystemRef fsRef = getFilesystem(fsrl.getFS(), monitor)) {
+ return getFullyQualifiedFSRL(fsRef.getFilesystem(), fsrl, monitor);
}
-
- return resultFSRL;
}
- /**
- * Returns true if the specified file is on the local computer's
- * filesystem.
- *
- * @param gfile file to query
- * @return true if local, false if the path points to an embedded file in a container.
- */
- public boolean isLocal(GFile gfile) {
- return gfile.getFSRL().getFS().hasContainer() == false;
+ private void assertFullyQualifiedFSRL(FSRL fsrl) throws IOException {
+ if (fsrl.getMD5() == null) {
+ throw new IOException("Bad FSRL, expected fully qualified: " + fsrl);
+ }
}
- /**
- * Returns true if the specified location is a path on the local computer's
- * filesystem.
- *
- * @param fsrl {@link FSRL} path to query
- * @return true if local, false if the path points to an embedded file in a container.
- */
- public boolean isLocal(FSRL fsrl) {
- return fsrl.getFS().hasContainer() == false;
- }
-
- public String getFileHash(GFile gfile, TaskMonitor monitor)
+ private FSRL getFullyQualifiedFSRL(GFileSystem fs, FSRL fsrl, TaskMonitor monitor)
throws CancelledException, IOException {
- if (isLocal(gfile)) {
- File f = localFS.getLocalFile(gfile.getFSRL());
- if (f.isFile()) {
- return FSUtilities.getFileMD5(f, monitor);
- }
+ if (fsrl.getMD5() != null) {
+ return fsrl;
}
- else {
- try (InputStream dataStream = gfile.getFilesystem().getInputStream(gfile, monitor)) {
- if (dataStream == null) {
- throw new IOException("Unable to get datastream for " + gfile.getFSRL());
+ GFile file = fs.lookup(fsrl.getPath());
+ if (file == null) {
+ throw new IOException("File not found: " + fsrl);
+ }
+ if (file.getFSRL().getMD5() != null || file.isDirectory()) {
+ return file.getFSRL();
+ }
+
+ FSRL containerFSRL = fsrl.getFS().getContainer();
+ if (containerFSRL != null && containerFSRL.getMD5() == null) {
+ // re-home the fsrl to the parent container's fsrl since
+ // filesystems will always have fully qualified fsrl
+ containerFSRL = fs.getFSRL().getContainer();
+ fsrl = FSRLRoot.nestedFS(containerFSRL, fsrl.getFS()).withPath(fsrl);
+ }
+
+ if (fs instanceof GFileHashProvider) {
+ GFileHashProvider hashProvider = (GFileHashProvider) fs;
+ return fsrl.withMD5(hashProvider.getMD5Hash(file, true, monitor));
+ }
+
+ String md5 = (containerFSRL != null)
+ ? fileCacheNameIndex.get(containerFSRL.getMD5(), fsrl.getPath())
+ : null;
+ if (md5 == null) {
+ try (ByteProvider bp = fs.getByteProvider(file, monitor)) {
+ if (bp == null) {
+ throw new IOException("Unable to get bytes for " + fsrl);
}
- monitor.setMessage("Caching " + gfile.getName());
- monitor.initialize(gfile.getLength());
- FileCacheEntry cfi = fileCache.addStream(dataStream, monitor);
- return cfi.md5;
+ md5 = (bp.getFSRL().getMD5() != null)
+ ? bp.getFSRL().getMD5()
+ : FSUtilities.getMD5(bp, monitor);
}
}
- return null;
+ if (containerFSRL != null && fs.isStatic()) {
+ fileCacheNameIndex.add(containerFSRL.getMD5(), fsrl.getPath(), md5);
+ }
+ return fsrl.withMD5(md5);
}
/**
@@ -875,8 +849,8 @@ public class FileSystemService {
* @return {@link List} of {@link FSRLRoot} of currently mounted filesystems.
*/
public List getMountedFilesystems() {
- synchronized (filesystemCache) {
- return filesystemCache.getMountedFilesystems();
+ synchronized (fsInstanceManager) {
+ return fsInstanceManager.getMountedFilesystems();
}
}
@@ -891,58 +865,30 @@ public class FileSystemService {
* @return new {@link FileSystemRef} or null if requested file system not mounted.
*/
public FileSystemRef getMountedFilesystem(FSRLRoot fsFSRL) {
- synchronized (filesystemCache) {
- return filesystemCache.getRef(fsFSRL);
+ synchronized (fsInstanceManager) {
+ return fsInstanceManager.getRef(fsFSRL);
}
-
}
/**
- * Interns the FSRLRoot so that its parent parts are shared with already interned instances.
+ * Returns a new {@link CryptoSession} that the caller can use to query for
+ * passwords and such. Caller is responsible for closing the instance when done.
*
- * Caller needs to hold sync mutex
- *
- * @param fsrl {@link FSRLRoot} to intern-alize.
- * @return possibly different {@link FSRLRoot} instance that has shared parent references
- * instead of unique bespoke instances.
+ * Later callers to this method will receive a nested CryptoSession that shares it's
+ * state with the initial CryptoSession, until the initial CryptoSession is closed().
+ *
+ * @return new {@link CryptoSession} instance, never null
*/
- private FSRLRoot intern(FSRLRoot fsrl) {
- if (localFSRL.equals(fsrl)) {
- return localFSRL;
+ public synchronized CryptoSession newCryptoSession() {
+ if (currentCryptoSession == null || currentCryptoSession.isClosed()) {
+ // If no this no current open cryptosession, return a new full/independent
+ // cryptosession, and use it as the parent for any subsequent sessions
+ currentCryptoSession = CryptoProviders.getInstance().newSession();
+ return currentCryptoSession;
}
- FSRL container = fsrl.getContainer();
- if (container != null) {
- FSRLRoot parentFSRL = intern(container.getFS());
- if (parentFSRL != container.getFS()) {
- FSRL internedContainer = parentFSRL.withPath(container);
- fsrl = FSRLRoot.nestedFS(internedContainer, fsrl);
- }
- }
- FSRLRoot existing = fsrlInternMap.get(fsrl);
- if (existing == null) {
- fsrlInternMap.put(fsrl, fsrl);
- existing = fsrl;
- }
-
- return existing;
- }
-
- /**
- * Interns the FSRL so that its parent parts are shared with already interned instances.
- *
- * Caller needs to hold sync mutex.
- *
- * Only {@link FSRLRoot} instances are cached in the intern map, {@link FSRL} instances
- * are not.
- *
- * @param fsrl {@link FSRL} to intern-alize.
- * @return possibly different {@link FSRL} instance that has shared parent references
- * instead of unique bespoke instances.
- */
- private FSRL intern(FSRL fsrl) {
- FSRLRoot internedRoot = intern(fsrl.getFS());
- return internedRoot == fsrl.getFS() ? fsrl : internedRoot.withPath(fsrl);
+ // return a nested / dependent cryptosession
+ return new CryptoProviderSessionChildImpl(currentCryptoSession);
}
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileHashProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileHashProvider.java
new file mode 100644
index 0000000000..dd139a0b23
--- /dev/null
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileHashProvider.java
@@ -0,0 +1,40 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.formats.gfilesystem;
+
+import java.io.IOException;
+
+import ghidra.util.exception.CancelledException;
+import ghidra.util.task.TaskMonitor;
+
+/**
+ * GFileSystem add-on interface that provides MD5 hashing for file located within the filesystem
+ */
+public interface GFileHashProvider {
+ /**
+ * Returns the MD5 hash of the specified file.
+ *
+ * @param file the {@link GFile}
+ * @param required boolean flag, if true the hash will always be returned, even if it has to
+ * be calculated. If false, the hash will be returned if easily available
+ * @param monitor {@link TaskMonitor}
+ * @return MD5 hash as a string
+ * @throws CancelledException if cancelled
+ * @throws IOException if error
+ */
+ String getMD5Hash(GFile file, boolean required, TaskMonitor monitor)
+ throws CancelledException, IOException;
+}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileImpl.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileImpl.java
index 09b43fa5bc..b369796fa9 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileImpl.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileImpl.java
@@ -15,7 +15,7 @@
*/
package ghidra.formats.gfilesystem;
-import ghidra.util.SystemUtilities;
+import java.util.Objects;
/**
* Base implementation of file in a {@link GFileSystem filesystem}.
@@ -140,11 +140,11 @@ public class GFileImpl implements GFile {
return new GFileImpl(fileSystem, parent, isDirectory, length, fsrl);
}
- private GFileSystem fileSystem;
- private GFile parentFile;
- private boolean isDirectory = false;
- private long length = -1;
- private FSRL fsrl;
+ private final GFileSystem fileSystem;
+ private final GFile parentFile;
+ private final boolean isDirectory;
+ private long length;
+ private final FSRL fsrl;
/**
* Protected constructor, use static helper methods to create new instances.
@@ -197,21 +197,6 @@ public class GFileImpl implements GFile {
return getPath();
}
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof GFile)) {
- return false;
- }
-
- GFile other = (GFile) obj;
- return SystemUtilities.isEqual(fsrl, other.getFSRL()) && isDirectory == other.isDirectory();
- }
-
- @Override
- public int hashCode() {
- return fsrl.hashCode() ^ Boolean.hashCode(isDirectory());
- }
-
@Override
public String getPath() {
return fsrl.getPath();
@@ -226,7 +211,22 @@ public class GFileImpl implements GFile {
return fsrl;
}
- public void setFSRL(FSRL fsrl) {
- this.fsrl = fsrl;
+ @Override
+ public int hashCode() {
+ return Objects.hash(fileSystem, fsrl.getPath(), isDirectory);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof GFile)) {
+ return false;
+ }
+ GFile other = (GFile) obj;
+ return Objects.equals(fileSystem, other.getFilesystem()) &&
+ Objects.equals(fsrl.getPath(), other.getFSRL().getPath()) &&
+ isDirectory == other.isDirectory();
}
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java
index ed7143acb5..975367dab3 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java
@@ -15,12 +15,14 @@
*/
package ghidra.formats.gfilesystem;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
import java.util.List;
+import ghidra.app.util.bin.ByteProvider;
+import ghidra.app.util.bin.ByteProviderInputStream;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
+import ghidra.formats.gfilesystem.fileinfo.FileAttribute;
+import ghidra.formats.gfilesystem.fileinfo.FileAttributes;
import ghidra.util.classfinder.ExtensionPoint;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@@ -30,8 +32,9 @@ import ghidra.util.task.TaskMonitor;
*
* Operations take a {@link TaskMonitor} if they need to be cancel-able.
*
- * Use {@link FileSystemService} to discover and open instances of filesystems in files or
- * to open a known {@link FSRL} path.
+ * Use a {@link FileSystemService FileSystemService instance} to discover and
+ * open instances of filesystems in files or to open a known {@link FSRL} path or to
+ * deal with creating {@link FileSystemService#createTempFile(long) temp files}.
*
* NOTE:
* ALL GFileSystem sub-CLASSES MUST END IN "FileSystem". If not, the ClassSearcher
@@ -137,7 +140,24 @@ public interface GFileSystem extends Closeable, ExtensionPoint {
* @throws IOException if IO problem
* @throws CancelledException if user cancels.
*/
- public InputStream getInputStream(GFile file, TaskMonitor monitor)
+ default public InputStream getInputStream(GFile file, TaskMonitor monitor)
+ throws IOException, CancelledException {
+ return getInputStreamHelper(file, this, monitor);
+ }
+
+ /**
+ * Returns a {@link ByteProvider} that contains the contents of the specified {@link GFile}.
+ *
+ * The caller is responsible for closing the provider.
+ *
+ * @param file {@link GFile} to get bytes for
+ * @param monitor {@link TaskMonitor} to watch and update progress
+ * @return new {@link ByteProvider} that contains the contents of the file, or NULL if file
+ * doesn't have data
+ * @throws IOException if error
+ * @throws CancelledException if user cancels
+ */
+ public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException;
/**
@@ -151,17 +171,36 @@ public interface GFileSystem extends Closeable, ExtensionPoint {
public List getListing(GFile directory) throws IOException;
/**
- * Returns a multi-line string with information about the specified {@link GFile file}.
+ * Returns a container of {@link FileAttribute} values.
*
- * TODO:{@literal this method needs to be refactored to return a Map; instead of}
- * a pre-formatted multi-line string.
- *
- * @param file {@link GFile} to get info message for.
- * @param monitor {@link TaskMonitor} to watch and update progress.
- * @return multi-line formatted string with info about the file, or null.
+ * Implementors of this method are not required to add FSRL, NAME, or PATH values unless
+ * the values are non-standard.
+ *
+ * @param file {@link GFile} to get the attributes for
+ * @param monitor {@link TaskMonitor}
+ * @return {@link FileAttributes} instance (possibly read-only), maybe empty but never null
*/
- default public String getInfo(GFile file, TaskMonitor monitor) {
- return null;
+ default public FileAttributes getFileAttributes(GFile file, TaskMonitor monitor) {
+ return FileAttributes.EMPTY;
+ }
+
+ /**
+ * Default implementation of getting an {@link InputStream} from a {@link GFile}'s
+ * {@link ByteProvider}.
+ *
+ *
+ * @param file {@link GFile}
+ * @param fs the {@link GFileSystem filesystem} containing the file
+ * @param monitor {@link TaskMonitor} to allow canceling
+ * @return new {@link InputStream} containing bytes of the file
+ * @throws CancelledException if canceled
+ * @throws IOException if error
+ */
+ public static InputStream getInputStreamHelper(GFile file, GFileSystem fs, TaskMonitor monitor)
+ throws CancelledException, IOException {
+ ByteProvider bp = fs.getByteProvider(file, monitor);
+ return (bp != null) ? new ByteProviderInputStream.ClosingInputStream(bp) : null;
+
}
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemBase.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemBase.java
index 21ab576ccb..d860246646 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemBase.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemBase.java
@@ -129,26 +129,6 @@ public abstract class GFileSystemBase implements GFileSystem {
@Override
abstract public List getListing(GFile directory) throws IOException;
- /**
- * Legacy implementation of {@link #getInputStream(GFile, TaskMonitor)}.
- *
- * @param file {@link GFile} to get an InputStream for
- * @param monitor {@link TaskMonitor} to watch and update progress
- * @return new {@link InputStream} contains the contents of the file or NULL if the
- * file doesn't have data.
- * @throws IOException if IO problem
- * @throws CancelledException if user cancels.
- * @throws CryptoException if crypto problem.
- */
- abstract protected InputStream getData(GFile file, TaskMonitor monitor)
- throws IOException, CancelledException, CryptoException;
-
- @Override
- public InputStream getInputStream(GFile file, TaskMonitor monitor)
- throws CancelledException, IOException {
- return getData(file, monitor);
- }
-
/**
* Writes the given bytes to a tempfile in the temp directory.
* @param bytes the bytes to write
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GIconProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GIconProvider.java
index 468e42962c..200547b953 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GIconProvider.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GIconProvider.java
@@ -15,18 +15,18 @@
*/
package ghidra.formats.gfilesystem;
-import ghidra.util.Msg;
-import ghidra.util.exception.CancelledException;
-import ghidra.util.task.TaskMonitor;
-
import java.awt.Image;
-import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import javax.imageio.ImageIO;
import javax.swing.Icon;
import javax.swing.ImageIcon;
+import ghidra.util.Msg;
+import ghidra.util.exception.CancelledException;
+import ghidra.util.task.TaskMonitor;
+
/**
* {@link GFileSystem} add-on interface to allow filesystems to override how image files
* are converted into viewable {@link Icon} instances.
@@ -61,9 +61,8 @@ public interface GIconProvider {
return ((GIconProvider) fs).getIcon(file, monitor);
}
- File data = FileSystemService.getInstance().getFile(file.getFSRL(), monitor);
- try {
- Image image = ImageIO.read(data);
+ try (InputStream is = file.getFilesystem().getInputStream(file, monitor)) {
+ Image image = ImageIO.read(is);
if (image == null) {
return null;
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystem.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystem.java
index 01b6ad37c0..313f407ec7 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystem.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystem.java
@@ -15,15 +15,22 @@
*/
package ghidra.formats.gfilesystem;
+import static ghidra.formats.gfilesystem.fileinfo.FileAttributeType.*;
+
import java.io.*;
-import java.nio.file.Files;
+import java.nio.file.*;
import java.util.*;
+import org.apache.commons.collections4.map.ReferenceMap;
import org.apache.commons.io.FilenameUtils;
+import ghidra.app.util.bin.ByteProvider;
+import ghidra.app.util.bin.FileByteProvider;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.factory.GFileSystemFactory;
import ghidra.formats.gfilesystem.factory.GFileSystemFactoryIgnore;
+import ghidra.formats.gfilesystem.fileinfo.*;
+import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
@@ -36,7 +43,7 @@ import ghidra.util.task.TaskMonitor;
* Closing() this filesystem does nothing.
*/
@FileSystemInfo(type = LocalFileSystem.FSTYPE, description = "Local filesystem", factory = GFileSystemFactoryIgnore.class)
-public class LocalFileSystem implements GFileSystem {
+public class LocalFileSystem implements GFileSystem, GFileHashProvider {
public static final String FSTYPE = "file";
/**
@@ -48,9 +55,11 @@ public class LocalFileSystem implements GFileSystem {
return new LocalFileSystem(FSRLRoot.makeRoot(FSTYPE));
}
- private final List emptyDir = Collections.emptyList();
+ private final List emptyDir = List.of();
private final FSRLRoot fsFSRL;
private final FileSystemRefManager refManager = new FileSystemRefManager(this);
+ private final ReferenceMap fileFingerprintToMD5Map =
+ new ReferenceMap<>();
private LocalFileSystem(FSRLRoot fsrl) {
this.fsFSRL = fsrl;
@@ -60,6 +69,21 @@ public class LocalFileSystem implements GFileSystem {
return fsFSRL.equals(fsrl.getFS());
}
+ /**
+ * Creates a new file system instance that is a sub-view limited to the specified directory.
+ *
+ * @param fsrl {@link FSRL} that must be a directory in this local filesystem
+ * @return new {@link LocalFileSystemSub} instance
+ * @throws IOException if bad FSRL
+ */
+ public LocalFileSystemSub getSubFileSystem(FSRL fsrl) throws IOException {
+ if (isLocalSubdir(fsrl)) {
+ File localDir = getLocalFile(fsrl);
+ return new LocalFileSystemSub(localDir, this);
+ }
+ return null;
+ }
+
/**
* Returns true if the {@link FSRL} is a local filesystem subdirectory.
*
@@ -74,6 +98,13 @@ public class LocalFileSystem implements GFileSystem {
return localFile.isDirectory();
}
+ /**
+ * Convert a FSRL that points to this file system into a java {@link File}.
+ *
+ * @param fsrl {@link FSRL}
+ * @return {@link File}
+ * @throws IOException if FSRL does not point to this file system
+ */
public File getLocalFile(FSRL fsrl) throws IOException {
if (!isSameFS(fsrl)) {
throw new IOException("FSRL does not specify local file: " + fsrl);
@@ -82,6 +113,17 @@ public class LocalFileSystem implements GFileSystem {
return localFile;
}
+ /**
+ * Converts a {@link File} into a {@link FSRL}.
+ *
+ * @param f {@link File}
+ * @return {@link FSRL}
+ */
+ public FSRL getLocalFSRL(File f) {
+ return fsFSRL
+ .withPath(FSUtilities.appendPath("/", FilenameUtils.separatorsToUnix(f.getPath())));
+ }
+
@Override
public String getName() {
return "Root Filesystem";
@@ -130,14 +172,48 @@ public class LocalFileSystem implements GFileSystem {
}
@Override
- public String getInfo(GFile file, TaskMonitor monitor) {
- File localFile = new File(file.getPath());
+ public FileAttributes getFileAttributes(GFile file, TaskMonitor monitor) {
+ File f = new File(file.getPath());
+ return getFileAttributes(f);
+ }
- StringBuilder buffer = new StringBuilder();
- buffer.append("Name: " + localFile.getName() + "\n");
- buffer.append("Size: " + localFile.length() + "\n");
- buffer.append("Date: " + new Date(localFile.lastModified()).toString() + "\n");
- return buffer.toString();
+ /**
+ * Create a {@link FileAttributes} container with info about the specified local file.
+ *
+ * @param f {@link File} to query
+ * @return {@link FileAttributes} instance
+ */
+ public FileAttributes getFileAttributes(File f) {
+ Path p = f.toPath();
+ FileType fileType = fileToFileType(p);
+ Path symLinkDest = null;
+ try {
+ symLinkDest = fileType == FileType.SYMBOLIC_LINK ? Files.readSymbolicLink(p) : null;
+ }
+ catch (IOException e) {
+ // ignore and continue with symLinkDest == null
+ }
+ return FileAttributes.of(
+ FileAttribute.create(NAME_ATTR, f.getName()),
+ FileAttribute.create(FILE_TYPE_ATTR, fileType),
+ FileAttribute.create(SIZE_ATTR, f.length()),
+ FileAttribute.create(MODIFIED_DATE_ATTR, new Date(f.lastModified())),
+ symLinkDest != null
+ ? FileAttribute.create(SYMLINK_DEST_ATTR, symLinkDest.toString())
+ : null);
+ }
+
+ private static FileType fileToFileType(Path p) {
+ if (Files.isSymbolicLink(p)) {
+ return FileType.SYMBOLIC_LINK;
+ }
+ if (Files.isDirectory(p)) {
+ return FileType.DIRECTORY;
+ }
+ if (Files.isRegularFile(p)) {
+ return FileType.FILE;
+ }
+ return FileType.UNKNOWN;
}
@Override
@@ -153,12 +229,6 @@ public class LocalFileSystem implements GFileSystem {
return gf;
}
- @Override
- public InputStream getInputStream(GFile file, TaskMonitor monitor) throws IOException {
- File f = new File(file.getPath());
- return new FileInputStream(f);
- }
-
@Override
public boolean isClosed() {
return false;
@@ -169,8 +239,103 @@ public class LocalFileSystem implements GFileSystem {
return refManager;
}
+ @Override
+ public InputStream getInputStream(GFile file, TaskMonitor monitor) throws IOException {
+ return getInputStream(file.getFSRL(), monitor);
+ }
+
+ InputStream getInputStream(FSRL fsrl, TaskMonitor monitor) throws IOException {
+ return new FileInputStream(getLocalFile(fsrl));
+ }
+
+ @Override
+ public ByteProvider getByteProvider(GFile file, TaskMonitor monitor) throws IOException {
+ return getByteProvider(file.getFSRL(), monitor);
+ }
+
+ ByteProvider getByteProvider(FSRL fsrl, TaskMonitor monitor) throws IOException {
+ File f = getLocalFile(fsrl);
+ return new FileByteProvider(f, fsrl, AccessMode.READ);
+ }
+
@Override
public String toString() {
return "Local file system " + fsFSRL;
}
+
+ @Override
+ public String getMD5Hash(GFile file, boolean required, TaskMonitor monitor)
+ throws CancelledException, IOException {
+ return getMD5Hash(file.getFSRL(), required, monitor);
+ }
+
+ synchronized String getMD5Hash(FSRL fsrl, boolean required, TaskMonitor monitor)
+ throws CancelledException, IOException {
+ File f = getLocalFile(fsrl);
+ if ( !f.isFile() ) {
+ return null;
+ }
+
+ FileFingerprintRec fileFingerprintRec = new FileFingerprintRec(f.getPath(), f.lastModified(), f.length());
+ String md5 = fileFingerprintToMD5Map.get(fileFingerprintRec);
+ if (md5 == null && required) {
+ md5 = FSUtilities.getFileMD5(f, monitor);
+ fileFingerprintToMD5Map.put(fileFingerprintRec, md5);
+ }
+
+ return md5;
+ }
+
+ //-----------------------------------------------------------------------------------
+
+ private static class FileFingerprintRec {
+ final String path;
+ final long timestamp;
+ final long length;
+
+ FileFingerprintRec(String path, long timestamp, long length) {
+ this.path = path;
+ this.timestamp = timestamp;
+ this.length = length;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (int) (length ^ (length >>> 32));
+ result = prime * result + ((path == null) ? 0 : path.hashCode());
+ result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof FileFingerprintRec)) {
+ return false;
+ }
+ FileFingerprintRec other = (FileFingerprintRec) obj;
+ if (length != other.length) {
+ return false;
+ }
+ if (path == null) {
+ if (other.path != null) {
+ return false;
+ }
+ }
+ else if (!path.equals(other.path)) {
+ return false;
+ }
+ if (timestamp != other.timestamp) {
+ return false;
+ }
+ return true;
+ }
+ }
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystemSub.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystemSub.java
index 94701c4a91..41a5505984 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystemSub.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystemSub.java
@@ -17,10 +17,13 @@ package ghidra.formats.gfilesystem;
import java.io.*;
import java.nio.file.Files;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.lang3.StringUtils;
+import ghidra.app.util.bin.ByteProvider;
+import ghidra.formats.gfilesystem.fileinfo.FileAttributes;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@@ -35,24 +38,20 @@ import ghidra.util.task.TaskMonitor;
* by the FileSystemFactoryMgr.
*
*/
-public class LocalFileSystemSub implements GFileSystem {
+public class LocalFileSystemSub implements GFileSystem, GFileHashProvider {
private final FSRLRoot fsFSRL;
- private final GFileSystem rootFS;
- private final List emptyDir = Collections.emptyList();
+ private final LocalFileSystem rootFS;
private File localfsRootDir;
private FileSystemRefManager refManager = new FileSystemRefManager(this);
private GFileLocal rootGFile;
- public LocalFileSystemSub(File rootDir, GFileSystem rootFS) throws IOException {
+ public LocalFileSystemSub(File rootDir, LocalFileSystem rootFS) throws IOException {
this.rootFS = rootFS;
this.localfsRootDir = rootDir.getCanonicalFile();
- GFile containerDir = rootFS.lookup(localfsRootDir.getPath());
- if (containerDir == null) {
- throw new IOException("Bad root dir: " + rootDir);
- }
- this.fsFSRL = FSRLRoot.nestedFS(containerDir.getFSRL(), rootFS.getFSRL().getProtocol());
- this.rootGFile = new GFileLocal(localfsRootDir, "/", containerDir.getFSRL(), this, null);
+ FSRL containerFSRL = rootFS.getLocalFSRL(localfsRootDir);
+ this.fsFSRL = FSRLRoot.nestedFS(containerFSRL, rootFS.getFSRL().getProtocol());
+ this.rootGFile = new GFileLocal(localfsRootDir, "/", containerFSRL, this, null);
}
@Override
@@ -97,17 +96,17 @@ public class LocalFileSystemSub implements GFileSystem {
directory = rootGFile;
}
if (!directory.isDirectory()) {
- return emptyDir;
+ return List.of();
}
File localDir = getFileFromGFile(directory);
if (Files.isSymbolicLink(localDir.toPath())) {
- return emptyDir;
+ return List.of();
}
File[] localFiles = localDir.listFiles();
if (localFiles == null) {
- return emptyDir;
+ return List.of();
}
List tmp = new ArrayList<>(localFiles.length);
@@ -130,19 +129,15 @@ public class LocalFileSystemSub implements GFileSystem {
}
@Override
- public String getInfo(GFile file, TaskMonitor monitor) {
+ public FileAttributes getFileAttributes(GFile file, TaskMonitor monitor) {
try {
File localFile = getFileFromGFile(file);
- StringBuilder buffer = new StringBuilder();
- buffer.append("Name: " + localFile.getName() + "\n");
- buffer.append("Size: " + localFile.length() + "\n");
- buffer.append("Date: " + new Date(localFile.lastModified()).toString() + "\n");
- return buffer.toString();
+ return rootFS.getFileAttributes(localFile);
}
catch (IOException e) {
// fail and return null
}
- return null;
+ return FileAttributes.EMPTY;
}
@Override
@@ -179,8 +174,7 @@ public class LocalFileSystemSub implements GFileSystem {
@Override
public InputStream getInputStream(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
-
- return new FileInputStream(getFileFromGFile(file));
+ return rootFS.getInputStream(file.getFSRL(), monitor);
}
@Override
@@ -192,4 +186,16 @@ public class LocalFileSystemSub implements GFileSystem {
public String toString() {
return getName();
}
+
+ @Override
+ public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
+ throws IOException, CancelledException {
+ return rootFS.getByteProvider(file.getFSRL(), monitor);
+ }
+
+ @Override
+ public String getMD5Hash(GFile file, boolean required, TaskMonitor monitor)
+ throws CancelledException, IOException {
+ return rootFS.getMD5Hash(file.getFSRL(), required, monitor);
+ }
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdByteProvider.java
new file mode 100644
index 0000000000..bb481b2cc8
--- /dev/null
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdByteProvider.java
@@ -0,0 +1,98 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.formats.gfilesystem;
+
+import java.io.File;
+import java.io.IOException;
+
+import ghidra.app.util.bin.ByteProvider;
+
+/**
+ * A {@link ByteProvider} along with a {@link FileSystemRef} to keep the filesystem pinned
+ * in memory.
+ *
+ * The caller is responsible for {@link #close() closing} this object, which releases
+ * the FilesystemRef.
+ */
+public class RefdByteProvider implements ByteProvider {
+ private final FileSystemRef fsRef;
+ private final ByteProvider provider;
+ private final FSRL fsrl;
+
+ /**
+ * Creates a RefdByteProvider instance, taking ownership of the supplied FileSystemRef.
+ *
+ * @param fsRef {@link FileSystemRef} that contains the specified ByteProvider
+ * @param provider {@link ByteProvider} inside the filesystem held open by the ref
+ * @param fsrl {@link FSRL} identity of this new ByteProvider
+ */
+ public RefdByteProvider(FileSystemRef fsRef, ByteProvider provider, FSRL fsrl) {
+ this.fsRef = fsRef;
+ this.provider = provider;
+ this.fsrl = fsrl;
+ }
+
+ @Override
+ public void close() throws IOException {
+ provider.close();
+ fsRef.close();
+ }
+
+ @Override
+ public FSRL getFSRL() {
+ return fsrl;
+ }
+
+ @Override
+ public File getFile() {
+ return provider.getFile();
+ }
+
+ @Override
+ public String getName() {
+ return fsrl != null ? fsrl.getName() : provider.getName();
+ }
+
+ @Override
+ public String getAbsolutePath() {
+ return fsrl != null ? fsrl.getPath() : provider.getAbsolutePath();
+ }
+
+ @Override
+ public long length() throws IOException {
+ return provider.length();
+ }
+
+ @Override
+ public boolean isValidIndex(long index) {
+ return provider.isValidIndex(index);
+ }
+
+ @Override
+ public byte readByte(long index) throws IOException {
+ return provider.readByte(index);
+ }
+
+ @Override
+ public byte[] readBytes(long index, long length) throws IOException {
+ return provider.readBytes(index, length);
+ }
+
+ @Override
+ public String toString() {
+ return "ByteProvider " + provider.getFSRL() + " in file system " + fsRef.getFilesystem();
+ }
+}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdFile.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdFile.java
index 6ce850dac5..112a4951d8 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdFile.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdFile.java
@@ -29,6 +29,12 @@ public class RefdFile implements Closeable {
public final FileSystemRef fsRef;
public final GFile file;
+ /**
+ * Creates a RefdFile instance, taking ownership of the supplied fsRef.
+ *
+ * @param fsRef {@link FileSystemRef} that pins the filesystem open
+ * @param file GFile file inside the specified filesystem
+ */
public RefdFile(FileSystemRef fsRef, GFile file) {
this.fsRef = fsRef;
this.file = file;
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/SelectFromListDialog.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/SelectFromListDialog.java
index 60869e5594..b1236c7a6f 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/SelectFromListDialog.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/SelectFromListDialog.java
@@ -15,8 +15,6 @@
*/
package ghidra.formats.gfilesystem;
-import ghidra.util.SystemUtilities;
-
import java.awt.BorderLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
@@ -29,6 +27,7 @@ import docking.DialogComponentProvider;
import docking.DockingWindowManager;
import docking.widgets.MultiLineLabel;
import docking.widgets.list.ListPanel;
+import ghidra.util.SystemUtilities;
/**
* Dialog that presents the user with a list of strings and returns the object
@@ -96,17 +95,17 @@ public class SelectFromListDialog extends DialogComponentProvider {
private void doSelect() {
selectedObject = null;
actionComplete = false;
- DockingWindowManager activeInstance = DockingWindowManager.getActiveInstance();
- activeInstance.showDialog(this);
+ DockingWindowManager.showDialog(this);
if (actionComplete) {
selectedObject = list.get(listPanel.getSelectedIndex());
}
}
private JPanel buildWorkPanel(String prompt) {
- DefaultListModel