development/ghidra
development/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra synced 2024-07-02 16:00:15 +00:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch

Browse Source 
'origin/GP-253_dev747368_refactor_gfilesystem_byteproviders_passwords_and_android--SQUASHED'
(Closes #377)
This commit is contained in:
ghidra1 2021-10-01 11:49:44 -04:00
commit 54bbbcf44b
156 changed files with 7948 additions and 4714 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Ghidra/Features/Base/data/file_extension_icons.xml
View File

@ -38,6 +38,7 @@
<file_overlay name="imported" icon="images/checkmark_green.gif" />
<file_overlay name="filesystem" icon="images/nuvola/16x16/ledgreen.png" quadrant="LL" />
<file_overlay name="password_missing" icon="images/lock.png" quadrant="UR" />
</file_extensions>

27
Ghidra/Features/Base/src/main/help/help/topics/FileSystemBrowserPlugin/FileSystemBrowserPlugin.html
View File

@ -123,6 +123,19 @@
of the file systems will display that file system's browser tree.</P>
</BLOCKQUOTE>
<H3><A name="FSB_Clear_Cached_Passwords"></A>Clear Cached Passwords</H3>
<BLOCKQUOTE>
<P>Clears any cached passwords that previously entered when accessing a password
protected container file.</P>
</BLOCKQUOTE>
<H3><A name="FSB_Refresh"></A>Refresh</H3>
<BLOCKQUOTE>
<P>Refreshes the status of the selected items.</P>
</BLOCKQUOTE>
<H3><A name="FSB_Close"></A>Close</H3>
<BLOCKQUOTE>
@ -150,6 +163,20 @@
</BLOCKQUOTE>
</BLOCKQUOTE>
<H2><A name="PasswordDialog"></A>Password Dialog</H2>
<BLOCKQUOTE>
<P>This dialog is used to prompt for a password when opening a password protected container.</P>
<P>If the password isn't known, the <b>Cancel</b> button will stop Ghidra from re-prompting
for the password for the current file during the current operation (but the user may be
re-prompted again later depending on the logic of the operation), and <b>Cancel All</b>
will stop Ghidra from prompting for passwords for any file during the current operation.</P>
<P>Passwords can also be provided via a text file specified on the java command line, useful
for headless operations. See <b>ghidra.formats.gfilesystem.crypto.CmdLinePasswordProvider</b></P>
</BLOCKQUOTE>
<H2>How To Handle Unsupported File Systems</H2>
<BLOCKQUOTE>

16
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteArrayProvider.java
View File

@ -20,7 +20,7 @@ import java.io.*;
import ghidra.formats.gfilesystem.FSRL;
/**
* An implementation of {@link ByteProvider} where the underlying bytes are supplied by a static
* An implementation of {@link ByteProvider} where the underlying bytes are supplied by a
* byte array.
* <p>
* NOTE: Use of this class is discouraged when the byte array could be large.
@ -66,6 +66,16 @@ public class ByteArrayProvider implements ByteProvider {
// don't do anything for now
}
/**
* Releases the byte storage of this instance.
* <p>
* This is separate from the normal close() to avoid changing existing
* behavior of this class.
*/
public void hardClose() {
srcBytes = new byte[0];
}
@Override
public FSRL getFSRL() {
return fsrl;
@ -78,12 +88,12 @@ public class ByteArrayProvider implements ByteProvider {
@Override
public String getName() {
return name;
return fsrl != null ? fsrl.getName() : name;
}
@Override
public String getAbsolutePath() {
return "";
return fsrl != null ? fsrl.getPath() : "";
}
@Override

15
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProvider.java
View File

@ -25,6 +25,11 @@ import ghidra.formats.gfilesystem.FileSystemService;
*/
public interface ByteProvider extends Closeable {
/**
* A static re-usable empty {@link ByteProvider} instance.
*/
public static final ByteProvider EMPTY_BYTEPROVIDER = new EmptyByteProvider();
/**
* Returns the {@link FSRL} of the underlying file for this byte provider,
* or null if this byte provider is not associated with a file.
@ -109,9 +114,17 @@ public interface ByteProvider extends Closeable {
* <p>
* The caller is responsible for closing the returned {@link InputStream} instance.
* <p>
* If you need to override this default implementation, please document why your inputstream
* is needed.
*
* @param index where in the {@link ByteProvider} to start the {@link InputStream}
* @return the {@link InputStream}
* @throws IOException if an I/O error occurs
*/
public InputStream getInputStream(long index) throws IOException;
default public InputStream getInputStream(long index) throws IOException {
if (index < 0 || index > length()) {
throw new IOException("Invalid start position: " + index);
}
return new ByteProviderInputStream(this, index);
}
}

119
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderInputStream.java
View File

@ -18,25 +18,122 @@ package ghidra.app.util.bin;
import java.io.IOException;
import java.io.InputStream;
/**
* An {@link InputStream} that reads from a {@link ByteProvider}.
* <p>
* Does not close the underlying ByteProvider when closed itself.
*
*/
public class ByteProviderInputStream extends InputStream {
private ByteProvider provider;
private long offset;
private long length;
private long nextOffset;
public ByteProviderInputStream( ByteProvider provider, long offset, long length ) {
/**
* An {@link InputStream} that reads from a {@link ByteProvider}, and <b>DOES</b>
* {@link ByteProvider#close() close()} the underlying ByteProvider when
* closed itself.
* <p>
*/
public static class ClosingInputStream extends ByteProviderInputStream {
/**
* Creates an {@link InputStream} that reads from a {@link ByteProvider}, that
* <b>DOES</b> {@link ByteProvider#close() close()} the underlying ByteProvider when
* closed itself.
* <p>
* @param provider the {@link ByteProvider} to read from (and close)
*/
public ClosingInputStream(ByteProvider provider) {
super(provider);
}
@Override
public void close() throws IOException {
super.close();
if (provider != null) {
provider.close();
provider = null;
}
}
}
protected ByteProvider provider;
private long currentPosition;
private long markPosition;
/**
* Creates an InputStream that uses a ByteProvider as its source of bytes.
*
* @param provider the {@link ByteProvider} to wrap
*/
public ByteProviderInputStream(ByteProvider provider) {
this(provider, 0);
}
/**
* Creates an InputStream that uses a ByteProvider as its source of bytes.
*
* @param provider the {@link ByteProvider} to wrap
* @param startPosition starting position in the provider
*/
public ByteProviderInputStream(ByteProvider provider, long startPosition) {
this.provider = provider;
this.offset = offset;
this.length = length;
this.nextOffset = offset;
this.markPosition = startPosition;
this.currentPosition = startPosition;
}
@Override
public void close() throws IOException {
// nothing to do here
}
@Override
public int available() throws IOException {
return (int) Math.min(provider.length() - currentPosition, Integer.MAX_VALUE);
}
@Override
public boolean markSupported() {
return true;
}
@Override
public synchronized void mark(int readlimit) {
this.markPosition = currentPosition;
}
@Override
public synchronized void reset() throws IOException {
// synchronized because the base class's method is synchronized.
this.currentPosition = markPosition;
}
@Override
public long skip(long n) throws IOException {
if (n <= 0) {
return 0;
}
long newPosition = Math.min(provider.length(), currentPosition + n);
long skipped = newPosition - currentPosition;
currentPosition = newPosition;
return skipped;
}
@Override
public int read() throws IOException {
if ( nextOffset < offset + length ) {
return provider.readByte( nextOffset++ ) & 0xff;
return (currentPosition < provider.length())
? Byte.toUnsignedInt(provider.readByte(currentPosition++))
: -1;
}
@Override
public int read(byte[] b, int bufferOffset, int len) throws IOException {
long eof = provider.length();
if (currentPosition >= eof) {
return -1;
}
return -1;
len = (int) Math.min(len, eof - currentPosition);
byte[] bytes = provider.readBytes(currentPosition, len);
System.arraycopy(bytes, 0, b, bufferOffset, len);
currentPosition += len;
return len;
}
}

53
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ByteProviderWrapper.java
View File

@ -15,12 +15,13 @@
*/
package ghidra.app.util.bin;
import java.io.*;
import java.io.File;
import java.io.IOException;
import ghidra.formats.gfilesystem.FSRL;
/**
* Creates a {@link ByteProvider} constrained to a sub-section of an existing {@link ByteProvider}.
* A {@link ByteProvider} constrained to a sub-section of an existing {@link ByteProvider}.
*/
public class ByteProviderWrapper implements ByteProvider {
private ByteProvider provider;
@ -29,7 +30,21 @@ public class ByteProviderWrapper implements ByteProvider {
private FSRL fsrl;
/**
* Constructs a {@link ByteProviderWrapper} around the specified {@link ByteProvider}
* Creates a wrapper around a {@link ByteProvider} that contains the same bytes as the specified
* provider, but with a new {@link FSRL} identity.
* <p>
*
* @param provider {@link ByteProvider} to wrap
* @param fsrl {@link FSRL} identity for the instance
* @throws IOException if error
*/
public ByteProviderWrapper(ByteProvider provider, FSRL fsrl) throws IOException {
this(provider, 0, provider.length(), fsrl);
}
/**
* Constructs a {@link ByteProviderWrapper} around the specified {@link ByteProvider},
* constrained to a subsection of the provider.
*
* @param provider the {@link ByteProvider} to wrap
* @param subOffset the offset in the {@link ByteProvider} of where to start the new
@ -41,13 +56,14 @@ public class ByteProviderWrapper implements ByteProvider {
}
/**
* Constructs a {@link ByteProviderWrapper} around the specified {@link ByteProvider}
* Constructs a {@link ByteProviderWrapper} around the specified {@link ByteProvider},
* constrained to a subsection of the provider.
*
* @param provider the {@link ByteProvider} to wrap
* @param subOffset the offset in the {@link ByteProvider} of where to start the new
* {@link ByteProviderWrapper}
* @param subLength the length of the new {@link ByteProviderWrapper}
* @param fsrl FSRL identity of the file this ByteProvider represents
* @param fsrl {@link FSRL} identity of the file this ByteProvider represents
*/
public ByteProviderWrapper(ByteProvider provider, long subOffset, long subLength, FSRL fsrl) {
this.provider = provider;
@ -57,8 +73,8 @@ public class ByteProviderWrapper implements ByteProvider {
}
@Override
public void close() {
// don't do anything for now
public void close() throws IOException {
// do not close the wrapped provider
}
@Override
@ -68,24 +84,22 @@ public class ByteProviderWrapper implements ByteProvider {
@Override
public File getFile() {
return provider.getFile();
}
@Override
public InputStream getInputStream(long index) throws IOException {
return new ByteProviderInputStream(this, index, subLength - index);
// there is no file that represents the actual contents of the subrange, so return null
return null;
}
@Override
public String getName() {
return provider.getName() + "[0x" + Long.toHexString(subOffset) + ",0x" +
Long.toHexString(subLength) + "]";
return (fsrl != null)
? fsrl.getName()
: String.format("%s[0x%x,0x%x]", provider.getName(), subOffset, subLength);
}
@Override
public String getAbsolutePath() {
return provider.getAbsolutePath() + "[0x" + Long.toHexString(subOffset) + ",0x" +
Long.toHexString(subLength) + "]";
return (fsrl != null)
? fsrl.getPath()
: String.format("%s[0x%x,0x%x]", provider.getAbsolutePath(), subOffset, subLength);
}
@Override
@ -95,10 +109,7 @@ public class ByteProviderWrapper implements ByteProvider {
@Override
public boolean isValidIndex(long index) {
if (provider.isValidIndex(index)) {
return index >= subOffset && index < subLength;
}
return false;
return (0 <= index && index < subLength) && provider.isValidIndex(subOffset + index);
}
@Override

103
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/EmptyByteProvider.java Normal file
View File

@ -0,0 +1,103 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.bin;
import java.io.*;
import ghidra.formats.gfilesystem.FSRL;
/**
* A {@link ByteProvider} that has no contents.
*
*/
public class EmptyByteProvider implements ByteProvider {
private final FSRL fsrl;
/**
* Create an instance with a null identity
*/
public EmptyByteProvider() {
this(null);
}
/**
* Create an instance with the specified {@link FSRL} identity.
*
* @param fsrl {@link FSRL} identity for this instance
*/
public EmptyByteProvider(FSRL fsrl) {
this.fsrl = fsrl;
}
@Override
public FSRL getFSRL() {
return fsrl;
}
@Override
public File getFile() {
return null;
}
@Override
public String getName() {
return fsrl != null ? fsrl.getName() : null;
}
@Override
public String getAbsolutePath() {
return fsrl != null ? fsrl.getPath() : null;
}
@Override
public byte readByte(long index) throws IOException {
throw new IOException("Not supported");
}
@Override
public byte[] readBytes(long index, long length) throws IOException {
if (index != 0 || length != 0) {
throw new IOException("Not supported");
}
return new byte[0];
}
@Override
public long length() {
return 0;
}
@Override
public boolean isValidIndex(long index) {
return false;
}
@Override
public void close() throws IOException {
// do nothing
}
@Override
public InputStream getInputStream(long index) throws IOException {
if (index != 0) {
throw new IOException("Invalid offset");
}
return InputStream.nullInputStream();
}
}

323
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/FileByteProvider.java Normal file
View File

@ -0,0 +1,323 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.bin;
import java.io.*;
import java.nio.file.AccessMode;
import org.apache.commons.collections4.map.ReferenceMap;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.util.Msg;
import ghidra.util.datastruct.LRUMap;
/**
* A {@link ByteProvider} that reads its bytes from a file.
*
*/
public class FileByteProvider implements ByteProvider, MutableByteProvider {
static final int BUFFER_SIZE = 64 * 1024;
private static final int BUFFERS_TO_PIN = 4;
private FSRL fsrl;
private File file;
private RandomAccessFile raf;
private ReferenceMap<Long, Buffer> buffers = new ReferenceMap<>();
private LRUMap<Long, Buffer> lruBuffers = new LRUMap<>(BUFFERS_TO_PIN); // only used to pin a small set of recently used buffers in memory. not used for lookup
private long currentLength;
private AccessMode accessMode; // probably wrong Enum class, but works for now
/**
* Creates a new instance.
*
* @param file {@link File} to open
* @param fsrl {@link FSRL} identity of the file
* @param accessMode {@link AccessMode#READ} or {@link AccessMode#WRITE}
* @throws IOException if error
*/
public FileByteProvider(File file, FSRL fsrl, AccessMode accessMode)
throws IOException {
this.file = file;
this.fsrl = fsrl;
this.accessMode = accessMode;
this.raf = new RandomAccessFile(file, accessModeToString(accessMode));
this.currentLength = raf.length();
}
/**
* Returns the access mode the file was opened with.
*
* @return {@link AccessMode} used to open file
*/
public AccessMode getAccessMode() {
return accessMode;
}
@Override
public void close() throws IOException {
if (raf != null) {
raf.close();
raf = null;
}
buffers.clear();
lruBuffers.clear();
}
@Override
public File getFile() {
return file;
}
@Override
public String getName() {
return fsrl != null ? fsrl.getName() : file.getName();
}
@Override
public String getAbsolutePath() {
return fsrl != null ? fsrl.getPath() : file.getAbsolutePath();
}
@Override
public FSRL getFSRL() {
return fsrl;
}
@Override
public long length() throws IOException {
return currentLength;
}
@Override
public boolean isValidIndex(long index) {
return 0 <= index && index < currentLength;
}
@Override
public byte readByte(long index) throws IOException {
ensureBounds(index, 1);
Buffer fileBuffer = getBufferFor(index);
int ofs = fileBuffer.getBufferOffset(index);
return fileBuffer.bytes[ofs];
}
@Override
public byte[] readBytes(long index, long length) throws IOException {
ensureBounds(index, length);
if (length > Integer.MAX_VALUE) {
throw new IllegalArgumentException();
}
int len = (int) length;
byte[] result = new byte[len];
int bytesRead = readBytes(index, result, 0, len);
if (bytesRead != len) {
throw new IOException("Unable to read " + len + " bytes at " + index);
}
return result;
}
/**
* Read bytes at the specified index into the given byte array.
* <p>
* See {@link InputStream#read(byte[], int, int)}.
* <p>
*
* @param index file offset to start reading
* @param buffer byte array that will receive the bytes
* @param offset offset inside the byte array to place the bytes
* @param length number of bytes to read
* @return number of actual bytes read
* @throws IOException if error
*/
public int readBytes(long index, byte[] buffer, int offset, int length) throws IOException {
ensureBounds(index, 0);
length = (int) Math.min(currentLength - index, length);
int totalBytesRead = 0;
while (length > 0) {
Buffer fileBuffer = getBufferFor(index);
int ofs = fileBuffer.getBufferOffset(index);
int bytesToReadFromThisBuffer = Math.min(fileBuffer.len - ofs, length);
System.arraycopy(fileBuffer.bytes, ofs, buffer, totalBytesRead,
bytesToReadFromThisBuffer);
length -= bytesToReadFromThisBuffer;
index += bytesToReadFromThisBuffer;
totalBytesRead += bytesToReadFromThisBuffer;
}
return totalBytesRead;
}
@Override
protected void finalize() {
if (raf != null) {
Msg.warn(this, "FAIL TO CLOSE " + file);
}
}
/**
* Writes bytes to the specified offset in the file.
*
* @param index the location in the file to starting writing
* @param buffer bytes to write
* @param offset offset in the buffer byte array to start
* @param length number of bytes to write
* @throws IOException if bad {@link AccessMode} or other io error
*/
public synchronized void writeBytes(long index, byte[] buffer, int offset, int length)
throws IOException {
if (accessMode != AccessMode.WRITE) {
throw new IOException("Not write mode");
}
doWriteBytes(index, buffer, offset, length);
long writeEnd = index + length;
currentLength = Math.max(currentLength, writeEnd);
// after writing new bytes to the file, update
// any buffers that we can completely fill with the contents of
// this write buffer, and invalidate any buffers that we can't
// completely fill (they can be re-read in a normal fashion later when needed)
while (length > 0) {
long bufferPos = getBufferPos(index);
int bufferOfs = (int) (index - bufferPos);
int bytesAvailForThisBuffer = Math.min(length, BUFFER_SIZE - bufferOfs);
Buffer fileBuffer = buffers.get(bufferPos);
if (fileBuffer != null) {
if (bufferOfs == 0 && length >= BUFFER_SIZE) {
System.arraycopy(buffer, offset, fileBuffer.bytes, 0, BUFFER_SIZE);
fileBuffer.len = BUFFER_SIZE;
}
else {
buffers.remove(bufferPos);
lruBuffers.remove(bufferPos);
}
}
index += bytesAvailForThisBuffer;
offset += bytesAvailForThisBuffer;
length -= bytesAvailForThisBuffer;
}
}
@Override
public void writeByte(long index, byte value) throws IOException {
writeBytes(index, new byte[] { value }, 0, 1);
}
@Override
public void writeBytes(long index, byte[] values) throws IOException {
writeBytes(index, values, 0, values.length);
}
//------------------------------------------------------------------------------------
/**
* Reads bytes from the file.
* <p>
* Protected by synchronized lock. (See {@link #getBufferFor(long)}).
*
* @param index file position of where to read
* @param buffer byte array that will receive bytes
* @return actual number of byte read
* @throws IOException if error
*/
protected int doReadBytes(long index, byte[] buffer) throws IOException {
raf.seek(index);
return raf.read(buffer, 0, buffer.length);
}
/**
* Writes the specified bytes to the file.
* <p>
* Protected by synchronized lock (See {@link #writeBytes(long, byte[], int, int)})
*
* @param index file position of where to write
* @param buffer byte array containing bytes to write
* @param offset offset inside of byte array to start
* @param length number of bytes from buffer to write
* @throws IOException if error
*/
protected void doWriteBytes(long index, byte[] buffer, int offset, int length)
throws IOException {
raf.seek(index);
raf.write(buffer, offset, length);
}
//------------------------------------------------------------------------------------
private void ensureBounds(long index, long length) throws IOException {
if (index < 0 || index > currentLength) {
throw new IOException("Invalid index: " + index);
}
if (index + length > currentLength) {
throw new IOException("Unable to read past EOF: " + index + ", " + length);
}
}
private long getBufferPos(long index) {
return (index / BUFFER_SIZE) * BUFFER_SIZE;
}
private synchronized Buffer getBufferFor(long pos) throws IOException {
long bufferPos = getBufferPos(pos);
if (bufferPos >= currentLength) {
throw new EOFException();
}
Buffer buffer = buffers.get(bufferPos);
if (buffer == null) {
buffer = new Buffer(bufferPos, (int) Math.min(currentLength - bufferPos, BUFFER_SIZE));
int bytesRead = doReadBytes(bufferPos, buffer.bytes);
if (bytesRead != buffer.len) {
buffer.len = bytesRead;
// warn?
}
buffers.put(bufferPos, buffer);
}
lruBuffers.put(bufferPos, buffer);
return buffer;
}
private static class Buffer {
long pos; // absolute position in file of this buffer
int len; // number of valid bytes in buffer
byte[] bytes;
Buffer(long pos, int len) {
this.pos = pos;
this.len = len;
this.bytes = new byte[len];
}
int getBufferOffset(long filePos) throws EOFException {
int ofs = (int) (filePos - pos);
if (ofs >= len) {
throw new EOFException();
}
return ofs;
}
}
private String accessModeToString(AccessMode mode) {
switch (mode) {
default:
case READ:
return "r";
case WRITE:
return "rw";
}
}
}

49
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/MemBufferByteProvider.java
View File

@ -105,53 +105,4 @@ public class MemBufferByteProvider implements ByteProvider {
return bytes;
}
@Override
public InputStream getInputStream(long index) throws IOException {
if (index < 0 || index > Integer.MAX_VALUE) {
throw new IOException("index out of range");
}
return new MemBufferProviderInputStream((int) index);
}
private class MemBufferProviderInputStream extends InputStream {
private int initialOffset;
private int offset;
MemBufferProviderInputStream(int offset) {
this.offset = offset;
this.initialOffset = offset;
}
@Override
public int read() throws IOException {
byte b = readByte(offset++);
return b & 0xff;
}
@Override
public int read(byte[] b, int off, int len) {
byte[] bytes = new byte[len];
int count = buffer.getBytes(bytes, offset);
System.arraycopy(bytes, 0, b, off, count);
offset += count;
return count;
}
@Override
public int available() {
return (int) length() - offset;
}
@Override
public synchronized void reset() throws IOException {
offset = initialOffset;
}
@Override
public void close() throws IOException {
// not applicable
}
}
}

97
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedFileByteProvider.java Normal file
View File

@ -0,0 +1,97 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.bin;
import java.io.File;
import java.io.IOException;
import java.nio.file.AccessMode;
import ghidra.formats.gfilesystem.FSRL;
/**
* A {@link ByteProvider} that reads from an on-disk file, but obfuscates / de-obfuscates the
* contents of the file when reading / writing.
*/
public class ObfuscatedFileByteProvider extends FileByteProvider {
// @formatter:off
// copied from ChainedBuffer
static final byte[] XOR_MASK_BYTES = new byte[] {
(byte)0x59, (byte)0xea, (byte)0x67, (byte)0x23, (byte)0xda, (byte)0xb8, (byte)0x00, (byte)0xb8,
(byte)0xc3, (byte)0x48, (byte)0xdd, (byte)0x8b, (byte)0x21, (byte)0xd6, (byte)0x94, (byte)0x78,
(byte)0x35, (byte)0xab, (byte)0x2b, (byte)0x7e, (byte)0xb2, (byte)0x4f, (byte)0x82, (byte)0x4e,
(byte)0x0e, (byte)0x16, (byte)0xc4, (byte)0x57, (byte)0x12, (byte)0x8e, (byte)0x7e, (byte)0xe6,
(byte)0xb6, (byte)0xbd, (byte)0x56, (byte)0x91, (byte)0x57, (byte)0x72, (byte)0xe6, (byte)0x91,
(byte)0xdc, (byte)0x52, (byte)0x2e, (byte)0xf2, (byte)0x1a, (byte)0xb7, (byte)0xd6, (byte)0x6f,
(byte)0xda, (byte)0xde, (byte)0xe8, (byte)0x48, (byte)0xb1, (byte)0xbb, (byte)0x50, (byte)0x6f,
(byte)0xf4, (byte)0xdd, (byte)0x11, (byte)0xee, (byte)0xf2, (byte)0x67, (byte)0xfe, (byte)0x48,
(byte)0x8d, (byte)0xae, (byte)0x69, (byte)0x1a, (byte)0xe0, (byte)0x26, (byte)0x8c, (byte)0x24,
(byte)0x8e, (byte)0x17, (byte)0x76, (byte)0x51, (byte)0xe2, (byte)0x60, (byte)0xd7, (byte)0xe6,
(byte)0x83, (byte)0x65, (byte)0xd5, (byte)0xf0, (byte)0x7f, (byte)0xf2, (byte)0xa0, (byte)0xd6,
(byte)0x4b, (byte)0xbd, (byte)0x24, (byte)0xd8, (byte)0xab, (byte)0xea, (byte)0x9e, (byte)0xa6,
(byte)0x48, (byte)0x94, (byte)0x3e, (byte)0x7b, (byte)0x2c, (byte)0xf4, (byte)0xce, (byte)0xdc,
(byte)0x69, (byte)0x11, (byte)0xf8, (byte)0x3c, (byte)0xa7, (byte)0x3f, (byte)0x5d, (byte)0x77,
(byte)0x94, (byte)0x3f, (byte)0xe4, (byte)0x8e, (byte)0x48, (byte)0x20, (byte)0xdb, (byte)0x56,
(byte)0x32, (byte)0xc1, (byte)0x87, (byte)0x01, (byte)0x2e, (byte)0xe3, (byte)0x7f, (byte)0x40,
};
// @formatter:on
/**
* Creates an instance of {@link ObfuscatedFileByteProvider}.
*
* @param file {@link File} to read from / write to
* @param fsrl {@link FSRL} identity of this file
* @param accessMode {@link AccessMode#READ} or {@link AccessMode#WRITE}
* @throws IOException if error
*/
public ObfuscatedFileByteProvider(File file, FSRL fsrl, AccessMode accessMode)
throws IOException {
super(file, fsrl, accessMode);
}
@Override
public File getFile() {
// obfuscated file isn't readable, so force null
return null;
}
@Override
protected int doReadBytes(long index, byte[] buffer) throws IOException {
int bytesRead = super.doReadBytes(index, buffer);
for (int i = 0; i < bytesRead; i++) {
long byteIndex = index + i;
int xorMaskIndex = (int) (byteIndex % XOR_MASK_BYTES.length);
byte xorMask = XOR_MASK_BYTES[xorMaskIndex];
buffer[i] ^= xorMask;
}
return bytesRead;
}
@Override
protected void doWriteBytes(long index, byte[] buffer, int offset, int length)
throws IOException {
byte[] tmpBuffer = new byte[length];
for (int i = 0; i < length; i++) {
long byteIndex = index + i;
int xorMaskIndex = (int) (byteIndex % XOR_MASK_BYTES.length);
byte xorMask = XOR_MASK_BYTES[xorMaskIndex];
tmpBuffer[i] = (byte) (buffer[i + offset] ^ xorMask);
}
super.doWriteBytes(index, tmpBuffer, 0, length);
}
}

99
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedInputStream.java Normal file
View File

@ -0,0 +1,99 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.bin;
import java.io.*;
/**
* An {@link InputStream} wrapper that de-obfuscates the bytes being read from the underlying
* stream.
*/
public class ObfuscatedInputStream extends InputStream {
private InputStream delegate;
private long currentPosition;
/**
* Creates instance.
*
* @param delegate {@link InputStream} to wrap
*/
public ObfuscatedInputStream(InputStream delegate) {
this.delegate = delegate;
}
@Override
public void close() throws IOException {
delegate.close();
super.close();
}
@Override
public int read() throws IOException {
byte[] buffer = new byte[1];
int bytesRead = read(buffer, 0, 1);
return bytesRead == 1 ? Byte.toUnsignedInt(buffer[0]) : -1;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
int bytesRead = delegate.read(b, off, len);
for (int i = 0; i < bytesRead; i++, currentPosition++) {
int xorMaskIndex =
(int) (currentPosition % ObfuscatedFileByteProvider.XOR_MASK_BYTES.length);
byte xorMask = ObfuscatedFileByteProvider.XOR_MASK_BYTES[xorMaskIndex];
b[off + i] ^= xorMask;
}
return bytesRead;
}
/**
* Entry point to enable command line users to retrieve the contents of an obfuscated
* file.
*
* @param args either ["--help"], or [ "input_filename", "output_filename" ]
* @throws IOException if error
*/
public static void main(String[] args) throws IOException {
if (args.length != 2 || (args.length > 1 && args[0].equals("--help"))) {
System.err.println("De-Obfuscator Usage:");
System.err.println("\t" + ObfuscatedInputStream.class.getName() +
" obfuscated_input_filename_path plain_dest_output_filename_path");
System.err.println("");
System.err.println("\tExample:");
System.err.println("\t\t" + ObfuscatedInputStream.class.getName() +
" /tmp/myuserid-Ghidra/fscache2/aa/bb/aabbccddeeff00112233445566778899 /tmp/aabbccddeeff00112233445566778899.plaintext");
System.err.println("");
return;
}
File obfuscatedInputFile = new File(args[0]);
File plainTextOutputFile = new File(args[1]);
try (InputStream is = new ObfuscatedInputStream(new FileInputStream(obfuscatedInputFile));
OutputStream os = new FileOutputStream(plainTextOutputFile)) {
byte buffer[] = new byte[4096];
int bytesRead;
while ((bytesRead = is.read(buffer)) > 0) {
os.write(buffer, 0, bytesRead);
}
}
}
}

69
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ObfuscatedOutputStream.java Normal file
View File

@ -0,0 +1,69 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.bin;
import java.io.IOException;
import java.io.OutputStream;
/**
* An {@link OutputStream} wrapper that obfuscates the bytes being written to the underlying
* stream.
*/
public class ObfuscatedOutputStream extends OutputStream {
private OutputStream delegate;
private long currentPosition;
/**
* Creates instance.
*
* @param delegate {@link OutputStream} to wrap
*/
public ObfuscatedOutputStream(OutputStream delegate) {
this.delegate = delegate;
}
@Override
public void close() throws IOException {
delegate.close();
super.close();
}
@Override
public void flush() throws IOException {
delegate.flush();
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
byte[] tmpBuffer = new byte[len];
for (int i = 0; i < len; i++) {
long byteIndex = currentPosition + i;
int xorMaskIndex =
(int) (byteIndex % ObfuscatedFileByteProvider.XOR_MASK_BYTES.length);
byte xorMask = ObfuscatedFileByteProvider.XOR_MASK_BYTES[xorMaskIndex];
tmpBuffer[i] = (byte) (b[i + off] ^ xorMask);
}
delegate.write(tmpBuffer, 0, tmpBuffer.length);
currentPosition += len;
}
@Override
public void write(int b) throws IOException {
write(new byte[] { (byte) b }, 0, 1);
}
}

3
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RandomAccessByteProvider.java
View File

@ -29,7 +29,10 @@ import ghidra.util.Msg;
* {@link ArrayIndexOutOfBoundsException}s.
* <p>
* See {@link SynchronizedByteProvider} as a solution.
* <p>
* @deprecated See {@link FileByteProvider} as replacement ByteProvider.
*/
@Deprecated(since = "10.1", forRemoval = true)
public class RandomAccessByteProvider implements ByteProvider {
protected File file;
protected GhidraRandomAccessFile randomAccessFile;

5
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RangeMappedByteProvider.java
View File

@ -220,11 +220,6 @@ public class RangeMappedByteProvider implements ByteProvider {
return totalBytesRead;
}
@Override
public InputStream getInputStream(long index) throws IOException {
return new ByteProviderInputStream(this, 0, length);
}
private void ensureBounds(long index, long count) throws IOException {
if (index < 0 || index > length) {
throw new IOException("Invalid index: " + index);

7
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/SynchronizedByteProvider.java
View File

@ -78,9 +78,12 @@ public class SynchronizedByteProvider implements ByteProvider {
public synchronized byte[] readBytes(long index, long length) throws IOException {
return provider.readBytes(index, length);
}
@Override
public synchronized InputStream getInputStream(long index) throws IOException {
return provider.getInputStream(index);
// Return a ByteProviderInputStream that reads its bytes via this wrapper so that it is completely
// synchronized. Returning the delegate provider's getInputStream() would subvert
// synchronization and allow direct access to the underlying delegate provider.
return ByteProvider.super.getInputStream(index);
}
}

23
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/CoffArchiveMemberHeader.java
View File

@ -15,14 +15,14 @@
*/
package ghidra.app.util.bin.format.coff.archive;
import java.io.IOException;
import ghidra.app.util.bin.*;
import ghidra.program.model.data.*;
import ghidra.util.Msg;
import ghidra.util.StringUtilities;
import ghidra.util.exception.DuplicateNameException;
import java.io.IOException;
public class CoffArchiveMemberHeader implements StructConverter {
public static final String SLASH = "/";
public static final String SLASH_SLASH = "//";
@ -249,6 +249,24 @@ public class CoffArchiveMemberHeader implements StructConverter {
return groupId;
}
public int getUserIdInt() {
try {
return Integer.parseInt(userId);
}
catch (NumberFormatException nfe) {
return 0;
}
}
public int getGroupIdInt() {
try {
return Integer.parseInt(groupId);
}
catch (NumberFormatException nfe) {
return 0;
}
}
public String getMode() {
return mode;
}
@ -274,6 +292,7 @@ public class CoffArchiveMemberHeader implements StructConverter {
!name.equals(CoffArchiveMemberHeader.SLASH_SLASH);
}
@Override
public DataType toDataType() throws DuplicateNameException, IOException {
String camh_name = StructConverterUtil.parseName(CoffArchiveMemberHeader.class);
Structure struct = new StructureDataType(camh_name, 0);

25
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/coff/archive/FirstLinkerMember.java
View File

@ -15,31 +15,30 @@
*/
package ghidra.app.util.bin.format.coff.archive;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import ghidra.app.util.bin.*;
import ghidra.program.model.data.*;
import ghidra.util.BigEndianDataConverter;
import ghidra.util.DataConverter;
import ghidra.util.exception.DuplicateNameException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public final class FirstLinkerMember implements StructConverter {
private int numberOfSymbols;
private int [] offsets;
private List<String> stringTable = new ArrayList<String>();
private List<String> stringTable = new ArrayList<>();
private long _fileOffset;
private List<Integer> stringLengths = new ArrayList<Integer>();
private List<Integer> stringLengths = new ArrayList<>();
public FirstLinkerMember(BinaryReader reader, CoffArchiveMemberHeader header, boolean skip)
throws IOException {
_fileOffset = reader.getPointerIndex();
boolean isLittleEndian = reader.isLittleEndian();
reader.setLittleEndian(false);//this entire structure is stored as big-endian..
BinaryReader origReader = reader;
reader = reader.asBigEndian(); //this entire structure is stored as big-endian..
numberOfSymbols = readNumberOfSymbols(reader);
@ -57,7 +56,7 @@ public final class FirstLinkerMember implements StructConverter {
}
}
else {
stringTable = new ArrayList<String>(numberOfSymbols);
stringTable = new ArrayList<>(numberOfSymbols);
for (int i = 0 ; i < numberOfSymbols ; ++i) {
String string = reader.readNextAsciiString();
stringTable.add( string );
@ -65,8 +64,7 @@ public final class FirstLinkerMember implements StructConverter {
}
}
reader.setLittleEndian(isLittleEndian);
reader.setPointerIndex(_fileOffset + header.getSize());
origReader.setPointerIndex(_fileOffset + header.getSize());
}
/**
@ -100,9 +98,10 @@ public final class FirstLinkerMember implements StructConverter {
if (stringTable.isEmpty()) {
throw new RuntimeException("FirstLinkerMember::getStringTable() has been skipped.");
}
return new ArrayList<String>(stringTable);
return new ArrayList<>(stringTable);
}
@Override
public DataType toDataType() throws DuplicateNameException, IOException {
String name = StructConverterUtil.parseName(FirstLinkerMember.class);
Structure struct = new StructureDataType(name + "_" + numberOfSymbols, 0);

7
Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/sectionprovider/NullSectionProvider.java
View File

@ -15,11 +15,10 @@
*/
package ghidra.app.util.bin.format.dwarf4.next.sectionprovider;
import ghidra.app.util.bin.ByteArrayProvider;
import ghidra.app.util.bin.ByteProvider;
import java.io.IOException;
import ghidra.app.util.bin.ByteProvider;
public class NullSectionProvider implements DWARFSectionProvider {
public NullSectionProvider() {
@ -28,7 +27,7 @@ public class NullSectionProvider implements DWARFSectionProvider {
@Override
public ByteProvider getSectionAsByteProvider(String sectionName) throws IOException {
return new ByteArrayProvider(new byte[] {});
return ByteProvider.EMPTY_BYTEPROVIDER;
}
@Override

5
Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java
View File

@ -22,6 +22,7 @@ import java.util.List;
import ghidra.app.util.Option;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.importer.MessageLog;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.framework.model.DomainFolder;
import ghidra.framework.model.DomainObject;
import ghidra.program.model.listing.Program;
@ -169,7 +170,9 @@ public interface Loader extends ExtensionPoint, Comparable<Loader> {
* @return The preferred file name to use when loading.
*/
public default String getPreferredFileName(ByteProvider provider) {
return provider.getName().replaceAll("[\\\\:|]+", "/");
FSRL fsrl = provider.getFSRL();
String name = (fsrl != null) ? fsrl.getName() : provider.getName();
return name.replaceAll("[\\\\:|]+", "/");
}
/**

9
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/AbstractFileExtractorTask.java
View File

@ -15,15 +15,10 @@
*/
package ghidra.formats.gfilesystem;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.*;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.CryptoException;
import ghidra.util.exception.IOCancelledException;
import ghidra.util.task.Task;
import ghidra.util.task.TaskMonitor;
@ -190,7 +185,7 @@ public abstract class AbstractFileExtractorTask extends Task {
}
protected void extractFile(GFile srcFile, File outputFile, TaskMonitor monitor)
throws CancelledException, CryptoException {
throws CancelledException {
monitor.setMessage(srcFile.getName());
try (InputStream in = getSourceFileInputStream(srcFile, monitor)) {

44
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/DerivedFileProducer.java
View File

@ -1,44 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import java.io.*;
/**
* Used by {@link FileSystemService#getDerivedFile(FSRL, String, DerivedFileProducer, TaskMonitor)}
* to produce a derived file from a source file.
* <p>
* The {@link InputStream} returned from the method will be closed by the caller.
*/
public interface DerivedFileProducer {
/**
* Callback method intended to be implemented by the caller to
* {@link FileSystemService#getDerivedFile(FSRL, String, DerivedFileProducer, TaskMonitor)}.
* <p>
* The implementation needs to return an {@link InputStream} that contains the bytes
* of the derived file.
* <p>
* @param srcFile {@link File} location of the source file (usually in the file cache)
* @return a new {@link InputStream} that will produce all the bytes of the derived file.
* @throws IOException if there is a problem while producing the InputStream.
* @throws CancelledException if the user canceled.
*/
public InputStream produceDerivedStream(File srcFile) throws IOException, CancelledException;
}

41
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/DerivedFilePushProducer.java
View File

@ -1,41 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import java.io.IOException;
import java.io.OutputStream;
/**
* Used by {@link FileSystemService#getDerivedFilePush(FSRL, String, DerivedFilePushProducer, TaskMonitor)}
* to produce a derived file from a source file.
*/
public interface DerivedFilePushProducer {
/**
* Callback method intended to be implemented by the caller to
* {@link FileSystemService#getDerivedFilePush(FSRL, String, DerivedFilePushProducer, TaskMonitor)}.
* <p>
* The implementation needs to write bytes to the supplied {@link OutputStream}.
* <p>
* @param os {@link OutputStream} that the implementor should write the bytes to. Do
* not close the stream when done.
* @throws IOException if there is a problem while writing to the OutputStream.
* @throws CancelledException if the user canceled.
*/
public void push(OutputStream os) throws IOException, CancelledException;
}

35
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSRL.java
View File

@ -96,6 +96,19 @@ public class FSRL {
return parent;
}
/**
* Ensures that a FSRL instance is a file type reference by converting any FSRLRoots
* into the container file that hosts the FSRLRoot.
*
* @param fsrl FSRL or FSRLRoot instance to possibly convert
* @return original FSRL if already a normal FSRL, or the container if it was a FSRLRoot
*/
public static FSRL convertRootToContainer(FSRL fsrl) {
return (fsrl instanceof FSRLRoot && fsrl.getFS().hasContainer())
? fsrl.getFS().getContainer()
: fsrl;
}
/**
* Creates a single {@link FSRL} from a FSRL-part string.
* <p>
@ -282,15 +295,29 @@ public class FSRL {
return md5;
}
/**
* Tests specified MD5 value against MD5 in this FSRL.
*
* @param otherMD5 md5 in a hex string
* @return boolean true if equal, or that both are null, false otherwise
*/
public boolean isMD5Equal(String otherMD5) {
if (this.md5 == null) {
return otherMD5 == null;
}
return this.md5.equalsIgnoreCase(otherMD5);
}
/**
* Creates a new {@link FSRL} instance, using the same information as this instance,
* but with a new {@link #getMD5() MD5} value.
*
* @param newMD5 string md5
* @return new {@link FSRL} instance with the same path and the specified md5 value.
* @return new {@link FSRL} instance with the same path and the specified md5 value,
* or if newMD5 is same as existing, returns this
*/
public FSRL withMD5(String newMD5) {
return new FSRL(getFS(), path, newMD5);
return Objects.equals(md5, newMD5) ? this : new FSRL(getFS(), path, newMD5);
}
/**
@ -312,7 +339,7 @@ public class FSRL {
* <p>
* Used when re-root'ing a FSRL path onto another parent object (usually during intern()'ing)
*
* @param copyPath
* @param copyPath another FSRL to copy path and md5 from
* @return new FSRL instance
*/
public FSRL withPath(FSRL copyPath) {
@ -323,7 +350,7 @@ public class FSRL {
* Creates a new {@link FSRL} instance, using the same {@link FSRLRoot} as this instance,
* combining the current {@link #getPath() path} with the {@code relPath} value.
* <p>
* @param relPath
* @param relPath relative path string to append, '/'s will be automatically added
* @return new {@link FSRL} instance with additional path appended.
*/
public FSRL appendPath(String relPath) {

173
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java
View File

@ -20,6 +20,7 @@ import java.io.*;
import java.net.MalformedURLException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.*;
@ -27,9 +28,8 @@ import java.util.Map.Entry;
import org.apache.commons.io.FilenameUtils;
import com.google.common.io.ByteStreams;
import docking.widgets.OptionDialog;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.util.*;
import ghidra.util.exception.CancelledException;
@ -45,16 +45,6 @@ public class FSUtilities {
private static final char DOT = '.';
private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
public static class StreamCopyResult {
public long bytesCopied;
public byte[] md5;
public StreamCopyResult(long bytesCopied, byte[] md5) {
this.bytesCopied = bytesCopied;
this.md5 = md5;
}
}
private static char[] hexdigit = "0123456789abcdef".toCharArray();
/**
@ -62,25 +52,17 @@ public class FSUtilities {
* case-insensitive.
*/
public static final Comparator<GFile> GFILE_NAME_TYPE_COMPARATOR = (o1, o2) -> {
String n1 = o1.getName();
String n2 = o2.getName();
if (n1 == null) {
return -1;
int result = Boolean.compare(!o1.isDirectory(), !o2.isDirectory());
if (result == 0) {
String n1 = Objects.requireNonNullElse(o1.getName(), "");
String n2 = Objects.requireNonNullElse(o2.getName(), "");
result = n1.compareToIgnoreCase(n2);
}
if (o1.isDirectory()) {
if (o2.isDirectory()) {
return n1.compareToIgnoreCase(n2);
}
return -1;
}
else if (o2.isDirectory()) {
return 1;
}
return n1.compareToIgnoreCase(n2);
return result;
};
/**
* Converts a string -&gt; string mapping into a "key: value" multi-line string.
* Converts a string-to-string mapping into a "key: value\n" multi-line string.
*
* @param info map of string key to string value.
* @return Multi-line string "key: value" string.
@ -349,55 +331,61 @@ public class FSUtilities {
}
/**
* Copies a stream and calculates the md5 at the same time.
* <p>
* Does not close the passed-in InputStream or OutputStream.
*
* @param is {@link InputStream} to copy. NOTE: not closed by this method.
* @param os {@link OutputStream} to write to. NOTE: not closed by this method.
* @return {@link StreamCopyResult} with md5 and bytes copied count, never null.
* Copy the contents of a {@link ByteProvider} to a file.
*
* @param provider {@link ByteProvider} source of bytes
* @param destFile {@link File} destination file
* @param monitor {@link TaskMonitor} to update
* @return number of bytes copied
* @throws IOException if error
* @throws CancelledException if canceled
* @throws CancelledException if cancelled
*/
@SuppressWarnings("resource")
public static StreamCopyResult streamCopy(InputStream is, OutputStream os, TaskMonitor monitor)
throws IOException, CancelledException {
HashingOutputStream hos;
try {
// This wrapping outputstream is not closed on purpose.
hos = new HashingOutputStream(os, "MD5");
}
catch (NoSuchAlgorithmException e) {
throw new IOException("Could not get MD5 hash algo", e);
public static long copyByteProviderToFile(ByteProvider provider, File destFile,
TaskMonitor monitor) throws IOException, CancelledException {
try (InputStream is = provider.getInputStream(0);
FileOutputStream fos = new FileOutputStream(destFile)) {
return streamCopy(is, fos, monitor);
}
}
// TODO: use FileUtilities.copyStreamToStream()
/**
* Copy a stream while updating a TaskMonitor.
*
* @param is {@link InputStream} source of bytes
* @param os {@link OutputStream} destination of bytes
* @param monitor {@link TaskMonitor} to update
* @return number of bytes copied
* @throws IOException if error
* @throws CancelledException if cancelled
*/
public static long streamCopy(InputStream is, OutputStream os, TaskMonitor monitor)
throws IOException, CancelledException {
byte buffer[] = new byte[FileUtilities.IO_BUFFER_SIZE];
int bytesRead;
long totalBytesCopied = 0;
while ((bytesRead = is.read(buffer)) > 0) {
hos.write(buffer, 0, bytesRead);
os.write(buffer, 0, bytesRead);
totalBytesCopied += bytesRead;
monitor.setProgress(totalBytesCopied);
monitor.checkCanceled();
}
hos.flush();
return new StreamCopyResult(totalBytesCopied, hos.getDigest());
os.flush();
return totalBytesCopied;
}
/**
* Calculate the MD5 of a stream.
*
* @param is {@link InputStream} to read
* @param monitor {@link TaskMonitor} to watch for cancel
* @return md5 as a hex encoded string, never null.
* Returns the text lines in the specified ByteProvider.
* <p>
* See {@link FileUtilities#getLines(InputStream)}
*
* @param byteProvider {@link ByteProvider} to read
* @return list of text lines
* @throws IOException if error
* @throws CancelledException if cancelled
*/
public static String getStreamMD5(InputStream is, TaskMonitor monitor)
throws IOException, CancelledException {
StreamCopyResult results = streamCopy(is, ByteStreams.nullOutputStream(), monitor);
return NumericUtilities.convertBytesToString(results.md5);
public static List<String> getLines(ByteProvider byteProvider) throws IOException {
try (InputStream is = byteProvider.getInputStream(0)) {
return FileUtilities.getLines(is);
}
}
/**
@ -412,7 +400,54 @@ public class FSUtilities {
public static String getFileMD5(File f, TaskMonitor monitor)
throws IOException, CancelledException {
try (FileInputStream fis = new FileInputStream(f)) {
return getStreamMD5(fis, monitor);
monitor.initialize(f.length());
monitor.setMessage("Hashing file: " + f.getName());
return getMD5(fis, monitor);
}
}
/**
* Calculate the MD5 of a file.
*
* @param provider {@link ByteProvider}
* @param monitor {@link TaskMonitor} to watch for cancel
* @return md5 as a hex encoded string, never null.
* @throws IOException if error
* @throws CancelledException if cancelled
*/
public static String getMD5(ByteProvider provider, TaskMonitor monitor)
throws IOException, CancelledException {
try (InputStream is = provider.getInputStream(0)) {
monitor.initialize(provider.length());
monitor.setMessage("Hashing file: " + provider.getName());
return getMD5(is, monitor);
}
}
/**
* Calculate the hash of an {@link InputStream}.
*
* @param is {@link InputStream}
* @param monitor {@link TaskMonitor} to update
* @return md5 as a hex encoded string, never null
* @throws IOException if error
* @throws CancelledException if cancelled
*/
public static String getMD5(InputStream is, TaskMonitor monitor)
throws IOException, CancelledException {
try {
MessageDigest messageDigest = MessageDigest.getInstance(HashUtilities.MD5_ALGORITHM);
byte[] buf = new byte[16 * 1024];
int bytesRead;
while ((bytesRead = is.read(buf)) >= 0) {
messageDigest.update(buf, 0, bytesRead);
monitor.incrementProgress(bytesRead);
monitor.checkCanceled();
}
return NumericUtilities.convertBytesToString(messageDigest.digest());
}
catch (NoSuchAlgorithmException e) {
throw new IOException(e);
}
}
@ -534,4 +569,22 @@ public class FSUtilities {
: "NA";
}
/**
* Helper method to invoke close() on a Closeable without having to catch
* an IOException.
*
* @param c {@link Closeable} to close
* @param msg optional msg to log if exception is thrown, null is okay
*/
public static void uncheckedClose(Closeable c, String msg) {
try {
if (c != null) {
c.close();
}
}
catch (IOException e) {
Msg.warn(FSUtilities.class, Objects.requireNonNullElse(msg, "Problem closing object"),
e);
}
}
}

668
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileCache.java
View File

@ -16,12 +16,14 @@
package ghidra.formats.gfilesystem;
import java.io.*;
import java.nio.file.*;
import java.security.NoSuchAlgorithmException;
import java.util.Date;
import java.util.UUID;
import java.util.*;
import java.util.regex.Pattern;
import ghidra.formats.gfilesystem.FSUtilities.StreamCopyResult;
import org.apache.commons.collections4.map.ReferenceMap;
import ghidra.app.util.bin.*;
import ghidra.util.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@ -31,44 +33,58 @@ import utilities.util.FileUtilities;
* File caching implementation.
* <p>
* Caches files based on a hash of the contents of the file.<br>
* Files are retrieved using the hash string.<p>
* Cached files are stored in a file with a name that is the hex encoded value of the hash.
* Files are retrieved using the hash string.<br>
* Cached files are stored in a file with a name that is the hex encoded value of the hash.<br>
* Cached files are obfuscated/de-obfuscated when written/read to/from disk. See
* {@link ObfuscatedFileByteProvider}, {@link ObfuscatedInputStream},
* {@link ObfuscatedOutputStream}.<br>
* Cached files are organized into a nested directory structure to prevent
* overwhelming a single directory with thousands of files.
* <p>
* Nested directory structure is based on the file's name:
* File: AABBCCDDEEFF...
* Directory (2 level nesting): AA/BB/AABBCCDDEEFF...
* Nested directory structure is based on the file's name:<br>
* <pre> File: AABBCCDDEEFF... &rarr; AA/AABBCCDDEEFF...</pre>
* <p>
* Cache size is not bounded.
* <p>
* Cache maint is done during startup if interval since last maint has been exceeded
* Cache maintenance is done during startup if interval since last maintenance has been exceeded.
* <p>
* No file data is maintained in memory.
* <p>
* No file is moved or removed from the cache after being added (except during startup)
* as there is no use count or reference tracking of the files.
* Files are not removed from the cache after being added, except during startup maintenance.
*
*/
public class FileCache {
/**
* Max size of a file that will be kept in {@link #memCache} (2Mb)
*/
public static final int MAX_INMEM_FILESIZE = 2 * 1024 * 1024; // 2mb
private static final long FREESPACE_RESERVE_BYTES = 50 * 1024 * 1024; // 50mb
private static final Pattern NESTING_DIR_NAME_REGEX = Pattern.compile("[0-9a-fA-F][0-9a-fA-F]");
private static final Pattern FILENAME_REGEX = Pattern.compile("[0-9a-fA-F]{32}");
private static final int MD5_BYTE_LEN = 16;
public static final int MD5_HEXSTR_LEN = MD5_BYTE_LEN * 2;
private static final int NESTING_LEVEL = 2;
private static final long MAX_FILE_AGE_MS = DateUtils.MS_PER_DAY;
private static final long MAINT_INTERVAL_MS = DateUtils.MS_PER_DAY * 2;
private final File cacheDir;
private final FileStore cacheDirFileStore;
private final File newDir;
private final File lastMaintFile;
private FileCacheMaintenanceDaemon cleanDaemon;
private ReferenceMap<String, FileCacheEntry> memCache = new ReferenceMap<>();
private int fileAddCount;
private int fileReUseCount;
private long storageEstimateBytes;
private long lastMaintTS;
/**
* Backwards compatible with previous cache directories to age off the files located
* therein.
*
* @param oldCacheDir the old 2-level cache directory
* @deprecated Marked as deprecated to ensure this is removed in a few versions after most
* user's old-style cache dirs have been cleaned up.
*/
@Deprecated(forRemoval = true, since = "10.1")
public static void performCacheMaintOnOldDirIfNeeded(File oldCacheDir) {
if (oldCacheDir.isDirectory()) {
performCacheMaintIfNeeded(oldCacheDir, 2 /* old nesting level */);
}
}
/**
* Creates a new {@link FileCache} instance where files are stored under the specified
@ -81,12 +97,13 @@ public class FileCache {
public FileCache(File cacheDir) throws IOException {
this.cacheDir = cacheDir;
this.newDir = new File(cacheDir, "new");
this.lastMaintFile = new File(cacheDir, ".lastmaint");
if ((!cacheDir.exists() && !cacheDir.mkdirs()) || (!newDir.exists() && !newDir.mkdirs())) {
throw new IOException("Unable to initialize cache dir " + cacheDir);
}
performCacheMaintIfNeeded();
cacheDirFileStore = Files.getFileStore(cacheDir.toPath());
cleanDaemon = performCacheMaintIfNeeded(cacheDir, 1 /* current nesting level */);
}
/**
@ -102,22 +119,26 @@ public class FileCache {
FileUtilities.deleteDir(f);
}
}
memCache.clear();
}
/**
* Adds a {@link File} to the cache, returning a {@link FileCacheEntry}.
*
* @param f {@link File} to add to cache.
* @param monitor {@link TaskMonitor} to monitor for cancel and to update progress.
* @return {@link FileCacheEntry} with new File and md5.
* @throws IOException if error
* @throws CancelledException if canceled
*/
public FileCacheEntry addFile(File f, TaskMonitor monitor)
throws IOException, CancelledException {
try (FileInputStream fis = new FileInputStream(f)) {
return addStream(fis, monitor);
synchronized boolean hasEntry(String md5) {
FileCacheEntry fce = memCache.get(md5);
if (fce == null) {
fce = getFileByMD5(md5);
}
return fce != null;
}
private void ensureAvailableSpace(long sizeHint) throws IOException {
if ( sizeHint > MAX_INMEM_FILESIZE ) {
long usableSpace = cacheDirFileStore.getUsableSpace();
if (usableSpace >= 0 && usableSpace < sizeHint + FREESPACE_RESERVE_BYTES) {
throw new IOException("Not enough storage available in " + cacheDir +
" to store file sized: " + sizeHint);
}
}
}
/**
@ -130,12 +151,26 @@ public class FileCache {
* @return {@link FileCacheEntry} with a File and it's md5 string or {@code null} if no
* matching file exists in cache.
*/
public FileCacheEntry getFile(String md5) {
FileCacheEntry cfi = getFileByMD5(md5);
if (cfi != null) {
cfi.file.setLastModified(System.currentTimeMillis());
synchronized FileCacheEntry getFileCacheEntry(String md5) {
if (md5 == null) {
return null;
}
FileCacheEntry fce = memCache.get(md5);
if (fce == null) {
fce = getFileByMD5(md5);
if (fce != null) {
fce.file.setLastModified(System.currentTimeMillis());
}
}
return fce;
}
synchronized void releaseFileCacheEntry(String md5) {
FileCacheEntry fce = memCache.get(md5);
if (fce != null) {
memCache.remove(md5);
Msg.debug(this, "Releasing memCache entry: " + fce.md5 + ", " + fce.bytes.length);
}
return cfi;
}
/**
@ -150,165 +185,53 @@ public class FileCache {
}
/**
* Prunes cache if interval since last maintenance exceeds {@link #MAINT_INTERVAL_MS}
* <p>
* Only called during construction, and the only known multi-process conflict that can occur
* is when re-writing the "lastMaint" timestamp file, which isn't a problem as its the
* approximate timestamp of that file that is important, not the contents.
*
* @throws IOException if error when writing metadata file.
* Creates a randomly generated file name in the temp directory.
*
* @return randomly generated file name in the cache's temp directory
*/
private void performCacheMaintIfNeeded() throws IOException {
lastMaintTS = (lastMaintTS == 0) ? lastMaintFile.lastModified() : lastMaintTS;
if (lastMaintTS + MAINT_INTERVAL_MS > System.currentTimeMillis()) {
return;
}
cleanDaemon = new FileCacheMaintenanceDaemon();
cleanDaemon.start();
private File createTempFile() {
return new File(newDir, UUID.randomUUID().toString());
}
/**
* Prunes files in cache if they are old, calculates space used by cache.
*/
private void performCacheMaint() {
storageEstimateBytes = 0;
Msg.info(this, "Starting cache cleanup: " + cacheDir);
// TODO: add check for orphan files in ./new
cacheMaintForDir(cacheDir, 0);
Msg.info(this, "Finished cache cleanup, estimated storage used: " + storageEstimateBytes);
}
private void cacheMaintForDir(File dir, int nestingLevel) {
if (nestingLevel < NESTING_LEVEL) {
for (File f : dir.listFiles()) {
String name = f.getName();
if (f.isDirectory() && NESTING_DIR_NAME_REGEX.matcher(name).matches()) {
cacheMaintForDir(f, nestingLevel + 1);
}
}
}
else if (nestingLevel == NESTING_LEVEL) {
cacheMaintForLeafDir(dir);
}
}
private void cacheMaintForLeafDir(File dir) {
long cutoffMS = System.currentTimeMillis() - MAX_FILE_AGE_MS;
for (File f : dir.listFiles()) {
if (f.isFile() && isCacheFileName(f.getName())) {
if (f.lastModified() < cutoffMS) {
if (!f.delete()) {
Msg.error(this, "Failed to delete cache file " + f);
}
else {
Msg.info(this, "Expired cache file " + f);
continue;
}
}
storageEstimateBytes += f.length();
}
}
}
private boolean isCacheFileName(String s) {
try {
byte[] bytes = NumericUtilities.convertStringToBytes(s);
return (bytes != null) && bytes.length == MD5_BYTE_LEN;
}
catch (IllegalArgumentException e) {
return false;
}
}
/**
* Adds a contents of a stream to the cache, returning the md5 identifier of the stream.
* <p>
* The stream is copied into a temp file in the cacheDir/new directory while its md5
* is calculated. The temp file is then moved into its final location
* based on the md5 of the stream: AA/BB/AABBCCDDEEFF....
* <p>
* The monitor progress is updated with the number of bytes that are being copied. No
* message or maximum is set.
* <p>
* @param is {@link InputStream} to add to the cache. Not closed when done.
* @param monitor {@link TaskMonitor} that will be checked for canceling and updating progress.
* @return {@link FileCacheEntry} with file info and md5, never null.
* Creates a new {@link FileCacheEntryBuilder} that will accept bytes written to it
* (via its {@link OutputStream} methods). When finished writing, the {@link FileCacheEntryBuilder}
* will give the caller a {@link FileCacheEntry}.
*
* @param sizeHint a hint about the size of the file being added. Use -1 if unsure or unknown
* @return new {@link FileCacheEntryBuilder}
* @throws IOException if error
* @throws CancelledException if canceled
*/
public FileCacheEntry addStream(InputStream is, TaskMonitor monitor)
throws IOException, CancelledException {
File tmpFile = new File(newDir, UUID.randomUUID().toString());
try (FileOutputStream fos = new FileOutputStream(tmpFile)) {
StreamCopyResult copyResults = FSUtilities.streamCopy(is, fos, monitor);
// Close the fos so the tmpFile can be moved even though
// the try(){} will attempt to close it as well.
fos.close();
String md5 = NumericUtilities.convertBytesToString(copyResults.md5);
return addTmpFileToCache(tmpFile, md5, copyResults.bytesCopied);
}
finally {
if (tmpFile.exists()) {
Msg.debug(this, "Removing left-over temp file " + tmpFile);
tmpFile.delete();
}
}
FileCacheEntryBuilder createCacheEntryBuilder(long sizeHint) throws IOException {
ensureAvailableSpace(sizeHint);
return new FileCacheEntryBuilder(sizeHint);
}
/**
* Adds a file to the cache, using a 'pusher' strategy where the producer is given a
* {@link OutputStream} to write to.
* Adds a plaintext file to this cache, consuming it.
* <p>
* Unbeknownst to the producer, but knownst to us, the outputstream is really a
* {@link HashingOutputStream} that will allow us to get the MD5 hash when the producer
* is finished pushing.
*
* @param pusher functional callback that will accept an {@link OutputStream} and write
* to it.
* <pre> (os) -&gt; { os.write(.....); }</pre>
* @param monitor {@link TaskMonitor} that will be checked for cancel and updated with
* file io progress.
* @return a new {@link FileCacheEntry} with the newly added cache file's File and MD5,
* never null.
* @throws IOException if an IO error
* @throws CancelledException if the user cancels
* @param file plaintext file
* @param monitor {@link TaskMonitor}
* @return a {@link FileCacheEntry} that controls the contents of the newly added file
* @throws IOException if error
* @throws CancelledException if cancelled
*/
public FileCacheEntry pushStream(DerivedFilePushProducer pusher, TaskMonitor monitor)
throws IOException, CancelledException {
File tmpFile = new File(newDir, UUID.randomUUID().toString());
try (HashingOutputStream hos =
new HashingOutputStream(new FileOutputStream(tmpFile), "MD5")) {
pusher.push(hos);
// early hos.close() so it can be renamed/moved on the filesystem
hos.close();
String md5 = NumericUtilities.convertBytesToString(hos.getDigest());
long fileSize = tmpFile.length();
return addTmpFileToCache(tmpFile, md5, fileSize);
}
catch (NoSuchAlgorithmException e) {
throw new IOException("Error getting MD5 algo", e);
}
catch (Throwable th) {
throw new IOException("Error while pushing stream into cache", th);
FileCacheEntry giveFile(File file, TaskMonitor monitor) throws IOException, CancelledException {
try (InputStream fis = new FileInputStream(file);
FileCacheEntryBuilder fceBuilder = createCacheEntryBuilder(file.length())) {
FSUtilities.streamCopy(fis, fceBuilder, monitor);
return fceBuilder.finish();
}
finally {
if (tmpFile.exists()) {
Msg.debug(this, "Removing left-over temp file " + tmpFile);
tmpFile.delete();
if (!file.delete()) {
Msg.warn(this, "Failed to delete temporary file: " + file);
}
}
}
/**
* Adds a File to this cache, consuming the file.
* Adds an already obfuscated File to this cache, consuming the file.
* <p>
* This method makes some assumptions:
* <p>
@ -318,16 +241,14 @@ public class FileCache {
* existence and the attempt to place the file into the directory. Solution: no
* process may remove a nested directory after it has been created.
* 2) The source file is co-located with the cache directory to ensure its on the
* same physical filesystem volume.
* same physical filesystem volume, and is already obfuscated.
* <p>
* @param tmpFile the File to add to the cache
* @param md5 hex string md5 of the file
* @param fileLen the length in bytes of the file being added
* @return a new {@link FileCacheEntry} with the File's location and its md5
* @throws IOException if an file error occurs
*/
private FileCacheEntry addTmpFileToCache(File tmpFile, String md5, long fileLen)
throws IOException {
private FileCacheEntry addTmpFileToCache(File tmpFile, String md5) throws IOException {
String relPath = getCacheRelPath(md5);
File destCacheFile = new File(cacheDir, relPath);
@ -337,91 +258,28 @@ public class FileCache {
throw new IOException("Failed to create cache dir " + destCacheFileDir);
}
boolean moved = false;
boolean reused = false;
if (destCacheFile.exists()) {
reused = true;
try {
tmpFile.renameTo(destCacheFile);
}
else {
moved = tmpFile.renameTo(destCacheFile);
// test again to see if another process was racing us if the rename failed
reused = !moved && destCacheFile.exists();
}
if (!moved && reused) {
//Msg.info(this, "File already exists in cache, reusing: " + destCacheFile);
finally {
tmpFile.delete();
}
else if (!moved) {
throw new IOException("Failed to move " + tmpFile + " to " + destCacheFile);
}
synchronized (this) {
fileAddCount++;
if (reused) {
fileReUseCount++;
destCacheFile.setLastModified(System.currentTimeMillis());
}
else {
storageEstimateBytes += fileLen;
if (!destCacheFile.exists()) {
throw new IOException("Failed to move " + tmpFile + " to " + destCacheFile);
}
}
destCacheFile.setLastModified(System.currentTimeMillis());
return new FileCacheEntry(destCacheFile, md5);
}
private String getCacheRelPath(String md5) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < NESTING_LEVEL; i++) {
sb.append(md5.substring(i * 2, (i + 1) * 2));
sb.append('/');
}
sb.append(md5);
return sb.toString();
return String.format("%s/%s",
md5.substring(0, 2),
md5);
}
@Override
public String toString() {
return "FileCache [cacheDir=" + cacheDir + ", fileAddCount=" + fileAddCount +
", storageEstimateBytes=" + storageEstimateBytes + ", lastMaintTS=" + lastMaintTS + "]";
}
/**
* Number of files added to this cache.
*
* @return Number of files added to this cache
*/
public int getFileAddCount() {
return fileAddCount;
}
/**
* Number of times a file-add was a no-op and the contents were already present
* in the cache.
*
* @return Number of times a file-add was a no-op and the contents were already present
* in the cache.
*/
public int getFileReUseCount() {
return fileReUseCount;
}
/**
* Estimate of the number of bytes in the cache.
*
* @return estimate of the number of bytes in the cache - could be very wrong
*/
public long getStorageEstimateBytes() {
return storageEstimateBytes;
}
/**
* How old (in milliseconds) files must be before being aged-off during cache maintenance.
*
* @return Max cache file age in milliseconds.
*/
public long getMaxFileAgeMS() {
return MAX_FILE_AGE_MS;
return "FileCache [cacheDir=" + cacheDir + "]";
}
/**
@ -433,19 +291,53 @@ public class FileCache {
return cleanDaemon != null && cleanDaemon.isAlive();
}
private class FileCacheMaintenanceDaemon extends Thread {
/**
* Prunes cache if interval since last maintenance exceeds {@link #MAINT_INTERVAL_MS}
* <p>
* Only called during construction, and the only known multi-process conflict that can occur
* is when re-writing the "lastMaint" timestamp file, which isn't a problem as its the
* approximate timestamp of that file that is important, not the contents.
*
* @param cacheDir cache directory location
* @param nestingLevel the depth of directory nesting, 2 for old style, 1 for newer style
* @return {@link FileCacheMaintenanceDaemon} instance if started, null otherwise
*/
private static FileCacheMaintenanceDaemon performCacheMaintIfNeeded(File cacheDir,
int nestingLevel) {
File lastMaintFile = new File(cacheDir, ".lastmaint");
long lastMaintTS = lastMaintFile.isFile() ? lastMaintFile.lastModified() : 0;
if (lastMaintTS + MAINT_INTERVAL_MS > System.currentTimeMillis()) {
return null;
}
FileCacheMaintenanceDaemon() {
FileCacheMaintenanceDaemon cleanDaemon =
new FileCacheMaintenanceDaemon(cacheDir, lastMaintFile, nestingLevel);
cleanDaemon.start();
return cleanDaemon;
}
private static class FileCacheMaintenanceDaemon extends Thread {
private File lastMaintFile;
private File cacheDir;
private long storageEstimateBytes;
private int nestingLevel;
FileCacheMaintenanceDaemon(File cacheDir, File lastMaintFile, int nestingLevel) {
setDaemon(true);
setName("FileCacheMaintenanceDaemon for " + cacheDir.getName());
this.cacheDir = cacheDir;
this.lastMaintFile = lastMaintFile;
this.nestingLevel = nestingLevel;
}
@Override
public void run() {
performCacheMaint();
Msg.info(this, "Starting cache cleanup: " + cacheDir);
cacheMaintForDir(cacheDir, 0);
Msg.info(this,
"Finished cache cleanup, estimated storage used: " + storageEstimateBytes);
// stamp the file after we finish, in case the VM stopped this daemon thread
lastMaintTS = System.currentTimeMillis();
try {
FileUtilities.writeStringToFile(lastMaintFile, "Last maint run at " + (new Date()));
}
@ -453,5 +345,255 @@ public class FileCache {
Msg.error(this, "Unable to write file cache maintenance file: " + lastMaintFile, e);
}
}
private void cacheMaintForDir(File dir, int dirLevel) {
if (dirLevel < nestingLevel) {
for (File f : dir.listFiles()) {
String name = f.getName();
if (f.isDirectory() && NESTING_DIR_NAME_REGEX.matcher(name).matches()) {
cacheMaintForDir(f, dirLevel + 1);
}
}
}
else if (dirLevel == nestingLevel) {
cacheMaintForLeafDir(dir);
}
}
private void cacheMaintForLeafDir(File dir) {
long cutoffMS = System.currentTimeMillis() - MAX_FILE_AGE_MS;
for (File f : dir.listFiles()) {
if (f.isFile() && isCacheFileName(f.getName())) {
if (f.lastModified() < cutoffMS) {
if (f.delete()) {
Msg.debug(this, "Expired cache file " + f);
continue;
}
Msg.error(this, "Failed to delete cache file " + f);
}
storageEstimateBytes += f.length();
}
}
}
private boolean isCacheFileName(String s) {
return FILENAME_REGEX.matcher(s).matches();
}
}
/**
* Helper class, keeps a FileCacheEntry pinned while the ByteProvider is alive. When
* the ByteProvider is closed, the FileCacheEntry is allowed to be garbage collected
* if there is enough memory pressure to also remove its entry from the {@link FileCache#memCache}
* map.
*/
private static class RefPinningByteArrayProvider extends ByteArrayProvider {
@SuppressWarnings("unused")
private FileCacheEntry fce; // its just here to be pinned in memory
public RefPinningByteArrayProvider(FileCacheEntry fce, FSRL fsrl) {
super(fce.bytes, fsrl);
this.fce = fce;
}
@Override
public void close() {
fce = null;
super.hardClose();
}
}
/**
* Allows creating {@link FileCacheEntry file cache entries} at the caller's convenience.
* <p>
*/
public class FileCacheEntryBuilder extends OutputStream {
private OutputStream delegate;
private HashingOutputStream hos;
private FileCacheEntry fce;
private long delegateLength;
private File tmpFile;
private FileCacheEntryBuilder(long sizeHint) throws IOException {
sizeHint = sizeHint <= 0 ? 512 : sizeHint;
if (sizeHint < MAX_INMEM_FILESIZE) {
delegate = new ByteArrayOutputStream((int) sizeHint);
}
else {
tmpFile = createTempFile();
delegate = new ObfuscatedOutputStream(new FileOutputStream(tmpFile));
}
initHashingOutputStream();
}
@Override
protected void finalize() throws Throwable {
if (hos != null) {
Msg.warn(this, "FAIL TO CLOSE FileCacheEntryBuilder, currentSize=" +
delegateLength + ", file=" + (tmpFile != null ? tmpFile : "not set"));
}
}
@Override
public void write(int b) throws IOException {
switchToTempFileIfNecessary(1);
hos.write(b);
}
@Override
public void write(byte[] b) throws IOException {
switchToTempFileIfNecessary(b.length);
hos.write(b);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
switchToTempFileIfNecessary(len);
hos.write(b, off, len);
}
@Override
public void flush() throws IOException {
hos.flush();
}
@Override
public void close() throws IOException {
finish();
}
private void initHashingOutputStream() throws IOException {
try {
hos = new HashingOutputStream(delegate, HashUtilities.MD5_ALGORITHM);
}
catch (NoSuchAlgorithmException e) {
throw new IOException("Error getting MD5 algo", e);
}
}
private void switchToTempFileIfNecessary(int bytesToAdd) throws IOException {
delegateLength += bytesToAdd;
if (tmpFile == null && delegateLength > MAX_INMEM_FILESIZE) {
tmpFile = createTempFile();
byte[] bytes = ((ByteArrayOutputStream) delegate).toByteArray();
delegate = new ObfuscatedOutputStream(new FileOutputStream(tmpFile));
initHashingOutputStream();
// send the old bytes through the new hasher and to the tmp file
hos.write(bytes);
}
}
/**
* Finalizes this builder, pushing the bytes that have been written to it into
* the FileCache.
* <p>
* @return new {@link FileCacheEntry}
* @throws IOException if error
*/
public FileCacheEntry finish() throws IOException {
if (hos != null) {
hos.close();
String md5 = NumericUtilities.convertBytesToString(hos.getDigest());
if (tmpFile != null) {
fce = addTmpFileToCache(tmpFile, md5);
}
else {
ByteArrayOutputStream baos = (ByteArrayOutputStream) delegate;
byte[] bytes = baos.toByteArray();
fce = new FileCacheEntry(bytes, md5);
synchronized (FileCache.this) {
memCache.put(md5, fce);
}
}
hos = null;
delegate = null;
}
return fce;
}
}
/**
* Represents a cached file. It may be an actual file if {@link FileCacheEntry#file file}
* is set, or if smaller than {@link FileCache#MAX_INMEM_FILESIZE 2Mb'ish} just an
* in-memory byte array that is weakly pinned in the {@link FileCache#memCache} map.
*/
public static class FileCacheEntry {
final String md5;
final File file;
final byte[] bytes;
private FileCacheEntry(File file, String md5) {
this.file = file;
this.bytes = null;
this.md5 = md5;
}
private FileCacheEntry(byte[] bytes, String md5) {
this.file = null;
this.bytes = bytes;
this.md5 = md5;
}
/**
* Returns the contents of this cache entry as a {@link ByteProvider}, using the specified
* {@link FSRL}.
* <p>
* @param fsrl {@link FSRL} that the returned {@link ByteProvider} should have as its
* identity
* @return new {@link ByteProvider} containing the contents of this cache entry, caller is
* responsible for {@link ByteProvider#close() closing}
* @throws IOException if error
*/
public ByteProvider asByteProvider(FSRL fsrl) throws IOException {
if (fsrl.getMD5() == null) {
fsrl = fsrl.withMD5(md5);
}
if (file != null) {
file.setLastModified(System.currentTimeMillis());
}
return (bytes != null)
? new RefPinningByteArrayProvider(this, fsrl)
: new ObfuscatedFileByteProvider(file, fsrl, AccessMode.READ);
}
/**
* Returns the MD5 of this cache entry.
*
* @return the MD5 (as a string) of this cache entry
*/
public String getMD5() {
return md5;
}
public long length() {
return bytes != null ? bytes.length : file.length();
}
@Override
public int hashCode() {
return Objects.hash(md5);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
FileCacheEntry other = (FileCacheEntry) obj;
return Objects.equals(md5, other.md5);
}
}
}

131
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileFingerprintCache.java
View File

@ -1,131 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem;
import org.apache.commons.collections4.map.ReferenceMap;
/**
* A best-effort cache of MD5 values of local files based on their {name,timestamp,length} fingerprint.
* <p>
* Used to quickly verify that a local file hasn't changed.
*
*/
public class FileFingerprintCache {
private ReferenceMap<FileFingerprintRec, String> fileFingerprintToMD5Map = new ReferenceMap<>();
/**
* Clears the cache.
*/
public synchronized void clear() {
fileFingerprintToMD5Map.clear();
}
/**
* Add a file's fingerprint to the cache.
*
* @param path String path to the file
* @param md5 hex-string md5 of the file
* @param timestamp long last modified timestamp of the file
* @param length long file size
*/
public synchronized void add(String path, String md5, long timestamp, long length) {
fileFingerprintToMD5Map.put(new FileFingerprintRec(path, timestamp, length), md5);
}
/**
* Returns true if the specified file with the specified fingerprints (timestamp, length)
* was previously added to the cache with the specified md5.
*
* @param path String path to the file
* @param md5 hex-string md5 of the file
* @param timestamp long last modified timestamp of the file
* @param length long file size
* @return true if the fingerprint has previously been added to the cache.
*/
public synchronized boolean contains(String path, String md5, long timestamp, long length) {
String prevMD5 =
fileFingerprintToMD5Map.get(new FileFingerprintRec(path, timestamp, length));
return prevMD5 != null && prevMD5.equals(md5);
}
/**
* Retrieves the md5 for the specified file that has the specified fingerprint (timestamp, length).
*
* @param path String path to the file
* @param timestamp long last modified timestamp of the file
* @param length long file size
* @return hex-string md5 or null if not present in the cache.
*/
public synchronized String getMD5(String path, long timestamp, long length) {
String prevMD5 =
fileFingerprintToMD5Map.get(new FileFingerprintRec(path, timestamp, length));
return prevMD5;
}
//-----------------------------------------------------------------------------------
static class FileFingerprintRec {
final String path;
final long timestamp;
final long length;
FileFingerprintRec(String path, long timestamp, long length) {
this.path = path;
this.timestamp = timestamp;
this.length = length;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (length ^ (length >>> 32));
result = prime * result + ((path == null) ? 0 : path.hashCode());
result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof FileFingerprintRec)) {
return false;
}
FileFingerprintRec other = (FileFingerprintRec) obj;
if (length != other.length) {
return false;
}
if (path == null) {
if (other.path != null) {
return false;
}
}
else if (!path.equals(other.path)) {
return false;
}
if (timestamp != other.timestamp) {
return false;
}
return true;
}
}
}

270
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemIndexHelper.java
View File

@ -15,14 +15,17 @@
*/
package ghidra.formats.gfilesystem;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import ghidra.util.Msg;
/**
* A helper class used by GFilesystem implementors to track mappings between GFile
* instances and the underlying container filesystem's native file objects.
* <p>
* Threadsafe after initial use of {@link #storeFile(String, int, boolean, long, Object) storeFile()}
* by the owning filesystem.
* Threadsafe (methods are synchronized).
* <p>
* This class also provides filename 'unique-ifying' (per directory) where an auto-incrementing
* number will be added to a file's filename if it is not unique in the directory.
@ -33,9 +36,16 @@ import java.util.*;
public class FileSystemIndexHelper<METADATATYPE> {
private GFile rootDir;
static class FileData<METADATATYPE> {
GFile file;
METADATATYPE metaData;
long fileIndex;
}
protected Map<GFile, METADATATYPE> fileToEntryMap = new HashMap<>();
protected Map<GFile, Map<String, GFile>> directoryToListing = new HashMap<>();
protected Map<GFile, FileData<METADATATYPE>> fileToEntryMap = new HashMap<>();
protected Map<Long, FileData<METADATATYPE>> fileIndexToEntryMap = new HashMap<>();
protected Map<GFile, Map<String, FileData<METADATATYPE>>> directoryToListing = new HashMap<>();
/**
* Creates a new {@link FileSystemIndexHelper} for the specified {@link GFileSystem}.
@ -49,6 +59,17 @@ public class FileSystemIndexHelper<METADATATYPE> {
*/
public FileSystemIndexHelper(GFileSystem fs, FSRLRoot fsFSRL) {
this.rootDir = GFileImpl.fromFSRL(fs, null, fsFSRL.withPath("/"), true, -1);
initRootDir(null);
}
private void initRootDir(METADATATYPE metadata) {
FileData<METADATATYPE> fileData = new FileData<>();
fileData.file = rootDir;
fileData.fileIndex = -1;
fileData.metaData = metadata;
fileToEntryMap.put(rootDir, fileData);
directoryToListing.put(rootDir, new HashMap<>());
}
/**
@ -63,7 +84,7 @@ public class FileSystemIndexHelper<METADATATYPE> {
/**
* Removes all file info from this index.
*/
public void clear() {
public synchronized void clear() {
fileToEntryMap.clear();
directoryToListing.clear();
}
@ -71,56 +92,84 @@ public class FileSystemIndexHelper<METADATATYPE> {
/**
* Number of files in this index.
*
* @return number of file in this index.
* @return number of file in this index
*/
public int getFileCount() {
public synchronized int getFileCount() {
return fileToEntryMap.size();
}
/**
* Gets the opaque filesystem specific blob that was associated with the specified file.
*
* @param f {@link GFile} to look for.
* @return Filesystem specific blob associated with the specified file, or null if not found.
* @param f {@link GFile} to look for
* @return Filesystem specific blob associated with the specified file, or null if not found
*/
public METADATATYPE getMetadata(GFile f) {
return fileToEntryMap.get(f);
public synchronized METADATATYPE getMetadata(GFile f) {
FileData<METADATATYPE> fileData = fileToEntryMap.get(f);
return fileData != null ? fileData.metaData : null;
}
/**
* Sets the associated metadata blob for the specified file.
*
* @param f GFile to update
* @param metaData new metdata blob
* @throws IOException if unknown file
*/
public synchronized void setMetadata(GFile f, METADATATYPE metaData) throws IOException {
FileData<METADATATYPE> fileData = fileToEntryMap.get(f);
if ( fileData == null ) {
throw new IOException("Unknown file: " + f);
}
fileData.metaData = metaData;
}
/**
* Gets the GFile instance that was associated with the filesystem file index.
*
* @param fileIndex index of the file in its filesystem
* @return the associated GFile instance, or null if not found
*/
public synchronized GFile getFileByIndex(long fileIndex) {
FileData<METADATATYPE> fileData = fileIndexToEntryMap.get(fileIndex);
return (fileData != null) ? fileData.file : null;
}
/**
* Mirror's {@link GFileSystem#getListing(GFile)} interface.
*
* @param directory {@link GFile} directory to get the list of child files that have been
* added to this index, null means root directory.
* @return {@link List} of GFile files that are in the specified directory, never null.
* added to this index, null means root directory
* @return {@link List} of GFile files that are in the specified directory, never null
*/
public List<GFile> getListing(GFile directory) {
Map<String, GFile> dirListing = getDirectoryContents(directory, false);
List<GFile> results =
(dirListing != null) ? new ArrayList<>(dirListing.values()) : Collections.emptyList();
return results;
public synchronized List<GFile> getListing(GFile directory) {
Map<String, FileData<METADATATYPE>> dirListing = getDirectoryContents(directory, false);
if (dirListing == null) {
return List.of();
}
return dirListing.values()
.stream()
.map(fd -> fd.file)
.collect(Collectors.toList());
}
/**
* Mirror's {@link GFileSystem#lookup(String)} interface.
*
* @param path path and filename of a file to find.
* @return {@link GFile} instance or null if no file was added to the index at that path.
* @param path path and filename of a file to find
* @return {@link GFile} instance or null if no file was added to the index at that path
*/
public GFile lookup(String path) {
public synchronized GFile lookup(String path) {
String[] nameparts = (path != null ? path : "").split("/");
GFile parent = lookupParent(nameparts);
if (nameparts.length == 0) {
return parent;
}
String name = nameparts[nameparts.length - 1];
String name = (nameparts.length > 0) ? nameparts[nameparts.length - 1] : null;
if (name == null || name.isEmpty()) {
return parent;
}
Map<String, GFile> dirListing = getDirectoryContents(parent, false);
return (dirListing != null) ? dirListing.get(name) : null;
Map<String, FileData<METADATATYPE>> dirListing = getDirectoryContents(parent, false);
FileData<METADATATYPE> fileData = (dirListing != null) ? dirListing.get(name) : null;
return (fileData != null) ? fileData.file : null;
}
/**
@ -133,38 +182,27 @@ public class FileSystemIndexHelper<METADATATYPE> {
* suffix added to the resultant GFile name, where nnn is the file's
* order of occurrence in the container file.
* <p>
*
* @param path string path and filename of the file being added to the index. Back
* slashes are normalized to forward slashes.
* slashes are normalized to forward slashes
* @param fileIndex the filesystem specific unique index for this file, or -1
* if not available.
* if not available
* @param isDirectory boolean true if the new file is a directory
* @param length number of bytes in the file or -1 if not known or directory.
* @param fileInfo opaque blob that will be stored and associated with the new
* GFile instance.
* @return new GFile instance.
* @param length number of bytes in the file or -1 if not known or directory
* @param metadata opaque blob that will be stored and associated with the new
* GFile instance
* @return new GFile instance
*/
public GFileImpl storeFile(String path, int fileIndex, boolean isDirectory, long length,
METADATATYPE fileInfo) {
public synchronized GFile storeFile(String path, long fileIndex, boolean isDirectory,
long length, METADATATYPE metadata) {
String[] nameparts = path.replaceAll("[\\\\]", "/").split("/");
GFile parent = lookupParent(nameparts);
int fileNum = (fileIndex != -1) ? fileIndex : fileToEntryMap.size();
String lastpart = nameparts[nameparts.length - 1];
Map<String, GFile> dirContents = getDirectoryContents(parent, true);
String uniqueName = dirContents.containsKey(lastpart) && !isDirectory
? lastpart + "[" + Integer.toString(fileNum) + "]"
: lastpart;
GFileImpl file = createNewFile(parent, uniqueName, isDirectory, length, fileInfo);
dirContents.put(uniqueName, file);
if (file.isDirectory()) {
getDirectoryContents(file, true);
}
fileToEntryMap.put(file, fileInfo);
return file;
FileData<METADATATYPE> fileData =
doStoreFile(lastpart, parent, fileIndex, isDirectory, length, metadata);
return fileData.file;
}
/**
@ -176,47 +214,81 @@ public class FileSystemIndexHelper<METADATATYPE> {
* suffix added to the resultant GFile name, where nnn is the file's
* order of occurrence in the container file.
* <p>
*
* @param filename the new file's name
* @param parent the new file's parent directory
* @param fileIndex the filesystem specific unique index for this file, or -1
* if not available.
* if not available
* @param isDirectory boolean true if the new file is a directory
* @param length number of bytes in the file or -1 if not known or directory.
* @param fileInfo opaque blob that will be stored and associated with the new
* GFile instance.
* @return new GFile instance.
* @param length number of bytes in the file or -1 if not known or directory
* @param metadata opaque blob that will be stored and associated with the new
* GFile instance
* @return new GFile instance
*/
public GFile storeFileWithParent(String filename, GFile parent, int fileIndex,
boolean isDirectory, long length, METADATATYPE fileInfo) {
public synchronized GFile storeFileWithParent(String filename, GFile parent, long fileIndex,
boolean isDirectory, long length, METADATATYPE metadata) {
FileData<METADATATYPE> fileData =
doStoreFile(filename, parent, fileIndex, isDirectory, length, metadata);
return fileData.file;
}
private FileData<METADATATYPE> doStoreMissingDir(String filename, GFile parent) {
parent = (parent == null) ? rootDir : parent;
int fileNum = (fileIndex != -1) ? fileIndex : fileToEntryMap.size();
Map<String, GFile> dirContents = getDirectoryContents(parent, true);
String uniqueName = dirContents.containsKey(filename) && !isDirectory
? filename + "[" + Integer.toString(fileNum) + "]"
: filename;
Map<String, FileData<METADATATYPE>> dirContents = getDirectoryContents(parent, true);
GFile file = createNewFile(parent, filename, true, -1, null);
GFile file = createNewFile(parent, uniqueName, isDirectory, length, fileInfo);
FileData<METADATATYPE> fileData = new FileData<>();
fileData.file = file;
fileData.fileIndex = -1;
fileToEntryMap.put(file, fileData);
dirContents.put(filename, fileData);
getDirectoryContents(file, true);
dirContents.put(uniqueName, file);
if (file.isDirectory()) {
return fileData;
}
private FileData<METADATATYPE> doStoreFile(String filename, GFile parent, long fileIndex,
boolean isDirectory, long length, METADATATYPE metadata) {
parent = (parent == null) ? rootDir : parent;
long fileNum = (fileIndex != -1) ? fileIndex : fileToEntryMap.size();
if (fileIndexToEntryMap.containsKey(fileNum)) {
Msg.warn(this, "Duplicate fileNum for file " + parent.getPath() + "/" + filename);
}
Map<String, FileData<METADATATYPE>> dirContents = getDirectoryContents(parent, true);
String uniqueName = makeUniqueFilename(dirContents.containsKey(filename) && !isDirectory,
filename, fileNum);
GFile file = createNewFile(parent, uniqueName, isDirectory, length, metadata);
FileData<METADATATYPE> fileData = new FileData<>();
fileData.file = file;
fileData.fileIndex = fileNum;
fileData.metaData = metadata;
fileToEntryMap.put(file, fileData);
fileIndexToEntryMap.put(fileNum, fileData);
dirContents.put(uniqueName, fileData);
if (isDirectory) {
// side-effect of get will eagerly create the directorylisting entry
getDirectoryContents(file, true);
}
fileToEntryMap.put(file, fileInfo);
return file;
return fileData;
}
/**
* Returns a string-&gt;GFile map that holds the contents of a single directory.
* @param directoryFile
* @return
*/
protected Map<String, GFile> getDirectoryContents(GFile directoryFile,
private String makeUniqueFilename(boolean wasNameCollision, String filename, long fileIndex) {
return wasNameCollision
? filename + "[" + Long.toString(fileIndex) + "]"
: filename;
}
private Map<String, FileData<METADATATYPE>> getDirectoryContents(GFile directoryFile,
boolean createIfMissing) {
directoryFile = (directoryFile != null) ? directoryFile : rootDir;
Map<String, GFile> dirContents = directoryToListing.get(directoryFile);
Map<String, FileData<METADATATYPE>> dirContents = directoryToListing.get(directoryFile);
if (dirContents == null && createIfMissing) {
dirContents = new HashMap<>();
directoryToListing.put(directoryFile, dirContents);
@ -242,23 +314,21 @@ public class FileSystemIndexHelper<METADATATYPE> {
protected GFile lookupParent(String[] nameparts) {
GFile currentDir = rootDir;
GFile currentFile = rootDir;
for (int i = 0; i < nameparts.length - 1; i++) {
Map<String, GFile> currentDirContents = getDirectoryContents(currentDir, true);
Map<String, FileData<METADATATYPE>> currentDirContents =
getDirectoryContents(currentDir, true);
String name = nameparts[i];
if (name.isEmpty()) {
continue;
}
currentFile = currentDirContents.get(name);
if (currentFile == null) {
currentFile = createNewFile(currentDir, name, true, -1, null);
currentDirContents.put(name, currentFile);
getDirectoryContents(currentFile, true);
FileData<METADATATYPE> fileData = currentDirContents.get(name);
if (fileData == null) {
fileData = doStoreMissingDir(name, currentDir);
}
currentDir = currentFile;
currentDir = fileData.file;
}
return currentFile;
return currentDir;
}
/**
@ -282,6 +352,38 @@ public class FileSystemIndexHelper<METADATATYPE> {
size);
}
/**
* Updates the FSRL of a file already in the index.
*
* @param file current {@link GFile}
* @param newFSRL the new FSRL the new file will be given
*/
public synchronized void updateFSRL(GFile file, FSRL newFSRL) {
GFileImpl newFile = GFileImpl.fromFSRL(rootDir.getFilesystem(), file.getParentFile(),
newFSRL, file.isDirectory(), file.getLength());
FileData<METADATATYPE> fileData = fileToEntryMap.get(file);
if (fileData != null) {
fileToEntryMap.remove(file);
fileIndexToEntryMap.remove(fileData.fileIndex);
fileData.file = newFile;
fileToEntryMap.put(newFile, fileData);
if (fileData.fileIndex != -1) {
fileIndexToEntryMap.put(fileData.fileIndex, fileData);
}
}
Map<String, FileData<METADATATYPE>> dirListing = directoryToListing.get(file);
if ( dirListing != null) {
// typically this shouldn't ever happen as directory entries don't have MD5s and won't need to be updated
// after the fact
directoryToListing.remove(file);
directoryToListing.put(newFile, dirListing);
}
}
@Override
public String toString() {
return "FileSystemIndexHelper for " + rootDir.getFilesystem();

25
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemCache.java → Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemInstanceManager.java
View File

@ -27,7 +27,7 @@ import ghidra.util.Msg;
* Any filesystems that are not referenced by outside users (via a {@link FileSystemRef}) will
* be closed and removed from the cache when the next {@link #cacheMaint()} is performed.
*/
public class FileSystemCache implements FileSystemEventListener {
class FileSystemInstanceManager implements FileSystemEventListener {
private static class FSCacheInfo {
FileSystemRef ref;
@ -47,7 +47,7 @@ public class FileSystemCache implements FileSystemEventListener {
* @param rootFS reference to the global root file system, which is a special case
* file system that is not subject to eviction.
*/
public FileSystemCache(GFileSystem rootFS) {
public FileSystemInstanceManager(GFileSystem rootFS) {
this.rootFS = rootFS;
this.rootFSRL = rootFS.getFSRL();
}
@ -190,7 +190,7 @@ public class FileSystemCache implements FileSystemEventListener {
continue;
}
if (fsContainer.equals(containerFSRL)) {
if (containerFSRL.isEquivalent(fsContainer)) {
return ref.dup();
}
}
@ -267,4 +267,23 @@ public class FileSystemCache implements FileSystemEventListener {
Msg.error(this, "Error closing filesystem", e);
}
}
/**
* Closes the specified ref, and if no other refs to the file system remain, closes the file system.
*
* @param ref {@link FileSystemRef} to close
*/
public synchronized void releaseImmediate(FileSystemRef ref) {
FSCacheInfo fsci = filesystems.get(ref.getFilesystem().getFSRL());
ref.close();
if (fsci == null) {
Msg.warn(this, "Unknown file system reference: " + ref.getFilesystem().getFSRL());
return;
}
FileSystemRefManager refManager = fsci.ref.getFilesystem().getRefManager();
if (refManager.canClose(fsci.ref)) {
release(fsci);
}
}
}

9
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemRefManager.java
View File

@ -15,13 +15,13 @@
*/
package ghidra.formats.gfilesystem;
import java.util.ArrayList;
import java.util.List;
import ghidra.util.Msg;
import ghidra.util.datastruct.WeakDataStructureFactory;
import ghidra.util.datastruct.WeakSet;
import java.util.ArrayList;
import java.util.List;
/**
* A threadsafe helper class that manages creating and releasing {@link FileSystemRef} instances
* and broadcasting events to {@link FileSystemEventListener} listeners.
@ -166,7 +166,8 @@ public class FileSystemRefManager {
// where instances are created and thrown away without a close() to probe
// filesystem container files.
if (fs != null && !(fs instanceof GFileSystemBase)) {
Msg.warn(this, "Unclosed FilesytemRefManager for filesystem: " + fs.getClass());
Msg.warn(this, "Unclosed FilesytemRefManager for filesystem: " + fs.getClass() + ", " +
fs.getName());
}
}

914
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileSystemService.java
View File

File diff suppressed because it is too large Load Diff

40
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileHashProvider.java Normal file
View File

@ -0,0 +1,40 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem;
import java.io.IOException;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* GFileSystem add-on interface that provides MD5 hashing for file located within the filesystem
*/
public interface GFileHashProvider {
/**
* Returns the MD5 hash of the specified file.
*
* @param file the {@link GFile}
* @param required boolean flag, if true the hash will always be returned, even if it has to
* be calculated. If false, the hash will be returned if easily available
* @param monitor {@link TaskMonitor}
* @return MD5 hash as a string
* @throws CancelledException if cancelled
* @throws IOException if error
*/
String getMD5Hash(GFile file, boolean required, TaskMonitor monitor)
throws CancelledException, IOException;
}

46
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileImpl.java
View File

@ -15,7 +15,7 @@
*/
package ghidra.formats.gfilesystem;
import ghidra.util.SystemUtilities;
import java.util.Objects;
/**
* Base implementation of file in a {@link GFileSystem filesystem}.
@ -140,11 +140,11 @@ public class GFileImpl implements GFile {
return new GFileImpl(fileSystem, parent, isDirectory, length, fsrl);
}
private GFileSystem fileSystem;
private GFile parentFile;
private boolean isDirectory = false;
private long length = -1;
private FSRL fsrl;
private final GFileSystem fileSystem;
private final GFile parentFile;
private final boolean isDirectory;
private long length;
private final FSRL fsrl;
/**
* Protected constructor, use static helper methods to create new instances.
@ -197,21 +197,6 @@ public class GFileImpl implements GFile {
return getPath();
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof GFile)) {
return false;
}
GFile other = (GFile) obj;
return SystemUtilities.isEqual(fsrl, other.getFSRL()) && isDirectory == other.isDirectory();
}
@Override
public int hashCode() {
return fsrl.hashCode() ^ Boolean.hashCode(isDirectory());
}
@Override
public String getPath() {
return fsrl.getPath();
@ -226,7 +211,22 @@ public class GFileImpl implements GFile {
return fsrl;
}
public void setFSRL(FSRL fsrl) {
this.fsrl = fsrl;
@Override
public int hashCode() {
return Objects.hash(fileSystem, fsrl.getPath(), isDirectory);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (!(obj instanceof GFile)) {
return false;
}
GFile other = (GFile) obj;
return Objects.equals(fileSystem, other.getFilesystem()) &&
Objects.equals(fsrl.getPath(), other.getFSRL().getPath()) &&
isDirectory == other.isDirectory();
}
}

69
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java
View File

@ -15,12 +15,14 @@
*/
package ghidra.formats.gfilesystem;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.*;
import java.util.List;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.bin.ByteProviderInputStream;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.fileinfo.FileAttribute;
import ghidra.formats.gfilesystem.fileinfo.FileAttributes;
import ghidra.util.classfinder.ExtensionPoint;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@ -30,8 +32,9 @@ import ghidra.util.task.TaskMonitor;
* <p>
* Operations take a {@link TaskMonitor} if they need to be cancel-able.
* <p>
* Use {@link FileSystemService} to discover and open instances of filesystems in files or
* to open a known {@link FSRL} path.
* Use a {@link FileSystemService FileSystemService instance} to discover and
* open instances of filesystems in files or to open a known {@link FSRL} path or to
* deal with creating {@link FileSystemService#createTempFile(long) temp files}.
* <p>
* NOTE:<p>
* ALL GFileSystem sub-CLASSES MUST END IN "FileSystem". If not, the ClassSearcher
@ -137,7 +140,24 @@ public interface GFileSystem extends Closeable, ExtensionPoint {
* @throws IOException if IO problem
* @throws CancelledException if user cancels.
*/
public InputStream getInputStream(GFile file, TaskMonitor monitor)
default public InputStream getInputStream(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
return getInputStreamHelper(file, this, monitor);
}
/**
* Returns a {@link ByteProvider} that contains the contents of the specified {@link GFile}.
* <p>
* The caller is responsible for closing the provider.
*
* @param file {@link GFile} to get bytes for
* @param monitor {@link TaskMonitor} to watch and update progress
* @return new {@link ByteProvider} that contains the contents of the file, or NULL if file
* doesn't have data
* @throws IOException if error
* @throws CancelledException if user cancels
*/
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException;
/**
@ -151,17 +171,36 @@ public interface GFileSystem extends Closeable, ExtensionPoint {
public List<GFile> getListing(GFile directory) throws IOException;
/**
* Returns a multi-line string with information about the specified {@link GFile file}.
* Returns a container of {@link FileAttribute} values.
* <p>
* TODO:{@literal this method needs to be refactored to return a Map<String, String>; instead of}
* a pre-formatted multi-line string.
* <p>
* @param file {@link GFile} to get info message for.
* @param monitor {@link TaskMonitor} to watch and update progress.
* @return multi-line formatted string with info about the file, or null.
* Implementors of this method are not required to add FSRL, NAME, or PATH values unless
* the values are non-standard.
*
* @param file {@link GFile} to get the attributes for
* @param monitor {@link TaskMonitor}
* @return {@link FileAttributes} instance (possibly read-only), maybe empty but never null
*/
default public String getInfo(GFile file, TaskMonitor monitor) {
return null;
default public FileAttributes getFileAttributes(GFile file, TaskMonitor monitor) {
return FileAttributes.EMPTY;
}
/**
* Default implementation of getting an {@link InputStream} from a {@link GFile}'s
* {@link ByteProvider}.
* <p>
*
* @param file {@link GFile}
* @param fs the {@link GFileSystem filesystem} containing the file
* @param monitor {@link TaskMonitor} to allow canceling
* @return new {@link InputStream} containing bytes of the file
* @throws CancelledException if canceled
* @throws IOException if error
*/
public static InputStream getInputStreamHelper(GFile file, GFileSystem fs, TaskMonitor monitor)
throws CancelledException, IOException {
ByteProvider bp = fs.getByteProvider(file, monitor);
return (bp != null) ? new ByteProviderInputStream.ClosingInputStream(bp) : null;
}
}

20
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemBase.java
View File

@ -129,26 +129,6 @@ public abstract class GFileSystemBase implements GFileSystem {
@Override
abstract public List<GFile> getListing(GFile directory) throws IOException;
/**
* Legacy implementation of {@link #getInputStream(GFile, TaskMonitor)}.
*
* @param file {@link GFile} to get an InputStream for
* @param monitor {@link TaskMonitor} to watch and update progress
* @return new {@link InputStream} contains the contents of the file or NULL if the
* file doesn't have data.
* @throws IOException if IO problem
* @throws CancelledException if user cancels.
* @throws CryptoException if crypto problem.
*/
abstract protected InputStream getData(GFile file, TaskMonitor monitor)
throws IOException, CancelledException, CryptoException;
@Override
public InputStream getInputStream(GFile file, TaskMonitor monitor)
throws CancelledException, IOException {
return getData(file, monitor);
}
/**
* Writes the given bytes to a tempfile in the temp directory.
* @param bytes the bytes to write

15
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GIconProvider.java
View File

@ -15,18 +15,18 @@
*/
package ghidra.formats.gfilesystem;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import java.awt.Image;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import javax.imageio.ImageIO;
import javax.swing.Icon;
import javax.swing.ImageIcon;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* {@link GFileSystem} add-on interface to allow filesystems to override how image files
* are converted into viewable {@link Icon} instances.
@ -61,9 +61,8 @@ public interface GIconProvider {
return ((GIconProvider) fs).getIcon(file, monitor);
}
File data = FileSystemService.getInstance().getFile(file.getFSRL(), monitor);
try {
Image image = ImageIO.read(data);
try (InputStream is = file.getFilesystem().getInputStream(file, monitor)) {
Image image = ImageIO.read(is);
if (image == null) {
return null;
}

197
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystem.java
View File

@ -15,15 +15,22 @@
*/
package ghidra.formats.gfilesystem;
import static ghidra.formats.gfilesystem.fileinfo.FileAttributeType.*;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.*;
import java.util.*;
import org.apache.commons.collections4.map.ReferenceMap;
import org.apache.commons.io.FilenameUtils;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.bin.FileByteProvider;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.factory.GFileSystemFactory;
import ghidra.formats.gfilesystem.factory.GFileSystemFactoryIgnore;
import ghidra.formats.gfilesystem.fileinfo.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
@ -36,7 +43,7 @@ import ghidra.util.task.TaskMonitor;
* Closing() this filesystem does nothing.
*/
@FileSystemInfo(type = LocalFileSystem.FSTYPE, description = "Local filesystem", factory = GFileSystemFactoryIgnore.class)
public class LocalFileSystem implements GFileSystem {
public class LocalFileSystem implements GFileSystem, GFileHashProvider {
public static final String FSTYPE = "file";
/**
@ -48,9 +55,11 @@ public class LocalFileSystem implements GFileSystem {
return new LocalFileSystem(FSRLRoot.makeRoot(FSTYPE));
}
private final List<GFile> emptyDir = Collections.emptyList();
private final List<GFile> emptyDir = List.of();
private final FSRLRoot fsFSRL;
private final FileSystemRefManager refManager = new FileSystemRefManager(this);
private final ReferenceMap<FileFingerprintRec, String> fileFingerprintToMD5Map =
new ReferenceMap<>();
private LocalFileSystem(FSRLRoot fsrl) {
this.fsFSRL = fsrl;
@ -60,6 +69,21 @@ public class LocalFileSystem implements GFileSystem {
return fsFSRL.equals(fsrl.getFS());
}
/**
* Creates a new file system instance that is a sub-view limited to the specified directory.
*
* @param fsrl {@link FSRL} that must be a directory in this local filesystem
* @return new {@link LocalFileSystemSub} instance
* @throws IOException if bad FSRL
*/
public LocalFileSystemSub getSubFileSystem(FSRL fsrl) throws IOException {
if (isLocalSubdir(fsrl)) {
File localDir = getLocalFile(fsrl);
return new LocalFileSystemSub(localDir, this);
}
return null;
}
/**
* Returns true if the {@link FSRL} is a local filesystem subdirectory.
*
@ -74,6 +98,13 @@ public class LocalFileSystem implements GFileSystem {
return localFile.isDirectory();
}
/**
* Convert a FSRL that points to this file system into a java {@link File}.
*
* @param fsrl {@link FSRL}
* @return {@link File}
* @throws IOException if FSRL does not point to this file system
*/
public File getLocalFile(FSRL fsrl) throws IOException {
if (!isSameFS(fsrl)) {
throw new IOException("FSRL does not specify local file: " + fsrl);
@ -82,6 +113,17 @@ public class LocalFileSystem implements GFileSystem {
return localFile;
}
/**
* Converts a {@link File} into a {@link FSRL}.
*
* @param f {@link File}
* @return {@link FSRL}
*/
public FSRL getLocalFSRL(File f) {
return fsFSRL
.withPath(FSUtilities.appendPath("/", FilenameUtils.separatorsToUnix(f.getPath())));
}
@Override
public String getName() {
return "Root Filesystem";
@ -130,14 +172,48 @@ public class LocalFileSystem implements GFileSystem {
}
@Override
public String getInfo(GFile file, TaskMonitor monitor) {
File localFile = new File(file.getPath());
public FileAttributes getFileAttributes(GFile file, TaskMonitor monitor) {
File f = new File(file.getPath());
return getFileAttributes(f);
}
StringBuilder buffer = new StringBuilder();
buffer.append("Name: " + localFile.getName() + "\n");
buffer.append("Size: " + localFile.length() + "\n");
buffer.append("Date: " + new Date(localFile.lastModified()).toString() + "\n");
return buffer.toString();
/**
* Create a {@link FileAttributes} container with info about the specified local file.
*
* @param f {@link File} to query
* @return {@link FileAttributes} instance
*/
public FileAttributes getFileAttributes(File f) {
Path p = f.toPath();
FileType fileType = fileToFileType(p);
Path symLinkDest = null;
try {
symLinkDest = fileType == FileType.SYMBOLIC_LINK ? Files.readSymbolicLink(p) : null;
}
catch (IOException e) {
// ignore and continue with symLinkDest == null
}
return FileAttributes.of(
FileAttribute.create(NAME_ATTR, f.getName()),
FileAttribute.create(FILE_TYPE_ATTR, fileType),
FileAttribute.create(SIZE_ATTR, f.length()),
FileAttribute.create(MODIFIED_DATE_ATTR, new Date(f.lastModified())),
symLinkDest != null
? FileAttribute.create(SYMLINK_DEST_ATTR, symLinkDest.toString())
: null);
}
private static FileType fileToFileType(Path p) {
if (Files.isSymbolicLink(p)) {
return FileType.SYMBOLIC_LINK;
}
if (Files.isDirectory(p)) {
return FileType.DIRECTORY;
}
if (Files.isRegularFile(p)) {
return FileType.FILE;
}
return FileType.UNKNOWN;
}
@Override
@ -153,12 +229,6 @@ public class LocalFileSystem implements GFileSystem {
return gf;
}
@Override
public InputStream getInputStream(GFile file, TaskMonitor monitor) throws IOException {
File f = new File(file.getPath());
return new FileInputStream(f);
}
@Override
public boolean isClosed() {
return false;
@ -169,8 +239,103 @@ public class LocalFileSystem implements GFileSystem {
return refManager;
}
@Override
public InputStream getInputStream(GFile file, TaskMonitor monitor) throws IOException {
return getInputStream(file.getFSRL(), monitor);
}
InputStream getInputStream(FSRL fsrl, TaskMonitor monitor) throws IOException {
return new FileInputStream(getLocalFile(fsrl));
}
@Override
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor) throws IOException {
return getByteProvider(file.getFSRL(), monitor);
}
ByteProvider getByteProvider(FSRL fsrl, TaskMonitor monitor) throws IOException {
File f = getLocalFile(fsrl);
return new FileByteProvider(f, fsrl, AccessMode.READ);
}
@Override
public String toString() {
return "Local file system " + fsFSRL;
}
@Override
public String getMD5Hash(GFile file, boolean required, TaskMonitor monitor)
throws CancelledException, IOException {
return getMD5Hash(file.getFSRL(), required, monitor);
}
synchronized String getMD5Hash(FSRL fsrl, boolean required, TaskMonitor monitor)
throws CancelledException, IOException {
File f = getLocalFile(fsrl);
if ( !f.isFile() ) {
return null;
}
FileFingerprintRec fileFingerprintRec = new FileFingerprintRec(f.getPath(), f.lastModified(), f.length());
String md5 = fileFingerprintToMD5Map.get(fileFingerprintRec);
if (md5 == null && required) {
md5 = FSUtilities.getFileMD5(f, monitor);
fileFingerprintToMD5Map.put(fileFingerprintRec, md5);
}
return md5;
}
//-----------------------------------------------------------------------------------
private static class FileFingerprintRec {
final String path;
final long timestamp;
final long length;
FileFingerprintRec(String path, long timestamp, long length) {
this.path = path;
this.timestamp = timestamp;
this.length = length;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (length ^ (length >>> 32));
result = prime * result + ((path == null) ? 0 : path.hashCode());
result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof FileFingerprintRec)) {
return false;
}
FileFingerprintRec other = (FileFingerprintRec) obj;
if (length != other.length) {
return false;
}
if (path == null) {
if (other.path != null) {
return false;
}
}
else if (!path.equals(other.path)) {
return false;
}
if (timestamp != other.timestamp) {
return false;
}
return true;
}
}
}

52
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/LocalFileSystemSub.java
View File

@ -17,10 +17,13 @@ package ghidra.formats.gfilesystem;
import java.io.*;
import java.nio.file.Files;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.fileinfo.FileAttributes;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@ -35,24 +38,20 @@ import ghidra.util.task.TaskMonitor;
* by the FileSystemFactoryMgr.
*
*/
public class LocalFileSystemSub implements GFileSystem {
public class LocalFileSystemSub implements GFileSystem, GFileHashProvider {
private final FSRLRoot fsFSRL;
private final GFileSystem rootFS;
private final List<GFile> emptyDir = Collections.emptyList();
private final LocalFileSystem rootFS;
private File localfsRootDir;
private FileSystemRefManager refManager = new FileSystemRefManager(this);
private GFileLocal rootGFile;
public LocalFileSystemSub(File rootDir, GFileSystem rootFS) throws IOException {
public LocalFileSystemSub(File rootDir, LocalFileSystem rootFS) throws IOException {
this.rootFS = rootFS;
this.localfsRootDir = rootDir.getCanonicalFile();
GFile containerDir = rootFS.lookup(localfsRootDir.getPath());
if (containerDir == null) {
throw new IOException("Bad root dir: " + rootDir);
}
this.fsFSRL = FSRLRoot.nestedFS(containerDir.getFSRL(), rootFS.getFSRL().getProtocol());
this.rootGFile = new GFileLocal(localfsRootDir, "/", containerDir.getFSRL(), this, null);
FSRL containerFSRL = rootFS.getLocalFSRL(localfsRootDir);
this.fsFSRL = FSRLRoot.nestedFS(containerFSRL, rootFS.getFSRL().getProtocol());
this.rootGFile = new GFileLocal(localfsRootDir, "/", containerFSRL, this, null);
}
@Override
@ -97,17 +96,17 @@ public class LocalFileSystemSub implements GFileSystem {
directory = rootGFile;
}
if (!directory.isDirectory()) {
return emptyDir;
return List.of();
}
File localDir = getFileFromGFile(directory);
if (Files.isSymbolicLink(localDir.toPath())) {
return emptyDir;
return List.of();
}
File[] localFiles = localDir.listFiles();
if (localFiles == null) {
return emptyDir;
return List.of();
}
List<GFile> tmp = new ArrayList<>(localFiles.length);
@ -130,19 +129,15 @@ public class LocalFileSystemSub implements GFileSystem {
}
@Override
public String getInfo(GFile file, TaskMonitor monitor) {
public FileAttributes getFileAttributes(GFile file, TaskMonitor monitor) {
try {
File localFile = getFileFromGFile(file);
StringBuilder buffer = new StringBuilder();
buffer.append("Name: " + localFile.getName() + "\n");
buffer.append("Size: " + localFile.length() + "\n");
buffer.append("Date: " + new Date(localFile.lastModified()).toString() + "\n");
return buffer.toString();
return rootFS.getFileAttributes(localFile);
}
catch (IOException e) {
// fail and return null
}
return null;
return FileAttributes.EMPTY;
}
@Override
@ -179,8 +174,7 @@ public class LocalFileSystemSub implements GFileSystem {
@Override
public InputStream getInputStream(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
return new FileInputStream(getFileFromGFile(file));
return rootFS.getInputStream(file.getFSRL(), monitor);
}
@Override
@ -192,4 +186,16 @@ public class LocalFileSystemSub implements GFileSystem {
public String toString() {
return getName();
}
@Override
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
return rootFS.getByteProvider(file.getFSRL(), monitor);
}
@Override
public String getMD5Hash(GFile file, boolean required, TaskMonitor monitor)
throws CancelledException, IOException {
return rootFS.getMD5Hash(file.getFSRL(), required, monitor);
}
}

98
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdByteProvider.java Normal file
View File

@ -0,0 +1,98 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem;
import java.io.File;
import java.io.IOException;
import ghidra.app.util.bin.ByteProvider;
/**
* A {@link ByteProvider} along with a {@link FileSystemRef} to keep the filesystem pinned
* in memory.
* <p>
* The caller is responsible for {@link #close() closing} this object, which releases
* the FilesystemRef.
*/
public class RefdByteProvider implements ByteProvider {
private final FileSystemRef fsRef;
private final ByteProvider provider;
private final FSRL fsrl;
/**
* Creates a RefdByteProvider instance, taking ownership of the supplied FileSystemRef.
*
* @param fsRef {@link FileSystemRef} that contains the specified ByteProvider
* @param provider {@link ByteProvider} inside the filesystem held open by the ref
* @param fsrl {@link FSRL} identity of this new ByteProvider
*/
public RefdByteProvider(FileSystemRef fsRef, ByteProvider provider, FSRL fsrl) {
this.fsRef = fsRef;
this.provider = provider;
this.fsrl = fsrl;
}
@Override
public void close() throws IOException {
provider.close();
fsRef.close();
}
@Override
public FSRL getFSRL() {
return fsrl;
}
@Override
public File getFile() {
return provider.getFile();
}
@Override
public String getName() {
return fsrl != null ? fsrl.getName() : provider.getName();
}
@Override
public String getAbsolutePath() {
return fsrl != null ? fsrl.getPath() : provider.getAbsolutePath();
}
@Override
public long length() throws IOException {
return provider.length();
}
@Override
public boolean isValidIndex(long index) {
return provider.isValidIndex(index);
}
@Override
public byte readByte(long index) throws IOException {
return provider.readByte(index);
}
@Override
public byte[] readBytes(long index, long length) throws IOException {
return provider.readBytes(index, length);
}
@Override
public String toString() {
return "ByteProvider " + provider.getFSRL() + " in file system " + fsRef.getFilesystem();
}
}

6
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/RefdFile.java
View File

@ -29,6 +29,12 @@ public class RefdFile implements Closeable {
public final FileSystemRef fsRef;
public final GFile file;
/**
* Creates a RefdFile instance, taking ownership of the supplied fsRef.
*
* @param fsRef {@link FileSystemRef} that pins the filesystem open
* @param file GFile file inside the specified filesystem
*/
public RefdFile(FileSystemRef fsRef, GFile file) {
this.fsRef = fsRef;
this.file = file;

9
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/SelectFromListDialog.java
View File

@ -15,8 +15,6 @@
*/
package ghidra.formats.gfilesystem;
import ghidra.util.SystemUtilities;
import java.awt.BorderLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
@ -29,6 +27,7 @@ import docking.DialogComponentProvider;
import docking.DockingWindowManager;
import docking.widgets.MultiLineLabel;
import docking.widgets.list.ListPanel;
import ghidra.util.SystemUtilities;
/**
* Dialog that presents the user with a list of strings and returns the object
@ -96,17 +95,17 @@ public class SelectFromListDialog<T> extends DialogComponentProvider {
private void doSelect() {
selectedObject = null;
actionComplete = false;
DockingWindowManager activeInstance = DockingWindowManager.getActiveInstance();
activeInstance.showDialog(this);
DockingWindowManager.showDialog(this);
if (actionComplete) {
selectedObject = list.get(listPanel.getSelectedIndex());
}
}
private JPanel buildWorkPanel(String prompt) {
DefaultListModel<Object> listModel = new DefaultListModel<Object>() {
DefaultListModel<Object> listModel = new DefaultListModel<>() {
@Override
public String getElementAt(int index) {
@SuppressWarnings("unchecked")
T t = (T) super.getElementAt(index);
return toStringFunc.apply(t);
}

2
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/SingleFileSystemIndexHelper.java
View File

@ -141,7 +141,7 @@ public class SingleFileSystemIndexHelper {
* the payload file.
*/
public GFile lookup(String path) {
if (path.equals("/")) {
if (path == null || path.equals("/")) {
return rootDir;
}
else if (path.equals(payloadFile.getFSRL().getPath())) {

143
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/CachedPasswordProvider.java Normal file
View File

@ -0,0 +1,143 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.util.*;
import ghidra.formats.gfilesystem.FSRL;
/**
* Caches passwords used to unlock a file.
* <p>
* Threadsafe.
*/
public class CachedPasswordProvider implements PasswordProvider {
private Map<String, List<CryptoRec>> values = new HashMap<>();
private int count;
/**
* Adds a password / file combo to the cache.
*
* @param fsrl {@link FSRL} file
* @param password password to unlock the file. Specified PasswordValue is
* only copied, clearing is still callers responsibility
*/
public synchronized void addPassword(FSRL fsrl, PasswordValue password) {
CryptoRec rec = new CryptoRec();
rec.fsrl = fsrl;
rec.value = password.clone();
addRec(rec);
}
private void addRec(CryptoRec rec) {
// index the record by its full FSRL, a simplified FSRL, its plain filename, and any MD5
String fsrlStr = rec.fsrl.toString();
boolean isNewValue =
addIfUnique(values.computeIfAbsent(fsrlStr, x -> new ArrayList<>()), rec);
String fsrlStr2 = rec.fsrl.toPrettyString();
if (!fsrlStr2.equals(fsrlStr)) {
addIfUnique(values.computeIfAbsent(fsrlStr2, x -> new ArrayList<>()), rec);
}
addIfUnique(values.computeIfAbsent(rec.fsrl.getName(), x -> new ArrayList<>()), rec);
if (rec.fsrl.getMD5() != null) {
addIfUnique(values.computeIfAbsent(rec.fsrl.getMD5(), x -> new ArrayList<>()), rec);
}
if (isNewValue) {
count++;
}
}
private boolean addIfUnique(List<CryptoRec> recs, CryptoRec newRec) {
for (CryptoRec rec : recs) {
if (rec.value.equals(newRec.value)) {
return false;
}
}
recs.add(newRec);
return true;
}
/**
* Remove all cached information.
*/
public synchronized void clearCache() {
values.clear();
count = 0;
}
/**
* Returns the number of items in cache
*
* @return number of items in cache
*/
public synchronized int getCount() {
return count;
}
@Override
public synchronized Iterator<PasswordValue> getPasswordsFor(FSRL fsrl, String prompt,
Session session) {
Set<CryptoRec> uniqueFoundRecs = new LinkedHashSet<>();
uniqueFoundRecs.addAll(values.getOrDefault(fsrl.toString(), Collections.emptyList()));
uniqueFoundRecs.addAll(values.getOrDefault(fsrl.toPrettyString(), Collections.emptyList()));
uniqueFoundRecs.addAll(values.getOrDefault(fsrl.getName(), Collections.emptyList()));
if (fsrl.getMD5() != null) {
uniqueFoundRecs.addAll(values.getOrDefault(fsrl.getMD5(), Collections.emptyList()));
}
List<PasswordValue> results = new ArrayList<>();
for (CryptoRec rec : uniqueFoundRecs) {
results.add(rec.value);
}
// Use an iterator that clones the values before giving them to the caller
// so our internal values don't get cleared
return new CloningPasswordIterator(results.iterator());
}
private static class CryptoRec {
FSRL fsrl;
PasswordValue value;
}
private class CloningPasswordIterator implements Iterator<PasswordValue> {
Iterator<PasswordValue> delegate;
CloningPasswordIterator(Iterator<PasswordValue> it) {
this.delegate = it;
}
@Override
public boolean hasNext() {
return delegate.hasNext();
}
@Override
public PasswordValue next() {
PasswordValue result = delegate.next();
return result.clone();
}
}
}

115
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/CmdLinePasswordProvider.java Normal file
View File

@ -0,0 +1,115 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.*;
import org.apache.commons.io.FilenameUtils;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.util.Msg;
import utilities.util.FileUtilities;
/**
* A {@link PasswordProvider} that supplies passwords to decrypt files via the java jvm invocation.
* <p>
* Example: <pre>java -Dfilesystem.passwords=/fullpath/to/textfile</pre>
* <p>
* The password file is a plain text tabbed-csv file, where each line
* specifies a password and an optional file identifier.
* <p>
* Example file contents, where each line is divided into fields by a tab
* character where the first field is the password and the second optional field
* is the file's identifying information (name, path, etc):
* <p>
* <pre>
* <code>password1 [tab] myfirstzipfile.zip</code> <b>&larr; supplies a password for the named file located in any directory</b>
* <code>someOtherPassword [tab] /full/path/tozipfile.zip</code> <b>&larr; supplies password for file at specified location</b>
* <code>anotherPassword [tab] file:///full/path/tozipfile.zip|zip:///subdir/in/zip/somefile.txt</code> <b>&larr; supplies password for file embedded inside a zip</b>
* <code>yetAnotherPassword</code> <b>&larr; a password to try for any file that needs a password</b>
* </pre>
*
*
*/
public class CmdLinePasswordProvider implements PasswordProvider {
public static final String CMDLINE_PASSWORD_PROVIDER_PROPERTY_NAME = "filesystem.passwords";
@Override
public Iterator<PasswordValue> getPasswordsFor(FSRL fsrl, String prompt, Session session) {
String propertyValue = System.getProperty(CMDLINE_PASSWORD_PROVIDER_PROPERTY_NAME);
if (propertyValue == null) {
return Collections.emptyIterator();
}
File passwordFile = new File(propertyValue);
return load(passwordFile, fsrl).iterator();
}
private List<PasswordValue> load(File f, FSRL fsrl) {
List<PasswordValue> result = new ArrayList<>();
try {
for (String s : FileUtilities.getLines(f)) {
String[] fields = s.split("\t");
String password = fields[0];
if (password.isBlank()) {
continue;
}
String fileIdStr = fields.length > 1 ? fields[1] : null;
if (fileIdStr == null) {
// no file identifier string, always matches
result.add(PasswordValue.wrap(password.toCharArray()));
continue;
}
// try to match the name string as a FSRL, a path or a plain name.
try {
FSRL currentFSRL = FSRL.fromString(fileIdStr);
// was a fsrl string, only test as fsrl
if (currentFSRL.isEquivalent(fsrl)) {
result.add(PasswordValue.wrap(password.toCharArray()));
}
continue;
}
catch (MalformedURLException e) {
// ignore
}
String nameOnly = FilenameUtils.getName(fileIdStr);
if (!nameOnly.equals(fileIdStr)) {
// was a path str, only test against path component
if (fileIdStr.equals(fsrl.getPath())) {
result.add(PasswordValue.wrap(password.toCharArray()));
}
continue;
}
// was a plain name, only test against name component
if (nameOnly.equals(fsrl.getName())) {
result.add(PasswordValue.wrap(password.toCharArray()));
continue;
}
// no matches, try next line
}
}
catch (IOException e) {
Msg.warn(this, "Error reading passwords from file: " + f, e);
}
return result;
}
}

55
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/CryptoProvider.java Normal file
View File

@ -0,0 +1,55 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.util.function.Supplier;
/**
* Common interface for provider interfaces that provide crypto information.
* <p>
* TODO: add CryptoKeyProvider.
*/
public interface CryptoProvider {
interface Session {
/**
* Saves a state object into the session using the cryptoprovider's identity as the key
*
* @param cryptoProvider the instance storing the value
* @param value the value to store
*/
void setStateValue(CryptoProvider cryptoProvider, Object value);
/**
* Retrieves a state object from the session
*
* @param <T> the type of the state object
* @param cryptoProvider the CryptoProvider instance
* @param stateFactory supplier that will create a new instance of the requested
* state object if not present in the session
* @return state object (either previously saved or newly created by the factory supplier)
*/
<T> T getStateValue(CryptoProvider cryptoProvider, Supplier<T> stateFactory);
/**
* Returns the {@link CryptoProviders} instance that created this session.
*
* @return the {@link CryptoProviders} instance that created this session
*/
CryptoProviders getCryptoProviders();
}
}

53
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/CryptoProviderSessionChildImpl.java Normal file
View File

@ -0,0 +1,53 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.util.Iterator;
import ghidra.formats.gfilesystem.FSRL;
/**
* A stub implementation of CryptoSession that relies on a parent instance.
*/
public class CryptoProviderSessionChildImpl implements CryptoSession {
private CryptoSession parentSession;
public CryptoProviderSessionChildImpl(CryptoSession parentSession) {
this.parentSession = parentSession;
}
@Override
public void close() {
// don't close parent
}
@Override
public boolean isClosed() {
return parentSession.isClosed();
}
@Override
public Iterator<PasswordValue> getPasswordsFor(FSRL fsrl, String prompt) {
return parentSession.getPasswordsFor(fsrl, prompt);
}
@Override
public void addSuccessfulPassword(FSRL fsrl, PasswordValue password) {
parentSession.addSuccessfulPassword(fsrl, password);
}
}

205
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/CryptoProviders.java Normal file
View File

@ -0,0 +1,205 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import ghidra.formats.gfilesystem.FSRL;
/**
* Registry of {@link CryptoProvider crypto providers} and {@link #newSession() session creator}.
*/
public class CryptoProviders {
private static final CryptoProviders singletonInstance = new CryptoProviders();
/**
* Fetch the global {@link CryptoProviders} singleton instance.
*
* @return shared {@link CryptoProviders} singleton instance
*/
public static CryptoProviders getInstance() {
return singletonInstance;
}
private CachedPasswordProvider cachedCryptoProvider;
private List<CryptoProvider> cryptoProviders = new CopyOnWriteArrayList<>();
CryptoProviders() {
initPasswordCryptoProviders();
}
private void initPasswordCryptoProviders() {
cachedCryptoProvider = new CachedPasswordProvider();
CmdLinePasswordProvider runtimePasswords = new CmdLinePasswordProvider();
registerCryptoProvider(runtimePasswords);
registerCryptoProvider(cachedCryptoProvider);
}
/**
* Adds a {@link CryptoProvider} to this registry.
* <p>
* TODO: do we need provider priority ordering?
*
* @param provider {@link CryptoProvider}
*/
public void registerCryptoProvider(CryptoProvider provider) {
cryptoProviders.add(provider);
}
/**
* Removes a {@link CryptoProvider} from this registry.
*
* @param provider {@link CryptoProvider} to remove
*/
public void unregisterCryptoProvider(CryptoProvider provider) {
cryptoProviders.remove(provider);
}
/**
* Returns the {@link CachedPasswordProvider}.
* <p>
* (Used by GUI actions to manage the cache)
*
* @return cached crypto provider instance
*/
public CachedPasswordProvider getCachedCryptoProvider() {
return cachedCryptoProvider;
}
/**
* Returns the previously registered matching {@link CryptoProvider} instance.
*
* @param <T> CryptoProvider type
* @param providerClass {@link CryptoProvider} class
* @return previously registered CryptoProvider instance, or null if not found
*/
public <T extends CryptoProvider> T getCryptoProviderInstance(Class<T> providerClass) {
return cryptoProviders.stream()
.filter(providerClass::isInstance)
.map(providerClass::cast)
.findFirst()
.orElse(null);
}
/**
* Creates a new {@link CryptoSession}.
* <p>
* TODO: to truly be effective when multiple files
* are being opened (ie. batch import), nested sessions
* need to be implemented.
*
* @return new {@link CryptoSession} instance
*/
public CryptoSession newSession() {
return new CryptoProviderSessionImpl(cryptoProviders);
}
private class CryptoProviderSessionImpl
implements CryptoProvider.Session, CryptoSession {
private List<CryptoProvider> providers;
private Map<CryptoProvider, Object> sessionStateValues = new IdentityHashMap<>();
private boolean closed;
public CryptoProviderSessionImpl(List<CryptoProvider> providers) {
this.providers = new ArrayList<>(providers);
}
@Override
public void addSuccessfulPassword(FSRL fsrl, PasswordValue password) {
cachedCryptoProvider.addPassword(fsrl, password);
}
@Override
public void close() {
closed = true;
}
@Override
public boolean isClosed() {
return closed;
}
@Override
public void setStateValue(CryptoProvider cryptoProvider, Object value) {
sessionStateValues.put(cryptoProvider, value);
}
@Override
public <T> T getStateValue(CryptoProvider cryptoProvider,
Supplier<T> stateFactory) {
Object val = sessionStateValues.get(cryptoProvider);
if (val == null) {
T newVal = stateFactory.get();
sessionStateValues.put(cryptoProvider, newVal);
return newVal;
}
return (T) val;
}
@Override
public CryptoProviders getCryptoProviders() {
return CryptoProviders.this;
}
@Override
public Iterator<PasswordValue> getPasswordsFor(FSRL fsrl, String prompt) {
return new PasswordIterator(providers, fsrl, prompt);
}
/**
* Union iterator of all password providers
*/
class PasswordIterator implements Iterator<PasswordValue> {
private List<PasswordProvider> providers;
private Iterator<PasswordValue> currentIt;
private String prompt;
private FSRL fsrl;
PasswordIterator(List<CryptoProvider> providers, FSRL fsrl, String prompt) {
this.providers = providers.stream()
.filter(PasswordProvider.class::isInstance)
.map(PasswordProvider.class::cast)
.collect(Collectors.toList());
this.fsrl = fsrl;
this.prompt = prompt;
}
@Override
public boolean hasNext() {
while (currentIt == null || !currentIt.hasNext()) {
if (providers.isEmpty()) {
return false;
}
PasswordProvider provider = providers.remove(0);
currentIt = provider.getPasswordsFor(fsrl, prompt, CryptoProviderSessionImpl.this);
}
return true;
}
@Override
public PasswordValue next() {
return currentIt.next();
}
}
}
}

68
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/CryptoSession.java Normal file
View File

@ -0,0 +1,68 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.io.Closeable;
import java.util.Iterator;
import ghidra.formats.gfilesystem.FSRL;
/**
* Provides the caller with the ability to perform crypto querying operations
* for a group of related files.
* <p>
* Typically used to query passwords and to add known good passwords
* to caches for later re-retrieval.
* <p>
* Closing a CryptoSession instance does not invalidate the instance, instead is is a suggestion
* that the instance should not be used for any further nested sessions.
* <p>
* See {@link CryptoProviders#newSession()}.
*/
public interface CryptoSession extends Closeable {
/**
* Returns a sequence of passwords (sorted by quality) that may apply to
* the specified file.
*
* @param fsrl {@link FSRL} path to the password protected file
* @param prompt optional prompt that may be displayed to a user
* @return {@link Iterator} of possible passwords
*/
Iterator<PasswordValue> getPasswordsFor(FSRL fsrl, String prompt);
/**
* Pushes a known good password into a cache for later re-retrieval.
*
* @param fsrl {@link FSRL} path to the file that was unlocked by the password
* @param password the good password
*/
void addSuccessfulPassword(FSRL fsrl, PasswordValue password);
/**
* Returns true if this session has been closed.
*
* @return boolean true if closed
*/
boolean isClosed();
/**
* Closes this session.
*/
@Override
void close();
}

135
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/PasswordDialog.java Normal file
View File

@ -0,0 +1,135 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.awt.Toolkit;
import java.awt.event.KeyEvent;
import java.awt.event.KeyListener;
import javax.swing.*;
import docking.DialogComponentProvider;
import docking.widgets.label.GLabel;
import ghidra.util.HelpLocation;
import ghidra.util.MessageType;
import ghidra.util.layout.PairLayout;
/**
* Simple dialog with single input field to prompt user for password.
* <p>
* User can cancel, or cancel-all, which can be determined by inspecting
* the value of the semi-visible member variables.
* <p>
* Treat this as an internal detail of PopupGUIPasswordProvider.
*/
class PasswordDialog extends DialogComponentProvider {
enum RESULT_STATE {
OK, CANCELED
}
private JPanel workPanel;
JPasswordField passwordField;
RESULT_STATE resultState;
boolean cancelledAll;
PasswordDialog(String title, String prompt) {
super(title, true, true, true, false);
setRememberSize(false);
setStatusJustification(SwingConstants.CENTER);
setMinimumSize(300, 100);
passwordField = new JPasswordField(16);
passwordField.addKeyListener(new KeyListener() {
@Override
public void keyTyped(KeyEvent e) {
if (e.getModifiersEx() == 0 && e.getKeyChar() == KeyEvent.VK_ENTER) {
e.consume();
okCallback();
}
}
@Override
public void keyReleased(KeyEvent e) {
updateCapLockWarning();
}
@Override
public void keyPressed(KeyEvent e) {
updateCapLockWarning();
}
});
workPanel = new JPanel(new PairLayout(5, 5));
workPanel.setBorder(BorderFactory.createEmptyBorder(5, 10, 0, 10));
workPanel.add(new GLabel(prompt != null ? prompt : "Password:"));
workPanel.add(passwordField);
addWorkPanel(workPanel);
addOKButton();
addCancelButton();
JButton cancelAllButton = new JButton("Cancel All");
cancelAllButton.addActionListener(e -> {
cancelledAll = true;
cancelButton.doClick();
});
addButton(cancelAllButton);
updateCapLockWarning();
setFocusComponent(passwordField);
setHelpLocation(
new HelpLocation("FileSystemBrowserPlugin", "PasswordDialog"));
}
private void updateCapLockWarning() {
try {
boolean capsLockOn =
Toolkit.getDefaultToolkit().getLockingKeyState(KeyEvent.VK_CAPS_LOCK);
if (capsLockOn) {
setStatusText("Warning! Caps-Lock is on", MessageType.WARNING);
}
else {
clearStatusText();
}
}
catch (UnsupportedOperationException e) {
// unable to detect caps-lock
}
}
@Override
protected void okCallback() {
resultState = RESULT_STATE.OK;
close();
}
@Override
protected void cancelCallback() {
resultState = RESULT_STATE.CANCELED;
super.cancelCallback();
}
@Override
public void dispose() {
if (passwordField != null) {
passwordField.setText("");
workPanel.remove(passwordField);
passwordField = null;
}
}
}

51
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/PasswordProvider.java Normal file
View File

@ -0,0 +1,51 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.util.Iterator;
import ghidra.formats.gfilesystem.FSRL;
/**
* Instances of this interface provide passwords to decrypt files.
* <p>
* Instances are typically not called directly, instead are used
* by a {@link CryptoSession} along with other provider instances to provide
* a balanced breakfast.
* <p>
* Multiple passwords can be returned for each request with the
* assumption that the consumer of the values can test and validate each one
* to find the correct value. Conversely, it would not be appropriate to use this to get
* a password for a login service that may lock the requester out after a small number
* of failed attempts.
* <p>
* TODO: add negative password result that can be persisted / cached so
* user isn't spammed with requests for an unknown password during batch / recursive
* operations.
*/
public interface PasswordProvider extends CryptoProvider {
/**
* Returns a sequence of passwords (ordered by quality) that may apply to
* the specified file.
*
* @param fsrl {@link FSRL} path to the password protected file
* @param prompt optional prompt that may be displayed to a user
* @param session a place to hold state values that persist across
* related queries
* @return {@link Iterator} of possible passwords
*/
Iterator<PasswordValue> getPasswordsFor(FSRL fsrl, String prompt, Session session);
}

108
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/PasswordValue.java Normal file
View File

@ -0,0 +1,108 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.io.Closeable;
import java.util.Arrays;
/**
* Wrapper for a password, held in a char[] array.
* <p>
* {@link #close() Closing} an instance will clear the characters of the char array.
*/
public class PasswordValue implements Closeable {
/**
* Creates a new PasswordValue using a copy the specified characters.
*
* @param password password characters
* @return new PasswordValue instance
*/
public static PasswordValue copyOf(char[] password) {
PasswordValue result = new PasswordValue();
result.password = new char[password.length];
System.arraycopy(password, 0, result.password, 0, password.length);
return result;
}
/**
* Creates a new PasswordValue by wrapping the specified character array.
* <p>
* The new instance will take ownership of the char array, and
* clear it when the instance is {@link #close() closed}.
*
* @param password password characters
* @return new PasswordValue instance
*/
public static PasswordValue wrap(char[] password) {
PasswordValue result = new PasswordValue();
result.password = password;
return result;
}
private char[] password;
private PasswordValue() {
// empty
}
@Override
public PasswordValue clone() {
return copyOf(password);
}
/**
* Clears the password characters by overwriting them with '\0's.
*/
@Override
public void close() {
Arrays.fill(password, '\0');
password = null;
}
/**
* Returns a reference to the current password characters.
*
* @return reference to the current password characters
*/
public char[] getPasswordChars() {
return password;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(password);
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
PasswordValue other = (PasswordValue) obj;
return Arrays.equals(password, other.password);
}
}

101
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/crypto/PopupGUIPasswordProvider.java Normal file
View File

@ -0,0 +1,101 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import java.util.Iterator;
import java.awt.Component;
import docking.DockingWindowManager;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.formats.gfilesystem.crypto.PasswordDialog.RESULT_STATE;
/**
* Pops up up a GUI dialog prompting the user to enter a password for the specified file.
* <p>
* The dialog is presented to the user when the iterator's hasNext() is called.
* <p>
* Repeated requests to the same iterator will adjust the dialog's title with a "try count" to
* help the user understand the previous password was unsuccessful.
* <p>
* Iterator's hasNext() will return false if the user has previously canceled the dialog,
*/
public class PopupGUIPasswordProvider implements PasswordProvider {
@Override
public Iterator<PasswordValue> getPasswordsFor(FSRL fsrl, String prompt, Session session) {
return new PasswordIterator(session, fsrl, prompt);
}
static class SessionState {
boolean cancelAll;
}
class PasswordIterator implements Iterator<PasswordValue> {
private SessionState sessionState;
private FSRL fsrl;
private boolean cancelled;
private PasswordValue password;
private String prompt;
private int tryCount;
PasswordIterator(Session session, FSRL fsrl, String prompt) {
this.sessionState =
session.getStateValue(PopupGUIPasswordProvider.this, SessionState::new);
this.fsrl = fsrl;
this.prompt = prompt;
}
private void showDlg() {
String dlgPrompt = (prompt != null && !prompt.isBlank()) ? prompt : fsrl.getName();
if (!dlgPrompt.endsWith(":")) {
dlgPrompt += ":";
}
tryCount++;
String dlgTitle =
"Enter Password" + (tryCount > 1 ? " (Try " + tryCount + ")" : "");
PasswordDialog pwdDialog = new PasswordDialog(dlgTitle, dlgPrompt);
DockingWindowManager winMgr = DockingWindowManager.getActiveInstance();
Component rootFrame = winMgr != null ? winMgr.getRootFrame() : null;
DockingWindowManager.showDialog(rootFrame, pwdDialog);
cancelled = pwdDialog.resultState == RESULT_STATE.CANCELED;
password = cancelled ? null : PasswordValue.wrap(pwdDialog.passwordField.getPassword());
sessionState.cancelAll |= cancelled && pwdDialog.cancelledAll;
pwdDialog.dispose();
}
@Override
public boolean hasNext() {
if (cancelled || sessionState.cancelAll) {
return false;
}
if (password == null) {
showDlg();
}
return !cancelled;
}
@Override
public PasswordValue next() {
PasswordValue result = password;
password = null;
return result;
}
}
}

148
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/FileSystemFactoryMgr.java
View File

@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import ghidra.app.util.bin.*;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.util.Msg;
@ -135,53 +135,53 @@ public class FileSystemFactoryMgr {
/**
* Creates a new {@link GFileSystem} instance when the filesystem type is already
* known.
* known, consuming the specified ByteProvider.
* <p>
*
* @param fsType filesystem type string, ie. "file", "zip".
* @param containerFSRL {@link FSRL} of the containing file.
* @param containerFile {@link File} the containing file.
* @param byteProvider {@link ByteProvider}, will be owned by the new file system
* @param fsService reference to the {@link FileSystemService} instance.
* @param monitor {@link TaskMonitor} to use for canceling and updating progress.
* @return new {@link GFileSystem} instance.
* @throws IOException if error when opening the filesystem or unknown fsType.
* @throws CancelledException if the user canceled the operation.
*/
public GFileSystem mountFileSystem(String fsType, FSRL containerFSRL, File containerFile,
public GFileSystem mountFileSystem(String fsType, ByteProvider byteProvider,
FileSystemService fsService, TaskMonitor monitor)
throws IOException, CancelledException {
FileSystemInfoRec fsir = fsByType.get(fsType);
if (fsir == null) {
byteProvider.close();
throw new IOException("Unknown file system type " + fsType);
}
GFileSystem result = mountUsingFactory(fsir, containerFSRL, containerFile, null,
containerFSRL.makeNested(fsType), fsService, monitor);
GFileSystem result = mountUsingFactory(fsir, byteProvider,
byteProvider.getFSRL().makeNested(fsType), fsService, monitor);
return result;
}
private GFileSystem mountUsingFactory(FileSystemInfoRec fsir, FSRL containerFSRL,
File containerFile, ByteProvider byteProvider, FSRLRoot targetFSRL,
FileSystemService fsService, TaskMonitor monitor)
private GFileSystem mountUsingFactory(FileSystemInfoRec fsir, ByteProvider byteProvider,
FSRLRoot targetFSRL, FileSystemService fsService, TaskMonitor monitor)
throws IOException, CancelledException {
GFileSystem result = null;
boolean bpTaken = false;
try {
if (fsir.getFactory() instanceof GFileSystemFactoryFull) {
byteProvider =
(byteProvider == null) ? makeBP(containerFile, containerFSRL) : byteProvider;
GFileSystemFactory<?> factory = fsir.getFactory();
if (factory instanceof GFileSystemFactoryByteProvider) {
GFileSystemFactoryByteProvider<?> bpFactory =
(GFileSystemFactoryByteProvider<?>) factory;
bpTaken = true;
result = ((GFileSystemFactoryFull<?>) fsir.getFactory()).create(containerFSRL,
targetFSRL, byteProvider, containerFile, fsService, monitor);
}
else if (fsir.getFactory() instanceof GFileSystemFactoryWithFile) {
result = ((GFileSystemFactoryWithFile<?>) fsir.getFactory()).create(containerFSRL,
targetFSRL, containerFile, fsService, monitor);
result = bpFactory.create(targetFSRL, byteProvider, fsService, monitor);
}
// add additional GFileSystemFactoryXYZ support blocks here
}
catch (IOException | CancelledException e) {
Msg.warn(this,
"Error during fs factory create: " + fsir.getType() + ", " + fsir.getFSClass(), e);
throw e;
}
finally {
if (byteProvider != null && !bpTaken) {
byteProvider.close();
@ -194,55 +194,45 @@ public class FileSystemFactoryMgr {
/**
* Returns true if the specified file contains a supported {@link GFileSystem}.
* <p>
* @param containerFSRL {@link FSRL} of the containing file.
* @param containerFile {@link File} the containing file.
* @param byteProvider
* @param fsService reference to the {@link FileSystemService} instance.
* @param monitor {@link TaskMonitor} to use for canceling and updating progress.
* @return {@code true} if the file seems to contain a filesystem, {@code false} if it does not.
* @throws IOException if error when accessing the containing file
* @throws CancelledException if the user canceled the operation
*/
public boolean test(FSRL containerFSRL, File containerFile, FileSystemService fsService,
public boolean test(ByteProvider byteProvider, FileSystemService fsService,
TaskMonitor monitor) throws IOException, CancelledException {
int pboByteCount = Math.min(
(int) Math.min(containerFile.length(), GFileSystemProbeBytesOnly.MAX_BYTESREQUIRED),
(int) Math.min(byteProvider.length(), GFileSystemProbeBytesOnly.MAX_BYTESREQUIRED),
largestBytesRequired);
try (ByteProvider bp = new RandomAccessByteProvider(containerFile, containerFSRL)) {
byte[] startBytes = bp.readBytes(0, pboByteCount);
for (FileSystemInfoRec fsir : sortedFactories) {
try {
if (fsir.getFactory() instanceof GFileSystemProbeBytesOnly) {
GFileSystemProbeBytesOnly factoryProbe =
(GFileSystemProbeBytesOnly) fsir.getFactory();
if (factoryProbe.getBytesRequired() <= startBytes.length) {
if (factoryProbe.probeStartBytes(containerFSRL, startBytes)) {
return true;
}
}
}
if (fsir.getFactory() instanceof GFileSystemProbeWithFile) {
GFileSystemProbeWithFile factoryProbe =
(GFileSystemProbeWithFile) fsir.getFactory();
if (factoryProbe.probe(containerFSRL, containerFile, fsService, monitor)) {
return true;
}
}
if (fsir.getFactory() instanceof GFileSystemProbeFull) {
GFileSystemProbeFull factoryProbe =
(GFileSystemProbeFull) fsir.getFactory();
if (factoryProbe.probe(containerFSRL, bp, containerFile, fsService,
monitor)) {
FSRL containerFSRL = byteProvider.getFSRL();
byte[] startBytes = byteProvider.readBytes(0, pboByteCount);
for (FileSystemInfoRec fsir : sortedFactories) {
try {
if (fsir.getFactory() instanceof GFileSystemProbeBytesOnly) {
GFileSystemProbeBytesOnly factoryProbe =
(GFileSystemProbeBytesOnly) fsir.getFactory();
if (factoryProbe.getBytesRequired() <= startBytes.length) {
if (factoryProbe.probeStartBytes(containerFSRL, startBytes)) {
return true;
}
}
}
catch (IOException e) {
Msg.trace(this, "File system probe error for " + fsir.getDescription() +
" with " + containerFSRL, e);
if (fsir.getFactory() instanceof GFileSystemProbeByteProvider) {
GFileSystemProbeByteProvider factoryProbe =
(GFileSystemProbeByteProvider) fsir.getFactory();
if (factoryProbe.probe(byteProvider, fsService, monitor)) {
return true;
}
}
}
catch (IOException e) {
Msg.trace(this, "File system probe error for " + fsir.getDescription() +
" with " + containerFSRL, e);
}
}
return false;
}
@ -263,26 +253,20 @@ public class FileSystemFactoryMgr {
* @throws IOException if error accessing the containing file
* @throws CancelledException if the user cancels the operation
*/
public GFileSystem probe(FSRL containerFSRL, File containerFile, FileSystemService fsService,
public GFileSystem probe(ByteProvider byteProvider, FileSystemService fsService,
FileSystemProbeConflictResolver conflictResolver, TaskMonitor monitor)
throws IOException, CancelledException {
return probe(containerFSRL, containerFile, fsService, conflictResolver,
FileSystemInfo.PRIORITY_LOWEST, monitor);
}
private ByteProvider makeBP(File containerFile, FSRL containerFSRL) throws IOException {
return new SynchronizedByteProvider(
new RandomAccessByteProvider(containerFile, containerFSRL));
return probe(byteProvider, fsService, conflictResolver, FileSystemInfo.PRIORITY_LOWEST,
monitor);
}
/**
* Probes the specified file for a supported {@link GFileSystem} implementation, and
* if found, creates a new filesystem instance.
* if found, creates a new filesystem instance. The ByteProvider is owned by the new
* file system.
* <p>
*
* @param containerFSRL {@link FSRL} of the containing file.
* @param containerFile {@link File} the containing file.
* @param byteProvider container {@link ByteProvider}, will be owned by the new filesystem
* @param fsService reference to the {@link FileSystemService} instance.
* @param conflictResolver {@link FileSystemProbeConflictResolver conflict resolver} to
* use when more than one {@link GFileSystem} implementation can handle the specified
@ -295,20 +279,20 @@ public class FileSystemFactoryMgr {
* @throws IOException if error accessing the containing file
* @throws CancelledException if the user cancels the operation
*/
public GFileSystem probe(FSRL containerFSRL, File containerFile, FileSystemService fsService,
public GFileSystem probe(ByteProvider byteProvider, FileSystemService fsService,
FileSystemProbeConflictResolver conflictResolver, int priorityFilter,
TaskMonitor monitor) throws IOException, CancelledException {
conflictResolver = (conflictResolver == null) ? FileSystemProbeConflictResolver.CHOOSEFIRST
: conflictResolver;
ByteProvider probeBP = makeBP(containerFile, containerFSRL);
FSRL containerFSRL = byteProvider.getFSRL();
try {
int pboByteCount = Math.min(
(int) Math.min(containerFile.length(), GFileSystemProbeBytesOnly.MAX_BYTESREQUIRED),
(int) Math.min(byteProvider.length(), GFileSystemProbeBytesOnly.MAX_BYTESREQUIRED),
largestBytesRequired);
byte[] startBytes = probeBP.readBytes(0, pboByteCount);
byte[] startBytes = byteProvider.readBytes(0, pboByteCount);
List<FileSystemInfoRec> probeMatches = new ArrayList<>();
for (FileSystemInfoRec fsir : sortedFactories) {
try {
@ -325,19 +309,10 @@ public class FileSystemFactoryMgr {
}
}
}
if (fsir.getFactory() instanceof GFileSystemProbeWithFile) {
GFileSystemProbeWithFile factoryProbe =
(GFileSystemProbeWithFile) fsir.getFactory();
if (factoryProbe.probe(containerFSRL, containerFile, fsService, monitor)) {
probeMatches.add(fsir);
continue;
}
}
if (fsir.getFactory() instanceof GFileSystemProbeFull) {
GFileSystemProbeFull factoryProbe =
(GFileSystemProbeFull) fsir.getFactory();
if (factoryProbe.probe(containerFSRL, probeBP, containerFile, fsService,
monitor)) {
if (fsir.getFactory() instanceof GFileSystemProbeByteProvider) {
GFileSystemProbeByteProvider factoryProbe =
(GFileSystemProbeByteProvider) fsir.getFactory();
if (factoryProbe.probe(byteProvider, fsService, monitor)) {
probeMatches.add(fsir);
continue;
}
@ -355,16 +330,19 @@ public class FileSystemFactoryMgr {
return null;
}
ByteProvider mountBP = probeBP;
probeBP = null;
GFileSystem fs = mountUsingFactory(fsir, containerFSRL, containerFile, mountBP,
// After this point, the byteProvider will be closed by the new filesystem,
// or by the factory method if there is an error during mount
ByteProvider mountBP = byteProvider;
byteProvider = null;
GFileSystem fs = mountUsingFactory(fsir, mountBP,
containerFSRL.makeNested(fsir.getType()), fsService, monitor);
monitor.setMessage("Found file system " + fs.getDescription());
return fs;
}
finally {
if (probeBP != null) {
probeBP.close();
if (byteProvider != null) {
byteProvider.close();
}
}

21
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/GFileSystemBaseFactory.java
View File

@ -15,7 +15,6 @@
*/
package ghidra.formats.gfilesystem.factory;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
@ -39,7 +38,7 @@ import ghidra.util.task.TaskMonitor;
*
*/
public class GFileSystemBaseFactory
implements GFileSystemFactoryFull<GFileSystemBase>, GFileSystemProbeFull {
implements GFileSystemFactoryByteProvider<GFileSystemBase>, GFileSystemProbeByteProvider {
private Class<? extends GFileSystemBase> fsClass;
private static final Class<?>[] FS_CTOR_PARAM_TYPES =
@ -54,14 +53,15 @@ public class GFileSystemBaseFactory
}
@Override
public boolean probe(FSRL containerFSRL, ByteProvider byteProvider, File containerFile,
FileSystemService fsService, TaskMonitor monitor)
throws IOException, CancelledException {
public boolean probe(ByteProvider byteProvider, FileSystemService fsService,
TaskMonitor monitor) throws IOException, CancelledException {
try {
FSRL containerFSRL = byteProvider.getFSRL();
Constructor<? extends GFileSystemBase> ctor =
fsClass.getConstructor(FS_CTOR_PARAM_TYPES);
GFileSystemBase fs = ctor.newInstance(containerFSRL.getName(), byteProvider);
fs.setFilesystemService(fsService);
// do NOT close fs here because that would close the byteProvider
return fs.isValid(monitor);
}
@ -72,21 +72,22 @@ public class GFileSystemBaseFactory
}
@Override
public GFileSystemBase create(FSRL containerFSRL, FSRLRoot targetFSRL,
ByteProvider byteProvider, File containerFile, FileSystemService fsService,
TaskMonitor monitor) throws IOException, CancelledException {
public GFileSystemBase create(FSRLRoot targetFSRL, ByteProvider byteProvider,
FileSystemService fsService, TaskMonitor monitor)
throws IOException, CancelledException {
try {
FSRL containerFSRL = byteProvider.getFSRL();
Constructor<? extends GFileSystemBase> ctor =
fsClass.getConstructor(FS_CTOR_PARAM_TYPES);
GFileSystemBase fs = ctor.newInstance(containerFSRL.getName(), byteProvider);
fs.setFilesystemService(fsService);
fs.setFSRL(targetFSRL);
try {
if (!fs.isValid(monitor)) {
throw new IOException("Error when creating new filesystem " +
fsClass.getName() + ", isvalid failed");
}
fs.setFilesystemService(fsService);
fs.setFSRL(targetFSRL);
fs.open(monitor);
GFileSystemBase successFS = fs;

20
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/GFileSystemFactoryFull.java → Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/GFileSystemFactoryByteProvider.java
View File

@ -15,32 +15,28 @@
*/
package ghidra.formats.gfilesystem.factory;
import java.io.IOException;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import java.io.File;
import java.io.IOException;
/**
* A {@link GFileSystemFactory} interface for filesystem implementations that need all available
* references to the source file, including a {@link ByteProvider}.
* A {@link GFileSystemFactory} interface for filesystem implementations
* that use a {@link ByteProvider}.
* <p>
* @param <FSTYPE>
*/
public interface GFileSystemFactoryFull<FSTYPE extends GFileSystem>
public interface GFileSystemFactoryByteProvider<FSTYPE extends GFileSystem>
extends GFileSystemFactory<FSTYPE> {
/**
* Constructs a new {@link GFileSystem} instance that handles the specified file.
* <p>
* @param containerFSRL the {@link FSRL} of the file being opened.
* @param targetFSRL the {@link FSRLRoot} of the filesystem being created.
* @param byteProvider a {@link ByteProvider} containing the contents of the file being probed.
* This method is responsible for closing this byte provider instance.
* @param containerFile the {@link File} (probably in the filecache with non-useful filename)
* being opened.
* @param fsService a reference to the {@link FileSystemService} object
* @param monitor a {@link TaskMonitor} that should be polled to see if the user has
* requested to cancel the operation, and updated with progress information.
@ -48,7 +44,7 @@ public interface GFileSystemFactoryFull<FSTYPE extends GFileSystem>
* @throws IOException if there is an error reading files.
* @throws CancelledException if the user cancels
*/
public FSTYPE create(FSRL containerFSRL, FSRLRoot targetFSRL, ByteProvider byteProvider,
File containerFile, FileSystemService fsService, TaskMonitor monitor)
throws IOException, CancelledException;
public FSTYPE create(FSRLRoot targetFSRL, ByteProvider byteProvider,
FileSystemService fsService, TaskMonitor monitor)
throws IOException, CancelledException;
}

50
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/GFileSystemFactoryWithFile.java
View File

@ -1,50 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.factory;
import ghidra.formats.gfilesystem.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import java.io.File;
import java.io.IOException;
/**
* A {@link GFileSystemFactory} interface for filesystem implementations that can
* be constructed using just a reference to the source {@link File}.
* <p>
* @param <FSTYPE>
*/
public interface GFileSystemFactoryWithFile<FSTYPE extends GFileSystem>
extends GFileSystemFactory<FSTYPE> {
/**
* Constructs a new {@link GFileSystem} instance that handles the specified File.
* <p>
* @param containerFSRL the {@link FSRL} of the file being opened.
* @param targetFSRL the {@link FSRLRoot} of the filesystem being created.
* @param containerFile the {@link File} (probably in the filecache with non-useful filename)
* being opened.
* @param fsService a reference to the {@link FileSystemService} object
* @param monitor a {@link TaskMonitor} that should be polled to see if the user has
* requested to cancel the operation, and updated with progress information.
* @return a new {@link GFileSystem} derived instance.
* @throws IOException if there is an error reading files.
* @throws CancelledException if the user cancels
*/
public FSTYPE create(FSRL containerFSRL, FSRLRoot targetFSRL, File containerFile,
FileSystemService fsService, TaskMonitor monitor)
throws IOException, CancelledException;
}

3
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/GFileSystemProbe.java
View File

@ -18,8 +18,7 @@ package ghidra.formats.gfilesystem.factory;
/**
* An empty interface that is a common type for the real probe interfaces to derive from.
* <p>
* See {@link GFileSystemProbeBytesOnly}, {@link GFileSystemProbeFull}, or
* {@link GFileSystemProbeWithFile}
* See {@link GFileSystemProbeBytesOnly}, {@link GFileSystemProbeByteProvider}
*/
public interface GFileSystemProbe {
// empty interface

22
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/GFileSystemProbeFull.java → Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/GFileSystemProbeByteProvider.java
View File

@ -15,30 +15,25 @@
*/
package ghidra.formats.gfilesystem.factory;
import java.io.IOException;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.formats.gfilesystem.FileSystemService;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import java.io.File;
import java.io.IOException;
/**
* A {@link GFileSystemProbe} interface for filesystems that need all available
* references to the source file, including a {@link ByteProvider}.
* A {@link GFileSystemProbe} interface for filesystems that need to examine
* a {@link ByteProvider}.
*/
public interface GFileSystemProbeFull extends GFileSystemProbe {
public interface GFileSystemProbeByteProvider extends GFileSystemProbe {
/**
* Probes the specified {@code containerFile} to determine if this filesystem implementation
* Probes the specified {@code ByteProvider} to determine if this filesystem implementation
* can handle the file.
*
* @param containerFSRL the {@link FSRL} of the file being probed
* @param byteProvider a {@link ByteProvider} containing the contents of the file being probed.
* Implementors of this method should <b>NOT</b> {@link ByteProvider#close() close()} this
* object.
* @param containerFile the {@link File} (probably in the filecache with non-useful filename)
* being probed.
* @param fsService a reference to the {@link FileSystemService} object
* @param monitor a {@link TaskMonitor} that should be polled to see if the user has
* requested to cancel the operation, and updated with progress information.
@ -47,7 +42,6 @@ public interface GFileSystemProbeFull extends GFileSystemProbe {
* @throws IOException if there is an error reading files.
* @throws CancelledException if the user cancels
*/
public boolean probe(FSRL containerFSRL, ByteProvider byteProvider, File containerFile,
FileSystemService fsService, TaskMonitor monitor)
throws IOException, CancelledException;
public boolean probe(ByteProvider byteProvider, FileSystemService fsService,
TaskMonitor monitor) throws IOException, CancelledException;
}

48
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/factory/GFileSystemProbeWithFile.java
View File

@ -1,48 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.factory;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.formats.gfilesystem.FileSystemService;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import java.io.File;
import java.io.IOException;
/**
* A {@link GFileSystemProbe} interface for filesystems that only need a {@link File}
* reference to the source file.
*/
public interface GFileSystemProbeWithFile extends GFileSystemProbe {
/**
* Probes the specified {@code containerFile} to determine if this filesystem implementation
* can handle the file.
*
* @param containerFSRL the {@link FSRL} of the file being probed
* @param containerFile the {@link File} (probably in the filecache with non-useful filename)
* being probed.
* @param fsService a reference to the {@link FileSystemService} object
* @param monitor a {@link TaskMonitor} that should be polled to see if the user has
* requested to cancel the operation, and updated with progress information.
* @return {@code true} if the specified file is handled by this filesystem implementation,
* {@code false} if not.
* @throws IOException if there is an error reading files.
* @throws CancelledException if the user cancels
*/
public boolean probe(FSRL containerFSRL, File containerFile, FileSystemService fsService,
TaskMonitor monitor) throws IOException, CancelledException;
}

133
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/fileinfo/FileAttribute.java Normal file
View File

@ -0,0 +1,133 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.fileinfo;
import java.util.Objects;
/**
* A (type, type_display_string, value) tuple.
*
* @param <T> type of the value
*/
public class FileAttribute<T> {
private final FileAttributeType attributeType;
private final String attributeDisplayName;
private final T attributeValue;
/**
* Creates a new {@link FileAttribute} instance with an
* {@link FileAttributeType#UNKNOWN_ATTRIBUTE} type and the specified display name.
*
* @param <T> type of the value
* @param name custom display name for the value
* @param attributeValue value (should be .toString()'able)
* @return new FileAttribute instance
*/
public static <T> FileAttribute<T> create(String name, T attributeValue) {
return create(FileAttributeType.UNKNOWN_ATTRIBUTE, name, attributeValue);
}
/**
* Creates a new {@link FileAttribute} instance with the specified type and value.
*
* @param <T> type of the value
* @param attributeType {@link FileAttributeType} type
* @param attributeValue value (should match the
* type specified in {@link FileAttributeType#getValueType()})
* @return new FileAttribute instance
*/
public static <T> FileAttribute<T> create(FileAttributeType attributeType,
T attributeValue) {
return create(attributeType, attributeType.getDisplayName(), attributeValue);
}
/**
* Creates a new {@link FileAttribute} instance with the specified type, display name and
* value.
*
* @param <T> type of the value
* @param attributeType {@link FileAttributeType} type
* @param attributeDisplayName display name of the type
* @param attributeValue value (should match the
* type specified in {@link FileAttributeType#getValueType()})
* @return new FileAttribute instance
*/
public static <T> FileAttribute<T> create(FileAttributeType attributeType,
String attributeDisplayName, T attributeValue) {
if (!attributeType.getValueType().isInstance(attributeValue)) {
throw new IllegalArgumentException("FileAttribute type " + attributeType +
" does not match value: " + attributeValue.getClass());
}
return new FileAttribute<>(attributeType, attributeDisplayName, attributeValue);
}
private FileAttribute(FileAttributeType attributeType, String attributeDisplayName,
T attributeValue) {
this.attributeType = attributeType;
this.attributeDisplayName = attributeDisplayName;
this.attributeValue = attributeValue;
}
/**
* Returns the {@link FileAttributeType} of this instance.
*
* @return {@link FileAttributeType}
*/
public FileAttributeType getAttributeType() {
return attributeType;
}
/**
* Returns the display name of this instance. This is usually derived from
* the {@link FileAttributeType#getDisplayName()}.
*
* @return string display name
*/
public String getAttributeDisplayName() {
return attributeDisplayName;
}
/**
* Return the value.
*
* @return value
*/
public T getAttributeValue() {
return attributeValue;
}
@Override
public int hashCode() {
return Objects.hash(attributeDisplayName, attributeType, attributeValue);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
FileAttribute<?> other = (FileAttribute<?>) obj;
return Objects.equals(attributeDisplayName, other.attributeDisplayName) &&
attributeType == other.attributeType &&
Objects.equals(attributeValue, other.attributeValue);
}
}

99
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/fileinfo/FileAttributeType.java Normal file
View File

@ -0,0 +1,99 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.fileinfo;
import static ghidra.formats.gfilesystem.fileinfo.FileAttributeTypeGroup.*;
import java.util.Date;
import ghidra.formats.gfilesystem.FSRL;
/**
* Well known types of file attributes.
* <p>
* Uncommon information about a file should be added to the {@link FileAttributes} collection
* as an {@link #UNKNOWN_ATTRIBUTE} with a custom display name.
* <p>
* When adding new attribute types to this enum, add them adjacent to other types of the same
* {@link FileAttributeTypeGroup category}. The enum ordinal controls display ordering.
*/
public enum FileAttributeType {
FSRL_ATTR("FSRL", GENERAL_INFO, FSRL.class),
NAME_ATTR("Name", GENERAL_INFO, String.class),
PATH_ATTR("Path", GENERAL_INFO, String.class),
FILE_TYPE_ATTR("File type", GENERAL_INFO, FileType.class),
PROJECT_FILE_ATTR("Project file", GENERAL_INFO, String.class),
SIZE_ATTR("Size", SIZE_INFO, Long.class),
COMPRESSED_SIZE_ATTR("Compressed size", SIZE_INFO, Long.class),
CREATE_DATE_ATTR("Create date", DATE_INFO, Date.class),
MODIFIED_DATE_ATTR("Last modified date", DATE_INFO, Date.class),
ACCESSED_DATE_ATTR("Last accessed date", DATE_INFO, Date.class),
USER_NAME_ATTR("User", OWNERSHIP_INFO, String.class),
USER_ID_ATTR("UserId", OWNERSHIP_INFO, Long.class),
GROUP_NAME_ATTR("Group", OWNERSHIP_INFO, String.class),
GROUP_ID_ATTR("GroupId", OWNERSHIP_INFO, Long.class),
UNIX_ACL_ATTR("Unix acl", PERMISSION_INFO, Long.class),
IS_ENCRYPTED_ATTR("Is encrypted?", ENCRYPTION_INFO, Boolean.class),
HAS_GOOD_PASSWORD_ATTR("Password available?", ENCRYPTION_INFO, Boolean.class),
SYMLINK_DEST_ATTR("Symbolic link destination", MISC_INFO, String.class),
COMMENT_ATTR("Comment", MISC_INFO, String.class),
UNKNOWN_ATTRIBUTE("Other attribute", ADDITIONAL_INFO, Object.class);
private final String displayName;
private final FileAttributeTypeGroup group;
private final Class<?> valueType;
private FileAttributeType(String displayName, FileAttributeTypeGroup group,
Class<?> valueType) {
this.displayName = displayName;
this.group = group;
this.valueType = valueType;
}
/**
* Returns the display name of this attribute type.
*
* @return string display name
*/
public String getDisplayName() {
return displayName;
}
/**
* Returns the {@link FileAttributeTypeGroup group} this attribute belongs in.
*
* @return {@link FileAttributeTypeGroup}
*/
public FileAttributeTypeGroup getGroup() {
return group;
}
/**
* Returns the class the value should match.
*
* @return expected class of the value
*/
public Class<?> getValueType() {
return valueType;
}
}

46
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/fileinfo/FileAttributeTypeGroup.java Normal file
View File

@ -0,0 +1,46 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.fileinfo;
/**
* Categories of file attributes.
*/
public enum FileAttributeTypeGroup {
GENERAL_INFO("General"),
SIZE_INFO("Size Info"),
DATE_INFO("Date Info"),
OWNERSHIP_INFO("Ownership Info"),
PERMISSION_INFO("Permission Info"),
ENCRYPTION_INFO("Encryption Info"),
MISC_INFO("Misc"),
ADDITIONAL_INFO("Addional Info");
private final String descriptiveName;
private FileAttributeTypeGroup(String descriptiveName) {
this.descriptiveName = descriptiveName;
}
/**
* Returns the descriptive name of the group.
*
* @return string descriptive name
*/
public String getDescriptiveName() {
return descriptiveName;
}
}

153
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/fileinfo/FileAttributes.java Normal file
View File

@ -0,0 +1,153 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.fileinfo;
import java.util.ArrayList;
import java.util.List;
/**
* A collection of {@link FileAttribute} values that describe a file.
*/
public class FileAttributes {
/**
* Read-only empty instance.
*/
public static FileAttributes EMPTY = new FileAttributes(List.of()); // read-only because of List.of()
/**
* Creates a {@link FileAttributes} instance containing the specified attribute values.
*
* @param attribs var-arg list of {@link FileAttribute} values, null values are ignored and
* skipped
* @return a new {@link FileAttributes} instance
*/
public static FileAttributes of(FileAttribute<?>... attribs) {
FileAttributes result = new FileAttributes();
for (FileAttribute<?> fa : attribs) {
if (fa != null) {
result.attributes.add(fa);
}
}
return result;
}
private List<FileAttribute<?>> attributes;
/**
* Creates a new / empty {@link FileAttributes} instance.
*/
public FileAttributes() {
this.attributes = new ArrayList<>();
}
private FileAttributes(List<FileAttribute<?>> attributes) {
this.attributes = attributes;
}
@Override
public FileAttributes clone() {
return new FileAttributes(new ArrayList<>(attributes));
}
/**
* Adds a custom named file attribute.
* <p>
* The value class should have a reasonable toString() that converts the value to something
* that is presentable to the user.
*
* @param name name of the attribute
* @param attributeValue value of the attribute
*/
public void add(String name, Object attributeValue) {
add(FileAttributeType.UNKNOWN_ATTRIBUTE, name, attributeValue);
}
/**
* Adds a typed file attribute value.
* <p>
* The value class needs to match {@link FileAttributeType#getValueType()}.
*
* @param attributeType {@link FileAttributeType} type of this value
* @param attributeValue value of attribute
*/
public void add(FileAttributeType attributeType, Object attributeValue) {
add(attributeType, attributeType.getDisplayName(), attributeValue);
}
/**
* Adds a typed file attribute value.
* <p>
* The value class needs to match {@link FileAttributeType#getValueType()}.
*
* @param attributeType {@link FileAttributeType} type of this value
* @param displayName string used to label the value when displayed to the user
* @param attributeValue value of attribute
* @throws IllegalArgumentException if attributeValue does not match attributeType's
* {@link FileAttributeType#getValueType()}.
*/
public void add(FileAttributeType attributeType, String displayName, Object attributeValue) {
if (attributeValue != null) {
attributes.add(FileAttribute.create(attributeType, displayName, attributeValue));
}
}
/**
* Gets the value of the specified attribute.
*
* @param <T> expected class of the attribute value
* @param attributeType {@link FileAttributeType} enum type of attribute to search for
* @param valueClass java class of the value
* @param defaultValue value to return if attribute is not present
* @return value of requested attribute, or defaultValue if not present
*/
public <T> T get(FileAttributeType attributeType, Class<T> valueClass, T defaultValue) {
for (FileAttribute<?> attr : attributes) {
if (attr.getAttributeType() == attributeType) {
Object val = attr.getAttributeValue();
if (valueClass.isAssignableFrom(val.getClass())) {
return valueClass.cast(val);
}
break;
}
}
return defaultValue;
}
/**
* Return a list of all the attributes added to this instance.
*
* @return list of {@link FileAttribute}
*/
public List<FileAttribute<?>> getAttributes() {
return new ArrayList<>(attributes);
}
/**
* Returns true if the specified attribute is present.
*
* @param attributeType attribute to query
* @return boolean true if present
*/
public boolean contains(FileAttributeType attributeType) {
for (FileAttribute<?> attr : attributes) {
if (attr.getAttributeType() == attributeType) {
return true;
}
}
return false;
}
}

21
Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FileCacheEntry.java → Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/fileinfo/FileType.java
View File

@ -13,20 +13,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem;
import java.io.File;
package ghidra.formats.gfilesystem.fileinfo;
/**
* Simple class that contains a {@link File} and its MD5 string.
* Enumeration of file types
*/
public class FileCacheEntry {
public String md5;
public File file;
public FileCacheEntry(File file, String md5) {
this.file = file;
this.md5 = md5;
}
public enum FileType {
FILE,
DIRECTORY,
SYMBOLIC_LINK,
OTHER,
UNKNOWN
}

3
Ghidra/Features/Base/src/main/java/ghidra/framework/GhidraApplicationConfiguration.java
View File

@ -25,6 +25,8 @@ import docking.framework.ApplicationInformationDisplayFactory;
import docking.framework.SplashScreen;
import docking.widgets.PopupKeyStorePasswordProvider;
import ghidra.docking.util.DockingWindowsLookAndFeelUtils;
import ghidra.formats.gfilesystem.crypto.CryptoProviders;
import ghidra.formats.gfilesystem.crypto.PopupGUIPasswordProvider;
import ghidra.framework.main.GhidraApplicationInformationDisplayFactory;
import ghidra.framework.main.UserAgreementDialog;
import ghidra.framework.preferences.Preferences;
@ -58,6 +60,7 @@ public class GhidraApplicationConfiguration extends HeadlessGhidraApplicationCon
ApplicationKeyManagerFactory.setKeyStorePasswordProvider(
new PopupKeyStorePasswordProvider());
CryptoProviders.getInstance().registerCryptoProvider(new PopupGUIPasswordProvider());
}
private static void platformSpecificFixups() {

2
Ghidra/Features/Base/src/main/java/ghidra/plugin/importer/AddToProgramDialog.java
View File

@ -44,6 +44,8 @@ public class AddToProgramDialog extends ImporterDialog {
* @param fsrl the FileSystemURL for where the imported data can be read.
* @param loaderMap the loaders and their corresponding load specifications
* @param byteProvider the ByteProvider from which the bytes from the source can be read.
* The dialog takes ownership of the ByteProvider and it will be closed when
* the dialog is closed
* @param addToProgram the program to which the newly imported data will be added
*/
protected AddToProgramDialog(PluginTool tool, FSRL fsrl, LoaderMap loaderMap,

20
Ghidra/Features/Base/src/main/java/ghidra/plugin/importer/ImporterDialog.java
View File

@ -89,7 +89,8 @@ public class ImporterDialog extends DialogComponentProvider {
* @param tool the active tool that spawned this dialog.
* @param programManager program manager to open imported file with or null
* @param loaderMap the loaders and their corresponding load specifications
* @param byteProvider the ByteProvider for getting the bytes from the file to be imported.
* @param byteProvider the ByteProvider for getting the bytes from the file to be imported. The
* dialog takes ownership of the ByteProvider and it will be closed when the dialog is closed
* @param suggestedDestinationPath optional string path that will be pre-pended to the destination
* filename. Any path specified in the destination filename field will be created when
* the user performs the import (as opposed to the {@link #setDestinationFolder(DomainFolder) destination folder}
@ -166,8 +167,10 @@ public class ImporterDialog extends DialogComponentProvider {
}
private Component buildFilenameTextField() {
filenameTextField = new JTextField();
filenameTextField.setText(getSuggestedFilename());
String initalSuggestedFilename =
FSUtilities.appendPath(suggestedDestinationPath, getSuggestedFilename());
int columns = (initalSuggestedFilename.length() > 50) ? 50 : 0;
filenameTextField = new JTextField(initalSuggestedFilename, columns);
// Use a key listener to track users edits. We can't use the document listener, as
// we change the name field ourselves when other fields are changed.
@ -302,13 +305,9 @@ public class ImporterDialog extends DialogComponentProvider {
if (loader != null) {
languageNeeded = isLanguageNeeded(loader);
setSelectedLanguage(getPreferredLanguage(loader));
if (suggestedDestinationPath != null) {
setFilename(
FSUtilities.appendPath(suggestedDestinationPath, getSuggestedFilename()));
}
else {
setFilename(getSuggestedFilename());
}
String newSuggestedFilename =
FSUtilities.appendPath(suggestedDestinationPath, getSuggestedFilename());
setFilename(newSuggestedFilename);
}
else {
languageNeeded = true;
@ -556,6 +555,7 @@ public class ImporterDialog extends DialogComponentProvider {
}
filenameTextField.setText(s);
filenameTextField.setCaretPosition(s.length());
}
protected void setSelectedLanguage(LanguageCompilerSpecPair lcsPair) {

6
Ghidra/Features/Base/src/main/java/ghidra/plugin/importer/ImporterPlugin.java
View File

@ -30,7 +30,8 @@ import ghidra.app.events.ProgramActivatedPluginEvent;
import ghidra.app.plugin.PluginCategoryNames;
import ghidra.app.services.*;
import ghidra.app.util.importer.LibrarySearchPathManager;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.formats.gfilesystem.FileSystemService;
import ghidra.framework.main.*;
import ghidra.framework.main.datatree.DomainFolderNode;
import ghidra.framework.model.*;
@ -350,8 +351,7 @@ public class ImporterPlugin extends Plugin
}
private void addToProgram(File file) {
GFile gFile = FileSystemService.getInstance().getLocalGFile(file);
if (gFile.getLength() == 0) {
if (file.length() == 0) {
Msg.showInfo(this, null, "Import File Failed",
"File " + file.getName() + " is empty (0 bytes).");
return;

21
Ghidra/Features/Base/src/main/java/ghidra/plugin/importer/ImporterUtilities.java
View File

@ -72,6 +72,8 @@ public class ImporterUtilities {
ExtensionFileFilter.forExtensions("Container files", "zip", "tar", "tgz", "jar", "gz",
"ipsw", "img3", "dmg", "apk", "cpio", "rpm", "lib");
private static final FileSystemService fsService = FileSystemService.getInstance();
static List<LanguageCompilerSpecPair> getPairs(Collection<LoadSpec> loadSpecs) {
Set<LanguageCompilerSpecPair> pairs = new HashSet<>();
for (LoadSpec loadSpec : loadSpecs) {
@ -99,7 +101,7 @@ public class ImporterUtilities {
int id = program.startTransaction("setImportProperties");
try {
fsrl = FileSystemService.getInstance().getFullyQualifiedFSRL(fsrl, monitor);
fsrl = fsService.getFullyQualifiedFSRL(fsrl, monitor);
Options propertyList = program.getOptions(Program.PROGRAM_INFO);
propertyList.setString(ProgramMappingService.PROGRAM_SOURCE_FSRL, fsrl.toString());
@ -165,11 +167,10 @@ public class ImporterUtilities {
RefdFile referencedFile = null;
try {
FileSystemService service = FileSystemService.getInstance();
referencedFile = service.getRefdFile(fsrl, monitor);
referencedFile = fsService.getRefdFile(fsrl, monitor);
FSRL fullFsrl = service.getFullyQualifiedFSRL(fsrl, monitor);
boolean isFSContainer = service.isFileFilesystemContainer(fullFsrl, monitor);
FSRL fullFsrl = fsService.getFullyQualifiedFSRL(fsrl, monitor);
boolean isFSContainer = fsService.isFileFilesystemContainer(fullFsrl, monitor);
if (referencedFile.file.getLength() == 0) {
Msg.showError(ImporterUtilities.class, null, "File is empty",
"File " + fsrl.getPath() + " is empty, nothing to import");
@ -264,10 +265,11 @@ public class ImporterUtilities {
Objects.requireNonNull(monitor);
try {
ByteProvider provider = FileSystemService.getInstance().getByteProvider(fsrl, monitor);
ByteProvider provider = fsService.getByteProvider(fsrl, false, monitor);
if (provider.length() == 0) {
Msg.showWarn(null, null, "Error opening " + fsrl.getName(),
"The item does not correspond to a valid file.");
provider.close();
return;
}
@ -307,8 +309,7 @@ public class ImporterUtilities {
TaskMonitor monitor) {
try {
ByteProvider provider = FileSystemService.getInstance().getByteProvider(fsrl, monitor);
ByteProvider provider = fsService.getByteProvider(fsrl, true, monitor);
LoaderMap loaderMap = LoaderService.getAllSupportedLoadSpecs(provider);
SystemUtilities.runSwingLater(() -> {
@ -393,7 +394,7 @@ public class ImporterUtilities {
Objects.requireNonNull(monitor);
try (ByteProvider bp = FileSystemService.getInstance().getByteProvider(fsrl, monitor)) {
try (ByteProvider bp = fsService.getByteProvider(fsrl, false, monitor)) {
Object consumer = new Object();
MessageLog messageLog = new MessageLog();
@ -464,7 +465,7 @@ public class ImporterUtilities {
Objects.requireNonNull(monitor);
MessageLog messageLog = new MessageLog();
try (ByteProvider bp = FileSystemService.getInstance().getByteProvider(fsrl, monitor)) {
try (ByteProvider bp = fsService.getByteProvider(fsrl, false, monitor)) {
loadSpec.getLoader().loadInto(bp, loadSpec, options, messageLog, program, monitor);
displayResults(tool, program, program.getDomainFile(), messageLog.toString());
}

312
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FSBActionContext.java
View File

@ -15,9 +15,17 @@
*/
package ghidra.plugins.fsbrowser;
import java.awt.event.MouseEvent;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.swing.tree.TreePath;
import docking.ActionContext;
import docking.ComponentProvider;
import docking.widgets.tree.GTree;
import docking.widgets.tree.GTreeNode;
import ghidra.formats.gfilesystem.*;
/**
* {@link FileSystemBrowserPlugin}-specific action.
@ -30,15 +38,32 @@ public class FSBActionContext extends ActionContext {
* Creates a new {@link FileSystemBrowserPlugin}-specific action context.
*
* @param provider the ComponentProvider that generated this context.
* @param contextObject an optional contextObject that the ComponentProvider can provide to the
* action.
* @param selectedNodes selected nodes in the tree
* @param event MouseEvent that caused the update, or null
* @param gTree {@link FileSystemBrowserPlugin} provider tree.
*/
public FSBActionContext(ComponentProvider provider, Object contextObject, GTree gTree) {
super(provider, contextObject, gTree);
public FSBActionContext(FileSystemBrowserComponentProvider provider, FSBNode[] selectedNodes,
MouseEvent event, GTree gTree) {
super(provider, selectedNodes, gTree);
this.gTree = gTree;
}
/**
* Returns true if the GTree is not busy
* @return boolean true if GTree is not busy
*/
public boolean notBusy() {
return !gTree.isBusy();
}
/**
* Returns true if the GTree is busy
* @return boolean true if the GTree is busy
*/
public boolean isBusy() {
return gTree.isBusy();
}
/**
* Gets the {@link FileSystemBrowserPlugin} provider's tree.
*
@ -47,4 +72,281 @@ public class FSBActionContext extends ActionContext {
public GTree getTree() {
return gTree;
}
/**
* Returns true if there are selected nodes in the browser tree.
*
* @return boolean true if there are selected nodes in the browser tree
*/
public boolean hasSelectedNodes() {
FSBNode[] selectedNodes = (FSBNode[]) getContextObject();
return selectedNodes.length > 0;
}
/**
* Returns a list of the currently selected tree nodes.
*
* @return list of currently selected tree nodes
*/
public List<FSBNode> getSelectedNodes() {
return List.of((FSBNode[]) getContextObject());
}
/**
* Returns the {@link FSRL} of the currently selected item, as long as it conforms to
* the dirsOk requirement.
*
* @param dirsOk boolean flag, if true the selected item can be either a file or directory
* element, if false, it must be a file or the root of a file system that has a container
* file
* @return FSRL of the single selected item, null if no items selected or more than 1 item
* selected
*/
public FSRL getFSRL(boolean dirsOk) {
FSBNode[] selectedNodes = (FSBNode[]) getContextObject();
if (selectedNodes.length != 1) {
return null;
}
FSBNode node = selectedNodes[0];
FSRL fsrl = node.getFSRL();
if (!dirsOk && node instanceof FSBRootNode && fsrlHasContainer(fsrl.getFS())) {
// 'convert' a file system root node back into its container file
return fsrl.getFS().getContainer();
}
boolean isDir = (node instanceof FSBDirNode) || (node instanceof FSBRootNode);
if (isDir && !dirsOk) {
return null;
}
return fsrl;
}
private boolean fsrlHasContainer(FSRLRoot fsFSRL) {
return fsFSRL.hasContainer() && !fsFSRL.getProtocol().equals(LocalFileSystem.FSTYPE);
}
/**
* Returns true if the currently selected items are all directory items
* @return boolean true if the currently selected items are all directory items
*/
public boolean isSelectedAllDirs() {
FSBNode[] selectedNodes = (FSBNode[]) getContextObject();
for (FSBNode node : selectedNodes) {
boolean isDir = (node instanceof FSBDirNode) || (node instanceof FSBRootNode);
if (!isDir) {
return false;
}
}
return true;
}
/**
* Returns the currently selected tree node
*
* @return the currently selected tree node, or null if no nodes or more than 1 node is selected
*/
public FSBNode getSelectedNode() {
FSBNode[] selectedNodes = (FSBNode[]) getContextObject();
return selectedNodes.length == 1 ? selectedNodes[0] : null;
}
/**
* Returns the FSBRootNode that contains the currently selected tree node.
*
* @return FSBRootNode that contains the currently selected tree node, or null nothing
* selected
*/
public FSBRootNode getRootOfSelectedNode() {
return getRootOfNode(getSelectedNode());
}
private FSBRootNode getRootOfNode(GTreeNode tmp) {
while (tmp != null && !(tmp instanceof FSBRootNode)) {
tmp = tmp.getParent();
}
return (tmp instanceof FSBRootNode) ? (FSBRootNode) tmp : null;
}
/**
* Returns the number of selected nodes in the tree.
*
* @return returns the number of selected nodes in the tree.
*/
public int getSelectedCount() {
FSBNode[] selectedNodes = (FSBNode[]) getContextObject();
return selectedNodes.length;
}
private List<FSRL> getFSRLsFromNodes(FSBNode[] nodes, boolean dirsOk) {
List<FSRL> fsrls = new ArrayList<>();
for (FSBNode node : nodes) {
FSRL fsrl = node.getFSRL();
if (!node.isLeaf() && !dirsOk) {
boolean canConvertToContainerNode =
node instanceof FSBRootNode && fsrl.getFS().hasContainer();
if (!canConvertToContainerNode) {
continue; // skip this node
}
// 'convert' a file system root node back into its container file node
fsrl = fsrl.getFS().getContainer();
}
fsrls.add(fsrl);
}
return fsrls;
}
/**
* Returns a list of FSRLs of the currently selected nodes in the tree.
*
* @param dirsOk boolean flag, if true the selected items can be either a file or directory
* element, if false, it must be a file or the root of a file system that has a container
* file before being included in the resulting list
* @return list of FSRLs of the currently selected items, maybe empty but never null
*/
public List<FSRL> getFSRLs(boolean dirsOk) {
FSBNode[] selectedNodes = (FSBNode[]) getContextObject();
return getFSRLsFromNodes(selectedNodes, dirsOk);
}
/**
* Returns a list of FSRLs of the currently selected file nodes in the tree.
*
* @return list of FSRLs of the currently selected file items, maybe empty but never null
*/
public List<FSRL> getFileFSRLs() {
return getFSRLs(false);
}
/**
* Returns the FSRL of the currently selected file node
*
* @return FSRL of the currently selected file, or null if not file or more than 1 selected
*/
public FSRL getFileFSRL() {
return getFSRL(false);
}
/**
* Converts the tree-node hierarchy of the currently selected item into a string path using
* "/" separators.
*
* @return string path of the currently selected tree item
*/
public String getFormattedTreePath() {
FSBNode[] selectedNodes = (FSBNode[]) getContextObject();
if (selectedNodes.length != 1) {
return null;
}
TreePath treePath = selectedNodes[0].getTreePath();
StringBuilder path = new StringBuilder();
for (Object pathElement : treePath.getPath()) {
if (pathElement instanceof FSBNode) {
FSBNode node = (FSBNode) pathElement;
FSRL fsrl = node.getFSRL();
if (path.length() != 0) {
path.append("/");
}
String s;
if (fsrl instanceof FSRLRoot) {
s = fsrl.getFS().hasContainer() ? fsrl.getFS().getContainer().getName()
: "/";
}
else {
s = fsrl.getName();
}
path.append(s);
}
}
return path.toString();
}
/**
* Returns the FSRL of the currently selected item, if it is a 'loadable' item.
*
* @return FSRL of the currently selected loadable item, or null if nothing selected or
* more than 1 selected
*/
public FSRL getLoadableFSRL() {
FSBNode node = getSelectedNode();
if (node == null) {
return null;
}
FSRL fsrl = node.getFSRL();
if ((node instanceof FSBDirNode) || (node instanceof FSBRootNode)) {
FSBRootNode rootNode = getRootOfSelectedNode();
GFileSystem fs = rootNode.getFSRef().getFilesystem();
if (fs instanceof GFileSystemProgramProvider) {
GFile gfile;
try {
gfile = fs.lookup(node.getFSRL().getPath());
if (gfile != null &&
((GFileSystemProgramProvider) fs).canProvideProgram(gfile)) {
return fsrl;
}
}
catch (IOException e) {
// ignore error and fall thru to normal file handling
}
}
}
if (node instanceof FSBRootNode && fsrl.getFS().hasContainer()) {
// 'convert' a file system root node back into its container file
return fsrl.getFS().getContainer();
}
return (node instanceof FSBFileNode) ? fsrl : null;
}
/**
* Returns a list of FSRLs of the currently selected loadable items.
*
* @return list of FSRLs of currently selected loadable items, maybe empty but never null
*/
public List<FSRL> getLoadableFSRLs() {
FSBNode[] selectedNodes = (FSBNode[]) getContextObject();
List<FSRL> fsrls = new ArrayList<>();
for (FSBNode node : selectedNodes) {
FSRL fsrl = node.getFSRL();
FSRL validated = vaildateFsrl(fsrl, node);
if (validated != null) {
fsrls.add(validated);
continue;
}
else if (node instanceof FSBRootNode && fsrl.getFS().hasContainer()) {
// 'convert' a file system root node back into its container file
fsrls.add(fsrl.getFS().getContainer());
}
else if (node instanceof FSBFileNode) {
fsrls.add(fsrl);
}
}
return fsrls;
}
private FSRL vaildateFsrl(FSRL fsrl, FSBNode node) {
if ((node instanceof FSBDirNode) || (node instanceof FSBRootNode)) {
FSBRootNode rootNode = getRootOfNode(node);
GFileSystem fs = rootNode.getFSRef().getFilesystem();
if (fs instanceof GFileSystemProgramProvider) {
GFile gfile;
try {
gfile = fs.lookup(node.getFSRL().getPath());
if (gfile != null &&
((GFileSystemProgramProvider) fs).canProvideProgram(gfile)) {
return fsrl;
}
}
catch (IOException e) {
// ignore error and return null
}
}
}
return null;
}
}

1556
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FSBActionManager.java
View File

File diff suppressed because it is too large Load Diff

55
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FSBDirNode.java
View File

@ -16,58 +16,44 @@
package ghidra.plugins.fsbrowser;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.swing.Icon;
import docking.widgets.tree.GTreeNode;
import ghidra.formats.gfilesystem.*;
import ghidra.util.Msg;
import ghidra.formats.gfilesystem.GFile;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* GTreeNode that represents a directory on a filesystem.
* <p>
* Visible to just this package.
*/
public class FSBDirNode extends FSBNode {
private FSRL fsrl;
public class FSBDirNode extends FSBFileNode {
FSBDirNode(FSRL fsrl) {
this.fsrl = fsrl;
FSBDirNode(GFile dirFile) {
super(dirFile);
}
@Override
public List<GTreeNode> generateChildren(TaskMonitor monitor) throws CancelledException {
try (RefdFile dir = FileSystemService.getInstance().getRefdFile(fsrl, monitor)) {
return FSBNode.getNodesFromFileList(dir.file.getListing());
try {
return FSBNode.createNodesFromFileList(file.getListing(), monitor);
}
catch (IOException e) {
Msg.showError(this, null, "loadChildren", e);
// fall thru, return empty list
}
return Collections.emptyList();
return List.of();
}
@Override
public Icon getIcon(boolean expanded) {
return null;
}
@Override
public String getName() {
return fsrl.getName();
}
@Override
public FSRL getFSRL() {
return fsrl;
}
@Override
public String getToolTip() {
return fsrl.getName();
public void updateFileAttributes(TaskMonitor monitor) {
for (GTreeNode node : getChildren()) {
if (node instanceof FSBFileNode) {
((FSBFileNode) node).updateFileAttributes(monitor);
}
if (monitor.isCancelled()) {
break;
}
}
super.updateFileAttributes(monitor);
}
@Override
@ -75,9 +61,4 @@ public class FSBDirNode extends FSBNode {
return false;
}
@Override
public int hashCode() {
return fsrl.hashCode();
}
}

86
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FSBFileNode.java
View File

@ -15,46 +15,34 @@
*/
package ghidra.plugins.fsbrowser;
import java.util.Collections;
import java.util.List;
import static ghidra.formats.gfilesystem.fileinfo.FileAttributeType.*;
import javax.swing.Icon;
import java.util.List;
import docking.widgets.tree.GTreeNode;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.formats.gfilesystem.GFile;
import ghidra.formats.gfilesystem.fileinfo.FileAttributeType;
import ghidra.formats.gfilesystem.fileinfo.FileAttributes;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* GTreeNode that represents a file on a filesystem.
* <p>
* Visible to just this package.
*/
public class FSBFileNode extends FSBNode {
protected FSRL fsrl;
FSBFileNode(FSRL fsrl) {
this.fsrl = fsrl;
}
protected GFile file;
protected boolean isEncrypted;
protected boolean hasPassword;
@Override
public Icon getIcon(boolean expanded) {
return null;
}
@Override
public String getName() {
return fsrl.getName();
FSBFileNode(GFile file) {
this.file = file;
}
@Override
public FSRL getFSRL() {
return fsrl;
}
@Override
public String getToolTip() {
return fsrl.getName();
return file.getFSRL();
}
@Override
@ -64,12 +52,60 @@ public class FSBFileNode extends FSBNode {
@Override
public int hashCode() {
return fsrl.hashCode();
return file.hashCode();
}
@Override
protected void updateFileAttributes(TaskMonitor monitor) {
FileAttributes fattrs = file.getFilesystem().getFileAttributes(file, monitor);
isEncrypted = fattrs.get(IS_ENCRYPTED_ATTR, Boolean.class, false);
hasPassword = fattrs.get(HAS_GOOD_PASSWORD_ATTR, Boolean.class, false);
}
@Override
public List<GTreeNode> generateChildren(TaskMonitor monitor) throws CancelledException {
return Collections.emptyList();
return List.of();
}
/**
* Local copy of the original GFile's {@link FileAttributeType#IS_ENCRYPTED_ATTR} attribute.
*
* @return boolean true if file needs a password to be read
*/
public boolean isEncrypted() {
return isEncrypted;
}
/**
* Local copy of the original GFile's {@link FileAttributeType#HAS_GOOD_PASSWORD_ATTR} attribute.
*
* @return boolean true if a password for the file has been found, false if missing the password
*/
public boolean hasPassword() {
return hasPassword;
}
/**
* Returns true if this file is missing its password
* @return boolean true if this file is missing its password
*/
public boolean hasMissingPassword() {
return isEncrypted && !hasPassword;
}
/**
* Returns true if this node's password status has changed, calling for a complete refresh
* of the status of all files in the file system.
*
* @param monitor {@link TaskMonitor}
* @return boolean true if this nodes password status has changed
*/
public boolean needsFileAttributesUpdate(TaskMonitor monitor) {
if (hasMissingPassword()) {
updateFileAttributes(monitor);
return hasPassword; // if true then the attribute has changed and everything should be refreshed
}
return false;
}
}

52
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FSBNode.java
View File

@ -17,12 +17,16 @@ package ghidra.plugins.fsbrowser;
import java.util.*;
import javax.swing.Icon;
import docking.widgets.tree.GTreeNode;
import docking.widgets.tree.GTreeSlowLoadingNode;
import ghidra.formats.gfilesystem.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* Base interface for all filesystem browser gtree nodes.
* Base class for all filesystem browser gtree nodes.
*/
public abstract class FSBNode extends GTreeSlowLoadingNode {
@ -35,6 +39,31 @@ public abstract class FSBNode extends GTreeSlowLoadingNode {
*/
public abstract FSRL getFSRL();
@Override
public String getToolTip() {
return getName();
}
@Override
public Icon getIcon(boolean expanded) {
return null;
}
@Override
public String getName() {
return getFSRL().getName();
}
public FSBRootNode getFSBRootNode() {
GTreeNode node = getParent();
while (node != null && !(node instanceof FSBRootNode)) {
node = node.getParent();
}
return (node instanceof FSBRootNode) ? (FSBRootNode) node : null;
}
protected abstract void updateFileAttributes(TaskMonitor monitor) throws CancelledException;
/**
* Returns the {@link FSBRootNode} that represents the root of the file system that
* contains the specified file node.
@ -54,15 +83,21 @@ public abstract class FSBNode extends GTreeSlowLoadingNode {
* Helper method to convert {@link GFile} objects to FSBNode objects.
*
* @param files {@link List} of {@link GFile} objects to convert
* @param monitor {@link TaskMonitor}
* @return {@link List} of {@link FSBNode} instances (return typed as a GTreeNode list),
* specific to each GFile instance's type.
*/
public static List<GTreeNode> getNodesFromFileList(List<GFile> files) {
List<GTreeNode> nodes = new ArrayList<>(files.size());
public static List<GTreeNode> createNodesFromFileList(List<GFile> files, TaskMonitor monitor) {
files = new ArrayList<>(files);
Collections.sort(files, FSUtilities.GFILE_NAME_TYPE_COMPARATOR);
List<GTreeNode> nodes = new ArrayList<>(files.size());
for (GFile child : files) {
nodes.add((GTreeNode) getNodeFromFile(child));
FSBFileNode node = createNodeFromFile(child);
if (node.isLeaf()) {
node.updateFileAttributes(monitor);
}
nodes.add(node);
}
return nodes;
}
@ -71,11 +106,10 @@ public abstract class FSBNode extends GTreeSlowLoadingNode {
* Helper method to convert a single {@link GFile} object into a FSBNode object.
*
* @param file {@link GFile} to convert
* @return a new {@link FSBNode} with type specific to the GFile's type.
* @return a new {@link FSBFileNode} with type specific to the GFile's type.
*/
public static FSBNode getNodeFromFile(GFile file) {
return file.isDirectory() ? new FSBDirNode(file.getFSRL())
: new FSBFileNode(file.getFSRL());
public static FSBFileNode createNodeFromFile(GFile file) {
return file.isDirectory() ? new FSBDirNode(file) : new FSBFileNode(file);
}
}

109
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FSBRootNode.java
View File

@ -16,9 +16,8 @@
package ghidra.plugins.fsbrowser;
import java.io.IOException;
import java.util.*;
import javax.swing.Icon;
import java.util.ArrayList;
import java.util.List;
import docking.widgets.tree.GTreeNode;
import ghidra.formats.gfilesystem.*;
@ -31,46 +30,88 @@ import ghidra.util.task.TaskMonitor;
* <p>
* The {@link FileSystemRef} is released when this node is {@link #dispose()}d.
* <p>
* Visible to just this package.
* Since GTreeNodes are cloned during GTree filtering, and this class has a reference to an external
* resource that needs managing, this class needs to keeps track of the original modelNode
* and does all state modification using the modelNode's context.
*/
public class FSBRootNode extends FSBNode {
private FileSystemRef fsRef;
private FSBFileNode prevNode;
private List<FSBRootNode> subRootNodes = new ArrayList<>();
private FSBRootNode modelNode;
FSBRootNode(FileSystemRef fsRef) {
this.fsRef = fsRef;
this(fsRef, null);
}
FSBRootNode(FileSystemRef fsRef, FSBFileNode prevNode) {
this.fsRef = fsRef;
this.prevNode = prevNode;
}
public FileSystemRef getFSRef() {
return fsRef;
}
public void releaseFSRefs() {
for (FSBRootNode subFSBRootNode : subRootNodes) {
subFSBRootNode.releaseFSRefs();
}
subRootNodes.clear();
if (fsRef != null) {
fsRef.close();
fsRef = null;
}
this.modelNode = this;
}
@Override
public Icon getIcon(boolean expanded) {
return null;
public GTreeNode clone() throws CloneNotSupportedException {
FSBRootNode clone = (FSBRootNode) super.clone();
clone.fsRef = null; // stomp on the clone's fsRef to force it to use modelNode's fsRef
return clone;
}
@Override
public void dispose() {
releaseFSRefsIfModelNode();
super.dispose();
}
void swapBackPrevModelNodeAndDispose() {
if (this != modelNode) {
modelNode.swapBackPrevModelNodeAndDispose();
return;
}
int indexInParent = getIndexInParent();
GTreeNode parent = getParent();
parent.removeNode(this);
parent.addNode(indexInParent, prevNode);
dispose(); // releases the fsRef
}
public FileSystemRef getFSRef() {
return modelNode.fsRef;
}
private void releaseFSRefsIfModelNode() {
if (this != modelNode) {
return;
}
for (FSBRootNode subFSBRootNode : subRootNodes) {
subFSBRootNode.releaseFSRefsIfModelNode();
}
subRootNodes.clear();
FileSystemService.getInstance().releaseFileSystemImmediate(fsRef);
fsRef = null;
}
@Override
public void updateFileAttributes(TaskMonitor monitor) throws CancelledException {
if (this != modelNode) {
modelNode.updateFileAttributes(monitor);
return;
}
for (GTreeNode node : getChildren()) {
monitor.checkCanceled();
if (node instanceof FSBFileNode) {
((FSBFileNode) node).updateFileAttributes(monitor);
}
}
}
@Override
public String getName() {
return fsRef != null && !fsRef.isClosed() ? fsRef.getFilesystem().getName() : " Missing ";
return modelNode.fsRef != null && !modelNode.fsRef.isClosed()
? modelNode.fsRef.getFilesystem().getName()
: " Missing ";
}
@Override
@ -83,37 +124,23 @@ public class FSBRootNode extends FSBNode {
return false;
}
@Override
public void dispose() {
releaseFSRefs();
super.dispose();
}
@Override
public List<GTreeNode> generateChildren(TaskMonitor monitor) throws CancelledException {
if (fsRef != null) {
try {
return FSBNode.getNodesFromFileList(fsRef.getFilesystem().getListing(null));
return FSBNode.createNodesFromFileList(fsRef.getFilesystem().getListing(null),
monitor);
}
catch (IOException e) {
FSUtilities.displayException(this, null, "Error Opening File System",
"Problem generating children at root of file system", e);
}
}
return Collections.emptyList();
return List.of();
}
@Override
public FSRL getFSRL() {
return fsRef.getFilesystem().getFSRL();
}
public FSBFileNode getPrevNode() {
return prevNode;
}
public List<FSBRootNode> getSubRootNodes() {
return subRootNodes;
return modelNode.fsRef.getFilesystem().getFSRL();
}
}

39
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FSBUtils.java
View File

@ -18,10 +18,8 @@ package ghidra.plugins.fsbrowser;
import java.util.ArrayList;
import java.util.List;
import docking.ActionContext;
import docking.widgets.tree.GTreeNode;
import ghidra.app.services.ProgramManager;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.SelectFromListDialog;
import ghidra.framework.plugintool.PluginTool;
import ghidra.util.Msg;
@ -30,41 +28,6 @@ import ghidra.util.Msg;
*/
public class FSBUtils {
public static FSRL getFileFSRLFromContext(ActionContext context) {
return getFSRLFromContext(context, false);
}
public static FSRL getFSRLFromContext(ActionContext context, boolean dirsOk) {
if (context == null || !(context.getContextObject() instanceof FSBNode)) {
return null;
}
FSBNode node = (FSBNode) context.getContextObject();
FSRL fsrl = node.getFSRL();
if (!dirsOk && node instanceof FSBRootNode && fsrlHasContainer(fsrl.getFS())) {
// 'convert' a file system root node back into its container file
return fsrl.getFS().getContainer();
}
boolean isDir = (node instanceof FSBDirNode) || (node instanceof FSBRootNode);
if (isDir && !dirsOk) {
return null;
}
return fsrl;
}
public static boolean fsrlHasContainer(FSRLRoot fsFSRL) {
return fsFSRL.hasContainer() && !fsFSRL.getProtocol().equals(LocalFileSystem.FSTYPE);
}
public static FSBRootNode getNodesRoot(FSBNode node) {
GTreeNode tmp = node;
while (tmp != null && !(tmp instanceof FSBRootNode)) {
tmp = tmp.getParent();
}
return (tmp instanceof FSBRootNode) ? (FSBRootNode) tmp : null;
}
/**
* Returns the {@link ProgramManager} associated with this fs browser plugin.

1
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FileIconService.java
View File

@ -60,6 +60,7 @@ public class FileIconService {
public static final String OVERLAY_IMPORTED = "imported";
public static final String OVERLAY_FILESYSTEM = "filesystem";
public static final String OVERLAY_MISSING_PASSWORD = "password_missing";
private static final String FILEEXT_MAPPING_FILE = "file_extension_icons.xml";

174
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FileSystemBrowserComponentProvider.java
View File

@ -17,7 +17,6 @@ package ghidra.plugins.fsbrowser;
import java.awt.Color;
import java.awt.Component;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
import java.util.ArrayList;
import java.util.List;
@ -26,8 +25,8 @@ import javax.swing.*;
import javax.swing.tree.TreePath;
import javax.swing.tree.TreeSelectionModel;
import docking.ActionContext;
import docking.WindowPosition;
import docking.event.mouse.GMouseListenerAdapter;
import docking.widgets.tree.GTree;
import docking.widgets.tree.GTreeNode;
import docking.widgets.tree.support.GTreeRenderer;
@ -37,7 +36,7 @@ import ghidra.formats.gfilesystem.*;
import ghidra.framework.plugintool.ComponentProviderAdapter;
import ghidra.plugin.importer.ProgramMappingService;
import ghidra.program.model.listing.Program;
import ghidra.util.HelpLocation;
import ghidra.util.*;
/**
* Plugin component provider for the {@link FileSystemBrowserPlugin}.
@ -46,13 +45,15 @@ import ghidra.util.HelpLocation;
* <p>
* Visible to just this package.
*/
class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
class FileSystemBrowserComponentProvider extends ComponentProviderAdapter
implements FileSystemEventListener {
private static final String TITLE = "Filesystem Viewer";
private FileSystemBrowserPlugin plugin;
private FSBActionManager actionManager;
private GTree gTree;
private FSBRootNode rootNode;
private FileSystemService fsService = FileSystemService.getInstance();
/**
* Creates a new {@link FileSystemBrowserComponentProvider} instance, taking
@ -80,31 +81,17 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
handleSingleClick(clickedNode);
}
});
gTree.addMouseListener(new MouseAdapter() {
/**
* Keep track of the previous mouse button that was clicked so we
* can ensure that it was two left clicks that activated
* our dbl-click handler.
*/
int prevMouseButton = -1;
gTree.addMouseListener(new GMouseListenerAdapter() {
@Override
public void mouseReleased(MouseEvent e) {
// keep track of the mouse button so it can be checked next time
int localPrevMouseButton = prevMouseButton;
prevMouseButton = e.getButton();
if (e.isPopupTrigger()) {
return;
}
GTreeNode clickedNode = gTree.getNodeForLocation(e.getX(), e.getY());
if (e.getClickCount() == 1) {
handleSingleClick(clickedNode);
}
if (e.getClickCount() == 2 && e.getButton() == MouseEvent.BUTTON1 &&
localPrevMouseButton == MouseEvent.BUTTON1) {
handleDoubleClick(clickedNode);
public void doubleClickTriggered(MouseEvent e) {
handleDoubleClick(gTree.getNodeForLocation(e.getX(), e.getY()));
e.consume();
}
@Override
public void mouseClicked(MouseEvent e) {
super.mouseClicked(e);
if (!e.isConsumed()) {
handleSingleClick(gTree.getNodeForLocation(e.getX(), e.getY()));
}
}
});
@ -119,6 +106,9 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
if (value instanceof FSBRootNode) {
renderFS((FSBRootNode) value, selected);
}
else if (value instanceof FSBDirNode) {
// do nothing special
}
else if (value instanceof FSBFileNode) {
renderFile((FSBFileNode) value, selected);
}
@ -145,14 +135,22 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
private void renderFile(FSBFileNode node, boolean selected) {
FSRL fsrl = node.getFSRL();
String filename = fsrl.getName();
Icon ico = FileIconService.getInstance().getImage(filename,
ProgramMappingService.isFileImportedIntoProject(fsrl)
? FileIconService.OVERLAY_IMPORTED
: null,
FileSystemService.getInstance().isFilesystemMountedAt(fsrl)
? FileIconService.OVERLAY_FILESYSTEM
: null);
String importOverlay = ProgramMappingService.isFileImportedIntoProject(fsrl)
? FileIconService.OVERLAY_IMPORTED
: null;
String mountedOverlay = fsService.isFilesystemMountedAt(fsrl)
? FileIconService.OVERLAY_FILESYSTEM
: null;
String missingPasswordOverlay = node.hasMissingPassword()
? FileIconService.OVERLAY_MISSING_PASSWORD
: null;
Icon ico = FileIconService.getInstance()
.getImage(filename, importOverlay, mountedOverlay, missingPasswordOverlay);
setIcon(ico);
if (ProgramMappingService.isFileOpen(fsrl)) {
// TODO: change this to a OVERLAY_OPEN option when fetching icon
setForeground(selected ? Color.CYAN : Color.MAGENTA);
@ -171,6 +169,7 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
setHelpLocation(
new HelpLocation("FileSystemBrowserPlugin", "FileSystemBrowserIntroduction"));
fsRef.getFilesystem().getRefManager().addListener(this);
}
/**
@ -182,16 +181,37 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
return gTree;
}
FSRL getFSRL() {
return rootNode != null ? rootNode.getFSRL() : null;
}
FSBActionManager getActionManager() {
return actionManager;
}
void dispose() {
if (rootNode != null && rootNode.getFSRef() != null && !rootNode.getFSRef().isClosed()) {
rootNode.getFSRef().getFilesystem().getRefManager().removeListener(this);
}
removeFromTool();
if (actionManager != null) {
actionManager.dispose();
actionManager = null;
}
if (gTree != null) {
gTree.dispose(); // calls dispose() on tree's rootNode, which will release the fsRefs
gTree = null;
}
rootNode = null;
plugin = null;
}
@Override
public void componentHidden() {
// if the component is 'closed', nuke ourselves via the plugin
if (plugin != null && rootNode.getFSRef() != null &&
rootNode.getFSRef().getFilesystem() != null) {
plugin.removeFileSystemBrowser(rootNode.getFSRef().getFilesystem().getFSRL());
// if the component is 'closed', nuke ourselves
if (plugin != null) {
plugin.removeFileSystemBrowserComponent(this);
dispose();
}
}
@ -199,6 +219,17 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
actionManager.registerComponentActionsInTool();
}
@Override
public void onFilesystemClose(GFileSystem fs) {
Msg.info(this, "File system " + fs.getFSRL() + " was closed! Closing browser window");
Swing.runIfSwingOrRunLater(() -> componentHidden());
}
@Override
public void onFilesystemRefChange(GFileSystem fs, FileSystemRefManager refManager) {
// nothing
}
/*****************************************/
/**
@ -230,12 +261,26 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
FSBFileNode node = (FSBFileNode) clickedNode;
if (node.getFSRL() != null) {
quickShowProgram(node.getFSRL());
updatePasswordStatus(node);
}
}
}
private void updatePasswordStatus(FSBFileNode node) {
// currently this is the only state that might change
// and that effect the node display
if (node.hasMissingPassword()) {
// check and see if its status has changed
gTree.runTask(monitor -> {
if (node.needsFileAttributesUpdate(monitor)) {
actionManager.doRefreshInfo(List.of(node), monitor);
}
});
}
}
private void handleDoubleClick(GTreeNode clickedNode) {
if (clickedNode instanceof FSBFileNode) {
if (clickedNode instanceof FSBFileNode && clickedNode.isLeaf()) {
FSBFileNode node = (FSBFileNode) clickedNode;
if (node.getFSRL() != null && !quickShowProgram(node.getFSRL())) {
@ -247,40 +292,34 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
/*****************************************/
@Override
public ActionContext getActionContext(MouseEvent event) {
public FSBActionContext getActionContext(MouseEvent event) {
return new FSBActionContext(this, getSelectedNodes(event), event, gTree);
}
private FSBNode[] getSelectedNodes(MouseEvent event) {
TreePath[] selectionPaths = gTree.getSelectionPaths();
if (selectionPaths != null && selectionPaths.length == 1) {
Object lastPathComponent = selectionPaths[0].getLastPathComponent();
return new FSBActionContext(this, lastPathComponent, gTree);
}
if (selectionPaths != null && selectionPaths.length > 0) {
List<FSBNode> list = new ArrayList<>();
for (TreePath selectionPath : selectionPaths) {
Object lastPathComponent = selectionPath.getLastPathComponent();
if (lastPathComponent instanceof FSBNode) {
FSBNode node = (FSBNode) lastPathComponent;
list.add(node);
}
List<FSBNode> list = new ArrayList<>(selectionPaths.length);
for (TreePath selectionPath : selectionPaths) {
Object lastPathComponent = selectionPath.getLastPathComponent();
if (lastPathComponent instanceof FSBNode) {
list.add((FSBNode) lastPathComponent);
}
if (list.size() == 1) {
return new FSBActionContext(this, list.get(0), gTree);
}
FSBNode[] nodes = new FSBNode[list.size()];
list.toArray(nodes);
return new FSBActionContext(this, nodes, gTree);
}
if (event != null) {
if (list.isEmpty() && event != null) {
Object source = event.getSource();
int x = event.getX();
int y = event.getY();
if (source instanceof JTree) {
JTree sourceTree = (JTree) source;
if (gTree.isMyJTree(sourceTree)) {
return new FSBActionContext(this, gTree.getNodeForLocation(x, y), gTree);
GTreeNode nodeAtEventLocation = gTree.getNodeForLocation(x, y);
if (nodeAtEventLocation != null && nodeAtEventLocation instanceof FSBNode) {
list.add((FSBNode) nodeAtEventLocation);
}
}
}
}
return null;
return list.toArray(FSBNode[]::new);
}
@Override
@ -298,15 +337,4 @@ class FileSystemBrowserComponentProvider extends ComponentProviderAdapter {
return WindowPosition.WINDOW;
}
void dispose() {
if (actionManager != null) {
actionManager.dispose();
actionManager = null;
}
if (gTree != null) {
gTree.dispose();
gTree = null;
}
plugin = null;
}
}

95
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/FileSystemBrowserPlugin.java
View File

@ -22,8 +22,10 @@ import java.io.File;
import java.io.IOException;
import java.util.*;
import docking.ActionContext;
import docking.action.*;
import javax.swing.KeyStroke;
import docking.action.DockingAction;
import docking.action.builder.ActionBuilder;
import docking.widgets.filechooser.GhidraFileChooser;
import docking.widgets.filechooser.GhidraFileChooserMode;
import ghidra.app.CorePluginPackage;
@ -50,7 +52,7 @@ import utilities.util.FileUtilities;
/**
* A {@link Plugin} that supplies a {@link GFileSystem filesystem} browser component
* that allows the user to view the contents of filesystems and perform actions on the
* files inside those filesystems.x
* files inside those filesystems.
*/
//@formatter:off
@PluginInfo(
@ -65,14 +67,13 @@ import utilities.util.FileUtilities;
)
//@formatter:on
public class FileSystemBrowserPlugin extends Plugin implements FrontEndable, ProjectListener,
FileSystemEventListener, FileSystemBrowserService {
private static final String MENU_GROUP = "Import";
FileSystemBrowserService {
/* package */ DockingAction openFilesystemAction;
private GhidraFileChooser chooserOpen;
private FrontEndService frontEndService;
private Map<FSRL, FileSystemBrowserComponentProvider> currentBrowsers = new HashMap<>();
private FileSystemService fsService; // don't use this directly, use fsService() instead
public FileSystemBrowserPlugin(PluginTool tool) {
super(tool);
@ -138,7 +139,6 @@ public class FileSystemBrowserPlugin extends Plugin implements FrontEndable, Pro
fsRef.close();
}
else {
fsRef.getFilesystem().getRefManager().addListener(this);
provider = new FileSystemBrowserComponentProvider(this, fsRef);
currentBrowsers.put(fsFSRL, provider);
getTool().addComponentProvider(provider, false);
@ -148,25 +148,14 @@ public class FileSystemBrowserPlugin extends Plugin implements FrontEndable, Pro
if (show) {
getTool().showComponentProvider(provider, true);
getTool().toFront(provider);
provider.contextChanged();
}
}
/**
* Closes any FilesystemBrowser window component that is showing the specified filesystem.
*
* @param fsFSRL {@link FSRLRoot} of the filesystem to close.
*/
/* package */ void removeFileSystemBrowser(FSRLRoot fsFSRL) {
Swing.runIfSwingOrRunLater(() -> {
FileSystemBrowserComponentProvider fsbcp = currentBrowsers.get(fsFSRL);
if (fsbcp == null) {
return;
}
currentBrowsers.remove(fsFSRL);
fsbcp.removeFromTool();
fsbcp.dispose();
});
void removeFileSystemBrowserComponent(FileSystemBrowserComponentProvider componentProvider) {
if (componentProvider != null) {
Swing.runIfSwingOrRunLater(() -> currentBrowsers.remove(componentProvider.getFSRL()));
}
}
/**
@ -176,7 +165,6 @@ public class FileSystemBrowserPlugin extends Plugin implements FrontEndable, Pro
Swing.runIfSwingOrRunLater(() -> {
for (FileSystemBrowserComponentProvider fsbcp : new ArrayList<>(
currentBrowsers.values())) {
fsbcp.removeFromTool();
fsbcp.dispose();
}
currentBrowsers.clear();
@ -197,7 +185,7 @@ public class FileSystemBrowserPlugin extends Plugin implements FrontEndable, Pro
public void projectClosed(Project project) {
removeAllFileSystemBrowsers();
if (FileSystemService.isInitialized()) {
FileSystemService.getInstance().closeUnusedFileSystems();
fsService().closeUnusedFileSystems();
}
}
@ -207,28 +195,14 @@ public class FileSystemBrowserPlugin extends Plugin implements FrontEndable, Pro
}
private void setupOpenFileSystemAction() {
String actionName = "Open File System";
openFilesystemAction = new DockingAction(actionName, this.getName()) {
@Override
public void actionPerformed(ActionContext context) {
doOpenFileSystem();
}
@Override
public boolean isEnabledForContext(ActionContext context) {
return tool.getProject() != null;
}
};
openFilesystemAction.setMenuBarData(
new MenuData(new String[] { "&File", actionName + "..." }, null, MENU_GROUP,
MenuData.NO_MNEMONIC, "z"));
openFilesystemAction.setKeyBindingData(
new KeyBindingData(KeyEvent.VK_I, InputEvent.CTRL_DOWN_MASK));
openFilesystemAction.setDescription(getPluginDescription().getDescription());
openFilesystemAction.setEnabled(tool.getProject() != null);
tool.addAction(openFilesystemAction);
openFilesystemAction = new ActionBuilder("Open File System", this.getName())
.description(getPluginDescription().getDescription())
.enabledWhen(ac -> tool.getProject() != null)
.menuPath("File", "Open File System...")
.menuGroup("Import", "z")
.keyBinding(KeyStroke.getKeyStroke(KeyEvent.VK_I, InputEvent.CTRL_DOWN_MASK))
.onAction(ac -> doOpenFileSystem())
.buildAndInstall(tool);
}
private void openChooser(String title, String buttonText, boolean multiSelect) {
@ -254,18 +228,15 @@ public class FileSystemBrowserPlugin extends Plugin implements FrontEndable, Pro
private void doOpenFilesystem(FSRL containerFSRL, Component parent, TaskMonitor monitor) {
try {
monitor.setMessage("Probing " + containerFSRL.getName() + " for filesystems");
FileSystemRef ref = FileSystemService.getInstance()
.probeFileForFilesystem(
containerFSRL, monitor, FileSystemProbeConflictResolver.GUI_PICKER);
FileSystemRef ref = fsService().probeFileForFilesystem(containerFSRL, monitor,
FileSystemProbeConflictResolver.GUI_PICKER);
if (ref == null) {
Msg.showWarn(this, parent, "Open Filesystem",
"No filesystem provider for " + containerFSRL.getName());
return;
}
Swing.runLater(() -> {
createNewFileSystemBrowser(ref, true);
});
createNewFileSystemBrowser(ref, true);
}
catch (IOException | CancelledException e) {
FSUtilities.displayException(this, parent, "Open Filesystem Error",
@ -301,22 +272,18 @@ public class FileSystemBrowserPlugin extends Plugin implements FrontEndable, Pro
return;
}
FSRL containerFSRL = FileSystemService.getInstance().getLocalFSRL(file);
FSRL containerFSRL = fsService().getLocalFSRL(file);
TaskLauncher.launchModal("Open File System", (monitor) -> {
doOpenFilesystem(containerFSRL, parent, monitor);
});
}
@Override
public void onFilesystemClose(GFileSystem fs) {
Msg.info(this, "File system " + fs.getFSRL() + " was closed! Closing browser window");
removeFileSystemBrowser(fs.getFSRL());
}
@Override
public void onFilesystemRefChange(GFileSystem fs, FileSystemRefManager refManager) {
//Msg.info(this, "File system " + fs.getFSRL() + " ref changed");
// nada
private FileSystemService fsService() {
// use a delayed initialization so we don't force the FileSystemService to initialize
if (fsService == null) {
fsService = FileSystemService.getInstance();
}
return fsService;
}
/**

21
Ghidra/Features/Base/src/main/java/ghidra/plugins/fsbrowser/tasks/GFileSystemExtractAllTask.java
View File

@ -17,22 +17,13 @@ package ghidra.plugins.fsbrowser.tasks;
import java.awt.Component;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.LinkedHashMap;
import java.util.Map;
import docking.widgets.OptionDialog;
import ghidra.formats.gfilesystem.AbstractFileExtractorTask;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.formats.gfilesystem.FileSystemService;
import ghidra.formats.gfilesystem.GFile;
import ghidra.formats.gfilesystem.GFileSystem;
import ghidra.formats.gfilesystem.RefdFile;
import ghidra.util.DateUtils;
import ghidra.util.HTMLUtilities;
import ghidra.util.Msg;
import ghidra.formats.gfilesystem.*;
import ghidra.util.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.Task;
import ghidra.util.task.TaskMonitor;
@ -154,12 +145,4 @@ public class GFileSystemExtractAllTask extends AbstractFileExtractorTask {
}
return true;
}
@Override
protected InputStream getSourceFileInputStream(GFile file, TaskMonitor monitor)
throws CancelledException, IOException {
File cacheFile = FileSystemService.getInstance().getFile(file.getFSRL(), monitor);
return new FileInputStream(cacheFile);
}
}

18
Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchImportDialog.java
View File

@ -39,7 +39,8 @@ import docking.widgets.filechooser.GhidraFileChooserMode;
import docking.widgets.label.GDLabel;
import docking.widgets.table.*;
import ghidra.app.services.ProgramManager;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.FSRL;
import ghidra.formats.gfilesystem.FileSystemService;
import ghidra.framework.model.DomainFolder;
import ghidra.framework.plugintool.PluginTool;
import ghidra.plugin.importer.ImporterUtilities;
@ -411,18 +412,9 @@ public class BatchImportDialog extends DialogComponentProvider {
private boolean addSources(List<FSRL> filesToAdd) {
//@formatter:off
List<FSRL> updatedFiles = filesToAdd
.stream()
.map(fsrl -> {
if (fsrl instanceof FSRLRoot && fsrl.getFS().hasContainer()) {
fsrl = fsrl.getFS().getContainer();
}
return fsrl;
})
.collect(Collectors.toList())
;
//@formatter:on
List<FSRL> updatedFiles = filesToAdd.stream()
.map(FSRL::convertRootToContainer)
.collect(Collectors.toList());
List<FSRL> badFiles = batchInfo.addFiles(updatedFiles);
if (!badFiles.isEmpty()) {

86
Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java
View File

@ -25,6 +25,7 @@ import javax.swing.SwingConstants;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.opinion.*;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.crypto.CryptoSession;
import ghidra.plugins.importer.batch.BatchGroup.BatchLoadConfig;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
@ -42,6 +43,8 @@ public class BatchInfo {
public static final int MAXDEPTH_UNLIMITED = -1;
public static final int MAXDEPTH_DEFAULT = 2;
private FileSystemService fsService = FileSystemService.getInstance();
/*
* These structures need to be synchronized to ensure thread visibility, since they are
* written to by a background thread and read from the Swing thread.
@ -181,7 +184,7 @@ public class BatchInfo {
public boolean addFile(FSRL fsrl, TaskMonitor taskMonitor)
throws IOException, CancelledException {
fsrl = FileSystemService.getInstance().getFullyQualifiedFSRL(fsrl, taskMonitor);
fsrl = fsService.getFullyQualifiedFSRL(fsrl, taskMonitor);
if (userAddedFSRLs.contains(fsrl)) {
throw new IOException("Batch already contains file " + fsrl);
}
@ -216,7 +219,7 @@ public class BatchInfo {
// use the fsrl param instead of file.getFSRL() as param may have more info (ie. md5)
try (RefdFile refdFile = FileSystemService.getInstance().getRefdFile(fsrl, taskMonitor)) {
try (RefdFile refdFile = fsService.getRefdFile(fsrl, taskMonitor)) {
GFile file = refdFile.file;
if (file.isDirectory()) {
processFS(file.getFilesystem(), file, taskMonitor);
@ -231,6 +234,9 @@ public class BatchInfo {
return true;
}
// the file was not of interest, let it be removed from the cache
fsService.releaseFileCache(fsrl);
return false;
}
}
@ -294,8 +300,8 @@ public class BatchInfo {
private boolean processAsFS(FSRL fsrl, TaskMonitor taskMonitor)
throws CancelledException {
try (FileSystemRef fsRef = FileSystemService.getInstance().probeFileForFilesystem(fsrl,
taskMonitor, FileSystemProbeConflictResolver.CHOOSEFIRST)) {
try (FileSystemRef fsRef = fsService.probeFileForFilesystem(fsrl, taskMonitor,
FileSystemProbeConflictResolver.CHOOSEFIRST)) {
if (fsRef == null) {
return false;
}
@ -333,9 +339,15 @@ public class BatchInfo {
// TODO: drop FSUtils.listFileSystem and do recursion here.
for (GFile file : FSUtilities.listFileSystem(fs, startDir, null, taskMonitor)) {
taskMonitor.checkCanceled();
doAddFile(
FileSystemService.getInstance().getFullyQualifiedFSRL(file.getFSRL(), taskMonitor),
taskMonitor);
FSRL fqFSRL;
try {
fqFSRL = fsService.getFullyQualifiedFSRL(file.getFSRL(), taskMonitor);
}
catch (IOException e) {
Msg.warn(this, "Error getting info for " + file.getFSRL());
continue;
}
doAddFile(fqFSRL, taskMonitor);
currentUASI.incRawFileCount();
}
}
@ -355,8 +367,7 @@ public class BatchInfo {
private boolean processWithLoader(FSRL fsrl, TaskMonitor monitor)
throws IOException, CancelledException {
try (ByteProvider provider =
FileSystemService.getInstance().getByteProvider(fsrl, monitor)) {
try (ByteProvider provider = fsService.getByteProvider(fsrl, false, monitor)) {
LoaderMap loaderMap = pollLoadersForLoadSpecs(provider, fsrl, monitor);
for (Loader loader : loaderMap.keySet()) {
Collection<LoadSpec> loadSpecs = loaderMap.get(loader);
@ -478,36 +489,39 @@ public class BatchInfo {
BatchTaskMonitor batchMonitor = new BatchTaskMonitor(monitor);
List<FSRL> badFiles = new ArrayList<>();
int size = filesToAdd.size();
for (int i = 0; i < size; i++) {
// start a new CryptoSession to group all password prompting by multiple container
// files into a single session, enabling "Cancel All" to really cancel all password
// prompts
try (CryptoSession cryptoSession = fsService.newCryptoSession()) {
List<FSRL> badFiles = new ArrayList<>();
for (FSRL fsrl : filesToAdd) {
Msg.trace(this, "Adding " + fsrl);
batchMonitor.setPrefix("Processing " + fsrl.getName() + ": ");
FSRL fsrl = filesToAdd.get(i);
Msg.trace(this, "Adding " + fsrl);
batchMonitor.setPrefix("Processing " + fsrl.getName() + ": ");
try {
monitor.checkCanceled();
addFile(fsrl, batchMonitor);
}
catch (CryptoException ce) {
FSUtilities.displayException(this, null, "Error Adding File To Batch Import",
"Error while adding " + fsrl.getName() + " to batch import", ce);
}
catch (IOException ioe) {
Msg.error(this, "Error while adding " + fsrl.getName() + " to batch import",
ioe);
badFiles.add(fsrl);
}
catch (CancelledException e) {
Msg.debug(this, "Cancelling Add File task while adding " + fsrl.getName());
// Note: the workflow for this felt odd: press cancel; confirm cancel; press Ok
// on dialog showing files not processed.
// It seems like the user should not have to see the second dialog
// badFiles.addAll(filesToAdd.subList(i, filesToAdd.size()));
}
}
try {
monitor.checkCanceled();
addFile(fsrl, batchMonitor);
}
catch (CryptoException ce) {
FSUtilities.displayException(this, null, "Error Adding File To Batch Import",
"Error while adding " + fsrl.getName() + " to batch import", ce);
}
catch (IOException ioe) {
Msg.error(this, "Error while adding " + fsrl.getName() + " to batch import", ioe);
badFiles.add(fsrl);
}
catch (CancelledException e) {
Msg.debug(this, "Cancelling Add File task while adding " + fsrl.getName());
// Note: the workflow for this felt odd: press cancel; confirm cancel; press Ok
// on dialog showing files not processed.
// It seems like the user should not have to see the second dialog
// badFiles.addAll(filesToAdd.subList(i, filesToAdd.size()));
}
return badFiles;
}
return badFiles;
}
//==================================================================================================

5
Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/tasks/ImportBatchTask.java
View File

@ -131,8 +131,9 @@ public class ImportBatchTask extends Task {
private void doImportApp(BatchLoadConfig batchLoadConfig,
BatchGroupLoadSpec selectedBatchGroupLoadSpec, TaskMonitor monitor)
throws CancelledException, IOException {
try (ByteProvider byteProvider =
FileSystemService.getInstance().getByteProvider(batchLoadConfig.getFSRL(), monitor)) {
Msg.info(this, "Importing " + batchLoadConfig.getFSRL());
try (ByteProvider byteProvider = FileSystemService.getInstance()
.getByteProvider(batchLoadConfig.getFSRL(), true, monitor)) {
LoadSpec loadSpec = batchLoadConfig.getLoadSpec(selectedBatchGroupLoadSpec);
if (loadSpec == null) {
Msg.error(this,

33
Ghidra/Features/Base/src/test.slow/java/ghidra/formats/gfilesystem/FileSystemServiceTest.java
View File

@ -15,12 +15,16 @@
*/
package ghidra.formats.gfilesystem;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import org.junit.*;
import org.junit.Before;
import org.junit.Test;
import generic.test.AbstractGenericTest;
import ghidra.app.util.bin.ByteProvider;
import ghidra.test.AbstractGhidraHeadedIntegrationTest;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@ -43,34 +47,27 @@ public class FileSystemServiceTest extends AbstractGhidraHeadedIntegrationTest {
File localFile = new File(fssTestDir, "file.txt");
FileUtilities.writeStringToFile(localFile, "this is a test");
FSRL localFSRL = fsService.getLocalFSRL(localFile);
localFSRL = fsService.getFullyQualifiedFSRL(localFSRL, monitor);
File localResult = fsService.getFile(localFSRL, monitor);
Assert.assertNotNull(localFSRL.getMD5());
Assert.assertEquals(localFile, localResult);
try (ByteProvider byteProvider = fsService.getByteProvider(localFSRL, true, monitor)) {
assertEquals(localFile, byteProvider.getFile());
}
}
/**
* Verifies that a fully qualified FSRL with MD5 generates a IOException failure
* when the original file was changed.
*
* @throws IOException
* @throws CancelledException
*/
@Test
public void testChangedLocalFile() throws IOException, CancelledException {
// Verifies that a fully qualified FSRL with MD5 generates a IOException failure
// when the original file was changed.
File localFile = new File(fssTestDir, "file.txt");
FileUtilities.writeStringToFile(localFile, "this is a test");
FSRL localFSRL = fsService.getLocalFSRL(localFile);
localFSRL = fsService.getFullyQualifiedFSRL(localFSRL, monitor);
FileUtilities.writeStringToFile(localFile, "this is a test with additional bytes");
try {
File localResult2 = fsService.getFile(localFSRL, monitor);
Assert.fail("Should not get here, got: " + localResult2);
try (ByteProvider byteProvider = fsService.getByteProvider(localFSRL, false, monitor)) {
fail("Should not get here, got: " + byteProvider.getFSRL());
}
catch (IOException ioe) {
Assert.assertTrue(ioe.getMessage().contains("Exact file no longer exists"));
catch (IOException e) {
assertTrue(e.getMessage().contains("hash has changed"));
}
}
}

110
Ghidra/Features/Base/src/test/java/ghidra/app/util/bin/FileByteProviderTest.java Normal file
View File

@ -0,0 +1,110 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.bin;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import java.nio.file.AccessMode;
import java.util.Arrays;
import org.junit.Test;
import generic.test.AbstractGenericTest;
import ghidra.formats.gfilesystem.FSUtilities;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import utilities.util.FileUtilities;
public class FileByteProviderTest extends AbstractGenericTest {
/*
* "NN 01 NN 03 NN 05 NN 07 NN 09"... (NN = blockNumber, 00-FF = offset in block)
*/
private ByteArrayProvider patternedBAP(int bs, int count) {
byte[] bytes = new byte[bs * count];
for (int blockNum = 0; blockNum < count; blockNum++) {
int blockStart = blockNum * bs;
Arrays.fill(bytes, blockStart, blockStart + bs, (byte) blockNum);
for (int i = 1; i < bs; i += 2) {
bytes[i + blockStart] = (byte) (i % 256);
}
}
return new ByteArrayProvider(bytes);
}
@Test
public void testSmallRead() throws IOException {
File file1 = createTempFileForTest("file1");
FileUtilities.writeStringToFile(file1, "testing\nsecond line");
try (FileByteProvider fbp = new FileByteProvider(file1, null, AccessMode.READ)) {
BinaryReader br = new BinaryReader(fbp, true);
assertEquals("testing", br.readAsciiString(0));
assertEquals("second line", br.readAsciiString(8));
}
}
@Test
public void testReadAtBuffersBoundaries() throws IOException, CancelledException {
File file1 = createTempFileForTest("file1");
int bs = FileByteProvider.BUFFER_SIZE;
FSUtilities.copyByteProviderToFile(patternedBAP(bs, 5), file1, TaskMonitor.DUMMY);
try (FileByteProvider fbp = new FileByteProvider(file1, null, AccessMode.READ)) {
BinaryReader br = new BinaryReader(fbp, false /*BE*/);
assertEquals(5 * bs, fbp.length());
assertEquals(0x0001, br.readUnsignedShort(0));
assertEquals(0x00ff, br.readUnsignedShort(bs - 2));
assertEquals(0x0101, br.readUnsignedShort(bs));
assertEquals(0x01ff, br.readUnsignedShort(bs + bs - 2));
assertEquals(0x0401, br.readUnsignedShort(bs * 4));
}
}
@Test
public void testReadStraddleBuffersBoundaries() throws IOException, CancelledException {
File file1 = createTempFileForTest("file1");
int bs = FileByteProvider.BUFFER_SIZE;
FSUtilities.copyByteProviderToFile(patternedBAP(bs, 5), file1, TaskMonitor.DUMMY);
try (FileByteProvider fbp = new FileByteProvider(file1, null, AccessMode.READ)) {
BinaryReader br = new BinaryReader(fbp, false /*BE*/);
assertEquals(5 * bs, fbp.length());
assertEquals(0x00ff0101, br.readUnsignedInt(bs - 2));
assertEquals(0x01ff0201, br.readUnsignedInt(bs + bs - 2));
}
}
@Test
public void testReadMultiStraddleBuffersBoundaries() throws IOException, CancelledException {
File file1 = createTempFileForTest("file1");
int bs = FileByteProvider.BUFFER_SIZE;
FSUtilities.copyByteProviderToFile(patternedBAP(bs, 5), file1, TaskMonitor.DUMMY);
try (FileByteProvider fbp = new FileByteProvider(file1, null, AccessMode.READ)) {
assertEquals(5 * bs, fbp.length());
byte[] bytes = fbp.readBytes(bs - 2, bs + 4); // read from 3 adjacent blocks, 2+bs+2
BinaryReader br = new BinaryReader(new ByteArrayProvider(bytes), false /*BE*/);
assertEquals(0x00ff0101, br.readUnsignedInt(0));
assertEquals(0x01ff0201, br.readUnsignedInt(bs));
}
}
}

163
Ghidra/Features/Base/src/test/java/ghidra/formats/gfilesystem/FileCacheTest.java
View File

@ -15,21 +15,24 @@
*/
package ghidra.formats.gfilesystem;
import static org.junit.Assert.*;
import java.io.*;
import java.util.Arrays;
import org.junit.*;
import generic.test.AbstractGenericTest;
import ghidra.app.util.bin.ObfuscatedInputStream;
import ghidra.formats.gfilesystem.FileCache.FileCacheEntry;
import ghidra.formats.gfilesystem.FileCache.FileCacheEntryBuilder;
import ghidra.util.DateUtils;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import utilities.util.FileUtilities;
public class FileCacheTest extends AbstractGenericTest {
private FileCache cache;
private File cacheDir;
private TaskMonitor monitor = TaskMonitor.DUMMY;
@Before
public void setup() throws IOException {
@ -46,12 +49,22 @@ public class FileCacheTest extends AbstractGenericTest {
return new ByteArrayInputStream(s.getBytes());
}
FileCacheEntry createCacheFile(String payload) throws IOException {
// lie about the sizeHint to force the cache entry to be saved into a file instead of
// memory only
FileCacheEntryBuilder fceBuilder = cache.createCacheEntryBuilder(Integer.MAX_VALUE);
fceBuilder.write(payload.getBytes());
FileCacheEntry fce = fceBuilder.finish();
return fce;
}
@Test
public void testPurge() throws IOException, CancelledException {
public void testPurge() throws IOException {
cache = new FileCache(cacheDir);
FileCacheEntry cfi = cache.addStream(toIS("This is a test1"), monitor);
Assert.assertTrue(cfi.file.exists());
FileCacheEntry fce = createCacheFile("blah");
Assert.assertTrue(fce.file.exists());
File f1 = new File(cacheDir, "file1");
FileUtilities.writeStringToFile(f1, "test");
@ -61,64 +74,148 @@ public class FileCacheTest extends AbstractGenericTest {
FileUtilities.writeStringToFile(f2, "test2");
cache.purge();
Assert.assertFalse(cfi.file.exists());
Assert.assertFalse(fce.file.exists());
Assert.assertTrue(f1.exists());
Assert.assertTrue(f2.exists());
}
@Test
public void testAgeOff() throws IOException, CancelledException {
public void testAgeOff() throws IOException {
// hack, delete lastmaint file to force a maint event during next cache startup
File lastMaintFile = new File(cacheDir, ".lastmaint");
lastMaintFile.delete();
cache = new FileCache(cacheDir);
waitForCleanup(); // don't let the cache delete the file we are about to create
FileCacheEntry cfi = cache.addStream(toIS("This is a test1"), monitor);
Assert.assertTrue(cfi.file.exists());
FileCacheEntry fce = createCacheFile("test");
Assert.assertTrue(fce.file.exists());
cfi.file.setLastModified(System.currentTimeMillis() - (DateUtils.MS_PER_DAY * 5));
// backdate the file so it should appear to be old and age-off-able
fce.file.setLastModified(System.currentTimeMillis() - (DateUtils.MS_PER_DAY * 5));
// hack, delete lastmaint file to force a maint event during next cache startup
lastMaintFile.delete();
cache = new FileCache(cacheDir);
// the file added before should have been deleted by the startup of cache2
waitForCleanup();
assertFalse(fce.file.exists());
}
@Test
public void testCacheFileBadFilename() throws IOException {
// test to ensure bad filenames in the cache dir don't cause problem
// hack, delete lastmaint file to force a maint event during next cache startup
File lastMaintFile = new File(cacheDir, ".lastmaint");
lastMaintFile.delete();
cache = new FileCache(cacheDir);
waitForCleanup(); // don't let the cache delete the file we are about to create
// the file added before should have been purged by the startup of cache2
waitForCondition(() -> cfi.file.exists());
FileCacheEntry fce = createCacheFile("test");
Assert.assertTrue(fce.file.exists());
// backdate the file so it should appear to be old and age-off-able
fce.file.setLastModified(System.currentTimeMillis() - (DateUtils.MS_PER_DAY * 5));
// do same for file with bad filename
File badFile = new File(fce.file.getParentFile(), "bad_filename");
FileUtilities.writeStringToFile(badFile, "bad file contents");
badFile.setLastModified(System.currentTimeMillis() - (DateUtils.MS_PER_DAY * 5));
// hack, delete lastmaint file to force a maint event during next cache startup
lastMaintFile.delete();
cache = new FileCache(cacheDir);
// the file added before should have been deleted by the startup of cache2
waitForCleanup();
assertFalse(fce.file.exists());
assertTrue(badFile.exists());
}
@Test
public void testAddFile() throws IOException, CancelledException {
public void testAddFile() throws IOException {
cache = new FileCache(cacheDir);
File tmpFile = createTempFile("filecacheaddfile");
FileUtilities.writeStringToFile(tmpFile, "This is a test1");
FileCacheEntry fce = cache.addFile(tmpFile, monitor);
Assert.assertTrue(fce.file.exists());
Assert.assertEquals("10428da10f5aa2793cb73c0b680e1621", fce.md5);
Assert.assertEquals(1, cache.getFileAddCount());
FileCacheEntry fce = createCacheFile("This is a test1");
assertTrue(fce.file.exists());
assertEquals("10428da10f5aa2793cb73c0b680e1621", fce.md5);
}
@Test
public void testAddStream() throws IOException, CancelledException {
public void testFileObfuscated() throws IOException {
cache = new FileCache(cacheDir);
FileCacheEntry fce = cache.addStream(toIS("This is a test1"), monitor);
Assert.assertTrue(fce.file.exists());
Assert.assertEquals("10428da10f5aa2793cb73c0b680e1621", fce.md5);
Assert.assertEquals(1, cache.getFileAddCount());
FileCacheEntry fce = createCacheFile("This is a test1");
assertTrue(fce.file.exists());
assertEquals("10428da10f5aa2793cb73c0b680e1621", fce.md5);
byte[] fileBytes = FileUtilities.getBytesFromFile(fce.file);
assertFalse(Arrays.equals("This is a test1".getBytes(), fileBytes));
try (ObfuscatedInputStream ois = new ObfuscatedInputStream(new FileInputStream(fce.file))) {
byte[] buffer = new byte[100];
int bytesRead = ois.read(buffer);
assertEquals(15, bytesRead);
assertTrue(Arrays.equals("This is a test1".getBytes(), 0, 15, buffer, 0, 15));
}
}
@Test
public void testAddStreamPush() throws IOException, CancelledException {
public void testAddSmallFile() throws IOException {
cache = new FileCache(cacheDir);
FileCacheEntry fce = cache.pushStream((os) -> {
FileUtilities.copyStreamToStream(toIS("This is a test1"), os, monitor);
}, monitor);
Assert.assertTrue(fce.file.exists());
Assert.assertEquals("10428da10f5aa2793cb73c0b680e1621", fce.md5);
Assert.assertEquals(1, cache.getFileAddCount());
FileCacheEntryBuilder fceBuilder = cache.createCacheEntryBuilder(-1);
fceBuilder.write("This is a test1".getBytes());
FileCacheEntry fce = fceBuilder.finish();
assertNull(fce.file);
assertEquals("10428da10f5aa2793cb73c0b680e1621", fce.md5);
assertEquals(15, fce.bytes.length);
}
@Test
public void testBoundaryCondition() throws IOException {
// test that writing 1 byte less than size cutoff doesn't trigger switch to disk file
cache = new FileCache(cacheDir);
FileCacheEntryBuilder fceBuilder = cache.createCacheEntryBuilder(-1);
byte[] bytes = new byte[FileCache.MAX_INMEM_FILESIZE - 1];
fceBuilder.write(bytes);
FileCacheEntry fce = fceBuilder.finish();
assertNull(fce.file);
assertEquals(FileCache.MAX_INMEM_FILESIZE - 1, fce.bytes.length);
}
@Test
public void testBoundaryCondition_Grow() throws IOException {
// test that writing more than size cutoff does trigger switch to disk file
cache = new FileCache(cacheDir);
FileCacheEntryBuilder fceBuilder = cache.createCacheEntryBuilder(-1);
byte[] bytes = new byte[FileCache.MAX_INMEM_FILESIZE - 1];
fceBuilder.write(bytes);
fceBuilder.write(0);
fceBuilder.write(0);
FileCacheEntry fce = fceBuilder.finish();
assertNotNull(fce.file);
assertEquals("4eda5bcf5ef0cd4066425006dba9ffaa", fce.md5);
}
@Test
public void testLargeFile() throws IOException {
cache = new FileCache(cacheDir);
FileCacheEntryBuilder fceBuilder =
cache.createCacheEntryBuilder(FileCache.MAX_INMEM_FILESIZE + 1);
byte[] bytes = new byte[FileCache.MAX_INMEM_FILESIZE + 1];
fceBuilder.write(bytes);
FileCacheEntry fce = fceBuilder.finish();
assertNotNull(fce.file);
assertEquals("4eda5bcf5ef0cd4066425006dba9ffaa", fce.md5);
}
private void waitForCleanup() {

71
Ghidra/Features/Base/src/test/java/ghidra/formats/gfilesystem/crypto/CachedPasswordProviderTest.java Normal file
View File

@ -0,0 +1,71 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import static org.junit.Assert.*;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.List;
import org.junit.*;
import generic.test.AbstractGenericTest;
import ghidra.formats.gfilesystem.FSRL;
import util.CollectionUtils;
public class CachedPasswordProviderTest extends AbstractGenericTest {
private CryptoProviders cryptoProviders = CryptoProviders.getInstance();
private List<PasswordValue> getPasswords(CryptoSession cryptoSession, String fsrlStr)
throws MalformedURLException {
return CollectionUtils
.asList(cryptoSession.getPasswordsFor(FSRL.fromString(fsrlStr), "badbeef"));
}
private PopupGUIPasswordProvider popupGUIPasswordProvider;
@Before
public void setUp() {
popupGUIPasswordProvider =
cryptoProviders.getCryptoProviderInstance(PopupGUIPasswordProvider.class);
cryptoProviders.unregisterCryptoProvider(popupGUIPasswordProvider);
cryptoProviders.getCachedCryptoProvider().clearCache();
}
@After
public void tearDown() {
if (popupGUIPasswordProvider != null) {
cryptoProviders.registerCryptoProvider(popupGUIPasswordProvider);
}
}
@Test
public void testPassword() throws IOException {
try (CryptoSession cryptoSession = cryptoProviders.newSession()) {
assertEquals(0, getPasswords(cryptoSession, "file:///fake/path/file1.txt").size());
// shouldn't match passwords: file1.txt to file2.txt
cryptoSession.addSuccessfulPassword(FSRL.fromString("file:///fake/path/file1.txt"),
PasswordValue.wrap("password_for_file2.txt".toCharArray()));
assertEquals(1, getPasswords(cryptoSession, "file:///fake/path/file1.txt").size());
assertEquals(0, getPasswords(cryptoSession, "file:///fake/path/file2.txt").size());
// should match file1.txt in 2 directories
assertEquals(1, getPasswords(cryptoSession, "file:///2nd/fake/path/file1.txt").size());
}
}
}

105
Ghidra/Features/Base/src/test/java/ghidra/formats/gfilesystem/crypto/CmdLinePasswordProviderTest.java Normal file
View File

@ -0,0 +1,105 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.formats.gfilesystem.crypto;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.List;
import org.junit.*;
import generic.test.AbstractGenericTest;
import ghidra.formats.gfilesystem.FSRL;
import util.CollectionUtils;
import utilities.util.FileUtilities;
public class CmdLinePasswordProviderTest extends AbstractGenericTest {
private CryptoProviders cryptoProviders = CryptoProviders.getInstance();
private List<PasswordValue> getPasswords(CryptoSession cryptoSession, String fsrlStr)
throws MalformedURLException {
return CollectionUtils
.asList(cryptoSession.getPasswordsFor(FSRL.fromString(fsrlStr), "badbeef"));
}
private String origCmdLinePasswordValue;
private PopupGUIPasswordProvider popupGUIPasswordProvider;
@Before
public void setUp() {
popupGUIPasswordProvider =
cryptoProviders.getCryptoProviderInstance(PopupGUIPasswordProvider.class);
cryptoProviders.unregisterCryptoProvider(popupGUIPasswordProvider);
cryptoProviders.getCachedCryptoProvider().clearCache();
origCmdLinePasswordValue = System
.getProperty(CmdLinePasswordProvider.CMDLINE_PASSWORD_PROVIDER_PROPERTY_NAME, null);
}
@After
public void tearDown() {
if (popupGUIPasswordProvider != null) {
cryptoProviders.registerCryptoProvider(popupGUIPasswordProvider);
}
if (origCmdLinePasswordValue == null) {
System.clearProperty(CmdLinePasswordProvider.CMDLINE_PASSWORD_PROVIDER_PROPERTY_NAME);
}
else {
System.setProperty(CmdLinePasswordProvider.CMDLINE_PASSWORD_PROVIDER_PROPERTY_NAME,
origCmdLinePasswordValue);
}
}
@Test
public void testPassword() throws IOException {
File pwdFile = createTempFile("password_test");
FileUtilities.writeStringToFile(pwdFile,
"password_for_file1.txt\tfile1.txt\n\npassword_for_file2.txt\t/path/to/file2.txt\ngeneral_password\n");
System.setProperty(CmdLinePasswordProvider.CMDLINE_PASSWORD_PROVIDER_PROPERTY_NAME,
pwdFile.getPath());
try (CryptoSession cryptoSession = cryptoProviders.newSession()) {
List<PasswordValue> pwdList =
getPasswords(cryptoSession, "file:///fake/path/file1.txt");
assertEquals(2, pwdList.size());
assertEquals("password_for_file1.txt", String.valueOf(pwdList.get(0).getPasswordChars()));
assertEquals("general_password", String.valueOf(pwdList.get(1).getPasswordChars()));
}
}
@Test
public void testPassword2() throws IOException {
File pwdFile = createTempFile("password_test");
FileUtilities.writeStringToFile(pwdFile, "password_for_file1.txt\t/path/to/a/file1.txt");
System.setProperty(CmdLinePasswordProvider.CMDLINE_PASSWORD_PROVIDER_PROPERTY_NAME,
pwdFile.getPath());
try (CryptoSession cryptoSession = cryptoProviders.newSession()) {
List<PasswordValue> pwdList =
getPasswords(cryptoSession, "file:///not_matching/path/file1.txt");
assertEquals(0, pwdList.size());
List<PasswordValue> list2 = getPasswords(cryptoSession, "file:///path/to/a/file1.txt");
assertEquals(1, list2.size());
}
}
}

58
Ghidra/Features/FileFormats/src/main/java/ghidra/file/eclipse/AndroidProjectCreator.java
View File

@ -21,6 +21,7 @@ import java.util.List;
import org.jdom.*;
import generic.jar.ResourceFile;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.importer.MessageLog;
import ghidra.file.formats.android.dex.DexToJarFileSystem;
import ghidra.file.formats.android.xml.AndroidXmlFileSystem;
@ -53,7 +54,7 @@ public class AndroidProjectCreator {
androidDirectory = directory;
}
private GFile apkFile;
private FSRL apkFileFSRL;
private File eclipseProjectDirectory;
private File srcDirectory;
private File genDirectory;
@ -65,20 +66,20 @@ public class AndroidProjectCreator {
new ResourceFile(androidDirectory, "eclipse-classpath");
private MessageLog log = new MessageLog();
private FileSystemService fsService = FileSystemService.getInstance();
public AndroidProjectCreator(GFile apkFile, File eclipseProjectDirectory) {
this.apkFile = apkFile;
public AndroidProjectCreator(FSRL apkFileFSRL, File eclipseProjectDirectory) {
this.apkFileFSRL = apkFileFSRL;
this.eclipseProjectDirectory = eclipseProjectDirectory;
}
public void create(TaskMonitor monitor) throws IOException, CancelledException {
createEclipseProjectDirectories();
try (ZipFileSystem fs = FileSystemService.getInstance()
.mountSpecificFileSystem(
apkFile.getFSRL(), ZipFileSystem.class, monitor)) {
try (ZipFileSystem fs =
fsService.mountSpecificFileSystem(apkFileFSRL, ZipFileSystem.class, monitor)) {
List<GFile> listing = fs.getListing(null);
processListing(eclipseProjectDirectory, listing, monitor);
processListing(eclipseProjectDirectory, fs, listing, monitor);
}
File destProjectFile =
@ -97,7 +98,7 @@ public class AndroidProjectCreator {
Document projectDoc = XmlUtilities.readDocFromFile(projectFile);
Element nameElement = projectDoc.getRootElement().getChild("name");
if (nameElement != null) {
nameElement.setText(apkFile.getName());
nameElement.setText(apkFileFSRL.getName());
XmlUtilities.writeDocToFile(projectDoc, projectFile);
}
}
@ -113,7 +114,8 @@ public class AndroidProjectCreator {
assetDirectory = FileUtilities.checkedMkdirs(new File(eclipseProjectDirectory, "asset"));
}
private void processListing(File outputDirectory, List<GFile> listing, TaskMonitor monitor)
private void processListing(File outputDirectory, GFileSystem fs, List<GFile> listing,
TaskMonitor monitor)
throws IOException, CancelledException {
for (GFile child : listing) {
@ -130,14 +132,13 @@ public class AndroidProjectCreator {
}
File subDir = new File(outputDirectory, childName);
FileUtilities.checkedMkdir(subDir);
processListing(subDir, child.getListing(), monitor);
processListing(subDir, fs, child.getListing(), monitor);
continue;
}
File cacheFile = FileSystemService.getInstance().getFile(child.getFSRL(), monitor);
try {
try (ByteProvider childBP = fs.getByteProvider(child, monitor)) {
if (childName.endsWith(".xml") &&
AndroidXmlFileSystem.isAndroidXmlFile(cacheFile, monitor)) {
AndroidXmlFileSystem.isAndroidXmlFile(childBP, monitor)) {
processXML(outputDirectory, child, monitor);
}
else if (childName.endsWith("classes.dex")) {
@ -145,13 +146,13 @@ public class AndroidProjectCreator {
}
else if (childName.endsWith("resources.arsc")) {
//TODO convert resources file back into actual resources
copyFile(cacheFile, outputDirectory, child.getName(), monitor);
copyStream(childBP, outputDirectory, child.getName(), monitor);
}
else if (childName.endsWith(".class")) {
processClass(outputDirectory, child, monitor);
}
else {
copyFile(cacheFile, outputDirectory, childName, monitor);
copyStream(childBP, outputDirectory, childName, monitor);
}
}
catch (Exception e) {
@ -164,9 +165,8 @@ public class AndroidProjectCreator {
private void processDex(File outputDirectory, GFile dexFile, TaskMonitor monitor)
throws IOException, CancelledException {
try (DexToJarFileSystem fs = FileSystemService.getInstance()
.mountSpecificFileSystem(
dexFile.getFSRL(), DexToJarFileSystem.class, monitor)) {
try (DexToJarFileSystem fs = fsService.mountSpecificFileSystem(dexFile.getFSRL(),
DexToJarFileSystem.class, monitor)) {
GFile jarFile = fs.getJarFile();
processJar(srcDirectory, jarFile.getFSRL(), monitor);
}
@ -190,8 +190,8 @@ public class AndroidProjectCreator {
File destClassFile = new File(outputDirectory, classFileName);
//File destJavaFile = new File(outputDirectory, PathUtils.stripExt(classFileName) + ".java");
File classFile = FileSystemService.getInstance().getFile(classGFile.getFSRL(), monitor);
copyFile(classFile, outputDirectory, classFileName, monitor);
InputStream is = classGFile.getFilesystem().getInputStream(classGFile, monitor);
copyStream(is, outputDirectory, classFileName, monitor);
JadProcessWrapper wrapper = new JadProcessWrapper(destClassFile);
JadProcessController controller = new JadProcessController(wrapper, classGFile.getName());
@ -201,9 +201,8 @@ public class AndroidProjectCreator {
private void processXML(File outputDirectory, GFile containerFile, TaskMonitor monitor)
throws CancelledException {
try (AndroidXmlFileSystem fs = FileSystemService.getInstance()
.mountSpecificFileSystem(
containerFile.getFSRL(), AndroidXmlFileSystem.class, monitor)) {
try (AndroidXmlFileSystem fs = fsService.mountSpecificFileSystem(containerFile.getFSRL(),
AndroidXmlFileSystem.class, monitor)) {
GFile xmlFile = fs.getPayloadFile();
copyStream(fs.getInputStream(xmlFile, monitor), outputDirectory,
containerFile.getName(), monitor);
@ -229,16 +228,9 @@ public class AndroidProjectCreator {
}
}
private static File copyFile(File inputFile, File outputDirectory, String outputName,
TaskMonitor monitor) throws IOException {
FileUtilities.checkedMkdirs(outputDirectory);
File destFile = new File(outputDirectory, outputName);
monitor.setMessage("Copying [" + inputFile.getName() + "] to Eclipse project...");
FileUtilities.copyFile(inputFile, destFile, false, monitor);
return destFile;
private static File copyStream(ByteProvider provider, File outputDirectory,
String outputName, TaskMonitor monitor) throws IOException {
return copyStream(provider.getInputStream(0), outputDirectory, outputName, monitor);
}
private static File copyStream(InputStream streamToCopy, File outputDirectory,

4
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/apk/ApkFileSystem.java
View File

@ -16,7 +16,6 @@
package ghidra.file.formats.android.apk;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
@ -56,8 +55,7 @@ public class ApkFileSystem extends GFileSystemBase {
}
@Override
protected InputStream getData(GFile file, TaskMonitor monitor)
throws IOException, CancelledException, CryptoException {
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor) {
return null;
}

55
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/bootimg/BootImageFileSystem.java
View File

@ -15,20 +15,15 @@
*/
package ghidra.file.formats.android.bootimg;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.*;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.GFile;
import ghidra.formats.gfilesystem.GFileImpl;
import ghidra.formats.gfilesystem.GFileSystemBase;
import ghidra.app.util.bin.ByteProviderWrapper;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.factory.GFileSystemBaseFactory;
import ghidra.formats.gfilesystem.fileinfo.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.CryptoException;
import ghidra.util.task.TaskMonitor;
@ -95,38 +90,46 @@ public class BootImageFileSystem extends GFileSystemBase {
}
@Override
public String getInfo(GFile file, TaskMonitor monitor) {
public FileAttributes getFileAttributes(GFile file, TaskMonitor monitor) {
if (file == kernelFile) {
return "This is the actual KERNEL for the android device. You can analyze this file.";
return FileAttributes.of(
FileAttribute.create(FileAttributeType.COMMENT_ATTR,
"This is the actual KERNEL for the android device. You can analyze this file."));
}
else if (file == ramdiskFile) {
return "This is a ramdisk, it is a GZIP file containing a CPIO archive.";
if (file == ramdiskFile) {
return FileAttributes.of(
FileAttribute.create(FileAttributeType.COMMENT_ATTR,
"This is a ramdisk, it is a GZIP file containing a CPIO archive."));
}
else if (file == secondStageFile) {
return "This is a second stage loader file. It appears unused at this time.";
return FileAttributes.of(
FileAttribute.create(FileAttributeType.COMMENT_ATTR,
"This is a second stage loader file. It appears unused at this time."));
}
return null;
}
@Override
protected InputStream getData(GFile file, TaskMonitor monitor)
throws IOException, CancelledException, CryptoException {
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
long offset;
long size;
if (file == kernelFile) {
byte[] kernelBytes =
provider.readBytes(header.getKernelOffset(), header.getKernelSize());
return new ByteArrayInputStream(kernelBytes);
offset = header.getKernelOffset();
size = header.getKernelSize();
}
else if (file == ramdiskFile) {
byte[] ramDiskBytes =
provider.readBytes(header.getRamdiskOffset(), header.getRamdiskSize());
return new ByteArrayInputStream(ramDiskBytes);
offset = header.getRamdiskOffset();
size = header.getRamdiskSize();
}
else if (file == secondStageFile) {
byte[] secondStageBytes =
provider.readBytes(header.getSecondOffset(), header.getSecondSize());
return new ByteArrayInputStream(secondStageBytes);
offset = header.getSecondOffset();
size = header.getSecondSize();
}
return null;
else {
return null;
}
return new ByteProviderWrapper(provider, offset, size, file.getFSRL());
}
}

11
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/bootimg/BootImageHeader.java
View File

@ -16,6 +16,7 @@
package ghidra.file.formats.android.bootimg;
import ghidra.app.util.bin.StructConverter;
import ghidra.util.NumericUtilities;
public abstract class BootImageHeader implements StructConverter {
/**
@ -31,6 +32,16 @@ public abstract class BootImageHeader implements StructConverter {
*/
public abstract int getPageSize();
/**
* Aligns a value upwards to nearest page boundary.
*
* @param value unsigned value to align
* @return value rounded up to next page size (if not already aligned)
*/
public long pageAlign(long value) {
return NumericUtilities.getUnsignedAlignedValue(value, getPageSize());
}
/**
* Returns the kernel size, as defined in the header.
* @return the kernel size, as defined in the header

13
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/bootimg/BootImageHeaderV0.java
View File

@ -19,7 +19,6 @@ import java.io.IOException;
import ghidra.app.util.bin.BinaryReader;
import ghidra.program.model.data.*;
import ghidra.util.NumericUtilities;
import ghidra.util.exception.DuplicateNameException;
/**
@ -83,9 +82,9 @@ public class BootImageHeaderV0 extends BootImageHeader {
/**
* n = (kernel_size + page_size - 1) / page_size
*/
@Override
public int getKernelPageCount() {
return (int) NumericUtilities.getUnsignedAlignedValue(Integer.toUnsignedLong(kernel_size),
Integer.toUnsignedLong(page_size));
return (int) (pageAlign(kernel_size) / page_size);
}
@Override
@ -105,9 +104,9 @@ public class BootImageHeaderV0 extends BootImageHeader {
/**
* m = (ramdisk_size + page_size - 1) / page_size
*/
@Override
public int getRamdiskPageCount() {
return (int) NumericUtilities.getUnsignedAlignedValue(Integer.toUnsignedLong(ramdisk_size),
Integer.toUnsignedLong(page_size));
return (int) (pageAlign(ramdisk_size) / page_size);
}
@Override
@ -127,9 +126,9 @@ public class BootImageHeaderV0 extends BootImageHeader {
/**
* o = (second_size + page_size - 1) / page_size
*/
@Override
public int getSecondPageCount() {
return (int) NumericUtilities.getUnsignedAlignedValue(Integer.toUnsignedLong(second_size),
Integer.toUnsignedLong(page_size));
return (int) (pageAlign(second_size) / page_size);
}
/**

6
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/bootimg/BootImageHeaderV1.java
View File

@ -9,7 +9,6 @@ import ghidra.app.util.bin.BinaryReader;
import ghidra.program.model.data.DataType;
import ghidra.program.model.data.Structure;
import ghidra.util.InvalidNameException;
import ghidra.util.NumericUtilities;
import ghidra.util.exception.DuplicateNameException;
/**
@ -38,11 +37,10 @@ public class BootImageHeaderV1 extends BootImageHeaderV0 {
/**
* p = (recovery_dtbo_size + page_size - 1) / page_size
* @return the recovery DTBO adjusted size
* @return the recovery DTBO adjusted size, as page counts
*/
public int getRecoveryDtboSizeAdjusted() {
return (int) NumericUtilities.getUnsignedAlignedValue(
Integer.toUnsignedLong(recovery_dtbo_size), Integer.toUnsignedLong(getPageSize()));
return (int) (pageAlign(Integer.toUnsignedLong(recovery_dtbo_size)) / getPageSize());
}
/**

6
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/bootimg/BootImageHeaderV2.java
View File

@ -21,7 +21,6 @@ import ghidra.app.util.bin.BinaryReader;
import ghidra.program.model.data.DataType;
import ghidra.program.model.data.Structure;
import ghidra.util.InvalidNameException;
import ghidra.util.NumericUtilities;
import ghidra.util.exception.DuplicateNameException;
/**
@ -48,11 +47,10 @@ public class BootImageHeaderV2 extends BootImageHeaderV1 {
/**
* q = (dtb_size + page_size - 1) / page_size
* @return the DTB adjusted size
* @return the DTB adjusted size, as page counts
*/
public int getDtbSizeAdjusted() {
return (int) NumericUtilities.getUnsignedAlignedValue(Integer.toUnsignedLong(dtb_size),
Integer.toUnsignedLong(getPageSize()));
return (int) (pageAlign(Integer.toUnsignedLong(dtb_size)) / getPageSize());
}
/**

14
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/bootimg/BootImageHeaderV3.java
View File

@ -18,11 +18,7 @@ package ghidra.file.formats.android.bootimg;
import java.io.IOException;
import ghidra.app.util.bin.BinaryReader;
import ghidra.program.model.data.ArrayDataType;
import ghidra.program.model.data.DataType;
import ghidra.program.model.data.Structure;
import ghidra.program.model.data.StructureDataType;
import ghidra.util.NumericUtilities;
import ghidra.program.model.data.*;
import ghidra.util.exception.DuplicateNameException;
/**
@ -71,8 +67,7 @@ public class BootImageHeaderV3 extends BootImageHeader {
*/
@Override
public int getKernelPageCount() {
return (int) NumericUtilities.getUnsignedAlignedValue(Integer.toUnsignedLong(kernel_size),
Integer.toUnsignedLong(BootImageConstants.V3_PAGE_SIZE));
return (int)(pageAlign(kernel_size) / BootImageConstants.V3_PAGE_SIZE);
}
@Override
@ -90,8 +85,8 @@ public class BootImageHeaderV3 extends BootImageHeader {
*/
@Override
public int getRamdiskPageCount() {
return (int) NumericUtilities.getUnsignedAlignedValue(Integer.toUnsignedLong(ramdisk_size),
Integer.toUnsignedLong(BootImageConstants.V3_PAGE_SIZE));
return (int) (pageAlign(Integer.toUnsignedLong(ramdisk_size)) /
BootImageConstants.V3_PAGE_SIZE);
}
@Override
@ -134,6 +129,7 @@ public class BootImageHeaderV3 extends BootImageHeader {
return header_version;
}
@Override
public String getCommandLine() {
return cmdline;
}

36
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/bootimg/VendorBootImageFileSystem.java
View File

@ -15,20 +15,15 @@
*/
package ghidra.file.formats.android.bootimg;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.*;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.GFile;
import ghidra.formats.gfilesystem.GFileImpl;
import ghidra.formats.gfilesystem.GFileSystemBase;
import ghidra.app.util.bin.ByteProviderWrapper;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.factory.GFileSystemBaseFactory;
import ghidra.formats.gfilesystem.fileinfo.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.CryptoException;
import ghidra.util.task.TaskMonitor;
@ -87,28 +82,31 @@ public class VendorBootImageFileSystem extends GFileSystemBase {
}
@Override
public String getInfo(GFile file, TaskMonitor monitor) {
public FileAttributes getFileAttributes(GFile file, TaskMonitor monitor) {
if (file == ramdiskFile) {
return "This is a ramdisk, it is a GZIP file containing a CPIO archive.";
return FileAttributes.of(
FileAttribute.create(FileAttributeType.COMMENT_ATTR,
"This is a ramdisk, it is a GZIP file containing a CPIO archive."));
}
else if (file == dtbFile) {
return "This is a DTB file. It appears unused at this time.";
return FileAttributes.of(
FileAttribute.create(FileAttributeType.COMMENT_ATTR,
"This is a DTB file. It appears unused at this time."));
}
return null;
}
@Override
protected InputStream getData(GFile file, TaskMonitor monitor)
throws IOException, CancelledException, CryptoException {
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
if (file == ramdiskFile) {
byte[] ramDiskBytes =
provider.readBytes(header.getVendorRamdiskOffset(), header.getVendorRamdiskSize());
return new ByteArrayInputStream(ramDiskBytes);
return new ByteProviderWrapper(provider, header.getVendorRamdiskOffset(),
Integer.toUnsignedLong(header.getVendorRamdiskSize()), file.getFSRL());
}
else if (file == dtbFile) {
byte[] dtbBytes = provider.readBytes(header.getDtbOffset(), header.getDtbSize());
return new ByteArrayInputStream(dtbBytes);
return new ByteProviderWrapper(provider, header.getDtbOffset(),
Integer.toUnsignedLong(header.getDtbSize()), file.getFSRL());
}
return null;
}

28
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/bootldr/AndroidBootLoaderFileSystem.java
View File

@ -16,24 +16,18 @@
package ghidra.file.formats.android.bootldr;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import ghidra.app.util.bin.BinaryReader;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.GFile;
import ghidra.formats.gfilesystem.GFileImpl;
import ghidra.formats.gfilesystem.GFileSystemBase;
import ghidra.app.util.bin.*;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.factory.GFileSystemBaseFactory;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.CryptoException;
import ghidra.util.task.TaskMonitor;
@FileSystemInfo(type = "androidbootloader", // ([a-z0-9]+ only)
description = "Android Boot Loader Image", factory = GFileSystemBaseFactory.class)
@FileSystemInfo(type = "androidbootloader", description = "Android Boot Loader Image", factory = GFileSystemBaseFactory.class)
public class AndroidBootLoaderFileSystem extends GFileSystemBase {
private List<GFileImpl> fileList = new ArrayList<>();
private List<Integer> offsetList = new ArrayList<>();
@ -71,18 +65,14 @@ public class AndroidBootLoaderFileSystem extends GFileSystemBase {
}
@Override
public String getInfo(GFile file, TaskMonitor monitor) {
return null;
}
@Override
protected InputStream getData(GFile file, TaskMonitor monitor)
throws IOException, CancelledException, CryptoException {
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
int index = fileList.indexOf(file);
if (index < 0) {
throw new IOException("Unknown file: " + file);
}
int offset = offsetList.get(index);
return provider.getInputStream(offset);
return new ByteProviderWrapper(provider, offset, file.getLength(), file.getFSRL());
}
}

35
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/dex/DexToJarFileSystem.java
View File

@ -15,7 +15,7 @@
*/
package ghidra.file.formats.android.dex;
import java.io.*;
import java.io.IOException;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@ -36,10 +36,8 @@ import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.factory.GFileSystemBaseFactory;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.CryptoException;
import ghidra.util.task.TaskMonitor;
import ghidra.util.task.UnknownProgressWrappingTaskMonitor;
import utilities.util.FileUtilities;
/**
* {@link GFileSystem} that converts a DEX file into a JAR file.
@ -58,14 +56,9 @@ public class DexToJarFileSystem extends GFileSystemBase {
}
@Override
protected InputStream getData(GFile file, TaskMonitor monitor)
throws IOException, CancelledException, CryptoException {
if (file.equals(jarFile)) {
FileCacheEntry jarFileInfo = getJarFile(monitor);
return new FileInputStream(jarFileInfo.file);
}
return null;
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
return file.equals(jarFile) ? getJarFile(jarFile.getFSRL(), monitor) : null;
}
@Override
@ -79,22 +72,22 @@ public class DexToJarFileSystem extends GFileSystemBase {
return DexConstants.isDexFile(provider);
}
private FileCacheEntry getJarFile(TaskMonitor monitor) throws CancelledException, IOException {
private ByteProvider getJarFile(FSRL jarFSRL, TaskMonitor monitor)
throws CancelledException, IOException {
TaskMonitor upwtm = new UnknownProgressWrappingTaskMonitor(monitor, 1);
upwtm.setMessage("Converting DEX to JAR...");
FSRLRoot targetFSRL = getFSRL();
FSRL containerFSRL = targetFSRL.getContainer();
File containerFile = fsService.getFile(containerFSRL, monitor);
FileCacheEntry derivedFileInfo =
fsService.getDerivedFilePush(containerFSRL, "dex2jar", (os) -> {
ByteProvider jarBP = fsService.getDerivedByteProviderPush(containerFSRL, jarFSRL,
"dex2jar", -1, (os) -> {
try (ZipOutputStream outputStream = new ZipOutputStream(os)) {
DexToJarExceptionHandler exceptionHandler = new DexToJarExceptionHandler();
DexFileReader reader =
new DexFileReader(FileUtilities.getBytesFromFile(containerFile));
byte[] containerFileBytes = provider.readBytes(0, provider.length());
DexFileReader reader = new DexFileReader(containerFileBytes);
DexFileNode fileNode = new DexFileNode();
try {
@ -154,21 +147,21 @@ public class DexToJarFileSystem extends GFileSystemBase {
}, monitor);
return derivedFileInfo;
return jarBP;
}
@Override
public void open(TaskMonitor monitor) throws CancelledException, IOException {
FileCacheEntry jarFileInfo = getJarFile(monitor);
ByteProvider jarBP = getJarFile(null, monitor);
FSRLRoot targetFSRL = getFSRL();
FSRL containerFSRL = targetFSRL.getContainer();
String baseName = FilenameUtils.removeExtension(containerFSRL.getName());
String jarName = baseName + ".jar";
FSRL jarFSRL = targetFSRL.withPathMD5(jarName, jarFileInfo.md5);
FSRL jarFSRL = targetFSRL.withPathMD5(jarName, jarBP.getFSRL().getMD5());
this.jarFile = GFileImpl.fromFilename(this, root, baseName + ".jar", false,
jarFileInfo.file.length(), jarFSRL);
jarBP.length(), jarFSRL);
}
@Override

11
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/dex/DexToSmaliFileSystem.java
View File

@ -15,7 +15,9 @@
*/
package ghidra.file.formats.android.dex;
import java.io.*;
import java.io.File;
import java.io.IOException;
import java.nio.file.AccessMode;
import java.util.*;
import org.apache.commons.io.FileUtils;
@ -24,6 +26,7 @@ import org.jf.baksmali.baksmali;
import org.jf.dexlib.DexFile;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.bin.FileByteProvider;
import ghidra.file.formats.android.dex.format.DexConstants;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
@ -42,10 +45,10 @@ public class DexToSmaliFileSystem extends GFileSystemBase {
}
@Override
protected InputStream getData(GFile file, TaskMonitor monitor)
throws IOException, CancelledException, CryptoException {
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
File entry = map.get(file);
return new FileInputStream(entry);
return new FileByteProvider(entry, file.getFSRL(), AccessMode.READ);
}
@Override

31
Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/android/fbpk/FBPK_FileSystem.java
View File

@ -16,17 +16,10 @@
package ghidra.file.formats.android.fbpk;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import ghidra.app.util.bin.BinaryReader;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.GFile;
import ghidra.formats.gfilesystem.GFileImpl;
import ghidra.formats.gfilesystem.GFileSystemBase;
import ghidra.app.util.bin.*;
import ghidra.formats.gfilesystem.*;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.factory.GFileSystemBaseFactory;
import ghidra.util.exception.CancelledException;
@ -71,17 +64,15 @@ public class FBPK_FileSystem extends GFileSystemBase {
}
@Override
public String getInfo(GFile file, TaskMonitor monitor) {
public ByteProvider getByteProvider(GFile file, TaskMonitor monitor)
throws IOException, CancelledException {
FBPK_Partition partition = map.get(file);
if (partition != null) {
return new ByteProviderWrapper(provider, partition.getDataStartOffset(),
Integer.toUnsignedLong(partition.getDataSize()), file.getFSRL());
}
return null;
}
@Override
protected InputStream getData(GFile file, TaskMonitor monitor)
throws IOException, CancelledException, CryptoException {
FBPK_Partition partition = map.get(file);
return provider.getInputStream(partition.getDataStartOffset());
}
}

Some files were not shown because too many files have changed in this diff Show More