GP-4245 - Split PDB Universal Analyzer into multiple phases with

analyzer state
This commit is contained in:
ghizard 2024-03-03 11:44:54 -05:00
parent 8746542a86
commit 76994b47a1
13 changed files with 540 additions and 218 deletions

View file

@ -196,9 +196,10 @@ public class Pagedump extends DumpFile {
try (AbstractPdb pdb = PdbParser.parse(pdbFile, readerOptions, monitor)) {
monitor.setMessage("PDB: Parsing " + pdbFile + "...");
pdb.deserialize();
DefaultPdbApplicator applicator = new DefaultPdbApplicator(pdb);
applicator.applyTo(program, dtm, program.getImageBase(), applicatorOptions,
(MessageLog) null);
DefaultPdbApplicator applicator =
new DefaultPdbApplicator(pdb, program, program.getDataTypeManager(),
program.getImageBase(), applicatorOptions, (MessageLog) null);
applicator.applyNoAnalysisState();
}
catch (PdbException | IOException | CancelledException e) {
Msg.error(this, e.getMessage());
@ -557,7 +558,7 @@ public class Pagedump extends DumpFile {
private boolean isValid(int flags) {
return (flags & 0x1) > 0;
}
private void walkPages(int page, long va, int depth, boolean lp) throws IOException {
long fileOffset = fileOffset(page);
if (fileOffset < 0) {

View file

@ -80,9 +80,9 @@ public class PdbDeveloperApplyDummyScript extends GhidraScript {
try (AbstractPdb pdb = PdbParser.parse(pdbFile, pdbReaderOptions, monitor)) {
monitor.setMessage("PDB: Parsing " + pdbFile + "...");
pdb.deserialize();
DefaultPdbApplicator applicator = new DefaultPdbApplicator(pdb);
applicator.applyTo(program, program.getDataTypeManager(), program.getImageBase(),
pdbApplicatorOptions, log);
DefaultPdbApplicator applicator = new DefaultPdbApplicator(pdb, program,
program.getDataTypeManager(), program.getImageBase(), pdbApplicatorOptions, log);
applicator.applyNoAnalysisState();
}
catch (PdbException | IOException e) {
log.appendMsg(getClass().getName(),

View file

@ -26,6 +26,8 @@ import ghidra.app.util.pdb.PdbProgramAttributes;
import ghidra.app.util.pdb.pdbapplicator.DefaultPdbApplicator;
import ghidra.app.util.pdb.pdbapplicator.PdbApplicatorOptions;
import ghidra.framework.*;
import ghidra.framework.cmd.BackgroundCommand;
import ghidra.framework.model.DomainObject;
import ghidra.framework.options.OptionType;
import ghidra.framework.options.Options;
import ghidra.program.model.address.AddressSetView;
@ -170,6 +172,29 @@ public class PdbUniversalAnalyzer extends AbstractAnalyzer {
return false;
}
return doAnalysis(program, pdbFile, pdbReaderOptions, pdbApplicatorOptions, log, monitor);
}
/**
* Initializes and calls the methods of the PdbApplicator pertaining to the various phases
* of PDB analysis. These methods can be called via scheduled background commands set with
* the appropriate analysis priorities to allow the work to be done at the appropriate time
* amongst other analyzers. The return PDB
* @param program the program to which the PDB is being applied
* @param pdbFile the PDB file to be applied
* @param pdbReaderOptions the PDB "reader" options to use
* @param pdbApplicatorOptions the PDB "applicator" options to use
* @param log the message log to which messages will be written
* @param monitor the task monitor
* @return {@code true} if the first phase of analysis has completed without error. Follow-on
* background commands will also have return values, which will include a {@code false} value
* upon user cancellation during those phases.
* @throws CancelledException upon user cancellation
*/
public static boolean doAnalysis(Program program, File pdbFile,
PdbReaderOptions pdbReaderOptions,
PdbApplicatorOptions pdbApplicatorOptions, MessageLog log, TaskMonitor monitor)
throws CancelledException {
PdbLog.message(
"================================================================================");
PdbLog.message(new Date(System.currentTimeMillis()).toString() + "\n");
@ -181,14 +206,43 @@ public class PdbUniversalAnalyzer extends AbstractAnalyzer {
try (AbstractPdb pdb = PdbParser.parse(pdbFile, pdbReaderOptions, monitor)) {
monitor.setMessage("PDB: Parsing " + pdbFile + "...");
pdb.deserialize();
DefaultPdbApplicator applicator = new DefaultPdbApplicator(pdb);
applicator.applyTo(program, program.getDataTypeManager(), program.getImageBase(),
pdbApplicatorOptions, log);
DefaultPdbApplicator applicator = new DefaultPdbApplicator(pdb, program,
program.getDataTypeManager(), program.getImageBase(), pdbApplicatorOptions, log);
applicator.applyDataTypesAndMainSymbolsAnalysis();
AutoAnalysisManager aam = AutoAnalysisManager.getAnalysisManager(program);
// TODO: Consider the various types of work we might want to do... they probably
// shouldn't all be the same priority.
// * function nested scopes, local/scope variables, static locals (might be in
// scopes). Includes register/stack scopes of local/scoped variables
// * parameters
// * code source module/file/line numbers
Msg.info(PdbUniversalAnalyzer.class,
NAME + ": scheduling PDB Function Internals Analysis");
// TODO: set this to appropriate priority (this is a guess for locals/params/scopes)
// Initial thought on priority: AnalysisPriority.FUNCTION_ANALYSIS.priority()
// From meeting:
// * before/after parameter ID
// different statement:
// * run before stack reference analysis runs (maybe turn it that one off)
// 902 Decompiler Parameter ID (DecompilerFunctionAnalyzer)
// 903 Stack (StackVariableAnalyzer)
aam.schedule(
new ProcessPdbFunctionInternalsCommand(pdbFile, pdbReaderOptions,
pdbApplicatorOptions, log),
AnalysisPriority.DATA_TYPE_PROPOGATION.after().after().after().priority());
// Following is intended to be the last PDB analysis background command
aam.schedule(
new PdbReportingBackgroundCommand(),
AnalysisPriority.DATA_TYPE_PROPOGATION.after().after().after().after().priority());
}
catch (PdbException | IOException e) {
log.appendMsg(getName(),
"Issue processing PDB file: " + pdbFile + ":\n " + e.toString());
log.appendMsg(NAME, "Issue processing PDB file: " + pdbFile + ":\n " + e.toString());
return false;
}
@ -274,4 +328,78 @@ public class PdbUniversalAnalyzer extends AbstractAnalyzer {
public static void setAllowRemoteOption(Program program, boolean allowRemote) {
PdbAnalyzerCommon.setAllowRemoteOption(NAME, program, allowRemote);
}
//==============================================================================================
/**
* A background command that performs additional PDB analysis after after other analysis
* works on function internals. The first phase must have been run, as it reads the PDB
* and processes data types, retaining the information needed for this step. Not what all
* processing we will do here and whether we might need additional commands like this one.
* For now, we want this one for doing global/module symbol function processing to
* set up locals/params/scopes, but some of the data encountered could be for static
* local variables and other things that might make sense to process in the first phase
* (for now, they will be in the second phase).
*/
private static class ProcessPdbFunctionInternalsCommand extends BackgroundCommand {
File pdbFile;
private PdbReaderOptions pdbReaderOptions;
private PdbApplicatorOptions pdbApplicatorOptions;
private MessageLog log;
public ProcessPdbFunctionInternalsCommand(File pdbFile, PdbReaderOptions pdbReaderOptions,
PdbApplicatorOptions pdbApplicatorOptions, MessageLog log) {
super("PDB Universal Function Internals", false, false, false);
this.pdbFile = pdbFile;
this.pdbReaderOptions = pdbReaderOptions;
this.pdbApplicatorOptions = pdbApplicatorOptions;
this.log = log;
}
@Override
public boolean applyTo(DomainObject obj, TaskMonitor monitor) {
Program program = (Program) obj;
try (AbstractPdb pdb = PdbParser.parse(pdbFile, pdbReaderOptions, monitor)) {
monitor.setMessage("PDB: Parsing " + pdbFile + "...");
pdb.deserialize();
DefaultPdbApplicator applicator = new DefaultPdbApplicator(pdb, program,
program.getDataTypeManager(), program.getImageBase(), pdbApplicatorOptions,
log);
applicator.applyFunctionInternalsAnalysis();
return true;
}
catch (PdbException | IOException e) {
log.appendMsg(getName(),
"Issue processing PDB file: " + pdbFile + ":\n " + e.toString());
return false;
}
catch (CancelledException e) {
return false;
}
}
}
/**
* A background command that performs final PDB analysis reporting.
*/
private static class PdbReportingBackgroundCommand extends BackgroundCommand {
public PdbReportingBackgroundCommand() {
super("PDB Universal Reporting", false, false, false);
}
@Override
public boolean applyTo(DomainObject obj, TaskMonitor monitor) {
Program program = (Program) obj;
try {
DefaultPdbApplicator.applyAnalysisReporting(program);
return true;
}
catch (CancelledException e) {
return false;
}
}
}
}

View file

@ -58,6 +58,10 @@ public class BlockNestingContext {
return nestingLevel;
}
public int getLevel() {
return nestingLevel;
}
public void beginBlock(Address startAddress, String name, long length) {
++nestingLevel;
currentBlockAddress = startAddress;

View file

@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -25,6 +25,8 @@ import ghidra.app.cmd.comments.SetCommentCmd;
import ghidra.app.cmd.disassemble.DisassembleCommand;
import ghidra.app.cmd.function.CreateFunctionCmd;
import ghidra.app.cmd.label.SetLabelPrimaryCmd;
import ghidra.app.plugin.core.analysis.TransientProgramProperties;
import ghidra.app.plugin.core.analysis.TransientProgramProperties.SCOPE;
import ghidra.app.util.*;
import ghidra.app.util.bin.format.pdb.PdbParserConstants;
import ghidra.app.util.bin.format.pdb2.pdbreader.*;
@ -49,20 +51,23 @@ import ghidra.util.task.TaskMonitor;
/**
* The main engine for applying an AbstractPdb to Ghidra, whether a Program or DataTypeManager.
* The class is to be constructed first.
* The class is to be constructed first with {@link Program} and/or {@link DataTypeManager}.
* Either, but not both can be null. If the Program is not null but the DatatypeManager is null,
* then the DataTypeManager is gotten from the Program. If the Program is null, then data types
* can be applied to a DataTypeManager.
* <p>
* The
* {@link #applyTo(Program, DataTypeManager, Address, PdbApplicatorOptions, MessageLog)} method is
* then called with the appropriate {@link PdbApplicatorOptions} along with
* a {@link Program} and/or {@link DataTypeManager}. Either, but not both can be null.
* If the Program is not null but the DatatypeManager is null, then the DataTypeManager is gotten
* from the Program. If the Program is null, then data types can be applied to a DataTypeManager.
* The validation logic for the parameters is found in
* {@link #validateAndSetParameters(Program, DataTypeManager, Address, PdbApplicatorOptions,
* MessageLog)}.
* The validation logic for the parameters is found in {@link #validateAndSetParameters(Program,
* DataTypeManager, Address, PdbApplicatorOptions, MessageLog)}.
* <p>
* Once the parameters are validated, appropriate classes and storage containers are constructed.
* Then processing commences, first with data types, followed by symbol-related processing.
* <p>
* Then the user either calls a series of methods if processing is done under an analysis state
* or the user calls a different single method if not running as analysis.
* For analysis, the methods to use are {@link #applyDataTypesAndMainSymbolsAnalysis()},
* {@link #applyFunctionInternalsAnalysis()}, and {@link #applyAnalysisReporting(Program)}.
* For non-analysis state the method to use is {@link #applyNoAnalysisState()}.
* <p>
* Processing commences, first with data types, followed by symbol-related processing.
* <p>
* {@link PdbApplicatorMetrics} are captured during the processing and status and logging is
* reported to various mechanisms including {@link Msg}, {@link MessageLog}, and {@link PdbLog}.
@ -71,6 +76,62 @@ public class DefaultPdbApplicator implements PdbApplicator {
private static final String THUNK_NAME_PREFIX = "[thunk]:";
//==============================================================================================
private static final String PDB_ANALYSIS_LOOKUP_STATE = "PDB_UNIVERSAL_ANALYSIS_STATE";
/**
* Analysis state for items that need to be passed from the first PDB analysis phase to
* subsequent PDB analysis phase(s).
*/
static class PdbUniversalAnalysisState {
private PdbApplicatorMetrics pdbApplicatorMetrics; // Required
private Map<RecordNumber, DataType> dataTypeByMsTypeNum; // Required
private Map<RecordNumber, CppCompositeType> classTypeByMsTypeNum; // Move to program state
private PdbAddressManager pdbAddressManager; // Could recreate each time
private ComplexTypeMapper complexTypeMapper; // Could recreate each time
PdbUniversalAnalysisState() {
pdbApplicatorMetrics = new PdbApplicatorMetrics();
dataTypeByMsTypeNum = new HashMap<>();
classTypeByMsTypeNum = new HashMap<>();
pdbAddressManager = new PdbAddressManager();
complexTypeMapper = new ComplexTypeMapper();
}
PdbApplicatorMetrics getPdbApplicatorMetrics() {
return pdbApplicatorMetrics;
}
PdbAddressManager getPdbAddressManager() {
return pdbAddressManager;
}
ComplexTypeMapper getComplexTypeMapper() {
return complexTypeMapper;
}
Map<RecordNumber, DataType> getDataTypeByMsTypeNumMap() {
return dataTypeByMsTypeNum;
}
Map<RecordNumber, CppCompositeType> getClassTypeByMsTypeNumMap() {
return classTypeByMsTypeNum;
}
}
public static PdbUniversalAnalysisState getPdbAnalysisLookupState(Program program,
boolean asAnalysis) {
if (program == null || !asAnalysis) {
return new PdbUniversalAnalysisState();
}
PdbUniversalAnalysisState pdbAnalysisLookupState = TransientProgramProperties.getProperty(
program, PDB_ANALYSIS_LOOKUP_STATE, SCOPE.ANALYSIS_SESSION,
PdbUniversalAnalysisState.class, () -> new PdbUniversalAnalysisState());
return pdbAnalysisLookupState;
}
//==============================================================================================
/**
* Returns integer value of BigInteger or Long.MAX_VALUE if does not fit
@ -109,6 +170,8 @@ public class DefaultPdbApplicator implements PdbApplicator {
//==============================================================================================
private AbstractPdb pdb;
private PdbUniversalAnalysisState pdbAnalysisLookupState;
private PdbApplicatorMetrics pdbApplicatorMetrics;
//==============================================================================================
@ -167,25 +230,15 @@ public class DefaultPdbApplicator implements PdbApplicator {
private Map<String, Set<RecordNumber>> recordNumbersByFileName;
private Map<Integer, Set<RecordNumber>> recordNumbersByModuleNumber;
//==============================================================================================
// Addresses of locations of functions for disassembly/function-creation.
AddressSet disassembleAddresses;
List<DeferrableFunctionSymbolApplier> deferredFunctionWorkAppliers;
//==============================================================================================
private int currentModuleNumber = 0;
//==============================================================================================
public DefaultPdbApplicator(AbstractPdb pdb) {
Objects.requireNonNull(pdb, "pdb cannot be null");
this.pdb = pdb;
}
//==============================================================================================
//==============================================================================================
/**
* Constructor for DefaultPdbApplicator.
* Applies the PDB to the {@link Program} or {@link DataTypeManager}. Either, but not both,
* can be null
* @param pdb the parsed PDB to apply
* @param programParam the {@link Program} to which to apply the PDB. Can be null in certain
* circumstances
* @param dataTypeManagerParam the {@link DataTypeManager} to which to apply data types. Can be
@ -195,19 +248,98 @@ public class DefaultPdbApplicator implements PdbApplicator {
* @param applicatorOptionsParam {@link PdbApplicatorOptions} used for applying the PDB
* @param logParam the MessageLog to which to output messages
* @throws PdbException if there was a problem processing the data
* @throws CancelledException upon user cancellation
*/
public void applyTo(Program programParam, DataTypeManager dataTypeManagerParam,
Address imageBaseParam, PdbApplicatorOptions applicatorOptionsParam,
MessageLog logParam) throws PdbException, CancelledException {
public DefaultPdbApplicator(AbstractPdb pdb, Program programParam,
DataTypeManager dataTypeManagerParam, Address imageBaseParam,
PdbApplicatorOptions applicatorOptionsParam, MessageLog logParam) throws PdbException {
Objects.requireNonNull(pdb, "pdb cannot be null");
this.pdb = pdb;
// FIXME: should not support use of DataTypeManager-only since it will not have the correct
// data organization if it corresponds to a data type archive. Need to evaluate archive
// use case and determine if a program must always be used.
initializeApplyTo(programParam, dataTypeManagerParam, imageBaseParam,
applicatorOptionsParam, logParam);
initialize(programParam, dataTypeManagerParam, imageBaseParam, applicatorOptionsParam,
logParam);
}
//==============================================================================================
//==============================================================================================
/**
* First of the PDB analysis phases. This phase creates data types and also lays down public
* symbols and some global symbols pertaining to functions, but not necessarily the global
* symbols pertaining to block scopes, local variables, and parameters. See other methods
* below.
* @throws PdbException upon error processing the PDB
* @throws CancelledException upon user cancellation
*/
public void applyDataTypesAndMainSymbolsAnalysis()
throws PdbException, CancelledException {
pdbAnalysisLookupState = getPdbAnalysisLookupState(program, true);
doPdbPreWork();
doPdbTypesAndMainSymbolsWork();
// Flag is set after the first phase... additional phases might not have been run.
// If it is determined that other analyzers need to know if work from follow-on phases
// have been completed, then we will possibly need to set another flag or have more
// that true/false values for this flag (maybe tri-state).
// Also see applyNoAnalysisState()
if (program != null) {
Options options = program.getOptions(Program.PROGRAM_INFO);
options.setBoolean(PdbParserConstants.PDB_LOADED, true);
}
}
/**
* Follow-on PDB analysis phase method that intends to figure out block scopes, local
* variables, and parameters for functions. This should be called only after code processing
* has been completed for the program.
* @throws PdbException upon error processing the PDB
* @throws CancelledException upon user cancellation
*/
public void applyFunctionInternalsAnalysis()
throws PdbException, CancelledException {
pdbAnalysisLookupState = getPdbAnalysisLookupState(program, true);
doPdbPreWork();
doPdbFunctionInternalsWork();
}
/**
* Does final applicator reporting, including metrics
* @param program the program
* @throws CancelledException upon user cancellation
*/
public static void applyAnalysisReporting(Program program) throws CancelledException {
PdbUniversalAnalysisState state = getPdbAnalysisLookupState(program, true);
doReports(state);
}
/**
* This method can be used instead of the multi-phased analysis methods. Generally, this
* method should only be used when not processing in an analysis state. It does work of
* some other analyzers (the disassembly phase).
* @throws PdbException upon error processing the PDB
* @throws CancelledException upon user cancellation
*/
public void applyNoAnalysisState() throws PdbException, CancelledException {
pdbAnalysisLookupState = getPdbAnalysisLookupState(program, false);
doPdbPreWork();
doPdbTypesAndMainSymbolsWork();
doDisassemblyWork();
doPdbFunctionInternalsWork();
doReports(pdbAnalysisLookupState);
// Setting flag that indicates that PDB has been loaded. The flag is also set by a
// different method that is intended to be called when processing is done under the
// auspices of an analysis state, whereas this method that also sets the flag does so
// for non-analysis state processing.
if (program != null) {
Options options = program.getOptions(Program.PROGRAM_INFO);
options.setBoolean(PdbParserConstants.PDB_LOADED, true);
}
}
//==============================================================================================
private void doPdbTypesAndMainSymbolsWork() throws PdbException, CancelledException {
switch (applicatorOptions.getProcessingControl()) {
case DATA_TYPES_ONLY:
processTypes();
@ -223,34 +355,46 @@ public class DefaultPdbApplicator implements PdbApplicator {
throw new PdbException("PDB: Invalid Application Control: " +
applicatorOptions.getProcessingControl());
}
if (program != null) {
// TODO: if I'm PDB Universal Analyzer #1, schedule PDD Universal Analyzer #2; for now,
// just calling doDeferred here.
doDeferredProgramProcessing();
// Mark PDB analysis as complete.
Options options = program.getOptions(Program.PROGRAM_INFO);
options.setBoolean(PdbParserConstants.PDB_LOADED, true);
}
pdbAddressManager.logReport();
pdbApplicatorMetrics.logReport();
Msg.info(this, "PDB Terminated Normally");
Msg.info(this, "PDB Types and Main Symbols Processing Terminated Normally");
}
//==============================================================================================
private void doDeferredProgramProcessing() throws CancelledException, PdbException {
disassembleFunctions();
doDeferredFunctionProcessing();
private void doDisassemblyWork() throws PdbException, CancelledException {
if (program != null) {
disassembleFunctions();
}
Msg.info(this, "PDB Disassembly Terminated Normally");
}
private void doPdbFunctionInternalsWork() throws PdbException, CancelledException {
if (program != null) {
doDeferredFunctionProcessing();
// Options options = program.getOptions(Program.PROGRAM_INFO);
// options.setBoolean(PdbParserConstants.PDB_LOADED, true);
}
// // Where/when? Split up? reporting... mixed info... depends on what is carried in analysis
// // state: applicator, pdb, etc. If not applicator... then each has separate report
// // unless PdbApplicatorMetrics is put into the state... and then what happens if we
// // never kick off the second phase? Do we need a final phase to do the reporting?
// pdbAddressManager.logReport();
// pdbApplicatorMetrics.logReport();
Msg.info(this, "PDB Function Internals Processing Terminated Normally");
}
private static void doReports(PdbUniversalAnalysisState state) throws CancelledException {
Msg.info(DefaultPdbApplicator.class, "PDB Applicator Reporting");
state.getPdbAddressManager().logReport();
state.getPdbApplicatorMetrics().logReport();
Msg.info(DefaultPdbApplicator.class, "PDB Applicator Reporting Terminated Normally");
}
//==============================================================================================
/**
* Set the context for each function, disassemble them, and then do fix-ups
* @throws PdbException upon issue gathering the data
*/
private void disassembleFunctions() throws CancelledException {
private void disassembleFunctions() throws PdbException, CancelledException {
AddressSet disassembleAddresses = gatherAddressesForDisassembly();
TaskMonitor monitor = getMonitor();
Listing listing = program.getListing();
@ -300,11 +444,6 @@ public class DefaultPdbApplicator implements PdbApplicator {
}
doDeferredProcessGlobalSymbolsNoTypedefs();
doDeferredProcessModuleSymbols();
//
// for (DeferrableFunctionSymbolApplier applier : deferredFunctionWorkAppliers) {
// cancelOnlyWrappingMonitor.checkCancelled();
// applier.doDeferredProcessing();
// }
}
//==============================================================================================
@ -415,50 +554,27 @@ public class DefaultPdbApplicator implements PdbApplicator {
//==============================================================================================
/**
* Initializes helper classes and data items used for applying the PDB
* @throws CancelledException upon user cancellation
* @throws PdbException upon error in processing components
*/
private void initializeApplyTo(Program programParam, DataTypeManager dataTypeManagerParam,
private void initialize(Program programParam, DataTypeManager dataTypeManagerParam,
Address imageBaseParam, PdbApplicatorOptions applicatorOptionsParam,
MessageLog logParam) throws PdbException, CancelledException {
MessageLog logParam) throws PdbException {
validateAndSetParameters(programParam, dataTypeManagerParam, imageBaseParam,
applicatorOptionsParam, logParam);
cancelOnlyWrappingMonitor = new CancelOnlyWrappingTaskMonitor(getMonitor());
pdbApplicatorMetrics = new PdbApplicatorMetrics();
pdbPeHeaderInfoManager = new PdbPeHeaderInfoManager(this);
symbolGroups = createSymbolGroups();
linkerModuleNumber = findLinkerModuleNumber();
pdbAddressManager = new PdbAddressManager(this, imageBase);
categoryUtils = setPdbCatogoryUtils(pdb.getFilename());
multiphaseResolver = new MultiphaseDataTypeResolver(this);
pdbPrimitiveTypeApplicator = new PdbPrimitiveTypeApplicator(dataTypeManager);
dataTypeByMsTypeNum = new HashMap<>();
classTypeByMsTypeNum = new HashMap<>();
complexTypeMapper = new ComplexTypeMapper();
typeApplierParser = new TypeApplierFactory(this);
isClassByNamespace = new TreeMap<>();
symbolApplierParser = new SymbolApplierFactory(this);
if (program != null) {
disassembleAddresses = new AddressSet();
deferredFunctionWorkAppliers = new ArrayList<>();
// Currently, this must happen after symbolGroups are created.
PdbVbtManager pdbVbtManager = new PdbVbtManager(this);
//pdbVbtManager.CreateVirtualBaseTables(); // Depends on symbolGroups
vbtManager = pdbVbtManager;
registerNameToRegisterMapper = new PdbRegisterNameToProgramRegisterMapper(program);
}
else {
vbtManager = new VbtManager(getDataTypeManager());
}
// Investigations
labelsByAddress = new HashMap<>();
// Investigations into source/line info
@ -466,6 +582,37 @@ public class DefaultPdbApplicator implements PdbApplicator {
recordNumbersByModuleNumber = new HashMap<>();
}
/**
* Does some basic work based on the PDB and other parameters. This work can be redone
* for each analysis phase (results should not differ)
* @throws CancelledException upon user cancellation
* @throws PdbException upon error in processing components
*/
private void doPdbPreWork() throws CancelledException, PdbException {
pdbApplicatorMetrics = pdbAnalysisLookupState.getPdbApplicatorMetrics();
pdbAddressManager = pdbAnalysisLookupState.getPdbAddressManager();
complexTypeMapper = pdbAnalysisLookupState.getComplexTypeMapper();
dataTypeByMsTypeNum = pdbAnalysisLookupState.getDataTypeByMsTypeNumMap();
classTypeByMsTypeNum = pdbAnalysisLookupState.getClassTypeByMsTypeNumMap();
if (!pdbAddressManager.isInitialized()) {
pdbAddressManager.initialize(this, imageBase);
}
categoryUtils = setPdbCatogoryUtils(pdb.getFilename());
symbolGroups = createSymbolGroups();
linkerModuleNumber = findLinkerModuleNumber();
if (program != null) {
// Currently, this must happen after symbolGroups are created.
PdbVbtManager pdbVbtManager = new PdbVbtManager(this);
vbtManager = pdbVbtManager;
registerNameToRegisterMapper = new PdbRegisterNameToProgramRegisterMapper(program);
}
else {
vbtManager = new VbtManager(getDataTypeManager());
}
}
private void validateAndSetParameters(Program programParam,
DataTypeManager dataTypeManagerParam, Address imageBaseParam,
PdbApplicatorOptions applicatorOptionsParam, MessageLog logParam) throws PdbException {
@ -1130,18 +1277,12 @@ public class DefaultPdbApplicator implements PdbApplicator {
return myFunction;
}
//==============================================================================================
// void scheduleDeferredFunctionWork(DeferrableFunctionSymbolApplier applier) {
// // Not using normalized address is OK, as we should have already set the context and
// // used the normalized address when creating the one-byte function
// disassembleAddresses.add(applier.getAddress());
// //deferredFunctionWorkAppliers.add(applier);
// }
//
void scheduleDisassembly(Address address) {
// Not using normalized address is OK, as we should have already set the context and
// used the normalized address when creating the one-byte function
disassembleAddresses.add(address);
Function getExistingFunction(Address address) {
if (program == null) {
return null;
}
// TODO: do we have to have normalized function address to retrieve (see method above)
return program.getListing().getFunctionAt(address);
}
//==============================================================================================
@ -1345,6 +1486,48 @@ public class DefaultPdbApplicator implements PdbApplicator {
processSymbolGroup(0, iter);
}
private AddressSet gatherAddressesForDisassembly() throws CancelledException, PdbException {
if (program == null) {
return null;
}
PdbDebugInfo debugInfo = pdb.getDebugInfo();
int num = debugInfo.getNumModules();
TaskMonitor monitor = getMonitor();
monitor.setMessage("PDB: Deferred-applying module symbol components...");
monitor.initialize(num + 1); // add one because we doing 0 through num, inclusive
AddressSet addresses = new AddressSet();
// Process symbols list for each module
// moduleNumber = 0 is for global symbols
for (int moduleNumber = 0; moduleNumber <= num; moduleNumber++) {
monitor.checkCancelled();
setCurrentModuleNumber(moduleNumber);
// Process module symbols list
SymbolGroup symbolGroup = getSymbolGroupForModule(moduleNumber);
if (symbolGroup != null) {
MsSymbolIterator iter = symbolGroup.getSymbolIterator();
addresses.add(getDisassembleAddressForModule(moduleNumber, iter));
}
monitor.increment();
}
return addresses;
}
AddressSet getDisassembleAddressForModule(int moduleNumber, MsSymbolIterator iter)
throws CancelledException {
iter.initGet();
TaskMonitor monitor = getMonitor();
AddressSet addresses = new AddressSet();
while (iter.hasNext()) {
monitor.checkCancelled();
MsSymbolApplier applier = getSymbolApplier(iter);
if (applier instanceof DisassembleableAddressSymbolApplier disassembleApplier) {
addresses.add(disassembleApplier.getAddressForDisassembly());
}
iter.next();
}
return addresses;
}
//==============================================================================================
int getCurrentModuleNumber() {
return currentModuleNumber;
@ -1354,39 +1537,26 @@ public class DefaultPdbApplicator implements PdbApplicator {
currentModuleNumber = moduleNumber;
}
//==============================================================================================
private void doDeferredProcessModuleSymbols() throws CancelledException, PdbException {
PdbDebugInfo debugInfo = pdb.getDebugInfo();
if (debugInfo == null) {
return;
}
int totalCount = 0;
int num = debugInfo.getNumModules();
TaskMonitor monitor = getMonitor();
for (int moduleNumber = 1; moduleNumber <= num; moduleNumber++) {
monitor.checkCancelled();
SymbolGroup symbolGroup = getSymbolGroupForModule(moduleNumber);
if (symbolGroup == null) {
continue; // should not happen
}
totalCount++;
//totalCount += symbolGroup.size();
}
monitor.setMessage("PDB: Deferred-applying module symbol components...");
//monitor.setMessage("PDB: Applying " + totalCount + " module symbol components...");
monitor.initialize(totalCount);
int num = debugInfo.getNumModules();
monitor.initialize(num);
// Process symbols list for each module
for (int moduleNumber = 1; moduleNumber <= num; moduleNumber++) {
monitor.checkCancelled();
setCurrentModuleNumber(moduleNumber);
// Process module symbols list
SymbolGroup symbolGroup = getSymbolGroupForModule(moduleNumber);
if (symbolGroup == null) {
continue; // should not happen
if (symbolGroup != null) {
MsSymbolIterator iter = symbolGroup.getSymbolIterator();
doDeferredModuleSymbolGroup(moduleNumber, iter);
}
MsSymbolIterator iter = symbolGroup.getSymbolIterator();
doDeferredModuleSymbolGroup(moduleNumber, iter);
monitor.increment();
}
}
@ -1409,60 +1579,30 @@ public class DefaultPdbApplicator implements PdbApplicator {
}
}
//==============================================================================================
private void processModuleSymbols() throws CancelledException, PdbException {
PdbDebugInfo debugInfo = pdb.getDebugInfo();
if (debugInfo == null) {
return;
}
int totalCount = 0;
int num = debugInfo.getNumModules();
TaskMonitor monitor = getMonitor();
for (int moduleNumber = 1; moduleNumber <= num; moduleNumber++) {
monitor.checkCancelled();
SymbolGroup symbolGroup = getSymbolGroupForModule(moduleNumber);
if (symbolGroup == null) {
continue; // should not happen
}
totalCount++;
//totalCount += symbolGroup.size();
}
monitor.setMessage("PDB: Applying module symbol components...");
//monitor.setMessage("PDB: Applying " + totalCount + " module symbol components...");
monitor.initialize(totalCount);
monitor.initialize(num);
// Process symbols list for each module
for (int moduleNumber = 1; moduleNumber <= num; moduleNumber++) {
monitor.checkCancelled();
setCurrentModuleNumber(moduleNumber);
// Process module symbols list
SymbolGroup symbolGroup = getSymbolGroupForModule(moduleNumber);
if (symbolGroup == null) {
continue; // should not happen
if (symbolGroup != null) {
MsSymbolIterator iter = symbolGroup.getSymbolIterator();
processSymbolGroup(moduleNumber, iter);
}
MsSymbolIterator iter = symbolGroup.getSymbolIterator();
processSymbolGroup(moduleNumber, iter);
monitor.increment();
// catelogSymbols(index, symbolGroup);
// do not call monitor.incrementProgress(1) here, as it is updated inside of
// processSymbolGroup.
}
}
// private Set<Class<? extends AbstractMsSymbol>> moduleSymbols = new HashSet<>();
//
// private void catelogSymbols(int moduleNumber, SymbolGroup symbolGroup)
// throws CancelledException {
// symbolGroup.initGet();
// while (symbolGroup.hasNext()) {
// monitor.checkCancelled();
// AbstractMsSymbol symbol = symbolGroup.peek();
// moduleSymbols.add(symbol.getClass());
// symbolGroup.next();
// }
// }
//
//==============================================================================================
private void processSymbolGroup(int moduleNumber, MsSymbolIterator iter)
throws CancelledException {
iter.initGet();
@ -2233,11 +2373,11 @@ public class DefaultPdbApplicator implements PdbApplicator {
else if (plate.contains(comment)) {
return true;
}
else if (!plate.endsWith("\n")) {
plate += '\n';
else if (!comment.endsWith("\n")) {
comment += '\n';
}
plate += comment;
SetCommentCmd.createComment(program, address, comment, CodeUnit.PLATE_COMMENT);
plate = comment + plate; // putting new comment at top of existing plate
SetCommentCmd.createComment(program, address, plate, CodeUnit.PLATE_COMMENT);
return true;
}

View file

@ -0,0 +1,31 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.pdb.pdbapplicator;
import ghidra.program.model.address.Address;
/**
* Interface class for MsSymbolApplier that has deferrable function work.
*/
interface DisassembleableAddressSymbolApplier {
/**
* Returns the address for disassembly. Does not increment the iterator
* @return the address
*/
public Address getAddressForDisassembly();
}

View file

@ -35,11 +35,12 @@ import ghidra.util.task.TaskMonitor;
* Applier for {@link AbstractProcedureStartMsSymbol} and {@link AbstractThunkMsSymbol} symbols.
*/
public class FunctionSymbolApplier extends AbstractBlockContextApplier
implements BlockNestingSymbolApplier {
implements BlockNestingSymbolApplier, DisassembleableAddressSymbolApplier {
private Function function = null;
// Do not trust any of these variables... this is work in progress (possibly getting
// torn up), but non-functioning code in other classes or this class still depend on these
private Function function_x = null;
private long specifiedFrameSize_x = 0;
private long currentFrameSize_x = 0;
@ -69,6 +70,11 @@ public class FunctionSymbolApplier extends AbstractBlockContextApplier
processSymbol(iter);
}
@Override
public Address getAddressForDisassembly() {
return applicator.getAddress(symbol);
}
private void processSymbol(MsSymbolIterator iter)
throws CancelledException, PdbException {
@ -87,17 +93,11 @@ public class FunctionSymbolApplier extends AbstractBlockContextApplier
return;
}
Function function = applicator.getExistingOrCreateOneByteFunction(address);
function = applicator.getExistingOrCreateOneByteFunction(address);
if (function == null) {
return;
}
// Collecting all addresses from all functions to do one large bulk disassembly of the
// complete AddressSet of function addresses. We could consider removing this logic
// of collecting them all for bulk disassembly and do individual disassembly at the
// same deferred point in time.
applicator.scheduleDisassembly(address);
boolean succeededSetFunctionSignature = setFunctionDefinition(function, address);
// If signature was set, then override existing primary mangled symbol with
@ -178,6 +178,15 @@ public class FunctionSymbolApplier extends AbstractBlockContextApplier
String name = symbol.getName();
Address address = applicator.getAddress(symbol);
function = applicator.getExistingFunction(address);
if (function == null) {
// Skip all interim symbols records
if (!processEndSymbol(symbol.getEndPointer(), iter)) {
applicator.appendLogMsg("PDB: Failed to process function at address " + address);
}
return;
}
long start = getStartOffset();
long end = getEndOffset();
Address blockAddress = address.add(start);
@ -191,7 +200,7 @@ public class FunctionSymbolApplier extends AbstractBlockContextApplier
* @return the Function
*/
Function getFunction() {
return function_x;
return function;
}
/**
@ -268,21 +277,21 @@ public class FunctionSymbolApplier extends AbstractBlockContextApplier
*/
private int getFrameBaseOffset(TaskMonitor monitor) throws CancelledException {
int retAddrSize = function_x.getProgram().getDefaultPointerSize();
int retAddrSize = function.getProgram().getDefaultPointerSize();
if (retAddrSize != 8) {
// don't do this for 32 bit.
return -retAddrSize; // 32 bit has a -4 byte offset
}
Register frameReg = function_x.getProgram().getCompilerSpec().getStackPointer();
Address entryAddr = function_x.getEntryPoint();
Register frameReg = function.getProgram().getCompilerSpec().getStackPointer();
Address entryAddr = function.getEntryPoint();
AddressSet scopeSet = new AddressSet();
scopeSet.addRange(entryAddr, entryAddr.add(64));
CallDepthChangeInfo valueChange =
new CallDepthChangeInfo(function_x, scopeSet, frameReg, monitor);
new CallDepthChangeInfo(function, scopeSet, frameReg, monitor);
InstructionIterator instructions =
function_x.getProgram().getListing().getInstructions(scopeSet, true);
function.getProgram().getListing().getInstructions(scopeSet, true);
int max = 0;
while (instructions.hasNext()) {
monitor.checkCancelled();

View file

@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -33,7 +33,7 @@ import ghidra.util.task.TaskMonitor;
* Applier for {@link AbstractLabelMsSymbol} symbols.
*/
public class LabelSymbolApplier extends MsSymbolApplier
implements DirectSymbolApplier, NestableSymbolApplier {
implements DirectSymbolApplier, NestableSymbolApplier, DisassembleableAddressSymbolApplier {
private AbstractLabelMsSymbol symbol;
@ -81,6 +81,11 @@ public class LabelSymbolApplier extends MsSymbolApplier
}
}
@Override
public Address getAddressForDisassembly() {
return applicator.getAddress(symbol);
}
@Override
public void applyTo(NestingSymbolApplier applyToApplier, MsSymbolIterator iter)
throws PdbException, CancelledException {
@ -163,7 +168,6 @@ public class LabelSymbolApplier extends MsSymbolApplier
if (function == null) {
return false;
}
applicator.scheduleDisassembly(address);
if (!function.isThunk() &&
function.getSignatureSource().isLowerPriorityThan(SourceType.IMPORTED)) {

View file

@ -55,7 +55,6 @@ public class LocalOptimizedSymbolApplier extends MsSymbolApplier
private void doWork(FunctionSymbolApplier functionSymbolApplier, MsSymbolIterator iter)
throws CancelledException, PdbException {
getValidatedSymbol(iter, true);
// TODO: Not doing anything with the information yet.
symbol.getLocalVariableFlags();
symbol.getName();

View file

@ -47,7 +47,7 @@ import ghidra.util.task.TaskMonitor;
* Applier for {@link AbstractManagedProcedureMsSymbol} symbols.
*/
public class ManagedProcedureSymbolApplier extends AbstractBlockContextApplier
implements BlockNestingSymbolApplier {
implements BlockNestingSymbolApplier, DisassembleableAddressSymbolApplier {
private int symbolBlockNestingLevel;
private Address currentBlockAddress;
@ -79,6 +79,11 @@ public class ManagedProcedureSymbolApplier extends AbstractBlockContextApplier
processSymbol(iter);
}
@Override
public Address getAddressForDisassembly() {
return applicator.getAddress(symbol);
}
// TODO. Investigate more. This is not working for at least one CLI dll in that we are
// not getting correct addresses. There is no omap and the one section is unnamed.
private void processSymbol(MsSymbolIterator iter)
@ -110,12 +115,6 @@ public class ManagedProcedureSymbolApplier extends AbstractBlockContextApplier
return;
}
// Collecting all addresses from all functions to do one large bulk disassembly of the
// complete AddressSet of function addresses. We could consider removing this logic
// of collecting them all for bulk diassembly and do individual disassembly at the
// same deferred point in time.
applicator.scheduleDisassembly(address);
boolean succeededSetFunctionSignature = setFunctionDefinition(function, address, symbol);
// If signature was set, then override existing primary mangled symbol with

View file

@ -70,22 +70,28 @@ public class PdbAddressManager {
private PdbAddressCalculator addressCalculator;
private boolean isInitialized;
//==============================================================================================
// API
//==============================================================================================
public PdbAddressManager() {
isInitialized = false;
}
/**
* Manager
* @param applicator {@link DefaultPdbApplicator} for which this class is working.
* @param imageBase Address from which all other addresses are based.
* @param applicatorArg {@link DefaultPdbApplicator} for which this class is working.
* @param imageBaseArg Address from which all other addresses are based.
* @throws PdbException If Program is null;
* @throws CancelledException upon user cancellation
*/
PdbAddressManager(DefaultPdbApplicator applicator, Address imageBase)
void initialize(DefaultPdbApplicator applicatorArg, Address imageBaseArg)
throws PdbException, CancelledException {
Objects.requireNonNull(applicator, "applicator may not be null");
Objects.requireNonNull(imageBase, "imageBase may not be null");
this.applicator = applicator;
this.imageBase = imageBase;
Objects.requireNonNull(applicatorArg, "applicator may not be null");
Objects.requireNonNull(imageBaseArg, "imageBase may not be null");
this.applicator = applicatorArg;
this.imageBase = imageBaseArg;
memoryGroupRefinement = new ArrayList<>();
memorySectionRefinement = new ArrayList<>();
@ -100,6 +106,15 @@ public class PdbAddressManager {
// determineMemoryBlocks_orig();
mapPreExistingSymbols();
createAddressRemap();
isInitialized = true;
}
/**
* Returns {@code true} if already initialized
* @return {@code true}" if initialized
*/
boolean isInitialized() {
return isInitialized;
}
/**

View file

@ -27,7 +27,8 @@ import ghidra.util.exception.CancelledException;
/**
* Applier for {@link TrampolineMsSymbol} symbols.
*/
public class TrampolineSymbolApplier extends MsSymbolApplier implements DirectSymbolApplier {
public class TrampolineSymbolApplier extends MsSymbolApplier
implements DirectSymbolApplier, DisassembleableAddressSymbolApplier {
//public class TrampolineSymbolApplier extends MsSymbolApplier
// implements DeferrableFunctionSymbolApplier { // Question of whether we need to do work later
@ -84,8 +85,11 @@ public class TrampolineSymbolApplier extends MsSymbolApplier implements DirectSy
if (target != null && thunk != null) {
thunk.setThunkedFunction(target);
}
applicator.scheduleDisassembly(address);
// TODO: should we schedule at targetAddress too?
}
@Override
public Address getAddressForDisassembly() {
return applicator.getAddress(symbol);
}
private TrampolineMsSymbol getValidatedSymbol(MsSymbolIterator iter, boolean iterate) {

View file

@ -25,10 +25,10 @@ import ghidra.app.plugin.core.disassembler.EntryPointAnalyzer;
import ghidra.app.services.DataTypeManagerService;
import ghidra.app.util.bin.format.pdb.PdbException;
import ghidra.app.util.bin.format.pdb.PdbParser;
import ghidra.app.util.bin.format.pdb2.pdbreader.AbstractPdb;
import ghidra.app.util.bin.format.pdb2.pdbreader.PdbReaderOptions;
import ghidra.app.util.importer.MessageLog;
import ghidra.app.util.pdb.pdbapplicator.*;
import ghidra.app.util.pdb.pdbapplicator.PdbApplicatorControl;
import ghidra.app.util.pdb.pdbapplicator.PdbApplicatorOptions;
import ghidra.framework.options.Options;
import ghidra.program.model.address.AddressSetView;
import ghidra.program.model.listing.Program;
@ -134,7 +134,7 @@ class LoadPdbTask extends Task {
}
private boolean parseWithNewParser(MessageLog log, TaskMonitor monitor)
throws IOException, CancelledException {
throws CancelledException {
PdbReaderOptions pdbReaderOptions = new PdbReaderOptions(); // use defaults
@ -142,20 +142,8 @@ class LoadPdbTask extends Task {
pdbApplicatorOptions.setProcessingControl(control);
try (AbstractPdb pdb = ghidra.app.util.bin.format.pdb2.pdbreader.PdbParser.parse(pdbFile,
pdbReaderOptions, monitor)) {
monitor.setMessage("PDB: Parsing " + pdbFile + "...");
pdb.deserialize();
DefaultPdbApplicator applicator = new DefaultPdbApplicator(pdb);
applicator.applyTo(program, program.getDataTypeManager(), program.getImageBase(),
pdbApplicatorOptions, log);
return true;
}
catch (ghidra.app.util.bin.format.pdb2.pdbreader.PdbException e) {
log.appendMsg("PDB Error: " + e.getMessage());
}
return false;
return PdbUniversalAnalyzer.doAnalysis(program, pdbFile, pdbReaderOptions,
pdbApplicatorOptions, log, monitor);
}
// We need to kick off any byte analyzers (like getting import symbols), as they typically