Merge remote-tracking branch 'origin/GP-2286_BooleanValues'

(Closes #4457)

Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc

@@ -1120,7 +1120,7 @@ SymbolEntry *ActionConstantPtr::isPointer(AddrSpace *spc,Varnode *vn,PcodeOp *op
 int4 ActionConstantPtr::apply(Funcdata &data)
 
 {
-  if (!data.isTypeRecoveryOn()) return 0;
+  if (!data.hasTypeRecoveryStarted()) return 0;
 
   if (localcount >= 4)		// At most 4 passes (once type recovery starts)
     return 0;
@@ -1208,7 +1208,7 @@ int4 ActionDeindirect::apply(Funcdata &data)
 	continue;
       }
     }
-    if (data.isTypeRecoveryOn()) {
+    if (data.hasTypeRecoveryStarted()) {
       // Check for a function pointer that has an attached prototype
       Datatype *ct = op->getIn(0)->getTypeReadFacing(op);
       if ((ct->getMetatype()==TYPE_PTR)&&
@@ -1486,7 +1486,7 @@ void ActionFuncLink::funcLinkOutput(FuncCallSpecs *fc,Funcdata &data)
     Datatype *outtype = outparam->getType();
     if (outtype->getMetatype() != TYPE_VOID) {
       int4 sz = outparam->getSize();
-      if (sz == 1 && outtype->getMetatype() == TYPE_BOOL)
+      if (sz == 1 && outtype->getMetatype() == TYPE_BOOL && data.isTypeRecoveryOn())
 	data.opMarkCalculatedBool(fc->getOp());
       Address addr = outparam->getAddress();
       data.newVarnodeOut(sz,addr,fc->getOp());
@@ -4792,7 +4792,7 @@ int4 ActionInferTypes::apply(Funcdata &data)
 
 {
   // Make sure spacebase is accurate or bases could get typed and then ptrarithed
-  if (!data.isTypeRecoveryOn()) return 0;
+  if (!data.hasTypeRecoveryStarted()) return 0;
   TypeFactory *typegrp = data.getArch()->types;
   Varnode *vn;
   VarnodeLocSet::const_iterator iter;

Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.hh

@@ -65,9 +65,14 @@ public:
 };
 
 /// \brief Allow type recovery to start happening
+///
+/// The presence of \b this Action causes the function to be marked that data-type analysis
+/// will be performed.  Then when \b this action is applied during analysis, the function is marked
+/// that data-type analysis has started.
 class ActionStartTypes : public Action {
 public:
   ActionStartTypes(const string &g) : Action(0,"starttypes",g) {}	///< Constructor
+  virtual void reset(Funcdata &data) { data.setTypeRecovery(true); }
   virtual Action *clone(const ActionGroupList &grouplist) const {
     if (!grouplist.contains(getGroup())) return (Action *)0;
     return new ActionStartTypes(getGroup());

Ghidra/Features/Decompiler/src/decompile/cpp/fspec.cc

@@ -4893,7 +4893,7 @@ void FuncCallSpecs::commitNewOutputs(Funcdata &data,Varnode *newout)
     // We could conceivably truncate the output to the correct size to match the parameter
     activeoutput.registerTrial(param->getAddress(),param->getSize());
     PcodeOp *indop = newout->getDef();
-    if (newout->getSize() == 1 && param->getType()->getMetatype() == TYPE_BOOL)
+    if (newout->getSize() == 1 && param->getType()->getMetatype() == TYPE_BOOL && data.isTypeRecoveryOn())
       data.opMarkCalculatedBool(op);
     if (newout->getSize() == param->getSize()) {
       if (indop != op) {

Ghidra/Features/Decompiler/src/decompile/cpp/funcdata.cc

@@ -81,7 +81,8 @@ void Funcdata::clear(void)
 
 {				// Clear everything associated with decompilation (analysis)
 
-  flags &= ~(highlevel_on|blocks_generated|processing_started|typerecovery_on|restart_pending);
+  flags &= ~(highlevel_on|blocks_generated|processing_started|typerecovery_start|typerecovery_on|
+      double_precis_on|restart_pending);
   clean_up_index = 0;
   high_level_index = 0;
   cast_phase_index = 0;
@@ -174,8 +175,8 @@ void Funcdata::stopProcessing(void)
 bool Funcdata::startTypeRecovery(void)
 
 {
-  if ((flags & typerecovery_on)!=0) return false; // Already started
-  flags |= typerecovery_on;
+  if ((flags & typerecovery_start)!=0) return false; // Already started
+  flags |= typerecovery_start;
   return true;
 }
 

Ghidra/Features/Decompiler/src/decompile/cpp/funcdata.hh

@@ -58,14 +58,15 @@ class Funcdata {
     blocks_unreachable = 4,	///< Set if at least one basic block is currently unreachable
     processing_started = 8,	///< Set if processing has started
     processing_complete = 0x10,	///< Set if processing completed
-    typerecovery_on = 0x20,	///< Set if data-type recovery is started
-    no_code = 0x40,		///< Set if there is no code available for this function
-    jumptablerecovery_on = 0x80,	///< Set if \b this Funcdata object is dedicated to jump-table recovery
-    jumptablerecovery_dont = 0x100, 	///< Don't try to recover jump-tables, always truncate
-    restart_pending = 0x200,	///< Analysis must be restarted (because of new override info)
-    unimplemented_present = 0x400,	///< Set if function contains unimplemented instructions
-    baddata_present = 0x800,	///< Set if function flowed into bad data
-    double_precis_on = 0x1000	///< Set if we are performing double precision recovery
+    typerecovery_on = 0x20,	///< Set if data-type analysis will be performed
+    typerecovery_start = 0x40,	///< Set if data-type recovery is started
+    no_code = 0x80,		///< Set if there is no code available for this function
+    jumptablerecovery_on = 0x100,	///< Set if \b this Funcdata object is dedicated to jump-table recovery
+    jumptablerecovery_dont = 0x200, 	///< Don't try to recover jump-tables, always truncate
+    restart_pending = 0x400,	///< Analysis must be restarted (because of new override info)
+    unimplemented_present = 0x800,	///< Set if function contains unimplemented instructions
+    baddata_present = 0x1000,	///< Set if function flowed into bad data
+    double_precis_on = 0x2000	///< Set if we are performing double precision recovery
   };
   uint4 flags;			///< Boolean properties associated with \b this function
   uint4 clean_up_index;		///< Creation index of first Varnode created after start of cleanup
@@ -144,7 +145,8 @@ public:
   bool isProcStarted(void) const { return ((flags&processing_started)!=0); }	///< Has processing of the function started
   bool isProcComplete(void) const { return ((flags&processing_complete)!=0); }	///< Is processing of the function complete
   bool hasUnreachableBlocks(void) const { return ((flags&blocks_unreachable)!=0); }	///< Did this function exhibit unreachable code
-  bool isTypeRecoveryOn(void) const { return ((flags&typerecovery_on)!=0); }	///< Has data-type recovery processes started
+  bool isTypeRecoveryOn(void) const { return ((flags&typerecovery_on)!=0); }	///< Will data-type analysis be performed
+  bool hasTypeRecoveryStarted(void) const { return ((flags&typerecovery_start)!=0); }	///< Has data-type recovery processes started
   bool hasNoCode(void) const { return ((flags & no_code)!=0); }		///< Return \b true if \b this function has no code body
   void setNoCode(bool val) { if (val) flags |= no_code; else flags &= ~no_code; }	///< Toggle whether \b this has a body
   void setLanedRegGenerated(void) { minLanedSize = 1000000; }	///< Mark that laned registers have been collected
@@ -169,6 +171,11 @@ public:
   void startProcessing(void);					///< Start processing for this function
   void stopProcessing(void);					///< Mark that processing has completed for this function
   bool startTypeRecovery(void);					///< Mark that data-type analysis has started
+
+  /// \brief Toggle whether data-type recovery will be performed on \b this function
+  ///
+  /// \param val is \b true if data-type analysis is enabled
+  void setTypeRecovery(bool val) { flags = val ? (flags | typerecovery_on) : (flags & ~typerecovery_on); }
   void startCastPhase(void) { cast_phase_index = vbank.getCreateIndex(); }	///< Start the \b cast insertion phase
   uint4 getCastPhaseIndex(void) const { return cast_phase_index; }	///< Get creation index at the start of \b cast insertion
   uint4 getHighLevelIndex(void) const { return high_level_index; }	///< Get creation index at the start of HighVariable creation

Ghidra/Features/Decompiler/src/decompile/cpp/ruleaction.cc

@@ -2641,7 +2641,6 @@ int4 RuleBooleanNegate::applyOp(PcodeOp *op,Funcdata &data)
   OpCode opc;
   Varnode *constvn;
   Varnode *subbool;
-  PcodeOp *subop;
   bool negate;
   uintb val;
 
@@ -2656,9 +2655,7 @@ int4 RuleBooleanNegate::applyOp(PcodeOp *op,Funcdata &data)
   if (val==0)
     negate = !negate;
 
-  if (!subbool->isWritten()) return 0;
-  subop = subbool->getDef();
-  if (!subop->isCalculatedBool()) return 0; // Subexpression must be boolean output
+  if (!subbool->isBooleanValue(data.isTypeRecoveryOn())) return 0;
 
   data.opRemoveInput(op,1);	// Remove second parameter
   data.opSetInput(op,subbool,0); // Keep original boolean parameter
@@ -2687,15 +2684,15 @@ void RuleBoolZext::getOpList(vector<uint4> &oplist) const
 int4 RuleBoolZext::applyOp(PcodeOp *op,Funcdata &data)
 
 {
-  PcodeOp *boolop1,*multop1,*actionop;
-  PcodeOp *boolop2,*zextop2,*multop2;
+  Varnode *boolVn1,*boolVn2;
+  PcodeOp *multop1,*actionop;
+  PcodeOp *zextop2,*multop2;
   uintb coeff,val;
   OpCode opc;
   int4 size;
 
-  if (!op->getIn(0)->isWritten()) return 0;
-  boolop1 = op->getIn(0)->getDef();
-  if (!boolop1->isCalculatedBool()) return 0;
+  boolVn1 = op->getIn(0);
+  if (!boolVn1->isBooleanValue(data.isTypeRecoveryOn())) return 0;
 
   multop1 = op->getOut()->loneDescend();
   if (multop1 == (PcodeOp *)0) return 0;
@@ -2717,7 +2714,7 @@ int4 RuleBoolZext::applyOp(PcodeOp *op,Funcdata &data)
       PcodeOp *newop = data.newOp(1,op->getAddr());
       data.opSetOpcode(newop,CPUI_BOOL_NEGATE);	// Negate the boolean
       vn = data.newUniqueOut(1,newop);
-      data.opSetInput(newop,boolop1->getOut(),0);
+      data.opSetInput(newop,boolVn1,0);
       data.opInsertBefore(newop,op);
       data.opSetInput(op,vn,0);
       data.opRemoveInput(actionop,1); // eliminate the INT_ADD operator
@@ -2742,7 +2739,7 @@ int4 RuleBoolZext::applyOp(PcodeOp *op,Funcdata &data)
     else if (val != 0)
       return 0;			// Not comparing with 0 or -1
 
-    data.opSetInput(actionop,boolop1->getOut(),0);
+    data.opSetInput(actionop,boolVn1,0);
     data.opSetInput(actionop,data.newConstant(1,val),1);
     return 1;
   case CPUI_INT_AND:
@@ -2771,17 +2768,16 @@ int4 RuleBoolZext::applyOp(PcodeOp *op,Funcdata &data)
   zextop2 = multop2->getIn(0)->getDef();
   if (zextop2 == (PcodeOp *)0) return 0;
   if (zextop2->code() != CPUI_INT_ZEXT) return 0;
-  boolop2 = zextop2->getIn(0)->getDef();
-  if (boolop2 == (PcodeOp *)0) return 0;
-  if (!boolop2->isCalculatedBool()) return 0;
+  boolVn2 = zextop2->getIn(0);
+  if (!boolVn2->isBooleanValue(data.isTypeRecoveryOn())) return 0;
 
   // Do the boolean calculation on unextended boolean values
   // and then extend the result
   PcodeOp *newop = data.newOp(2,actionop->getAddr());
   Varnode *newres = data.newUniqueOut(1,newop);
   data.opSetOpcode(newop,opc);
-  data.opSetInput(newop, boolop1->getOut(), 0);
-  data.opSetInput(newop, boolop2->getOut(), 1);
+  data.opSetInput(newop, boolVn1, 0);
+  data.opSetInput(newop, boolVn2, 1);
   data.opInsertBefore(newop,actionop);
 
   PcodeOp *newzext = data.newOp(1,actionop->getAddr());
@@ -2811,19 +2807,17 @@ void RuleLogic2Bool::getOpList(vector<uint4> &oplist) const
 int4 RuleLogic2Bool::applyOp(PcodeOp *op,Funcdata &data)
 
 {
-  PcodeOp *boolop;
+  Varnode *boolVn;
 
-  if (!op->getIn(0)->isWritten()) return 0;
-  boolop = op->getIn(0)->getDef();
-  if (!boolop->isCalculatedBool()) return 0;
+  boolVn = op->getIn(0);
+  if (!boolVn->isBooleanValue(data.isTypeRecoveryOn())) return 0;
   Varnode *in1 = op->getIn(1);
-  if (!in1->isWritten()) {
-    if ((!in1->isConstant())||(in1->getOffset()>(uintb)1)) // If one side is a constant 0 or 1, this is boolean
+  if (in1->isConstant()) {
+    if (in1->getOffset()>(uintb)1) // If one side is a constant 0 or 1, this is boolean
       return 0;
   }
-  else {
-    boolop = op->getIn(1)->getDef();
-    if (!boolop->isCalculatedBool()) return 0;
+  else if (!in1->isBooleanValue(data.isTypeRecoveryOn())) {
+    return 0;
   }
   switch(op->code()) {
   case CPUI_INT_AND:
@@ -6292,7 +6286,7 @@ int4 RulePtrArith::applyOp(PcodeOp *op,Funcdata &data)
   int4 slot;
   const Datatype *ct = (const Datatype *)0; // Unnecessary initialization
 
-  if (!data.isTypeRecoveryOn()) return 0;
+  if (!data.hasTypeRecoveryStarted()) return 0;
 
   for(slot=0;slot<op->numInput();++slot) { // Search for pointer type
     ct = op->getIn(slot)->getTypeReadFacing(op);
@@ -6330,7 +6324,7 @@ int4 RuleStructOffset0::applyOp(PcodeOp *op,Funcdata &data)
 {
   int4 movesize;			// Number of bytes being moved by load or store
 
-  if (!data.isTypeRecoveryOn()) return 0;
+  if (!data.hasTypeRecoveryStarted()) return 0;
   if (op->code()==CPUI_LOAD) {
     movesize = op->getOut()->getSize();
   }
@@ -6478,7 +6472,7 @@ int4 RulePushPtr::applyOp(PcodeOp *op,Funcdata &data)
   int4 slot;
   Varnode *vni = (Varnode *)0;
 
-  if (!data.isTypeRecoveryOn()) return 0;
+  if (!data.hasTypeRecoveryStarted()) return 0;
   for(slot=0;slot<op->numInput();++slot) { // Search for pointer type
     vni = op->getIn(slot);
     if (vni->getTypeReadFacing(op)->getMetatype() == TYPE_PTR) break;
@@ -6542,7 +6536,7 @@ int4 RulePtraddUndo::applyOp(PcodeOp *op,Funcdata &data)
   Varnode *basevn;
   TypePointer *tp;
 
-  if (!data.isTypeRecoveryOn()) return 0;
+  if (!data.hasTypeRecoveryStarted()) return 0;
   int4 size = (int4)op->getIn(2)->getOffset(); // Size the PTRADD thinks we are pointing
   basevn = op->getIn(0);
   tp = (TypePointer *)basevn->getTypeReadFacing(op);
@@ -6572,7 +6566,7 @@ void RulePtrsubUndo::getOpList(vector<uint4> &oplist) const
 int4 RulePtrsubUndo::applyOp(PcodeOp *op,Funcdata &data)
 
 {
-  if (!data.isTypeRecoveryOn()) return 0;
+  if (!data.hasTypeRecoveryStarted()) return 0;
 
   Varnode *basevn = op->getIn(0);
   if (basevn->getTypeReadFacing(op)->isPtrsubMatching(op->getIn(1)->getOffset()))

Ghidra/Features/Decompiler/src/decompile/cpp/varmap.cc

@@ -317,7 +317,7 @@ void ScopeLocal::collectNameRecs(void)
 void ScopeLocal::annotateRawStackPtr(void)
 
 {
-  if (!fd->isTypeRecoveryOn()) return;
+  if (!fd->hasTypeRecoveryStarted()) return;
   Varnode *spVn = fd->findSpacebaseInput(space);
   if (spVn == (Varnode *)0) return;
   list<PcodeOp *>::const_iterator iter;

Ghidra/Features/Decompiler/src/decompile/cpp/varnode.cc

@@ -819,6 +819,24 @@ Datatype *Varnode::getLocalType(bool &blockup) const
   return ct;
 }
 
+/// If \b this varnode is produced by an operation with a boolean output, or if it is
+/// formally marked with a boolean data-type, return \b true. The parameter \b trustAnnotation
+/// toggles whether or not the formal data-type is trusted.
+/// \return \b true if \b this is a formal boolean, \b false otherwise
+bool Varnode::isBooleanValue(bool useAnnotation) const
+
+{
+  if (isWritten()) return def->isCalculatedBool();
+  if (!useAnnotation)
+    return false;
+  if ((flags & (input | typelock)) == (input | typelock)) {
+    if (size == 1 && type->getMetatype() == TYPE_BOOL)
+      return true;
+  }
+  return false;
+}
+
+
 /// Make a local determination if \b this and \b op2 hold the same value. We check if
 /// there is a common ancester for which both \b this and \b op2 are created from a direct
 /// sequence of COPY operations. NOTE: This is a transitive relationship

Ghidra/Features/Decompiler/src/decompile/cpp/varnode.hh

@@ -328,6 +328,7 @@ public:
   void copySymbol(const Varnode *vn); ///< Copy symbol info from \b vn
   void copySymbolIfValid(const Varnode *vn);	///< Copy symbol info from \b vn if constant value matches
   Datatype *getLocalType(bool &blockup) const; ///< Calculate type of Varnode based on local information
+  bool isBooleanValue(bool useAnnotation) const;	///< Does \b this Varnode hold a formal boolean value
   bool copyShadow(const Varnode *op2) const; ///< Are \b this and \b op2 copied from the same source?
   void encode(Encoder &encoder) const; ///< Encode a description of \b this to a stream
   static bool comparePointers(const Varnode *a,const Varnode *b) { return (*a < *b); }	///< Compare Varnodes as pointers