development/ghidra
development/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra synced 2024-09-13 21:56:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/adamopolous_test_script_fixes' into Ghidra_9.1

Browse source
This commit is contained in:
Ryan Kurtz 2019-09-17 09:52:53 -04:00
parent 95b58cbd94 d176879548
commit 0ed8fb63d7
3 changed files with 90 additions and 279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
gradle/root/test.gradle
View file

@ -367,7 +367,7 @@ def createTestTask(Project subproject, String testType, String bucketName, int t
group "test"
testClassesDirs = files subproject.sourceSets["$testType"].output.classesDirs
classpath = subproject.sourceSets["$testType"].runtimeClasspath
maxParallelForks = numMaxParallelForks
initTestJVM(t, testRootDirName)
@ -411,17 +411,13 @@ def createTestTask(Project subproject, String testType, String bucketName, int t
*********************************************************************************/
configure(subprojects.findAll {parallelMode == true}) { subproject ->
afterEvaluate {
// "subprojects { afterEvaluate { evaluate()"
// forces evaluation of subproject configuration. Needed for inheriting excludes
// from non-parallel counterpart.
// subproject.evaluate()
if (!shouldSkipTestTaskCreation(subproject)) {
logger.info("parallelCombinedTestReport: Creating 'test' tasks for " + subproject.name + " subproject.")
Map<String,Map> testMap = getTestsForSubProject(subproject.sourceSets.test.java)
Map<String,List> testMap = getTestsForSubProject(subproject.sourceSets.test.java)
for (Map.Entry<String,Map> classMap : testMap.entrySet()) {
for (Map.Entry<String,List> classMap : testMap.entrySet()) {
String bucketName = classMap.getKey();
@ -429,12 +425,10 @@ configure(subprojects.findAll {parallelMode == true}) { subproject ->
int taskNameCounter = 1 // task suffix
int numMaxParallelForks = 40 // unit tests are fast; 40 seems to be a reasonable number
Map<String,Long> tests = classMap.getValue();
def sorted = tests.sort { a, b -> b.value <=> a.value };
List<String> classesList = new ArrayList(sorted.keySet());
List<String> tests = classMap.getValue();
while (classesListPosition < classesList.size()) {
createTestTask(subproject, "test", bucketName, taskNameCounter, classesList, classesListPosition, numMaxParallelForks)
while (classesListPosition < tests.size()) {
createTestTask(subproject, "test", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks)
classesListPosition+=numMaxParallelForks
taskNameCounter+=1; // "test_1_appConfig", "test_2_appConfig, etc.
}
@ -444,9 +438,9 @@ configure(subprojects.findAll {parallelMode == true}) { subproject ->
if (!shouldSkipIntegrationTestTaskCreation(subproject)) {
logger.info("parallelCombinedTestReport: Creating 'integrationTest' tasks for " + subproject.name + " subproject.")
Map<String,Map> testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java)
Map<String,List> testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java)
for (Map.Entry<String,Map> classMap : testMap.entrySet()) {
for (Map.Entry<String,List> classMap : testMap.entrySet()) {
String bucketName = classMap.getKey();
@ -458,12 +452,10 @@ configure(subprojects.findAll {parallelMode == true}) { subproject ->
// 20 seems like a good balance of throughput vs resource usage for ghidratest server.
int numMaxParallelForks = 20
Map<String,Long> tests = classMap.getValue();
def sorted = tests.sort { a, b -> b.value <=> a.value };
List<String> classesList = new ArrayList(sorted.keySet());
while (classesListPosition < classesList.size()) {
createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, classesList, classesListPosition, numMaxParallelForks)
List<String> tests = classMap.getValue();
while (classesListPosition < tests.size()) {
createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks)
classesListPosition+=numMaxParallelForks
taskNameCounter+=1; // "integrationTest_1_appConfig", "integrationTest_2_appConfig, etc.
}

1
gradle/support/app_config_breakout.txt
View file

@ -82,6 +82,7 @@ AbstractToolSavingTest
AbstractVersionControlActionTest
AbstractVTCorrelatorTest
AbstractVTMarkupItemTest
CallTreePluginTest
DiffTestAdapter
DWARFTestBase
AbstractSelfSimilarCorrelatorTest

336
gradle/support/testUtils.gradle
View file

@ -4,9 +4,10 @@ import java.lang.reflect.Constructor;
import java.lang.*;
import java.io.*;
// This is a map of configuration type names (integration vs. non-integration vs. docking etc..)
// to tests (test name, duration)
ext.testReport = null;
ext.integrationConfigs = new ArrayList<>();
ext.dockingConfigs = new ArrayList<>();
ext.appConfigs = new ArrayList<>();
ext.ghidraConfigs = new ArrayList<>();
/*
* Checks if html test report for an individual test class has a valid name.
@ -15,190 +16,6 @@ boolean hasValidTestReportClassName(String name) {
return name.endsWith("Test.html") && !name.contains("Suite")
}
/*
* Returns duration for a test class report.
*/
long getDurationFromTestReportClass(String fileContents, String fileName) {
/* The duration for the entire test class appears in the test report as (multiline):
* <div class="infoBox" id="duration">
* <div class="counter">0s</div>
* The duration value may appear in the format of: 1m2s, 1m2.3s, 3.4s
*/
Pattern p = Pattern.compile("(?<=id=\"duration\">[\r\n]<div\\sclass=\"counter\">)(.*?)(?=</div)",
Pattern.MULTILINE);
Matcher m = p.matcher(fileContents);
assert m.find() == true
String duration = m.group()
assert duration != null && duration.trim().length() > 0
long durationInMillis
// Parse out the duration
if (duration.contains("m") && duration.contains("s")) { // has minute and seconds
int minutes = Integer.parseInt(duration.substring(0, duration.indexOf("m")))
double seconds = Double.parseDouble(duration.substring(duration.indexOf("m") + 1
, duration.length()-1))
durationInMillis = (minutes * 60 * 1000) + (seconds * 1000)
} else if (!duration.contains("m") && duration.contains("s")) { // has only seconds
double seconds = Double.parseDouble(duration.substring(0, duration.length()-1))
durationInMillis = (seconds * 1000)
} else { // unknown format
assert false : "getDurationFromTestReportClass: Unknown duration format in $fileName. 'duration' value is $duration"
}
logger.debug("getDurationFromTestReportClass: durationInMillis = '"+ durationInMillis
+"' parsed from duration = '" + duration + "' in $fileName")
return durationInMillis
}
/*
* Creates a map of tests to their durations, organized by the type of
* application configuration they require.
*
* When creating groups of tests to run, we have to ensure that we not only
* group them by duration (to make the parallelization more efficient) but also
* by the type of application config they require (to avoid a catastrophic test
* failure).
*
* This timing information is gleaned by parsing the html results of a previous
* test run.
*
* The application config information is contained in the resource file
* app_config_breakout.txt.
*
* eg: GhidraAppConfiguration -> DiffTestTypeAdapter, 0.135s
*/
def Map<String, Map<String, Long>> getTestReport() {
// If we have already created the test report, do not waste time creating
// it again. Just return it.
if (project.testReport != null) {
return project.testReport;
}
logger.debug("getTestReport: Populating 'testReport' using '$testTimeParserInputDir'")
testReport = new HashMap<String,Map>();
List<String> integrationConfigs = new ArrayList<>();
List<String> dockingConfigs = new ArrayList<>();
List<String> appConfigs = new ArrayList<>();
List<String> ghidraConfigs = new ArrayList<>();
parseApplicationConfigs(dockingConfigs, integrationConfigs, appConfigs, ghidraConfigs);
File classesReportDir = new File(testTimeParserInputDir)
if(!classesReportDir.exists()) {
logger.info("getTestReport: The path '$testTimeParserInputDir' does not exist on the file system." +
" Returning empty testReport map.")
return Collections.emptyMap();
}
// These are the configuration 'buckets' that each test class will be dumped
// into. Only tests in the same bucket will be run together.
Map dockingBucket = new HashMap<String, Long>();
Map integrationBucket = new HashMap<String, Long>();
Map appBucket = new HashMap<String, Long>();
Map ghidraBucket = new HashMap<String, Long>();
Map unknownBucket = new HashMap<String, Long>();
int excludedHtmlFiles = 0 // counter
int totalHtmlFiles = 0
String excludedHtmlFileNames = "" // for log.info summary message
classesReportDir.eachFileRecurse (FileType.FILES) { file ->
totalHtmlFiles++
// Only read html file for a Test and not a test Suite
if(hasValidTestReportClassName(file.name)) {
String fileContents = file.text
/* The fully qualified class name appears in the test report as:
* <h1>Class ghidra.app.plugin.assembler.sleigh.BuilderTest</h1>
*/
String fqNameFromTestReport = fileContents.find("(?<=<h1>Class\\s).*?(?=</h1>)")
int nameIndex = fqNameFromTestReport.lastIndexOf('.')
String shortName = fqNameFromTestReport.substring(nameIndex+1);
long durationInMillis = getDurationFromTestReportClass(fileContents, file.name)
File rootDir = project.rootDir.getParentFile();
File foundFile;
fileTree(rootDir.getAbsolutePath()).visit { FileVisitDetails details ->
if (details.getName().contains(shortName + ".java")) {
foundFile = details.getFile();
}
}
if (!foundFile.exists()) {
// throw error
}
String javaFileContents = foundFile.text;
if (javaFileContents.contains(shortName)) {
// Match the word right after "extends", if there is one
Pattern p = Pattern.compile("extends\\W+(\\w+)");
Matcher m = p.matcher(javaFileContents);
String extendsClass = "";
while (m.find()) {
extendsClass = m.group(1);
break;
}
if (extendsClass.isEmpty()) {
unknownBucket.put(fqNameFromTestReport, durationInMillis);
}
else {
if (integrationConfigs.contains(extendsClass)) {
integrationBucket.put(fqNameFromTestReport, durationInMillis);
}
else if (dockingConfigs.contains(extendsClass)) {
dockingBucket.put(fqNameFromTestReport, durationInMillis);
}
else if (appConfigs.contains(extendsClass)) {
appBucket.put(fqNameFromTestReport, durationInMillis);
}
else if (ghidraConfigs.contains(extendsClass)) {
ghidraBucket.put(fqNameFromTestReport, durationInMillis);
}
else {
unknownBucket.put(fqNameFromTestReport, durationInMillis);
}
}
}
testReport.put("integration", integrationBucket);
testReport.put("docking", dockingBucket);
testReport.put("app", appBucket);
testReport.put("ghidra", ghidraBucket);
testReport.put("unknown", unknownBucket);
logger.debug("getTestReport: Added to testReport: class name = '"
+ fqNameFromTestReport + "' and durationInMillis = '"+ durationInMillis
+"' from " + file.name)
}
else {
logger.debug("getTestReport: Excluding " + file.name + " from test report parsing.")
excludedHtmlFileNames += file.name + ", "
excludedHtmlFiles++
}
}
int processedFiles = 0;
for (Map.Entry<String,Map> entry : testReport.entrySet()) {
Map<String,Long> testMap = entry.getValue();
processedFiles += testMap.size();
}
assert totalHtmlFiles != 0 : "getTestReport: Did not parse any valid html files in $testTimeParserInputDir. Directory might be empty"
assert totalHtmlFiles == (processedFiles + excludedHtmlFiles) : "Not all html files processed."
logger.info("getTestReport:\n" +
"\tIncluded " + testReport.size() + " and excluded " + excludedHtmlFiles
+ " html files out of " + totalHtmlFiles + " in Junit test report.\n"
+ "\tExcluded html file names are: " + excludedHtmlFileNames + "\n"
+ "\tParsed test report located at " + testTimeParserInputDir)
return project.testReport
}
/**
* Parses the file containing the mapping of test classes to application configs and assigns those
* classes to the appropriate lists.
@ -299,12 +116,11 @@ String constructFullyQualifiedClassName(String fileContents, String fileName) {
return packageName + "." + fileName.replace(".java","")
}
/* Creates a list of test classes, sorted by duration, for a subproject.
* First parses JUnit test report located at 'testTimeParserInputDir' for <fully qualified class name, duration in milliseconds> .
* Then traverses a test sourceSet for a subproject for a test to include and assigns a duration value.
* Returns a sorted list of test classes for the sourceSet parameter.
/*
* Creates a map of config types to the test classes for that type. This should be
* used to creates sets of tests that can be run in parallel.
*/
def Map<String, Map> getTestsForSubProject(SourceDirectorySet sourceDirectorySet) {
def Map<String, List> getTestsForSubProject(SourceDirectorySet sourceDirectorySet) {
def testsForSubProject = new HashMap<String,LinkedHashMap>();
@ -317,9 +133,18 @@ def Map<String, Map> getTestsForSubProject(SourceDirectorySet sourceDirectorySet
logger.debug("getTestsForSubProject: Found " + sourceDirectorySet.files.size()
+ " file(s) in source set to process.")
Map<String,Map> testReports = getTestReport();
assert (testReports != null) : "getTestsForSubProject: testReport should not be null"
// Read in the config file that indicates which base test classes are associated with
// which application configs. This is not a comprehensive list of all test classes
// in Ghidra - it's the list of all classes that are extended in Ghidra.
parseApplicationConfigs(dockingConfigs, integrationConfigs, appConfigs, ghidraConfigs);
// "Buckets" that delineate which test classes should be run together.
List dockingBucket = new ArrayList<String>();
List integrationBucket = new ArrayList<String>();
List appBucket = new ArrayList<String>();
List ghidraBucket = new ArrayList<String>();
List unknownBucket = new ArrayList<String>();
for (File file : sourceDirectorySet.getFiles()) {
logger.debug("getTestsForSubProject: Found file in sourceSet = " + file.name)
@ -331,7 +156,7 @@ def Map<String, Map> getTestsForSubProject(SourceDirectorySet sourceDirectorySet
}
String fileContents = file.text
// Must not have a Category annotation
if (hasCategoryExcludes(fileContents)) {
logger.debug("getTestsForSubProject: Found category exclude for '"
@ -339,75 +164,68 @@ def Map<String, Map> getTestsForSubProject(SourceDirectorySet sourceDirectorySet
excludedClassFilesCategory++
continue
}
String fqName = constructFullyQualifiedClassName( fileContents, file.name)
boolean foundTest = false;
for (Map.Entry<String,Map> entry : testReports.entrySet()) {
String configName = entry.getKey();
Map<String,Long> tests = entry.getValue();
// Get any extending class so we can see what bucket it belongs to. Do this
// by grabbing the next word after "extends".
Pattern p = Pattern.compile("extends\\W+(\\w+)");
Matcher m = p.matcher(fileContents);
String extendsClass = "";
while (m.find()) {
extendsClass = m.group(1);
break;
}
// Get full package name of the class.
Pattern p2 = Pattern.compile("package\\s+([a-zA_Z_][\\.\\w]*);");
Matcher m2 = p2.matcher(fileContents);
String packageName = "";
while (m2.find()) {
packageName = m2.group(1);
break;
}
// Construct a var of the form "<package name>.<class name>". This will
// be stored in the appropriate bucket and be used when creating test
// tasks later on.
String className = packageName + "." + file.name
className = className.replace(".java", "")
if (tests.containsKey(fqName)) {
foundTest = true;
if (!testsForSubProject.containsKey(configName)) {
Map<String,Map> configToTestMap = new LinkedHashMap<>();
testsForSubProject.put(configName, configToTestMap);
}
Map<String,Long> subTests = testsForSubProject.get(configName);
long duration = tests.get(fqName);
if (duration > 0) {
subTests.put(fqName,duration);
logger.debug("getTestsForSubProject: Adding '" + fqName + "'")
includedClassFilesInTestReport++
}
else {
logger.debug("getTestsForSubProject: Excluding '" + fqName
+ "' because duration from test report is " + duration
+ "ms. Probably because all test methods are @Ignore'd." )
excludedClassAllTestsIgnored++
}
if (extendsClass.isEmpty()) {
unknownBucket.add(className);
}
else {
if (integrationConfigs.contains(extendsClass)) {
integrationBucket.add(className);
}
else if (dockingConfigs.contains(extendsClass)) {
dockingBucket.add(className);
}
else if (appConfigs.contains(extendsClass)) {
appBucket.add(className);
}
else if (ghidraConfigs.contains(extendsClass)) {
ghidraBucket.add(className);
}
else {
unknownBucket.add(className);
}
}
if (!foundTest) {
// Don't know what this test is so put it in the "unknown" bucket
if (!testsForSubProject.containsKey("unknown")) {
Map<String,Map> configToTestMap = new LinkedHashMap<>();
testsForSubProject.put("unknown", configToTestMap);
}
Map<String,Long> subTests = testsForSubProject.get("unknown");
logger.debug("getTestsForSubProject: Found test class not in test report."
+ " Bumping to front of tasks '" + fqName + "'")
subTests.put(fqName, 3600000) // cheap way to bump to front of (eventually) sorted list
includedClassFilesNotInTestReport++
}
}
// Sort by duration
for (Map.Entry<String,Map> entry : testsForSubProject.entrySet()) {
Map<String,Long> testMap = entry.getValue();
testMap.sort { a, b -> b.value <=> a.value }
}
/* logger.info ("getTestsForSubProject:\n"
+ "\tIncluding " + includedClassFilesInTestReport + " test classes for this sourceSet because they are in the test report.\n"
+ "\tIncluding/bumping " + includedClassFilesNotInTestReport + " not in test report.\n"
+ "\tExcluding "+ excludedClassFilesBadName +" based on name not ending in 'Test' or contains 'Abstract' or 'Suite', " + excludedClassFilesCategory
+ " based on '@Category, " + excludedClassAllTestsIgnored + " because duration = 0ms.\n"
+ "\tReturning sorted list of size "+ sorted.size() + " out of " + sourceDirectorySet.files.size()
+ " total files found in sourceSet.")
*/
int filesProcessed = includedClassFilesNotInTestReport + includedClassFilesInTestReport +
excludedClassFilesBadName + excludedClassFilesCategory + excludedClassAllTestsIgnored
assert sourceDirectorySet.files.size() == filesProcessed : "getTestsForSubProject did not process every file in sourceSet"
Map<String,List> testBuckets = new HashMap<String,List>();
testBuckets.put("docking", dockingBucket)
testBuckets.put("integration", integrationBucket)
testBuckets.put("app", appBucket)
testBuckets.put("ghidra", ghidraBucket)
testBuckets.put("unknown", unknownBucket)
logger.debug("integration bucket: " + integrationBucket)
logger.debug("docking bucket: " + dockingBucket)
logger.debug("app bucket: " + appBucket)
logger.debug("ghidra bucket: " + ghidraBucket)
logger.debug("unknown bucket: " + unknownBucket)
return testsForSubProject;
return testBuckets;
}
/*********************************************************************************