Skip to content

Commit

Permalink
Merge branch 'use-logical-signature-for-incrementality'
Browse files Browse the repository at this point in the history
  • Loading branch information
@PaulKlint
PaulKlint committed Oct 29, 2024
2 parents 99b91ba + 8ec8da0 commit 36f1b2c
Show file tree
Hide file tree
Showing 12 changed files with 1,255 additions and 732 deletions.
135 changes: 67 additions & 68 deletions src/org/rascalmpl/core/library/lang/rascalcore/check/AType.rsc
View file

Large diffs are not rendered by default.

65 changes: 58 additions & 7 deletions src/org/rascalmpl/core/library/lang/rascalcore/check/Checker.rsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,8 @@ ModuleStatus rascalTModelForLocs(
ms = getImportAndExtendGraph(topModuleNames, compilerConfig);

if(/error(_,_) := ms.messages){
return ms;
return clearTModelCache(ms);
}
if(compilerConfig.forceCompilationTopModule){
Expand All @@ -153,8 +154,8 @@ ModuleStatus rascalTModelForLocs(
}
}
imports_and_extends = ms.strPaths<0,2>;
imports_and_extends = ms.strPaths<0,2>;
<components, sorted> = stronglyConnectedComponentsAndTopSort(imports_and_extends);
map[str, set[str]] module2component = (m : c | c <- components, m <- c);
Expand Down Expand Up @@ -187,7 +188,7 @@ ModuleStatus rascalTModelForLocs(
component = module2component[ordered[mi]];
jobStep(jobName, intercalate(" + ", [*component]), work=size(component));
recheck = !all(m <- component, (tpl_uptodate() in ms.status[m] || checked() in ms.status[m]));
recheck = !all(m <- component, ms.status[m]?, (tpl_uptodate() in ms.status[m] || checked() in ms.status[m]));
for(m <- component){
mi += 1;
if(!recheck){
Expand All @@ -199,7 +200,34 @@ ModuleStatus rascalTModelForLocs(
}
}
}
if(!all(m <- component, tpl_uptodate() in ms.status[m] || checked() in ms.status[m])){
compatible_with_all_imports = true;
for(m <- component){
m_compatible = false;
<found, tm, ms> = getTModelForModule(m, ms);
if(found){
imports_extends_m = imports_and_extends[m];
<m_compatible, ms> = importsAndExtendsAreBinaryCompatible(tm, imports_extends_m, ms);
if(m_compatible){
ms.status[m] += {tpl_uptodate(), checked()};
}
}
compatible_with_all_imports = compatible_with_all_imports && m_compatible;
}
any_rsc_changed = any(m <- component, rsc_changed() in ms.status[m]);
all_tmodels_uptodate = true;
for(m <- component){
if(tpl_uptodate() notin ms.status[m] && checked() notin ms.status[m])
all_tmodels_uptodate = false;
}
recheckCond = !compatible_with_all_imports || any_rsc_changed || !all_tmodels_uptodate;
if(recheckCond){
if(ms.compilerConfig.verbose){
println("recheck <component>: compatible_with_all_imports: <compatible_with_all_imports>, any_rsc_changed: <any_rsc_changed>, all_tmodels_uptodate: <all_tmodels_uptodate>");
}
<tm, ms> = rascalTModelComponent(component, ms);
moduleScopes += getModuleScopes(tm);
map[str,TModel] tmodels_for_component = ();
Expand All @@ -218,6 +246,7 @@ ModuleStatus rascalTModelForLocs(
msgs = [];
<success, pt, ms> = getModuleParseTree(m, ms);
if(success){
msgs += [info("Checked <m>", pt.header.name@\loc)];
check_imports:
for(imod <- pt.header.imports, imod has \module){
iname = unescape("<imod.\module.name>");
Expand All @@ -228,10 +257,10 @@ ModuleStatus rascalTModelForLocs(
if(iname == "ParseTree" && implicitlyUsesParseTree(ms.moduleLocs[m].path, tm)){
continue check_imports;
}
if(ms.moduleLocs[iname]? && implicitlyUsesLayoutOrLexical(ms.moduleLocs[m].path, ms.moduleLocs[iname].path, tm)){
if(ms.moduleLocs[iname]? && ms.moduleLocs[m]? && implicitlyUsesLayoutOrLexical(ms.moduleLocs[m].path, ms.moduleLocs[iname].path, tm)){
continue check_imports;
}
if(ms.moduleLocs[iname]? && usesOrExtendsADT(ms.moduleLocs[m].path, ms.moduleLocs[iname].path, tm)){
if(ms.moduleLocs[iname]? && ms.moduleLocs[m]? && usesOrExtendsADT(ms.moduleLocs[m].path, ms.moduleLocs[iname].path, tm)){
continue check_imports;
}
if(checked() in ms.status[iname] && rsc_not_found() notin ms.status[iname]){
Expand Down Expand Up @@ -271,6 +300,16 @@ ModuleStatus rascalTModelForLocs(
}
}
ms = doSaveModule(component, m_imports, m_extends, ms, moduleScopes, transient_tms, compilerConfig);
for(m <- component){
ms.status[m] -= {rsc_changed()};
ms.status[m] += {tpl_uptodate()};
}
} else {
for(m <- component){
imports = { imp | <m1, importPath(), imp> <- ms.strPaths, m1 == m };
extends = { ext | <m1, extendPath(), ext > <- ms.strPaths, m1 == m };
updateBOM(m, imports, extends, ms);
}
}
}
} catch ParseError(loc src): {
Expand All @@ -292,7 +331,7 @@ ModuleStatus rascalTModelForLocs(
}
jobEnd(jobName);
return ms;
return clearTModelCache(ms);
}
bool implicitlyUsesParseTree(str modulePath, TModel tm){
Expand Down Expand Up @@ -440,9 +479,21 @@ list[ModuleMessages] checkAll(loc root, RascalCompilerConfig compilerConfig){
// ---- Convenience check function during development -------------------------
void find(str s, ModuleStatus ms){
if(ms.moduleLocs[s]?) println("moduleLocs[<s>] = <ms.moduleLocs[s]>");
for(mname <- ms.tmodels){
tm = ms.tmodels[mname];
for(d <- tm.definitions){
def = tm.definitions[d];
if(contains("<def>", s)) {println("<mname>: <def>"); }
}
}
}
map[str, list[Message]] checkModules(list[str] moduleNames, RascalCompilerConfig compilerConfig) {
ModuleStatus ms = rascalTModelForNames(moduleNames, compilerConfig, dummy_compile1);
tmodels = ms.tmodels;
//find("Exception.rsc|(0,", ms);
tmMsgs = (mname : tmodels[mname].messages | mname <- tmodels, !isEmpty(tmodels[mname].messages));
return //(mname : tmodels[mname].messages | mname <- tmodels, !isEmpty(tmodels[mname].messages))
(mname : ms.messages[mname] + (tmMsgs[mname] ? []) | mname <- ms.messages, !isEmpty(ms.messages[mname]));
Expand Down
79 changes: 58 additions & 21 deletions src/org/rascalmpl/core/library/lang/rascalcore/check/CheckerCommon.rsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ void checkSupportedByParserGenerator(Tree t, Collector c){
data MStatus =
rsc_not_found()
| tpl_not_found()
| rsc_changed()
| parsed()
| parse_error()
| module_dependencies_extracted()
Expand Down Expand Up @@ -178,9 +179,21 @@ tuple[bool, Module, ModuleStatus] getModuleParseTree(str qualifiedModuleName, Mo
}
}

/*
* We implement a caching mechanism for TModels with the following properties:
* - tmodelCacheSize tmodels are cached.
* - The most frequently uses modules are hardwired and are never removed.
They are determined by static analysis of the import/extend graph.
* - TModels on file (.tpl) physical locations have been replaced by logical locations where possible.
* - When a TModel is read in, physical locations are converted by logical logical locations
* - The policy is to keep TModels in the cache in this physical form as long as possible.
* - During its presence in the cache, the BOM of a TModel may get updated.
* - When a TModel has to be removed from the cache, it is converted back to the logical form and written back to file.
*/

set[str] hardwired = {};

int tmodelCacheSize = 4; // should be > 0
int tmodelCacheSize = 12; // should be > 0

int maxHardwired = tmodelCacheSize/4;
Expand All @@ -202,25 +215,54 @@ void analyzeTModels(ModuleStatus ms){
hardwired = toSet(domain(hardwire));
}

ModuleStatus clearTModelCache(ModuleStatus ms){
for(candidate <- ms.tmodelLIFO){
ms = removeOldestTModelFromCache(ms);
}
for(candidate <- hardwired){
ms = removeTModel(candidate, ms);
}
return ms;
}

ModuleStatus removeTModel(str candidate, ModuleStatus ms){
if(tpl_saved() notin ms.status[candidate]){
pcfg = ms.pathConfig;
if(ms.compilerConfig.verbose) println("Save <candidate> before removing from cache <ms.status[candidate]>");
tm = ms.tmodels[candidate];
<found, tplLoc> = getTPLWriteLoc(candidate, pcfg);
tm = convertTModel2LogicalLocs(tm, ms.tmodels);
ms.status[candidate] += tpl_saved();
try {
writeBinaryValueFile(tplLoc, tm);
if(traceTPL) println("Written <tplLoc>");
} catch value e: {
throw "Cannot write TPL file <tplLoc>, reason: <e>";
}
}
ms.tmodels = delete(ms.tmodels, candidate);
return ms;
}

ModuleStatus removeOldestTModelFromCache(ModuleStatus ms){
if(size(ms.tmodelLIFO) > 0){
candidate = ms.tmodelLIFO[-1];
ms = removeTModel(candidate, ms);
if(traceTModelCache) println("*** deleting tmodel <candidate>, tmodels: <size(ms.tmodels)>, lifo: <size(ms.tmodelLIFO)>");
ms.tmodelLIFO = ms.tmodelLIFO[..-1];
}
return ms;
}

ModuleStatus addTModel (str qualifiedModuleName, TModel tm, ModuleStatus ms){
if(traceTModelCache) println("addTModel: <qualifiedModuleName>");
if(tmodelCacheSize > 0){
if(tpl_saved() notin ms.status[qualifiedModuleName]){
iprintln(ms.status);
throw "Cannot add module <qualifiedModuleName> with unsaved tpl";
}
ms.tmodels[qualifiedModuleName] = tm;
if(qualifiedModuleName notin hardwired){
if(qualifiedModuleName notin ms.tmodelLIFO){
ms.tmodelLIFO = [qualifiedModuleName, *ms.tmodelLIFO];
while(size(ms.tmodels) >= tmodelCacheSize && size(ms.tmodelLIFO) > 0 && ms.tmodelLIFO[-1] != qualifiedModuleName){
if(tpl_saved() notin ms.status[ms.tmodelLIFO[-1]]){
iprintln(ms.status);
throw "Cannot remove unsaved tpl <ms.tmodelLIFO[-1]>, <ms.status[ms.tmodelLIFO[-1]]>";
}
ms.tmodels = delete(ms.tmodels, ms.tmodelLIFO[-1]);
if(traceTModelCache) println("*** deleting tmodel <ms.tmodelLIFO[-1]>, tmodels: <size(ms.tmodels)>, lifo: <size(ms.tmodelLIFO)>");
ms.tmodelLIFO = ms.tmodelLIFO[..-1];
ms = removeOldestTModelFromCache(ms);
}
}
}
Expand All @@ -231,16 +273,11 @@ ModuleStatus addTModel (str qualifiedModuleName, TModel tm, ModuleStatus ms){
tuple[bool, TModel, ModuleStatus] getTModelForModule(str qualifiedModuleName, ModuleStatus ms){
if(traceTModelCache) println("getTModelForModule: <qualifiedModuleName>");
pcfg = ms.pathConfig;
if(ms.tmodels[qualifiedModuleName]?){
if(tpl_saved() notin ms.status[qualifiedModuleName]){
throw "Unsaved tmodel for <qualifiedModuleName> in cache";
}
if(ms.tmodels[qualifiedModuleName]? /*&& tpl_uptodate() in ms.status[qualifiedModuleName]*/){
return <true, ms.tmodels[qualifiedModuleName], ms>;
}
while(size(ms.tmodels) >= tmodelCacheSize && size(ms.tmodelLIFO) > 0 && ms.tmodelLIFO[-1] != qualifiedModuleName){
ms.tmodels = delete(ms.tmodels, ms.tmodelLIFO[-1]);
if(traceTModelCache) println("*** deleting tmodel <ms.tmodelLIFO[-1]>, tmodels: <size(ms.tmodels)>, lifo: <size(ms.tmodelLIFO)>");
ms.tmodelLIFO = ms.tmodelLIFO[..-1];
ms = removeOldestTModelFromCache(ms);
}

<found, tplLoc> = getTPLReadLoc(qualifiedModuleName, pcfg);
Expand All @@ -252,7 +289,7 @@ tuple[bool, TModel, ModuleStatus] getTModelForModule(str qualifiedModuleName, Mo
tpl = convertTModel2PhysicalLocs(tpl);

ms.tmodels[qualifiedModuleName] = tpl;
ms.status[qualifiedModuleName] += {tpl_uptodate(), tpl_saved()};
ms.status[qualifiedModuleName] ? {} += {tpl_uptodate(), tpl_saved()};
if(qualifiedModuleName notin hardwired){
ms.tmodelLIFO = [qualifiedModuleName, *ms.tmodelLIFO];
}
Expand Down Expand Up @@ -285,4 +322,4 @@ int nextClosure(){

void resetClosureCounter(){
closureCounter = 0;
}
}
Loading

0 comments on commit 36f1b2c

Please sign in to comment.