diff --git a/src/main/java/com/educoder/bridge/tmp/ASTStringUtilExt.java b/src/main/java/com/educoder/bridge/tmp/ASTStringUtilExt.java new file mode 100644 index 0000000..c148844 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/ASTStringUtilExt.java @@ -0,0 +1,169 @@ +package depends.extractor.cpp.cdt; + +import depends.entity.GenericName; +import depends.entity.TypeEntity; +import org.eclipse.cdt.core.dom.ast.*; +import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTNameSpecifier; +import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTQualifiedName; +import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTUsingDeclaration; +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTTemplateId; +import org.eclipse.cdt.internal.core.model.ASTStringUtil; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +/** + * This extends the CDT ASTStringUtil class. + * A tricky point here is that we have to use some of the reflection mechanism to invoke + * some private functions in ASTStringUtils class + * It is not good, but it seems the most easiest one to reuse existing functions + */ +public class ASTStringUtilExt extends ASTStringUtil { + public static String getName(IASTDeclSpecifier decl) { + StringBuilder buffer = new StringBuilder(); + String name = appendBareDeclSpecifierString(buffer, decl).toString().replace("::", ".").replace("...", ""); + return name; + } + + public static String getName(IASTLiteralExpression expr) { + return expr.getRawSignature().replace("::", ".").replace("...", ""); + } + + public static String getTypeIdString(IASTTypeId typeId) { + StringBuilder sb = new StringBuilder(); + return appendBareTypeIdString(sb, typeId).toString().replace("::", "."); + } + + + /** + * retrieve template parameters from declSpecifier + */ + public static List getTemplateParameters(IASTDeclSpecifier declSpecifier) { + List parameters = new ArrayList<>(); + declSpecifier.accept(new TemplateParameterASTVisitor(parameters)); + return parameters; + } + + + + private static StringBuilder appendBareDeclSpecifierString(StringBuilder buffer, IASTDeclSpecifier declSpecifier) { + if (declSpecifier instanceof IASTCompositeTypeSpecifier) { + final IASTCompositeTypeSpecifier compositeTypeSpec = (IASTCompositeTypeSpecifier) declSpecifier; + appendBareNameString(buffer, compositeTypeSpec.getName()); + } else if (declSpecifier instanceof IASTElaboratedTypeSpecifier) { + final IASTElaboratedTypeSpecifier elaboratedTypeSpec = (IASTElaboratedTypeSpecifier) declSpecifier; + appendBareNameString(buffer, elaboratedTypeSpec.getName()); + } else if (declSpecifier instanceof IASTEnumerationSpecifier) { + final IASTEnumerationSpecifier enumerationSpec = (IASTEnumerationSpecifier) declSpecifier; + appendBareNameString(buffer, enumerationSpec.getName()); + } else if (declSpecifier instanceof IASTSimpleDeclSpecifier) { + buffer.append(TypeEntity.buildInType.getRawName()); + } else if (declSpecifier instanceof IASTNamedTypeSpecifier) { + final IASTNamedTypeSpecifier namedTypeSpec = (IASTNamedTypeSpecifier) declSpecifier; + appendBareNameString(buffer, namedTypeSpec.getName()); + } + return buffer; + } + + private static StringBuilder appendBareNameString(StringBuilder buffer, IASTName name) { + if (name instanceof ICPPASTQualifiedName) { + final ICPPASTQualifiedName qualifiedName = (ICPPASTQualifiedName) name; + final ICPPASTNameSpecifier[] segments = qualifiedName.getAllSegments(); + for (int i = 0; i < segments.length; i++) { + if (i > 0) { + buffer.append("."); + } + appendQualifiedNameStringWithReflection(buffer, segments[i]); + } + } else if (name instanceof CPPASTTemplateId) { + appendQualifiedNameStringWithReflection(buffer,(CPPASTTemplateId)name); + } else if (name != null) { + buffer.append(name.getSimpleID()); + } + return buffer; + } + + private static void appendQualifiedNameStringWithReflection(StringBuilder buffer, IASTName name) { + try { + Method m = ASTStringUtil.class.getDeclaredMethod("appendQualifiedNameString", StringBuilder.class, + IASTName.class); + m.setAccessible(true); // if security settings allow this + m.invoke(null, buffer, name); // use null if the method is static + } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException e) { + System.err.println("Error: cannot invoke ASTStringUtils method of "); + } + } + + private static void appendQualifiedNameStringWithReflection(StringBuilder buffer, + CPPASTTemplateId templateId) { + appendQualifiedNameStringWithReflection(buffer,templateId.getTemplateName()); + } + + private static void appendQualifiedNameStringWithReflection(StringBuilder buffer, + ICPPASTNameSpecifier nameSpecifier) { + if (nameSpecifier instanceof CPPASTTemplateId) { + appendQualifiedNameStringWithReflection(buffer,(CPPASTTemplateId)nameSpecifier); + return; + } + try { + Method m = ASTStringUtil.class.getDeclaredMethod("appendQualifiedNameString", StringBuilder.class, + ICPPASTNameSpecifier.class); + m.setAccessible(true); // if security settings allow this + m.invoke(null, buffer, nameSpecifier); // use null if the method is static + } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException e) { + System.err.println("Error: cannot invoke ASTStringUtils method of "); + } + } + + private static StringBuilder appendBareTypeIdString(StringBuilder buffer, IASTTypeId typeId) { + return appendBareDeclSpecifierString(buffer, typeId.getDeclSpecifier()); + } + + + public static String getName(IASTDeclarator declarator) { + return declarator.getName().toString().replace("::", "."); + } + + + public static String getName(ICPPASTUsingDeclaration declaration) { + return declaration.getName().toString().replace("::", "."); + } + + + public static String getName(IASTName name) { + return name.getRawSignature().toString().replace("::", "."); + } + + + private static StringBuilder appendBareNameString(StringBuilder buffer, ICPPASTNameSpecifier name) { + if (name instanceof ICPPASTQualifiedName) { + final ICPPASTQualifiedName qualifiedName = (ICPPASTQualifiedName) name; + final ICPPASTNameSpecifier[] segments = qualifiedName.getAllSegments(); + for (int i = 0; i < segments.length; i++) { + if (i > 0) { + buffer.append("."); + } + appendQualifiedNameStringWithReflection(buffer, segments[i]); + } + } else if (name instanceof CPPASTTemplateId) { + appendQualifiedNameStringWithReflection(buffer,(CPPASTTemplateId)name); + } else if (name != null) { + buffer.append(name.getRawSignature()); + } + return buffer; + } + + public static String getName(ICPPASTNameSpecifier nameSpecifier) { + StringBuilder buffer = new StringBuilder(); + String name = appendBareNameString(buffer, nameSpecifier).toString().replace("::", ".").replace("...", ""); + return name; + } + + + + +} \ No newline at end of file diff --git a/src/main/java/com/educoder/bridge/tmp/AbstractLangProcessor.java b/src/main/java/com/educoder/bridge/tmp/AbstractLangProcessor.java new file mode 100644 index 0000000..da5728c --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/AbstractLangProcessor.java @@ -0,0 +1,248 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor; + +import depends.entity.Entity; +import depends.entity.FileEntity; +import depends.entity.repo.BuiltInType; +import depends.entity.repo.EntityRepo; +import depends.entity.repo.InMemoryEntityRepo; +import depends.relations.ImportLookupStrategy; +import depends.relations.IBindingResolver; +import multilang.depends.util.file.FileTraversal; +import multilang.depends.util.file.FileUtil; +import org.codehaus.plexus.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +abstract public class AbstractLangProcessor { + + /** + * The name of the lang + * + * @return + */ + public abstract String supportedLanguage(); + + /** + * The file suffixes in the lang + * + * @return + */ + public abstract String[] fileSuffixes(); + + /** + * Strategy of how to lookup types and entities in the lang. + * + * @return + */ + public abstract ImportLookupStrategy getImportLookupStrategy(); + + /** + * The builtInType of the lang. + * + * @return + */ + public abstract BuiltInType getBuiltInType(); + + /** + * The language specific file parser + * + * @param fileFullPath + * @return + */ + public abstract FileParser createFileParser(); + + public IBindingResolver bindingResolver; + protected EntityRepo entityRepo; + protected String inputSrcPath; + public String[] includeDirs; + private Set potentialExternalDependencies; + private List includePaths; + private static Logger logger = LoggerFactory.getLogger(AbstractLangProcessor.class); + + public AbstractLangProcessor() { + entityRepo = new InMemoryEntityRepo(); + } + + /** + * The process steps of build dependencies. Step 1: parse all files, add + * entities and expression into repositories Step 2: resolve bindings of files + * (if not resolved yet) Step 3: identify dependencies + * + * @param inputDir + * @param includeDir + * @param bindingResolver + * @return + */ + public EntityRepo buildDependencies(String inputDir, String[] includeDir, IBindingResolver bindingResolver) { + this.inputSrcPath = inputDir; + this.includeDirs = includeDir; + this.bindingResolver = bindingResolver; + logger.info("Start parsing files..."); + parseAllFiles(); + markAllEntitiesScope(); + if (logger.isInfoEnabled()) { + logger.info("Resolve types and bindings of variables, methods and expressions.... " + this.inputSrcPath); + logger.info("Heap Information: " + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()); + } + resolveBindings(); + if (logger.isInfoEnabled()) { + System.gc(); + logger.info("Heap Information: " + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()); + } + return entityRepo; + } + + private void markAllEntitiesScope() { + entityRepo.getFileEntities().stream().forEach(entity -> { + Entity file = entity.getAncestorOfType(FileEntity.class); + try { + if (!file.getQualifiedName().startsWith(this.inputSrcPath)) { + entity.setInScope(false); + } + } catch (Exception e) { + + } + }); + } + + /** + * @return unsolved bindings + */ + public void resolveBindings() { + System.out.println("Resolve types and bindings of variables, methods and expressions...."); + this.potentialExternalDependencies = bindingResolver.resolveAllBindings(this.isEagerExpressionResolve()); + if (getExternalDependencies().size() > 0) { + System.out.println("There are " + getExternalDependencies().size() + " items are potential external dependencies."); + } + System.out.println("types and bindings resolved successfully..."); + } + + + + private final void parseAllFiles() { + System.out.println("Start parsing files..."); + Set phase2Files = new HashSet<>(); + FileTraversal fileTransversal = new FileTraversal(new FileTraversal.IFileVisitor() { + @Override + public void visit(File file) { + String fileFullPath = file.getAbsolutePath(); + if (!fileFullPath.startsWith(inputSrcPath)) { + return; + } + parseFile(fileFullPath, phase2Files); + } + + }); + fileTransversal.extensionFilter(this.fileSuffixes()); + fileTransversal.travers(this.inputSrcPath); + for (String f : phase2Files) { + parseFile(f, phase2Files); + } + System.out.println("all files procceed successfully..."); + + } + + protected void parseFile(String fileFullPath, Set phase2Files) { + FileParser fileParser = createFileParser(); + try { + if (fileParser.isPhase2Files(fileFullPath)){ + phase2Files.add(fileFullPath); + }else { + fileParser.parse(fileFullPath); + } + } catch (IOException e) { + e.printStackTrace(); + } catch (Exception e) { + System.err.println("error occoured during parse file " + fileFullPath); + e.printStackTrace(); + } + } + + public List includePaths() { + if (this.includePaths ==null) { + this.includePaths = buildIncludePath(); + } + return includePaths; + } + + private List buildIncludePath() { + includePaths = new ArrayList(); + for (String path : includeDirs) { + if (FileUtils.fileExists(path)) { + path = FileUtil.uniqFilePath(path); + if (!includePaths.contains(path)) + includePaths.add(path); + } + path = this.inputSrcPath + File.separator + path; + if (FileUtils.fileExists(path)) { + path = FileUtil.uniqFilePath(path); + if (!includePaths.contains(path)) + includePaths.add(path); + } + } + return includePaths; + } + + + public EntityRepo getEntityRepo() { + return this.entityRepo; + } + + + public abstract List supportedRelations(); + + public Set getExternalDependencies() { + return potentialExternalDependencies; + } + + public String getRelationMapping(String relation) { + return relation; + } + + /** + * Whether to resolve expression immediately during parse + * @return + */ + public boolean isEagerExpressionResolve(){ + return false; + } + + /** + * Call as Impl: + * implicit call (for example polymorphic in cpp) + * @return + */ + public boolean supportCallAsImpl(){return false;}; +} diff --git a/src/main/java/com/educoder/bridge/tmp/AliasEntity.java b/src/main/java/com/educoder/bridge/tmp/AliasEntity.java new file mode 100644 index 0000000..219fba5 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/AliasEntity.java @@ -0,0 +1,203 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.relations.IBindingResolver; + +import java.util.*; + +public class AliasEntity extends Entity { + private Entity referToEntity = new EmptyTypeEntity(); + private GenericName originName; + private List referPath = new ArrayList<>(); + private boolean deepResolve = false; + public AliasEntity() { + + } + public AliasEntity(GenericName simpleName, Entity parent, Integer id, GenericName originTypeName) { + super(simpleName, parent, id); + this.originName = originTypeName; + } + + public void inferLocalLevelEntities(IBindingResolver bindingResolver) { + if (!(referToEntity instanceof EmptyTypeEntity)) return; + Entity entity = bindingResolver.resolveName(this, originName, true); + while(entity instanceof AliasEntity) { + AliasEntity aliasEntity = (AliasEntity)entity; + if (this.referPath.contains(aliasEntity)) { + entity = null; + break; + } + this.referPath.add(aliasEntity); + entity = bindingResolver.resolveName(aliasEntity, aliasEntity.originName,true); + if (entity==null) break; + if (entity.equals(this)) { + entity = null; + break; + } + } + if (entity != null) + referToEntity = entity; + } + + public Collection getResolvedTypeParameters() { + if (!(referToEntity instanceof DecoratedEntity)) + return new ArrayList<>(); + DecoratedEntity origin = (DecoratedEntity) referToEntity; + return origin.getResolvedTypeParameters(); + } + + public Collection getResolvedAnnotations() { + if (!(referToEntity instanceof DecoratedEntity)) + return new ArrayList<>(); + DecoratedEntity origin = (DecoratedEntity) referToEntity; + return origin.getResolvedAnnotations(); + } + + public ArrayList getVars() { + if (!(referToEntity instanceof ContainerEntity)) + return new ArrayList<>(); + ContainerEntity origin = (ContainerEntity) referToEntity; + return origin.getVars(); + } + + public ArrayList getFunctions() { + if (!(referToEntity instanceof ContainerEntity)) + return new ArrayList<>(); + ContainerEntity origin = (ContainerEntity) referToEntity; + return origin.getFunctions(); + } + + protected FunctionEntity lookupFunctionLocally(GenericName functionName) { + if (!(referToEntity instanceof ContainerEntity)) + return null; + ContainerEntity origin = (ContainerEntity) referToEntity; + return origin.lookupFunctionLocally(functionName); + } + + public List lookupFunctionInVisibleScope(GenericName functionName) { + if (!(referToEntity instanceof ContainerEntity)) + return null; + ContainerEntity origin = (ContainerEntity) referToEntity; + return origin.lookupFunctionInVisibleScope(functionName); + } + + public Entity lookupVarsInVisibleScope(GenericName varName) { + if (!(referToEntity instanceof ContainerEntity)) + return null; + ContainerEntity origin = (ContainerEntity) referToEntity; + return origin.lookupVarInVisibleScope(varName); + } + + public Collection getResolvedMixins() { + if (!(referToEntity instanceof ContainerEntity)) + return new ArrayList<>(); + ContainerEntity origin = (ContainerEntity) referToEntity; + return origin.getResolvedMixins(); + } + + public Collection getInheritedTypes() { + if (referToEntity instanceof TypeEntity) + return ((TypeEntity) referToEntity).getInheritedTypes(); + return new ArrayList<>(); + } + + public Collection getImplementedTypes() { + if (referToEntity instanceof TypeEntity) + return ((TypeEntity) referToEntity).getImplementedTypes(); + return new ArrayList<>(); + } + + public TypeEntity getInheritedType() { + if (referToEntity instanceof TypeEntity) + return ((TypeEntity) referToEntity).getInheritedType(); + return null; + } + + public Collection getReturnTypes() { + if (!(referToEntity instanceof FunctionEntity)) + return new ArrayList<>(); + FunctionEntity origin = (FunctionEntity) referToEntity; + return origin.getReturnTypes(); + } + + public TypeEntity getType() { + return referToEntity.getType(); + } + + public Collection getParameters() { + if (!(referToEntity instanceof FunctionEntity)) + return new ArrayList<>(); + FunctionEntity origin = (FunctionEntity) referToEntity; + return origin.getParameters(); + } + + public Collection getThrowTypes() { + if (!(referToEntity instanceof FunctionEntity)) + return new ArrayList<>(); + FunctionEntity origin = (FunctionEntity) referToEntity; + return origin.getThrowTypes(); + } + + public Entity getOriginType() { + return referToEntity; + } + public Entity getReferToEntity() { + return referToEntity; + } + public void setReferToEntity(Entity referToEntity) { + this.referToEntity = referToEntity; + } + public Entity deepResolve() { + if (!deepResolve) return this; + Set searched = new HashSet<>(); + int i=0; + Entity current = this; + while(i<100) { //maximum 100 levels + if (searched.contains(current)) return current; //avoid a loop + if (!(current instanceof AliasEntity)) return current; + + searched.add(current); + Entity originalFile = current.getAncestorOfType(FileEntity.class); + current = ((AliasEntity)current).getReferToEntity(); + + if (current ==null) return this; + //if already out of current file, return current + if (!current.getAncestorOfType(FileEntity.class).equals(originalFile)) { + return current; + } + i++; + } + return current; + } + public boolean isDeepResolve() { + return deepResolve; + } + public void setDeepResolve(boolean deepResolve) { + this.deepResolve = deepResolve; + } + + +} diff --git a/src/main/java/com/educoder/bridge/tmp/AnnotationProcessor.java b/src/main/java/com/educoder/bridge/tmp/AnnotationProcessor.java new file mode 100644 index 0000000..cf40e49 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/AnnotationProcessor.java @@ -0,0 +1,121 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.java.context; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import org.antlr.v4.runtime.RuleContext; +import org.codehaus.plexus.util.StringUtils; + +import depends.entity.ContainerEntity; +import depends.entity.GenericName; +import depends.extractor.java.JavaParser.AnnotationContext; + +public class AnnotationProcessor { + + public AnnotationProcessor() { + } + + public void processAnnotationModifier(RuleContext ctx, @SuppressWarnings("rawtypes") Class rootClass, + String toAnnotationPath,ContainerEntity container) { + List list = new ArrayList<>() ; + list.add(container); + processAnnotationModifier(ctx, rootClass, + toAnnotationPath, list); + } + + public void processAnnotationModifier(RuleContext ctx, @SuppressWarnings("rawtypes") Class rootClass, + String toAnnotationPath, List containers) { + + while (true) { + if (ctx == null) + break; + if (ctx.getClass().equals(rootClass)) + break; + ctx = ctx.parent; + } + if (ctx == null) + return; + + + try { + Object r = ctx; + String[] paths = toAnnotationPath.split("\\."); + for (String path : paths) { + r = invokeMethod(r, path); + if (r == null) + return; + } + Collection contexts = new HashSet<>(); + mergeElements(contexts, r); + for (Object item : contexts) { + AnnotationContext annotation = (AnnotationContext) item; + String name = QualitiedNameContextHelper.getName(annotation.qualifiedName()); + containers.stream().forEach(container->((ContainerEntity)container).addAnnotation(GenericName.build(name))); + } + } catch (Exception e) { + return; + } + } + + + private void mergeElements(Collection collection, Object r) { + if (r instanceof Collection) { + for (Object item : (Collection) r) { + mergeElements(collection, item); + } + } else { + if (r instanceof AnnotationContext) + collection.add((AnnotationContext) r); + } + } + + private Object invokeMethod(Object r, String path) { + if (StringUtils.isEmpty(path)) + return null; + if (r instanceof Collection) { + Collection list = (Collection) r; + return list.stream().map(item -> invokeMethod(item, path)).filter(item -> item != null) + .collect(Collectors.toSet()); + } + try { + Method m = r.getClass().getMethod(path); + return m.invoke(r); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException e) { + return null; + } + } + + + + +} diff --git a/src/main/java/com/educoder/bridge/tmp/BindingResolver.java b/src/main/java/com/educoder/bridge/tmp/BindingResolver.java new file mode 100644 index 0000000..66134db --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/BindingResolver.java @@ -0,0 +1,297 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.relations; +import depends.entity.*; +import depends.entity.repo.BuiltInType; +import depends.entity.repo.EntityRepo; +import depends.extractor.AbstractLangProcessor; +import depends.extractor.UnsolvedBindings; +import depends.extractor.empty.EmptyBuiltInType; +import depends.importtypes.Import; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.management.ManagementFactory; +import java.util.*; + +public class BindingResolver implements IBindingResolver{ + + private BuiltInType buildInTypeManager = new EmptyBuiltInType(); + private ImportLookupStrategy importLookupStrategy; + private Set unsolvedSymbols = new HashSet<>(); + private EntityRepo repo; + + private boolean eagerExpressionResolve = false; + private boolean isCollectUnsolvedBindings = false; + private boolean isDuckTypingDeduce = true; + private static Logger logger = LoggerFactory.getLogger(IBindingResolver.class); + + public BindingResolver(AbstractLangProcessor langProcessor, + boolean isCollectUnsolvedBindings, boolean isDuckTypingDeduce) { + this.repo = langProcessor.getEntityRepo(); + this.importLookupStrategy = langProcessor.getImportLookupStrategy(); + this.buildInTypeManager = langProcessor.getBuiltInType(); + this.isCollectUnsolvedBindings = isCollectUnsolvedBindings; + this.isDuckTypingDeduce = isDuckTypingDeduce; + unsolvedSymbols= new HashSet<>(); + importLookupStrategy.setBindingResolver(this); + } + + + @Override + public Set resolveAllBindings(boolean isEagerExpressionResolve) { + System.out.println("Resolve type bindings...."); + if (logger.isInfoEnabled()) { + logger.info("Resolve type bindings..."); + } + resolveTypes(isEagerExpressionResolve); + System.out.println("Dependency analaysing...."); + if (logger.isInfoEnabled()) { + logger.info("Dependency analaysing..."); + } + logger.info("Heap Information: " + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()); + return unsolvedSymbols; + } + + + private void resolveTypes(boolean eagerExpressionResolve) { + this.eagerExpressionResolve = eagerExpressionResolve; + Iterator iterator = repo.sortedFileIterator(); + while(iterator.hasNext()) { + Entity entity= iterator.next(); + entity.inferEntities(this); + } + } + + + @Override + public Collection getImportedRelationEntities(List importedNames) { + return importLookupStrategy.getImportedRelationEntities(importedNames); + } + + @Override + public Collection getImportedTypes(List importedNames, FileEntity fileEntity) { + HashSet unsolved = new HashSet(); + Collection result = importLookupStrategy.getImportedTypes(importedNames,unsolved); + for (UnsolvedBindings item:unsolved) { + item.setFromEntity(fileEntity); + addUnsolvedBinding(item); + } + return result; + } + + private void addUnsolvedBinding(UnsolvedBindings item) { + if (!isCollectUnsolvedBindings) return; + this.unsolvedSymbols.add(item); + } + @Override + public Collection getImportedFiles(List importedNames) { + return importLookupStrategy.getImportedFiles(importedNames); + } + + + @Override + public TypeEntity inferTypeFromName(Entity fromEntity, GenericName rawName) { + Entity data = resolveName(fromEntity, rawName, true); + if (data == null) + return null; + return data.getType(); + } + + + @Override + public Entity resolveName(Entity fromEntity, GenericName rawName, boolean searchImport) { + if (rawName==null) return null; + Entity entity = resolveNameInternal(fromEntity,rawName,searchImport); + if (entity==null ) { + if (!this.buildInTypeManager.isBuiltInType(rawName.getName())) { + addUnsolvedBinding(new UnsolvedBindings(rawName.getName(), fromEntity)); + } + } + return entity; + } + + private Entity resolveNameInternal(Entity fromEntity, GenericName rawName, boolean searchImport) { + if (rawName==null || rawName.getName()==null) + return null; + if (buildInTypeManager.isBuiltInType(rawName.getName())) { + return TypeEntity.buildInType; + } + // qualified name will first try global name directly + if (rawName.startsWith(".")) { + rawName = rawName.substring(1); + if (repo.getEntity(rawName) != null) + return repo.getEntity(rawName); + } + Entity entity = null; + int indexCount = 0; + String name = rawName.getName(); + if (fromEntity==null) return null; + do { + entity = lookupEntity(fromEntity, name, searchImport); + if (entity!=null ) { + break; + } + if (importLookupStrategy.supportGlobalNameLookup()) { + if (repo.getEntity(name) != null) { + entity = repo.getEntity(name); + break; + } + } + + indexCount++; + if (name.contains(".")) + name = name.substring(0,name.lastIndexOf('.')); + else + break; + }while (true); + if (entity == null) { + return null; + } + String[] names = rawName.getName().split("\\."); + if (names.length == 0) + return null; + if (names.length == 1) { + return entity; + } + // then find the subsequent symbols + return findEntitySince(entity, names, names.length-indexCount); + } + + private Entity lookupEntity(Entity fromEntity, String name, boolean searchImport) { + if (name.equals("this") || name.equals("class") ) { + TypeEntity entityType = (TypeEntity) (fromEntity.getAncestorOfType(TypeEntity.class)); + return entityType; + } else if (name.equals("super")) { + TypeEntity parent = (TypeEntity) (fromEntity.getAncestorOfType(TypeEntity.class)); + if (parent != null) { + TypeEntity parentType = parent.getInheritedType(); + if (parentType!=null) + return parentType; + } + } + Entity inferData = findEntityUnderSamePackage(fromEntity, name); + if (inferData != null) { + return inferData; + } + if (searchImport) + inferData = lookupTypeInImported((FileEntity)(fromEntity.getAncestorOfType(FileEntity.class)), name); + return inferData; + } + /** + * To lookup entity in case of a.b.c from a; + * @param precendenceEntity + * @param names + * @param nameIndex + * @return + */ + private Entity findEntitySince(Entity precendenceEntity, String[] names, int nameIndex) { + if (nameIndex >= names.length) { + return precendenceEntity; + } + if (nameIndex == -1) { + System.err.println("No expected symbols: names"+Arrays.toString(names) +", index=" + nameIndex); + return null; + } + //If it is not an entity with types (not a type, var, function), fall back to itself + if (precendenceEntity.getType()==null) + return precendenceEntity; + + for (Entity child : precendenceEntity.getType().getChildren()) { + if (child.getRawName().getName().equals(names[nameIndex])) { + return findEntitySince(child, names, nameIndex + 1); + } + } + return null; + } + + @Override + public Entity lookupTypeInImported(FileEntity fileEntity, String name) { + if (fileEntity == null) + return null; + Entity type = importLookupStrategy.lookupImportedType(name, fileEntity); + if (type != null) + return type; + return null; + } + + + /** + * In Java/C++ etc, the same package names should take priority of resolving. + * the entity lookup is implemented recursively. + * @param fromEntity + * @param name + * @return + */ + private Entity findEntityUnderSamePackage(Entity fromEntity, String name) { + while (true) { + Entity entity = fromEntity.getByName(name, new HashSet<>()); + if (entity!=null) return entity; + fromEntity = fromEntity.getParent(); + if (fromEntity == null) + break; + } + return null; + } + + + @Override + public List calculateCandidateTypes(VarEntity fromEntity, List functionCalls) { + if (buildInTypeManager.isBuildInTypeMethods(functionCalls)) { + return new ArrayList<>(); + } + if (!isDuckTypingDeduce) + return new ArrayList<>(); + return searchTypesInRepo(fromEntity, functionCalls); + } + + private List searchTypesInRepo(VarEntity fromEntity, List functionCalls) { + List types = new ArrayList<>(); + Iterator iterator = repo.sortedFileIterator(); + while(iterator.hasNext()) { + Entity f = iterator.next(); + if (f instanceof FileEntity) { + for (TypeEntity type:((FileEntity)f).getDeclaredTypes()) { + FunctionMatcher functionMatcher = new FunctionMatcher(type.getFunctions()); + if (functionMatcher.containsAll(functionCalls)) { + types.add(type); + } + } + } + } + return types; + } + + @Override + public boolean isEagerExpressionResolve() { + return eagerExpressionResolve; + } + + @Override + public EntityRepo getRepo() { + return repo; + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/BuiltInType.java b/src/main/java/com/educoder/bridge/tmp/BuiltInType.java new file mode 100644 index 0000000..1771cc9 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/BuiltInType.java @@ -0,0 +1,110 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity.repo; + +import depends.entity.FunctionCall; +import depends.entity.TypeEntity; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public abstract class BuiltInType { + + public BuiltInType(){ + createBuiltInTypes(); + } + /** + * Init the build in types data + */ + private void createBuiltInTypes() { + for(String prefix: getBuiltInTypePrefix()) { + builtInPrefix.add(prefix); + } + for (String type: getBuiltInTypeName()) { + builtInType.add(type); + } + for (String method:getBuiltInMethods()) { + builtInMethod.add(method); + } + } + + protected String[] getBuiltInMethods(){return new String[]{};} + protected String[] getBuiltInTypeName(){return new String[]{};} + protected String[] getBuiltInTypePrefix() {return new String[]{};} + + private Set builtInType = new HashSet<>(); + private Set builtInPrefix = new HashSet<>(); + private Set builtInMethod = new HashSet<>(); + + /** + * To determine whether a type name is built-in + * @param typeName + * @return + */ + public boolean isBuiltInType(String typeName) { + return TypeEntity.buildInType.getRawName().uniqName().equals(typeName) || + builtInType.contains(typeName)|| + isBuiltInTypePrefix(typeName); + } + + /** + * To determine a typeName is a built-in type based on prefix. + * For example, in Java language, name start with java.*, javax.*, com.sun.* + * is build-in types + * @param typeName + * @return + */ + private boolean isBuiltInTypePrefix(String typeName) { + for (String prefix:builtInPrefix) { + if (typeName.startsWith(prefix)) return true; + } + return false; + } + + /** + * In some language, there are common methods, like in ruby, + * object_id is a method for all type + * @param name + * @return + */ + public boolean isBuildInMethod(String name) { + return builtInMethod.contains(name); + } + + /** + * Used by duck typing deduce feature: + * - if all calls of a type are build in method, + * then no duck typing is deduced + * Refer to Python built-in type for example + * + * @param functionCalls + * @return + */ + public boolean isBuildInTypeMethods(List functionCalls) { + return false; + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/CandidateTypes.java b/src/main/java/com/educoder/bridge/tmp/CandidateTypes.java new file mode 100644 index 0000000..04d669b --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/CandidateTypes.java @@ -0,0 +1,311 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.relations.IBindingResolver; +import depends.relations.Relation; + +import java.util.*; + +public class CandidateTypes extends TypeEntity { + private List candidateTypes; + + public CandidateTypes() { + candidateTypes = new ArrayList<>(); + } + public CandidateTypes(List candidateTypes, Integer id) { + super(GenericName.build("candidateTypes"), null, id); + this.candidateTypes = candidateTypes; + } + + public List getCandidateTypes() { + return candidateTypes; + } + + @Override + public Collection getInheritedTypes() { + List result = new ArrayList<>(); + for (TypeEntity type:candidateTypes) { + result.addAll(type.getInheritedTypes()); + } + return result; + } + + @Override + public Collection getImplementedTypes() { + List result = new ArrayList<>(); + for (TypeEntity type:candidateTypes) { + result.addAll(type.getImplementedTypes()); + } + return result; + } + + @Override + public ArrayList getFunctions() { + ArrayList result = new ArrayList<>(); + for (TypeEntity type:candidateTypes) { + result.addAll(type.getFunctions()); + } + return result; + } + + @Override + public TypeEntity getInheritedType() { + return inheritedType; + } + @Override + public List lookupFunctionInVisibleScope(GenericName functionName) { + List functions = new ArrayList<>(); + for (TypeEntity type:candidateTypes) { + List f = type.lookupFunctionInVisibleScope(functionName); + if (f!=null) { + functions.addAll(f); + } + } + if (functions.size()==0) + return null; + return functions; + } + + @Override + public Entity lookupVarInVisibleScope(GenericName varName) { + for (TypeEntity type:candidateTypes) { + Entity v = type.lookupVarInVisibleScope(varName); + if (v!=null) return v; + } + return null; + } + + + @Override + public VarEntity lookupVarLocally(String varName) { + for (TypeEntity type:candidateTypes) { + VarEntity v = type.lookupVarLocally(varName); + if (v!=null) return v; + } + return null; + } + + @Override + public TypeEntity getType() { + if (candidateTypes.size()>0) return candidateTypes.get(0); + return null; + } + + @Override + public void inferLocalLevelEntities(IBindingResolver bindingResolver) { + System.err.println("error: inferLocalLevelEntities should not been invoked"); + super.inferLocalLevelEntities(bindingResolver); + } + + @Override + public void addImplements(GenericName typeName) { + System.err.println("error: addImplements should not been invoked"); + super.addImplements(typeName); + } + + @Override + public void addExtends(GenericName typeName) { + System.err.println("error: addExtends should not been invoked"); + super.addExtends(typeName); + } + + @Override + public void addVar(VarEntity var) { + System.err.println("error: addVar should not been invoked"); + super.addVar(var); + } + + @Override + public ArrayList getVars() { + System.err.println("error: getVars should not been invoked"); + return super.getVars(); + } + + @Override + public void addFunction(FunctionEntity functionEntity) { + System.err.println("error: addFunction should not been invoked"); + super.addFunction(functionEntity); + } + + @Override + public HashMap expressions() { + System.err.println("error: expressions should not been invoked"); + return super.expressions(); + } + + @Override + public void addExpression(Object key, Expression expression) { + System.err.println("error: addExpression should not been invoked"); + super.addExpression(key, expression); + } + + public void resolveExpressions(IBindingResolver bindingResolver) { + System.err.println("error: resolveExpressions should not been invoked"); + super.resolveExpressions(bindingResolver); + } + + + @Override + public void addMixin(GenericName moduleName) { + System.err.println("error: addMixin should not been invoked"); + super.addMixin(moduleName); + } + + @Override + public Collection getResolvedMixins() { + System.err.println("error: getResolvedMixins should not been invoked"); + return super.getResolvedMixins(); + } + + @Override + public void addAnnotation(GenericName name) { + System.err.println("error: addAnnotation should not been invoked"); + super.addAnnotation(name); + } + + + @Override + public Collection getResolvedTypeParameters() { + System.err.println("error: getResolvedTypeParameters should not been invoked"); + return super.getResolvedTypeParameters(); + } + + @Override + public Collection getResolvedAnnotations() { + System.err.println("error: getResolvedAnnotations should not been invoked"); + return super.getResolvedAnnotations(); + } + + @Override + public boolean isGenericTypeParameter(GenericName rawType) { + System.err.println("error: isGenericTypeParameter should not been invoked"); + return super.isGenericTypeParameter(rawType); + } + + @Override + protected Collection identiferToEntities(IBindingResolver bindingResolver, Collection identifiers) { + System.err.println("error: identiferToTypes should not been invoked"); + return super.identiferToEntities(bindingResolver, identifiers); + } + + @Override + public GenericName getRawName() { + return super.getRawName(); + } + + @Override + public Integer getId() { + return super.getId(); + } + + @Override + public void addRelation(Relation relation) { + System.err.println("error: addRelation should not been invoked"); + super.addRelation(relation); + } + + @Override + public ArrayList getRelations() { + System.err.println("error: getRelations should not been invoked"); + return super.getRelations(); + } + + @Override + public void addChild(Entity child) { + System.err.println("error: addChild should not been invoked"); + super.addChild(child); + } + + @Override + public Entity getParent() { + return null; + } + + @Override + public void setParent(Entity parent) { + System.err.println("error: setParent should not been invoked"); + super.setParent(parent); + } + + @Override + public Collection getChildren() { + List children = new ArrayList<>(); + for (Entity entity:this.candidateTypes) { + children.addAll(entity.getChildren()); + } + return children; + } + + @Override + public void setQualifiedName(String qualifiedName) { + System.err.println("error: setQualifiedName should not been invoked"); + super.setQualifiedName(qualifiedName); + } + + @Override + public void setRawName(GenericName rawName) { + System.err.println("error: setRawName should not been invoked"); + super.setRawName(rawName); + } + + @Override + public String getQualifiedName(boolean overrideFileWithPackage) { + System.err.println("error: getQualifiedName should not been invoked"); + return super.getQualifiedName(overrideFileWithPackage); + } + + + @Override + public Entity getAncestorOfType(@SuppressWarnings("rawtypes") Class classType) { + return null; + } + + @Override + public void inferEntities(IBindingResolver bindingResolver) { + System.err.println("error: inferEntities should not been invoked"); + super.inferEntities(bindingResolver); + } + + @Override + public String getDisplayName() { + System.err.println("error: getDisplayName should not been invoked"); + return super.getDisplayName(); + } + @Override + public Entity getByName(String name, HashSet searched) { + Entity entity = super.getByName(name, searched); + if (entity!=null) return entity; + for (TypeEntity type:getCandidateTypes()) { + if (searched.contains(type)) continue; + Entity e = type.getByName(name, searched); + if (e !=null) return e; + } + return null; + } + + + +} diff --git a/src/main/java/com/educoder/bridge/tmp/CdtCppFileParser.java b/src/main/java/com/educoder/bridge/tmp/CdtCppFileParser.java new file mode 100644 index 0000000..42360f6 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/CdtCppFileParser.java @@ -0,0 +1,86 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.cpp.cdt; + +import depends.entity.repo.EntityRepo; +import depends.extractor.cpp.CppFileParser; +import depends.extractor.cpp.MacroRepo; +import depends.relations.IBindingResolver; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class CdtCppFileParser extends CppFileParser { + private PreprocessorHandler preprocessorHandler ; + private IBindingResolver bindingResolver; + private MacroRepo macroRepo; + + public CdtCppFileParser(EntityRepo entityRepo, PreprocessorHandler preprocessorHandler, IBindingResolver bindingResolver, MacroRepo macroRepo) { + super(entityRepo); + this.preprocessorHandler = preprocessorHandler; + this.bindingResolver = bindingResolver; + this.macroRepo= macroRepo; + } + @Override + protected void parseFile(String fileFullPath) throws IOException { + Map macroMap = new HashMap<>(macroRepo.getDefaultMap()); + parse(fileFullPath,macroMap); + } + + /** + * + * @param isInScope whether the parse is invoked by project file or an 'indirect' included file + * @return + */ + public void parse(String fileFullPath,Map macroMap) throws IOException { + CppVisitor bridge = new CppVisitor(fileFullPath, entityRepo, preprocessorHandler, bindingResolver); + IASTTranslationUnit tu = (new CDTParser(preprocessorHandler.getIncludePaths())).parse(fileFullPath,macroMap); + boolean containsIncludes = false; + for (String incl:preprocessorHandler.getDirectIncludedFiles(tu.getAllPreprocessorStatements(),fileFullPath)) { + CdtCppFileParser importedParser = new CdtCppFileParser(entityRepo, preprocessorHandler, bindingResolver,macroRepo); + importedParser.parse(incl); + Map macros = macroRepo.get(incl); + if (macros!=null) + macroMap.putAll(macros); + containsIncludes = true; + } + if (containsIncludes) { + tu = (new CDTParser(preprocessorHandler.getIncludePaths())).parse(fileFullPath,macroMap); + } + macroRepo.putMacros(fileFullPath,macroMap,tu.getMacroDefinitions()); + tu.accept(bridge); + return; + } + + @Override + protected boolean isPhase2Files(String fileFullPath) { + if (fileFullPath.endsWith(".h") || fileFullPath.endsWith(".hh") || fileFullPath.endsWith(".hpp") + || fileFullPath.endsWith(".hxx")) + return true; + return false; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/ContainerEntity.java b/src/main/java/com/educoder/bridge/tmp/ContainerEntity.java new file mode 100644 index 0000000..b299763 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/ContainerEntity.java @@ -0,0 +1,467 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.entity.repo.EntityRepo; +import depends.relations.IBindingResolver; +import depends.relations.Relation; +import multilang.depends.util.file.TemporaryFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.lang.ref.WeakReference; +import java.util.*; + +/** + * ContainerEntity for example file, class, method, etc. they could contain + * vars, functions, ecpressions, type parameters, etc. + */ +public abstract class ContainerEntity extends DecoratedEntity { + private static final Logger logger = LoggerFactory.getLogger(ContainerEntity.class); + + private ArrayList vars; + private ArrayList functions; + WeakReference> expressionWeakReference; + private ArrayList expressionList; + private int expressionCount = 0; + private Collection mixins; + private Collection resolvedMixins; + + private ArrayList vars() { + if (vars==null) + vars = new ArrayList<>(); + return this.vars; + } + + private Collection mixins() { + if (mixins==null) + mixins = new ArrayList<>(); + return this.mixins; + } + + private ArrayList functions() { + if (functions==null) + functions = new ArrayList<>(); + return this.functions; + } + + public ContainerEntity() { + } + + public ContainerEntity(GenericName rawName, Entity parent, Integer id) { + super(rawName, parent, id); + } + + public void addVar(VarEntity var) { + if (logger.isDebugEnabled()) { + logger.debug("var found: " + var.getRawName() + ":" + var.getRawType()); + } + this.vars().add(var); + } + + public ArrayList getVars() { + if (vars==null) + return new ArrayList<>(); + return this.vars(); + } + + public void addFunction(FunctionEntity functionEntity) { + this.functions().add(functionEntity); + } + + public ArrayList getFunctions() { + if (functions==null) + return new ArrayList<>(); + return this.functions; + } + + public HashMap expressions() { + if (expressionWeakReference==null) + expressionWeakReference= new WeakReference>(new HashMap<>()); + HashMap r = expressionWeakReference.get(); + if (r==null) return new HashMap<>(); + return r; + } + + public void addExpression(Object key, Expression expression) { + expressions().put(key, expression); + expressionList().add(expression); + expressionCount = expressionList.size(); + } + + public boolean containsExpression(Object key) { + return expressions().containsKey(key); + } + /** + * For all data in the class, infer their types. Should be override in + * sub-classes + */ + public void inferLocalLevelEntities(IBindingResolver bindingResolver) { + super.inferLocalLevelEntities(bindingResolver); + for (VarEntity var : this.vars()) { + if (var.getParent()!=this) { + var.inferLocalLevelEntities(bindingResolver); + } + } + for (FunctionEntity func : this.getFunctions()) { + if (func.getParent()!=this) { + func.inferLocalLevelEntities(bindingResolver); + } + } + if (bindingResolver.isEagerExpressionResolve()) { + reloadExpression(bindingResolver.getRepo()); + resolveExpressions(bindingResolver); + cacheExpressions(); + } + resolvedMixins = identiferToContainerEntity(bindingResolver, getMixins()); + } + + private Collection getMixins() { + if (mixins==null) + return new ArrayList<>(); + return mixins; + } + + private Collection identiferToContainerEntity(IBindingResolver bindingResolver, Collection identifiers) { + if (identifiers.size()==0) return null; + ArrayList r = new ArrayList<>(); + for (GenericName identifier : identifiers) { + Entity entity = bindingResolver.resolveName(this, identifier, true); + if (entity == null) { + continue; + } + if (entity instanceof ContainerEntity) + r.add((ContainerEntity) entity); + } + return r; + } + + /** + * Resolve all expression's type + * + * @param bindingResolver + */ + public void resolveExpressions(IBindingResolver bindingResolver) { + if (this instanceof FunctionEntity) { + ((FunctionEntity)this).linkReturnToLastExpression(); + } + + if (expressionList==null) return; + if(expressionList.size()>10000) return; + + + for (Expression expression : expressionList) { + // 1. if expression's type existed, break; + if (expression.getType() != null) + continue; + if (expression.isDot()) { // wait for previous + continue; + } + if (expression.getRawType() == null && expression.getIdentifier() == null) + continue; + + // 2. if expression's rawType existed, directly infer type by rawType + // if expression's rawType does not existed, infer type based on identifiers + if (expression.getRawType() != null) { + expression.setType(bindingResolver.inferTypeFromName(this, expression.getRawType()), null, bindingResolver); + if (expression.getType() != null) { + continue; + } + } + if (expression.getIdentifier() != null) { + Entity entity = bindingResolver.resolveName(this, expression.getIdentifier(), true); + String composedName = expression.getIdentifier().toString(); + Expression theExpr = expression; + if (entity==null) { + while(theExpr.getParent()!=null && theExpr.getParent().isDot()) { + theExpr = theExpr.getParent(); + if (theExpr.getIdentifier()==null) break; + composedName = composedName + "." + theExpr.getIdentifier().toString(); + entity = bindingResolver.resolveName(this, GenericName.build(composedName), true); + if (entity!=null) + break; + } + } + if (entity != null) { + expression.setType(entity.getType(), entity, bindingResolver); + continue; + } + if (expression.isCall()) { + List funcs = this.lookupFunctionInVisibleScope(expression.getIdentifier()); + if (funcs != null) { + for (Entity func:funcs) { + expression.setType(func.getType(), func, bindingResolver); + } + } + } else { + + Entity varEntity = this.lookupVarInVisibleScope(expression.getIdentifier()); + if (varEntity != null) { + expression.setType(varEntity.getType(), varEntity, bindingResolver); + } + } + } + } + } + + public void cacheChildExpressions() { + cacheExpressions(); + for (Entity child:getChildren()) { + if (child instanceof ContainerEntity) { + ((ContainerEntity)child).cacheChildExpressions(); + } + } + } + + + public void cacheExpressions() { + if (expressionWeakReference==null) return; + if (expressionList==null) return; + this.expressions().clear(); + this.expressionWeakReference.clear(); + cacheExpressionListToFile(); + this.expressionList.clear(); + this.expressionList=null; + this.expressionList = new ArrayList<>(); + } + + public void clearExpressions() { + if (expressionWeakReference==null) return; + if (expressionList==null) return; + this.expressions().clear(); + this.expressionWeakReference.clear(); + this.expressionList.clear(); + this.expressionList=null; + this.expressionList = new ArrayList<>(); + this.expressionUseList = null; + } + + private void cacheExpressionListToFile() { + if (expressionCount ==0) return; + try { + FileOutputStream fileOut = new FileOutputStream(TemporaryFile.getInstance().exprPath(this.id)); + ObjectOutputStream out = new ObjectOutputStream(fileOut); + out.writeObject(this.expressionList); + out.close(); + fileOut.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @SuppressWarnings("unchecked") + public void reloadExpression(EntityRepo repo) { + if (expressionCount ==0) return; + try + { + FileInputStream fileIn = new FileInputStream(TemporaryFile.getInstance().exprPath(this.id)); + ObjectInputStream in = new ObjectInputStream(fileIn); + expressionList = (ArrayList) in.readObject(); + if (expressionList==null) expressionList = new ArrayList<>(); + for (Expression expr:expressionList) { + expr.reload(repo,expressionList); + } + in.close(); + fileIn.close(); + }catch(IOException | ClassNotFoundException i) + { + return; + } + } + + + public List expressionList() { + if (expressionList==null) + expressionList = new ArrayList<>(); + return expressionList; + } + + public boolean containsExpression() { + return expressions().size() > 0; + } + + /** + * The entry point of lookup functions. It will treat multi-declare entities and + * normal entity differently. - for multiDeclare entity, it means to lookup all + * entities - for normal entity, it means to lookup entities from current scope + * still root + * + * @param functionName + * @return + */ + public List lookupFunctionInVisibleScope(GenericName functionName) { + List functions = new ArrayList<>(); + if (this.getMutliDeclare() != null) { + for (Entity fromEntity : this.getMutliDeclare().getEntities()) { + Entity f = lookupFunctionBottomUpTillTopContainer(functionName, fromEntity); + if (f != null) { + functions.add(f); + return functions; + } + } + } else { + ContainerEntity fromEntity = this; + Entity f = lookupFunctionBottomUpTillTopContainer(functionName, fromEntity); + if (f != null) { + functions.add(f); + return functions; + } + } + return null; + } + + /** + * lookup function bottom up till the most outside container + * + * @param functionName + * @param fromEntity + * @return + */ + private Entity lookupFunctionBottomUpTillTopContainer(GenericName functionName, Entity fromEntity) { + while (fromEntity != null) { + if (fromEntity instanceof ContainerEntity) { + FunctionEntity func = ((ContainerEntity) fromEntity).lookupFunctionLocally(functionName); + if (func != null) + return func; + } + for (Entity child:this.getChildren()) { + if (child instanceof AliasEntity) { + if (child.getRawName().equals(functionName)) + return child; + } + } + fromEntity = (ContainerEntity) this.getAncestorOfType(ContainerEntity.class); + } + return null; + } + + /** + * lookup function in local entity. It could be override such as the type entity + * (it should also lookup the inherit/implemented types + * + * @param functionName + * @return + */ + public FunctionEntity lookupFunctionLocally(GenericName functionName) { + for (FunctionEntity func : getFunctions()) { + if (func.getRawName().equals(functionName)) + return func; + } + return null; + } + + /** + * The entry point of lookup var. It will treat multi-declare entities and + * normal entity differently. - for multiDeclare entity, it means to lookup all + * entities - for normal entity, it means to lookup entities from current scope + * still root + * + * @param varName + * @return + */ + public Entity lookupVarInVisibleScope(GenericName varName) { + ContainerEntity fromEntity = this; + return lookupVarBottomUpTillTopContainer(varName, fromEntity); + } + + /** + * To found the var. + * + * @param fromEntity + * @param varName + * @return + */ + private Entity lookupVarBottomUpTillTopContainer(GenericName varName, ContainerEntity fromEntity) { + while (fromEntity != null) { + if (fromEntity instanceof ContainerEntity) { + VarEntity var = ((ContainerEntity) fromEntity).lookupVarLocally(varName); + if (var != null) + return var; + } + for (Entity child:this.getChildren()) { + if (child instanceof AliasEntity) { + if (child.getRawName().equals(varName)) + return child; + } + } + fromEntity = (ContainerEntity) this.getAncestorOfType(ContainerEntity.class); + } + return null; + } + + public VarEntity lookupVarLocally(GenericName varName) { + for (VarEntity var : getVars()) { + if (var.getRawName().equals(varName)) + return var; + } + return null; + } + + public VarEntity lookupVarLocally(String varName) { + return this.lookupVarLocally(GenericName.build(varName)); + } + + public void addMixin(GenericName moduleName) { + mixins().add(moduleName); + } + + public Collection getResolvedMixins() { + if (resolvedMixins==null) return new ArrayList<>(); + return resolvedMixins; + } + + HashMap> expressionUseList = null; + public void addRelation(Expression expression, Relation relation) { + String key = relation.getEntity().qualifiedName+relation.getType(); + if (this.expressionUseList==null) + expressionUseList = new HashMap<>(); + if (expressionUseList.containsKey(key)){ + Set expressions = expressionUseList.get(key); + for (Expression expr:expressions){ + if (linkedExpr(expr,expression)) return; + } + }else{ + expressionUseList.put(key,new HashSet<>()); + } + + expressionUseList.get(key).add(expression); + super.addRelation(relation); + } + + private boolean linkedExpr(Expression a, Expression b) { + Expression parent = a.getParent(); + while(parent!=null){ + if (parent==b) return true; + parent = parent.getParent(); + } + parent = b.getParent(); + while(parent!=null){ + if (parent==a) return true; + parent = parent.getParent(); + } + return false; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/CppBuiltInType.java b/src/main/java/com/educoder/bridge/tmp/CppBuiltInType.java new file mode 100644 index 0000000..e94528f --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/CppBuiltInType.java @@ -0,0 +1,66 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.cpp; + +import depends.entity.repo.BuiltInType; + +public class CppBuiltInType extends BuiltInType { + + @Override + protected String[] getBuiltInTypeName() { + return new String[] { "alignas", "alignof", "asm", "auto", "bool", "break", "case", "catch", "char", + "char16_t", "char32_t", "class", "const", "constexpr", "const_cast", "continue", "decltype", + "default", "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", "export", "extern", + "false", "final", "float", "for", "friend", "goto", "if", "inline", "int", "long", "mutable", + "namespace", "new", "noexcept", "nullptr", "operator", "override", "private", "protected", "public", + "register", "reinterpret_cast", "return", "short", "signed", "sizeof", "static", "static_assert", + "static_cast", "struct", "switch", "template", "this", "thread_local", "throw", "true", "try", + "typedef", "typeid", "typename", "union", "unsigned", "using", "virtual", "void", "volatile", + "wchar_t", "while", "", + "__cplusplus","_cpp_aggregate_bases","__cpp_aggregate_nsdmi","__cpp_alias_templates","__cpp_aligned_new", + "__cpp_attributes","__cpp_binary_literals","__cpp_capture_star_this","__cpp_constexpr","__cpp_decltype", + "__cpp_decltype_auto","__cpp_deduction_guides","__cpp_delegating_constructors", + "__cpp_enumerator_attributes","__cpp_explicit_bool","__cpp_fold_expressions","__cpp_generic_lambdas", + "__cpp_guaranteed_copy_elision","__cpp_hex_float","__cpp_if_constexpr","__cpp_inheriting_constructors", + "__cpp_init_captures","__cpp_initializer_lists","__cpp_inline_variables","__cpp_lambdas", + "__cpp_namespace_attributes","__cpp_noexcept_function_type","__cpp_nontype_template_args", + "__cpp_nontype_template_parameter_auto","__cpp_nontype_template_parameter_class","__cpp_nsdmi" + + "","__cpp_range_based_for","__cpp_raw_strings","__cpp_ref_qualifiers","__cpp_return_type_deduction" + ,"__cpp_rvalue_references","__cpp_sized_deallocation","__cpp_static_assert","__cpp_structured_bindings", + "__cpp_template_template_args","__cpp_threadsafe_static_init","__cpp_unicode_characters","__cpp_unicode_literals", + "__cpp_user_defined_literals","__cpp_variable_templates","__cpp_variadic_templates","__cpp_variadic_using", + "__DATE__","__FILE__","__LINE__","__STDC__","__STDC_ANALYZABLE__","__STDC_HOSTED__","__STDC_IEC_559__", + "__STDC_IEC_559_COMPLEX__","__STDC_ISO_10646__","__STDC_LIB_EXT1__","__STDC_MB_MIGHT_NEQ_WC__", + "__STDC_NO_ATOMICS__","__STDC_NO_COMPLEX__","__STDC_NO_THREADS__","__STDC_NO_VLA__", + "__STDCPP_DEFAULT_NEW_ALIGNMENT__","__STDCPP_STRICT_POINTER_SAFETY__","__STDCPP_THREADS__", + "__STDC_UTF_16__","__STDC_UTF_32__","__STDC_VERSION__","__TIME__" + }; + } + + @Override + protected String[] getBuiltInTypePrefix() { + return new String[] {"__"}; + } +} \ No newline at end of file diff --git a/src/main/java/com/educoder/bridge/tmp/CppImportLookupStrategy.java b/src/main/java/com/educoder/bridge/tmp/CppImportLookupStrategy.java new file mode 100644 index 0000000..b723455 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/CppImportLookupStrategy.java @@ -0,0 +1,133 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.cpp; + +import depends.entity.Entity; +import depends.entity.FileEntity; +import depends.entity.GenericName; +import depends.entity.repo.EntityRepo; +import depends.extractor.UnsolvedBindings; +import depends.importtypes.FileImport; +import depends.importtypes.Import; +import depends.relations.ImportLookupStrategy; + +import java.util.*; + +public class CppImportLookupStrategy extends ImportLookupStrategy { + + + public CppImportLookupStrategy(EntityRepo repo){ + super(repo); + } + @Override + public Entity lookupImportedType(String name, FileEntity fileEntity) { + String importedString = fileEntity.importedSuffixMatch(name); + if (importedString!=null) { + Entity r = repo.getEntity(importedString); + if (r!=null) return r; + } + + HashSet fileSet = getIncludedFiles(fileEntity); + + for (Integer file:fileSet) { + Entity importedItem = repo.getEntity(file); + if (importedItem instanceof FileEntity) { + FileEntity importedFile = (FileEntity) repo.getEntity(file); + if (importedFile==null) continue; + Entity entity = bindingResolver.resolveName(importedFile,GenericName.build(name), false); + if (entity!=null) return entity; + Collection namespaces = fileEntity.getImportedTypes(); + for (Entity ns:namespaces) { + String nameWithPrefix = ns.getQualifiedName() + "." + name; + entity = bindingResolver.resolveName(importedFile,GenericName.build(nameWithPrefix), false); + if (entity!=null) return entity; + } + } + } + return null; + } + + private Map > includedFiles = new HashMap<>(); + private HashSet getIncludedFiles(FileEntity fileEntity) { + + if (includedFiles.containsKey(fileEntity.getId())) { + return includedFiles.get(fileEntity.getId()); + } + HashSet fileSet = new HashSet<>(); + foundIncludedFiles(fileSet, fileEntity.getImportedFiles()); + includedFiles.put(fileEntity.getId(), fileSet); + return fileSet; + } + + private void foundIncludedFiles(HashSet fileSet, Collection importedFiles) { + for (Entity file:importedFiles) { + if (file==null ) continue; + if (!(file instanceof FileEntity)) continue; + if (fileSet.contains(file.getId())) continue; + fileSet.add(file.getId()); + foundIncludedFiles(fileSet,((FileEntity)file).getImportedFiles()); + } + } + + + @Override + public List getImportedRelationEntities(List importedList) { + ArrayList result = new ArrayList<>(); + for (Import importedItem:importedList) { + if (importedItem instanceof FileImport) { + Entity imported = repo.getEntity(importedItem.getContent()); + if (imported==null) continue; + result.add(imported); + } + } + return result; + } + + @Override + public List getImportedTypes(List importedList, Set unsolvedBindings) { + ArrayList result = new ArrayList<>(); + for (Import importedItem:importedList) { + if (!(importedItem instanceof FileImport)) { + Entity imported = repo.getEntity(importedItem.getContent()); + if (imported==null) { + unsolvedBindings.add(new UnsolvedBindings(importedItem.getContent(),null)); + continue; + } + result.add(imported); + } + } + return result; + } + + @Override + public List getImportedFiles(List importedList) { + return getImportedRelationEntities(importedList); + } + + @Override + public boolean supportGlobalNameLookup() { + return false; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/CppVisitor.java b/src/main/java/com/educoder/bridge/tmp/CppVisitor.java new file mode 100644 index 0000000..e291c7f --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/CppVisitor.java @@ -0,0 +1,338 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.cpp.cdt; + +import depends.entity.*; +import depends.entity.repo.EntityRepo; +import depends.entity.repo.IdGenerator; +import depends.extractor.cpp.CppHandlerContext; +import depends.importtypes.ExactMatchImport; +import depends.importtypes.FileImport; +import depends.importtypes.PackageWildCardImport; +import depends.relations.IBindingResolver; +import org.codehaus.plexus.util.StringUtils; +import org.eclipse.cdt.core.dom.ast.*; +import org.eclipse.cdt.core.dom.ast.IASTEnumerationSpecifier.IASTEnumerator; +import org.eclipse.cdt.core.dom.ast.cpp.*; +import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTCompositeTypeSpecifier.ICPPASTBaseSpecifier; +import org.eclipse.cdt.internal.core.dom.parser.cpp.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +public class CppVisitor extends ASTVisitor { + private static final Logger logger = LoggerFactory.getLogger(CppVisitor.class); + private CppHandlerContext context; + private IdGenerator idGenerator; + private PreprocessorHandler preprocessorHandler; + IBindingResolver bindingResolver; + private ExpressionUsage expressionUsage; + HashSet file; + public CppVisitor(String fileFullPath, EntityRepo entityRepo, PreprocessorHandler preprocessorHandler, IBindingResolver bindingResolver) { + super(true); + this.shouldVisitAmbiguousNodes = true; + this.shouldVisitImplicitNames = true; + this.includeInactiveNodes = true; + this.context = new CppHandlerContext(entityRepo, bindingResolver); + idGenerator = entityRepo; + this.bindingResolver = bindingResolver; + this.preprocessorHandler = preprocessorHandler; + expressionUsage = new ExpressionUsage(context,entityRepo); + file = new HashSet<>(); + context.startFile(fileFullPath); + file.add(this.context.currentFile().getQualifiedName()); + } + + @Override + public int visit(IASTTranslationUnit tu) { + for (String incl:preprocessorHandler.getDirectIncludedFiles(tu.getAllPreprocessorStatements(),context.currentFile().getQualifiedName())) { + context.foundNewImport(new FileImport(incl)); + } + MacroExtractor macroExtractor = new MacroExtractor(tu.getAllPreprocessorStatements(),context.currentFile().getQualifiedName()); + macroExtractor.extract(context); + + for (IASTNode child:tu.getChildren()) { + if (notLocalFile(child)) continue; + child.accept(this); + } + return ASTVisitor.PROCESS_SKIP; + } + + + @Override + public int visit(IASTProblem problem) { + if (notLocalFile(problem)) return ASTVisitor.PROCESS_SKIP; + System.out.println("warning: parse error " + problem.getOriginalNode().getRawSignature() + problem.getMessageWithLocation()); + return super.visit(problem); + } + + private boolean notLocalFile(IASTNode node) { + if (file.contains(node.getFileLocation().getFileName())) { + return false; + } + return true; + } + + // PACKAGES + @Override + public int visit(ICPPASTNamespaceDefinition namespaceDefinition) { + if (notLocalFile(namespaceDefinition)) return ASTVisitor.PROCESS_SKIP; + String ns = namespaceDefinition.getName().toString().replace("::", "."); + logger.trace("enter ICPPASTNamespaceDefinition " + ns); + Entity pkg = context.foundNamespace(ns,namespaceDefinition.getFileLocation().getStartingLineNumber()); + context.foundNewImport(new PackageWildCardImport(ns)); + return super.visit(namespaceDefinition); + } + + @Override + public int leave(ICPPASTNamespaceDefinition namespaceDefinition) { + if (notLocalFile(namespaceDefinition)) return ASTVisitor.PROCESS_SKIP; + context.exitLastedEntity(); + return super.leave(namespaceDefinition); + } + + // Types + @Override + public int visit(IASTDeclSpecifier declSpec) { + if (notLocalFile(declSpec)) return ASTVisitor.PROCESS_SKIP; + logger.trace("enter IASTDeclSpecifier " + declSpec.getClass().getSimpleName()); + if (declSpec instanceof IASTCompositeTypeSpecifier) { + IASTCompositeTypeSpecifier type = (IASTCompositeTypeSpecifier)declSpec; + String name = ASTStringUtilExt.getName(type); + List param = ASTStringUtilExt.getTemplateParameters(type); + TypeEntity typeEntity = context.foundNewType(name, type.getFileLocation().getStartingLineNumber()); + if (declSpec instanceof ICPPASTCompositeTypeSpecifier) { + ICPPASTBaseSpecifier[] baseSpecififers = ((ICPPASTCompositeTypeSpecifier)declSpec).getBaseSpecifiers(); + for (ICPPASTBaseSpecifier baseSpecififer:baseSpecififers) { + String extendName = ASTStringUtilExt.getName(baseSpecififer.getNameSpecifier()); + context.foundExtends(extendName); + } + } + } + else if (declSpec instanceof IASTEnumerationSpecifier) { + context.foundNewType(ASTStringUtilExt.getName(declSpec), declSpec.getFileLocation().getStartingLineNumber()); + }else { + //we do not care other types + } + return super.visit(declSpec); + } + + @Override + public int leave(IASTDeclSpecifier declSpec) { + if (notLocalFile(declSpec)) return ASTVisitor.PROCESS_SKIP; + if (declSpec instanceof IASTCompositeTypeSpecifier) { + context.exitLastedEntity(); + } + else if (declSpec instanceof IASTEnumerationSpecifier) { + context.exitLastedEntity(); + }else { + //we do not care other types + } + return super.leave(declSpec); + } + + //Function or Methods + @Override + public int visit(IASTDeclarator declarator) { + if (notLocalFile(declarator)) return ASTVisitor.PROCESS_SKIP; + logger.trace("enter IASTDeclarator " + declarator.getClass().getSimpleName()); + if (declarator instanceof IASTFunctionDeclarator){ + GenericName returnType = null; + if ( declarator.getParent() instanceof IASTSimpleDeclaration) { + IASTSimpleDeclaration decl = (IASTSimpleDeclaration)(declarator.getParent()); + returnType = buildGenericNameFromDeclSpecifier(decl.getDeclSpecifier()); + String rawName = ASTStringUtilExt.getName(declarator); + List namedEntity = context.currentFile().lookupFunctionInVisibleScope(GenericName.build(rawName)); + if (namedEntity!=null) { + rawName = namedEntity.get(0).getQualifiedName(); + } + returnType = reMapIfConstructDeconstruct(rawName,returnType); + context.foundMethodDeclaratorProto(rawName, returnType,decl.getFileLocation().getStartingLineNumber()); + } + else if ( declarator.getParent() instanceof IASTFunctionDefinition) { + IASTFunctionDefinition decl = (IASTFunctionDefinition)declarator.getParent(); + returnType = buildGenericNameFromDeclSpecifier(decl.getDeclSpecifier()); + String rawName = ASTStringUtilExt.getName(declarator); + List namedEntity = context.currentFile().lookupFunctionInVisibleScope(GenericName.build(rawName)); + if (namedEntity!=null) { + rawName = namedEntity.get(0).getQualifiedName(); + } + returnType = reMapIfConstructDeconstruct(rawName,returnType); + context.foundMethodDeclaratorImplementation(rawName, returnType,decl.getFileLocation().getStartingLineNumber()); + } + } + return super.visit(declarator); + } + + private GenericName buildGenericNameFromDeclSpecifier(IASTDeclSpecifier decl) { + String name = ASTStringUtilExt.getName(decl); + List templateParams = ASTStringUtilExt.getTemplateParameters(decl); + if (name==null) + return null; + return new GenericName(name,templateParams); + } + + /** + * In case of return type is empty, it maybe a construct/deconstruct function + * @param functionname + * @param returnType + * @return + */ + private GenericName reMapIfConstructDeconstruct(String functionname, GenericName returnType) { + if (returnType!=null && returnType.uniqName().length()>0) + return returnType; + if (functionname.contains("::")) { + return new GenericName(functionname.substring(0, functionname.indexOf("::"))); + }else { + return new GenericName(functionname); + } + } + + @Override + public int leave(IASTDeclarator declarator) { + if (notLocalFile(declarator)) return ASTVisitor.PROCESS_SKIP; + if (declarator instanceof IASTFunctionDeclarator){ + if ( declarator.getParent() instanceof IASTSimpleDeclaration) { + String rawName = ASTStringUtilExt.getName(declarator); + if (rawName.equals(context.lastContainer().getRawName().getName())) { + context.exitLastedEntity(); + }else { + System.err.println("unexpected symbol"); + } + } + } + return super.leave(declarator); + } + + @Override + public int leave(IASTDeclaration declaration) { + if (notLocalFile(declaration)) return ASTVisitor.PROCESS_SKIP; + if ( declaration instanceof IASTFunctionDefinition) { + context.exitLastedEntity(); + } + return super.leave(declaration); + } + + // Variables + @Override + public int visit(IASTDeclaration declaration) { + if (notLocalFile(declaration)) return ASTVisitor.PROCESS_SKIP; + logger.trace("enter IASTDeclaration " + declaration.getClass().getSimpleName()); + + if (declaration instanceof ICPPASTUsingDeclaration) { + String ns = ASTStringUtilExt.getName((ICPPASTUsingDeclaration)declaration); + context.foundNewImport(new PackageWildCardImport(ns)); + } + else if (declaration instanceof ICPPASTUsingDirective) { + String ns = ((ICPPASTUsingDirective)declaration).getQualifiedName().toString().replace("::", "."); + context.foundNewImport(new ExactMatchImport(ns)); + } + else if (declaration instanceof IASTSimpleDeclaration ) { + for (IASTDeclarator declarator:((IASTSimpleDeclaration) declaration).getDeclarators()) { + IASTDeclSpecifier declSpecifier = ((IASTSimpleDeclaration) declaration).getDeclSpecifier(); + //Found new typedef definition + if (declSpecifier.getStorageClass()==IASTDeclSpecifier.sc_typedef) { + context.foundNewAlias(ASTStringUtilExt.getName(declarator),ASTStringUtilExt.getName(declSpecifier)); + }else if (!(declarator instanceof IASTFunctionDeclarator)) { + String varType = ASTStringUtilExt.getName(declSpecifier); + String varName = ASTStringUtilExt.getName(declarator); + if (!StringUtils.isEmpty(varType)) { + context.foundVarDefinition(varName, GenericName.build(varType), ASTStringUtilExt.getTemplateParameters(declSpecifier),declarator.getFileLocation().getStartingLineNumber()); + }else { + expressionUsage.foundCallExpressionOfFunctionStyle(varName,declarator); + } + + } + } + }else if (declaration instanceof IASTFunctionDefinition){ + //handled in declarator + }else if (declaration instanceof CPPASTVisibilityLabel){ + //we ignore the visibility in dependency check + }else if (declaration instanceof CPPASTLinkageSpecification){ + + }else if (declaration instanceof CPPASTTemplateDeclaration){ + + }else if (declaration instanceof CPPASTProblemDeclaration){ + System.err.println("parsing error \n" + declaration.getRawSignature()); + }else if (declaration instanceof ICPPASTAliasDeclaration){ + IASTName name = ((ICPPASTAliasDeclaration)declaration).getAlias(); + String alias = ASTStringUtilExt.getSimpleName(name).replace("::", "."); + ICPPASTTypeId mapped = ((ICPPASTAliasDeclaration)declaration).getMappingTypeId(); + String originalName1 = ASTStringUtilExt.getTypeIdString(mapped); + context.foundNewAlias(alias, originalName1); + }else if (declaration instanceof CPPASTNamespaceAlias){ + IASTName name = ((CPPASTNamespaceAlias)declaration).getAlias(); + String alias = ASTStringUtilExt.getSimpleName(name).replace("::", "."); + IASTName mapped = ((CPPASTNamespaceAlias)declaration).getMappingName(); + String originalName = ASTStringUtilExt.getName(mapped); + context.foundNewAlias(alias, originalName); + } + else if(declaration instanceof CPPASTStaticAssertionDeclaration) + { + + }else if (declaration instanceof CPPASTTemplateSpecialization) { + } + else{ + System.out.println("not handled type: " + declaration.getClass().getName()); + System.out.println(declaration.getRawSignature()); + } + return super.visit(declaration); + } + + @Override + public int visit(IASTEnumerator enumerator) { + if (notLocalFile(enumerator)) return ASTVisitor.PROCESS_SKIP; + logger.trace("enter IASTEnumerator " + enumerator.getClass().getSimpleName()); + VarEntity var = context.foundVarDefinition(enumerator.getName().toString(), context.currentType().getRawName(), new ArrayList<>(),enumerator.getFileLocation().getStartingLineNumber()); + return super.visit(enumerator); + } + + @Override + public int visit(IASTExpression expression) { + if (notLocalFile(expression)) return ASTVisitor.PROCESS_SKIP; + Expression expr = expressionUsage.foundExpression(expression); + expr.setLine(expression.getFileLocation().getStartingLineNumber()); + return super.visit(expression); + } + + @Override + public int visit(IASTParameterDeclaration parameterDeclaration) { + if (notLocalFile(parameterDeclaration)) return ASTVisitor.PROCESS_SKIP; + + logger.trace("enter IASTParameterDeclaration " + parameterDeclaration.getClass().getSimpleName()); + String parameterName = ASTStringUtilExt.getName(parameterDeclaration.getDeclarator()); + String parameterType = ASTStringUtilExt.getName(parameterDeclaration.getDeclSpecifier()); + if (context.currentFunction()!=null) { + VarEntity var = new VarEntity(GenericName.build(parameterName),GenericName.build(parameterType),context.currentFunction(),idGenerator.generateId()); + context.currentFunction().addParameter(var ); + }else { + //System.out.println("** parameterDeclaration = " + parameter); + } + return super.visit(parameterDeclaration); + } +} \ No newline at end of file diff --git a/src/main/java/com/educoder/bridge/tmp/DecoratedEntity.java b/src/main/java/com/educoder/bridge/tmp/DecoratedEntity.java new file mode 100644 index 0000000..a18344d --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/DecoratedEntity.java @@ -0,0 +1,148 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.relations.IBindingResolver; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public abstract class DecoratedEntity extends Entity{ + private Collection annotations; + private Collection resolvedAnnotations; + private Collection resolvedTypeParameters; + public DecoratedEntity() { + } + + public DecoratedEntity(GenericName rawName, Entity parent, Integer id) { + super(rawName, parent, id); + } + + public void addAnnotation(GenericName name) { + if(this.annotations==null) + annotations = new ArrayList<>(); + this.annotations.add(name); + } + + public void addTypeParameter(List parameters) { + this.getRawName().appendArguments(parameters); + } + + + + public void addTypeParameter(GenericName parameter) { + this.getRawName().appendArguments(parameter); + } + + protected void appendTypeParameters(Collection typeParameterEntities) { + if (resolvedTypeParameters==null) + resolvedTypeParameters = new ArrayList<>(); + resolvedTypeParameters.addAll(typeParameterEntities); + } + + /** + * For all data in the class, infer their types. + * Should be override in sub-classes + */ + public void inferLocalLevelEntities(IBindingResolver bindingResolver) { + Collection typeParameterEntities = typeParametersToEntities(bindingResolver); + appendTypeParameters(typeParameterEntities); +// if (this.getAncestorOfType(FileEntity.class).getRawName().contains("/examples/usersession/server.py") +// ) { +// System.out.print("dd"); +// } + resolvedAnnotations = identiferToEntities(bindingResolver, annotations); + } + + + + + private Collection typeParametersToEntities(IBindingResolver bindingResolver) { + ArrayList r = new ArrayList<>(); + for (GenericName typeParameter:this.getRawName().getArguments()) { + toEntityList(bindingResolver, r,typeParameter); + } + return r; + } + + protected void toEntityList(IBindingResolver bindingResolver, ArrayList r, GenericName typeParameter) { + Entity entity = resolveEntity(bindingResolver, typeParameter); + if (entity != null) + r.add(entity); + for (GenericName arg: typeParameter.getArguments()) { + toEntityList(bindingResolver,r,arg); + } + } + + public Collection getResolvedTypeParameters() { + if (resolvedTypeParameters==null) + return new ArrayList<>(); + return resolvedTypeParameters; + } + + + public Collection getResolvedAnnotations() { + if (resolvedAnnotations==null) + return new ArrayList<>(); + return resolvedAnnotations; + } + + public boolean isGenericTypeParameter(GenericName rawType) { + boolean foundInCurrentLevel = rawType.find(rawType); + if (foundInCurrentLevel) return true; + if (this.getParent()==null || !(this.getParent() instanceof ContainerEntity)) + return false; + return ((ContainerEntity)getParent()).isGenericTypeParameter(rawType); + } + /** + * A common utility function used to transfer the identifiers + * to types. + * @param bindingResolver - the inferer object + * @param identifiers - the identifiers will be translated + * @return The translated Types + */ + protected Collection identiferToEntities(IBindingResolver bindingResolver, Collection identifiers) { + if (identifiers==null) return null; + if (identifiers.size()==0) return null; + ArrayList r = new ArrayList<>(); + for (GenericName name : identifiers) { + Entity entity = resolveEntity(bindingResolver, name); + if (entity != null) + r.add(entity); + } + return r; + } + + private Entity resolveEntity(IBindingResolver bindingResolver, GenericName name) { + Entity entity = bindingResolver.resolveName(this, name, true); + if (entity==null) { + if (((ContainerEntity)getParent()).isGenericTypeParameter(name)) { + entity = TypeEntity.genericParameterType; + } + } + return entity; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/DependencyGenerator.java b/src/main/java/com/educoder/bridge/tmp/DependencyGenerator.java new file mode 100644 index 0000000..6f943f5 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/DependencyGenerator.java @@ -0,0 +1,160 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.generator; + +import depends.entity.CandidateTypes; +import depends.entity.Entity; +import depends.entity.EntityNameBuilder; +import depends.entity.FileEntity; +import depends.entity.repo.EntityRepo; +import depends.matrix.core.DependencyDetail; +import depends.matrix.core.DependencyMatrix; +import depends.matrix.core.LocationInfo; +import depends.matrix.transform.OrderedMatrixGenerator; +import depends.relations.Relation; +import multilang.depends.util.file.path.EmptyFilenameWritter; +import multilang.depends.util.file.path.FilenameWritter; +import multilang.depends.util.file.strip.EmptyLeadingNameStripper; +import multilang.depends.util.file.strip.ILeadingNameStrippper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import static depends.deptypes.DependencyType.DuckTypingLabel; + +public abstract class DependencyGenerator { + + private static Logger logger = LoggerFactory.getLogger(DependencyGenerator.class); + + public DependencyMatrix identifyDependencies(EntityRepo entityRepo, List typeFilter) { + System.out.println("dependencie data generating..."); + DependencyMatrix dependencyMatrix = build(entityRepo, typeFilter); + entityRepo.clear(); + System.out.println("reorder dependency matrix..."); + dependencyMatrix = new OrderedMatrixGenerator(dependencyMatrix).build(); + System.out.println("Dependencies data generating done successfully..."); + logger.info("Dependencies data generating done successfully..."); + return dependencyMatrix; + } + + /** + * Build the dependency matrix (without re-mapping file id) + * @param entityRepo which contains entities and relations + * @return the generated dependency matrix + */ + public DependencyMatrix build(EntityRepo entityRepo,List typeFilter) { + DependencyMatrix dependencyMatrix = new DependencyMatrix(typeFilter); + Iterator iterator = entityRepo.entityIterator(); + System.out.println("Start create dependencies matrix...."); + while(iterator.hasNext()) { + Entity entity = iterator.next(); + if (!entity.inScope()) continue; + if (outputLevelMatch(entity)){ + dependencyMatrix.addNode(nameOf(entity),entity.getId()); + } + int entityFrom = upToOutputLevelEntityId(entityRepo, entity); + if (entityFrom==-1) continue; + for (Relation relation:entity.getRelations()) { + Entity relatedEntity = relation.getEntity(); + if (relatedEntity==null) continue; + List relatedEntities = expandEntity(relatedEntity); + String duckTypingFlag = relatedEntity instanceof CandidateTypes? DuckTypingLabel:""; + relatedEntities.forEach(theEntity->{ + if (theEntity.getId()>=0) { + int entityTo = upToOutputLevelEntityId(entityRepo,theEntity); + if (entityTo!=-1) { + DependencyDetail detail = buildDescription(entity, theEntity, relation.getFromLine()); + detail = rewriteDetail(detail); + dependencyMatrix.addDependency(relation.getType()+duckTypingFlag, entityFrom,entityTo,1,detail); + } + } + }); + } + } + System.out.println("Finish create dependencies matrix...."); + return dependencyMatrix; + } + + private List expandEntity(Entity relatedEntity) { + List entities = new ArrayList<>(); + if (relatedEntity instanceof CandidateTypes) { + entities = Collections.unmodifiableList((List) ((CandidateTypes) relatedEntity).getCandidateTypes()); + }else { + entities.add(relatedEntity); + } + return entities; + } + + private DependencyDetail rewriteDetail(DependencyDetail detail) { + if (detail==null) return null; + String srcFile = filenameWritter.reWrite( + stripper.stripFilename(detail.getSrc().getFile()) + ); + String dstFile = filenameWritter.reWrite( + stripper.stripFilename(detail.getDest().getFile())); + return new DependencyDetail( + new LocationInfo(detail.getSrc().getObject(), + srcFile, detail.getSrc().getLineNumber()) + , + new LocationInfo(detail.getDest().getObject(), + dstFile, detail.getDest().getLineNumber())); + } + + protected abstract int upToOutputLevelEntityId(EntityRepo entityRepo, Entity entity); + + protected abstract String nameOf(Entity entity); + + protected abstract boolean outputLevelMatch(Entity entity); + + protected ILeadingNameStrippper stripper = new EmptyLeadingNameStripper(); + protected FilenameWritter filenameWritter = new EmptyFilenameWritter(); + private boolean generateDetail = false; + + public void setLeadingStripper(ILeadingNameStrippper stripper) { + this.stripper = stripper; + } + protected DependencyDetail buildDescription(Entity fromEntity, Entity toEntity, Integer fromLineNumber) { + if (!generateDetail) return null; + String fromObject = EntityNameBuilder.build(fromEntity); + String toObject = EntityNameBuilder.build(toEntity); + + Entity fromFile = fromEntity.getAncestorOfType(FileEntity.class); + Entity toFile = toEntity.getAncestorOfType(FileEntity.class); + + return new DependencyDetail( + new LocationInfo(stripper.stripFilename(fromObject),stripper.stripFilename(fromFile.getQualifiedName()),fromLineNumber), + new LocationInfo(stripper.stripFilename(toObject),stripper.stripFilename(toFile.getQualifiedName()),toEntity.getLine())); + } + public void setFilenameRewritter(FilenameWritter filenameWritter) { + this.filenameWritter = filenameWritter; + } + public void setGenerateDetail(boolean generateDetail) { + this.generateDetail = generateDetail; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/DependencyMatrix.java b/src/main/java/com/educoder/bridge/tmp/DependencyMatrix.java new file mode 100644 index 0000000..f74cc84 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/DependencyMatrix.java @@ -0,0 +1,102 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.matrix.core; + +import multilang.depends.util.file.path.FilenameWritter; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; + +import static depends.deptypes.DependencyType.DuckTypingLabel; + +public class DependencyMatrix { + private HashMap dependencyPairs = new HashMap<>(); + private ArrayList nodes = new ArrayList<>(); + private HashMap nodeIdToName = new HashMap<>(); + private List typeFilter; + public DependencyMatrix() { + } + public DependencyMatrix(int size) { + dependencyPairs = new HashMap<>(size); + } + public DependencyMatrix(List typeFilter) { + this.typeFilter = typeFilter; + } + public Collection getDependencyPairs() { + return dependencyPairs.values(); + } + + public void addNode(String name, int id) { + this.nodes.add(name); + this.nodeIdToName.put(id, name); + } + + public void addDependency(String depType, Integer from, Integer to, int weight,List details) { + if (typeFilter!=null && (!typeFilter.contains(depType))) + return; + if(from.equals(to) || from == -1 || to == -1) { + return; + } + if (dependencyPairs.get(DependencyPair.key(from,to))==null) { + dependencyPairs.put(DependencyPair.key(from,to),new DependencyPair(from,to)); + } + DependencyPair dependencyPair = dependencyPairs.get(DependencyPair.key(from,to)); + dependencyPair.addDependency(depType,weight,details); + } + + public void addDependency(String depType, Integer from, Integer to, int weight,DependencyDetail detail) { + if (typeFilter!=null && (!typeFilter.contains(depType.replace(DuckTypingLabel,"")))) + return; + if(from.equals(to) || from == -1 || to == -1) { + return; + } + if (dependencyPairs.get(DependencyPair.key(from,to))==null) { + dependencyPairs.put(DependencyPair.key(from,to),new DependencyPair(from,to)); + } + DependencyPair dependencyPair = dependencyPairs.get(DependencyPair.key(from,to)); + dependencyPair.addDependency(depType,weight,detail); + } + + public ArrayList getNodes() { + return nodes; + } + + + public DependencyMatrix reWriteFilenamePattern(FilenameWritter filenameRewritter) { + this.nodeIdToName = new HashMap<>(); + for (int i=0;i { + private static final long serialVersionUID = 1L; + public SupportedLangs() { super( LangProcessorRegistration.getRegistry().getLangs()); } + } + + public static class SupportedTypes extends ArrayList { + private static final long serialVersionUID = 1L; + public SupportedTypes() { super( DependencyType.allDependencies()); } + } + + @Parameters(index = "0", completionCandidates = DependsCommand.SupportedLangs.class, description = "The lanauge of project files: [${COMPLETION-CANDIDATES}]") + private String lang; + @Parameters(index = "1", description = "The directory to be analyzed") + private String src; + @Parameters(index = "2", description = "The output file name") + private String output; + @Option(names = {"-f", "--format"},split=",", description = "the output format: [json(default),xml,excel,detail,dot,plantuml]") + private String[] format=new String[]{"json"}; + @Option(names = {"-d", "--dir"}, description = "The output directory") + private String dir; + @Option(names = {"-m", "--map"}, description = "Output DV8 dependency map file.") + private boolean dv8map = true; + @Option(names = {"-s", "--strip-leading-path"}, description = "Strip the leading path.") + private boolean stripLeadingPath = false; + @Option(names = {"--strip-paths"}, split=",", description = "The path(s) to be stripped. if -s enabled, the path(s) start after . " + + "Otherwise, the path(s) should be valid.") + private String[] strippedPaths = new String[]{}; + @Option(names = {"-g", "--granularity"}, description = "Granularity of dependency.[file(default),method,structure,L#(the level of folder. e.g. L1=1st level folder)]") + private String granularity="file"; + @Option(names = {"-p", "--namepattern"}, description = "The name path pattern.[dot(.), unix(/) or windows(\\)") + private String namePathPattern=""; + @Option(names = {"-i","--includes"},split=",", description = "The files of searching path") + private String[] includes = new String[] {}; + @Option(names = {"--auto-include"},split=",", description = "auto include all paths under the source path (please notice the potential side effect)") + private boolean autoInclude = false; + @Option(names = {"--detail"},split=",", description = "add detail dependency information to output (only applicable for JSON output format)") + private boolean detail = false; + @Option(names = {"--auto-stub"},split=",", description = "create stub files for unsolved symbols (exprimental feature, only for java)") + private boolean autoStub = false; + @Option(names = {"--type-filter"},split=",", completionCandidates = DependsCommand.SupportedTypes.class, description = "only filter the listed dependency types[${COMPLETION-CANDIDATES}]") + private String[] typeFilter=new String[]{}; + @Option(names = {"--external-deps"}, description = "Output external dependencies") + private boolean outputExternalDependencies = false; + @Option(names = {"--duck-typing-deduce"}, description = "Deduce implicit variable types") + private boolean duckTypingDeduce = true; + @Option(names = {"-h","--help"}, usageHelp = true, description = "display this help and exit") + boolean help; + public DependsCommand() { + } + public String getLang() { + return lang; + } + public void setLang(String lang) { + this.lang = lang; + } + public String getSrc() { + return src; + } + public void setSrc(String src) { + this.src = src; + } + public String getOutputName() { + return output; + } + public void setOutput(String output) { + this.output = output; + } + public String[] getFormat() { + return format; + } + public String getOutputDir() { + if (dir==null) { + dir = System.getProperty("user.dir"); + } + return dir; + } + public boolean isDv8map() { + return dv8map; + } + public String[] getIncludes() { + return includes; + } + public boolean isHelp() { + return help; + } + public String getGranularity() { + return granularity; + } + public String getNamePathPattern() { + return namePathPattern; + } + public boolean isStripLeadingPath() { + return stripLeadingPath; + } + + public boolean isAutoInclude () { + return autoInclude; + } + public boolean isDetail () { + return detail; + } + public String[] getStrippedPaths() { + return strippedPaths; + } + public void setStrippedPaths(String[] strippedPaths) { + this.strippedPaths = strippedPaths; + } + public boolean isAutoStub() { + return autoStub; + } + public List getTypeFilter() { + if (typeFilter.length==0) { + return DependencyType.allDependencies(); + } + return java.util.Arrays.asList(typeFilter); + } + public boolean isOutputExternalDependencies() { + return outputExternalDependencies; + } + + public boolean isDuckTypingDeduce() { + return this.duckTypingDeduce; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/EclipseTestBase_No_ResponseDuirngTypeResolve.java b/src/main/java/com/educoder/bridge/tmp/EclipseTestBase_No_ResponseDuirngTypeResolve.java new file mode 100644 index 0000000..25f2020 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/EclipseTestBase_No_ResponseDuirngTypeResolve.java @@ -0,0 +1,222 @@ +/******************************************************************************* + * Copyright (c) 2007, 2014 BEA Systems, Inc. and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * BEA Systems, Inc. - initial API and implementation + * Jesper Steen Moller - Bug 412150 [1.8] [compiler] Enable reflected parameter names during annotation processing + *******************************************************************************/ +package org.eclipse.jdt.apt.pluggable.tests; + +import java.io.File; +import java.io.InputStream; +import java.net.URL; +import java.util.Map; + +import javax.lang.model.SourceVersion; + +import org.eclipse.core.resources.IFile; +import org.eclipse.core.resources.IProject; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IPath; +import org.eclipse.jdt.apt.core.internal.util.FactoryContainer; +import org.eclipse.jdt.apt.core.internal.util.FactoryContainer.FactoryType; +import org.eclipse.jdt.apt.core.internal.util.FactoryPath; +import org.eclipse.jdt.apt.core.util.AptConfig; +import org.eclipse.jdt.core.IClasspathAttribute; +import org.eclipse.jdt.core.IJavaProject; +import org.eclipse.jdt.core.JavaCore; +import org.eclipse.jdt.core.tests.builder.BuilderTests; +import org.eclipse.jdt.core.tests.util.Util; +import org.eclipse.jdt.internal.core.ClasspathEntry; + +import junit.framework.Test; + +public class TestBase extends BuilderTests +{ + + protected static final String JAVA_16_COMPLIANCE = "1.6"; + protected static final String JAVA_18_COMPLIANCE = "1.8"; + protected static final String JAVA_9_COMPLIANCE = "9"; + + protected String _projectName; + protected static int _projectSerial = 0; // used to create unique project names, to avoid resource deletion problems + + public TestBase(String name) { + super(name); + } + + public static Test suite() { + throw new IllegalStateException("This is a base test class whose suite() method must not be called.\n" + + "This exception is thrown to avoid running org.eclipse.jdt.core.tests.builder.BuilderTests#suite() twice."); + } + + /** + * Extract lib/annotations.jar from the test bundle and add it to the specified project + */ + private static void addAnnotationJar(IJavaProject jproj, boolean addToModulePath) throws Exception { + final String resName = "lib/annotations.jar"; // name in bundle + final String libName = resName; // name in destination project + InputStream is = null; + URL resURL = Apt6TestsPlugin.thePlugin().getBundle().getEntry(resName); + is = resURL.openStream(); + IPath projPath = jproj.getPath(); + IProject proj = jproj.getProject(); + IFile libFile = proj.getFile(libName); + env.addFolder(projPath, "lib"); + if (libFile.exists()) { + libFile.setContents(is, true, false, null); + } else { + libFile.create(is, true, null); + } + if (addToModulePath) { + IClasspathAttribute[] attributes = { JavaCore.newClasspathAttribute(IClasspathAttribute.MODULE, "true") }; + env.addEntry(projPath, JavaCore.newLibraryEntry(libFile.getFullPath(), null, null, + ClasspathEntry.NO_ACCESS_RULES, attributes, false)); + } else { + env.addLibrary(projPath, libFile.getFullPath(), null, null); + } + } + + /** + * Create a java project with java libraries and test annotations on classpath + * (compiler level is 1.6). Use "src" as source folder and "bin" as output folder. APT + * is not enabled. + * + * @param projectName + * @return a java project that has been added to the current workspace. + * @throws Exception + */ + protected static IJavaProject createJavaProject(final String projectName) throws Exception + { + IPath projectPath = env.addProject(projectName, JAVA_16_COMPLIANCE); + env.addExternalJars(projectPath, Util.getJavaClassLibs()); + // remove old package fragment root so that names don't collide + env.removePackageFragmentRoot(projectPath, ""); //$NON-NLS-1$ + env.addPackageFragmentRoot(projectPath, "src"); //$NON-NLS-1$ + env.setOutputFolder(projectPath, "bin"); //$NON-NLS-1$ + final IJavaProject javaProj = env.getJavaProject(projectPath); + addAnnotationJar(javaProj, false); + return javaProj; + } + + /** + * Create a java project with java libraries and test annotations on classpath + * (compiler level is 1.8). Use "src" as source folder and "bin" as output folder. APT + * is not enabled. + * + * @param projectName + * @return a java project that has been added to the current workspace. + * @throws Exception + */ + protected static IJavaProject createJava8Project(final String projectName) throws Exception { + // Note, make sure this is run only with a JRE 8 and above. + IPath projectPath = env.addProject(projectName, JAVA_18_COMPLIANCE); + env.addExternalJars(projectPath, Util.getJavaClassLibs()); + + // remove old package fragment root so that names don't collide + env.removePackageFragmentRoot(projectPath, ""); //$NON-NLS-1$ + env.addPackageFragmentRoot(projectPath, "src"); //$NON-NLS-1$ + env.setOutputFolder(projectPath, "bin"); //$NON-NLS-1$ + final IJavaProject javaProj = env.getJavaProject(projectPath); + javaProj.getProject().getFolder("prebuilt").create(true, true, null); + javaProj.getProject().getFolder("prebuilt").getFolder("p").create(true, true, null); + env.addClassFolder(projectPath, projectPath.append("prebuilt"), true); + addAnnotationJar(javaProj, false); + return javaProj; + } + + /** + * Create a java project with java libraries and test annotations on modulepath + * (compiler level is 1.9). Use "src" as source folder and "bin" as output folder. APT + * is not enabled. + * + * @param projectName + * @return a java project that has been added to the current workspace. + * @throws Exception + */ + protected static IJavaProject createJava9Project(final String projectName) throws Exception { + // Note, make sure this is run only with a JRE 9 and above. + IPath projectPath = env.addProject(projectName, JAVA_9_COMPLIANCE); + env.addExternalJars(projectPath, Util.getJavaClassLibs()); + // remove old package fragment root so that names don't collide + env.removePackageFragmentRoot(projectPath, ""); //$NON-NLS-1$ + env.addPackageFragmentRoot(projectPath, "src"); //$NON-NLS-1$ + env.setOutputFolder(projectPath, "bin"); //$NON-NLS-1$ + final IJavaProject javaProj = env.getJavaProject(projectPath); + addAnnotationJar(javaProj, true); + return javaProj; + } + + /** + * Ensure that there are no Java 5 processors on the factory path, as they can cause + * units to be multiply compiled, which can mess up tests that expect a certain number + * of compilations to occur. + * @param jproj the project whose factory path will be edited + * @throws CoreException + */ + protected void disableJava5Factories(IJavaProject jproj) throws CoreException { + FactoryPath fp = (FactoryPath) AptConfig.getFactoryPath(jproj); + for (Map.Entry entry : fp.getAllContainers().entrySet()) { + if (entry.getKey().getType() == FactoryType.PLUGIN) { + String id = entry.getKey().getId(); + if (!Apt6TestsPlugin.PLUGIN_ID.equals(id)) { + fp.disablePlugin(id); + } + } + } + AptConfig.setFactoryPath(jproj, fp); + } + + /** + * Verify that an expected file exists within a project. + * @param fileName the filename relative to the project root. + */ + protected void expectingFile(IProject proj, String fileName) throws Exception + { + IPath path = proj.getLocation().append(fileName); + File file = new File(path.toOSString()); + assertTrue("Expected file " + fileName + " was missing from project", file != null && file.exists()); + } + + /** + * Verify that an expected file exists within a project. + * @param fileName the filename relative to the project root. + */ + protected void expectingNoFile(IProject proj, String fileName) throws Exception + { + IPath path = proj.getLocation().append(fileName); + File file = new File(path.toOSString()); + boolean exists = file.exists(); + // work around a timing bug in some versions of JRE 1.6 on Linux: + // Before assuming the test has failed, wait half a second and try again. + // This delay is not encountered when the test is passing normally. + if (exists) { + Thread.sleep(500); + exists = file.exists(); + } + assertTrue("File " + fileName + " was expected to not exist", file == null || !exists); + } + + @Override + protected void setUp() throws Exception + { + super.setUp(); + env.setAutoBuilding(false); + _projectName = String.format("testproj%04d", ++_projectSerial); + } + public boolean canRunJava9() { + try { + SourceVersion.valueOf("RELEASE_9"); + } catch(IllegalArgumentException iae) { + return false; + } + return true; + } +} \ No newline at end of file diff --git a/src/main/java/com/educoder/bridge/tmp/Entity.java b/src/main/java/com/educoder/bridge/tmp/Entity.java new file mode 100644 index 0000000..75174c5 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/Entity.java @@ -0,0 +1,275 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.relations.IBindingResolver; +import depends.relations.Relation; + +import java.util.*; + +/** + * Entity is the root of all entities, including file, package, module, + * class, method/function etc. + * Each entity has unique id, name,qualifiedName, parent, children + * We also use entity to record relations + */ +public abstract class Entity { + + Integer id=-1; + String qualifiedName = null; + GenericName rawName = GenericName.build(""); + Entity parent; + private MultiDeclareEntities mutliDeclare = null; + private Set children; + ArrayList relations; + private Entity actualReferTo = null; + private boolean inScope = true; + protected HashMap visibleNames = new HashMap<>(); + private Location location = new Location(); + public Entity() {}; + public Entity(GenericName rawName, Entity parent, Integer id) { + this.qualifiedName = null; + this.rawName = rawName; + this.parent = parent; + this.id = id; + if (parent!=null) + parent.addChild(this); + deduceQualifiedName(); + visibleNames.put(rawName.getName(), this); + visibleNames.put(qualifiedName, this); + } + + private Set children() { + if (children==null) + children = new HashSet<>(); + return children; + } + /** + * Rule 1: if it start with '.' , then the name is equal to raw name + * Rule 2: if parent not exists, the name is equal to raw name + * Rule 3: if parent exists but no qualified name exists or empty, the name is equal to raw name + * Rule 4: otherwise, qualified name = parent_qualfied_name + "."+rawName + * Rule 5: make sure the qualified name do not start with '.' + */ + private void deduceQualifiedName() { + rawName = rawName.replace("::","." ); + if (this.rawName.startsWith(".")) { + this.qualifiedName = this.rawName.uniqName().substring(1); + return; //already qualified + } + if (parent==null) { + this.qualifiedName = this.rawName.uniqName(); + return; + } + if (parent.getQualifiedName(true)==null) { + this.qualifiedName = this.rawName.uniqName(); + return; + } + if (parent.getQualifiedName(true).isEmpty()) { + this.qualifiedName = rawName.uniqName(); + return; + } + this.qualifiedName= parent.getQualifiedName(true)+"." + rawName.uniqName(); + } + + + public GenericName getRawName() { + return rawName; + } + + public Integer getId() { + return id; + } + + public void addRelation(Relation relation) { + if (relations==null) + relations = new ArrayList<>(); + if (relation.getEntity()==null) return; + relations.add(relation); + } + + public ArrayList getRelations() { + if (relations==null) + return new ArrayList<>(); + return relations; + } + + public void addChild(Entity child) { + children().add(child); + visibleNames.put(child.getRawName().getName(), child); + visibleNames.put(child.getQualifiedName(), child); + } + + public Entity getParent() { + return parent; + } + + public void setParent(Entity parent) { + this.parent = parent; + } + + public Collection getChildren() { + if (children==null) + return new HashSet<>(); + return children; + } + + public void setQualifiedName(String qualifiedName) { + this.qualifiedName = qualifiedName; + } + + public void setRawName(GenericName rawName) { + this.rawName = rawName; + deduceQualifiedName(); + } + + public final String getQualifiedName() { + return qualifiedName; + } + + public String getQualifiedName(boolean overrideFileWithPackage) { + return qualifiedName; + } + + @Override + public String toString() { + return "Entity [id=" + id + ", qualifiedName=" + qualifiedName + ", rawName=" + rawName + "]"; + } + + /** + * Get ancestor of type. + * @param classType + * @return null (if not exist) or the type + */ + public Entity getAncestorOfType(@SuppressWarnings("rawtypes") Class classType) { + Entity fromEntity = this; + while(fromEntity!=null) { + if (fromEntity.getClass().equals(classType)) + return fromEntity; + if (fromEntity.getParent()==null) return null; + fromEntity = fromEntity.getParent(); + } + return null; + } + + /** + * Invoke inferer to resolve the entity type etc. + * */ + public void inferEntities(IBindingResolver bindingResolver) { + inferLocalLevelEntities(bindingResolver); + for (Entity child:this.getChildren()) { + child.inferEntities(bindingResolver); + } + } + public abstract void inferLocalLevelEntities(IBindingResolver bindingResolver); + + public TypeEntity getType() { + return null; + } + + public String getDisplayName() { + return getRawName().uniqName(); + } + + public MultiDeclareEntities getMutliDeclare() { + return mutliDeclare; + } + + public void setMutliDeclare(MultiDeclareEntities mutliDeclare) { + this.mutliDeclare = mutliDeclare; + } + + public Entity getActualReferTo() { + if (this.actualReferTo ==null) + return this; + return actualReferTo; + } + + public void setActualReferTo(Entity actualReferTo) { + this.actualReferTo = actualReferTo; + } + + public static void setParent(Entity child, Entity parent) { + if (parent == null) + return; + if (child == null) + return; + if (parent.equals(child.getParent())) + return; + child.setParent(parent); + parent.addChild(child); + } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Entity other = (Entity) obj; + if (id == null) { + if (other.id != null) + return false; + } else if (!id.equals(other.id)) + return false; + return true; + } + + public void setInScope(boolean value) { + this.inScope = value; + children().forEach(child->child.setInScope(value)); + } + + public boolean inScope() { + return inScope; + } + public Entity getByName(String name, HashSet searched) { + if (searched.contains(this)) return null; + searched.add(this); + return visibleNames.get(name); + } + + public Integer getLine() { + return location.getLine(); + } + + public void setLine(int lineNumber) { + this.location.setLine(lineNumber); + } + + public Location getLocation() { + return this.location; + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/EntityExtractTest.java b/src/main/java/com/educoder/bridge/tmp/EntityExtractTest.java new file mode 100644 index 0000000..9c5e1a1 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/EntityExtractTest.java @@ -0,0 +1,101 @@ +package depends.extractor.pom; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class EntityExtractTest extends MavenParserTest{ + @Before + public void setUp() { + super.init(); + } + + @Test + public void use_package_contains() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/maven-code-examples/simple/log4j.pom", + }; + + for (String src:srcs) { + PomFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + assertEquals(0,entityRepo.getEntity("org.log4j-test.log4j_1.2.12_").getRelations().size()); + } + + + @Test + public void should_use_parent_groupId() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/maven-code-examples/use_parent_groupId_and_version.pom", + }; + + for (String src:srcs) { + PomFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + assertNotNull(entityRepo.getEntity("org.apache.maven.surefire.surefire-junit4_2.12.4_")); + } + + @Test + public void should_parse_properties_in_same_pom() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/maven-code-examples/properties-test1.pom", + }; + + for (String src:srcs) { + PomFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + PomArtifactEntity entity = (PomArtifactEntity)(entityRepo.getEntity("properties-test.test_1_")); + /* + 1.00 + 3.1.4 + Apache ActiveMQ + activemq-${project.version} */ + assertEquals("1.00",entity.getProperty("project.version")); + assertEquals("activemq-1.00",entity.getProperty("siteId")); + } + + + @Test + public void should_parse_multiple_properties_in_same_pom() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/maven-code-examples/properties-test1.pom", + }; + + for (String src:srcs) { + PomFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + PomArtifactEntity entity = (PomArtifactEntity)(entityRepo.getEntity("properties-test.test_1_")); + /* + 1.00 + 3.1.4 + Apache ActiveMQ + activemq-${project.version}--${activeio-version} */ + assertEquals("activemq-1.00-3.1.4",entity.getProperty("anotherId")); + } + + @Test + public void should_parse_multiple_properties_in_parent_pom() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/maven-code-examples/properties-test-child.pom" + }; + + for (String src:srcs) { + PomFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + PomArtifactEntity entity = (PomArtifactEntity)(entityRepo.getEntity("properties-test.test_1_")); + assertEquals("13",entity.getProperty("project.version")); + } +} \ No newline at end of file diff --git a/src/main/java/com/educoder/bridge/tmp/ExcelXlsFormatDependencyDumper.java b/src/main/java/com/educoder/bridge/tmp/ExcelXlsFormatDependencyDumper.java new file mode 100644 index 0000000..6831f18 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/ExcelXlsFormatDependencyDumper.java @@ -0,0 +1,120 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.format.excel; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +import org.apache.poi.hssf.usermodel.HSSFCell; +import org.apache.poi.hssf.usermodel.HSSFRow; +import org.apache.poi.hssf.usermodel.HSSFSheet; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; + +import depends.format.AbstractFormatDependencyDumper; +import depends.matrix.core.DependencyMatrix; +import depends.matrix.core.DependencyPair; +import depends.matrix.core.DependencyValue; + +public class ExcelXlsFormatDependencyDumper extends AbstractFormatDependencyDumper { + private HSSFWorkbook workbook; + private HSSFSheet sheet; + @Override + public String getFormatName() { + return "xls"; + } + public ExcelXlsFormatDependencyDumper(DependencyMatrix dependencyMatrix, String projectName, String outputDir) { + super(dependencyMatrix, projectName,outputDir); + } + + @Override + public boolean output() { + String filename = composeFilename() + ".xls"; + if (matrix.getNodes().size() > 255) { + System.out.println("We can only export small matrix(<256 items) to excel" + "due to MS Office limitation"); + return false; + } + startFile(); + Collection dependencyPairs = matrix.getDependencyPairs(); + HSSFRow[] row = new HSSFRow[matrix.getNodes().size()]; + + // create header row + HSSFRow header = sheet.createRow(0); + for (int i = 0; i < matrix.getNodes().size(); i++) { + HSSFCell cell = header.createCell(i + 2); + cell.setCellValue(i); + } + ; + + // create header col + for (int i = 0; i < matrix.getNodes().size(); i++) { + row[i] = sheet.createRow(i + 1); + String node = matrix.getNodes().get(i); + HSSFCell cell = row[i].createCell(0); + cell.setCellValue(i); + cell = row[i].createCell(1); + cell.setCellValue(node); + } + ; + + // create header col + for (int i = 0; i < matrix.getNodes().size(); i++) { + HSSFCell cell = row[i].createCell(i + 2); + cell.setCellValue("(" + i + ")"); + } + ; + + for (DependencyPair dependencyPair : dependencyPairs) { + HSSFCell cell = row[dependencyPair.getFrom()].createCell(dependencyPair.getTo() + 2); + cell.setCellValue(buildDependencyValues(dependencyPair.getDependencies())); + } + closeFile(filename); + return true; + } + + private String buildDependencyValues(Collection dependencies) { + StringBuilder sb = new StringBuilder(); + for (DependencyValue dependency : dependencies) { + String comma = sb.length() > 0 ? "," : ""; + sb.append(comma).append(dependency.getType()).append("(").append(dependency.getWeight()).append(")"); + } + return sb.toString(); + } + + private void closeFile(String filename) { + try { + workbook.write(new File(filename)); + workbook.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private void startFile() { + workbook = new HSSFWorkbook(); + sheet = workbook.createSheet("DSM"); + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/ExcelXlsxFormatDependencyDumper.java b/src/main/java/com/educoder/bridge/tmp/ExcelXlsxFormatDependencyDumper.java new file mode 100644 index 0000000..c27b68a --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/ExcelXlsxFormatDependencyDumper.java @@ -0,0 +1,119 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.format.excel; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Collection; + +import org.apache.poi.xssf.usermodel.XSSFCell; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import depends.format.AbstractFormatDependencyDumper; +import depends.matrix.core.DependencyMatrix; +import depends.matrix.core.DependencyPair; +import depends.matrix.core.DependencyValue; + +public class ExcelXlsxFormatDependencyDumper extends AbstractFormatDependencyDumper { + private XSSFWorkbook workbook; + private XSSFSheet sheet; + @Override + public String getFormatName() { + return "xlsx"; + } + public ExcelXlsxFormatDependencyDumper(DependencyMatrix dependencyMatrix, String projectName, String outputDir) { + super(dependencyMatrix, projectName,outputDir); + } + + @Override + public boolean output() { + String filename = composeFilename() + ".xlsx"; + + startFile(); + Collection dependencyPairs = matrix.getDependencyPairs(); + XSSFRow[] row = new XSSFRow[matrix.getNodes().size()]; + + // create header row + XSSFRow header = sheet.createRow(0); + for (int i = 0; i < matrix.getNodes().size(); i++) { + XSSFCell cell = header.createCell(i + 2); + cell.setCellValue(i); + } + ; + + // create header col + for (int i = 0; i < matrix.getNodes().size(); i++) { + row[i] = sheet.createRow(i + 1); + String node = matrix.getNodes().get(i); + XSSFCell cell = row[i].createCell(0); + cell.setCellValue(i); + cell = row[i].createCell(1); + cell.setCellValue(node); + } + ; + + // create header col + for (int i = 0; i < matrix.getNodes().size(); i++) { + XSSFCell cell = row[i].createCell(i + 2); + cell.setCellValue("(" + i + ")"); + } + ; + + for (DependencyPair dependencyPair : dependencyPairs) { + XSSFCell cell = row[dependencyPair.getFrom()].createCell(dependencyPair.getTo() + 2); + cell.setCellValue(buildDependencyValues(dependencyPair.getDependencies())); + } + closeFile(filename); + return true; + } + + private String buildDependencyValues(Collection dependencies) { + StringBuilder sb = new StringBuilder(); + for (DependencyValue dependency : dependencies) { + String comma = sb.length() > 0 ? "," : ""; + sb.append(comma).append(dependency.getType()).append("(").append(dependency.getWeight()).append(")"); + } + return sb.toString(); + } + + private void closeFile(String filename) { + try { + FileOutputStream out = new FileOutputStream(filename); + workbook.write(out); + workbook.close(); + out.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private void startFile() { + workbook = new XSSFWorkbook(); + sheet = workbook.createSheet("DSM"); + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/Expression.java b/src/main/java/com/educoder/bridge/tmp/Expression.java new file mode 100644 index 0000000..5639248 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/Expression.java @@ -0,0 +1,413 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.entity.repo.EntityRepo; +import depends.relations.IBindingResolver; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Expression + */ +public class Expression implements Serializable{ + private static final long serialVersionUID = 1L; + + public Integer id; + private String text; // for debug purpose + private GenericName rawType; // the raw type name + private GenericName identifier; // the varName, or method name, etc. + private boolean isSet = false; // is a set relation from right to leftHand + private boolean isDot = false; // is a dot expression, will decuce variable tfype left to right + private boolean isCall = false; + private boolean isLogic = false; + private boolean isCreate = false; + private boolean isCast = false; + private boolean isThrow = false; + private boolean isStatement = false; //statement is only used for return type calcuation in some langs such as ruby + //they will not be treat as real expressions in case of relation calculation + private boolean deriveTypeFromChild = true; + + private Integer deduceTypeBasedId; //by default, parent expression type determined by most left child + + private Integer parentId = -1; + private transient Expression parent; + + private transient List deducedTypeVars = new ArrayList<>(); + private List deducedTypeVarsId = new ArrayList<>(); + + private transient List deducedTypeFunctions= new ArrayList<>(); + private List deducedTypeFunctionsId = new ArrayList<>(); + + private Integer referredEntityId; + private transient Entity referredEntity; + + private transient TypeEntity type; // the type we care - for relation calculation. + private Location location = new Location(); + //for leaf, it equals to referredEntity.getType. otherwise, depends on child's type strategy + + /* + * */ + + public Expression() { + deducedTypeVars = new ArrayList<>(); + deducedTypeFunctions = new ArrayList<>(); + } + + public Expression(Integer id) { + this.id = id; + deducedTypeVars = new ArrayList<>(); + deducedTypeFunctions = new ArrayList<>(); + } + + public void reload(EntityRepo repo, ArrayList expressionList) { + this.deducedTypeFunctions = new ArrayList<>(); + this.deducedTypeVars = new ArrayList<>(); + + //recover parent relation + if (parentId!=-1) { + for (Expression expr:expressionList) { + if (expr.id==parentId) { + parent = expr; + break; + } + } + } + + //recover deducedTypeFunctionsId + if (deducedTypeFunctionsId!=null) { + for (Integer funcId:this.deducedTypeFunctionsId) { + this.deducedTypeFunctions.add((FunctionEntity) repo.getEntity(funcId)); + } + } + + //recover deducedTypeVars + if (deducedTypeVarsId!=null) { + for (Integer varId:this.deducedTypeVarsId) { + this.deducedTypeVars.add((VarEntity) repo.getEntity(varId)); + } + } + + //referer referredEntity -- TODO:maybe not require + if (this.referredEntityId!=null && this.referredEntity==null) { + this.referredEntity = repo.getEntity(this.referredEntityId); + if (this.referredEntity ==null){ + System.err.println("unexpected: referred Entity is null" + this.referredEntityId + this.text+this.id); + } + } + } + + /** + * Set type of the expression + * @param type + * @param referredEntity + * @param bindingResolver + */ + public void setType(TypeEntity type, Entity referredEntity, IBindingResolver bindingResolver) { + if (this.getReferredEntity()==null && referredEntity!=null) { + this.setReferredEntity(referredEntity); + } + + boolean changedType = false; + if (this.type==null && type!=null) { + this.type = type; + for (VarEntity var:deducedTypeVars) { + if (var!=null) { + var.setType(this.type); + } + } + for (FunctionEntity func:deducedTypeFunctions) { + if (func!=null) { + func.addReturnType(this.type); + } + } + changedType = true; + } + if (this.referredEntity==null) + this.setReferredEntity(this.type); + + if (changedType) + deduceTheParentType(bindingResolver); + } + + + /** + * deduce type of parent based on child's type + * @param bindingResolver + */ + private void deduceTheParentType(IBindingResolver bindingResolver) { + if (this.type==null) return; + if (this.parent==null) return; + Expression parent = this.parent; + if (parent.type != null)return; + if (!parent.deriveTypeFromChild) return; + //parent's type depends on first child's type + if (parent.deduceTypeBasedId!=this.id) return; + + //if child is a built-in/external type, then parent must also a built-in/external type + if (this.type.equals(TypeEntity.buildInType)) { + parent.setType(TypeEntity.buildInType,TypeEntity.buildInType, bindingResolver); + return; + } + + /* if it is a logic expression, the return type/type is boolean. */ + if (parent.isLogic) { + parent.setType(TypeEntity.buildInType,null, bindingResolver); + } + /* if it is a.b, and we already get a's type, b's type could be identified easily */ + else if (parent.isDot) { + if (parent.isCall()) { + List funcs = this.getType().lookupFunctionInVisibleScope(parent.identifier); + setReferredFunctions(bindingResolver, parent, funcs); + }else { + Entity var = this.getType().lookupVarInVisibleScope(parent.identifier); + if (var!=null) { + parent.setType(var.getType(),var, bindingResolver); + parent.setReferredEntity(var); + }else { + List funcs = this.getType().lookupFunctionInVisibleScope(parent.identifier); + setReferredFunctions(bindingResolver,parent,funcs); + } + } + if (parent.getType()==null) { + parent.setType(bindingResolver.inferTypeFromName(this.getType(), parent.identifier),null, bindingResolver); + } + } + /* if other situation, simple make the parent and child type same */ + else { + parent.setType(type, null, bindingResolver); + } + if (parent.getReferredEntity()==null) + parent.setReferredEntity(parent.type); + } + + private void setReferredFunctions(IBindingResolver bindingResolver, Expression expr, List funcs) { + if (funcs ==null ||funcs.size()==0) return; + Entity func = funcs.get(0); + if (funcs.size()==1){ + expr.setType(func.getType(), func, bindingResolver); + expr.setReferredEntity(func); + return; + } + MultiDeclareEntities m = new MultiDeclareEntities(func, bindingResolver.getRepo().generateId()); + bindingResolver.getRepo().add(m); + for (int i = 1; i< funcs.size(); i++) { + m.add(funcs.get(i)); + } + expr.setType(func.getType(), m, bindingResolver); + expr.setReferredEntity(m); + } + + private void setReferredEntity(Entity referredEntity) { + this.referredEntity = referredEntity; + if (this.referredEntity!=null) { + this.referredEntityId = referredEntity.getId(); + } + } + + public void addDeducedTypeVar(VarEntity var) { + this.deducedTypeVars.add(var); + this.deducedTypeVarsId.add(var.getId()); + } + + public void addDeducedTypeFunction(FunctionEntity function) { + this.deducedTypeFunctions.add(function); + this.deducedTypeFunctionsId.add(function.id); + } + + public void setParent(Expression parent) { + this.parent = parent; + if (parent!=null) + this.parentId = parent.id; + if (parent!=null) { + if (parent.deduceTypeBasedId==null) + parent.deduceTypeBasedId = id; + if (parent.isSet) { + parent.deduceTypeBasedId = id; + } + } + } + + + public GenericName getIdentifier() { + return this.identifier; + } + + public GenericName getRawType() { + return this.rawType; + } + + public void setIdentifier(String name) { + if (!validName(name)){ + return; + } + this.identifier = GenericName.build(name); + } + + public void setIdentifier(GenericName name) { + if (name==null) return; + if (!validName(name.getName())){ + return; + } + this.identifier = name; + } + + public void setRawType(GenericName name) { + if (name==null) return; + if (!validName(name.getName())){ + return; + } + this.rawType = name; + + } + + public void setRawType(String name) { + if (name==null) return; + if (!validName(name)){ + return; + } + this.rawType = GenericName.build(name); + } + + public Expression getParent() { + return this.parent; + } + + public void setText(String text) { + this.text = text; + } + + + public boolean isCall() { + return isCall; + } + + public boolean isSet() { + return isSet; + } + + public void setSet(boolean isSet) { + this.isSet = isSet; + } + + public boolean isDot() { + return isDot; + } + + public void setDot(boolean isDot) { + this.isDot = isDot; + } + + public boolean isLogic() { + return isLogic; + } + + public void setLogic(boolean isLogic) { + this.isLogic = isLogic; + } + + public boolean isCreate() { + return isCreate; + } + + public void setCreate(boolean isCreate) { + this.isCreate = isCreate; + } + + public boolean isCast() { + return isCast; + } + + public void setCast(boolean isCast) { + this.isCast = isCast; + } + + public boolean isThrow() { + return isThrow; + } + + public void setThrow(boolean isThrow) { + this.isThrow = isThrow; + } + + public boolean isStatement() { + return isStatement; + } + + public void setStatement(boolean isStatement) { + this.isStatement = isStatement; + } + + public void setCall(boolean isCall) { + this.isCall = isCall; + } + + public void disableDriveTypeFromChild() { + deriveTypeFromChild = false ; + } + + public Entity getReferredEntity() { + return referredEntity; + } + + public TypeEntity getType() { + return type; + } + + + private boolean validName(String name) { + if (name==null) return false; + if (name.toLowerCase().equals("")) return true; + if (name.toLowerCase().equals("")) return true; + return true; + } + + @Override + public String toString() { + StringBuilder s = new StringBuilder(); + s.append("[").append(text).append("]").append("|") + .append("rawType:").append(rawType).append("|") + .append("identifier:").append(identifier).append("|") + .append("prop:").append(isDot?"[dot]":"") + .append(isSet?"[set]":"") + .append(isLogic?"[bool]":"") + .append(isCall?"[call]":"") + .append(isCreate?"[new]":"") + .append(isThrow?"[throw]":"").append("|") + .append("parent:").append(parent==null?"nil":parent.text).append("|") + .append("type:").append(type).append("|"); + return s.toString(); + } + + public void setLine(int lineNumber) { + this.location.setLine(lineNumber); + } + + public Location getLocation() { + return location; + } +} \ No newline at end of file diff --git a/src/main/java/com/educoder/bridge/tmp/ExpressionUsage.java b/src/main/java/com/educoder/bridge/tmp/ExpressionUsage.java new file mode 100644 index 0000000..2d9762a --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/ExpressionUsage.java @@ -0,0 +1,298 @@ +package depends.extractor.python.union; + +import depends.entity.*; +import depends.entity.repo.IdGenerator; +import depends.extractor.HandlerContext; +import depends.extractor.python.PythonHandlerContext; +import depends.extractor.python.PythonParser.*; +import depends.extractor.python.PythonParserBaseVisitor; +import depends.relations.IBindingResolver; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RuleContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +public class ExpressionUsage { + HandlerContext context; + IdGenerator idGenerator; + private boolean exprStarted=false; + private IBindingResolver bindingResolver; + public ExpressionUsage(PythonHandlerContext context, IdGenerator idGenerator, IBindingResolver bindingResolver) { + this.context = context; + this.idGenerator = idGenerator; + this.bindingResolver = bindingResolver; + } + + /** + * Auto deduce variable type from assignment. for example: c = new C() then c is + * type of C + * + */ + private void deduceVarTypeInCaseOfAssignment(Expr_stmtContext expr, Expression expression) { + List names = getName(expr.testlist_star_expr()); + // TODO: should handle list properly; + String varName = null; + if (names.size() == 1) + varName = names.get(0); + if (varName == null) + return; + VarEntity var = context.lastContainer().lookupVarLocally(varName); + if (var != null) { + expression.addDeducedTypeVar(var); + } + } + + private List getName(Testlist_star_exprContext testlist_star_expr) { + List names = new ArrayList<>(); + testlist_star_expr.accept(new NameCollector(names)); + return names; + } + + public void foundExpression(ParserRuleContext ctx) { + if (!isStartOfContainerRule(ctx)) { + return ; + } + if (context.lastContainer().containsExpression(ctx)) return; + if (ctx.getParent() instanceof TrailerContext) return; + + Expression parent = findParentInStack(ctx); + Expression expression = parent; + + if (ctx.getParent().getChildCount()==1 && parent!=null) { + //如果就是自己，则无需创建新的Expression + }else { + /* create expression and link it with parent*/ + expression = new Expression(idGenerator.generateId()); + expression.setLine(ctx.getStart().getLine()); + + expression.setText(ctx.getText()); + context.lastContainer().addExpression(ctx,expression); + expression.setParent(parent); + } + + + if (ctx instanceof Expr_stmtContext) { + Expr_stmtContext exprAssign = (Expr_stmtContext)ctx; + if (exprAssign.assign_part()!=null) { + expression.setSet(true); + expression.setIdentifier(exprAssign.testlist_star_expr().getText()); + if (isValidIdentifier(expression.getIdentifier())) { + makeSureVarExist(expression.getIdentifier(), ctx); + } + deduceVarTypeInCaseOfAssignment((Expr_stmtContext)ctx,expression); + } + } + if (ctx instanceof Raise_stmtContext) { + expression.setThrow (true); + } + if (ctx instanceof Return_stmtContext) { + deduceReturnTypeInCaseOfReturn((Return_stmtContext)ctx,expression); + } + if (ctx instanceof ExprContext) { + processExprContext((ExprContext)ctx, expression); + } + + } + + private void deduceReturnTypeInCaseOfReturn(Return_stmtContext ctx, Expression expression) { + FunctionEntity currentFunction = context.currentFunction(); + if (currentFunction == null) + return; + expression.addDeducedTypeFunction(currentFunction); + } + + + private void makeSureVarExist(GenericName identifier, ParserRuleContext ctx) { + if (null==context.foundEntityWithName(identifier)) { + VarEntity var = context.foundVarDefinition(context.lastContainer(), identifier.getName(),ctx.getStart().getLine()); + var.setLine(ctx.getStart().getLine()); + + } + } + + private boolean isValidIdentifier(GenericName identifier) { + Pattern p = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*"); + Matcher m = p.matcher(identifier.getName()); + return m.matches(); + } + + + + private void processExprContext(ExprContext exprCtx, Expression expression) { + //func_call, member_access, subscript member, and atom + Expression lastExpression = null; + if (exprCtx.atom()!=null) { + //atom + Expression atomExpr = new Expression(idGenerator.generateId()); + atomExpr.setLine(exprCtx.atom().getStart().getLine()); + atomExpr.setParent(expression); + atomExpr.setText(exprCtx.atom().getText()); + atomExpr.setIdentifier(exprCtx.atom().getText()); + context.lastContainer().addExpression(exprCtx.atom(),atomExpr); + processAtom(exprCtx.atom(),atomExpr); + lastExpression = atomExpr; + if (exprCtx.trailer()==null || exprCtx.trailer().size()==0) { + //do nothing; it is just an id; + }else { + for (TrailerContext trailer:exprCtx.trailer()) { + if (trailer.name()!=null) { + Expression trailerExpr = new Expression(idGenerator.generateId()); + trailerExpr.setLine(trailer.getStart().getLine()); + trailerExpr.setText(trailer.getText()); + context.lastContainer().addExpression(trailer,trailerExpr); + trailerExpr.setParent(expression); + + //doted name = member access or method call + trailerExpr.setDot(true);; + trailerExpr.setIdentifier(trailer.name().getText()); + if (trailer.arguments()!=null) { + if (trailer.arguments().OPEN_PAREN()!=null) { + foundCallStyleExpressionWithDot(trailerExpr,lastExpression.getIdentifier(), trailer); + }else { + //subscript list, do nothing + } + } + lastExpression.setParent(trailerExpr); + lastExpression = trailerExpr; + }else { + //direct call, or direct data access + if (trailer.arguments()!=null) { + if (trailer.arguments().OPEN_PAREN()!=null) { + foundCallStyleExpressionWithoutDot(lastExpression, trailer.arguments()); + }else { + //subscript list, do nothing + } + } + } + } + } + }else { +/** expr + | expr op=POWER expr + | op=(ADD | MINUS | NOT_OP) expr + | expr op=(STAR | DIV | MOD | IDIV | AT) expr + | expr op=(ADD | MINUS) expr + | expr op=(LEFT_SHIFT | RIGHT_SHIFT) expr + | expr op=AND_OP expr + | expr op=XOR expr + | expr op=OR_OP expr + ;*/ + + } + } + + + private boolean isStartOfContainerRule(ParserRuleContext ctx) { + if (this.exprStarted) return true; + return ctx instanceof ExprContext || + ctx instanceof Expr_stmtContext || + ctx instanceof Del_stmtContext || + ctx instanceof Return_stmtContext || + ctx instanceof Raise_stmtContext || + ctx instanceof Raise_stmtContext || + ctx instanceof Yield_stmtContext || + ctx instanceof Assert_stmtContext; + } + + + + private void foundCallStyleExpressionWithDot(Expression theExpression, GenericName varName, ParserRuleContext ctx) { + GenericName funcName = theExpression.getIdentifier(); + Entity prefixEntity = context.foundEntityWithName(varName); + if (prefixEntity instanceof VarEntity) { + ((VarEntity) prefixEntity).addFunctionCall(funcName); + } + Entity typeEntity = context.foundEntityWithName(funcName); + if (typeEntity instanceof TypeEntity && typeEntity.getId() > 0) { + theExpression.setCreate(true); + theExpression.setType(typeEntity.getType(), typeEntity, bindingResolver); + theExpression.setRawType(typeEntity.getRawName()); + return; + } + theExpression.setCall(true); + } + + + + private void foundCallStyleExpressionWithoutDot(Expression theExpression, ParserRuleContext ctx) { + GenericName funcName = theExpression.getIdentifier(); + Entity typeEntity = context.foundEntityWithName(funcName); + if (typeEntity instanceof TypeEntity && typeEntity.getId() > 0) { + theExpression.getParent().setCreate(true); + theExpression.setType(typeEntity.getType(), typeEntity, bindingResolver); + theExpression.getParent().setRawType(typeEntity.getRawName()); + return; + } + theExpression.setCall(true); + } + + + private void processAtom(AtomContext atom, Expression expression) { + if (atom.name()!=null) { + expression.setIdentifier(atom.getText()); + return; + } + if (atom.STRING()!=null + || atom.NONE()!=null + || atom.number()!=null) { + expression.setRawType(""); + expression.setIdentifier(""); + return; + } + + if (atom.EXEC()!=null + || atom.PRINT()!=null + || atom.ELLIPSIS()!=null) { + return; + } +// : OPEN_PAREN (yield_expr | testlist_comp)? CLOSE_PAREN +// | OPEN_BRACKET testlist_comp? CLOSE_BRACKET +// | OPEN_BRACE dictorsetmaker? CLOSE_BRACE +// | REVERSE_QUOTE testlist COMMA? REVERSE_QUOTE + return; + } + + + + + + private Expression findParentInStack(RuleContext ctx) { + if (ctx==null) return null; + if (ctx.parent==null) return null; + if (context.lastContainer()==null) { + return null; + } + if (context.lastContainer().expressions().containsKey(ctx.parent)) + return context.lastContainer().expressions().get(ctx.parent); + return findParentInStack(ctx.parent); + } + + + + public void startExpr() { + this.exprStarted = true; + } + + + + public void stopExpr() { + this.exprStarted = false; + } + + +} + +class NameCollector extends PythonParserBaseVisitor{ + private List names; + NameCollector(List names){ + this.names = names; + } + @Override + public Void visitAtom(AtomContext ctx) { + if (ctx.name()!=null) + names.add(ctx.name().getText()); + return super.visitAtom(ctx); + } +} \ No newline at end of file diff --git a/src/main/java/com/educoder/bridge/tmp/ExtractText.java b/src/main/java/com/educoder/bridge/tmp/ExtractText.java new file mode 100644 index 0000000..4ba8951 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/ExtractText.java @@ -0,0 +1,129 @@ +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.lang.StringBuffer; + + +// https://svn.apache.org/repos/asf/poi/trunk/src/ooxml/testcases/org/apache/poi/extractor/TestExtractorFactory.java +import org.apache.poi.POIOLE2TextExtractor; +import org.apache.poi.POITextExtractor; +//import org.apache.poi.POIDataSamples; +//import org.apache.poi.extractor.*; +import org.apache.poi.extractor.ExtractorFactory; +import org.apache.poi.hdgf.extractor.VisioTextExtractor; +import org.apache.poi.hpbf.extractor.PublisherTextExtractor; +import org.apache.poi.hslf.extractor.PowerPointExtractor; +import org.apache.poi.hsmf.extractor.OutlookTextExtactor; +import org.apache.poi.hssf.extractor.EventBasedExcelExtractor; +import org.apache.poi.hssf.extractor.ExcelExtractor; +import org.apache.poi.hwpf.extractor.Word6Extractor; +import org.apache.poi.hwpf.extractor.WordExtractor; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.xslf.extractor.XSLFPowerPointExtractor; +import org.apache.poi.xssf.extractor.XSSFEventBasedExcelExtractor; +import org.apache.poi.xssf.extractor.XSSFExcelExtractor; +import org.apache.poi.xwpf.extractor.XWPFWordExtractor; + +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.apache.poi.poifs.filesystem.OfficeXmlFileException; + +import org.apache.poi.xslf.usermodel.XMLSlideShow; // pptx 2007, http://poi.apache.org/apidocs/org/apache/poi/xslf/ +import org.apache.poi.xwpf.usermodel.XWPFDocument; // docx 2007, http://poi.apache.org/apidocs/org/apache/poi/xwpf/ +import org.apache.poi.xssf.usermodel.XSSFWorkbook; // xlsx 2007, http://poi.apache.org/apidocs/org/apache/poi/xssf/ + +class ExtractText +{ + public static String file(String path) { + try { return pptx(new FileInputStream(path)); } catch(Exception e) { } + try { return docx(new FileInputStream(path)); } catch(Exception e) { } + try { return xlsx(new FileInputStream(path)); } catch(Exception e) { } + return ""; + } + public static String pptx(InputStream in) throws Exception { + XSLFPowerPointExtractor o = new XSLFPowerPointExtractor( new XMLSlideShow(in) ); + o.setSlidesByDefault(true); + o.setNotesByDefault(true); + return o.getText(); + } + public static String docx(InputStream in) throws Exception { + XWPFWordExtractor o = new XWPFWordExtractor(new XWPFDocument(in)); + return o.getText(); + } + public static String xlsx(InputStream in) throws Exception { + XSSFExcelExtractor o = new XSSFExcelExtractor(new XSSFWorkbook(in)); + return o.getText(); + } + public static void main(String argv[]) { + try { + InputStream in = null; + if (argv.length < 1) + in = System.in; + else + in = new FileInputStream(argv[0]); + StringBuffer output = new StringBuffer(); + POITextExtractor textExtractor = ExtractorFactory.createExtractor(in); + + if (textExtractor instanceof ExcelExtractor) // xls, excel 97-2003 + { + ExcelExtractor extractor = (ExcelExtractor) textExtractor; + output.append(extractor.getText()); + } + else if (textExtractor instanceof XSSFExcelExtractor) // xlsx, excel 2007 + { + XSSFExcelExtractor extractor = (XSSFExcelExtractor) textExtractor; + output.append(extractor.getText()); + } + else if (textExtractor instanceof Word6Extractor) // doc, word 95 + { + Word6Extractor extractor = (Word6Extractor) textExtractor; + output.append(extractor.getText()); + } + else if (textExtractor instanceof WordExtractor) // doc, word 97-2003 + { + WordExtractor extractor = (WordExtractor) textExtractor; + output.append(extractor.getText()); + } + else if (textExtractor instanceof XWPFWordExtractor) // docx, word 2007 + { + XWPFWordExtractor extractor = (XWPFWordExtractor) textExtractor; + output.append(extractor.getText()); + } + else if (textExtractor instanceof PowerPointExtractor) // ppt, ppt 97-2003 + { + PowerPointExtractor extractor = (PowerPointExtractor) textExtractor; + output.append(extractor.getText()); + output.append(extractor.getNotes()); + } + else if (textExtractor instanceof XSLFPowerPointExtractor ) // pptx, powerpoint 2007 + { + XSLFPowerPointExtractor extractor = (XSLFPowerPointExtractor) textExtractor; + extractor.setSlidesByDefault(true); + extractor.setNotesByDefault(true); + output.append(extractor.getText()); + } + else if (textExtractor instanceof VisioTextExtractor) // vsd, visio + { + VisioTextExtractor extractor = (VisioTextExtractor) textExtractor; + output.append(extractor.getText()); + } + else if (textExtractor instanceof PublisherTextExtractor) // pub, publisher + { + PublisherTextExtractor extractor = (PublisherTextExtractor) textExtractor; + output.append(extractor.getText()); + } + else if (textExtractor instanceof OutlookTextExtactor) // msg, outlook + { + OutlookTextExtactor extractor = (OutlookTextExtactor) textExtractor; + output.append(extractor.getText()); + } + System.out.println(output.toString().replaceAll( "[\n\t\r ]+"," ")); + } + catch (Exception e) + { + // TODO Auto-generated catch block + //e.printStackTrace(); + //System.out.println(e); + } + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/FileEntity.java b/src/main/java/com/educoder/bridge/tmp/FileEntity.java new file mode 100644 index 0000000..8765a3a --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/FileEntity.java @@ -0,0 +1,169 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.importtypes.Import; +import depends.relations.IBindingResolver; + +import java.util.*; + +public class FileEntity extends TypeEntity { + private List importedNames = new ArrayList<>(); + private boolean isInProjectScope = false; + private Collection importedRelationEntities = new ArrayList<>(); + private Collection importedFiles = new ArrayList<>(); + private Collection importedTypes = new ArrayList<>(); + private List declaredTypes = new ArrayList<>(); + private ImportedFileCollector importedFileCollector = null; + public FileEntity() {} + + public FileEntity(String fullName, int fileId, boolean isInProjectScope) { + super(GenericName.build(fullName), null, fileId); + setQualifiedName(fullName); + this.isInProjectScope = isInProjectScope; + } + + public FileEntity(String fullName, int fileId) { + this(fullName, fileId, true); + } + + public void addImport(Import imported) { + if (!importedNames.contains(imported)) + importedNames.add(imported); + } + + /** + * To match the imported name by suffix + * for example: + * import a.b.ClassX; + * the b.ClassX, ClassX , a.b.classX should be matched + * @param lastName + * @return + */ + public String importedSuffixMatch(String lastName) { + if (!lastName.startsWith(".")) + lastName = "." + lastName; + for (Entity imported : this.importedTypes) { + String name = imported.getQualifiedName(true); + if (!name.startsWith(".")) + name = "." + name; + if (imported.getQualifiedName(true).endsWith(lastName)) + return imported.getQualifiedName(true); + } + return null; + } + + + @Override + public String getQualifiedName(boolean overrideFileWithPackage) { + if (!overrideFileWithPackage) { + return super.getQualifiedName(); + } + if (this.getParent() == null) { + return ""; + } + if (this.getParent() instanceof PackageEntity) + return this.getParent().getQualifiedName(); + else + return super.getQualifiedName(); + } + + @Override + public void inferLocalLevelEntities(IBindingResolver bindingResolver) { + this.importedRelationEntities = bindingResolver.getImportedRelationEntities(importedNames); + this.importedTypes = bindingResolver.getImportedTypes(importedNames,this); + this.importedFiles = bindingResolver.getImportedFiles(importedNames); + + super.inferLocalLevelEntities(bindingResolver); + } + + public boolean isInProjectScope() { + return isInProjectScope; + } + + public void setInProjectScope(boolean isInProjectScope) { + this.isInProjectScope = isInProjectScope; + } + + public Collection getImportedRelationEntities() { + return importedRelationEntities; + } + + public Collection getImportedFiles() { + return importedFiles; + } + + public Collection getImportedTypes() { + return importedTypes; + } + + public List getDeclaredTypes() { + return this.declaredTypes; + } + + public void addType(TypeEntity currentTypeEntity) { + this.declaredTypes.add(currentTypeEntity); + } + + public Set getImportedFilesInAllLevel() { + if (importedFileCollector==null) + importedFileCollector = new ImportedFileCollector(this); + + return importedFileCollector.getFiles(); + } + + public List getImportedNames() { + return importedNames; + } + + public void cacheAllExpressions() { + this.cacheChildExpressions(); + } + + + @Override + public Entity getByName(String name, HashSet searched) { + Entity entity = super.getByName(name, searched); + if (entity!=null) return entity; + for (TypeEntity type:getDeclaredTypes()) { + if (type.getRawName().getName().equals(name)|| + suffixMatch(name,type.getQualifiedName())) { + return type; + } + } + return null; + } + + private boolean suffixMatch(String name, String qualifiedName) { + if (qualifiedName.contains(".")) { + if (!name.startsWith(".")) name = "." +name; + return qualifiedName.endsWith(name); + } + else { + return qualifiedName.equals(name); + } + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/FormalParameterListContextHelper.java b/src/main/java/com/educoder/bridge/tmp/FormalParameterListContextHelper.java new file mode 100644 index 0000000..4c24ca9 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/FormalParameterListContextHelper.java @@ -0,0 +1,99 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.java.context; + +import java.util.ArrayList; +import java.util.List; + +import org.antlr.v4.runtime.tree.TerminalNode; + +import depends.entity.FunctionEntity; +import depends.entity.GenericName; +import depends.entity.VarEntity; +import depends.entity.repo.IdGenerator; +import depends.extractor.java.JavaParser.FormalParameterContext; +import depends.extractor.java.JavaParser.FormalParameterListContext; +import depends.extractor.java.JavaParser.FormalParametersContext; +import depends.extractor.java.JavaParser.LastFormalParameterContext; +import depends.extractor.java.JavaParser.TypeTypeContext; +import depends.extractor.java.JavaParser.VariableModifierContext; + +public class FormalParameterListContextHelper { + + FormalParameterListContext context; + private IdGenerator idGenerator; + private List annotations; + private FunctionEntity container; + + public FormalParameterListContextHelper(FormalParameterListContext formalParameterListContext,FunctionEntity container, IdGenerator idGenerator) { + this.context = formalParameterListContext; + this.container = container; + annotations = new ArrayList<>(); + this.idGenerator = idGenerator; + if (context!=null) + extractParameterTypeList(); + } + + public FormalParameterListContextHelper(FormalParametersContext formalParameters,FunctionEntity container, IdGenerator idGenerator) { + this(formalParameters.formalParameterList(),container,idGenerator); + } + + + + public void extractParameterTypeList() { + if (context != null) { + if (context.formalParameter() != null) { + for (FormalParameterContext p : context.formalParameter()) { + foundParameterDefintion(p.typeType(),p.variableDeclaratorId().IDENTIFIER(),p.variableModifier()); + } + if (context.lastFormalParameter()!=null) { + LastFormalParameterContext p = context.lastFormalParameter(); + foundParameterDefintion(p.typeType(),p.variableDeclaratorId().IDENTIFIER(),p.variableModifier()); + } + } + } + return; + } + + private void foundParameterDefintion(TypeTypeContext typeType, TerminalNode identifier, List variableModifier) { + GenericName type = GenericName.build(ClassTypeContextHelper.getClassName(typeType)); + GenericName varName = GenericName.build(identifier.getText()); + VarEntity varEntity = new VarEntity(varName,type,container,idGenerator.generateId()); + container.addParameter(varEntity); + + for ( VariableModifierContext modifier:variableModifier) { + if (modifier.annotation()!=null) { + this.annotations.add(QualitiedNameContextHelper.getName(modifier.annotation().qualifiedName())); + } + } + + } + + public List getAnnotations() { + return annotations; + } + + +} diff --git a/src/main/java/com/educoder/bridge/tmp/FunctionEntity.java b/src/main/java/com/educoder/bridge/tmp/FunctionEntity.java new file mode 100644 index 0000000..91f7e7a --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/FunctionEntity.java @@ -0,0 +1,149 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.relations.IBindingResolver; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class FunctionEntity extends ContainerEntity{ + private List returnTypeIdentifiers = new ArrayList<>(); + Collection parameters; + Collection throwTypesIdentifiers = new ArrayList<>(); + private Collection returnTypes = new ArrayList<>(); + private Collection throwTypes = new ArrayList<>(); + public FunctionEntity() { + this.parameters = new ArrayList<>(); + } + public FunctionEntity(GenericName simpleName, Entity parent, Integer id, GenericName returnType) { + super(simpleName, parent,id); + this.returnTypes = new ArrayList<>(); + returnTypeIdentifiers = new ArrayList<>(); + this.parameters = new ArrayList<>(); + throwTypesIdentifiers = new ArrayList<>(); + addReturnType(returnType); + } + public Collection getReturnTypes() { + return returnTypes; + } + + @Override + public TypeEntity getType() { + if (returnTypes.size()>0){ + Object type = returnTypes.iterator().next(); + if (type instanceof TypeEntity) + return (TypeEntity)type; + } + return null; + } + + public void addReturnType(GenericName returnType) { + if (returnType==null) return; + this.returnTypeIdentifiers.add(returnType); + } + + public void addReturnType(TypeEntity returnType) { + if (returnType==null) return; + if (!this.returnTypeIdentifiers.contains(returnType.rawName)){ + this.returnTypeIdentifiers.add(returnType.rawName); + this.returnTypes.add(returnType); + } + } + + public void addThrowTypes(List throwedType) { + throwTypesIdentifiers.addAll(throwedType); + } + + @Override + public void inferLocalLevelEntities(IBindingResolver bindingResolver) { + for (VarEntity param:parameters) { + param.fillCandidateTypes(bindingResolver); + param.inferLocalLevelEntities(bindingResolver); + } + if (returnTypes.size() typeEntities = typeParametersToEntities(bindingResolver, returnTypeName); + this.appendTypeParameters(typeEntities); + } + } + if (throwTypes.size() typeParametersToEntities(IBindingResolver bindingResolver, GenericName name) { + ArrayList r = new ArrayList<>(); + for (GenericName typeParameter:name.getArguments()) { + toEntityList(bindingResolver, r,typeParameter); + } + return r; + } + + + public Collection getParameters() { + return parameters; + } + public Collection getThrowTypes() { + return throwTypes; + } + @Override + public Entity lookupVarInVisibleScope(GenericName varName) { + for (VarEntity param:parameters) { + if (varName.equals(param.getRawName())) { + return param; + } + } + return super.lookupVarInVisibleScope(varName); + } + public void addParameter(VarEntity var) { + this.parameters.add(var); + } + @Override + public String getDisplayName() { + FileEntity f = (FileEntity) this.getAncestorOfType(FileEntity.class); + return f.getRawName()+"("+this.getQualifiedName()+")"; + } + @Override + public VarEntity lookupVarLocally(GenericName varName) { + for (VarEntity var:this.parameters) { + if (var.getRawName().equals(varName)) + return var; + } + return super.lookupVarLocally(varName); + } + + public void linkReturnToLastExpression() { + if (expressionList()==null) return; + for (int i = expressionList().size() - 1; i >= 0; i--) { + Expression expr = expressionList().get(i); + if (expr.isStatement()) + expr.addDeducedTypeFunction(this); + } + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/GenericName.java b/src/main/java/com/educoder/bridge/tmp/GenericName.java new file mode 100644 index 0000000..1645f52 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/GenericName.java @@ -0,0 +1,117 @@ +package depends.entity; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class GenericName implements Serializable{ + private static final long serialVersionUID = 1L; + private char[] name; + List arguments; + public GenericName(String name) { + this.name = name.toCharArray(); + } + public GenericName(String name, List arguments) { + this.name = name.toCharArray(); + this.arguments = arguments; + } + public boolean contains(String rawType) { + if (new String(name).contains(rawType)) return true; + return false; + } + public String getName() { + return new String(name); + } + public List getArguments() { + if (arguments==null) return new ArrayList<>(); + return arguments; + } + + @Override + public String toString() { + return new String(name) + (getArguments().size()>0?"(" + arguments + ")":""); + } + + public GenericName replace(String from, String to) { + name = new String(name).replace(from, to).toCharArray(); + for (GenericName arg:getArguments()) { + arg.replace(from, to); + } + return this; + } + + public boolean startsWith(String prefix) { + if (name==null) return false; + return new String(name).startsWith(prefix); + } + public String uniqName() { + if (getArguments().size()==0) return new String(name); + StringBuffer sb = new StringBuffer(); + sb.append(name); + if (getArguments().size()>0) { + for (GenericName arg:getArguments()) { + sb.append("__").append(arg.uniqName()).append("__"); + } + } + return sb.toString(); + } + public GenericName substring(int start) { + return new GenericName(new String(this.name).substring(start)); + } + public boolean isNull() { + return name==null; + } + public static GenericName build(String name) { + if (name==null) return null; + return new GenericName(name); + } + public static GenericName build(String name, List arguments) { + return new GenericName(name,arguments); + } + public boolean find(GenericName rawType) { + //if (this.equals(rawType)) return true; + for (GenericName subType:this.getArguments()) { + if (subType.equals(rawType)) return true; + boolean found = subType.find(rawType); + if (found) return true; + } + return false; + } + public void appendArguments(List parameters) { + if (this.arguments==null) this.arguments = new ArrayList<>(); + this.arguments.addAll(parameters); + } + public void appendArguments(GenericName parameter) { + if (this.arguments==null) this.arguments = new ArrayList<>(); + this.arguments.add(parameter); + } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((arguments == null) ? 0 : arguments.hashCode()); + result = prime * result + Arrays.hashCode(name); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + GenericName other = (GenericName) obj; + if (this.getArguments() == null) { + if (other.getArguments() != null) + return false; + } else if (!getArguments().equals(other.getArguments())) + return false; + if (!Arrays.equals(name, other.name)) + return false; + return true; + } + + +} diff --git a/src/main/java/com/educoder/bridge/tmp/GoImportLookupStrategy.java b/src/main/java/com/educoder/bridge/tmp/GoImportLookupStrategy.java new file mode 100644 index 0000000..cda04d7 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/GoImportLookupStrategy.java @@ -0,0 +1,103 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.golang; + +import depends.entity.Entity; +import depends.entity.FileEntity; +import depends.entity.PackageEntity; +import depends.entity.repo.EntityRepo; +import depends.extractor.UnsolvedBindings; +import depends.importtypes.Import; +import depends.relations.ImportLookupStrategy; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class GoImportLookupStrategy extends ImportLookupStrategy{ + public GoImportLookupStrategy(EntityRepo repo) { + super(repo); + } + + public Entity lookupImportedType(String name, FileEntity fileEntity) { + //Java Strategy + String importedString = fileEntity.importedSuffixMatch(name); + if (importedString==null) return null; + return repo.getEntity(importedString); + } + + + @Override + public List getImportedRelationEntities(List importedList) { + ArrayList result = new ArrayList<>(); + for (Import importedItem:importedList) { + Entity imported = repo.getEntity(importedItem.getContent()); + if (imported==null) continue; + if (imported instanceof PackageEntity) { + //ignore wildcard import relation + }else { + result.add(imported); + } + } + return result; + } + + @Override + public List getImportedTypes(List importedList, Set unsolvedBindings) { + ArrayList result = new ArrayList<>(); + for (Import importedItem:importedList) { + Entity imported = repo.getEntity(importedItem.getContent()); + if (imported==null) { + unsolvedBindings.add(new UnsolvedBindings(importedItem.getContent(),null)); + continue; + } + if (imported instanceof PackageEntity) { + //expand import of package to all classes under the package due to we dis-courage the behavior + for (Entity child:imported.getChildren()) { + if (child instanceof FileEntity) { + child.getChildren().forEach(item->result.add(item)); + }else { + result.add(child); + } + } + }else { + result.add(imported); + } + } + return result; + } + + @Override + public List getImportedFiles(List importedList) { + return new ArrayList(); + } + + + @Override + public boolean supportGlobalNameLookup() { + return true; + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/GoListener.java b/src/main/java/com/educoder/bridge/tmp/GoListener.java new file mode 100644 index 0000000..b8fafe7 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/GoListener.java @@ -0,0 +1,136 @@ +package depends.extractor.golang; + +import depends.entity.FunctionEntity; +import depends.entity.GenericName; +import depends.entity.VarEntity; +import depends.entity.repo.EntityRepo; +import depends.relations.IBindingResolver; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayList; + +public class GoListener extends GoParserBaseListener { + private final EntityRepo entityRepo; + GoHandlerContext context; + @Override + public void enterSourceFile(GoParser.SourceFileContext ctx) { + super.enterSourceFile(ctx); + } + + public GoListener(String fileFullPath, EntityRepo entityRepo, IBindingResolver bindingResolver) { + context = new GoHandlerContext(entityRepo, bindingResolver); + context.startFile(fileFullPath); + this.entityRepo = entityRepo; + } + + @Override + public void enterFunctionDecl(GoParser.FunctionDeclContext ctx) { + String funcName = ctx.IDENTIFIER().getText(); + context.foundMethodDeclarator(funcName,ctx.getStart().getLine()); + foundFuncSignature(ctx.signature()); + super.enterFunctionDecl(ctx); + } + + @Override + public void exitFunctionDecl(GoParser.FunctionDeclContext ctx) { + context.exitLastedEntity(); + super.exitFunctionDecl(ctx); + } + + @Override + public void enterPackageClause(GoParser.PackageClauseContext ctx) { + context.foundPackageDeclaration(ctx.IDENTIFIER().getText()); + super.enterPackageClause(ctx); + } + + private void foundFuncSignature(GoParser.SignatureContext signature) { + FunctionEntity func = (FunctionEntity) context.lastContainer(); + if (signature.parameters()!=null) { + for (GoParser.ParameterDeclContext param : signature.parameters().parameterDecl()) { + if (param.identifierList() != null) { + TypeDefHelper typeDefHelper = new TypeDefHelper(param.type_()); + for (TerminalNode id : param.identifierList().IDENTIFIER()) { + VarEntity varEntity = new VarEntity(GenericName.build(id.getText()), + GenericName.build(typeDefHelper.getTypeRefName()), context.lastContainer(), + entityRepo.generateId()); + func.addParameter(varEntity); + } + } else/* with ... parameters*/ { + + } + } + } + if (signature.result()!=null){ + if(signature.result().parameters()!=null){ + for (GoParser.ParameterDeclContext paramDecl:signature.result().parameters().parameterDecl()){ + TypeDefHelper typeDefHelper = new TypeDefHelper(paramDecl.type_()); + if (typeDefHelper.isTypeRef()) { + func.addReturnType(GenericName.build(typeDefHelper.getTypeRefName())); + }else{ + System.err.println("TODO: unsupport return type"); + } + } + } + if (signature.result().type_()!=null){ + TypeDefHelper typeDefHelper = new TypeDefHelper(signature.result().type_()); + if (typeDefHelper.isTypeRef()) { + func.addReturnType(GenericName.build(typeDefHelper.getTypeRefName())); + }else{ + System.err.println("TODO: unsupport return type"); + } + } + System.err.println(signature.result().getText()); + } + } + + @Override + public void enterTypeSpec(GoParser.TypeSpecContext ctx) { + TypeSpecHelper specHelper = new TypeSpecHelper(ctx); + if (specHelper.getTypeDefHelper().isTypeRef()){ + context.foundNewAlias(specHelper.getIdentifier(),specHelper.getTypeDefHelper().getTypeRefName()); + }else if (specHelper.getTypeDefHelper().isStruct()){ + context.foundNewType(specHelper.getIdentifier(),ctx.getStart().getLine()); + } + super.enterTypeSpec(ctx); + } + + @Override + public void exitTypeSpec(GoParser.TypeSpecContext ctx) { + TypeSpecHelper specHelper = new TypeSpecHelper(ctx); + if (specHelper.getTypeDefHelper().isStruct()){ + context.exitLastedEntity(); + } + super.exitTypeSpec(ctx); + } + +// fieldDecl +// : ({noTerminatorBetween(2)}? identifierList type_ | anonymousField) string_? +// ; + + @Override + public void enterFieldDecl(GoParser.FieldDeclContext ctx) { + FieldDeclHelper fieldDeclHelper = new FieldDeclHelper(ctx); + String typeName = fieldDeclHelper.getTypeName(); + + if (fieldDeclHelper.getIdentifiers()!=null){ + for (String id:fieldDeclHelper.getIdentifiers()){ + if (typeName==null){ + System.err.println("TODO: unsupport fieldDecl without type"); + } + context.foundVarDefinition(id, GenericName.build(typeName),new ArrayList<>(),ctx.getStart().getLine()); + } + } + + if (fieldDeclHelper.getAnonymousField()!=null){ + System.err.println("TODO: unsupport anonymousField"); + } + if (fieldDeclHelper.getString()!=null){ + System.err.println("TODO: unsupport field with string " + fieldDeclHelper.getString()); + } + super.enterFieldDecl(ctx); + } + + public void done() { + + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/GoParserBase.java b/src/main/java/com/educoder/bridge/tmp/GoParserBase.java new file mode 100644 index 0000000..9f3c099 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/GoParserBase.java @@ -0,0 +1,132 @@ +package depends.extractor.golang; + +import org.antlr.v4.runtime.*; + +import java.util.List; + +/** + * All parser methods that used in grammar (p, prev, notLineTerminator, etc.) + * should start with lower case char similar to parser rules. + */ +public abstract class GoParserBase extends Parser +{ + protected GoParserBase(TokenStream input) { + super(input); + } + + /** + * Returns {@code true} iff on the current index of the parser's + * token stream a token exists on the {@code HIDDEN} channel which + * either is a line terminator, or is a multi line comment that + * contains a line terminator. + * + * @return {@code true} iff on the current index of the parser's + * token stream a token exists on the {@code HIDDEN} channel which + * either is a line terminator, or is a multi line comment that + * contains a line terminator. + */ + protected boolean lineTerminatorAhead() { + // Get the token ahead of the current index. + int possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 1; + + if (possibleIndexEosToken == -1) + { + return true; + } + + Token ahead = _input.get(possibleIndexEosToken); + if (ahead.getChannel() != Lexer.HIDDEN) { + // We're only interested in tokens on the HIDDEN channel. + return false; + } + + if (ahead.getType() == GoLexer.TERMINATOR) { + // There is definitely a line terminator ahead. + return true; + } + + if (ahead.getType() == GoLexer.WS) { + // Get the token ahead of the current whitespaces. + possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 2; + ahead = _input.get(possibleIndexEosToken); + } + + // Get the token's text and type. + String text = ahead.getText(); + int type = ahead.getType(); + + // Check if the token is, or contains a line terminator. + return (type == GoLexer.COMMENT && (text.contains("\r") || text.contains("\n"))) || + (type == GoLexer.TERMINATOR); + } + + /** + * Returns {@code true} if no line terminator exists between the specified + * token offset and the prior one on the {@code HIDDEN} channel. + * + * @return {@code true} if no line terminator exists between the specified + * token offset and the prior one on the {@code HIDDEN} channel. + */ + protected boolean noTerminatorBetween(int tokenOffset) { + BufferedTokenStream stream = (BufferedTokenStream)_input; + List tokens = stream.getHiddenTokensToLeft(stream.LT(tokenOffset).getTokenIndex()); + + if (tokens == null) { + return true; + } + + for (Token token : tokens) { + if (token.getText().contains("\n")) + return false; + } + + return true; + } + + /** + * Returns {@code true} if no line terminator exists after any encounterd + * parameters beyond the specified token offset and the next on the + * {@code HIDDEN} channel. + * + * @return {@code true} if no line terminator exists after any encounterd + * parameters beyond the specified token offset and the next on the + * {@code HIDDEN} channel. + */ + protected boolean noTerminatorAfterParams(int tokenOffset) { + BufferedTokenStream stream = (BufferedTokenStream)_input; + int leftParams = 1; + int rightParams = 0; + int valueType; + + if (stream.LT(tokenOffset).getType() == GoLexer.L_PAREN) { + // Scan past parameters + while (leftParams != rightParams) { + tokenOffset++; + valueType = stream.LT(tokenOffset).getType(); + + if (valueType == GoLexer.L_PAREN){ + leftParams++; + } + else if (valueType == GoLexer.R_PAREN) { + rightParams++; + } + } + + tokenOffset++; + return noTerminatorBetween(tokenOffset); + } + + return true; + } + + protected boolean checkPreviousTokenText(String text) + { + BufferedTokenStream stream = (BufferedTokenStream)_input; + String prevTokenText = stream.LT(1).getText(); + + if (prevTokenText == null) + return false; + + return prevTokenText.equals(text); + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/HandlerContext.java b/src/main/java/com/educoder/bridge/tmp/HandlerContext.java new file mode 100644 index 0000000..a18547c --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/HandlerContext.java @@ -0,0 +1,333 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor; + +import depends.entity.*; +import depends.entity.repo.EntityRepo; +import depends.entity.repo.IdGenerator; +import depends.importtypes.Import; +import depends.relations.IBindingResolver; + +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; +import java.util.stream.Collectors; + +public abstract class HandlerContext { + protected EntityRepo entityRepo; + protected IdGenerator idGenerator; + + protected FileEntity currentFileEntity; + protected IBindingResolver bindingResolver; + + + public HandlerContext(EntityRepo entityRepo, IBindingResolver bindingResolver) { + this.entityRepo = entityRepo; + this.idGenerator = entityRepo; + entityStack = new Stack(); + this.bindingResolver = bindingResolver; + } + + public FileEntity startFile(String fileName) { + currentFileEntity = new FileEntity(fileName, idGenerator.generateId(),true); + pushToStack(currentFileEntity); + addToRepo(currentFileEntity); + return currentFileEntity; + } + + public TypeEntity foundNewType(GenericName name, Integer startLine) { + TypeEntity currentTypeEntity = new TypeEntity(name, this.latestValidContainer(), + idGenerator.generateId()); + currentTypeEntity.setLine(startLine); + pushToStack(currentTypeEntity); + addToRepo(currentTypeEntity); + currentFileEntity.addType(currentTypeEntity); + return currentTypeEntity; + } + + /** + * Tell the context object that a new type founded. + * @param name + * @param startLine + * @return + */ + public TypeEntity foundNewType(String name, Integer startLine) { + return foundNewType(GenericName.build(name),startLine); + } + + public AliasEntity foundNewAlias(String aliasName, String originalName) { + if (aliasName.equals(originalName)) return null; //it is a tricky, we treat same name no different. + //indeed it is not perfect -> the right match should depends on no-bare format like "struct a" instead of "a" + AliasEntity currentTypeEntity = new AliasEntity(GenericName.build(aliasName), this.latestValidContainer(), + idGenerator.generateId(),GenericName.build(originalName) ); + addToRepo(currentTypeEntity); + return currentTypeEntity; + } + + public AliasEntity foundNewAlias(GenericName aliasName, Entity referToEntity) { + AliasEntity currentTypeEntity = new AliasEntity(aliasName, this.latestValidContainer(), + idGenerator.generateId(),aliasName); + currentTypeEntity.setReferToEntity(referToEntity); + addToRepo(currentTypeEntity); + return currentTypeEntity; + } + + /** + * Tell the context that a new method was found. + * Do not forget to tell the context leave the method when you finish + * the process of the method + * @param methodName + * @param returnType - if no return type information avaliable, keep it as null; + * @param throwedType - if no throwed type information avaliable, keep it as empty list; + * @return the new function enity + */ + public FunctionEntity foundMethodDeclarator(String methodName, String returnType, List throwedType, Integer startLine) { + FunctionEntity functionEntity = new FunctionEntity(GenericName.build(methodName), this.latestValidContainer(), + idGenerator.generateId(),GenericName.build(returnType)); + functionEntity.setLine(startLine); + addToRepo(functionEntity); + this.typeOrFileContainer().addFunction(functionEntity); + pushToStack(functionEntity); + functionEntity.addThrowTypes(throwedType.stream().map(item->GenericName.build(item)).collect(Collectors.toList())); + return functionEntity; + } + + public FunctionEntity foundMethodDeclarator(String methodName, Integer startLine) { + FunctionEntity functionEntity = new FunctionEntity(GenericName.build(methodName), this.latestValidContainer(), + idGenerator.generateId(),null); + functionEntity.setLine(startLine); + addToRepo(functionEntity); + this.typeOrFileContainer().addFunction(functionEntity); + pushToStack(functionEntity); + return functionEntity; + } + + + public FunctionEntity foundMethodDeclarator(ContainerEntity containerEntity, String methodName, Integer startLine) { + FunctionEntity functionEntity = new FunctionEntity(GenericName.build(methodName), containerEntity, + idGenerator.generateId(),null); + functionEntity.setLine(startLine); + addToRepo(functionEntity); + containerEntity.addFunction(functionEntity); + pushToStack(functionEntity); + functionEntity.addThrowTypes(new ArrayList<>()); + return functionEntity; + } + + public void foundNewImport(Import imported) { + currentFileEntity.addImport(imported); + } + + public TypeEntity currentType() { + for (int i = entityStack.size() - 1; i >= 0; i--) { + Entity t = entityStack.get(i); + if (t instanceof TypeEntity) + return (TypeEntity) t; + } + return null; + } + + public ContainerEntity typeOrFileContainer() { + for (int i = entityStack.size() - 1; i >= 0; i--) { + Entity t = entityStack.get(i); + if (t instanceof TypeEntity) + return (ContainerEntity) t; + if (t instanceof FileEntity) { + return (ContainerEntity)t; + } + } + return null; + } + + + public FunctionEntity currentFunction() { + for (int i = entityStack.size() - 1; i >= 0; i--) { + Entity t = entityStack.get(i); + if (t instanceof FunctionEntity) + return (FunctionEntity) t; + } + return null; + } + + public FileEntity currentFile() { + return currentFileEntity; + } + + public ContainerEntity globalScope() { + Entity global = entityRepo.getEntity(EntityRepo.GLOBAL_SCOPE_NAME); + if (global==null) { + global = new PackageEntity(EntityRepo.GLOBAL_SCOPE_NAME,idGenerator.generateId()); + addToRepo(global); + } + return (ContainerEntity)global; + } + + public Entity latestValidContainer() { + for (int i = entityStack.size() - 1; i >= 0; i--) { + Entity t = entityStack.get(i); + if (t instanceof FunctionEntity) + return t; + if (t instanceof TypeEntity) + return t; + if (t instanceof FileEntity) + return t; + } + return null; + } + + public ContainerEntity lastContainer() { + for (int i = entityStack.size() - 1; i >= 0; i--) { + Entity t = entityStack.get(i); + if (t instanceof ContainerEntity) + return (ContainerEntity) t; + } + return null; + } + + public void foundImplements(GenericName typeName) { + currentType().addImplements(typeName); + } + + public void foundExtends(String className) { + foundExtends(GenericName.build(className)); + } + + public void foundExtends(GenericName typeName) { + if (currentType()==null) { + System.out.println("error: type do not exist"); + return ; + } + currentType().addExtends(typeName); + } + + public void foundMixin(String name) { + foundMixin(GenericName.build(name)); + + } + + public void foundMixin(GenericName name) { + lastContainer().addMixin(name); + + } + + public void foundTypeParametes(GenericName typeName) { + lastContainer().addTypeParameter(typeName); + } + + + public List foundVarDefinitions(List varNames, String type, List typeArguments, Integer line) { + return varNames.stream().map(item->foundVarDefinition(item,GenericName.build(type),typeArguments,line)).collect(Collectors.toList()); + } + + public VarEntity foundVarDefinition(ContainerEntity container,String varName,Integer line) { + if (container==null) { + System.out.println("fallback to file container for var " + varName + " in file "+ currentFile().getRawName()); + container = currentFile(); + } + + VarEntity var = getVarInLocalFile(container,GenericName.build(varName)); + if (var!=null) return var; + var = new VarEntity(GenericName.build(varName), null, container, idGenerator.generateId()); + var.setLine(line); + container.addVar(var); + addToRepo(var); + + return var; + } + + public VarEntity foundGlobalVarDefinition(ContainerEntity container,String varName,Integer line) { + if (container==null) { + System.out.println("fallback to file container for var " + varName + " in file "+ currentFile().getRawName()); + container = currentFile(); + } + + VarEntity var = getVarInLocalFile(container,GenericName.build(varName)); + if (var!=null) return var; + var = new VarEntity(GenericName.build(varName), null, container, idGenerator.generateId()); + container.addVar(var); + var.setLine(line); + var.setQualifiedName(var.getRawName().toString()); + addToRepo(var); + return var; + } + + public VarEntity foundVarDefinition(String varName, GenericName type, List typeArguments,Integer line) { + VarEntity var = new VarEntity(GenericName.build(varName), type, lastContainer(), idGenerator.generateId()); + var.setLine(line); + var.addTypeParameter(typeArguments); + lastContainer().addVar(var); + addToRepo(var); + return var; + } + + public VarEntity addMethodParameter(String paramName) { + if (currentFunction()==null) return null; + VarEntity varEntity = new VarEntity(GenericName.build(paramName),null,currentFunction(),idGenerator.generateId()); + currentFunction().addParameter(varEntity); + addToRepo(varEntity); + return varEntity; + } + + public VarEntity foundEnumConstDefinition(String varName,Integer line) { + GenericName type = lastContainer().getRawName(); + return foundVarDefinition(varName,type,new ArrayList<>(),line); + } + + protected Stack entityStack = new Stack(); + + protected void pushToStack(Entity entity) { + entityStack.push(entity); + } + + + public void exitLastedEntity() { + //we never pop up the lastest one (FileEntity) + if (entityStack.size()>1) + entityStack.pop(); + } + + private VarEntity getVarInLocalFile(ContainerEntity container, GenericName varName) { + Entity entity = bindingResolver.resolveName(container, varName, false); + if (entity ==null ) return null; + Entity fileEntity = entity.getAncestorOfType(FileEntity.class); + if (fileEntity ==null ){ + //may not exist in fileEntity, for example global vars + }else{ + if (!fileEntity.equals(currentFileEntity)) return null; + if (entity instanceof VarEntity) return (VarEntity)entity; + } + return null; + } + + public Entity foundEntityWithName(GenericName rawName) { + return bindingResolver.resolveName(lastContainer(), rawName, true); + } + + public void addToRepo(Entity entity) { + entityRepo.add(entity); + } + + +} \ No newline at end of file diff --git a/src/main/java/com/educoder/bridge/tmp/InMemoryEntityRepo.java b/src/main/java/com/educoder/bridge/tmp/InMemoryEntityRepo.java new file mode 100644 index 0000000..435676b --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/InMemoryEntityRepo.java @@ -0,0 +1,135 @@ +package depends.entity.repo; + +import depends.entity.Entity; +import depends.entity.FileEntity; +import depends.entity.GenericName; +import depends.entity.MultiDeclareEntities; +import multilang.depends.util.file.FileUtil; + +import java.util.*; +import java.util.Map.Entry; + + +public class InMemoryEntityRepo extends SimpleIdGenerator implements EntityRepo { + + public class EntityＭapIterator implements Iterator{ + + private Iterator> entryIterator; + + public EntityＭapIterator(Set> entries) { + this.entryIterator = entries.iterator(); + } + @Override + public boolean hasNext() { + return entryIterator.hasNext(); + } + + @Override + public Entity next() { + return entryIterator.next().getValue(); + } + + } + + private Map allEntieisByName; + private Map allEntitiesById; + private List allFileEntitiesByOrder; + + public InMemoryEntityRepo() { + allEntieisByName = new TreeMap<>(); + allEntitiesById = new TreeMap<>(); + allFileEntitiesByOrder = new LinkedList<>(); + } + + @Override + public Entity getEntity(String entityName) { + return allEntieisByName.get(entityName); + } + + @Override + public Entity getEntity(Integer entityId) { + return allEntitiesById.get(entityId); + } + + @Override + public void add(Entity entity) { + allEntitiesById.put(entity.getId(), entity); + String name = entity.getRawName().uniqName(); + if (entity.getQualifiedName() != null && !(entity.getQualifiedName().isEmpty())) { + name = entity.getQualifiedName(); + } + if (allEntieisByName.containsKey(name)) { + Entity existedEntity = allEntieisByName.get(name); + if (existedEntity instanceof MultiDeclareEntities) { + ((MultiDeclareEntities) existedEntity).add(entity); + } else { + MultiDeclareEntities eMultiDeclare = new MultiDeclareEntities(existedEntity, this.generateId()); + eMultiDeclare.add(entity); + allEntieisByName.put(name, eMultiDeclare); + } + } else { + allEntieisByName.put(name, entity); + } + if (entity.getParent() != null) + Entity.setParent(entity, entity.getParent()); + } + + @Override + public Iterator entityIterator() { + return new EntityＭapIterator(allEntitiesById.entrySet()); + } + + + @Override + public void update(Entity entity) { + } + + @Override + public Entity getEntity(GenericName rawName) { + return this.getEntity(rawName.uniqName()); + } + + @Override + public Collection getFileEntities() { + return allFileEntitiesByOrder; + } + + @Override + public Iterator sortedFileIterator() { + return allFileEntitiesByOrder.iterator(); + } + + @Override + public void clear() { + allEntieisByName.clear(); + allEntitiesById.clear(); + allFileEntitiesByOrder.clear(); + } + + @Override + public FileEntity getFileEntity(String fileFullPath) { + fileFullPath = FileUtil.uniqFilePath(fileFullPath); + Entity entity = this.getEntity(fileFullPath); + if (entity ==null) return null; + if (entity instanceof FileEntity) return (FileEntity) entity; + if (entity instanceof MultiDeclareEntities){ + MultiDeclareEntities multiDeclare = (MultiDeclareEntities) entity; + for (Entity theEntity: multiDeclare.getEntities()){ + if (theEntity instanceof FileEntity){ + return (FileEntity) theEntity; + } + } + } + return null; + } + + @Override + public void completeFile(String fileFullPath) { + FileEntity fileEntity = getFileEntity(fileFullPath); + // in case of parse error(throw exception), the file entity may not exists + if (fileEntity!=null) { + fileEntity.cacheAllExpressions(); + allFileEntitiesByOrder.add(fileEntity); + } + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/IncludeRelationTest.java b/src/main/java/com/educoder/bridge/tmp/IncludeRelationTest.java new file mode 100644 index 0000000..3356fb4 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/IncludeRelationTest.java @@ -0,0 +1,109 @@ +package depends.extractor.cpp; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; + +import depends.deptypes.DependencyType; +import depends.entity.AliasEntity; +import multilang.depends.util.file.FileUtil; + +public class IncludeRelationTest extends CppParserTest{ + @Before + public void setUp() { + super.init(); + } + + @Test + public void test_includefiles_should_be_imported_relations() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/cpp-code-examples/includesTest/EntryFile.cpp", + "./src/test/resources/cpp-code-examples/includesTest/LocalHeader.h", + "./src/test/resources/cpp-code-examples/includesTest/IndirectIncluded.h", + "./src/test/resources/cpp-code-examples/includesTest/RelativeInclude.h", + "./src/test/resources/cpp-code-examples/includesTest/path/Header.h", + }; + + for (String src:srcs) { + CppFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + File f = new File(srcs[0]); + assertEquals(3, entityRepo.getEntity(f.getCanonicalPath()).getRelations().size()); + } + + + @Test + public void test_could_found_files_in_include_path() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/cpp-code-examples/includesTest/EntryFileIncludePathTest.cpp", + "./src/test/resources/cpp-code-examples/includesTest/path/HeadersWithoutPath.h", + }; + + List includePaths = new ArrayList<>(); + includePaths.add("./src/test/resources/cpp-code-examples/includesTest/path/"); + for (String src:srcs) { + CppFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + File f = new File(srcs[0]); + assertEquals(1, entityRepo.getEntity(f.getCanonicalPath()).getRelations().size()); + } + + + @Test + public void test_type_t_should_be_treat_as_structure() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/cpp-code-examples/typedeftest2.cpp", + }; + + for (String src:srcs) { + CppFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + assertEquals("abc",((AliasEntity) entityRepo.getEntity("abc_t")).getOriginType().getRawName().uniqName()); + + } + + @Test + public void test_call_should_only_in_relations_with_include() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/cpp-code-examples/includeTest2/f0.cpp", + "./src/test/resources/cpp-code-examples/includeTest2/f_with_include.cpp", + "./src/test/resources/cpp-code-examples/includeTest2/f_without_include.cpp", + }; + + for (String src:srcs) { + CppFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + this.assertContainsRelation(this.entityRepo.getEntity("foo"), DependencyType.CALL, "bar"); + this.assertNotContainsRelation(this.entityRepo.getEntity("foo2"), DependencyType.CALL, "bar"); + } + + @Test + public void should_find_include_relation_in_conditional_macro_block() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/cpp-code-examples/includeTest3/inc_macro_test.c", + "./src/test/resources/cpp-code-examples/includeTest3/fx.h", + "./src/test/resources/cpp-code-examples/includeTest3/fy.h", + }; + + for (String src:srcs) { + CppFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + this.assertContainsRelation(this.entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])), DependencyType.IMPORT, FileUtil.uniqFilePath(srcs[1])); + this.assertContainsRelation(this.entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])), DependencyType.IMPORT, FileUtil.uniqFilePath(srcs[2])); + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/JDataBuilder.java b/src/main/java/com/educoder/bridge/tmp/JDataBuilder.java new file mode 100644 index 0000000..5e44240 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/JDataBuilder.java @@ -0,0 +1,88 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.format.json; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import depends.format.FileAttributes; +import depends.matrix.core.DependencyDetail; +import depends.matrix.core.DependencyMatrix; +import depends.matrix.core.DependencyPair; +import depends.matrix.core.DependencyValue; + +public class JDataBuilder { + public JDepObject build(DependencyMatrix dependencyMatrix, FileAttributes attribute) { + ArrayList files = dependencyMatrix.getNodes(); + Collection dependencyPairs = dependencyMatrix.getDependencyPairs(); + ArrayList cellObjects = buildCellObjects(dependencyPairs); // transform finalRes into cellObjects + + JDepObject depObject = new JDepObject(); + depObject.setVariables(files); + depObject.setName(attribute.getAttributeName()); + depObject.setSchemaVersion(attribute.getSchemaVersion()); + depObject.setCells(cellObjects); + + return depObject; + } + + private ArrayList buildCellObjects(Collection dependencyPairs) { + ArrayList cellObjects = new ArrayList(); + + for (DependencyPair dependencyPair : dependencyPairs) { + Map valueObject = buildValueObject(dependencyPair.getDependencies()); + List details = buildDetails(dependencyPair.getDependencies()); + JCellObject cellObject = new JCellObject(); + cellObject.setSrc(dependencyPair.getFrom()); + cellObject.setDest(dependencyPair.getTo()); + cellObject.setValues(valueObject); + cellObject.setDetails(details); + cellObjects.add(cellObject); + } + return cellObjects; + } + + private List buildDetails(Collection dependencies) { + List r = new ArrayList<>(); + for (DependencyValue dependency : dependencies) { + for (DependencyDetail detail:dependency.getDetails()) { + r.add(new DetailItem(detail.getSrc(),detail.getDest(),dependency.getType())); + } + } + if (r.size()==0) return null; + return r; + } + + private Map buildValueObject(Collection dependencies) { + Map valueObject = new HashMap(); + for (DependencyValue dependency : dependencies) { + valueObject.put(dependency.getType(), (float) dependency.getWeight()); + } + return valueObject; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/JRubyVisitor.java b/src/main/java/com/educoder/bridge/tmp/JRubyVisitor.java new file mode 100644 index 0000000..052b755 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/JRubyVisitor.java @@ -0,0 +1,281 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.ruby.jruby; + +import depends.entity.*; +import depends.entity.repo.EntityRepo; +import depends.extractor.ParserCreator; +import depends.extractor.IncludedFileLocator; +import depends.extractor.ruby.RubyHandlerContext; +import depends.relations.IBindingResolver; +import org.jrubyparser.ast.*; +import org.jrubyparser.util.NoopVisitor; + +import java.util.ArrayList; +import java.util.Collection; + +public class JRubyVisitor extends NoopVisitor { + + private RubyHandlerContext context; + RubyParserHelper helper = RubyParserHelper.getInst(); + private ExpressionUsage expressionUsage; + + public JRubyVisitor(String fileFullPath, EntityRepo entityRepo, IncludedFileLocator includedFileLocator, + IBindingResolver bindingResolver, ParserCreator parserCreator) { + this.context = new RubyHandlerContext(entityRepo, includedFileLocator, bindingResolver, parserCreator); + expressionUsage = new ExpressionUsage(context, entityRepo, helper, bindingResolver); + context.startFile(fileFullPath); + + } + + @Override + public Object visitAliasNode(AliasNode node) { + context.foundNewAlias(node.getNewNameString(), node.getOldNameString()); + return super.visitAliasNode(node); + } + + @Override + public Object visitModuleNode(ModuleNode node) { + String name = helper.getName(node.getCPath()); + context.foundNamespace(name,node.getPosition().getStartLine()+1); + super.visitModuleNode(node); + context.exitLastedEntity(); + return null; + } + + @Override + public Object visitClassNode(ClassNode node) { + TypeEntity type = context.foundNewType(helper.getName(node.getCPath()),node.getPosition().getStartLine()+1); + Node superNode = node.getSuper(); + + if (superNode instanceof ConstNode || + superNode instanceof SymbolNode || + superNode instanceof Colon2ConstNode || + superNode instanceof Colon3Node) { + String superName = helper.getName(superNode); + context.foundExtends(superName); + }else{ + if (superNode != null) { + System.err.println("cannot support the super node style" + superNode.toString()); + } + } + + super.visitClassNode(node); + context.exitLastedEntity(); + return null; + } + + @Override + public Object visitRootNode(RootNode node) { + return super.visitRootNode(node); + } + + @Override + public Object visitFCallNode(FCallNode node) { + String fname = helper.getName(node); + Collection params = getParams(node); + context.processSpecialFuncCall(fname, params, node.getPosition().getStartLine()+1); + return super.visitFCallNode(node); + } + + private Collection getParams(IArgumentNode node) { + Node args = node.getArgs(); + Collection params = new ArrayList<>(); + if (args instanceof ArrayNode) { + ArrayNode argArray = (ArrayNode) args; + for (Node arg : argArray.childNodes()) { + if (arg instanceof StrNode) { + params.add(((StrNode) arg).getValue()); + } else if (arg instanceof ConstNode) { + params.add(((ConstNode) arg).getName()); + } + } + } + return params; + } + + @Override + public Object visitCallNode(CallNode node) { + String fname = helper.getName(node); + Collection params = getParams(node); + addCallToReceiverVar(node, fname); + context.processSpecialFuncCall(fname, params, node.getPosition().getStartLine()+1); + return super.visitCallNode(node); + } + + private void addCallToReceiverVar(CallNode node, String fname) { + if (helper.isCommonOperator(fname))return; + Node varNode = node.getReceiver(); + + GenericName varName = GenericName.build(helper.getName(varNode)); + if (varName==null) return; + Entity var = context.foundEntityWithName(varName); + if (var != null && var instanceof VarEntity) { + VarEntity varEntity = (VarEntity) var; + varEntity.addFunctionCall(GenericName.build(fname)); + } + } + + @Override + public Object visitUnaryCallNode(UnaryCallNode node) { + String fname = helper.getName(node); + Collection params = new ArrayList<>(); + context.processSpecialFuncCall(fname, params, node.getPosition().getStartLine()+1); + return super.visitUnaryCallNode(node); + } + + /** + * VCallNode is just a function call without parameter + */ + @Override + public Object visitVCallNode(VCallNode node) { + return super.visitVCallNode(node); + } + + @Override + public Object visitDefnNode(DefnNode node) { + FunctionEntity method = context.foundMethodDeclarator(node.getName(),node.getPosition().getStartLine()+1); + method.setLine(node.getPosition().getStartLine()+1); + + super.visitDefnNode(node); + context.exitLastedEntity(); + return null; + } + + @Override + public Object visitDefsNode(DefsNode node) { + boolean handled = false; + Node varNode = node.getReceiver(); + if (varNode instanceof SelfNode) { + //will be handled by context.foundMethodDeclarator(node.getName(), null, new ArrayList<>()); + } else if (varNode instanceof ConstNode) { + String className = ((INameNode) varNode).getName(); + Entity entity = context.foundEntityWithName(GenericName.build(className)); + if (entity != null && entity instanceof ContainerEntity) { + FunctionEntity method = context.foundMethodDeclarator(((ContainerEntity) entity), node.getName(),node.getPosition().getStartLine()+1); + method.setLine(node.getPosition().getStartLine()+1); + handled = true; + } + + } else if (varNode instanceof INameNode) { + String varName = ((INameNode) varNode).getName(); + Entity var = context.foundEntityWithName(GenericName.build(varName)); + if (var != null && var instanceof ContainerEntity) { + FunctionEntity method = context.foundMethodDeclarator(((ContainerEntity) var), node.getName(),node.getPosition().getStartLine()+1); + method.setLine(node.getPosition().getStartLine()+1); + handled = true; + } + } + + if (!handled) { + // fallback to add it to last container + FunctionEntity method = context.foundMethodDeclarator(node.getName(),node.getPosition().getStartLine()+1); + method.setLine(node.getPosition().getStartLine()+1); + } + super.visitDefsNode(node); + context.exitLastedEntity(); + return null; + } + + @Override + public Object visitGlobalVarNode(GlobalVarNode node) { + context.foundVarDefinition(context.globalScope(), node.getName(),node.getPosition().getStartLine()+1); + return super.visitGlobalVarNode(node); + } + + @Override + public Object visitInstVarNode(InstVarNode node) { + context.foundVarDefinition(context.currentType(), node.getName(),node.getPosition().getStartLine()+1); + return super.visitInstVarNode(node); + } + + @Override + public Object visitClassVarAsgnNode(ClassVarAsgnNode node) { + context.foundVarDefinition(helper.getScopeOfVar(node,context), node.getName(),node.getPosition().getStartLine()+1); + return super.visitClassVarAsgnNode(node); + } + + @Override + public Object visitClassVarDeclNode(ClassVarDeclNode node) { + context.foundVarDefinition(context.currentType(), node.getName(),node.getPosition().getStartLine()+1); + return super.visitClassVarDeclNode(node); + } + + @Override + public Object visitClassVarNode(ClassVarNode node) { + context.foundVarDefinition(context.currentType(), node.getName(),node.getPosition().getStartLine()+1); + return super.visitClassVarNode(node); + } + + @Override + public Object visitLocalVarNode(LocalVarNode node) { + return super.visitLocalVarNode(node); + } + + @Override + public Object visitDVarNode(DVarNode node) { + context.foundVarDefinition(context.lastContainer(), node.getName(),node.getPosition().getStartLine()+1); + return super.visitDVarNode(node); + } + + @Override + public Object visitDAsgnNode(DAsgnNode node) { + context.foundVarDefinition(helper.getScopeOfVar(node,context), node.getName(),node.getPosition().getStartLine()+1); + expressionUsage.foundExpression(node.getValue()); + return super.visitDAsgnNode(node); + } + + @Override + public Object visitGlobalAsgnNode(GlobalAsgnNode node) { + context.foundVarDefinition(helper.getScopeOfVar(node,context), node.getName(),node.getPosition().getStartLine()+1); + return super.visitGlobalAsgnNode(node); + } + + @Override + public Object visitInstAsgnNode(InstAsgnNode node) { + context.foundVarDefinition(helper.getScopeOfVar(node,context), node.getName(),node.getPosition().getStartLine()+1); + return super.visitInstAsgnNode(node); + } + + @Override + public Object visitArgumentNode(ArgumentNode node) { + String paramName = node.getName(); + context.addMethodParameter(paramName); + return super.visitArgumentNode(node); + } + + @Override + public Object visitLocalAsgnNode(LocalAsgnNode node) { + context.foundVarDefinition(helper.getScopeOfVar(node,context), node.getName(),node.getPosition().getStartLine()+1); + return super.visitLocalAsgnNode(node); + } + + @Override + protected Object visit(Node node) { + expressionUsage.foundExpression(node); + return super.visit(node); + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/JavaBuiltInType.java b/src/main/java/com/educoder/bridge/tmp/JavaBuiltInType.java new file mode 100644 index 0000000..7402f58 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/JavaBuiltInType.java @@ -0,0 +1,84 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.java; + +import depends.entity.repo.BuiltInType; + +public class JavaBuiltInType extends BuiltInType{ + + @Override + protected String[] getBuiltInTypeName() { + return new String[]{ + "void","int","double","char","byte","boolean","long","short","float", + "BigDecimal","Integer","Double","Char","Byte","Boolean","Long","Short","Float", + "String","Object","Class","Exception","StringBuilder", + "Appendable","AutoCloseable","Cloneable","Comparable","Iterable","Readable", + "Runnable","Thread.UncaughtExceptionHandler","Boolean","Byte","Character","Character.Subset", + "Character.UnicodeBlock","ClassLoader","ClassValue","Compiler","Double","Enum", + "InheritableThreadLocal","Math","Number","Package","Process", + "ProcessBuilder","ProcessBuilder.Redirect","Runtime","RuntimePermission", + "SecurityManager","StackTraceElement","StrictMath","StringBuffer", + "System","Thread","ThreadGroup","ThreadLocal","Throwable","Void","ProcessBuilder.Redirect.Type", + "Thread.State","ArithmeticException","ArrayIndexOutOfBoundsException", + "ArrayStoreException","ClassCastException","ClassNotFoundException","CloneNotSupportedException", + "EnumConstantNotPresentException","Exception","IllegalAccessException","IllegalArgumentException", + "IllegalMonitorStateException","IllegalStateException","IllegalThreadStateException", + "IndexOutOfBoundsException","InstantiationException","InterruptedException", + "NegativeArraySizeException","NoSuchFieldException","NoSuchMethodException","NullPointerException", + "NumberFormatException","ReflectiveOperationException","RuntimeException","SecurityException", + "StringIndexOutOfBoundsException","TypeNotPresentException","UnsupportedOperationException","AbstractMethodError", + "AssertionError","BootstrapMethodError","ClassCircularityError","ClassFormatError","Error","ExceptionInInitializerError", + "IllegalAccessError","IncompatibleClassChangeError","InstantiationError","InternalError","LinkageError","NoClassDefFoundError" + ,"NoSuchFieldError","NoSuchMethodError","OutOfMemoryError","StackOverflowError","ThreadDeath","UnknownError", + "UnsatisfiedLinkError","UnsupportedClassVersionError","VerifyError","VirtualMachineError","Deprecated","Override", + "SafeVarargs","SuppressWarnings", + "Collection","Comparator","Deque","Enumeration","EventListener","Formattable","Iterator","List", + "ListIterator","Map","Map.Entry","NavigableMap","NavigableSet","Observer","Queue","RandomAccess", + "Set","SortedMap","SortedSet","AbstractCollection","AbstractList","AbstractMap","AbstractMap.SimpleEntry", + "AbstractMap.SimpleImmutableEntry","AbstractQueue","AbstractSequentialList","AbstractSet","ArrayDeque", + "ArrayList","Arrays","BitSet","Calendar","Collections","Currency","Date","Dictionary","EnumMap","EnumSet", + "EventListenerProxy","EventObject","FormattableFlags","Formatter","GregorianCalendar","HashMap","HashSet", + "Hashtable","IdentityHashMap","LinkedHashMap","LinkedHashSet","LinkedList","ListResourceBundle","Locale", + "Locale.Builder","Objects","Observable","PriorityQueue","Properties","PropertyPermission", + "PropertyResourceBundle","Random","ResourceBundle","ResourceBundle.Control","Scanner", + "ServiceLoader","SimpleTimeZone","Stack","StringTokenizer","Timer","TimerTask","TimeZone", + "TreeMap","TreeSet","UUID","Vector","WeakHashMap","Formatter.BigDecimalLayoutForm", + "Locale.Category","ConcurrentModificationException","DuplicateFormatFlagsException", + "EmptyStackException","FormatFlagsConversionMismatchException","FormatterClosedException", + "IllegalFormatCodePointException","IllegalFormatConversionException","IllegalFormatException", + "IllegalFormatFlagsException","IllegalFormatPrecisionException","IllegalFormatWidthException", + "IllformedLocaleException","InputMismatchException","InvalidPropertiesFormatException","MissingFormatArgumentException", + "MissingFormatWidthException","MissingResourceException","NoSuchElementException","TooManyListenersException", + "UnknownFormatConversionException","UnknownFormatFlagsException","ServiceConfigurationError", + "" + }; + } + @Override + protected String[] getBuiltInTypePrefix() { + return new String[]{ + "java.","javax.","com.sun." + }; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/JavaImportLookupStrategy.java b/src/main/java/com/educoder/bridge/tmp/JavaImportLookupStrategy.java new file mode 100644 index 0000000..d791697 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/JavaImportLookupStrategy.java @@ -0,0 +1,105 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.java; + +import depends.entity.Entity; +import depends.entity.FileEntity; +import depends.entity.PackageEntity; +import depends.entity.repo.EntityRepo; +import depends.extractor.UnsolvedBindings; +import depends.importtypes.Import; +import depends.relations.ImportLookupStrategy; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class JavaImportLookupStrategy extends ImportLookupStrategy{ + + public JavaImportLookupStrategy(EntityRepo repo) { + super(repo); + } + + @Override + public Entity lookupImportedType(String name, FileEntity fileEntity) { + //Java Strategy + String importedString = fileEntity.importedSuffixMatch(name); + if (importedString==null) return null; + return repo.getEntity(importedString); + } + + + @Override + public List getImportedRelationEntities(List importedList) { + ArrayList result = new ArrayList<>(); + for (Import importedItem:importedList) { + Entity imported = repo.getEntity(importedItem.getContent()); + if (imported==null) continue; + if (imported instanceof PackageEntity) { + //ignore wildcard import relation + }else { + result.add(imported); + } + } + return result; + } + + @Override + public List getImportedTypes(List importedList,Set unsolvedBindings) { + ArrayList result = new ArrayList<>(); + for (Import importedItem:importedList) { + Entity imported = repo.getEntity(importedItem.getContent()); + if (imported==null) { + unsolvedBindings.add(new UnsolvedBindings(importedItem.getContent(),null)); + continue; + } + if (imported instanceof PackageEntity) { + //expand import of package to all classes under the package due to we dis-courage the behavior + for (Entity child:imported.getChildren()) { + if (child instanceof FileEntity) { + child.getChildren().forEach(item->result.add(item)); + }else { + result.add(child); + } + } + }else { + result.add(imported); + } + } + return result; + } + + @Override + public List getImportedFiles(List importedList) { + return new ArrayList(); + } + + + @Override + public boolean supportGlobalNameLookup() { + return true; + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/JavaListener.java b/src/main/java/com/educoder/bridge/tmp/JavaListener.java new file mode 100644 index 0000000..326e788 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/JavaListener.java @@ -0,0 +1,337 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.java; + +import depends.entity.*; +import depends.entity.repo.EntityRepo; +import depends.extractor.java.JavaParser.*; +import depends.extractor.java.context.*; +import depends.importtypes.ExactMatchImport; +import depends.relations.IBindingResolver; + +import java.util.ArrayList; +import java.util.List; + +public class JavaListener extends JavaParserBaseListener { + private final JavaHandlerContext context; + private final AnnotationProcessor annotationProcessor; + private final ExpressionUsage expressionUsage; + private final EntityRepo entityRepo; + + public JavaListener(String fileFullPath, EntityRepo entityRepo, IBindingResolver bindingResolver) { + this.context = new JavaHandlerContext(entityRepo, bindingResolver); + this.entityRepo = entityRepo; + annotationProcessor = new AnnotationProcessor(); + expressionUsage = new ExpressionUsage(context,entityRepo); + context.startFile(fileFullPath); + } + + //////////////////////// + // Package + @Override + public void enterPackageDeclaration(PackageDeclarationContext ctx) { + context.foundNewPackage(QualitiedNameContextHelper.getName(ctx.qualifiedName())); + super.enterPackageDeclaration(ctx); + } + + + //////////////////////// + // Import + @Override + public void enterImportDeclaration(ImportDeclarationContext ctx) { + context.foundNewImport(new ExactMatchImport(ctx.qualifiedName().getText())); + super.enterImportDeclaration(ctx); + } + + /////////////////////// + // Class or Interface + // classDeclaration | enumDeclaration | interfaceDeclaration | + /////////////////////// annotationTypeDeclaration + @Override + public void enterClassDeclaration(ClassDeclarationContext ctx) { + if (ctx.IDENTIFIER()==null) return; + context.foundNewType(GenericName.build(ctx.IDENTIFIER().getText()), ctx.getStart().getLine()); + // implements + if (ctx.typeList() != null) { + for (int i = 0; i < ctx.typeList().typeType().size(); i++) { + context.foundImplements(GenericName.build(ClassTypeContextHelper.getClassName(ctx.typeList().typeType().get(i)))); + } + } + // extends relation + if (ctx.typeType() != null) { + context.foundExtends(GenericName.build(ClassTypeContextHelper.getClassName(ctx.typeType()))); + } + + if (ctx.typeParameters() != null) { + foundTypeParametersUse(ctx.typeParameters()); + } + annotationProcessor.processAnnotationModifier(ctx, TypeDeclarationContext.class ,"classOrInterfaceModifier.annotation",context.lastContainer()); + super.enterClassDeclaration(ctx); + } + + @Override + public void exitClassDeclaration(ClassDeclarationContext ctx) { + exitLastEntity(); + super.exitClassDeclaration(ctx); + } + + @Override + public void enterEnumDeclaration(EnumDeclarationContext ctx) { + context.foundNewType(GenericName.build(ctx.IDENTIFIER().getText()), ctx.getStart().getLine()); + annotationProcessor.processAnnotationModifier(ctx, TypeDeclarationContext.class ,"classOrInterfaceModifier.annotation",context.lastContainer()); + super.enterEnumDeclaration(ctx); + } + + @Override + public void enterAnnotationTypeDeclaration(AnnotationTypeDeclarationContext ctx) { + context.foundNewType(GenericName.build(ctx.IDENTIFIER().getText()), ctx.getStart().getLine()); + annotationProcessor.processAnnotationModifier(ctx, TypeDeclarationContext.class ,"classOrInterfaceModifier.annotation",context.lastContainer()); + super.enterAnnotationTypeDeclaration(ctx); + } + + @Override + public void exitEnumDeclaration(EnumDeclarationContext ctx) { + exitLastEntity(); + super.exitEnumDeclaration(ctx); + } + + /** + * interfaceDeclaration : INTERFACE IDENTIFIER typeParameters? (EXTENDS + * typeList)? interfaceBody ; + */ + @Override + public void enterInterfaceDeclaration(InterfaceDeclarationContext ctx) { + context.foundNewType(GenericName.build(ctx.IDENTIFIER().getText()), ctx.getStart().getLine()); + // type parameters + if (ctx.typeParameters() != null) { + foundTypeParametersUse(ctx.typeParameters()); + } + // extends relation + if (ctx.typeList() != null) { + for (int i = 0; i < ctx.typeList().typeType().size(); i++) { + context.foundExtends(ClassTypeContextHelper.getClassName(ctx.typeList().typeType().get(i))); + } + } + annotationProcessor.processAnnotationModifier(ctx, TypeDeclarationContext.class ,"classOrInterfaceModifier.annotation",context.lastContainer()); + super.enterInterfaceDeclaration(ctx); + } + + @Override + public void exitInterfaceDeclaration(InterfaceDeclarationContext ctx) { + exitLastEntity(); + super.exitInterfaceDeclaration(ctx); + } + + + + @Override + public void exitAnnotationTypeDeclaration(AnnotationTypeDeclarationContext ctx) { + exitLastEntity(); + super.exitAnnotationTypeDeclaration(ctx); + } + + ///////////////////////// + // Method + @Override + public void enterMethodDeclaration(MethodDeclarationContext ctx) { + List throwedType = QualitiedNameContextHelper.getNames(ctx.qualifiedNameList()); + String methodName = ctx.IDENTIFIER().getText(); + String returnedType = ClassTypeContextHelper.getClassName(ctx.typeTypeOrVoid()); + FunctionEntity method = context.foundMethodDeclarator(methodName, returnedType, throwedType,ctx.getStart().getLine()); + new FormalParameterListContextHelper(ctx.formalParameters(), method, entityRepo); + if (ctx.typeParameters() != null) { + List parameters = TypeParameterContextHelper.getTypeParameters(ctx.typeParameters()); + method.addTypeParameter(parameters); + } + annotationProcessor.processAnnotationModifier(ctx, ClassBodyDeclarationContext.class,"modifier.classOrInterfaceModifier.annotation",context.lastContainer()); + super.enterMethodDeclaration(ctx); + } + + @Override + public void exitMethodDeclaration(MethodDeclarationContext ctx) { + exitLastEntity(); + super.exitMethodDeclaration(ctx); + } + + private void exitLastEntity() { + context.exitLastedEntity(); + } + +// interfaceMethodDeclaration +// : interfaceMethodModifier* (typeTypeOrVoid | typeParameters annotation* typeTypeOrVoid) +// IDENTIFIER formalParameters ('[' ']')* (THROWS qualifiedNameList)? methodBody + + @Override + public void enterInterfaceMethodDeclaration(InterfaceMethodDeclarationContext ctx) { + List throwedType = QualitiedNameContextHelper.getNames(ctx.qualifiedNameList()); + FunctionEntity method = context.foundMethodDeclarator(ctx.IDENTIFIER().getText(), + ClassTypeContextHelper.getClassName(ctx.typeTypeOrVoid()), throwedType,ctx.getStart().getLine()); + new FormalParameterListContextHelper(ctx.formalParameters(), method, entityRepo); + if (ctx.typeParameters() != null) { + foundTypeParametersUse(ctx.typeParameters()); + } + annotationProcessor.processAnnotationModifier(ctx, InterfaceBodyDeclarationContext.class,"modifier.classOrInterfaceModifier.annotation",context.lastContainer()); + super.enterInterfaceMethodDeclaration(ctx); + } + + @Override + public void exitInterfaceMethodDeclaration(InterfaceMethodDeclarationContext ctx) { + exitLastEntity(); + super.exitInterfaceMethodDeclaration(ctx); + } + + @Override + public void enterConstructorDeclaration(ConstructorDeclarationContext ctx) { + List throwedType = QualitiedNameContextHelper.getNames(ctx.qualifiedNameList()); + FunctionEntity method = context.foundMethodDeclarator(ctx.IDENTIFIER().getText(), ctx.IDENTIFIER().getText(), + throwedType,ctx.getStart().getLine()); + new FormalParameterListContextHelper(ctx.formalParameters(), method, entityRepo); + method.addReturnType(context.currentType()); + annotationProcessor.processAnnotationModifier(ctx, ClassBodyDeclarationContext.class,"modifier.classOrInterfaceModifier.annotation",context.lastContainer()); + super.enterConstructorDeclaration(ctx); + } + + @Override + public void exitConstructorDeclaration(ConstructorDeclarationContext ctx) { + exitLastEntity(); + super.exitConstructorDeclaration(ctx); + } + + ///////////////////////////////////////////////////////// + // Field + @Override + public void enterFieldDeclaration(FieldDeclarationContext ctx) { + List varNames = VariableDeclaratorsContextHelper.getVariables(ctx.variableDeclarators()); + String type = ClassTypeContextHelper.getClassName(ctx.typeType()); + List typeArguments = ClassTypeContextHelper.getTypeArguments(ctx.typeType()); + List vars = context.foundVarDefinitions(varNames, type,typeArguments,ctx.getStart().getLine()); + annotationProcessor.processAnnotationModifier(ctx, ClassBodyDeclarationContext.class,"modifier.classOrInterfaceModifier.annotation",vars); + super.enterFieldDeclaration(ctx); + } + + + + @Override + public void enterConstDeclaration(ConstDeclarationContext ctx) { + List typeArguments = ClassTypeContextHelper.getTypeArguments(ctx.typeType()); + List vars = context.foundVarDefinitions(VariableDeclaratorsContextHelper.getVariables(ctx.constantDeclarator()), + ClassTypeContextHelper.getClassName(ctx.typeType()),typeArguments, ctx.getStart().getLine()); + + annotationProcessor.processAnnotationModifier(ctx, InterfaceBodyDeclarationContext.class,"modifier.classOrInterfaceModifier.annotation",vars); + super.enterConstDeclaration(ctx); + } + + @Override + public void enterEnumConstant(EnumConstantContext ctx) { + if (ctx.IDENTIFIER() != null) { + context.foundEnumConstDefinition(ctx.IDENTIFIER().getText(),ctx.getStart().getLine()); + } + super.enterEnumConstant(ctx); + } + + @Override + public void enterAnnotationMethodRest(AnnotationMethodRestContext ctx) { + context.foundMethodDeclarator(ctx.IDENTIFIER().getText(), ClassTypeContextHelper.getClassName(ctx.typeType()), + new ArrayList<>(),ctx.getStart().getLine()); + super.enterAnnotationMethodRest(ctx); + } + + @Override + public void exitAnnotationMethodRest(AnnotationMethodRestContext ctx) { + exitLastEntity(); + super.exitAnnotationMethodRest(ctx); + } + + @Override + public void enterAnnotationConstantRest(AnnotationConstantRestContext ctx) { + // TODO: no variable type defined in annotation const？ + context.foundVarDefinitions(VariableDeclaratorsContextHelper.getVariables(ctx.variableDeclarators()), "", new ArrayList<>(), ctx.getStart().getLine()); + super.enterAnnotationConstantRest(ctx); + } + + /////////////////////////////////////////// + // variables + // TODO: all modifier have not processed yet. + @Override + public void enterLocalVariableDeclaration(LocalVariableDeclarationContext ctx) { + List typeArguments = ClassTypeContextHelper.getTypeArguments(ctx.typeType()); + context.foundVarDefinitions(VariableDeclaratorsContextHelper.getVariables((ctx.variableDeclarators())), + ClassTypeContextHelper.getClassName(ctx.typeType()), typeArguments, ctx.getStart().getLine()); + + super.enterLocalVariableDeclaration(ctx); + } + + public void enterEnhancedForControl(EnhancedForControlContext ctx) { + List typeArguments = ClassTypeContextHelper.getTypeArguments(ctx.typeType()); + context.foundVarDefinitions(VariableDeclaratorsContextHelper.getVariable((ctx.variableDeclaratorId())), + ClassTypeContextHelper.getClassName(ctx.typeType()), typeArguments, ctx.getStart().getLine()); + super.enterEnhancedForControl(ctx); + } + +// resource +// : variableModifier* classOrInterfaceType variableDeclaratorId '=' expression +// ; + @Override + public void enterResource(ResourceContext ctx) { + List typeArguments = ClassTypeContextHelper.getTypeArguments(ctx.classOrInterfaceType()); + context.foundVarDefinition(ctx.variableDeclaratorId().IDENTIFIER().getText(), + GenericName.build(IdentifierContextHelper.getName(ctx.classOrInterfaceType().IDENTIFIER())), typeArguments,ctx.getStart().getLine()); + super.enterResource(ctx); + } + + @Override + public void enterExpression(ExpressionContext ctx) { + Expression expr = expressionUsage.foundExpression(ctx); + expr.setLine(ctx.getStart().getLine()); + super.enterExpression(ctx); + } + + ///////////////////////////////////////////// + // Block + @Override + public void enterBlock(BlockContext ctx) { + // TODO support block in java + super.enterBlock(ctx); + } + + @Override + public void exitBlock(BlockContext ctx) { + // TODO support block in java + super.exitBlock(ctx); + } + + /* type parameters , <> treat as USE */ + private void foundTypeParametersUse(TypeParametersContext typeParameters) { + for (int i = 0; i < typeParameters.typeParameter().size(); i++) { + TypeParameterContext typeParam = typeParameters.typeParameter(i); + if (typeParam.typeBound() != null) { + for (int j = 0; j < typeParam.typeBound().typeType().size(); j++) { + context.foundTypeParametes(GenericName.build(ClassTypeContextHelper.getClassName(typeParam.typeBound().typeType(j)))); + } + } + context.currentType().addTypeParameter(GenericName.build(typeParam.IDENTIFIER().getText())); + } + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/JavaVarResolveTest.java b/src/main/java/com/educoder/bridge/tmp/JavaVarResolveTest.java new file mode 100644 index 0000000..4660844 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/JavaVarResolveTest.java @@ -0,0 +1,100 @@ +package depends.extractor.java; + +import depends.deptypes.DependencyType; +import depends.entity.ContainerEntity; +import depends.entity.Entity; +import depends.entity.FunctionEntity; +import depends.entity.TypeEntity; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +public class JavaVarResolveTest extends JavaParserTest{ + @Before + public void setUp() { + super.init(); + } + + @Test + public void test_field_var_should_be_parsed() throws IOException { + String src = "./src/test/resources/java-code-examples/FieldVar.java"; + JavaFileParser parser = createParser(); + parser.parse(src); + resolveAllBindings(); + Entity classEntity = entityRepo.getEntity("FieldVar"); + assertEquals(3,((TypeEntity)classEntity).getVars().size()); + } + + @Test + public void test_local_var_should_be_parsed() throws IOException { + String src = "./src/test/resources/java-code-examples/LocalVar.java"; + JavaFileParser parser = createParser(); + parser.parse(src); + resolveAllBindings(); + assertEquals(1,((TypeEntity)entityRepo.getEntity("LocalVar")).getVars().size()); + assertEquals(1,((FunctionEntity)entityRepo.getEntity("LocalVar.foo")).getVars().size()); + } + + @Test + public void test_local_var_type_could_be_inferred() throws IOException { + String src = "./src/test/resources/java-code-examples/LocalVarInferExample.java"; + JavaFileParser parser = createParser(); + parser.parse(src); + resolveAllBindings(); + ContainerEntity e = (ContainerEntity) entityRepo.getEntity("LocalVarInferExample.setExample"); + this.assertContainsRelation(e, DependencyType.CONTAIN, "MyInteger"); + } + + @Test + public void test_field_access_could_be_inferred() throws IOException { + String src = "./src/test/resources/java-code-examples/ComplexExpressionExample.java"; + JavaFileParser parser = createParser(); + parser.parse(src); + resolveAllBindings(); + Entity e = entityRepo.getEntity("test.ComplexExpressionExample.setExample"); + this.assertContainsRelation(e, DependencyType.PARAMETER, "test.ClassA"); + this.assertContainsRelation(e, DependencyType.CREATE, "test.ClassA"); + this.assertContainsRelation(e, DependencyType.CALL, "test.ClassA"); + this.assertContainsRelation(e, DependencyType.CAST, "test.ClassA"); + this.assertContainsRelation(e, DependencyType.CALL, "test.ClassA.foo"); + this.assertContainsRelation(e, DependencyType.CALL, "test.ClassA"); + this.assertContainsRelation(e, DependencyType.USE, "test.ComplexExpressionExample.setExample.a3"); + this.assertContainsRelation(e, DependencyType.USE, "test.ClassX.m"); + this.assertContainsRelation(e, DependencyType.USE, "test.ComplexExpressionExample.setExample.a2"); + this.assertContainsRelation(e, DependencyType.USE, "test.ClassA.x"); + this.assertContainsRelation(e, DependencyType.USE, "test.ComplexExpressionExample.setExample.a"); + } + + @Test + public void test_long_static_function_should_be_inferred() throws IOException { + String src = "./src/test/resources/java-code-examples/LongExpressionWithAbsolutePath.java"; + JavaFileParser parser = createParser(); + parser.parse(src); + resolveAllBindings(); + assertEquals(5,entityRepo.getEntity("x.LongExpressionWithAbsolutePath.setExample").getRelations().size()); + } + + + + @Test + public void test_call_should_be_referred() throws IOException { + String src = "./src/test/resources/java-code-examples/ExpressionCallTest.java"; + JavaFileParser parser = createParser(); + parser.parse(src); + resolveAllBindings(); + assertEquals(10,entityRepo.getEntity("ValidateAll.validate").getRelations().size()); + } + + @Test + public void test_could_detect_type_argument_in_field() throws IOException { + String src = "./src/test/resources/java-code-examples/TypeArgument.java"; + JavaFileParser parser = createParser(); + parser.parse(src); + resolveAllBindings(); + this.assertContainsRelation(entityRepo.getEntity("JDepObject.cells"),DependencyType.PARAMETER, "JCellObject"); + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/JchService.java b/src/main/java/com/educoder/bridge/tmp/JchService.java new file mode 100644 index 0000000..8657d1d --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/JchService.java @@ -0,0 +1,261 @@ +package com.educoder.bridge.service; + +import com.alibaba.fastjson.JSONObject; +import com.educoder.bridge.model.SSHInfo; +import com.educoder.bridge.model.SSHSession; +import com.educoder.bridge.utils.Base64Util; +import com.jcraft.jsch.ChannelShell; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.UserInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.socket.TextMessage; +import org.springframework.web.socket.WebSocketSession; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Service +public class JchService { + + private static List sshSessionQueue = new CopyOnWriteArrayList<>(); + private ExecutorService executorService = Executors.newCachedThreadPool(); + private Logger logger = LoggerFactory.getLogger(getClass()); + + com.jcraft.jsch.Logger jschLogger = new com.jcraft.jsch.Logger() { + + @Override + public boolean isEnabled(int arg0) { + return true; + } + + @Override + public void log(int arg0, String arg1) { + if (logger.isTraceEnabled()) { + logger.trace("JSch Log [Level " + arg0 + "]: " + arg1); + } + } + }; + + + /** + * 在webSocket连接时，初始化一个ssh连接 + * + * @param webSocketSession webSocket连接 + */ + public void add(WebSocketSession webSocketSession) { + + SSHSession sshSession = new SSHSession(); + sshSession.setWebSocketSession(webSocketSession); + + sshSessionQueue.add(sshSession); + } + + /** + * 处理客户端发过来的数据 + * @param buffer 数据 + * @param webSocketSession webSocket连接 + */ + public void recv(String buffer, WebSocketSession webSocketSession) { + + SSHSession sshSession = null; + try { + logger.debug("webSocketSessionID: {}, 信息: {}", webSocketSession.getId(), buffer); + JSONObject info = JSONObject.parseObject(buffer); + String tp = info.getString("tp"); + sshSession = findByWebSocketSession(webSocketSession); + + //初始化连接 + if ("init".equals(tp)) { +// {"tp":"init","data":{"host":"127.0.0.1","port":"41080","username":"root","password":"123123"}} + SSHInfo sshInfo = info.getObject("data", SSHInfo.class); + sshSession.setSSHInfo(sshInfo); + + if (sshSession != null) { + SSHSession finalSSHSession = sshSession; + + // 新开一个线程建立连接，连接开启之后以一直监听来自客户端的输入 + executorService.execute(() -> { + connectTossh(finalSSHSession); + }); + } + } else if ("client".equals(tp)) { + String data = info.getString("data"); + + // 将网页输入的数据传送给后端服务器 + if (sshSession != null) { + transTossh(sshSession.getOutputStream(), data); + } + } + } catch (Exception e) { + logger.error("转发命令到ssh出错: {}", e); + + close(sshSession); + } + + } + + /** + * 将数据传送给服务端作为SSH的输入 + * + * @param outputStream + * @param data + * @throws IOException + */ + private void transTossh(OutputStream outputStream, String data) throws IOException { + if (outputStream != null) { + outputStream.write(data.getBytes()); + outputStream.flush(); + } + } + + /** + * 连接ssh + * + * @param sshSession ssh连接需要的信息 + */ + private void connectTossh(SSHSession sshSession){ + Session jschSession = null; + SSHInfo SSHInfo = sshSession.getSSHInfo(); + try { + JSch jsch = new JSch(); + JSch.setLogger(jschLogger); + + //启动线程 + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + jschSession = jsch.getSession(SSHInfo.getUsername(), SSHInfo.getHost(), SSHInfo.getPort()); + + jschSession.setConfig(config); + jschSession.setPassword(SSHInfo.getPassword()); + jschSession.setUserInfo(new UserInfo() { + @Override + public String getPassphrase() { + return null; + } + + @Override + public String getPassword() { + return null; + } + + @Override + public boolean promptPassword(String s) { + return false; + } + + @Override + public boolean promptPassphrase(String s) { + return false; + } + + @Override + public boolean promptYesNo(String s) { + return true; + } // Accept all server keys + + @Override + public void showMessage(String s) { + } + }); + + jschSession.connect(); + ChannelShell channel = (ChannelShell) jschSession.openChannel("shell"); + channel.setPtyType("xterm"); + + channel.connect(); + + sshSession.setChannel(channel); + InputStream inputStream = channel.getInputStream(); + sshSession.setOutputStream(channel.getOutputStream()); + + sshSession.setSSHInfo(SSHInfo); + logger.debug("主机: {} 连接成功!", SSHInfo.getHost()); + + // 循环读取，jsch的输入为服务器执行命令之后的返回数据 + byte[] buf = new byte[1024]; + while (true) { + int length = inputStream.read(buf); + if (length < 0) { + close(sshSession); + throw new Exception("读取出错，数据长度：" + length); + } + sendMsg(sshSession.getWebSocketSession(), Arrays.copyOfRange(buf, 0, length)); + } + + } catch (Exception e) { + logger.error("ssh连接出错, e: {}", e); + } finally { + logger.info("连接关闭, {}", SSHInfo.getHost()); + if (jschSession != null) { + jschSession.disconnect(); + } + + close(sshSession); + } + } + + + /** + * 发送数据回websocket + * + * @param webSocketSession webSocket连接 + * @param buffer 数据 + * @throws IOException + */ + public void sendMsg(WebSocketSession webSocketSession, byte[] buffer) throws IOException { + logger.debug("服务端返回的数据: {}", new String(buffer, "UTF-8")); + + webSocketSession.sendMessage(new TextMessage(Base64Util.encodeBytes(buffer))); + } + + /** + * 通过webSocket连接在队列中找到对应的SSH连接 + * + * @param webSocketSession webSocket连接 + */ + public SSHSession findByWebSocketSession(WebSocketSession webSocketSession) { + Optional optional = sshSessionQueue.stream().filter(webscoketObj -> webscoketObj.getWebSocketSession() == webSocketSession).findFirst(); + if (optional.isPresent()) { + return optional.get(); + } + return null; + } + + /** + * 关闭ssh和websocket连接 + * + * @param sshSession ssh连接 + */ + private void close(SSHSession sshSession) { + if (sshSession != null) { + sshSession.getChannel().disconnect(); + try { + sshSession.getWebSocketSession().close(); + sshSession.getOutputStream().close(); + } catch (IOException e) { + logger.error("连接关闭失败！e: {}", e); + } + + sshSessionQueue.remove(sshSession); + } + } + + /** + * 通过webSocketSession关闭ssh与webSocket连接 + * + * @param webSocketSession + */ + public void closeByWebSocket(WebSocketSession webSocketSession) { + close(findByWebSocketSession(webSocketSession)); + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/Main.java b/src/main/java/com/educoder/bridge/tmp/Main.java new file mode 100644 index 0000000..260ae11 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/Main.java @@ -0,0 +1,197 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends; + +import depends.addons.DV8MappingFileBuilder; +import depends.entity.repo.EntityRepo; +import depends.extractor.AbstractLangProcessor; +import depends.extractor.LangProcessorRegistration; +import depends.extractor.UnsolvedBindings; +import depends.format.DependencyDumper; +import depends.format.detail.UnsolvedSymbolDumper; +import depends.generator.DependencyGenerator; +import depends.generator.FileDependencyGenerator; +import depends.generator.FunctionDependencyGenerator; +import depends.generator.StructureDependencyGenerator; +import depends.matrix.core.DependencyMatrix; +import depends.matrix.transform.MatrixLevelReducer; +import depends.relations.BindingResolver; +import depends.relations.IBindingResolver; +import depends.relations.RelationCounter; +import edu.emory.mathcs.backport.java.util.Arrays; +import multilang.depends.util.file.FileUtil; +import multilang.depends.util.file.FolderCollector; +import multilang.depends.util.file.TemporaryFile; +import multilang.depends.util.file.path.*; +import multilang.depends.util.file.strip.LeadingNameStripper; +import net.sf.ehcache.CacheManager; +import org.codehaus.plexus.util.StringUtils; +import picocli.CommandLine; +import picocli.CommandLine.PicocliException; + +import java.io.File; +import java.util.List; +import java.util.Set; + +/** + * The entry pooint of depends + */ +public class Main { + + public static void main(String[] args) { + try { + LangRegister langRegister = new LangRegister(); + langRegister.register(); + DependsCommand appArgs = CommandLine.populateCommand(new DependsCommand(), args); + if (appArgs.help) { + CommandLine.usage(new DependsCommand(), System.out); + System.exit(0); + } + executeCommand(appArgs); + } catch (Exception e) { + if (e instanceof PicocliException) { + CommandLine.usage(new DependsCommand(), System.out); + } else if (e instanceof ParameterException){ + System.err.println(e.getMessage()); + }else { + System.err.println("Exception encountered. If it is a design error, please report issue to us." ); + e.printStackTrace(); + } + System.exit(0); + } + } + + @SuppressWarnings("unchecked") + private static void executeCommand(DependsCommand args) throws ParameterException { + String lang = args.getLang(); + String inputDir = args.getSrc(); + String[] includeDir = args.getIncludes(); + String outputName = args.getOutputName(); + String outputDir = args.getOutputDir(); + String[] outputFormat = args.getFormat(); + + inputDir = FileUtil.uniqFilePath(inputDir); + + if (args.isAutoInclude()) { + includeDir = appendAllFoldersToIncludePath(inputDir, includeDir); + } + + AbstractLangProcessor langProcessor = LangProcessorRegistration.getRegistry().getProcessorOf(lang); + if (langProcessor == null) { + System.err.println("Not support this language: " + lang); + return; + } + + IBindingResolver bindingResolver = new BindingResolver(langProcessor, args.isOutputExternalDependencies(), args.isDuckTypingDeduce()); + + long startTime = System.currentTimeMillis(); + //step1: build data + EntityRepo entityRepo = langProcessor.buildDependencies(inputDir, includeDir, bindingResolver); + + new RelationCounter(entityRepo,langProcessor, bindingResolver).computeRelations(); + System.out.println("Dependency done...."); + + //step2: generate dependencies matrix + DependencyGenerator dependencyGenerator = getDependencyGenerator(args, inputDir); + DependencyMatrix matrix = dependencyGenerator.identifyDependencies(entityRepo,args.getTypeFilter()); + //step3: output + if (args.getGranularity().startsWith("L")) { + matrix = new MatrixLevelReducer(matrix,args.getGranularity().substring(1)).shrinkToLevel(); + } + DependencyDumper output = new DependencyDumper(matrix); + output.outputResult(outputName,outputDir,outputFormat); + if (args.isOutputExternalDependencies()) { + Set unsolved = langProcessor.getExternalDependencies(); + UnsolvedSymbolDumper unsolvedSymbolDumper = new UnsolvedSymbolDumper(unsolved,args.getOutputName(),args.getOutputDir(), + new LeadingNameStripper(args.isStripLeadingPath(),inputDir,args.getStrippedPaths())); + unsolvedSymbolDumper.output(); + } + long endTime = System.currentTimeMillis(); + TemporaryFile.getInstance().delete(); + CacheManager.create().shutdown(); + System.out.println("Consumed time: " + (float) ((endTime - startTime) / 1000.00) + " s, or " + + (float) ((endTime - startTime) / 60000.00) + " min."); + if ( args.isDv8map()) { + DV8MappingFileBuilder dv8MapfileBuilder = new DV8MappingFileBuilder(langProcessor.supportedRelations()); + dv8MapfileBuilder.create(outputDir+ File.separator+"depends-dv8map.mapping"); + } + } + + private static String[] appendAllFoldersToIncludePath(String inputDir, String[] includeDir) { + FolderCollector includePathCollector = new FolderCollector(); + List additionalIncludePaths = includePathCollector.getFolders(inputDir); + additionalIncludePaths.addAll(Arrays.asList(includeDir)); + includeDir = additionalIncludePaths.toArray(new String[] {}); + return includeDir; + } + + private static DependencyGenerator getDependencyGenerator(DependsCommand app, String inputDir) throws ParameterException { + FilenameWritter filenameWritter = new EmptyFilenameWritter(); + if (!StringUtils.isEmpty(app.getNamePathPattern())) { + if (app.getNamePathPattern().equals("dot")|| + app.getNamePathPattern().equals(".")) { + filenameWritter = new DotPathFilenameWritter(); + }else if (app.getNamePathPattern().equals("unix")|| + app.getNamePathPattern().equals("/")) { + filenameWritter = new UnixPathFilenameWritter(); + }else if (app.getNamePathPattern().equals("windows")|| + app.getNamePathPattern().equals("\\")) { + filenameWritter = new WindowsPathFilenameWritter(); + }else{ + throw new ParameterException("Unknown name pattern paremater:" + app.getNamePathPattern()); + } + } + + + /* by default use file dependency generator */ + DependencyGenerator dependencyGenerator = new FileDependencyGenerator(); + if (!StringUtils.isEmpty(app.getGranularity())) { + /* method parameter means use method generator */ + if (app.getGranularity().equals("method")) + dependencyGenerator = new FunctionDependencyGenerator(); + else if (app.getGranularity().equals("structure")) + dependencyGenerator = new StructureDependencyGenerator(); + else if (app.getGranularity().equals("file")) + /*no action*/; + else if (app.getGranularity().startsWith("L")) + /*no action*/; + else + throw new ParameterException("Unknown granularity parameter:" + app.getGranularity()); + } + + if (app.isStripLeadingPath() || + app.getStrippedPaths().length>0) { + dependencyGenerator.setLeadingStripper(new LeadingNameStripper(app.isStripLeadingPath(), inputDir, app.getStrippedPaths())); + } + + if (app.isDetail()) { + dependencyGenerator.setGenerateDetail(true); + } + + dependencyGenerator.setFilenameRewritter(filenameWritter); + return dependencyGenerator; + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/MatrixLevelReducer.java b/src/main/java/com/educoder/bridge/tmp/MatrixLevelReducer.java new file mode 100644 index 0000000..4cb0d5d --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/MatrixLevelReducer.java @@ -0,0 +1,131 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.matrix.transform; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; + +import depends.matrix.core.DependencyMatrix; +import depends.matrix.core.DependencyPair; +import depends.matrix.core.DependencyValue; + +public class MatrixLevelReducer { + + private DependencyMatrix origin; + private int level; + HashMap nodesMap = new HashMap<>(); + + public MatrixLevelReducer(DependencyMatrix matrix, String levelString) { + this.origin = matrix; + this.level = stringToPositiveInt(levelString); + } + + public DependencyMatrix shrinkToLevel() { + if (level < 0) + return origin; + + ArrayList reMappedNodes = new ArrayList<>(); + for (String node : origin.getNodes()) { + String newNode = calcuateNodeAtLevel(node, level); + if (!reMappedNodes.contains(newNode)) { + reMappedNodes.add(newNode); + } + } + // sort nodes by name + reMappedNodes.sort(new Comparator() { + @Override + public int compare(String o1, String o2) { + return o1.compareTo(o2); + } + }); + DependencyMatrix ordered = new DependencyMatrix(); + for (int id=0;id 0) { + if (sb.length()>0) + sb.append(splitter); + sb.append(segments[i]); + count++; + } + } + return prefix + sb.toString(); + } + + private Integer translateToNewId(Integer id) { + String newNode = calcuateNodeAtLevel(origin.getNodeName(id), level); + return nodesMap.get(newNode); + } + + private int stringToPositiveInt(String level) { + int result = -1; + try { + result = Integer.parseInt(level); + } catch (Exception e) { + result = -1; + } + if (result <= 0) { + result = -1; + } + return result; + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/ParserTest.java b/src/main/java/com/educoder/bridge/tmp/ParserTest.java new file mode 100644 index 0000000..4df73d0 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/ParserTest.java @@ -0,0 +1,109 @@ +package depends.extractor; + +import depends.entity.ContainerEntity; +import depends.entity.Entity; +import depends.entity.FunctionEntity; +import depends.entity.VarEntity; +import depends.entity.repo.EntityRepo; +import depends.relations.BindingResolver; +import depends.relations.IBindingResolver; +import depends.relations.Relation; +import depends.relations.RelationCounter; +import multilang.depends.util.file.TemporaryFile; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; + +import static org.junit.Assert.fail; + +public abstract class ParserTest { + protected EntityRepo entityRepo ; + protected IBindingResolver bindingResolver; + protected AbstractLangProcessor langProcessor; + + protected void init(){ + entityRepo = langProcessor.getEntityRepo(); + bindingResolver = new BindingResolver(langProcessor,true,false); + langProcessor.bindingResolver = bindingResolver; + TemporaryFile.reset(); + } + + protected void init(boolean duckTypingDeduce){ + entityRepo = langProcessor.getEntityRepo(); + bindingResolver = new BindingResolver(langProcessor,false,duckTypingDeduce); + langProcessor.bindingResolver = bindingResolver; + TemporaryFile.reset(); + } + + public Set resolveAllBindings() { + Set result = bindingResolver.resolveAllBindings(langProcessor.isEagerExpressionResolve()); + new RelationCounter(entityRepo,langProcessor, bindingResolver).computeRelations(); + return result; + } + + protected Set resolveAllBindings(boolean callAsImpl) { + Set result = bindingResolver.resolveAllBindings(langProcessor.isEagerExpressionResolve()); + new RelationCounter(entityRepo,langProcessor, bindingResolver).computeRelations(); + return result; + } + + protected void assertNotContainsRelation(Entity inEntity, String dependencyType, String dependedEntityFullName) { + for (Relation r:inEntity.getRelations()) { + if (r.getType().equals(dependencyType)) { + if (r.getEntity().getQualifiedName().equals(dependedEntityFullName)) { + fail("found unexpected relation: type = " + dependencyType + " to entity " + dependedEntityFullName); + } + } + } + } + + protected void assertContainsRelation(Entity inEntity, String dependencyType, String dependedEntityFullName) { + Relation relation = null; + for (Relation r:inEntity.getRelations()) { + if (r.getType().equals(dependencyType)) { + relation = r; + if (r.getEntity()==null) continue; + if (r.getEntity().getQualifiedName().equals(dependedEntityFullName)) + return; + } + } + if (relation==null) { + fail("cannot found relation type of "+ dependencyType); + }else { + fail("cannot found relation type of " + dependencyType + " to entity " + dependedEntityFullName); + } + } + + protected void assertContainsVarWithRawName(Entity entity, String name) { + ContainerEntity container = (ContainerEntity)entity; + ArrayList vars = container.getVars(); + for (VarEntity var:vars) { + if (var.getRawName().uniqName().equals(name)) { + return; + } + } + fail("cannot found var with rawname " + name); + } + + protected void assertContainsParametersWithRawName(FunctionEntity function, String name) { + Collection vars = function.getParameters(); + for (VarEntity var:vars) { + if (var.getRawName().uniqName().equals(name)) { + return; + } + } + fail("cannot found parameter with rawname " + name); + } + + protected void assertContainReturnType(FunctionEntity function, String name) { + Collection types = function.getReturnTypes(); + for (Entity type:types) { + if (type.getRawName().uniqName().equals(name)) { + return; + } + } + fail("cannot found return type with rawname " + name); + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/PomListener.java b/src/main/java/com/educoder/bridge/tmp/PomListener.java new file mode 100644 index 0000000..8bcd2f1 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/PomListener.java @@ -0,0 +1,165 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.pom; + +import depends.entity.Expression; +import depends.entity.GenericName; +import depends.entity.VarEntity; +import depends.entity.repo.EntityRepo; +import depends.extractor.FileParser; +import depends.extractor.xml.XMLParser.ElementContext; +import depends.extractor.xml.XMLParserBaseListener; +import depends.relations.IBindingResolver; +import org.antlr.v4.runtime.ParserRuleContext; + +import java.io.IOException; +import java.util.List; +import java.util.Stack; + +public class PomListener extends XMLParserBaseListener { + private PomHandlerContext context; + private EntityRepo entityRepo; + PomArtifactEntity currentEntity; + private VarEntity currentVar; + Expression currentExpression; + private PomParent pomParent; + private PomProcessor parseCreator; + private List includePaths; + private IBindingResolver IBindingResolver; + private Stack pomCoords= new Stack<>(); + + public PomListener(String fileFullPath, EntityRepo entityRepo, List includePaths, PomProcessor parseCreator, + IBindingResolver bindingResolver) { + this.context = new PomHandlerContext(entityRepo); + this.entityRepo = entityRepo; + this.parseCreator = parseCreator; + this.includePaths = includePaths; + this.IBindingResolver = bindingResolver; + context.startFile(fileFullPath); + } + + @Override + public void enterElement(ElementContext ctx) { + String name = ctx.Name(0).getText(); + if (name.equals("project")) { + pomCoords.push(new PomCoords()); + currentEntity = new PomArtifactEntity("", context.currentFile(), entityRepo.generateId()); + } else if (name.equals("plugin")) { + pomCoords.push(new PomCoords()); + currentExpression = new Expression(entityRepo.generateId()); + currentExpression.setRawType(""); + } else if (name.equals("dependency")) { + pomCoords.push(new PomCoords()); + currentVar = new VarEntity(GenericName.build(""), GenericName.build(""), + currentEntity, entityRepo.generateId()); + } else if (name.equals("parent")) { + pomCoords.push(new PomCoords()); + pomParent = new PomParent(""); + } + + // Add attribute + else if (name.equals("groupId")) { + peekPomCoords().groupId = getContentValueOf(ctx); + } else if (name.equals("artifactId")) { + peekPomCoords().artifactId = getContentValueOf(ctx); + } else if (name.equals("version")) { + peekPomCoords().version = getContentValueOf(ctx); + } else if ("properties".equals(getParentElementName(ctx))) { + if (ctx.content() != null) { + currentEntity.addProperty(name, getContentValueOf(ctx)); + } + } + super.enterElement(ctx); + } + + private PomCoords peekPomCoords() { + return pomCoords.peek(); + } + + private String getContentValueOf(ElementContext ctx) { + String text = ctx.content().getText(); + if (text == null) + return ""; + if (text.contains("${")) + text = currentEntity.replaceProperty(text); + return text; + } + + @Override + public void exitElement(ElementContext ctx) { + String name = ctx.Name(0).getText(); + if (name.equals("project")) { + if (pomParent != null) { + peekPomCoords().fillFromIfNull(pomParent); + } + currentEntity.setRawName(peekPomCoords().getGenericNamePath()); + currentEntity.setQualifiedName(currentEntity.getRawName().uniqName()); + entityRepo.add(currentEntity); + pomCoords.pop(); + } else if (name.equals("plugin")) { + peekPomCoords().sureFillVersion(includePaths); + currentExpression.setRawType(peekPomCoords().getGenericNamePath()); + currentEntity.addExpression(ctx, currentExpression); + pomCoords.pop(); + } else if (name.equals("dependency")) { + peekPomCoords().sureFillVersion(includePaths); + currentVar.setRawType(peekPomCoords().getGenericNamePath()); + //TODO: Depends currently has a limitation: var name cannot be same as var type + //To be fixed in future + currentVar.setRawName(new GenericName("_" + currentVar.getRawType().getName())); + currentEntity.addVar(currentVar); + pomCoords.pop(); + } else if (name.equals("parent")) { + pomParent.buildFrom(peekPomCoords()); + context.currentFile().addImport(pomParent); + String parentFileName = new PomLocator(includePaths, pomParent).getLocation(); + if (parentFileName != null) { + FileParser importedParser = parseCreator.createFileParser(); + try { + importedParser.parse(parentFileName); + } catch (IOException e) { + System.err.println("error occurred during parse "+ parentFileName); + } + } + pomCoords.pop(); + context.currentFile().inferLocalLevelEntities(IBindingResolver); + } + super.exitElement(ctx); + } + + + private String getParentElementName(ParserRuleContext node) { + node = node.getParent(); + if (node == null) + return "project"; + node = node.getParent(); + if (!(node instanceof ElementContext)) + return "project"; + + ElementContext p = (ElementContext) node; + String name = p.Name().get(0).getText(); + return name; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/PreprocessorHandler.java b/src/main/java/com/educoder/bridge/tmp/PreprocessorHandler.java new file mode 100644 index 0000000..decaedf --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/PreprocessorHandler.java @@ -0,0 +1,126 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.cpp.cdt; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +import org.eclipse.cdt.core.dom.ast.IASTFileLocation; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIncludeStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorStatement; +import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility; + +import multilang.depends.util.file.FileTraversal; +import multilang.depends.util.file.FileTraversal.IFileVisitor; +import multilang.depends.util.file.FileUtil; + +public class PreprocessorHandler { + private List includePaths; + private String inputSrcPath; + private HashSet allFiles = new HashSet<>(); + public PreprocessorHandler(String inputSrcPath, List includePaths){ + this.inputSrcPath = inputSrcPath; + this.includePaths = includePaths; + buildAllFiles(); + } + + class AllFileVisitor implements IFileVisitor{ + @Override + public void visit(File file) { + try { + allFiles.add(file.getCanonicalPath()); + } catch (IOException e) { + } + } + } + private void buildAllFiles() { + allFiles = new HashSet<>(); + AllFileVisitor v = new AllFileVisitor(); + if (inputSrcPath!=null) { + FileTraversal ft = new FileTraversal(v,false,true); + ft.travers(inputSrcPath); + } + for (String includePath:includePaths) { + FileTraversal ft = new FileTraversal(v,false,true); + ft.travers(includePath); + } + } + + private boolean existFile(String checkPath) { + checkPath = FileUtil.uniformPath(checkPath); + return allFiles.contains(checkPath); + } + + public List getDirectIncludedFiles(IASTPreprocessorStatement[] statements, String fileLocation) { + ArrayList includedFullPathNames = new ArrayList<>(); + for (int statementIndex=0;statementIndex searchPath = new ArrayList<>(); + searchPath.add(locationDir); + searchPath.addAll(includePaths); + for (String includePath:searchPath) { + String checkPath = ScannerUtility.createReconciledPath(includePath,path); + if (existFile(checkPath)) { + return FileUtil.uniqFilePath(checkPath); + } + } + return ""; + } + + + public List getIncludePaths() { + return includePaths; + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/PythonBuiltInType.java b/src/main/java/com/educoder/bridge/tmp/PythonBuiltInType.java new file mode 100644 index 0000000..934c870 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/PythonBuiltInType.java @@ -0,0 +1,96 @@ +package depends.extractor.python; + +import depends.entity.FunctionCall; +import depends.entity.FunctionEntity; +import depends.entity.GenericName; +import depends.entity.TypeEntity; +import depends.entity.repo.BuiltInType; +import depends.relations.FunctionMatcher; + +import java.util.ArrayList; +import java.util.List; + +public class PythonBuiltInType extends BuiltInType { + + public static String[] BUILT_IN_FUNCTIONS = { "abs", "delattr", "hash", "memoryview", "set", "all", "dict", "help", + "min", "setattr", "any", "dir", "hex", "next", "slice", "exit", "ascii", "divmod", "id", "object", "sorted", + "bin", "enumerate", "input", "oct", "staticmethod", "bool", "eval", "int", "open", "str", "breakpoint", + "exec", "isinstance", "ord", "sum", "bytearray", "filter", "issubclass", "pow", "super", "bytes", "float", + "iter", "print", "tuple", "callable", "format", "len", "property", "type", "chr", "frozenset", "list", + "range", "vars", "classmethod", "getattr", "locals", "repr", "zip", "compile", "globals", "map", "reversed", + "__import__", "complex", "hasattr", "max", "round" }; + + /** + * methods of built-in String + */ + public static String[] BUILT_IN_STRING_METHODS = { "capitalize", "center", "casefold", "count", "endswith", + "expandtabs", "encode", "find", "format", "index", "isalnum", "isalpha", "isdecimal", "isdigit", + "isidentifier", "islower", "isnumeric", "isprintable", "isspace", "istitle", "isupper", "join", "ljust", + "rjust", "lower", "upper", "swapcase", "lstrip", "rstrip", "strip", "partition", "maketrans", "rpartition", + "translate", "replace", "rfind", "rindex", "split", "rsplit", "splitlines", "startswith", "title", "zfill", + "format_map" }; + + /** + * methods of built-in List + */ + public static String[] BUILT_IN_LIST_METHODS = { "index", "append", "extend", "insert", "remove", "count", "pop", + "reverse", "sort", "copy", "clear" }; + + /** + * methods of built-in Tuple + */ + public static String[] BUILT_IN_TUPLE_METHODS = { "index", "count" }; + + /** + * methods of built-in Dict + */ + public static String[] BUILT_IN_DICT_METHODS = { "clear", "copy", "fromkeys", "get", "items", "keys", "popitem", + "setdefault", "pop", "values", "update", }; + + /** + * methods of built-in Set + */ + public static String[] BUILT_IN_SET_METHODS = { "remove", "add", "copy", "clear", "difference", "difference_update", + "discard", "intersection", "intersection_update", "isdisjoint", "issubset", "pop", "symmetric_difference", + "symmetric_difference_update", "union", "update" }; + + /** + * methods of built-in File + */ + public static String[] BUILT_IN_FILE_METHOD = { "close", "flush", "fileno", "isatty", "next", "read", "readline", + "readlines", "seek", "tell", "truncate", "write", "writelines" }; + + List buildInTypes = new ArrayList<>(); + + + + public PythonBuiltInType() { + addBuildInType(BUILT_IN_FILE_METHOD); + addBuildInType(BUILT_IN_SET_METHODS); + addBuildInType(BUILT_IN_DICT_METHODS); + addBuildInType(BUILT_IN_TUPLE_METHODS); + addBuildInType(BUILT_IN_LIST_METHODS); + addBuildInType(BUILT_IN_STRING_METHODS); + } + + private void addBuildInType(String[] methods) { + TypeEntity type = new TypeEntity(); + for (String method:methods) { + FunctionEntity func = new FunctionEntity(GenericName.build(method),type,-1,GenericName.build("")); + type.addFunction(func); + } + buildInTypes.add(type); + } + + @Override + public boolean isBuildInTypeMethods(List functionCalls) { + for (TypeEntity type:buildInTypes) { + FunctionMatcher functionMatcher = new FunctionMatcher(type.getFunctions()); + if (functionMatcher.containsAll(functionCalls)) { + return true; + } + } + return false; + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/PythonCodeListener.java b/src/main/java/com/educoder/bridge/tmp/PythonCodeListener.java new file mode 100644 index 0000000..e406a13 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/PythonCodeListener.java @@ -0,0 +1,371 @@ +package depends.extractor.python.union; + +import depends.entity.*; +import depends.entity.repo.EntityRepo; +import depends.extractor.python.NameAliasImport; +import depends.extractor.python.PythonHandlerContext; +import depends.extractor.python.PythonParser.*; +import depends.extractor.python.PythonParserBaseListener; +import depends.extractor.IncludedFileLocator; +import depends.importtypes.FileImport; +import depends.relations.IBindingResolver; +import multilang.depends.util.file.FileUtil; +import org.antlr.v4.runtime.ParserRuleContext; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class PythonCodeListener extends PythonParserBaseListener{ + private final PythonHandlerContext context; + private final ExpressionUsage expressionUsage; + private final EntityRepo entityRepo; + private final IncludedFileLocator includeFileLocator; + private final PythonProcessor pythonProcessor; + private final IBindingResolver bindingResolver; + public PythonCodeListener(String fileFullPath, EntityRepo entityRepo, IBindingResolver bindingResolver, + IncludedFileLocator includeFileLocator, PythonProcessor pythonProcessor) { + this.context = new PythonHandlerContext(entityRepo, bindingResolver); + this.expressionUsage = new ExpressionUsage(context, entityRepo, bindingResolver); + FileEntity fileEntity = context.startFile(fileFullPath); + this.entityRepo = entityRepo; + this.includeFileLocator = includeFileLocator; + this.bindingResolver = bindingResolver; + this.pythonProcessor = pythonProcessor; + + String dir = FileUtil.uniqFilePath(FileUtil.getLocatedDir(fileFullPath)); + if (entityRepo.getEntity(dir) == null) { + PackageEntity pacakgeEntity = new PackageEntity(dir, entityRepo.generateId()); + entityRepo.add(pacakgeEntity); + } + + PackageEntity packageEntity = (PackageEntity) entityRepo.getEntity(dir); + String moduleName = fileEntity.getRawName().uniqName().substring(packageEntity.getRawName().uniqName().length() + 1); + if (moduleName.endsWith(".py")) + moduleName.substring(0, moduleName.length() - ".py".length()); + Entity.setParent(fileEntity, packageEntity); + packageEntity.addChild(FileUtil.getShortFileName(fileEntity.getRawName().uniqName()).replace(".py", ""), fileEntity); + } + @Override + public void enterImport_stmt(Import_stmtContext ctx) { + String moduleName = null; + for(Dotted_as_nameContext dotted_as_name:ctx.dotted_as_names().dotted_as_name()){ + moduleName = getName(dotted_as_name.dotted_name()); + String aliasName = moduleName; + if (dotted_as_name.name()!=null) { + aliasName = dotted_as_name.name().getText(); + } + List fullNames = foundImportedModuleOrPackage(0,moduleName); + + for (String fullName:fullNames) { + if (FileUtil.existFile(fullName) && !(FileUtil.isDirectory(fullName))) { + context.foundNewImport(new FileImport(fullName)); + } + context.foundNewImport(new NameAliasImport(fullName, entityRepo.getEntity(fullName), aliasName)); + } + } + super.enterImport_stmt(ctx); + } + @Override + public void enterFrom_stmt(From_stmtContext ctx) { + String moduleName = null; + if (ctx.dotted_name() != null) { + moduleName = ctx.dotted_name().getText(); + } + int prefixDotCount = getDotCounter(ctx); + + List fullNames = foundImportedModuleOrPackage(prefixDotCount, moduleName); + for (String fullName:fullNames) { + + if (ctx.import_as_names() == null) {// import * + ContainerEntity moduleEntity = (ContainerEntity) (entityRepo.getEntity(fullName)); + if (moduleEntity != null) { + for (Entity child:moduleEntity.getChildren()) { + context.foundNewImport(new NameAliasImport(fullName, child, child.getRawName().uniqName())); + context.foundNewAlias(child.getRawName(), child); + } + if (moduleEntity instanceof PackageEntity) { + for (Entity file : moduleEntity.getChildren()) { + if (file instanceof FileEntity) { + String fileName = file.getRawName().uniqName().substring(fullName.length()); + context.foundNewImport(new NameAliasImport(file.getRawName().uniqName(), file, fileName)); + context.foundNewAlias(GenericName.build(FileUtil.getShortFileName(fileName).replace(".py", "")), file); + }else { + context.foundNewImport(new NameAliasImport(file.getRawName().uniqName(), file, file.getRawName().uniqName())); + context.foundNewAlias(GenericName.build(FileUtil.getShortFileName(file.getRawName().uniqName())), file); + } + } + } + if (moduleEntity instanceof FileEntity) { + String fileName = moduleEntity.getRawName().uniqName().substring(fullName.length()); + context.foundNewImport(new NameAliasImport(moduleEntity.getRawName().uniqName(), moduleEntity, fileName)); + } + } + } else { + for (Import_as_nameContext item : ctx.import_as_names().import_as_name()) { + String name = item.name(0).getText(); + String alias = name; + if (item.name().size() > 1) + alias = item.name(1).getText(); + if (FileUtil.isDirectory(fullName)) { + String fileName = fullName + File.separator + name + ".py"; + if (FileUtil.existFile(fileName) && !(FileUtil.isDirectory(fileName))) { + context.foundNewImport(new FileImport(fileName)); + } + } + if (FileUtil.existFile(fullName) && !(FileUtil.isDirectory(fullName))) { + context.foundNewImport(new FileImport(fullName)); + } + Entity itemEntity = bindingResolver.resolveName(entityRepo.getEntity(fullName), GenericName.build(name), true); + if (itemEntity != null) { + context.foundNewAlias(GenericName.build(alias), itemEntity); + context.foundNewImport(new NameAliasImport(itemEntity.getQualifiedName(), itemEntity, alias)); + } + } + } + } + super.enterFrom_stmt(ctx); + } + + + private int getDotCounter(From_stmtContext ctx) { + int total = 0; + if (ctx.DOT()!=null){ + total = ctx.DOT().size(); + } + if (ctx.ELLIPSIS()!=null) { + total += ctx.ELLIPSIS().size()*3; + } + return total; + } + private List foundImportedModuleOrPackage(int prefixDotCount, String originalName) { + String dir = FileUtil.getLocatedDir(context.currentFile().getRawName().uniqName()); + String preFix = ""; + for (int i = 0; i < prefixDotCount - 1; i++) { + preFix = preFix + ".." + File.separator; + } + dir = dir + File.separator + preFix; + String fullName = null; + if (originalName != null) { + String importedName = originalName.replace(".", File.separator); + fullName = includeFileLocator.uniqFileName(dir, importedName); + if (fullName == null) { + fullName = includeFileLocator.uniqFileName(dir, importedName + ".py"); + } + } else { + fullName = FileUtil.uniqFilePath(dir); + } + if (fullName != null) { + if (FileUtil.isDirectory(fullName)) { + if (!FileUtil.uniqFilePath(fullName).equals(FileUtil.uniqFilePath(dir))) { + File d = new File(fullName); + File[] files = d.listFiles(); + for (File file : files) { + if (!file.isDirectory()) { + if (file.getAbsolutePath().endsWith(".py")) { + visitIncludedFile(FileUtil.uniqFilePath(file.getAbsolutePath())); + } + } + } + } + } else { + visitIncludedFile(fullName); + } + } + ArrayList r = new ArrayList<>(); + if (fullName==null) return r; + r.add(fullName); + if (FileUtil.existFile(fullName+File.separator + "__init__.py")) { + r.add( fullName+File.separator +"__init__.py"); + } + return r; + } + + private void visitIncludedFile(String fullName) { + PythonFileParser importedParser = new PythonFileParser(entityRepo, includeFileLocator, bindingResolver, + pythonProcessor); + try { + importedParser.parse(fullName); + } catch (IOException e) { + e.printStackTrace(); + } + } + @Override + public void enterFuncdef(FuncdefContext ctx) { + String functionName =""; + String name = getName(ctx.name()); + if (name!=null) { + functionName = name; + } + + FunctionEntity method = context.foundMethodDeclarator(functionName,ctx.getStart().getLine()); + if (ctx.typedargslist()!=null) { + List parameters = getParameterList(ctx.typedargslist().def_parameters()); + for (String param : parameters) { + VarEntity paramEntity = context.addMethodParameter(param); + if (param.equals("self")) { + paramEntity.setType(context.currentType()); + } + } + } + super.enterFuncdef(ctx); + } + + @Override + public void exitFuncdef(FuncdefContext ctx) { + context.exitLastedEntity(); + super.exitFuncdef(ctx); + } + + + @Override + public void enterClassdef(ClassdefContext ctx) { + String name = getName(ctx.name()); + TypeEntity type = context.foundNewType(name, ctx.getStart().getLine()); + List baseClasses = getArgList(ctx.arglist()); + baseClasses.forEach(base -> type.addExtends(GenericName.build(base))); + + super.enterClassdef(ctx); + } + + + @Override + public void exitClassdef(ClassdefContext ctx) { + context.exitLastedEntity(); + super.exitClassdef(ctx); + } + + private List getParameterList(List def_parameters) { + List result = new ArrayList<>(); + for (Def_parametersContext params:def_parameters) { + for (Def_parameterContext param:params.def_parameter()) { + String p = getName( param.named_parameter().name()); + result.add(p); + } + + } + return result; + } + private String getName(NameContext name) { + return name.getText(); + } + + private String getName(Dotted_nameContext dotted_name) { + return dotted_name.getText(); + } + + private String getDecoratedName(Class_or_func_def_stmtContext ctx) { + if (ctx.classdef()!=null) { + return getName(ctx.classdef().name()); + }else if (ctx.funcdef()!=null) { + return getName(ctx.funcdef().name()); + } + return null; + } + + private List getArgList(ArglistContext arglist) { + List r = new ArrayList<>(); + if (arglist==null) return r; + if (arglist.argument() == null) return r; + if (arglist.argument().isEmpty()) return r; + arglist.argument().forEach(arg->r.add(arg.getText())); + return r; + } + + + /** + * class_or_func_def_stmt: decorator+ (classdef | funcdef); + */ + @Override + public void exitClass_or_func_def_stmt(Class_or_func_def_stmtContext ctx) { + String decoratedName = getDecoratedName(ctx); + if (decoratedName!=null) { + Entity entity = context.foundEntityWithName(GenericName.build(decoratedName)); + entity.setLine(ctx.getStart().getLine()); + + if (entity instanceof DecoratedEntity) { + for (DecoratorContext decorator: ctx.decorator()) { + String decoratorName = getName(decorator.dotted_name()); + ((DecoratedEntity) entity).addAnnotation(GenericName.build(decoratorName)); + } + } + } + super.exitClass_or_func_def_stmt(ctx); + } + + + @Override + public void enterGlobal_stmt(Global_stmtContext ctx) { + for (NameContext name:ctx.name()){ + VarEntity var = context.foundGlobalVarDefinition(context.currentFile(), name.getText(),ctx.getStart().getLine()); + } + super.enterGlobal_stmt(ctx); + } + + @Override + public void enterEveryRule(ParserRuleContext ctx) { + expressionUsage.foundExpression(ctx); + super.enterEveryRule(ctx); + } + @Override + public void enterExpr_stmt(Expr_stmtContext ctx) { + expressionUsage.startExpr(); + super.enterExpr_stmt(ctx); + } + @Override + public void exitExpr_stmt(Expr_stmtContext ctx) { + expressionUsage.stopExpr(); + super.exitExpr_stmt(ctx); + } + @Override + public void enterDel_stmt(Del_stmtContext ctx) { + expressionUsage.startExpr(); + super.enterDel_stmt(ctx); + } + @Override + public void exitDel_stmt(Del_stmtContext ctx) { + expressionUsage.stopExpr(); + super.exitDel_stmt(ctx); + } + @Override + public void enterReturn_stmt(Return_stmtContext ctx) { + expressionUsage.startExpr(); + super.enterReturn_stmt(ctx); + } + @Override + public void exitReturn_stmt(Return_stmtContext ctx) { + expressionUsage.stopExpr(); + super.exitReturn_stmt(ctx); + } + @Override + public void enterRaise_stmt(Raise_stmtContext ctx) { + expressionUsage.startExpr(); + super.enterRaise_stmt(ctx); + } + @Override + public void exitRaise_stmt(Raise_stmtContext ctx) { + expressionUsage.stopExpr(); + super.exitRaise_stmt(ctx); + } + @Override + public void enterYield_stmt(Yield_stmtContext ctx) { + expressionUsage.startExpr(); + super.enterYield_stmt(ctx); + } + @Override + public void exitYield_stmt(Yield_stmtContext ctx) { + expressionUsage.stopExpr(); + super.exitYield_stmt(ctx); + } + @Override + public void enterAssert_stmt(Assert_stmtContext ctx) { + expressionUsage.startExpr(); + super.enterAssert_stmt(ctx); + } + @Override + public void exitAssert_stmt(Assert_stmtContext ctx) { + expressionUsage.stopExpr(); + super.exitAssert_stmt(ctx); + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/PythonImportTest.java b/src/main/java/com/educoder/bridge/tmp/PythonImportTest.java new file mode 100644 index 0000000..6b36c4f --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/PythonImportTest.java @@ -0,0 +1,284 @@ +package depends.extractor.python; + +import depends.deptypes.DependencyType; +import depends.entity.Entity; +import depends.entity.FileEntity; +import depends.entity.FunctionEntity; +import depends.entity.MultiDeclareEntities; +import depends.extractor.python.union.PythonFileParser; +import multilang.depends.util.file.FileUtil; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +public class PythonImportTest extends PythonParserTest { + @Before + public void setUp() { + super.init(); + } + + @Test + public void should_parse_module_in_same_package() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/imported_a.py", + "./src/test/resources/python-code-examples/importing.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + Entity file = entityRepo.getEntity(FileUtil.uniqFilePath(srcs[1])); + this.assertContainsRelation(file, DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[0])); + } + + + @Test + public void should_parse_module_in_same_package_order_robust() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/importing.py", + "./src/test/resources/python-code-examples/imported_a.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + Entity file = entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])); + this.assertContainsRelation(file, DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + this.assertContainsRelation(file, DependencyType.CALL,withPackageName(srcs[0],"foo")); + } + + @Test + public void should_parse_module_in_same_package_with_alias() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/importing_with_alias.py", + "./src/test/resources/python-code-examples/imported_a.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + Entity file = entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])); + this.assertContainsRelation(file, DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + this.assertContainsRelation(file, DependencyType.CALL,withPackageName(srcs[0],"foo")); + } + + @Test + public void should_parse_module_in_from_importing() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/from_importing.py", + "./src/test/resources/python-code-examples/imported_a.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + Entity file = entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])); + this.assertContainsRelation(file, DependencyType.CALL,withPackageName(srcs[0],"foo")); + this.assertContainsRelation(file, DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + } + + + @Test + public void should_parse_module_in_from_importing_star() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/from_importing_star.py", + "./src/test/resources/python-code-examples/imported_a.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + Entity file = entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])); + this.assertContainsRelation(file, DependencyType.CALL,withPackageName(srcs[0],"foo")); + this.assertContainsRelation(file, DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + } + + + @Test + public void should_parse_import_with_multi_dots() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/from_importing_multidot.py", + "./src/test/resources/python-code-examples/pkg/imported.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + Entity file = entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])); + this.assertContainsRelation(file, DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + this.assertContainsRelation(file, DependencyType.CALL,withPackageName(srcs[1],"foo")); + } + + @Test + public void should_parse_import_with_prefix_dots() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/import_with_dir/importing.py", + "./src/test/resources/python-code-examples/import_with_dir/imported_a.py", + "./src/test/resources/python-code-examples/import_with_dir/subdir/importing.py", + "./src/test/resources/python-code-examples/import_with_dir/subdir/importing2.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + this.assertContainsRelation(entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])), DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + this.assertContainsRelation(entityRepo.getEntity(FileUtil.uniqFilePath(srcs[2])), DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + this.assertContainsRelation(entityRepo.getEntity(FileUtil.uniqFilePath(srcs[3])), DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + } + + @Test + public void should_parse_import_with_prefix_dots2() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/import_with_dir/subdir/importing2.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + } + + + @Test + public void should_import_from_package__init__file() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/import_from_init/importing.py", + "./src/test/resources/python-code-examples/import_from_init/pkg/__init__.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + this.assertContainsRelation(entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])), DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + } + + + @Test + public void should_not_bypass_import_in_same_dir() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/import_of_same_dir/pkg/importing.py", + "./src/test/resources/python-code-examples/import_of_same_dir/pkg/a.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + this.assertContainsRelation(entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])), DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1])); + } + + + @Test + public void should_resolve_symbols_of_imported_in_same_dir() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/import_of_same_dir/pkg/importing.py", + "./src/test/resources/python-code-examples/import_of_same_dir/pkg/a.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + FunctionEntity func = (FunctionEntity) entityRepo.getEntity(withPackageName(srcs[0],"test")); + this.assertContainsRelation(func, DependencyType.CALL, withPackageName(srcs[1],"foo")); + } + + + + @Test + public void should_resolve_symbols_of_ducktyping() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/duck_typing/forest.py", + "./src/test/resources/python-code-examples/duck_typing/animals.py", + "./src/test/resources/python-code-examples/duck_typing/controller.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + + MultiDeclareEntities funcs = (MultiDeclareEntities) entityRepo.getEntity(withPackageName(srcs[1],"in_the_forest")); + Entity func = funcs.getEntities().get(0); + + this.assertContainsRelation(func, DependencyType.CALL, withPackageName(srcs[1],"Duck.quack")); + this.assertContainsRelation(func, DependencyType.CALL, withPackageName(srcs[1],"Doge.quack")); + this.assertContainsRelation(func, DependencyType.CALL, withPackageName(srcs[1],"Bird.quack")); + } + + @Test + public void should_resolve_symbols_of_ducktyping2() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/duck_typing/animals.py", + "./src/test/resources/python-code-examples/duck_typing/forest.py", + "./src/test/resources/python-code-examples/duck_typing/controller.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + + MultiDeclareEntities funcs = (MultiDeclareEntities) entityRepo.getEntity(withPackageName(srcs[1],"in_the_forest")); + Entity func = funcs.getEntities().get(0); + this.assertContainsRelation(func, DependencyType.CALL, withPackageName(srcs[0],"Duck.quack")); + this.assertContainsRelation(func, DependencyType.CALL, withPackageName(srcs[0],"Bird.quack")); + this.assertContainsRelation(func, DependencyType.CALL, withPackageName(srcs[0],"Doge.quack")); + } + + @Test + public void should_resolve_imported_symbols() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/import_from_init/use_imported.py", + "./src/test/resources/python-code-examples/import_from_init/pkg/__init__.py", + "./src/test/resources/python-code-examples/import_from_init/pkg/core.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + FunctionEntity func = (FunctionEntity) entityRepo.getEntity(withPackageName(srcs[0],"bar")); + this.assertContainsRelation(func, DependencyType.CALL, withPackageName(srcs[2],"C")); + } + + @Test + public void should_resolve_imported_vars() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/import_from_var/use_imported.py", + "./src/test/resources/python-code-examples/import_from_var/pkg/core.py", + }; + + for (String src:srcs) { + PythonFileParser parser = createParser(); + parser.parse(src); + } + resolveAllBindings(); + FileEntity f = (FileEntity) entityRepo.getEntity(FileUtil.uniqFilePath(srcs[0])); + this.assertContainsRelation(f, DependencyType.CALL, withPackageName(srcs[1],"Core.foo")); + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/PythonLexerBase.java b/src/main/java/com/educoder/bridge/tmp/PythonLexerBase.java new file mode 100644 index 0000000..ca98458 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/PythonLexerBase.java @@ -0,0 +1,186 @@ +package depends.extractor.python.union; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonToken; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.Token; + +import depends.extractor.python.PythonLexer; + +import java.util.Stack; + +public abstract class PythonLexerBase extends Lexer { + public static int TabSize = 8; + + // The amount of opened braces, brackets and parenthesis. + private int _opened; + + // The stack that keeps track of the indentation level. + private Stack _indents = new Stack<>(); + + // A circular buffer where extra tokens are pushed on (see the NEWLINE and WS lexer rules). + private int _firstTokensInd; + private int _lastTokenInd; + private Token[] _buffer = new Token[32]; + private Token _lastToken; + + protected PythonLexerBase(CharStream input) { + super(input); + } + + @Override + public void emit(Token token) { + super.setToken(token); + + if (_buffer[_firstTokensInd] != null) + { + _lastTokenInd = IncTokenInd(_lastTokenInd); + + if (_lastTokenInd == _firstTokensInd) + { + // Enlarge buffer + Token[] newArray = new Token[_buffer.length * 2]; + int destInd = newArray.length - (_buffer.length - _firstTokensInd); + + System.arraycopy(_buffer, 0, newArray, 0, _firstTokensInd); + System.arraycopy(_buffer, _firstTokensInd, newArray, destInd, _buffer.length - _firstTokensInd); + + _firstTokensInd = destInd; + _buffer = newArray; + } + } + + _buffer[_lastTokenInd] = token; + _lastToken = token; + } + + @Override + public Token nextToken() { + // Check if the end-of-file is ahead and there are still some DEDENTS expected. + if (_input.LA(1) == EOF && _indents.size() > 0) + { + if (_buffer[_lastTokenInd] == null || _buffer[_lastTokenInd].getType() != PythonLexer.LINE_BREAK) + { + // First emit an extra line break that serves as the end of the statement. + emit(PythonLexer.LINE_BREAK); + } + + // Now emit as much DEDENT tokens as needed. + while (_indents.size() != 0) + { + emit(PythonLexer.DEDENT); + _indents.pop(); + } + } + + Token next = super.nextToken(); + + if (_buffer[_firstTokensInd] == null) + { + return next; + } + + Token result = _buffer[_firstTokensInd]; + _buffer[_firstTokensInd] = null; + + if (_firstTokensInd != _lastTokenInd) + { + _firstTokensInd = IncTokenInd(_firstTokensInd); + } + + return result; + } + + protected void HandleNewLine() { + emit(PythonLexer.NEWLINE, HIDDEN, getText()); + + char next = (char) _input.LA(1); + + // Process whitespaces in HandleSpaces + if (next != ' ' && next != '\t' && IsNotNewLineOrComment(next)) + { + ProcessNewLine(0); + } + } + + protected void HandleSpaces() { + char next = (char) _input.LA(1); + + if ((_lastToken == null || _lastToken.getType() == PythonLexer.NEWLINE) && IsNotNewLineOrComment(next)) + { + // Calculates the indentation of the provided spaces, taking the + // following rules into account: + // + // "Tabs are replaced (from left to right) by one to eight spaces + // such that the total number of characters up to and including + // the replacement is a multiple of eight [...]" + // + // -- https://docs.python.org/3.1/reference/lexical_analysis.html#indentation + + int indent = 0; + String text = getText(); + + for (int i = 0; i < text.length(); i++) { + indent += text.charAt(i) == '\t' ? TabSize - indent % TabSize : 1; + } + + ProcessNewLine(indent); + } + + emit(PythonLexer.WS, HIDDEN, getText()); + } + + protected void IncIndentLevel() { + _opened++; + } + + protected void DecIndentLevel() { + if (_opened > 0) { + --_opened; + } + } + + private boolean IsNotNewLineOrComment(char next) { + return _opened == 0 && next != '\r' && next != '\n' && next != '\f' && next != '#'; + } + + private void ProcessNewLine(int indent) { + emit(PythonLexer.LINE_BREAK); + + int previous = _indents.size() == 0 ? 0 : _indents.peek(); + + if (indent > previous) + { + _indents.push(indent); + emit(PythonLexer.INDENT); + } + else + { + // Possibly emit more than 1 DEDENT token. + while (_indents.size() != 0 && _indents.peek() > indent) + { + emit(PythonLexer.DEDENT); + _indents.pop(); + } + } + } + + private int IncTokenInd(int ind) { + return (ind + 1) % _buffer.length; + } + + private void emit(int tokenType) { + emit(tokenType, DEFAULT_TOKEN_CHANNEL, ""); + } + + private void emit(int tokenType, int channel, String text) { + int charIndex = getCharIndex(); + CommonToken token = new CommonToken(_tokenFactorySourcePair, tokenType, channel, charIndex - text.length(), charIndex); + token.setLine(getLine()); + token.setCharPositionInLine(getCharPositionInLine()); + token.setText(text); + + emit(token); + } +} + diff --git a/src/main/java/com/educoder/bridge/tmp/PythonParameterTypeDedudceTest.java b/src/main/java/com/educoder/bridge/tmp/PythonParameterTypeDedudceTest.java new file mode 100644 index 0000000..8fcb6fd --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/PythonParameterTypeDedudceTest.java @@ -0,0 +1,118 @@ +package depends.extractor.python; + +import depends.entity.CandidateTypes; +import depends.entity.FunctionEntity; +import depends.entity.TypeEntity; +import depends.entity.VarEntity; +import depends.extractor.FileParser; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.*; + +public class PythonParameterTypeDedudceTest extends PythonParserTest { + @Before + public void setUp() { + super.init(); + } + + @Test + public void test_deduce_type_of_parameter() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/deducetype_parameter.py", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + resolveAllBindings(); + String name = withPackageName(srcs[0],"test"); + FunctionEntity function = (FunctionEntity)( entityRepo.getEntity(name)); + VarEntity var = function.lookupVarLocally("t1"); + TypeEntity type = var.getType(); + assertTrue(type instanceof CandidateTypes); + assertEquals(2,((CandidateTypes)type).getCandidateTypes().size()); + } + + + @Test + public void test_deduce_type_of_builtIn() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/deducetype_builtin.py", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + resolveAllBindings(); + String name = withPackageName(srcs[0],"test"); + FunctionEntity function = (FunctionEntity)( entityRepo.getEntity(name)); + VarEntity var = function.lookupVarLocally("t1"); + TypeEntity type = var.getType(); + assertTrue(type == null); + } + + + @Test + public void test_deduce_type_of_builtIn_cannot_override() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/deducetype_builtin.py", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + resolveAllBindings(); + String name = withPackageName(srcs[0],"test2"); + FunctionEntity function = (FunctionEntity)( entityRepo.getEntity(name)); + VarEntity var = function.lookupVarLocally("t1"); + TypeEntity type = var.getType(); + assertTrue(type instanceof CandidateTypes); + assertEquals(1,((CandidateTypes)type).getCandidateTypes().size()); + } + + + @Ignore + public void test_deduce_type_of_non_param_var() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/deducetype_nonparam.py", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + resolveAllBindings(); + String name = withPackageName(srcs[0],"test"); + FunctionEntity function = (FunctionEntity)( entityRepo.getEntity(name)); + VarEntity var = function.lookupVarLocally("t2"); + TypeEntity type = var.getType(); + assertTrue(type instanceof CandidateTypes); + assertEquals(2,((CandidateTypes)type).getCandidateTypes().size()); + } + + @Test + public void test_expression_count_should_be_() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/python-code-examples/expression_reload_issue_test.py", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + resolveAllBindings(); + String name = withPackageName(srcs[0],"test_expression"); + FunctionEntity function = (FunctionEntity)( entityRepo.getEntity(name)); + assertNotNull(function); + } + + +} + diff --git a/src/main/java/com/educoder/bridge/tmp/RelationCounter.java b/src/main/java/com/educoder/bridge/tmp/RelationCounter.java new file mode 100644 index 0000000..c3a163f --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/RelationCounter.java @@ -0,0 +1,246 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.relations; + +import depends.deptypes.DependencyType; +import depends.entity.*; +import depends.entity.repo.EntityRepo; +import depends.extractor.AbstractLangProcessor; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +public class RelationCounter { + + private Collection entities; + private IBindingResolver bindingResolver; + private EntityRepo repo; + private boolean callAsImpl; + private AbstractLangProcessor langProcessor; + + public RelationCounter(EntityRepo repo, AbstractLangProcessor langProcessor, IBindingResolver bindingResolver) { + this.entities = repo.getFileEntities(); + this.bindingResolver = bindingResolver; + this.repo = repo; + this.callAsImpl = langProcessor.supportCallAsImpl(); + this.langProcessor = langProcessor; + } + + public void computeRelations() { + entities.forEach(entity-> + computeRelationOf(entity)); + } + + private void computeRelationOf(Entity entity) { + if (!entity.inScope()) + return; + if (entity instanceof FileEntity) { + computeImports((FileEntity)entity); + } + else if (entity instanceof FunctionEntity) { + computeFunctionRelations((FunctionEntity)entity); + } + else if (entity instanceof TypeEntity) { + computeTypeRelations((TypeEntity)entity); + } + if (entity instanceof ContainerEntity) { + computeContainerRelations((ContainerEntity)entity); + } + entity.getChildren().forEach(child->computeRelationOf(child)); + } + + + + private void computeContainerRelations(ContainerEntity entity) { + for (VarEntity var:entity.getVars()) { + if (var.getType()!=null) + entity.addRelation(buildRelation(entity,DependencyType.CONTAIN,var.getType(),var.getLocation())); + for (Entity type:var.getResolvedTypeParameters()) { + var.addRelation(buildRelation(var, DependencyType.PARAMETER,type,type.getLocation())); + } + } + for (Entity type:entity.getResolvedAnnotations()) { + entity.addRelation(buildRelation(entity,DependencyType.ANNOTATION,type)); + } + for (Entity type:entity.getResolvedTypeParameters()) { + entity.addRelation(buildRelation(entity,DependencyType.USE,type)); + } + for (ContainerEntity mixin:entity.getResolvedMixins()) { + entity.addRelation(buildRelation(entity,DependencyType.MIXIN,mixin)); + } + + entity.reloadExpression(repo); + if (!bindingResolver.isEagerExpressionResolve()) + { + entity.resolveExpressions(bindingResolver); + } + for (Expression expression:entity.expressionList()){ + if (expression.isStatement()) { + continue; + } + Entity referredEntity = expression.getReferredEntity(); + addRelationFromExpression(entity, expression, referredEntity); + } + entity.clearExpressions(); + } + + private void addRelationFromExpression(ContainerEntity entity, Expression expression, Entity referredEntity) { + + if (referredEntity==null) { + return; + } + if (referredEntity.getId()<0){ + return; + } + if (referredEntity instanceof MultiDeclareEntities) { + for (Entity e:((MultiDeclareEntities)referredEntity).getEntities()) { + addRelationFromExpression(entity,expression,e); + } + return; + } + + boolean matched = false; + if (expression.isCall()) { + /* if it is a FunctionEntityProto, add Relation to all Impl Entities*/ + if (callAsImpl && referredEntity instanceof FunctionEntityProto) { + if (entity.getAncestorOfType(FileEntity.class).getId().equals(referredEntity.getAncestorOfType(FileEntity.class).getId())){ + entity.addRelation(buildRelation(entity,DependencyType.CALL,referredEntity,expression.getLocation())); + }else { + Entity multiDeclare = repo.getEntity(referredEntity.getQualifiedName()); + if (multiDeclare instanceof MultiDeclareEntities) { + MultiDeclareEntities m = (MultiDeclareEntities) multiDeclare; + List entities = m.getEntities().stream().filter(item -> (item instanceof FunctionEntityImpl)) + .collect(Collectors.toList()); + for (Entity e : entities) { + entity.addRelation(expression, buildRelation(entity, DependencyType.IMPLLINK, e, expression.getLocation())); + matched = true; + } + } + } + } + entity.addRelation(buildRelation(entity,DependencyType.CALL,referredEntity,expression.getLocation())); + matched = true; + + } + if (expression.isCreate()) { + entity.addRelation(buildRelation(entity,DependencyType.CREATE,referredEntity,expression.getLocation())); + matched = true; + } + if (expression.isThrow()) { + entity.addRelation(buildRelation(entity,DependencyType.THROW,referredEntity,expression.getLocation())); + matched = true; + } + if (expression.isCast()) { + entity.addRelation(buildRelation(entity,DependencyType.CAST,referredEntity,expression.getLocation())); + matched = true; + } + if (!matched) { + if (callAsImpl && repo.getEntity(referredEntity.getQualifiedName()) instanceof MultiDeclareEntities && + (referredEntity instanceof VarEntity ||referredEntity instanceof FunctionEntity)) { + if (entity.getAncestorOfType(FileEntity.class).getId().equals(referredEntity.getAncestorOfType(FileEntity.class).getId())){ + entity.addRelation(buildRelation(entity,DependencyType.USE,referredEntity,expression.getLocation())); + }else { + MultiDeclareEntities m = (MultiDeclareEntities) (repo.getEntity(referredEntity.getQualifiedName())); + for (Entity e : m.getEntities()) { + if (e == referredEntity) { + entity.addRelation(expression, buildRelation(entity, DependencyType.USE, e, expression.getLocation())); + } else { + entity.addRelation(expression, buildRelation(entity, DependencyType.IMPLLINK, e, expression.getLocation())); + } + matched = true; + } + } + } + else { + entity.addRelation(expression,buildRelation(entity,DependencyType.USE,referredEntity,expression.getLocation())); + } + } + } + + private Relation buildRelation(Entity from, String type, Entity referredEntity) { + return buildRelation(from,type,referredEntity,from.getLocation()); + } + + private Relation buildRelation(Entity from, String type, Entity referredEntity,Location location) { + if (referredEntity instanceof AliasEntity) { + if (from.getAncestorOfType(FileEntity.class).equals(referredEntity.getAncestorOfType(FileEntity.class))) { + AliasEntity alias = ((AliasEntity) referredEntity); + if (alias.deepResolve()!=null) { + referredEntity = alias.deepResolve(); + } + } + } + if (this.langProcessor==null) + return new Relation(type,referredEntity,location); + return new Relation(langProcessor.getRelationMapping(type),referredEntity,location); + } + + private void computeTypeRelations(TypeEntity type) { + for (TypeEntity superType:type.getInheritedTypes()) { + type.addRelation(buildRelation(type,DependencyType.INHERIT,superType)); + } + for (TypeEntity interfaceType:type.getImplementedTypes()) { + type.addRelation(buildRelation(type,DependencyType.IMPLEMENT,interfaceType)); + } + } + + private void computeFunctionRelations(FunctionEntity func) { + for (Entity returnType:func.getReturnTypes()) { + func.addRelation(buildRelation(func,DependencyType.RETURN,returnType.getActualReferTo())); + } + for (VarEntity parameter:func.getParameters()) { + if (parameter.getType()!=null) + func.addRelation(buildRelation(func,DependencyType.PARAMETER,parameter.getActualReferTo())); + } + for (Entity throwType:func.getThrowTypes()) { + func.addRelation(buildRelation(func,DependencyType.THROW,throwType)); + } + for (Entity type:func.getResolvedTypeParameters()) { + func.addRelation(buildRelation(func,DependencyType.PARAMETER,type)); + } + if (func instanceof FunctionEntityImpl) { + FunctionEntityImpl funcImpl = (FunctionEntityImpl)func; + if(funcImpl.getImplemented()!=null) { + func.addRelation(buildRelation(func,DependencyType.IMPLEMENT,funcImpl.getImplemented())); + } + } + } + + private void computeImports(FileEntity file) { + Collection imports = file.getImportedRelationEntities(); + if (imports==null) return; + for (Entity imported:imports) { + if (imported instanceof FileEntity) + { + if (((FileEntity)imported).isInProjectScope()) + file.addRelation(buildRelation(file,DependencyType.IMPORT,imported)); + }else { + file.addRelation(buildRelation(file,DependencyType.IMPORT,imported)); + } + } + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/RubyHandlerContext.java b/src/main/java/com/educoder/bridge/tmp/RubyHandlerContext.java new file mode 100644 index 0000000..2ce057c --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/RubyHandlerContext.java @@ -0,0 +1,105 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.ruby; + +import depends.entity.Entity; +import depends.entity.PackageEntity; +import depends.entity.repo.EntityRepo; +import depends.extractor.FileParser; +import depends.extractor.HandlerContext; +import depends.extractor.IncludedFileLocator; +import depends.extractor.ParserCreator; +import depends.importtypes.FileImport; +import depends.relations.IBindingResolver; +import multilang.depends.util.file.FileUtil; + +import java.util.Collection; + +public class RubyHandlerContext extends HandlerContext { + + private IncludedFileLocator includedFileLocator; + private ParserCreator parserCreator; + public RubyHandlerContext(EntityRepo entityRepo, + IncludedFileLocator includedFileLocator, + IBindingResolver bindingResolver, ParserCreator parserCreator) { + super(entityRepo, bindingResolver); + this.includedFileLocator = includedFileLocator; + this.parserCreator = parserCreator; + } + + public Entity foundNamespace(String nampespaceName, Integer startLine) { + Entity parentEntity = currentFile(); + if (latestValidContainer()!=null) + parentEntity = latestValidContainer(); + PackageEntity pkgEntity = new PackageEntity(nampespaceName, parentEntity,idGenerator.generateId()); + entityRepo.add(pkgEntity); + entityStack.push(pkgEntity); + return pkgEntity; + } + + public void processSpecialFuncCall(String methodName, Collection params, Integer startLine) { + // Handle Import relation + if(methodName.equals("require") || methodName.equals("require_relative")) { + for (String importedFilename:params) { + if (!importedFilename.endsWith(".rb")) importedFilename = importedFilename + ".rb"; + String dir = FileUtil.getLocatedDir(currentFile().getRawName().uniqName()); + String inclFileName = includedFileLocator.uniqFileName(dir,importedFilename); + if (inclFileName==null) { + System.err.println("Warning: cannot found included file " + importedFilename ); + continue; + } + FileParser importedParser = parserCreator.createFileParser(); + try { + importedParser.parse(inclFileName); + } catch (Exception e) { + System.err.println("parsing error in "+inclFileName); + } + foundNewImport(new FileImport(inclFileName)); + } + } + // Handle Extend relation + else if (methodName.equals("extend")) { + for (String moduleName:params) { + foundExtends(moduleName); + } + } + // Handle mixin relation + else if (methodName.equals("include")) { + for (String moduleName:params) { + foundMixin(moduleName); + } + } + // attribute methods + else if (methodName.equals("attr_accessor")||methodName.equals("attr_writer")||methodName.equals("attr_reader")) { + for (String name:params) { + name = name.replace(":", ""); //remove symbol + foundMethodDeclarator(name,startLine); + } + } + } + + + +} diff --git a/src/main/java/com/educoder/bridge/tmp/RubyParserHelper.java b/src/main/java/com/educoder/bridge/tmp/RubyParserHelper.java new file mode 100644 index 0000000..bfb80fb --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/RubyParserHelper.java @@ -0,0 +1,165 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.extractor.ruby.jruby; + +import java.util.ArrayList; +import java.util.List; + +import org.jrubyparser.ast.AssignableNode; +import org.jrubyparser.ast.CallNode; +import org.jrubyparser.ast.ClassVarAsgnNode; +import org.jrubyparser.ast.ClassVarDeclNode; +import org.jrubyparser.ast.Colon2Node; +import org.jrubyparser.ast.Colon3Node; +import org.jrubyparser.ast.ConstDeclNode; +import org.jrubyparser.ast.DAsgnNode; +import org.jrubyparser.ast.DefsNode; +import org.jrubyparser.ast.FCallNode; +import org.jrubyparser.ast.GlobalAsgnNode; +import org.jrubyparser.ast.INameNode; +import org.jrubyparser.ast.InstAsgnNode; +import org.jrubyparser.ast.ListNode; +import org.jrubyparser.ast.LocalAsgnNode; +import org.jrubyparser.ast.MultipleAsgnNode; +import org.jrubyparser.ast.Node; +import org.jrubyparser.ast.UnaryCallNode; +import org.jrubyparser.ast.VCallNode; + +import depends.entity.ContainerEntity; +import depends.extractor.ruby.RubyBuiltInType; +import depends.extractor.ruby.RubyHandlerContext; + +public class RubyParserHelper { + private static RubyParserHelper inst = new RubyParserHelper(); + public static RubyParserHelper getInst() { + return inst; + } + + private RubyBuiltInType buildInType; + + private RubyParserHelper() { + this.buildInType = new RubyBuiltInType(); + } + + + public String getName(Node node) { + String name = ""; + if (node instanceof INameNode) { + name = ((INameNode)node).getName(); + if (node instanceof Colon2Node) { + Node left = ((Colon2Node)node).getLeftNode(); + if (left!=null) { + name = getName(left) + "."+name; + } + }else if (node instanceof Colon3Node) { + name = "."+name; + } + } + return name.length()>0?name:null; + } + + public boolean isFunctionCall(Node ctx) { + return ctx instanceof CallNode || + ctx instanceof FCallNode || + ctx instanceof UnaryCallNode || + ctx instanceof VCallNode; + } + + public List getName(AssignableNode ctx) { + List names = new ArrayList<>(); + if (ctx instanceof ClassVarAsgnNode) { + names.add(((ClassVarAsgnNode)ctx).getName()); + }else if (ctx instanceof ClassVarDeclNode) { + names.add(((ClassVarDeclNode)ctx).getName()); + }else if (ctx instanceof ConstDeclNode) { + names.add(((ConstDeclNode)ctx).getName()); + }else if (ctx instanceof DAsgnNode) { + names.add(((DAsgnNode)ctx).getName()); + }else if (ctx instanceof GlobalAsgnNode) { + names.add(((GlobalAsgnNode)ctx).getName()); + }else if (ctx instanceof InstAsgnNode) { + names.add(((InstAsgnNode)ctx).getName()); + }else if (ctx instanceof LocalAsgnNode) { + names.add(((LocalAsgnNode)ctx).getName()); + }else if (ctx instanceof MultipleAsgnNode) { + ListNode pre = ((MultipleAsgnNode)ctx).getPre(); + if (pre!=null) { + //TODO: support multiple assignment + } + } + return names; + } + + public String getReciever(Node ctx) { + Node receiver = null; + if (ctx instanceof CallNode) { + receiver = ((CallNode)ctx).getReceiver(); + }else if (ctx instanceof DefsNode) { + receiver = ((DefsNode)ctx).getReceiver(); + } + if (receiver==null) { + return null; + } + if (receiver instanceof INameNode) { + return this.getName(receiver); + } + return null; + } + + public ContainerEntity getScopeOfVar(AssignableNode node, RubyHandlerContext context) { + if (node instanceof LocalAsgnNode) return context.lastContainer(); + if (node instanceof InstAsgnNode) return context.currentType(); + if (node instanceof ClassVarAsgnNode) return context.currentType(); + if (node instanceof GlobalAsgnNode) return context.globalScope(); + if (node instanceof DAsgnNode) return context.lastContainer(); + return context.lastContainer(); + } + + public boolean isArithMeticOperator(String name) { + return name.equals("+") || + name.equals("-") || + name.equals("*") || + name.equals("/") || + name.equals("**") || + name.equals("%") || + name.equals("&") || + name.equals("<") || + name.equals("<=") || + name.equals(">") || + name.equals(">=") || + name.equals("==") || + name.equals("!=") || + name.equals("===") || + name.equals("<<") || + name.equals(">>") || + name.equals("~") || + name.equals("!") || + name.equals("^"); + } + + public boolean isCommonOperator(String name) { + return this.buildInType.isBuildInMethod(name); + } +} diff --git a/src/main/java/com/educoder/bridge/tmp/RubyVarTest.java b/src/main/java/com/educoder/bridge/tmp/RubyVarTest.java new file mode 100644 index 0000000..fcd7dcc --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/RubyVarTest.java @@ -0,0 +1,172 @@ +package depends.extractor.ruby; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; + +import depends.entity.ContainerEntity; +import depends.entity.FunctionEntity; +import depends.entity.TypeEntity; +import depends.entity.repo.EntityRepo; +import depends.extractor.FileParser; + +public class RubyVarTest extends RubyParserTest { + @Before + public void setUp() { + super.init(); + } + + @Test + public void test_parameter_should_be_created() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + FunctionEntity function = (FunctionEntity)(entityRepo.getEntity("method1")); + assertEquals(1,function.getParameters().size()); + assertContainsParametersWithRawName(function, "param1"); + } + + + @Test + public void test_var_should_be_created_if_not_declared() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + FunctionEntity function = (FunctionEntity)(entityRepo.getEntity("method_with_local_var")); + assertEquals(1,function.getVars().size()); + assertContainsVarWithRawName(function, "var_1"); + } + + + @Test + public void test_var_should_only_create_once() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + FunctionEntity function = (FunctionEntity)(entityRepo.getEntity("method_with_local_var_2times")); + assertEquals(1,function.getVars().size()); + } + + @Test + public void test_var_should_not_be_created_if_it_actually_parameter() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + FunctionEntity function = (FunctionEntity)(entityRepo.getEntity("method_without_local_var_and_param")); + assertEquals(0,function.getVars().size()); + } + + + @Test + public void test_var_should_not_be_created_if_it_actually_a_file_level_var() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + FunctionEntity function = (FunctionEntity)(entityRepo.getEntity("method_access_file_level_var")); + assertEquals(0,function.getVars().size()); + } + + @Test + public void test_var_should_not_be_created_with_a_lot_of_levels() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + FunctionEntity function = (FunctionEntity)(entityRepo.getEntity("M.C.method")); + assertEquals(1,function.getVars().size()); + } + @Test + public void test_var_should_not_be_created_not_in_scope() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + FunctionEntity function = (FunctionEntity)(entityRepo.getEntity("M.C.method2")); + assertEquals(1,function.getVars().size()); + } + + + @Test + public void test_var_should_created_at_class_level() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + TypeEntity c = (TypeEntity)(entityRepo.getEntity("M.C")); + assertEquals(3,c.getVars().size()); + assertContainsVarWithRawName(c,"class_level_var"); + assertContainsVarWithRawName(c,"class_var"); + assertContainsVarWithRawName(c,"instance_var"); + } + + @Test + public void test_var_in_block() throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + ContainerEntity c = (ContainerEntity)(entityRepo.getEntity("Block")); + assertEquals(1,c.getVars().size()); + assertContainsVarWithRawName(c,"a"); + } + + @Test + public void test_global_var()throws IOException { + String[] srcs = new String[] { + "./src/test/resources/ruby-code-examples/auto_var.rb", + }; + + for (String src:srcs) { + FileParser parser = createFileParser(); + parser.parse(src); + } + ContainerEntity c = (ContainerEntity)(entityRepo.getEntity(EntityRepo.GLOBAL_SCOPE_NAME)); + assertEquals(1,c.getVars().size()); + assertContainsVarWithRawName(c,"global_var"); + } +} + diff --git a/src/main/java/com/educoder/bridge/tmp/TypeEntity.java b/src/main/java/com/educoder/bridge/tmp/TypeEntity.java new file mode 100644 index 0000000..6282ef4 --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/TypeEntity.java @@ -0,0 +1,204 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.relations.IBindingResolver; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; + +public class TypeEntity extends ContainerEntity { + static final public TypeEntity buildInType = new TypeEntity(GenericName.build("built-in"), null, -1); + static final public TypeEntity genericParameterType = new TypeEntity(GenericName.build("T"), null, -3); + Collection inheritedTypes = new ArrayList<>(); + Collection implementedTypes = new ArrayList<>(); + Collection inhertedTypeIdentifiers; + Collection implementedIdentifiers; + TypeEntity inheritedType; + public TypeEntity() {} + public TypeEntity(GenericName simpleName, Entity parent, Integer id) { + super(simpleName, parent, id); + inhertedTypeIdentifiers = new ArrayList<>(); + implementedIdentifiers = new ArrayList<>(); + } + + @Override + public void inferLocalLevelEntities(IBindingResolver bindingResolver) { + inheritedTypes = new ArrayList<>(); + Collection r = identiferToEntities(bindingResolver, this.inhertedTypeIdentifiers); + if (r!=null) { + r.forEach(item -> { + Entity typeItem = getTypeEntity(item); + if (typeItem !=null) { + inheritedTypes.add((TypeEntity) typeItem); + }else { + System.err.println(item.getRawName() + " expected a type, but actually it is "+ item.getClass().getSimpleName()); + } + }); + } + inheritedTypes.remove(this); + + implementedTypes = new ArrayList<>(); + r = identiferToEntities(bindingResolver, this.implementedIdentifiers); + if (r!=null) { + r.forEach(item -> { + Entity typeItem = getTypeEntity(item); + if (typeItem !=null) { + implementedTypes.add((TypeEntity) typeItem); + }else { + System.err.println(item.getRawName() + " expected a type, but actually it is "+ item.getClass().getSimpleName()); + } + }); + } + implementedTypes.remove(this); + if (inheritedTypes.size() > 0) + inheritedType = inheritedTypes.iterator().next(); + super.inferLocalLevelEntities(bindingResolver); + } + + private Entity getTypeEntity(Entity item) { + if (item==null) return null; + if (item instanceof TypeEntity) return item; + if (item instanceof MultiDeclareEntities) return ((MultiDeclareEntities)item).getType(); + if (item instanceof AliasEntity) return item.getType(); + return null; + } + public void addImplements(GenericName typeName) { + if (typeName==null) { + return; + } + if (typeName.equals(this.getRawName())) + return; + if (implementedIdentifiers.contains(typeName)) + return; + if (typeName.equals(this.rawName)) + return; + this.implementedIdentifiers.add(typeName); + } + + public void addExtends(GenericName typeName) { + if (typeName==null) { + return; + } + if (typeName.equals(this.getRawName())) + return; + if (inhertedTypeIdentifiers.contains(typeName)) + return; + if (typeName.equals(this.rawName)) + return; + this.inhertedTypeIdentifiers.add(typeName); + } + + public Collection getInheritedTypes() { + return inheritedTypes; + } + + public Collection getImplementedTypes() { + return implementedTypes; + } + + public TypeEntity getInheritedType() { + return inheritedType; + } + + @Override + public FunctionEntity lookupFunctionLocally(GenericName functionName) { + Collection searchedTypes = new ArrayList<>(); + return lookupFunctionLocally(functionName,searchedTypes); + } + + private FunctionEntity lookupFunctionLocally(GenericName functionName, Collection searched) { + if (searched.contains(this)) return null; + searched.add(this); + FunctionEntity func = super.lookupFunctionLocally(functionName); + if (func != null) + return func; + for (TypeEntity inhertedType : getInheritedTypes()) { + func = inhertedType.lookupFunctionLocally(functionName, searched); + if (func != null) + break; + } + if (func != null) + return func; + for (TypeEntity implType : getImplementedTypes()) { + func = implType.lookupFunctionLocally(functionName,searched); + if (func != null) + break; + } + return func; + } + + @Override + public VarEntity lookupVarLocally(GenericName varName) { + Collection searchedTypes = new ArrayList<>(); + return lookupVarLocally(varName,searchedTypes); + } + + private VarEntity lookupVarLocally(GenericName varName, Collection searched) { + if (searched.contains(this)) return null; + searched.add(this); + VarEntity var = super.lookupVarLocally(varName); + if (var != null) + return var; + for (TypeEntity inhertedType : getInheritedTypes()) { + var = inhertedType.lookupVarLocally(varName,searched); + if (var != null) + break; + } + if (var != null) + return var; + for (TypeEntity implType : getImplementedTypes()) { + var = implType.lookupVarLocally(varName,searched); + if (var != null) + break; + } + return var; + } + + @Override + public TypeEntity getType() { + return this; + } + @Override + public Entity getByName(String name, HashSet searched) { + Entity entity = super.getByName(name, searched); + if (entity!=null) + return entity; + for (TypeEntity child:getInheritedTypes()) { + if (searched.contains(child)) continue; + entity = child.getByName(name, searched); + if (entity!=null) return entity; + } + for (TypeEntity child:getImplementedTypes()) { + if (searched.contains(child)) continue; + entity = child.getByName(name,searched); + if (entity!=null) return entity; + } + return null; + } + + +} diff --git a/src/main/java/com/educoder/bridge/tmp/UnsolvedSymbolDumper.java b/src/main/java/com/educoder/bridge/tmp/UnsolvedSymbolDumper.java new file mode 100644 index 0000000..874369f --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/UnsolvedSymbolDumper.java @@ -0,0 +1,96 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.format.detail; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.util.HashSet; +import java.util.Set; +import java.util.TreeMap; + +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +import depends.extractor.UnsolvedBindings; +import multilang.depends.util.file.strip.LeadingNameStripper; + +public class UnsolvedSymbolDumper{ + private Set unsolved; + private String name; + private String outputDir; + private LeadingNameStripper leadingNameStripper; + + public UnsolvedSymbolDumper(Set unsolved, String name, String outputDir, LeadingNameStripper leadingNameStripper) { + this.unsolved = unsolved; + this.name = name; + this.outputDir = outputDir; + this.leadingNameStripper = leadingNameStripper; + } + + public void output() { + outputDetail(); + outputGrouped(); + } + + private void outputGrouped() { + TreeMap> grouped = new TreeMap>(); + for (UnsolvedBindings symbol: unsolved) { + String depended = symbol.getRawName(); + String from = leadingNameStripper.stripFilename(symbol.getSourceDisplay()); + Set list = grouped.get(depended); + if (list==null) { + list = new HashSet<>(); + grouped.put(depended, list); + } + list.add(from); + } + ObjectMapper om = new ObjectMapper(); + om.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false); + om.configure(SerializationFeature.INDENT_OUTPUT, true); + om.setSerializationInclusion(Include.NON_NULL); + try { + om.writerWithDefaultPrettyPrinter().writeValue(new File(outputDir + File.separator + name +"-PotentialExternalDependencies.json"), grouped); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void outputDetail() { + PrintWriter writer; + try { + writer = new PrintWriter(outputDir + File.separator + name +"-PotentialExternalDependencies.txt"); + for (UnsolvedBindings symbol: unsolved) { + String source = leadingNameStripper.stripFilename(symbol.getSourceDisplay()); + writer.println(""+symbol.getRawName()+", "+source); + } + writer.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/main/java/com/educoder/bridge/tmp/VarEntity.java b/src/main/java/com/educoder/bridge/tmp/VarEntity.java new file mode 100644 index 0000000..f81bdbc --- /dev/null +++ b/src/main/java/com/educoder/bridge/tmp/VarEntity.java @@ -0,0 +1,106 @@ +/* +MIT License + +Copyright (c) 2018-2019 Gang ZHANG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package depends.entity; + +import depends.relations.IBindingResolver; + +import java.util.ArrayList; +import java.util.List; + +public class VarEntity extends ContainerEntity { + private GenericName rawType; + private TypeEntity type; + private List functionCalls; + public VarEntity() { + + } + public VarEntity(GenericName simpleName, GenericName rawType, Entity parent, int id) { + super(simpleName, parent,id); + this.rawType = rawType; + } + + public void setRawType(GenericName rawType) { + this.rawType =rawType; + } + + public GenericName getRawType() { + return rawType; + } + + @Override + public TypeEntity getType() { + return type; + } + + public void setType(TypeEntity type) { + this.type = type; + } + + @Override + public void inferLocalLevelEntities(IBindingResolver bindingResolver) { + super.inferLocalLevelEntities(bindingResolver); + Entity entity = bindingResolver.resolveName(this, rawType, true); + if (entity!=null) { + this.setActualReferTo(entity); + type = entity.getType(); + if (type==null) { + if (((ContainerEntity)getParent()).isGenericTypeParameter(rawType)) { + type = TypeEntity.genericParameterType; + } + } + } + if (type==null) { + fillCandidateTypes(bindingResolver); + } + } + + public List getCalledFunctions() { + if (this.functionCalls!=null) + return functionCalls; + return new ArrayList<>(); + } + + public void addFunctionCall(GenericName fname) { + if (this.functionCalls==null) + { + functionCalls = new ArrayList<>(); + } + this.functionCalls.add(new FunctionCall(fname)); + } + + public void fillCandidateTypes(IBindingResolver bindingResolver) { + if (!bindingResolver.isEagerExpressionResolve()) return; + if (type!=null && !(type instanceof CandidateTypes)) return ; //it is a strong type lang, do not need deduce candidate types + if (functionCalls==null) return; + if (functionCalls.size()==0) return; //no information avaliable for type deduction + if (this.rawType==null) { + List candidateTypes = bindingResolver.calculateCandidateTypes(this,this.functionCalls); + if (candidateTypes.size()>0) { + this.type = new CandidateTypes(candidateTypes, bindingResolver.getRepo().generateId()); + bindingResolver.getRepo().add(this.type); + } + } + } +}