diff --git a/src/commands/scan.ts b/src/commands/scan.ts index b62e856..fe3ce32 100644 --- a/src/commands/scan.ts +++ b/src/commands/scan.ts @@ -5,7 +5,7 @@ import { Opts, pluginMap, PluginType, StorageEngine } from '@cloudgraph/sdk' import { range } from 'lodash' import Command from './base' -import { fileUtils } from '../utils' +import { cleanString, fileUtils, getStoredSchema } from '../utils' import DgraphEngine from '../storage/dgraph' import { scanReport } from '../reports' import { loadAllData, processConnectionsBetweenEntities } from '../utils/data' @@ -91,6 +91,14 @@ export default class Scan extends Command { } } + // Indicates when schema has new changes + private schemaHasChange(oldSchema: string, newSchema: string): boolean { + return !!Buffer.compare( + Buffer.from(cleanString(oldSchema), 'utf8'), + Buffer.from(cleanString(newSchema), 'utf8') + ) + } + async run(): Promise { const { argv, flags } = await this.parse(Scan) const { dev: devMode } = flags as { @@ -227,8 +235,14 @@ export default class Scan extends Command { if (storageEngine instanceof DgraphEngine) { await storageEngine.validateSchema(schema, dataFolder) } - await storageEngine.dropAll() // Delete schema before change it - await storageEngine.setSchema(schema) + + const currentSchema = getStoredSchema(dataDir) + + // Only drops and changes data when the schema changes + if (this.schemaHasChange(currentSchema, schema.join())) { + await storageEngine.dropAll() // Delete schema before change it + await storageEngine.setSchema(schema, { overwrite: dataDir }) + } } catch (error: any) { this.logger.error( `There was an issue pushing schema for providers: ${allProviders.join( diff --git a/src/reports/scan-report.ts b/src/reports/scan-report.ts index b51fab8..99378fe 100644 --- a/src/reports/scan-report.ts +++ b/src/reports/scan-report.ts @@ -29,6 +29,7 @@ const servicesToIgnore = [ /^account$/, /^tag$/, /^label$/, + /ruleMetadata$/, /^billing$/, /Findings$/, ] diff --git a/src/storage/dgraph/index.ts b/src/storage/dgraph/index.ts index b940b28..fcdde5a 100644 --- a/src/storage/dgraph/index.ts +++ b/src/storage/dgraph/index.ts @@ -3,8 +3,9 @@ import { StorageEngineConfig, StorageEngine, GraphQLInputData, - GraphQLQueryData + GraphQLQueryData, } from '@cloudgraph/sdk' +import { isEmpty } from 'lodash' import DGraphClientWrapper from './base' import { @@ -12,6 +13,7 @@ import { processGQLExecutionResult, UPDATE_SCHEMA_QUERY, } from './utils' +import { fileUtils, sleep } from '../../utils' export default class DgraphEngine extends DGraphClientWrapper @@ -80,11 +82,15 @@ export default class DgraphEngine }) } - async setSchema(schemas: string[]): Promise { + async setSchema( + schemas: string[], + config?: { overwrite: string } + ): Promise { + const schema = schemas.join() const data = { query: UPDATE_SCHEMA_QUERY, variables: { - schema: schemas.join(), + schema, }, } try { @@ -99,8 +105,13 @@ export default class DgraphEngine resData, errors, }) + + if (isEmpty(errors) && config?.overwrite) { + fileUtils.writeGraphqlSchemaToFile(`${config.overwrite}/cg`, schema) + } }) .catch(error => Promise.reject(error)) + sleep(3) } catch (error: any) { const { response: { data: resData, errors }, @@ -181,8 +192,7 @@ export default class DgraphEngine /** * Executes mutations sequentially into Dgraph */ - async run(dropData = true): Promise { - dropData && (await this.dropData()) + async run(): Promise { for (const mutation of this.axiosPromises) { try { await mutation() diff --git a/src/utils/index.ts b/src/utils/index.ts index b30ec4d..69a2960 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -150,8 +150,19 @@ export function deleteFolder(dirPath: string): void { fs.rmSync(dirPath, { recursive: true }) } +export const getStoredSchema = (dirPath: string): string => { + try { + const schemaPath = path.normalize(`${dirPath}/cg/schema.graphql`) + const schema = fs.readFileSync(schemaPath, 'utf8') + return schema + } catch (error) { + // Return an empty string if a schema was not found + return '' + } +} + export const sleep = (ms: number): Promise => - new Promise(resolve => setTimeout(resolve, ms)) + new Promise(resolve => setTimeout(resolve, ms * 1000)) export const calculateBackoff = (n: number): number => { const temp = Math.min( @@ -248,3 +259,6 @@ export const getNextPort = async (port: number): Promise => { const availablePort = await detect(port) return String(availablePort) } + +export const cleanString = (dirtyString: string): string => + dirtyString.replace(/(\r\n|\n|\r)/gm, '').replace(/\s+/g, '')