(para, log_format, rex)
| 43 | |
| 44 | |
| 45 | def SLCT(para, log_format, rex): |
| 46 | startTime = datetime.now() # start timing |
| 47 | logname = os.path.join(para["dataPath"], para["dataName"]) |
| 48 | print("Parsing file: {}".format(logname)) |
| 49 | |
| 50 | # SLCT compilation |
| 51 | if not os.path.isfile("../SLCT/slct"): |
| 52 | try: |
| 53 | print("Compile SLCT...\n>> gcc -o ./src/slct -O2 ./src/cslct.c") |
| 54 | subprocess.check_output( |
| 55 | "gcc -o ./src/slct -O2 ./src/cslct.c", |
| 56 | stderr=subprocess.STDOUT, |
| 57 | shell=True, |
| 58 | ) |
| 59 | except: |
| 60 | raise "Compile error! Please check GCC installed." |
| 61 | |
| 62 | headers, regex = generate_logformat_regex(log_format) |
| 63 | df_log = log_to_dataframe(logname, regex, headers, log_format) |
| 64 | print("load data done.") |
| 65 | # Generate input file |
| 66 | with open("slct_input.log", "w") as fw: |
| 67 | for line in df_log["Content"]: |
| 68 | if rex: |
| 69 | for currentRex in rex: |
| 70 | line = re.sub(currentRex, "<*>", line) |
| 71 | fw.write(line + "\n") |
| 72 | print("modify data done.") |
| 73 | # Run SLCT command |
| 74 | SLCT_command = extract_command(para, "slct_input.log") |
| 75 | try: |
| 76 | print("Run SLCT...\n>> {}".format(SLCT_command)) |
| 77 | subprocess.check_call(SLCT_command, shell=True) |
| 78 | except: |
| 79 | print("SLCT executable is invalid! Please compile it using GCC.\n") |
| 80 | raise |
| 81 | |
| 82 | # Collect and dump templates |
| 83 | tempParameter = TempPara( |
| 84 | path="./", savePath=para["savePath"], logname="slct_input.log" |
| 85 | ) |
| 86 | tempProcess(tempParameter) |
| 87 | print("temProcess done!") |
| 88 | matcher = RegexMatch(outdir=para["savePath"], logformat=log_format) |
| 89 | matched_df = matcher.match(logname, "temp_templates.csv") |
| 90 | print("regex match done!") |
| 91 | # sys.exit() |
| 92 | os.remove("slct_input.log") |
| 93 | os.remove("slct_outliers.log") |
| 94 | os.remove("slct_templates.txt") |
| 95 | os.remove("temp_templates.csv") |
| 96 | |
| 97 | for idx, line in matched_df.iterrows(): |
| 98 | if line["EventTemplate"] == "None": |
| 99 | content = line["Content"] |
| 100 | matched_df.loc[idx, "EventTemplate"] = content |
| 101 | matched_df.loc[idx, "EventId"] = hashlib.md5( |
| 102 | content.encode("utf-8") |
no test coverage detected