From 54d5c4651e365189f55174a493b9aaeee1187e76 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Fri, 18 Apr 2025 11:23:59 -0600 Subject: [PATCH 1/5] Support log config from a specified file Signed-off-by: Mikayla Thompson --- .../migrations/replay/TrafficReplayer.java | 24 +++++++++++++++++++ .../replayer/templates/deployment.yaml | 3 +++ 2 files changed, 27 insertions(+) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index ddf1456194..27f44741d7 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -1,5 +1,7 @@ package org.opensearch.migrations.replay; +import java.io.File; +import java.io.IOException; import java.lang.ref.WeakReference; import java.net.URI; import java.time.Clock; @@ -36,6 +38,10 @@ import io.netty.util.concurrent.DefaultThreadFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.ConfigurationFactory; +import org.apache.logging.log4j.core.config.ConfigurationSource; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; import software.amazon.awssdk.arns.Arn; @@ -45,6 +51,7 @@ @Slf4j public class TrafficReplayer { private static final String ALL_ACTIVE_CONTEXTS_MONITOR_LOGGER = "AllActiveWorkMonitor"; + public static final String LOG4J2_PROPERTIES_FILE = "LOG4J2_PROPERTIES_FILE"; public static final String SIGV_4_AUTH_HEADER_SERVICE_REGION_ARG = "--sigv4-auth-header-service-region"; public static final String AUTH_HEADER_VALUE_ARG = "--auth-header-value"; @@ -91,6 +98,21 @@ public static boolean validateRequiredKafkaParams(String brokers, String topic, return true; } + public static void reloadConfiguration(String configFile) throws IOException { + ConfigurationSource source; + var file = configFile == null || configFile.isEmpty() ? null : new File(configFile); + if (file != null && file.exists()) { + source = new ConfigurationSource(file.toURI().toURL().openStream(), file.toURI().toURL()); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + var config = ConfigurationFactory.getInstance().getConfiguration(context, source); + context.start(config); + log.atInfo().setMessage("Successfully reloaded Log4j2 configuration from: {}") + .addArgument(configFile).log(); + } else { + log.atInfo().setMessage("Using the default Log4j2 configuration").log(); + } + } + public static class Parameters { @Parameter( required = true, @@ -333,6 +355,8 @@ private static Parameters parseArgs(String[] args) { public static void main(String[] args) throws Exception { System.err.println("Got args: " + String.join("; ", args)); + reloadConfiguration(System.getenv(LOG4J2_PROPERTIES_FILE)); + final var workerId = ProcessHelpers.getNodeInstanceName(); log.info("Starting Traffic Replayer with id=" + workerId); diff --git a/deployment/k8s/charts/components/replayer/templates/deployment.yaml b/deployment/k8s/charts/components/replayer/templates/deployment.yaml index 91c7f51b16..698c6de9cb 100644 --- a/deployment/k8s/charts/components/replayer/templates/deployment.yaml +++ b/deployment/k8s/charts/components/replayer/templates/deployment.yaml @@ -47,6 +47,9 @@ spec: - name: shared-logs mountPath: /shared-logs-output {{- end }} + env: + - name: LOG4J2_PROPERTIES_FILE + value: "/shared-logs-output/replayer_log4j.properties" volumes: - name: {{ $envMountName }} emptyDir: {} From 85eee2345193c4562c60526d8e0a25f83e3db886 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Fri, 18 Apr 2025 14:28:26 -0600 Subject: [PATCH 2/5] Update logging config to write uncompressed files locally Signed-off-by: Mikayla Thompson --- .../src/main/resources/log4j2.properties | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties b/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties index a64261631b..a902bf4fac 100644 --- a/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties +++ b/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties @@ -2,6 +2,7 @@ status = warn property.logsDir = ${env:SHARED_LOGS_DIR_PATH:-./logs} property.tuplesFileNamePrefix = ${logsDir}/${hostName}/tuples/tuples +property.tempDir = ./logs/temp appenders = console, ReplayerLogFile, OUTPUT_TUPLES, TRANSACTION_SUMMARIES, TRANSACTION_SUMMARIES_LOGFILE, ALL_ACTIVE_WORK_MONITOR_LOGFILE @@ -14,8 +15,9 @@ appender.console.direct = true appender.ReplayerLogFile.type = RollingRandomAccessFile appender.ReplayerLogFile.name = ReplayerLogFile -appender.ReplayerLogFile.fileName = logs/replayer.log -appender.ReplayerLogFile.filePattern = logs/%d{yyyy-MM}{UTC}/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz +appender.ReplayerLogFile.fileName = ${tempDir}/replayer.log +appender.ReplayerLogFile.filePattern = ${logsDir}/${hostName}/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz +appender.ReplayerLogFile.tempCompressedFilePattern = ${tempDir}/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz.tmp appender.ReplayerLogFile.layout.type = PatternLayout appender.ReplayerLogFile.layout.pattern = [%-5level] %d{DEFAULT_MICROS}{UTC} [%t] %c{1} - %msg%equals{ ctx=%mdc}{ ctx={}}{}%n appender.ReplayerLogFile.policies.type = Policies @@ -24,18 +26,21 @@ appender.ReplayerLogFile.policies.time.interval = 60 appender.ReplayerLogFile.policies.time.modulate = true appender.ReplayerLogFile.strategy.type = DefaultRolloverStrategy appender.ReplayerLogFile.strategy.max = 288 +appender.ReplayerLogFile.strategy.compressionLevel = 9 appender.ReplayerLogFile.immediateFlush = false appender.OUTPUT_TUPLES.type = RollingRandomAccessFile appender.OUTPUT_TUPLES.name = OUTPUT_TUPLES -appender.OUTPUT_TUPLES.fileName = ${tuplesFileNamePrefix}.log -appender.OUTPUT_TUPLES.filePattern = ${tuplesFileNamePrefix}-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz +appender.OUTPUT_TUPLES.fileName = ${tempDir}/tuples.log +appender.OUTPUT_TUPLES.filePattern = ${logsDir}/${hostName}/tuples/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz +appender.OUTPUT_TUPLES.tempCompressedFilePattern = ${tempDir}/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz.tmp appender.OUTPUT_TUPLES.layout.type = PatternLayout appender.OUTPUT_TUPLES.layout.pattern = %m%n appender.OUTPUT_TUPLES.policies.type = Policies appender.OUTPUT_TUPLES.policies.size.type = SizeBasedTriggeringPolicy appender.OUTPUT_TUPLES.policies.size.size = 10 MB appender.OUTPUT_TUPLES.strategy.type = DefaultRolloverStrategy +appender.OUTPUT_TUPLES.strategy.compressionLevel = 9 appender.OUTPUT_TUPLES.immediateFlush = false appender.TRANSACTION_SUMMARIES.type = Console From d885e8f54c23589322ae48e9013699d18c718f66 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Tue, 22 Apr 2025 14:00:25 -0500 Subject: [PATCH 3/5] Update replayer log4j2 for minute rollover and size based triggering Signed-off-by: Andre Kurait --- .../src/main/resources/log4j2.properties | 106 ++++++++++++------ 1 file changed, 69 insertions(+), 37 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties b/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties index a902bf4fac..9a792ab9ab 100644 --- a/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties +++ b/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties @@ -1,7 +1,7 @@ status = warn -property.logsDir = ${env:SHARED_LOGS_DIR_PATH:-./logs} -property.tuplesFileNamePrefix = ${logsDir}/${hostName}/tuples/tuples +property.rootLogsDir = ${env:SHARED_LOGS_DIR_PATH:-./logs} +property.logFileNamePrefix = ${rootLogsDir}/${hostName} property.tempDir = ./logs/temp appenders = console, ReplayerLogFile, OUTPUT_TUPLES, TRANSACTION_SUMMARIES, TRANSACTION_SUMMARIES_LOGFILE, ALL_ACTIVE_WORK_MONITOR_LOGFILE @@ -15,32 +15,47 @@ appender.console.direct = true appender.ReplayerLogFile.type = RollingRandomAccessFile appender.ReplayerLogFile.name = ReplayerLogFile -appender.ReplayerLogFile.fileName = ${tempDir}/replayer.log -appender.ReplayerLogFile.filePattern = ${logsDir}/${hostName}/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz -appender.ReplayerLogFile.tempCompressedFilePattern = ${tempDir}/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz.tmp +appender.ReplayerLogFile.fileName = ${tempDir}/logs/replayer.log +appender.ReplayerLogFile.filePattern = ${logFileNamePrefix}/logs/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz +appender.ReplayerLogFile.tempCompressedFilePattern = ${logFileNamePrefix}/logs/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz.tmp appender.ReplayerLogFile.layout.type = PatternLayout appender.ReplayerLogFile.layout.pattern = [%-5level] %d{DEFAULT_MICROS}{UTC} [%t] %c{1} - %msg%equals{ ctx=%mdc}{ ctx={}}{}%n -appender.ReplayerLogFile.policies.type = Policies -appender.ReplayerLogFile.policies.time.type = TimeBasedTriggeringPolicy -appender.ReplayerLogFile.policies.time.interval = 60 -appender.ReplayerLogFile.policies.time.modulate = true -appender.ReplayerLogFile.strategy.type = DefaultRolloverStrategy -appender.ReplayerLogFile.strategy.max = 288 -appender.ReplayerLogFile.strategy.compressionLevel = 9 +appender.ReplayerLogFile.policies.type=Policies +# Make a new file once per minute +appender.ReplayerLogFile.policies.time.type=TimeBasedTriggeringPolicy +appender.ReplayerLogFile.policies.time.interval=1 +appender.ReplayerLogFile.policies.time.modulate=true +# Make a new file if size is over 1GB +appender.ReplayerLogFile.policies.size.type=SizeBasedTriggeringPolicy +appender.ReplayerLogFile.policies.size.size=1GB +# Make a new file on startup +appender.ReplayerLogFile.policies.startup.type=OnStartupTriggeringPolicy +appender.ReplayerLogFile.strategy.type=DefaultRolloverStrategy +appender.ReplayerLogFile.strategy.fileIndex=nomax +appender.ReplayerLogFile.strategy.compressionLevel = 7 appender.ReplayerLogFile.immediateFlush = false + appender.OUTPUT_TUPLES.type = RollingRandomAccessFile appender.OUTPUT_TUPLES.name = OUTPUT_TUPLES -appender.OUTPUT_TUPLES.fileName = ${tempDir}/tuples.log -appender.OUTPUT_TUPLES.filePattern = ${logsDir}/${hostName}/tuples/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz -appender.OUTPUT_TUPLES.tempCompressedFilePattern = ${tempDir}/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz.tmp +appender.OUTPUT_TUPLES.fileName = ${tempDir}/progress/tuples.log +appender.OUTPUT_TUPLES.filePattern = ${logFileNamePrefix}/tuples/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz +appender.OUTPUT_TUPLES.tempCompressedFilePattern = ${logFileNamePrefix}/tuples/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz.tmp appender.OUTPUT_TUPLES.layout.type = PatternLayout appender.OUTPUT_TUPLES.layout.pattern = %m%n -appender.OUTPUT_TUPLES.policies.type = Policies -appender.OUTPUT_TUPLES.policies.size.type = SizeBasedTriggeringPolicy -appender.OUTPUT_TUPLES.policies.size.size = 10 MB -appender.OUTPUT_TUPLES.strategy.type = DefaultRolloverStrategy -appender.OUTPUT_TUPLES.strategy.compressionLevel = 9 +appender.OUTPUT_TUPLES.policies.type=Policies +# Make a new file once per minute +appender.OUTPUT_TUPLES.policies.time.type=TimeBasedTriggeringPolicy +appender.OUTPUT_TUPLES.policies.time.interval=1 +appender.OUTPUT_TUPLES.policies.time.modulate=true +# Make a new file if size is over 1GB +appender.OUTPUT_TUPLES.policies.size.type=SizeBasedTriggeringPolicy +appender.OUTPUT_TUPLES.policies.size.size=1GB +# Make a new file on startup +appender.OUTPUT_TUPLES.policies.startup.type=OnStartupTriggeringPolicy +appender.OUTPUT_TUPLES.strategy.type=DefaultRolloverStrategy +appender.OUTPUT_TUPLES.strategy.fileIndex=nomax +appender.OUTPUT_TUPLES.strategy.compressionLevel = 7 appender.OUTPUT_TUPLES.immediateFlush = false appender.TRANSACTION_SUMMARIES.type = Console @@ -52,30 +67,46 @@ appender.TRANSACTION_SUMMARIES.direct = true appender.TRANSACTION_SUMMARIES_LOGFILE.type = RollingRandomAccessFile appender.TRANSACTION_SUMMARIES_LOGFILE.name = TransactionSummariesFile -appender.TRANSACTION_SUMMARIES_LOGFILE.fileName = logs/progress.log -appender.TRANSACTION_SUMMARIES_LOGFILE.filePattern = logs/%d{yyyy-MM}{UTC}/progress-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz +appender.TRANSACTION_SUMMARIES_LOGFILE.fileName = ${tempDir}/progress/progress.log +appender.TRANSACTION_SUMMARIES_LOGFILE.filePattern = ${logFileNamePrefix}/progress/progress-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz +appender.TRANSACTION_SUMMARIES_LOGFILE.tempCompressedFilePattern = ${logFileNamePrefix}/progress/progress-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.tmp appender.TRANSACTION_SUMMARIES_LOGFILE.layout.type = PatternLayout appender.TRANSACTION_SUMMARIES_LOGFILE.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS}{UTC}: %msg%n -appender.TRANSACTION_SUMMARIES_LOGFILE.policies.type = Policies -appender.TRANSACTION_SUMMARIES_LOGFILE.policies.time.type = TimeBasedTriggeringPolicy -appender.TRANSACTION_SUMMARIES_LOGFILE.policies.time.interval = 60 -appender.TRANSACTION_SUMMARIES_LOGFILE.policies.time.modulate = true -appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.type = DefaultRolloverStrategy -appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.max = 720 +appender.TRANSACTION_SUMMARIES_LOGFILE.policies.type=Policies +# Make a new file once per minute +appender.TRANSACTION_SUMMARIES_LOGFILE.policies.time.type=TimeBasedTriggeringPolicy +appender.TRANSACTION_SUMMARIES_LOGFILE.policies.time.interval=1 +appender.TRANSACTION_SUMMARIES_LOGFILE.policies.time.modulate=true +# Make a new file if size is over 100MB +appender.TRANSACTION_SUMMARIES_LOGFILE.policies.size.type=SizeBasedTriggeringPolicy +appender.TRANSACTION_SUMMARIES_LOGFILE.policies.size.size=100MB +# Make a new file on startup +appender.TRANSACTION_SUMMARIES_LOGFILE.policies.startup.type=OnStartupTriggeringPolicy +appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.type=DefaultRolloverStrategy +appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.fileIndex=nomax +appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.compressionLevel = 7 appender.TRANSACTION_SUMMARIES_LOGFILE.immediateFlush = false appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.type = RollingRandomAccessFile appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.name = AllActiveWorkMonitorFile -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.fileName = logs/longRunningActivity.log -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.filePattern = logs/%d{yyyy-MM}{UTC}/longRunningActivity-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log.gz +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.fileName = ${tempDir}/longRunningActivity/longRunningActivity.log +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.filePattern = ${logFileNamePrefix}/longRunningActivity/longRunningActivity-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.tempCompressedFilePattern = ${logFileNamePrefix}/longRunningActivity/longRunningActivity-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.tmp appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.layout.type = PatternLayout appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.layout.pattern = %msg%n -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.type = Policies -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.time.type = TimeBasedTriggeringPolicy -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.time.interval = 60 -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.time.modulate = true -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.type = DefaultRolloverStrategy -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.max = 4 +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.type=Policies +# Make a new file once per minute +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.time.type=TimeBasedTriggeringPolicy +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.time.interval=1 +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.time.modulate=true +# Make a new file if size is over 100MB +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.size.type=SizeBasedTriggeringPolicy +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.size.size=100MB +# Make a new file on startup +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.startup.type=OnStartupTriggeringPolicy +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.type=DefaultRolloverStrategy +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.fileIndex=nomax +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.compressionLevel = 7 appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.immediateFlush = false rootLogger.level = info @@ -90,8 +121,9 @@ logger.OutputTupleJsonLogger.appenderRef.OUTPUT_TUPLES.ref = OUTPUT_TUPLES logger.TransactionSummaryLogger.name = TransactionSummaryLogger logger.TransactionSummaryLogger.level = info logger.TransactionSummaryLogger.additivity = false -#logger.TransactionSummaryLogger.appenderRef.TRANSACTION_SUMMARIES.ref = TransactionSummariesConsole logger.TransactionSummaryLogger.appenderRef.TRANSACTION_SUMMARIES_LOGFILE.ref = TransactionSummariesFile +# Uncomment to enable progress logs by console output +#logger.TransactionSummaryLogger.appenderRef.TRANSACTION_SUMMARIES.ref = TransactionSummariesConsole logger.AllActiveWorkMonitor.name = AllActiveWorkMonitor logger.AllActiveWorkMonitor.level = info From c8381361b9434de143ea85160fe7d26016ef9464 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Wed, 23 Apr 2025 16:30:21 -0600 Subject: [PATCH 4/5] Fix tempCompressedFilePattern Signed-off-by: Mikayla Thompson --- .../trafficReplayer/src/main/resources/log4j2.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties b/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties index 9a792ab9ab..6ba3bc9744 100644 --- a/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties +++ b/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties @@ -17,7 +17,6 @@ appender.ReplayerLogFile.type = RollingRandomAccessFile appender.ReplayerLogFile.name = ReplayerLogFile appender.ReplayerLogFile.fileName = ${tempDir}/logs/replayer.log appender.ReplayerLogFile.filePattern = ${logFileNamePrefix}/logs/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz -appender.ReplayerLogFile.tempCompressedFilePattern = ${logFileNamePrefix}/logs/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz.tmp appender.ReplayerLogFile.layout.type = PatternLayout appender.ReplayerLogFile.layout.pattern = [%-5level] %d{DEFAULT_MICROS}{UTC} [%t] %c{1} - %msg%equals{ ctx=%mdc}{ ctx={}}{}%n appender.ReplayerLogFile.policies.type=Policies @@ -32,6 +31,7 @@ appender.ReplayerLogFile.policies.size.size=1GB appender.ReplayerLogFile.policies.startup.type=OnStartupTriggeringPolicy appender.ReplayerLogFile.strategy.type=DefaultRolloverStrategy appender.ReplayerLogFile.strategy.fileIndex=nomax +appender.ReplayerLogFile.strategy.tempCompressedFilePattern = ${logFileNamePrefix}/logs/replayer-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz.tmp appender.ReplayerLogFile.strategy.compressionLevel = 7 appender.ReplayerLogFile.immediateFlush = false @@ -40,7 +40,6 @@ appender.OUTPUT_TUPLES.type = RollingRandomAccessFile appender.OUTPUT_TUPLES.name = OUTPUT_TUPLES appender.OUTPUT_TUPLES.fileName = ${tempDir}/progress/tuples.log appender.OUTPUT_TUPLES.filePattern = ${logFileNamePrefix}/tuples/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz -appender.OUTPUT_TUPLES.tempCompressedFilePattern = ${logFileNamePrefix}/tuples/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz.tmp appender.OUTPUT_TUPLES.layout.type = PatternLayout appender.OUTPUT_TUPLES.layout.pattern = %m%n appender.OUTPUT_TUPLES.policies.type=Policies @@ -55,6 +54,7 @@ appender.OUTPUT_TUPLES.policies.size.size=1GB appender.OUTPUT_TUPLES.policies.startup.type=OnStartupTriggeringPolicy appender.OUTPUT_TUPLES.strategy.type=DefaultRolloverStrategy appender.OUTPUT_TUPLES.strategy.fileIndex=nomax +appender.OUTPUT_TUPLES.strategy.tempCompressedFilePattern = ${logFileNamePrefix}/tuples/tuples-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz.tmp appender.OUTPUT_TUPLES.strategy.compressionLevel = 7 appender.OUTPUT_TUPLES.immediateFlush = false @@ -69,7 +69,6 @@ appender.TRANSACTION_SUMMARIES_LOGFILE.type = RollingRandomAccessFile appender.TRANSACTION_SUMMARIES_LOGFILE.name = TransactionSummariesFile appender.TRANSACTION_SUMMARIES_LOGFILE.fileName = ${tempDir}/progress/progress.log appender.TRANSACTION_SUMMARIES_LOGFILE.filePattern = ${logFileNamePrefix}/progress/progress-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz -appender.TRANSACTION_SUMMARIES_LOGFILE.tempCompressedFilePattern = ${logFileNamePrefix}/progress/progress-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.tmp appender.TRANSACTION_SUMMARIES_LOGFILE.layout.type = PatternLayout appender.TRANSACTION_SUMMARIES_LOGFILE.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS}{UTC}: %msg%n appender.TRANSACTION_SUMMARIES_LOGFILE.policies.type=Policies @@ -84,6 +83,7 @@ appender.TRANSACTION_SUMMARIES_LOGFILE.policies.size.size=100MB appender.TRANSACTION_SUMMARIES_LOGFILE.policies.startup.type=OnStartupTriggeringPolicy appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.type=DefaultRolloverStrategy appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.fileIndex=nomax +appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.tempCompressedFilePattern = ${logFileNamePrefix}/progress/progress-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.tmp appender.TRANSACTION_SUMMARIES_LOGFILE.strategy.compressionLevel = 7 appender.TRANSACTION_SUMMARIES_LOGFILE.immediateFlush = false @@ -91,7 +91,6 @@ appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.type = RollingRandomAccessFile appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.name = AllActiveWorkMonitorFile appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.fileName = ${tempDir}/longRunningActivity/longRunningActivity.log appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.filePattern = ${logFileNamePrefix}/longRunningActivity/longRunningActivity-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.gz -appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.tempCompressedFilePattern = ${logFileNamePrefix}/longRunningActivity/longRunningActivity-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.tmp appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.layout.type = PatternLayout appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.layout.pattern = %msg%n appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.type=Policies @@ -106,6 +105,7 @@ appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.size.size=100MB appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.policies.startup.type=OnStartupTriggeringPolicy appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.type=DefaultRolloverStrategy appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.fileIndex=nomax +appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.tempCompressedFilePattern = ${logFileNamePrefix}/longRunningActivity/longRunningActivity-%d{yyyy-MM-dd-HH-mm}{UTC}-%03i.log.tmp appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.strategy.compressionLevel = 7 appender.ALL_ACTIVE_WORK_MONITOR_LOGFILE.immediateFlush = false From 8b563f9a4b2ac8a4615d40ceec50d894e9b95219 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Wed, 23 Apr 2025 16:31:22 -0600 Subject: [PATCH 5/5] Revert "Support log config from a specified file" This reverts commit 54d5c4651e365189f55174a493b9aaeee1187e76. Signed-off-by: Mikayla Thompson --- .../migrations/replay/TrafficReplayer.java | 24 ------------------- .../replayer/templates/deployment.yaml | 3 --- 2 files changed, 27 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 27f44741d7..ddf1456194 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -1,7 +1,5 @@ package org.opensearch.migrations.replay; -import java.io.File; -import java.io.IOException; import java.lang.ref.WeakReference; import java.net.URI; import java.time.Clock; @@ -38,10 +36,6 @@ import io.netty.util.concurrent.DefaultThreadFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LoggerContext; -import org.apache.logging.log4j.core.config.ConfigurationFactory; -import org.apache.logging.log4j.core.config.ConfigurationSource; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; import software.amazon.awssdk.arns.Arn; @@ -51,7 +45,6 @@ @Slf4j public class TrafficReplayer { private static final String ALL_ACTIVE_CONTEXTS_MONITOR_LOGGER = "AllActiveWorkMonitor"; - public static final String LOG4J2_PROPERTIES_FILE = "LOG4J2_PROPERTIES_FILE"; public static final String SIGV_4_AUTH_HEADER_SERVICE_REGION_ARG = "--sigv4-auth-header-service-region"; public static final String AUTH_HEADER_VALUE_ARG = "--auth-header-value"; @@ -98,21 +91,6 @@ public static boolean validateRequiredKafkaParams(String brokers, String topic, return true; } - public static void reloadConfiguration(String configFile) throws IOException { - ConfigurationSource source; - var file = configFile == null || configFile.isEmpty() ? null : new File(configFile); - if (file != null && file.exists()) { - source = new ConfigurationSource(file.toURI().toURL().openStream(), file.toURI().toURL()); - LoggerContext context = (LoggerContext) LogManager.getContext(false); - var config = ConfigurationFactory.getInstance().getConfiguration(context, source); - context.start(config); - log.atInfo().setMessage("Successfully reloaded Log4j2 configuration from: {}") - .addArgument(configFile).log(); - } else { - log.atInfo().setMessage("Using the default Log4j2 configuration").log(); - } - } - public static class Parameters { @Parameter( required = true, @@ -355,8 +333,6 @@ private static Parameters parseArgs(String[] args) { public static void main(String[] args) throws Exception { System.err.println("Got args: " + String.join("; ", args)); - reloadConfiguration(System.getenv(LOG4J2_PROPERTIES_FILE)); - final var workerId = ProcessHelpers.getNodeInstanceName(); log.info("Starting Traffic Replayer with id=" + workerId); diff --git a/deployment/k8s/charts/components/replayer/templates/deployment.yaml b/deployment/k8s/charts/components/replayer/templates/deployment.yaml index 698c6de9cb..91c7f51b16 100644 --- a/deployment/k8s/charts/components/replayer/templates/deployment.yaml +++ b/deployment/k8s/charts/components/replayer/templates/deployment.yaml @@ -47,9 +47,6 @@ spec: - name: shared-logs mountPath: /shared-logs-output {{- end }} - env: - - name: LOG4J2_PROPERTIES_FILE - value: "/shared-logs-output/replayer_log4j.properties" volumes: - name: {{ $envMountName }} emptyDir: {}