update script and execution design

Author: nikhilsinhaparseable

Cargo.lock

@@ -122,6 +122,7 @@ thiserror = "2.0"
 ulid = { version = "1.0", features = ["serde"] }
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 futures-core = "0.3.31"
+tempfile = "3.20.0"
 
 [build-dependencies]
 cargo_toml = "0.21"

Cargo.toml

@@ -1,36 +1,11 @@
 #!/usr/bin/env bash
 
-# Configuration with validation
+# Configuration
 P_URL=${P_URL:-"http://localhost:8000"}
 P_USERNAME=${P_USERNAME:-"admin"}
 P_PASSWORD=${P_PASSWORD:-"admin"}
 P_STREAM=${P_STREAM:-"demodata"}
 
-# Silent mode handling
-SILENT=${SILENT:-false}
-for arg in "$@"; do
-    case $arg in
-        --silent)
-            SILENT=true
-            shift
-            ;;
-        -h|--help)
-            echo "Usage: $0 [--silent]"
-            echo "  --silent    Run in silent mode"
-            exit 0
-            ;;
-    esac
-done
-
-# Only show config if not silent
-if [[ "$SILENT" != "true" ]]; then
-    echo "Configuration:"
-    echo "  URL: $P_URL"
-    echo "  Username: $P_USERNAME"
-    echo "  Stream: $P_STREAM"
-    echo
-fi
-
 # Pre-compute auth header
 AUTH_HEADER="Authorization: Basic $(echo -n "$P_USERNAME:$P_PASSWORD" | base64)"
 
@@ -41,26 +16,20 @@ curl_with_retry() {
     local data="$3"
     local content_type="${4:-application/json}"
     local max_retries="${5:-3}"
-    local base_timeout="${6:-15}"
     local retry_count=0
 
-    # Set timeout based on retry attempt: 10s, 20s, 30s
-    local max_time=$((10 + (retry_count * 10)))
-    local connect_timeout=5
-    
     # Create temp file if data is provided
     if [[ -n "$data" ]]; then
         temp_file=$(mktemp)
         if [[ $? -ne 0 ]]; then
-            print_error "Failed to create temporary file"
             return 1
         fi
         echo "$data" > "$temp_file"
     fi
 
     while [[ $retry_count -lt $max_retries ]]; do
-        # Current timeout: 10s, 20s, 30s for attempts 1, 2, 3
-        max_time=$((10 + (retry_count * 10)))
+        local max_time=$((10 + (retry_count * 10)))
+        local connect_timeout=5
 
         local curl_cmd="curl -s -w \"\n%{http_code}\" --max-time $max_time --connect-timeout $connect_timeout"
 
@@ -93,81 +62,38 @@ curl_with_retry() {
 
         # Check curl exit code
         if [[ $curl_exit_code -eq 0 ]]; then
-            # Success - extract status code and return
             local status_code
             if [[ -n "$response" ]]; then
                 status_code=$(echo "$response" | tail -n1)
-                local response_body=$(echo "$response" | sed '$d')
 
                 # Clean up temp file
                 [[ -n "$temp_file" ]] && rm -f "$temp_file"
 
                 if [[ "$status_code" == "200" || "$status_code" == "201" ]]; then
                     return 0
                 else
-                    print_error "HTTP $status_code: $response_body"
                     return 1
                 fi
             else
-                print_error "No response from server"
                 return 1
             fi
         elif [[ $curl_exit_code -eq 28 ]]; then
-            # Timeout - retry immediately with next timeout level
+            # Timeout - retry
             retry_count=$((retry_count + 1))
-            
-            if [[ "$SILENT" != "true" && -n "$data" ]]; then
-                echo "Timeout (${#data} chars) - retry $retry_count with $((10 + (retry_count * 10)))s timeout"
-            elif [[ "$SILENT" != "true" ]]; then
-                echo "Timeout - retry $retry_count with $((10 + (retry_count * 10)))s timeout"
-            fi
-            
-            # Brief pause before retry
             sleep 1
         else
-            # Other error - break and report
             break
         fi
     done
 
     # Clean up temp file on failure
     [[ -n "$temp_file" ]] && rm -f "$temp_file"
 
-    # Final error reporting
-    print_error "curl failed with exit code $curl_exit_code after $retry_count retries"
-    if [[ -n "$data" ]]; then
-        print_error "Data size: ${#data} characters, Final timeout: ${max_time}s"
-    fi
-    [[ "$SILENT" != "true" ]] && print_error "Response: $response"
-    
     return 1
 }
 
-# Colors
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-BLUE='\033[0;34m'
-NC='\033[0m'
-
-print_info() { [[ "$SILENT" != "true" ]] && echo -e "${BLUE}[INFO]${NC} $1"; }
-print_success() { [[ "$SILENT" != "true" ]] && echo -e "${GREEN}[SUCCESS]${NC} $1"; }
-print_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }
-
-# Test connection before creating filters
-if [[ "$SILENT" != "true" ]]; then
-    print_info "Testing connectivity..."
-    if curl_with_retry "$P_URL" "GET" "" "text/html" 1 5; then
-        print_info "Basic connectivity OK"
-    else
-        print_error "Cannot connect to $P_URL - check if server is running"
-        exit 1
-    fi
-fi
-
-# Create comprehensive SQL filters (10 filters)
+# Create SQL filters
 create_sql_filters() {
-    print_info "Creating 10 SQL filters..."
-    
     sql_filters=(
         "error_logs|Monitor all ERROR and FATAL severity events|SELECT * FROM $P_STREAM WHERE severity_text IN ('ERROR', 'FATAL') ORDER BY time_unix_nano DESC LIMIT 100"
         "high_response_time|Identify requests with extended response times|SELECT \"service.name\", \"url.path\", body FROM $P_STREAM WHERE body LIKE '%duration%' ORDER BY time_unix_nano DESC LIMIT 50"
@@ -182,43 +108,28 @@ create_sql_filters() {
     )
 
     sql_success_count=0
-    filter_number=1
 
     for filter_config in "${sql_filters[@]}"; do
         IFS='|' read -r name description query <<< "$filter_config"
 
-        [[ "$SILENT" != "true" ]] && echo "Creating SQL filter $filter_number/10: $name"
-        
         # Escape quotes for JSON
         escaped_query=$(echo "$query" | sed 's/"/\\"/g')
         escaped_desc=$(echo "$description" | sed 's/"/\\"/g')
 
         json="{\"stream_name\":\"sql\",\"filter_name\":\"$name\",\"filter_description\":\"$escaped_desc\",\"query\":{\"filter_type\":\"sql\",\"filter_query\":\"$escaped_query\"},\"time_filter\":null}"
 
-        # Add timeout and better error handling
         if curl_with_retry "$P_URL/api/v1/filters" "POST" "$json" "application/json" 3 10; then
-            [[ "$SILENT" != "true" ]] && echo "✓ SQL Filter: $name"
             sql_success_count=$((sql_success_count + 1))
-        else
-            [[ "$SILENT" != "true" ]] && echo "✗ Failed after retries: $name"
         fi
 
-        # Small delay between requests to avoid overwhelming server
         sleep 0.5
-        filter_number=$((filter_number + 1))
     done
 
-    [[ "$SILENT" != "true" ]] && print_success "Created $sql_success_count/10 SQL filters"
-    
-    # Wait a bit before creating saved filters
-    [[ "$SILENT" != "true" ]] && echo "Waiting 3 seconds before creating saved filters..."
     sleep 3
 }
 
-# Create comprehensive saved filters (10 filters)
+# Create saved filters
 create_saved_filters() {
-    print_info "Creating 10 saved filters..."
-    
     saved_filters=(
         "service_errors|Monitor service errors and failures|SELECT * FROM $P_STREAM WHERE severity_text IN ('ERROR', 'FATAL') LIMIT 500|Ingestion Time,Data,service.name,severity_text,url.path|service.name"
         "auth_security_events|Authentication and authorization monitoring|SELECT * FROM $P_STREAM WHERE url.path LIKE '%login%' AND severity_text IN ('WARN', 'ERROR', 'FATAL') LIMIT 500|Ingestion Time,Data,service.name,severity_text,source.address,user_agent.original|severity_text"
@@ -233,13 +144,10 @@ create_saved_filters() {
     )
 
     saved_success_count=0
-    filter_number=1
 
     for filter_config in "${saved_filters[@]}"; do
         IFS='|' read -r name description query visible_columns group_by <<< "$filter_config"
 
-        [[ "$SILENT" != "true" ]] && echo "Creating saved filter $filter_number/10: $name"
-        
         # Escape quotes
         escaped_query=$(echo "$query" | sed 's/"/\\"/g')
         escaped_desc=$(echo "$description" | sed 's/"/\\"/g')
@@ -254,27 +162,16 @@ create_saved_filters() {
 
         json="{\"stream_name\":\"$P_STREAM\",\"filter_name\":\"$name\",\"filter_description\":\"$escaped_desc\",\"query\":{\"filter_type\":\"filter\",\"filter_query\":\"$escaped_query\"},\"time_filter\":null,\"tableConfig\":{\"visibleColumns\":[$visible_cols_json],\"pinnedColumns\":[]},\"groupBy\":\"$group_by\"}"
 
-        # Add timeout and better error handling for saved filters
         if curl_with_retry "$P_URL/api/v1/filters" "POST" "$json" "application/json" 3 10; then
-            [[ "$SILENT" != "true" ]] && echo "✓ Saved Filter: $name"
             saved_success_count=$((saved_success_count + 1))
-        else
-            [[ "$SILENT" != "true" ]] && echo "✗ Failed after retries: $name"
         fi
 
-        # Small delay between requests
         sleep 0.5
-        filter_number=$((filter_number + 1))
     done
-    
-    [[ "$SILENT" != "true" ]] && print_success "Created $saved_success_count/10 saved filters"
 }
 
 # Create all filters
 create_sql_filters
 create_saved_filters
 
-print_success "Filter creation completed successfully!"
-
-# Always exit with success if we get here
 exit 0