chore: Azure SB Emulator for testing & local development

.env.example

@@ -55,16 +55,19 @@ RABBITMQ_LOG_QUEUE="outpost-log"
 # GCP_PUBSUB_LOG_SUBSCRIPTION="outpost-log-sub"
 
 ## Azure ServiceBus
-AZURE_SERVICEBUS_TENANT_ID=""
-AZURE_SERVICEBUS_CLIENT_ID=""
-AZURE_SERVICEBUS_CLIENT_SECRET=""
-AZURE_SERVICEBUS_SUBSCRIPTION_ID=""
-AZURE_SERVICEBUS_RESOURCE_GROUP=""
-AZURE_SERVICEBUS_NAMESPACE=""
-AZURE_SERVICEBUS_DELIVERY_TOPIC="outpost-delivery"
-AZURE_SERVICEBUS_DELIVERY_SUBSCRIPTION="outpost-delivery-sub"
-AZURE_SERVICEBUS_LOG_TOPIC="outpost-log"
-AZURE_SERVICEBUS_LOG_SUBSCRIPTION="outpost-log-sub"
+# AZURE_SERVICEBUS_TENANT_ID=""
+# AZURE_SERVICEBUS_CLIENT_ID=""
+# AZURE_SERVICEBUS_CLIENT_SECRET=""
+# AZURE_SERVICEBUS_SUBSCRIPTION_ID=""
+# AZURE_SERVICEBUS_RESOURCE_GROUP=""
+# AZURE_SERVICEBUS_NAMESPACE=""
+# AZURE_SERVICEBUS_DELIVERY_TOPIC="outpost-delivery"
+# AZURE_SERVICEBUS_DELIVERY_SUBSCRIPTION="outpost-delivery-sub"
+# AZURE_SERVICEBUS_LOG_TOPIC="outpost-log"
+# AZURE_SERVICEBUS_LOG_SUBSCRIPTION="outpost-log-sub"
+# or for local emulator:
+# AZURE_SERVICEBUS_CONNECTION_STRING="Endpoint=sb://azuresb;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;"
+
 
 
 # ============================== PublishMQ ==============================

.env.test

@@ -1,6 +1,10 @@
+# DBs
 TEST_POSTGRES_URL="localhost:35432"
 TEST_CLICKHOUSE_URL="localhost:39000"
+# MQs
+TEST_RABBITMQ_URL="localhost:35672"
 TEST_LOCALSTACK_URL="localhost:34566"
 TEST_GCP_URL="localhost:38085"
-TEST_RABBITMQ_URL="localhost:35672"
+TEST_AZURE_SB_CONNSTRING="Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;"
+# Misc
 TEST_MOCKSERVER_URL="localhost:35555"

Makefile

@@ -41,12 +41,21 @@ down/uptrace:
 up/portal:
 	cd internal/portal && npm install && npm run dev
 
+up/azure:
+	docker compose -f build/dev/azure/compose.yml up -d
+
+down/azure:
+	docker compose -f build/dev/azure/compose.yml down --volumes
+
 up/test:
 	docker-compose -f build/test/compose.yml up -d
 
 down/test:
 	docker-compose -f build/test/compose.yml down --volumes
 
+test/setup:
+	bash scripts/test-setup-info.sh
+
 test:
 	go test $(TEST) $(TESTARGS)
 

build/dev/azure/compose.yml

@@ -0,0 +1,28 @@
+name: "outpost-azure"
+
+services:
+  azuresb:
+    image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest
+    volumes:
+      - "./config.json:/ServiceBus_Emulator/ConfigFiles/Config.json"
+    ports:
+      - "5672:5672"
+      - "5300:5300"
+    environment:
+      SQL_SERVER: sqledge
+      MSSQL_SA_PASSWORD: "Password!"
+      ACCEPT_EULA: "Y"
+      SQL_WAIT_INTERVAL: "5"
+    depends_on:
+      - sqledge
+
+  sqledge:
+    image: mcr.microsoft.com/azure-sql-edge:latest
+    environment:
+      ACCEPT_EULA: "Y"
+      MSSQL_SA_PASSWORD: "Password!"
+
+networks:
+  default:
+    name: outpost
+    external: true

build/dev/azure/config.json

@@ -0,0 +1,55 @@
+{
+  "UserConfig": {
+    "Namespaces": [
+      {
+        "Name": "sbemulatorns",
+        "Queues": [],
+        "Topics": [
+          // dev
+          {
+            "Name": "outpost-delivery",
+            "Properties": {},
+            "Subscriptions": [
+              {
+                "Name": "outpost-delivery-sub",
+                "Properties": {
+                  "MaxDeliveryCount": 6
+                },
+                "Rules": []
+              }
+            ]
+          },
+          {
+            "Name": "outpost-log",
+            "Properties": {},
+            "Subscriptions": [
+              {
+                "Name": "outpost-log-sub",
+                "Properties": {
+                  "MaxDeliveryCount": 6
+                },
+                "Rules": []
+              }
+            ]
+          },
+
+          // tests
+          {
+            "Name": "TestIntegrationMQ_AzureServiceBus-topic",
+            "Properties": {},
+            "Subscriptions": [
+              {
+                "Name": "TestIntegrationMQ_AzureServiceBus-subscription",
+                "Properties": {},
+                "Rules": []
+              }
+            ]
+          }
+        ]
+      }
+    ],
+    "Logging": {
+      "Type": "Console"
+    }
+  }
+}

build/dev/deps/compose.yml

@@ -63,7 +63,9 @@ services:
   rabbitmq:
     image: rabbitmq:3-management
     ports:
-      - 5672:5672
+      # Reserve 5672 for AzureSB Emulator which doens't support custom ports.
+      # We may not need to expose 5672 at all if we run the full local dev env with Docker Compose.
+      # - 5673:5672
       - 15672:15672
     volumes:
       - rabbitmq:/var/lib/rabbitmq

internal/config/mqconfig_azure.go

@@ -8,6 +8,9 @@ import (
 )
 
 type AzureServiceBusConfig struct {
+	// Using AzureSB with ConnectionString will skip infra management
+	ConnectionString string `yaml:"connection_string" env:"AZURE_SERVICEBUS_CONNECTION_STRING" desc:"Azure Service Bus connection string" required:"N"`
+
 	TenantID       string `yaml:"tenant_id" env:"AZURE_SERVICEBUS_TENANT_ID" desc:"Azure Active Directory tenant ID" required:"Y"`
 	ClientID       string `yaml:"client_id" env:"AZURE_SERVICEBUS_CLIENT_ID" desc:"Service principal client ID" required:"Y"`
 	ClientSecret   string `yaml:"client_secret" env:"AZURE_SERVICEBUS_CLIENT_SECRET" desc:"Service principal client secret" required:"Y"`
@@ -16,17 +19,17 @@ type AzureServiceBusConfig struct {
 	Namespace      string `yaml:"namespace" env:"AZURE_SERVICEBUS_NAMESPACE" desc:"Azure Service Bus namespace" required:"Y"`
 
 	DeliveryTopic        string `yaml:"delivery_topic" env:"AZURE_SERVICEBUS_DELIVERY_TOPIC" desc:"Topic name for delivery queue" required:"N" default:"outpost-delivery"`
-	DeliverySubscription string `yaml:"delivery_subscription" env:"AZURE_SERVICEBUS_DELIVERY_SUBSCRIPTION" desc:"Subscription name for delivery queue" required:"N" default:"outpost-delivery-subscription"`
+	DeliverySubscription string `yaml:"delivery_subscription" env:"AZURE_SERVICEBUS_DELIVERY_SUBSCRIPTION" desc:"Subscription name for delivery queue" required:"N" default:"outpost-delivery-sub"`
 	LogTopic             string `yaml:"log_topic" env:"AZURE_SERVICEBUS_LOG_TOPIC" desc:"Topic name for log queue" required:"N" default:"outpost-log"`
-	LogSubscription      string `yaml:"log_subscription" env:"AZURE_SERVICEBUS_LOG_SUBSCRIPTION" desc:"Subscription name for log queue" required:"N" default:"outpost-log-subscription"`
+	LogSubscription      string `yaml:"log_subscription" env:"AZURE_SERVICEBUS_LOG_SUBSCRIPTION" desc:"Subscription name for log queue" required:"N" default:"outpost-log-sub"`
 
 	// connectionStringOnce  sync.Once
 	// connectionString      string
 	// connectionStringError error
 }
 
 func (c *AzureServiceBusConfig) IsConfigured() bool {
-	return c.TenantID != "" && c.ClientID != "" && c.ClientSecret != "" && c.SubscriptionID != "" && c.ResourceGroup != "" && c.Namespace != ""
+	return c.ConnectionString != "" || (c.TenantID != "" && c.ClientID != "" && c.ClientSecret != "" && c.SubscriptionID != "" && c.ResourceGroup != "" && c.Namespace != "")
 }
 
 func (c *AzureServiceBusConfig) GetProviderType() string {
@@ -65,14 +68,15 @@ func (c *AzureServiceBusConfig) ToInfraConfig(queueType string) *mqinfra.MQInfra
 
 	return &mqinfra.MQInfraConfig{
 		AzureServiceBus: &mqinfra.AzureServiceBusInfraConfig{
-			TenantID:       c.TenantID,
-			ClientID:       c.ClientID,
-			ClientSecret:   c.ClientSecret,
-			SubscriptionID: c.SubscriptionID,
-			ResourceGroup:  c.ResourceGroup,
-			Namespace:      c.Namespace,
-			Topic:          topic,
-			Subscription:   subscription,
+			ConnectionString: c.ConnectionString,
+			TenantID:         c.TenantID,
+			ClientID:         c.ClientID,
+			ClientSecret:     c.ClientSecret,
+			SubscriptionID:   c.SubscriptionID,
+			ResourceGroup:    c.ResourceGroup,
+			Namespace:        c.Namespace,
+			Topic:            topic,
+			Subscription:     subscription,
 		},
 	}
 }
@@ -85,6 +89,16 @@ func (c *AzureServiceBusConfig) ToQueueConfig(ctx context.Context, queueType str
 	topic := c.getTopicByQueueType(queueType)
 	subscription := c.getSubscriptionByQueueType(queueType)
 
+	if c.ConnectionString != "" {
+		return &mqs.QueueConfig{
+			AzureServiceBus: &mqs.AzureServiceBusConfig{
+				ConnectionString: c.ConnectionString,
+				Topic:            topic,
+				Subscription:     subscription,
+			},
+		}, nil
+	}
+
 	return &mqs.QueueConfig{
 		AzureServiceBus: &mqs.AzureServiceBusConfig{
 			Topic:          topic,

internal/mqinfra/azureservicebus.go

@@ -21,6 +21,10 @@ func (infra *infraAzureServiceBus) Exist(ctx context.Context) (bool, error) {
 
 	cfg := infra.cfg.AzureServiceBus
 
+	if cfg.ConnectionString != "" {
+		return true, nil
+	}
+
 	// Create credential for authentication
 	cred, err := azidentity.NewClientSecretCredential(
 		cfg.TenantID,
@@ -71,6 +75,10 @@ func (infra *infraAzureServiceBus) Declare(ctx context.Context) error {
 
 	cfg := infra.cfg.AzureServiceBus
 
+	if cfg.ConnectionString != "" {
+		return nil
+	}
+
 	// Create credential for authentication
 	cred, err := azidentity.NewClientSecretCredential(
 		cfg.TenantID,
@@ -146,6 +154,10 @@ func (infra *infraAzureServiceBus) TearDown(ctx context.Context) error {
 
 	cfg := infra.cfg.AzureServiceBus
 
+	if cfg.ConnectionString != "" {
+		return nil
+	}
+
 	// Create credential for authentication
 	cred, err := azidentity.NewClientSecretCredential(
 		cfg.TenantID,

internal/mqinfra/mqinfra.go

@@ -33,6 +33,8 @@ type AWSSQSInfraConfig struct {
 }
 
 type AzureServiceBusInfraConfig struct {
+	ConnectionString string // If set, skip infra management
+
 	TenantID       string
 	ClientID       string
 	ClientSecret   string

internal/mqinfra/mqinfra_test.go

@@ -324,7 +324,7 @@ func TestIntegrationMQInfra_GCPPubSub(t *testing.T) {
 }
 
 func TestIntegrationMQInfra_AzureServiceBus(t *testing.T) {
-	t.Skip("skip AzureServiceBus integration test for now since there's no emulator yet")
+	t.Skip("skip TestIntegrationMQInfra_AzureServiceBus integration test for now since the emulator doesn't support managing resources")
 
 	topic := uuid.New().String()
 	subscription := topic + "-subscription"

internal/mqs/queue_test.go

@@ -49,11 +49,9 @@ func TestIntegrationMQ_GCPPubSub(t *testing.T) {
 }
 
 func TestIntegrationMQ_AzureServiceBus(t *testing.T) {
-	t.Skip("skip AzureServiceBus integration test for now since there's no emulator yet")
-
 	t.Parallel()
 	t.Cleanup(testinfra.Start(t))
-	// config := testinfra.NewMQAzureServiceBusConfig(t, nil)
+	config := testinfra.GetMQAzureConfig(t, "TestIntegrationMQ_AzureServiceBus")
 
 	// config := mqs.QueueConfig{
 	// 	AzureServiceBus: &mqs.AzureServiceBusConfig{
@@ -62,18 +60,18 @@ func TestIntegrationMQ_AzureServiceBus(t *testing.T) {
 	// 		Subscription:     "test-subscription",
 	// 	},
 	// }
-	config := mqs.QueueConfig{
-		AzureServiceBus: &mqs.AzureServiceBusConfig{
-			TenantID:       "",
-			ClientID:       "",
-			ClientSecret:   "",
-			SubscriptionID: "",
-			ResourceGroup:  "",
-			Namespace:      "",
-			Topic:          "test-topic",
-			Subscription:   "test-subscription",
-		},
-	}
+	// config := mqs.QueueConfig{
+	// 	AzureServiceBus: &mqs.AzureServiceBusConfig{
+	// 		TenantID:       "",
+	// 		ClientID:       "",
+	// 		ClientSecret:   "",
+	// 		SubscriptionID: "",
+	// 		ResourceGroup:  "",
+	// 		Namespace:      "",
+	// 		Topic:          "test-topic",
+	// 		Subscription:   "test-subscription",
+	// 	},
+	// }
 	testMQ(t, func() mqs.QueueConfig { return config })
 }