Drop old code to sync files on server filesystem. Clean cli, init paths

This stale code was originally used to index files on server file
system directly by server. We currently push files to sync via API.

Server side syncing of remote content like Github and Notion is still
supported. But old, unused code for server side sync of files on
server fs is being cleaned out.

New --log-file cli args allows specifying where khoj server should
store logs on fs. This replaces the --config-file cli arg that was
only being used as a proxy for deciding where to store the log file.

- TODO
  - Tests are broken. They were relying on the server side content
    syncing for test setup

Author: debanjum

src/khoj/configure.py

@@ -50,13 +50,11 @@
 )
 from khoj.database.models import ClientApplication, KhojUser, ProcessLock, Subscription
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
-from khoj.routers.api_content import configure_content, configure_search
+from khoj.routers.api_content import configure_content
 from khoj.routers.twilio import is_twilio_enabled
 from khoj.utils import constants, state
 from khoj.utils.config import SearchType
-from khoj.utils.fs_syncer import collect_files
-from khoj.utils.helpers import is_none_or_empty, telemetry_disabled
-from khoj.utils.rawconfig import FullConfig
+from khoj.utils.helpers import is_none_or_empty
 
 logger = logging.getLogger(__name__)
 
@@ -232,14 +230,6 @@ async def authenticate(self, request: HTTPConnection):
         return AuthCredentials(), UnauthenticatedUser()
 
 
-def initialize_server(config: Optional[FullConfig]):
-    try:
-        configure_server(config, init=True)
-    except Exception as e:
-        logger.error(f"🚨 Failed to configure server on app load: {e}", exc_info=True)
-        raise e
-
-
 def clean_connections(func):
     """
     A decorator that ensures that Django database connections that have become unusable, or are obsolete, are closed
@@ -260,19 +250,7 @@ def func_wrapper(*args, **kwargs):
     return func_wrapper
 
 
-def configure_server(
-    config: FullConfig,
-    regenerate: bool = False,
-    search_type: Optional[SearchType] = None,
-    init=False,
-    user: KhojUser = None,
-):
-    # Update Config
-    if config == None:
-        logger.info(f"Initializing with default config.")
-        config = FullConfig()
-    state.config = config
-
+def initialize_server():
     if ConversationAdapters.has_valid_ai_model_api():
         ai_model_api = ConversationAdapters.get_ai_model_api()
         state.openai_client = openai.OpenAI(api_key=ai_model_api.api_key, base_url=ai_model_api.api_base_url)
@@ -309,43 +287,33 @@ def configure_server(
             )
 
         state.SearchType = configure_search_types()
-        state.search_models = configure_search(state.search_models, state.config.search_type)
-        setup_default_agent(user)
+        setup_default_agent()
 
-        message = (
-            "📡 Telemetry disabled"
-            if telemetry_disabled(state.config.app, state.telemetry_disabled)
-            else "📡 Telemetry enabled"
-        )
+        message = "📡 Telemetry disabled" if state.telemetry_disabled else "📡 Telemetry enabled"
         logger.info(message)
 
-        if not init:
-            initialize_content(user, regenerate, search_type)
-
     except Exception as e:
         logger.error(f"Failed to load some search models: {e}", exc_info=True)
 
 
-def setup_default_agent(user: KhojUser):
-    AgentAdapters.create_default_agent(user)
+def setup_default_agent():
+    AgentAdapters.create_default_agent()
 
 
 def initialize_content(user: KhojUser, regenerate: bool, search_type: Optional[SearchType] = None):
     # Initialize Content from Config
-    if state.search_models:
-        try:
-            logger.info("📬 Updating content index...")
-            all_files = collect_files(user=user)
-            status = configure_content(
-                user,
-                all_files,
-                regenerate,
-                search_type,
-            )
-            if not status:
-                raise RuntimeError("Failed to update content index")
-        except Exception as e:
-            raise e
+    try:
+        logger.info("📬 Updating content index...")
+        status = configure_content(
+            user,
+            {},
+            regenerate,
+            search_type,
+        )
+        if not status:
+            raise RuntimeError("Failed to update content index")
+    except Exception as e:
+        raise e
 
 
 def configure_routes(app):
@@ -438,8 +406,7 @@ async def dispatch(self, request: Request, call_next):
 
 def update_content_index():
     for user in get_all_users():
-        all_files = collect_files(user=user)
-        success = configure_content(user, all_files)
+        success = configure_content(user, {})
     if not success:
         raise RuntimeError("Failed to update content index")
     logger.info("📪 Content index updated via Scheduler")
@@ -464,7 +431,7 @@ def configure_search_types():
 @schedule.repeat(schedule.every(2).minutes)
 @clean_connections
 def upload_telemetry():
-    if telemetry_disabled(state.config.app, state.telemetry_disabled) or not state.telemetry:
+    if state.telemetry_disabled or not state.telemetry:
         return
 
     try:

src/khoj/database/adapters/__init__.py

@@ -788,8 +788,8 @@ def get_default_agent():
         return Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).first()
 
     @staticmethod
-    def create_default_agent(user: KhojUser):
-        default_chat_model = ConversationAdapters.get_default_chat_model(user)
+    def create_default_agent():
+        default_chat_model = ConversationAdapters.get_default_chat_model(user=None)
         if default_chat_model is None:
             logger.info("No default conversation config found, skipping default agent creation")
             return None

src/khoj/database/migrations/0093_remove_localorgconfig_user_and_more.py

@@ -0,0 +1,36 @@
+# Generated by Django 5.1.10 on 2025-07-25 23:30
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0092_alter_chatmodel_model_type_alter_chatmodel_name_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="localorgconfig",
+            name="user",
+        ),
+        migrations.RemoveField(
+            model_name="localpdfconfig",
+            name="user",
+        ),
+        migrations.RemoveField(
+            model_name="localplaintextconfig",
+            name="user",
+        ),
+        migrations.DeleteModel(
+            name="LocalMarkdownConfig",
+        ),
+        migrations.DeleteModel(
+            name="LocalOrgConfig",
+        ),
+        migrations.DeleteModel(
+            name="LocalPdfConfig",
+        ),
+        migrations.DeleteModel(
+            name="LocalPlaintextConfig",
+        ),
+    ]

src/khoj/database/models/__init__.py

@@ -488,34 +488,6 @@ def save(self, *args, **kwargs):
         super().save(*args, **kwargs)
 
 
-class LocalOrgConfig(DbBaseModel):
-    input_files = models.JSONField(default=list, null=True)
-    input_filter = models.JSONField(default=list, null=True)
-    index_heading_entries = models.BooleanField(default=False)
-    user = models.ForeignKey(KhojUser, on_delete=models.CASCADE)
-
-
-class LocalMarkdownConfig(DbBaseModel):
-    input_files = models.JSONField(default=list, null=True)
-    input_filter = models.JSONField(default=list, null=True)
-    index_heading_entries = models.BooleanField(default=False)
-    user = models.ForeignKey(KhojUser, on_delete=models.CASCADE)
-
-
-class LocalPdfConfig(DbBaseModel):
-    input_files = models.JSONField(default=list, null=True)
-    input_filter = models.JSONField(default=list, null=True)
-    index_heading_entries = models.BooleanField(default=False)
-    user = models.ForeignKey(KhojUser, on_delete=models.CASCADE)
-
-
-class LocalPlaintextConfig(DbBaseModel):
-    input_files = models.JSONField(default=list, null=True)
-    input_filter = models.JSONField(default=list, null=True)
-    index_heading_entries = models.BooleanField(default=False)
-    user = models.ForeignKey(KhojUser, on_delete=models.CASCADE)
-
-
 class SearchModelConfig(DbBaseModel):
     class ModelType(models.TextChoices):
         TEXT = "text"

src/khoj/main.py

@@ -138,10 +138,10 @@ def run(should_start_server=True):
     initialization(not args.non_interactive)
 
     # Create app directory, if it doesn't exist
-    state.config_file.parent.mkdir(parents=True, exist_ok=True)
+    state.log_file.parent.mkdir(parents=True, exist_ok=True)
 
     # Set Log File
-    fh = logging.FileHandler(state.config_file.parent / "khoj.log", encoding="utf-8")
+    fh = logging.FileHandler(state.log_file, encoding="utf-8")
     fh.setLevel(logging.DEBUG)
     logger.addHandler(fh)
 
@@ -194,7 +194,7 @@ def run(should_start_server=True):
     # Configure Middleware
     configure_middleware(app, state.ssl_config)
 
-    initialize_server(args.config)
+    initialize_server()
 
     # If the server is started through gunicorn (external to the script), don't start the server
     if should_start_server:
@@ -204,8 +204,7 @@ def run(should_start_server=True):
 
 
 def set_state(args):
-    state.config_file = args.config_file
-    state.config = args.config
+    state.log_file = args.log_file
     state.verbose = args.verbose
     state.host = args.host
     state.port = args.port

src/khoj/processor/content/github/github_to_entries.py

@@ -20,7 +20,6 @@
 
 class GithubToEntries(TextToEntries):
     def __init__(self, config: GithubConfig):
-        super().__init__(config)
         raw_repos = config.githubrepoconfig.all()
         repos = []
         for repo in raw_repos:

src/khoj/processor/content/notion/notion_to_entries.py

@@ -47,7 +47,6 @@ class NotionBlockType(Enum):
 
 class NotionToEntries(TextToEntries):
     def __init__(self, config: NotionConfig):
-        super().__init__(config)
         self.config = NotionContentConfig(
             token=config.token,
         )

src/khoj/processor/content/text_to_entries.py

@@ -27,7 +27,6 @@
 class TextToEntries(ABC):
     def __init__(self, config: Any = None):
         self.embeddings_model = state.embeddings_model
-        self.config = config
         self.date_filter = DateFilter()
 
     @abstractmethod

src/khoj/routers/api.py

@@ -87,22 +87,14 @@ def update(
     force: Optional[bool] = False,
 ):
     user = request.user.object
-    if not state.config:
-        error_msg = f"🚨 Khoj is not configured.\nConfigure it via http://localhost:42110/settings, plugins or by editing {state.config_file}."
-        logger.warning(error_msg)
-        raise HTTPException(status_code=500, detail=error_msg)
     try:
         initialize_content(user=user, regenerate=force, search_type=t)
     except Exception as e:
-        error_msg = f"🚨 Failed to update server via API: {e}"
+        error_msg = f"🚨 Failed to update server indexed content via API: {e}"
         logger.error(error_msg, exc_info=True)
         raise HTTPException(status_code=500, detail=error_msg)
     else:
-        components = []
-        if state.search_models:
-            components.append("Search models")
-        components_msg = ", ".join(components)
-        logger.info(f"📪 {components_msg} updated via API")
+        logger.info(f"📪 Server indexed content updated via API")
 
     update_telemetry_state(
         request=request,