PySyft
PySyft copied to clipboard
OpenMined
Remove Mongo and Add PostgreSQL
Check out this pull request on 
See visual diffs & provide feedback on Jupyter Notebooks.
Powered by ReviewNB
![review-notebook-app[bot] avatar](https://avatars.githubusercontent.com/in/15474?v=4 "Published by a pub.dev verified publisher"){kind=small}
Add helm templates for postgres
- add postgres service to devspace
- pass postgres creds to backend
diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile
index 256fdfd447..0989cc4f4f 100644
--- a/packages/grid/backend/backend.dockerfile
+++ b/packages/grid/backend/backend.dockerfile
@@ -83,9 +83,10 @@ ENV \
DEFAULT_ROOT_EMAIL="[email protected]" \
DEFAULT_ROOT_PASSWORD="changethis" \
STACK_API_KEY="changeme" \
- MONGO_HOST="localhost" \
- MONGO_PORT="27017" \
- MONGO_USERNAME="root" \
- MONGO_PASSWORD="example"
+ POSTGRESQL_DBNAME="syftdb_postgres" \
+ POSTGRESQL_HOST="localhost" \
+ POSTGRESQL_PORT="5432" \
+ POSTGRESQL_USERNAME="syft_postgres" \
+ POSTGRESQL_PASSWORD="example"
CMD ["bash", "./grid/start.sh"]
diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py
index 0d50d9c907..6a201629b3 100644
--- a/packages/grid/backend/grid/core/config.py
+++ b/packages/grid/backend/grid/core/config.py
@@ -128,7 +128,7 @@ class Settings(BaseSettings):
CONTAINER_HOST: str = str(os.getenv("CONTAINER_HOST", "docker"))
POSTGRESQL_DBNAME: str = str(os.getenv("POSTGRESQL_DBNAME", ""))
POSTGRESQL_HOST: str = str(os.getenv("POSTGRESQL_HOST", ""))
- POSTGRESQL_PORT: int = int(os.getenv("POSTGRESQL_PORT", 27017))
+ POSTGRESQL_PORT: int = int(os.getenv("POSTGRESQL_PORT", 5432))
POSTGRESQL_USERNAME: str = str(os.getenv("POSTGRESQL_USERNAME", ""))
POSTGRESQL_PASSWORD: str = str(os.getenv("POSTGRESQL_PASSWORD", ""))
DEV_MODE: bool = True if os.getenv("DEV_MODE", "false").lower() == "true" else False
diff --git a/packages/grid/backend/grid/core/server.py b/packages/grid/backend/grid/core/server.py
index d284c8edb4..a447d56ed1 100644
--- a/packages/grid/backend/grid/core/server.py
+++ b/packages/grid/backend/grid/core/server.py
@@ -91,6 +91,10 @@ single_container_mode = settings.SINGLE_CONTAINER_MODE
store_config = (
sql_store_config() if single_container_mode else postgresql_store_config()
)
+
+print("----------------------------Store Config----------------------------\n")
+print(store_config.model_dump())
+print("\n----------------------------Store Config----------------------------")
blob_storage_config = None if single_container_mode else seaweedfs_config()
queue_config = queue_config()
diff --git a/packages/grid/default.env b/packages/grid/default.env
index 791778206f..a95248b07b 100644
--- a/packages/grid/default.env
+++ b/packages/grid/default.env
@@ -124,4 +124,4 @@ POSTGRESQL_DBNAME=syftdb_postgres
POSTGRESQL_HOST=localhost
POSTGRESQL_PORT=5432
POSTGRESQL_USERNAME=syft_postgres
-POSTGRESQL_PASSWORD=changethis
\ No newline at end of file
+POSTGRESQL_PASSWORD=example
\ No newline at end of file
diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml
index 8a35cdef7e..06efe99f4a 100644
--- a/packages/grid/devspace.yaml
+++ b/packages/grid/devspace.yaml
@@ -80,12 +80,12 @@ deployments:
- ./helm/examples/dev/base.yaml
dev:
- mongo:
+ postgres:
labelSelector:
app.kubernetes.io/name: syft
- app.kubernetes.io/component: mongo
+ app.kubernetes.io/component: postgres
ports:
- - port: "27017"
+ - port: "5432"
seaweedfs:
labelSelector:
app.kubernetes.io/name: syft
@@ -188,8 +188,8 @@ profiles:
path: dev.seaweedfs
# Port Re-Mapping
- op: replace
- path: dev.mongo.ports[0].port
- value: 27018:27017
+ path: dev.postgres.ports[0].port
+ value: 5433:5432
- op: replace
path: dev.backend.ports[0].port
value: 5679:5678
@@ -251,8 +251,8 @@ profiles:
value: ./helm/examples/dev/enclave.yaml
# Port Re-Mapping
- op: replace
- path: dev.mongo.ports[0].port
- value: 27019:27017
+ path: dev.postgres.ports[0].port
+ value: 5434:5432
- op: replace
path: dev.backend.ports[0].port
value: 5680:5678
diff --git a/packages/grid/helm/examples/dev/base.yaml b/packages/grid/helm/examples/dev/base.yaml
index 9b14fbe29e..e173e9ac53 100644
--- a/packages/grid/helm/examples/dev/base.yaml
+++ b/packages/grid/helm/examples/dev/base.yaml
@@ -26,6 +26,13 @@ mongo:
secret:
rootPassword: example
+postgres:
+ resourcesPreset: null
+ resources: null
+
+ secret:
+ rootPassword: example
+
seaweedfs:
resourcesPreset: null
resources: null
diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml
index 3dcefcd0f6..693bb6820b 100644
--- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml
+++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml
@@ -99,18 +99,20 @@ spec:
- name: REVERSE_TUNNEL_ENABLED
value: "true"
{{- end }}
- # MongoDB
- - name: MONGO_PORT
- value: {{ .Values.mongo.port | quote }}
- - name: MONGO_HOST
- value: "mongo"
- - name: MONGO_USERNAME
- value: {{ .Values.mongo.username | quote }}
- - name: MONGO_PASSWORD
+ # Postgres
+ - name: POSTGRESQL_PORT
+ value: {{ .Values.postgres.port | quote }}
+ - name: POSTGRESQL_HOST
+ value: "postgres"
+ - name: POSTGRESQL_USERNAME
+ value: {{ .Values.postgres.username | quote }}
+ - name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
- name: {{ .Values.mongo.secretKeyName | required "mongo.secretKeyName is required" }}
+ name: {{ .Values.postgres.secretKeyName | required "postgres.secretKeyName is required" }}
key: rootPassword
+ - name: POSTGRESQL_DBNAME
+ value: {{ .Values.postgres.dbname | quote }}
# SMTP
- name: SMTP_HOST
value: {{ .Values.server.smtp.host | quote }}
diff --git a/packages/grid/helm/syft/templates/postgres/postgres-headless-service.yaml b/packages/grid/helm/syft/templates/postgres/postgres-headless-service.yaml
new file mode 100644
index 0000000000..b367e13722
--- /dev/null
+++ b/packages/grid/helm/syft/templates/postgres/postgres-headless-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: postgres-headless
+ labels:
+ {{- include "common.labels" . | nindent 4 }}
+ app.kubernetes.io/component: postgres
+spec:
+ clusterIP: None
+ ports:
+ - name: postgres
+ port: 5432
+ selector:
+ {{- include "common.selectorLabels" . | nindent 4 }}
+ app.kubernetes.io/component: postgres
diff --git a/packages/grid/helm/syft/templates/postgres/postgres-secret.yaml b/packages/grid/helm/syft/templates/postgres/postgres-secret.yaml
new file mode 100644
index 0000000000..9f1ad66cfa
--- /dev/null
+++ b/packages/grid/helm/syft/templates/postgres/postgres-secret.yaml
@@ -0,0 +1,17 @@
+{{- $secretName := "postgres-secret" }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ $secretName }}
+ labels:
+ {{- include "common.labels" . | nindent 4 }}
+ app.kubernetes.io/component: postgres
+type: Opaque
+data:
+ rootPassword: {{ include "common.secrets.set" (dict
+ "secret" $secretName
+ "key" "rootPassword"
+ "randomDefault" .Values.global.randomizedSecrets
+ "default" .Values.postgres.secret.rootPassword
+ "context" $)
+ }}
diff --git a/packages/grid/helm/syft/templates/postgres/postgres-service.yaml b/packages/grid/helm/syft/templates/postgres/postgres-service.yaml
new file mode 100644
index 0000000000..a4c5aa8778
--- /dev/null
+++ b/packages/grid/helm/syft/templates/postgres/postgres-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: postgres
+ labels:
+ {{- include "common.labels" . | nindent 4 }}
+ app.kubernetes.io/component: postgres
+spec:
+ type: ClusterIP
+ selector:
+ {{- include "common.selectorLabels" . | nindent 4 }}
+ app.kubernetes.io/component: postgres
+ ports:
+ - name: postgres
+ port: 5432
+ protocol: TCP
+ targetPort: 5432
diff --git a/packages/grid/helm/syft/templates/postgres/postgres-statefuleset.yaml b/packages/grid/helm/syft/templates/postgres/postgres-statefuleset.yaml
new file mode 100644
index 0000000000..425f9e8877
--- /dev/null
+++ b/packages/grid/helm/syft/templates/postgres/postgres-statefuleset.yaml
@@ -0,0 +1,72 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: postgres
+ labels:
+ {{- include "common.labels" . | nindent 4 }}
+ app.kubernetes.io/component: postgres
+spec:
+ replicas: 1
+ updateStrategy:
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ {{- include "common.selectorLabels" . | nindent 6 }}
+ app.kubernetes.io/component: postgres
+ serviceName: postgres-headless
+ podManagementPolicy: OrderedReady
+ template:
+ metadata:
+ labels:
+ {{- include "common.labels" . | nindent 8 }}
+ app.kubernetes.io/component: postgres
+ {{- if .Values.postgres.podLabels }}
+ {{- toYaml .Values.postgres.podLabels | nindent 8 }}
+ {{- end }}
+ {{- if .Values.postgres.podAnnotations }}
+ annotations: {{- toYaml .Values.postgres.podAnnotations | nindent 8 }}
+ {{- end }}
+ spec:
+ {{- if .Values.postgres.nodeSelector }}
+ nodeSelector: {{- .Values.postgres.nodeSelector | toYaml | nindent 8 }}
+ {{- end }}
+ containers:
+ - name: postgres-container
+ image: postgres:13
+ imagePullPolicy: Always
+ resources: {{ include "common.resources.set" (dict "resources" .Values.postgres.resources "preset" .Values.postgres.resourcesPreset) | nindent 12 }}
+ env:
+ - name: POSTGRES_USER
+ value: {{ .Values.postgres.username | required "postgres.username is required" | quote }}
+ - name: POSTGRES_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.postgres.secretKeyName | required "postgres.secretKeyName is required" }}
+ key: rootPassword
+ - name: POSTGRES_DB
+ value: {{ .Values.postgres.dbname | required "postgres.dbname is required" | quote }}
+ {{- if .Values.postgres.env }}
+ {{- toYaml .Values.postgres.env | nindent 12 }}
+ {{- end }}
+ volumeMounts:
+ - mountPath: /data/db
+ name: postgres-data
+ readOnly: false
+ subPath: ''
+ ports:
+ - name: postgres-port
+ containerPort: 5432
+ terminationGracePeriodSeconds: 5
+ volumeClaimTemplates:
+ - metadata:
+ name: postgres-data
+ labels:
+ {{- include "common.volumeLabels" . | nindent 8 }}
+ app.kubernetes.io/component: postgres
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.postgres.storageSize | quote }}
+
diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml
index d43c721203..7aea4e09f0 100644
--- a/packages/grid/helm/syft/values.yaml
+++ b/packages/grid/helm/syft/values.yaml
@@ -44,6 +44,39 @@ mongo:
# =================================================================================
+postgres:
+ # Postgres config
+ port: 5432
+ username: syft_postgres
+ dbname: syftdb_postgres
+ host: postgres
+
+ # Extra environment vars
+ env: null
+
+ # Pod labels & annotations
+ podLabels: null
+ podAnnotations: null
+
+ # Node selector for pods
+ nodeSelector: null
+
+ # Pod Resource Limits
+ resourcesPreset: large
+ resources: null
+
+ # PVC storage size
+ storageSize: 5Gi
+
+ # Mongo secret name. Override this if you want to use a self-managed secret.
+ secretKeyName: postgres-secret
+
+ # default/custom secret raw values
+ secret:
+ rootPassword: null
+
+# =================================================================================
+
frontend:
# Extra environment vars
env: null
diff --git a/packages/syft/src/syft/store/postgresql_document_store.py b/packages/syft/src/syft/store/postgresql_document_store.py
index f8de84876b..c0750dfded 100644
--- a/packages/syft/src/syft/store/postgresql_document_store.py
+++ b/packages/syft/src/syft/store/postgresql_document_store.py
@@ -48,6 +48,12 @@ class PostgreSQLStoreClientConfig(StoreClientConfig):
class Config:
frozen = True
+ def __hash__(self) -> int:
+ return hash((self.dbname, self.username, self.password, self.host, self.port))
+
+ def __str__(self) -> str:
+ return f"dbname={self.dbname} user={self.username} password={self.password} host={self.host} port={self.port}"
+
@serializable(canonical_name="PostgreSQLStorePartition", version=1)
class PostgreSQLStorePartition(SQLiteStorePartition):
@@ -95,6 +101,9 @@ class PostgreSQLBackingStore(SQLiteBackingStore):
port=self.store_config.client_config.port,
)
+ print(f"Connected to {self.store_config.client_config.dbname}")
+ print("PostgreSQL database connection:", connection._check_connection_ok())
+
_CONNECTION_POOL_DB[cache_key(self.dbname)] = connection
def create_table(self) -> None:
diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py
index c2a329d0ca..36903c2122 100644
--- a/packages/syft/src/syft/store/sqlite_document_store.py
+++ b/packages/syft/src/syft/store/sqlite_document_store.py
@@ -248,7 +248,9 @@ class SQLiteBackingStore(KeyValueBackingStore):
return _deserialize(data, from_bytes=True)
def _exists(self, key: UID) -> bool:
- select_sql = f"select uid from {self.table_name} where uid = {self.subs_char}" # nosec
+ select_sql = (
+ f"select uid from {self.table_name} where uid = {self.subs_char}" # nosec
+ )
res = self._execute(select_sql, [str(key)])
if res.is_err():
return False
@@ -294,7 +296,9 @@ class SQLiteBackingStore(KeyValueBackingStore):
return keys
def _delete(self, key: UID) -> None:
- select_sql = f"delete from {self.table_name} where uid = {self.subs_char}" # nosec
+ select_sql = (
+ f"delete from {self.table_name} where uid = {self.subs_char}" # nosec
+ )
self._execute(select_sql, [str(key)]).unwrap()
def _delete_all(self) -> None:
