pyrefly Tests failing in Nix builds

Hi, I'm one of the maintainers for the Pyrefly package on Nixpkgs, and I've been having trouble packaging the latest Pyrefly in Nix. There are 2 tests failing during the build — one on macOS and one on Linux.

macOS failing test

https://github.com/facebook/pyrefly/blob/231fc144694e913dfde1cdb6e95946f111b5f7d6/pyrefly/lib/test/lsp/lsp_interaction/notebook_sync.rs#L14-L44

A similar issue was there in #1630 but this one feels a bit more complicated than that.

Error Logs

test test::lsp::lsp_interaction::notebook_sync::test_notebook_publish_diagnostics ... FAILED(B

failures:

---- test::lsp::lsp_interaction::notebook_sync::test_notebook_publish_diagnostics stdout ----
client--->server {"id":1,"method":"initialize","params":{"capabilities":{"textDocument":{"publishDiagnostics":{"codeDescriptionSupport":true,"dataSupport":true,"relatedInformation":true,"tagSupport":{"valueSet":[1,2]},"versionSupport":false}},"workspace":{"configuration":true}},"clientInfo":{"name":"debug"},"processId":27532,"rootPath":"/","rootUri":null,"trace":"verbose"}}
client<---server {"id":1,"result":{"capabilities":{"codeActionProvider":{"codeActionKinds":["quickfix"]},"completionProvider":{"triggerCharacters":["."]},"declarationProvider":true,"definitionProvider":true,"documentHighlightProvider":true,"documentSymbolProvider":true,"foldingRangeProvider":true,"hoverProvider":true,"implementationProvider":true,"inlayHintProvider":true,"notebookDocumentSync":{"notebookSelector":[{"cells":[{"language":"python"}]}]},"positionEncoding":"utf-16","signatureHelpProvider":{"triggerCharacters":["(",","]},"textDocumentSync":2,"typeDefinitionProvider":true,"workspace":{"fileOperations":{"willRename":{"filters":[{"pattern":{"glob":"**/*.{py,pyi}","matches":"file"},"scheme":"file"}]}},"workspaceFolders":{"changeNotifications":true,"supported":true}},"workspaceSymbolProvider":true},"serverInfo":{"name":"pyrefly-lsp","version":"pyrefly-lsp-test-version"}}}
client--->server {"method":"initialized","params":{}}
 INFO Reading messages
DEBUG Running with 3 threads (5 MiB stack size)
client<---server {"id":1,"method":"workspace/configuration","params":{"items":[{"section":"python"}]}}
client--->server {"id":1,"result":[{"pyrefly":{"displayTypeErrors":"force-on"}}]}
 INFO Client configuration applied to workspace: None
DEBUG Enqueued task on recheck_queue heavy task queue
 INFO Language server processed event `LspResponse` in 0.00s (0.00s waiting)
DEBUG Dequeued task on recheck_queue heavy task queue
DEBUG Running epoch 1 of run 0
client--->server {"method":"notebookDocument/didOpen","params":{"cellTextDocuments":[{"languageId":"python","text":"z: str = ''\nz = 1","uri":"vscode-notebook-cell:///private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb#cell1","version":1}],"notebookDocument":{"cells":[{"document":"vscode-notebook-cell:///private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb#cell1","kind":2}],"metadata":{"language_info":{"name":"python"}},"notebookType":"jupyter-notebook","uri":"file:///private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb","version":1}}}
DEBUG Running epoch 1 of run 1
DEBUG Committing transaction
 INFO Invalidated config, prepare to recheck open files.
 INFO Ran task on recheck_queue heavy task queue. Queue time: 0.00, task time: 0.00
DEBUG Running epoch 1 of run 2
DEBUG Committing transaction
 INFO Validated open files and committed transaction.
DEBUG Enqueued task on sourcedb_queue heavy task queue
 INFO Language server processed event `RecheckFinished` in 0.00s (0.00s waiting)
DEBUG Dequeued task on sourcedb_queue heavy task queue
 INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
 INFO File /private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb opened, prepare to validate open files.
 WARN While finding Python interpreter: Python environment (version, platform, or site-package-path) has value unset, but no Python interpreter could be found to query for values. Falling back to Pyrefly defaults for missing values.
DEBUG Running epoch 1 of run 3
 WARN While finding Python interpreter: Python environment (version, platform, or site-package-path) has value unset, but no Python interpreter could be found to query for values. Falling back to Pyrefly defaults for missing values.
 INFO Validated open files and saved non-committable transaction.
DEBUG Enqueued task on sourcedb_queue heavy task queue
DEBUG Dequeued task on sourcedb_queue heavy task queue
 INFO Language server processed event `DidOpenNotebookDocument` in 0.19s (0.00s waiting)
client<---server {"method":"textDocument/publishDiagnostics","params":{"diagnostics":[{"code":"bad-assignment","codeDescription":{"href":"https://pyrefly.org/en/docs/error-kinds/#bad-assignment"},"message":"`Literal[1]` is not assignable to variable `z` with type `str`","range":{"end":{"character":5,"line":1},"start":{"character":4,"line":1}},"severity":1,"source":"Pyrefly"}],"uri":"vscode-notebook-cell:///private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb#cell1"}}
 INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00

thread 'test::lsp::lsp_interaction::notebook_sync::test_notebook_publish_diagnostics' panicked at pyrefly/lib/test/lsp/lsp_interaction/object_model.rs:683:21:
Timeout waiting for message: publishDiagnostics notification 1 errors for uri: vscode-notebook-cell:///private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb#cell1
 INFO waiting for connection to close


failures:
    test::lsp::lsp_interaction::notebook_sync::test_notebook_publish_diagnostics

test result: FAILED(B. 2842 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 81.29s

error: test failed, to rerun pass `--lib`
error: builder for '/nix/store/h2xc09m6cn13yx9ljfccf1hgrapr1fds-pyrefly-0.42.3.drv' failed with exit code 101;
       last 25 log lines:
       >  INFO Language server processed event `RecheckFinished` in 0.00s (0.00s waiting)
       > DEBUG Dequeued task on sourcedb_queue heavy task queue
       >  INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
       >  INFO File /private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb opened, prepare to validate open files.
       >  WARN While finding Python interpreter: Python environment (version, platform, or site-package-path) has value unset, but no Python interpreter could be found to query for values. Falling back to Pyrefly defaults for missing values.
       > DEBUG Running epoch 1 of run 3
       >  WARN While finding Python interpreter: Python environment (version, platform, or site-package-path) has value unset, but no Python interpreter could be found to query for values. Falling back to Pyrefly defaults for missing values.
       >  INFO Validated open files and saved non-committable transaction.
       > DEBUG Enqueued task on sourcedb_queue heavy task queue
       > DEBUG Dequeued task on sourcedb_queue heavy task queue
       >  INFO Language server processed event `DidOpenNotebookDocument` in 0.19s (0.00s waiting)
       > client<---server {"method":"textDocument/publishDiagnostics","params":{"diagnostics":[{"code":"bad-assignment","codeDescription":{"href":"https://pyrefly.org/en/docs/error-kinds/#bad-assignment"},"message":"`Literal[1]` is not assignable to variable `z` with type `str`","range":{"end":{"character":5,"line":1},"start":{"character":4,"line":1}},"severity":1,"source":"Pyrefly"}],"uri":"vscode-notebook-cell:///private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb#cell1"}}
       >  INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
       >
       > thread 'test::lsp::lsp_interaction::notebook_sync::test_notebook_publish_diagnostics' panicked at pyrefly/lib/test/lsp/lsp_interaction/object_model.rs:683:21:
       > Timeout waiting for message: publishDiagnostics notification 1 errors for uri: vscode-notebook-cell:///private/tmp/nix-build-pyrefly-0.42.3.drv-0/tmp.P1EIf4kzub/pyrefly_lsp_testjn9Xpe/notebook.ipynb#cell1
       >  INFO waiting for connection to close
       >
       >
       > failures:
       >     test::lsp::lsp_interaction::notebook_sync::test_notebook_publish_diagnostics
       >
       > test result: FAILED(B. 2842 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 81.29s
       >
       > error: test failed, to rerun pass `--lib`

Linux failing tests

https://github.com/facebook/pyrefly/blob/231fc144694e913dfde1cdb6e95946f111b5f7d6/pyrefly/lib/test/lsp/lsp_interaction/configuration.rs#L79-L249

PR where it's being discussed: https://github.com/NixOS/nixpkgs/pull/458544

Error Logs

test test::lsp::lsp_interaction::configuration::test_workspace_pythonpath_ignored_when_set_in_config_file ... FAILED
test test::lsp::lsp_interaction::configuration::test_pythonpath_change ... FAILED
failures:
---- test::lsp::lsp_interaction::configuration::test_workspace_pythonpath_ignored_when_set_in_config_file stdout ----
client--->server {"id":1,"method":"initialize","params":{"capabilities":{"textDocument":{"publishDiagnostics":{"codeDescriptionSupport":true,"dataSupport":true,"relatedInformation":true,"tagSupport":{"valueSet":[1,2]},"versionSupport":false}},"workspace":{"configuration":true}},"clientInfo":{"name":"debug"},"processId":5346,"rootPath":"/","rootUri":null,"trace":"verbose"}}
client<---server {"id":1,"result":{"capabilities":{"codeActionProvider":{"codeActionKinds":["quickfix"]},"completionProvider":{"triggerCharacters":["."]},"declarationProvider":true,"definitionProvider":true,"documentHighlightProvider":true,"documentSymbolProvider":true,"foldingRangeProvider":true,"hoverProvider":true,"implementationProvider":true,"inlayHintProvider":true,"notebookDocumentSync":{"notebookSelector":[{"cells":[{"language":"python"}]}]},"positionEncoding":"utf-16","signatureHelpProvider":{"triggerCharacters":["(",","]},"textDocumentSync":2,"typeDefinitionProvider":true,"workspace":{"fileOperations":{"willRename":{"filters":[{"pattern":{"glob":"**/*.{py,pyi}","matches":"file"},"scheme":"file"}]}},"workspaceFolders":{"changeNotifications":true,"supported":true}},"workspaceSymbolProvider":true},"serverInfo":{"name":"pyrefly-lsp","version":"pyrefly-lsp-test-version"}}}
client--->server {"method":"initialized","params":{}}
 INFO Reading messages
DEBUG Running with 3 threads (5 MiB stack size)
client<---server {"id":1,"method":"workspace/configuration","params":{"items":[{"section":"python"}]}}
client--->server {"id":1,"result":[{"pyrefly":{"displayTypeErrors":"force-on"}}]}
client--->server {"method":"textDocument/didOpen","params":{"textDocument":{"languageId":"python","text":"# Copyright (c) Meta Platforms, Inc. and affiliates.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom custom_module import CustomClass\n\nprint(CustomClass.custom_attr)\n","uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/foo.py","version":1}}}
 INFO Client configuration applied to workspace: None
DEBUG Enqueued task on recheck_queue heavy task queue
 INFO Language server processed event `LspResponse` in 0.00s (0.00s waiting)
 INFO File /build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/foo.py opened, prepare to validate open files.
DEBUG Dequeued task on recheck_queue heavy task queue
 WARN /build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/pyrefly.toml: Failed to query interpreter at /build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/bin/python, falling back to default Python environment settings
No such file or directory (os error 2)
DEBUG Running epoch 1 of run 0
 WARN /build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/pyrefly.toml: Failed to query interpreter at /build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/bin/python, falling back to default Python environment settings
No such file or directory (os error 2)
DEBUG Running epoch 1 of run 1
 INFO Validated open files and saved non-committable transaction.
DEBUG Enqueued task on sourcedb_queue heavy task queue
 INFO Language server processed event `DidOpenTextDocument` in 0.23s (0.00s waiting)
client<---server {"method":"textDocument/publishDiagnostics","params":{"diagnostics":[{"code":"missing-import","codeDescription":{"href":"https://pyrefly.org/en/docs/error-kinds/#missing-import"},"message":"Could not find import of `custom_module`\n  Looked in these locations (from config in `/build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/pyrefly.toml`):\n  Import root (inferred from project layout): \"/build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src\"","range":{"end":{"character":37,"line":5},"start":{"character":0,"line":5}},"severity":1,"source":"Pyrefly"}],"uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/foo.py"}}
DEBUG Dequeued task on sourcedb_queue heavy task queue
 INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
DEBUG Running epoch 1 of run 2
DEBUG Committing transaction
 INFO Invalidated config, prepare to recheck open files.
 INFO Ran task on recheck_queue heavy task queue. Queue time: 0.00, task time: 0.76
DEBUG Running epoch 1 of run 3
DEBUG Committing transaction
 INFO Validated open files and committed transaction.
DEBUG Enqueued task on sourcedb_queue heavy task queue
 INFO Language server processed event `RecheckFinished` in 0.01s (0.00s waiting)
client<---server {"method":"textDocument/publishDiagnostics","params":{"diagnostics":[{"code":"missing-import","codeDescription":{"href":"https://pyrefly.org/en/docs/error-kinds/#missing-import"},"message":"Could not find import of `custom_module`\n  Looked in these locations (from config in `/build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/pyrefly.toml`):\n  Import root (inferred from project layout): \"/build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src\"","range":{"end":{"character":37,"line":5},"start":{"character":0,"line":5}},"severity":1,"source":"Pyrefly"}],"uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/foo.py"}}
DEBUG Dequeued task on sourcedb_queue heavy task queue
 INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
thread 'test::lsp::lsp_interaction::configuration::test_workspace_pythonpath_ignored_when_set_in_config_file' panicked at pyrefly/lib/test/lsp/lsp_interaction/object_model.rs:610:21:
Timeout waiting for message: publishDiagnostics notification with 0 errors for file: /build/tmp.YPQKw8hFbv/pyrefly_lsp_testHNk0eC/custom_interpreter_config/src/foo.py
 INFO waiting for connection to close
---- test::lsp::lsp_interaction::configuration::test_pythonpath_change stdout ----
client--->server {"id":1,"method":"initialize","params":{"capabilities":{"textDocument":{"publishDiagnostics":{"codeDescriptionSupport":true,"dataSupport":true,"relatedInformation":true,"tagSupport":{"valueSet":[1,2]},"versionSupport":false}},"workspace":{"configuration":true}},"clientInfo":{"name":"debug"},"processId":5346,"rootPath":"/","rootUri":null,"trace":"verbose"}}
client<---server {"id":1,"result":{"capabilities":{"codeActionProvider":{"codeActionKinds":["quickfix"]},"completionProvider":{"triggerCharacters":["."]},"declarationProvider":true,"definitionProvider":true,"documentHighlightProvider":true,"documentSymbolProvider":true,"foldingRangeProvider":true,"hoverProvider":true,"implementationProvider":true,"inlayHintProvider":true,"notebookDocumentSync":{"notebookSelector":[{"cells":[{"language":"python"}]}]},"positionEncoding":"utf-16","signatureHelpProvider":{"triggerCharacters":["(",","]},"textDocumentSync":2,"typeDefinitionProvider":true,"workspace":{"fileOperations":{"willRename":{"filters":[{"pattern":{"glob":"**/*.{py,pyi}","matches":"file"},"scheme":"file"}]}},"workspaceFolders":{"changeNotifications":true,"supported":true}},"workspaceSymbolProvider":true},"serverInfo":{"name":"pyrefly-lsp","version":"pyrefly-lsp-test-version"}}}
client--->server {"method":"initialized","params":{}}
 INFO Reading messages
DEBUG Running with 3 threads (5 MiB stack size)
client<---server {"id":1,"method":"workspace/configuration","params":{"items":[{"section":"python"}]}}
client--->server {"id":1,"result":[{"pyrefly":{"displayTypeErrors":"force-on"}}]}
client--->server {"method":"textDocument/didOpen","params":{"textDocument":{"languageId":"python","text":"# Copyright (c) Meta Platforms, Inc. and affiliates.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom custom_module import CustomClass\n\nprint(CustomClass.custom_attr)\n","uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py","version":1}}}
 INFO Client configuration applied to workspace: None
DEBUG Enqueued task on recheck_queue heavy task queue
 INFO Language server processed event `LspResponse` in 0.00s (0.00s waiting)
 INFO File /build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py opened, prepare to validate open files.
 WARN While finding Python interpreter: Python environment (version, platform, or site-package-path) has value unset, but no Python interpreter could be found to query for values. Falling back to Pyrefly defaults for missing values.
DEBUG Running epoch 1 of run 0
DEBUG Dequeued task on recheck_queue heavy task queue
 WARN While finding Python interpreter: Python environment (version, platform, or site-package-path) has value unset, but no Python interpreter could be found to query for values. Falling back to Pyrefly defaults for missing values.
DEBUG Running epoch 1 of run 1
 WARN While finding Python interpreter: Python environment (version, platform, or site-package-path) has value unset, but no Python interpreter could be found to query for values. Falling back to Pyrefly defaults for missing values.
 INFO Validated open files and saved non-committable transaction.
DEBUG Enqueued task on sourcedb_queue heavy task queue
 INFO Language server processed event `DidOpenTextDocument` in 0.67s (0.00s waiting)
DEBUG Dequeued task on sourcedb_queue heavy task queue
 INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
DEBUG Running epoch 1 of run 2
DEBUG Committing transaction
 INFO Invalidated config, prepare to recheck open files.
 INFO Ran task on recheck_queue heavy task queue. Queue time: 0.00, task time: 0.67
client<---server {"method":"textDocument/publishDiagnostics","params":{"diagnostics":[{"code":"missing-import","codeDescription":{"href":"https://pyrefly.org/en/docs/error-kinds/#missing-import"},"message":"Could not find import of `custom_module`\n  Looked in these locations:\n  Fallback search path (guessed from importing file with heuristics): [\"/build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src\"]","range":{"end":{"character":37,"line":5},"start":{"character":0,"line":5}},"severity":1,"source":"Pyrefly"}],"uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py"}}
client--->server {"id":2,"method":"textDocument/definition","params":{"position":{"character":31,"line":5},"textDocument":{"uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py"}}}
DEBUG Running epoch 1 of run 3
DEBUG Committing transaction
 INFO Validated open files and committed transaction.
DEBUG Enqueued task on sourcedb_queue heavy task queue
 INFO Language server processed event `RecheckFinished` in 0.03s (0.00s waiting)
 INFO Handling non-canceled request textDocument/definition (2)
DEBUG Dequeued task on sourcedb_queue heavy task queue
 INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
client<---server {"method":"textDocument/publishDiagnostics","params":{"diagnostics":[{"code":"missing-import","codeDescription":{"href":"https://pyrefly.org/en/docs/error-kinds/#missing-import"},"message":"Could not find import of `custom_module`\n  Looked in these locations:\n  Fallback search path (guessed from importing file with heuristics): [\"/build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src\"]","range":{"end":{"character":37,"line":5},"start":{"character":0,"line":5}},"severity":1,"source":"Pyrefly"}],"uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py"}}
client<---server {"id":2,"result":{"range":{"end":{"character":37,"line":5},"start":{"character":26,"line":5}},"uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py"}}
 INFO Language server processed event `LspRequest(textDocument/definition)` in 0.00s (0.03s waiting)
client--->server {"method":"workspace/didChangeConfiguration","params":{"settings":{}}}
 INFO Language server processed event `DidChangeConfiguration` in 0.00s (0.00s waiting)
client<---server {"id":2,"method":"workspace/configuration","params":{"items":[{"section":"python"}]}}
client--->server {"id":2,"result":[{"pythonPath":"/build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/bin/python"}]}
ERROR Failed to query interpreter at /build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/bin/python, falling back to default Python environment settings
No such file or directory (os error 2)
 INFO Client configuration applied to workspace: None
DEBUG Enqueued task on recheck_queue heavy task queue
 INFO Language server processed event `LspResponse` in 0.02s (0.00s waiting)
DEBUG Dequeued task on recheck_queue heavy task queue
DEBUG Running epoch 1 of run 4
DEBUG Running epoch 1 of run 5
DEBUG Committing transaction
 INFO Invalidated config, prepare to recheck open files.
 INFO Ran task on recheck_queue heavy task queue. Queue time: 0.00, task time: 0.00
DEBUG Running epoch 1 of run 6
DEBUG Committing transaction
 INFO Validated open files and committed transaction.
DEBUG Enqueued task on sourcedb_queue heavy task queue
 INFO Language server processed event `RecheckFinished` in 0.00s (0.00s waiting)
DEBUG Dequeued task on sourcedb_queue heavy task queue
 INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
client<---server {"method":"textDocument/publishDiagnostics","params":{"diagnostics":[{"code":"missing-import","codeDescription":{"href":"https://pyrefly.org/en/docs/error-kinds/#missing-import"},"message":"Could not find import of `custom_module`\n  Looked in these locations:\n  Fallback search path (guessed from importing file with heuristics): [\"/build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src\"]","range":{"end":{"character":37,"line":5},"start":{"character":0,"line":5}},"severity":1,"source":"Pyrefly"}],"uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py"}}
thread 'test::lsp::lsp_interaction::configuration::test_pythonpath_change' panicked at pyrefly/lib/test/lsp/lsp_interaction/object_model.rs:610:21:
Timeout waiting for message: publishDiagnostics notification with 0 errors for file: /build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py
 INFO waiting for connection to close
failures:
    test::lsp::lsp_interaction::configuration::test_pythonpath_change
    test::lsp::lsp_interaction::configuration::test_workspace_pythonpath_ignored_when_set_in_config_file
test result: FAILED. 2794 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 65.79s
error: test failed, to rerun pass `--lib`
error: builder for '/nix/store/1nfnx402janinxxj640w442wjn38295h-pyrefly-0.42.2.drv' failed with exit code 101;
       last 25 log lines:
       > DEBUG Running epoch 1 of run 5
       > DEBUG Committing transaction
       >  INFO Invalidated config, prepare to recheck open files.
       >  INFO Ran task on recheck_queue heavy task queue. Queue time: 0.00, task time: 0.00
       > DEBUG Running epoch 1 of run 6
       > DEBUG Committing transaction
       >  INFO Validated open files and committed transaction.
       > DEBUG Enqueued task on sourcedb_queue heavy task queue
       >  INFO Language server processed event `RecheckFinished` in 0.00s (0.00s waiting)
       > DEBUG Dequeued task on sourcedb_queue heavy task queue
       >  INFO Ran task on sourcedb_queue heavy task queue. Queue time: 0.00, task time: 0.00
       > client<---server {"method":"textDocument/publishDiagnostics","params":{"diagnostics":[{"code":"missing-import","codeDescription":{"href":"https://pyrefly.org/en/docs/error-kinds/#missing-import"},"message":"Could not find import of `custom_module`\n  Looked in these locations:\n  Fallback search path (guessed from importing file with heuristics): [\"/build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src\"]","range":{"end":{"character":37,"line":5},"start":{"character":0,"line":5}},"severity":1,"source":"Pyrefly"}],"uri":"file:///build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py"}}
       >
       > thread 'test::lsp::lsp_interaction::configuration::test_pythonpath_change' panicked at pyrefly/lib/test/lsp/lsp_interaction/object_model.rs:610:21:
       > Timeout waiting for message: publishDiagnostics notification with 0 errors for file: /build/tmp.YPQKw8hFbv/pyrefly_lsp_testbs2Lb3/custom_interpreter/src/foo.py
       >  INFO waiting for connection to close
       >
       >
       > failures:
       >     test::lsp::lsp_interaction::configuration::test_pythonpath_change
       >     test::lsp::lsp_interaction::configuration::test_workspace_pythonpath_ignored_when_set_in_config_file
       >
       > test result: FAILED. 2794 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 65.79s
       >
       > error: test failed, to rerun pass `--lib`

I would love to help resolve it but I have almost zero Rust experience. 😓 Any help would be greatly appreciated. Thanks.

Alternatively if nothing else works, I can disable the failing tests specifically in the Nix build, but would be nice if we can have all tests work.

Nov 22 '25 18:11 cybardev

cc @yangdanny97 for the notebook failure, I'm pretty sure it's timing out on the duplicate publish diagnostics check because the logs show a publish diagnostics check succeeding before the timeout.

In other words, I think the timeout is happening on the check marked with a comment saying we ought to get rid of it:

    // TODO: Stop sending multiple publishDiagnostics on didOpen
    interaction
        .client
        .expect_publish_diagnostics_uri(&cell_uri, 1);

Do we know why there's a duplicate, and is it possible that something about that is distro-dependent, e.g. if the PATH is very different then it doesn't happen?

I think the other failures might be similar, although it's harder to tell I'm having a very hard time reading the test output and figuring out which line failed

Separately, I'm going to try to put up a diff to rewire the lsp testing types a bit - it's much harder to read the bug report because the line numbers are useless, we're panicking inside testing helper functions. I think it's always an architecture bug to do this when there's a viable alternative where we panic in the actual test - the failures are vastly easier to read if they come with a helpful line number.

Nov 24 '25 16:11 stroxler

@yangdanny97, @kinto0 I think at least some of these may be reproing on trunk, maybe nondeterministically.

I just observed the notebook test failure in a seemingly unrelated PR https://github.com/facebook/pyrefly/actions/runs/19606247892/job/56145548484?pr=1650 which makes me think at least that failure may not be distro-specific

Nov 24 '25 16:11 stroxler

Yes I've noticed non-deterministic results from various tests recently as well.

Nov 24 '25 17:11 yangdanny97

Just an update: originally the notebook sync test was failing on macOS (or so I noticed), but now it's failing on Linux too.

And yeah, it doesn't seem to always fail, it's kinda random...

Nov 26 '25 01:11 cybardev

test_notebook_publish_diagnostics also failed for me on macOS. Let me know if there is anything I can do to help reproduce this.

Nov 26 '25 22:11 DanielNoord

I believe @stroxler is right with linking the flakiness of test_notebook_publish_diagnostics to the dual publishDiagnostics.

I looked into what is happening and I think it is because the start up of the LSP is interacting with the events created by didOpen.

The initialization of the LSP triggers this code: https://github.com/facebook/pyrefly/blob/d555b28cab00389e73a9e18b4d65c35f2a36570a/pyrefly/lib/lsp/non_wasm/server.rs#L776

(I think) because the configuration is "new" it always triggers this: https://github.com/facebook/pyrefly/blob/d555b28cab00389e73a9e18b4d65c35f2a36570a/pyrefly/lib/lsp/non_wasm/server.rs#L3157

However, if the open_notebook is processed quickly enough the didOpen event is processed before the RecheckFinished event. RecheckFinished also triggers publishDiagnostics and will now publish diagnostics for the notebook that was just "added" to the LSP workspace.

If you add a short sleep to open_notebook the publishDiagnotics of RecheckFinished is empty and the test will fail. If you don't the publishDiagnostics of RecheckFinished will trigger the second expect as the notebook is now loaded into the workspace and is included in the diagnostics.

I'm not experienced enough with the codebase, but it seems possible that open_notebook is indeed somewhat slower sometimes and therefore is added to the event queue after RecheckFinished. A fix is also above my paygrade 😅

A short sleep in open_notebook ensures the LSP has time to process RecheckFinished before didOpen and the second expect can be removed, but adding sleeps doesn't seem the best way to fix race conditions in the test suite 😄 Hopefully there is a synchronization primitive in the codebase that can help with this.

Nov 29 '25 16:11 DanielNoord

Thanks for all the debugging steps @DanielNoord! Thanks to your investigation, I think I have a fix ready here.

I also think this could fix the flaky test_pythonpath_change

Dec 02 '25 01:12 kinto0

@cybardev can you confirm whether these tests are still flaky? I haven't seen these tests fail recently since the fix. but I do see some other ones (test_exit_without_shutdown, test_connection_closed_server_stops) fail on windows

Dec 09 '25 16:12 kinto0

@kinto0 No errors on darwin with all tests enabled.

test result: ok(B. 3040 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 36.25s

saw a warning tho:

   Compiling pyrefly_config v0.45.0 (/nix/var/nix/builds/nix-38616-3687158741/source/crates/pyrefly_config)
warning: unexpected `cfg` condition name: `fbcode_build`
    --> crates/pyrefly_config/src/config.rs:1055:17
     |
1055 |         if cfg!(fbcode_build) {
     |                 ^^^^^^^^^^^^
     |
     = help: expected names are: `docsrs`, `feature`, and `test` and 31 more
     = help: consider using a Cargo feature instead
     = help: or consider adding in `Cargo.toml` the `check-cfg` lint config for the lint:
              [lints.rust]
              unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fbcode_build)'] }
     = help: or consider adding `println!("cargo::rustc-check-cfg=cfg(fbcode_build)");` to the top of the `build.rs`
     = note: see <https://doc.rust-lang.org/nightly/rustc/check-cfg/cargo-specifics.html> for more information about checking conditional configuration
     = note: `#[warn(unexpected_cfgs)]` on by default

not a concern (or known issue), i assume?

I created PR https://github.com/NixOS/nixpkgs/pull/469349 cuz I can't check on Linux. awaiting tests to run in CI.

This is the most important one (x86_64-linux):
https://github.com/NixOS/nixpkgs/pull/469349/checks?check_run_id=57577299680

Dec 09 '25 17:12 cybardev

@kinto0 Ok, looks like tests are still failing on NixOS (x86_64-linux):

       > failures:
       >     test::lsp::lsp_interaction::configuration::test_interpreter_change_removes_type_errors
       >     test::lsp::lsp_interaction::configuration::test_pythonpath_change
       >     test::lsp::lsp_interaction::configuration::test_workspace_pythonpath_ignored_when_set_in_config_file
       >
       > test result: FAILED. 3037 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out; finished in 68.58s
       >
       > error: test failed, to rerun pass `--lib`
       For full logs, run:
         nix log /nix/store/53z20hr7s0bnxgl1nw99vr3cwca74pz1-pyrefly-0.45.0.drv

That test_interpreter_change_removes_type_errors looks like a new failure.

Dec 09 '25 19:12 cybardev

Interesting... Seems that it fails on darwin too: https://github.com/cybardev/nixpkgs-review-action/actions/runs/20086697819/job/57625825916#step:6:1

For some reason it works on my machine... I wonder if it's something to do with Nix's sandboxing. I'll do some more tests and see.

Dec 10 '25 07:12 cybardev

Interesting... Seems that it fails on darwin too: https://github.com/cybardev/nixpkgs-review-action/actions/runs/20086697819/job/57625825916#step:6:1

For some reason it works on my machine... I wonder if it's something to do with Nix's sandboxing. I'll do some more tests and see.

hmm.... this log line looks like the culprit:

pyrefly> ERROR Failed to query interpreter at /nix/build/nix-5922-879399443/tmp.AbDqWv025s/pyrefly_lsp_testDWtzlH/interpreter_with_no_site_packages/bin/python, falling back to default Python environment settings
pyrefly> Operation not permitted (os error 1)

does nix have write access in that temp dir? when we set up the dummy interpreter, we set it up with these permissiosn:

#[cfg(unix)]
fn setup_dummy_interpreter(custom_interpreter_path: &Path) -> PathBuf {
    // Create a mock Python interpreter script that returns the environment info
    // This simulates what a real Python interpreter would return when queried with the env script
    let python_script = format!(
        r#"#!/usr/bin/env bash
if [[ "$1" == "-c" && "$2" == *"import json, sys"* ]]; then
    cat << 'EOF'
{{"python_platform": "linux", "python_version": "3.12.0", "site_package_path": ["{site_packages}"]}}
EOF
else
    echo "Mock python interpreter - args: $@" >&2
    exit 1
fi
"#,
        site_packages = custom_interpreter_path
            .join("bin/site-packages")
            .to_str()
            .unwrap()
    );

    let interpreter_path = custom_interpreter_path.join("bin/python");
    write(&interpreter_path, python_script).unwrap();
    let mut perms = fs::metadata(&interpreter_path).unwrap().permissions();
    perms.set_mode(0o755); // rwxr-xr-x
    fs::set_permissions(&interpreter_path, perms).unwrap();

    interpreter_path
}

Dec 10 '25 15:12 kinto0

Aight so apparently /usr is not a thing in the build sandbox. I'll cook up a patch and test it.

Dec 10 '25 16:12 cybardev

Yep, that does it. had to change the /usr/bin/env bash to the path of bash in the Nix build sandbox. Tests pass, builds fine. You can see the changes here: https://github.com/NixOS/nixpkgs/pull/469349/files

Thanks all for the patience and support. I think this can be closed now.

Dec 10 '25 20:12 cybardev