diff --git a/.editorconfig b/.editorconfig index 1a05f3eb4ade..05b730f0e572 100644 --- a/.editorconfig +++ b/.editorconfig @@ -75,7 +75,7 @@ dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent # Modifier preferences dotnet_style_require_accessibility_modifiers = for_non_interface_members:error -dotnet_style_readonly_field = true:suggestion +dotnet_style_readonly_field = true:warning # Expression-level preferences dotnet_style_object_initializer = true:suggestion dotnet_style_collection_initializer = true:suggestion @@ -96,10 +96,6 @@ dotnet_style_prefer_compound_assignment = true:suggestion dotnet_code_quality_unused_parameters = all:suggestion [*.cs] - -# TODO: enable this but stop "dotnet format" from applying incorrect fixes and introducing a lot of unwanted changes. -dotnet_analyzer_diagnostic.severity = none - # Note: these settings cause "dotnet format" to fix the code. You should review each change if you uses "dotnet format". dotnet_diagnostic.RCS1036.severity = warning # Remove unnecessary blank line. dotnet_diagnostic.RCS1037.severity = warning # Remove trailing white-space. @@ -163,7 +159,7 @@ dotnet_diagnostic.CA1032.severity = none # We're using RCS1194 which seems to co dotnet_diagnostic.CA1034.severity = none # Do not nest type. Alternatively, change its accessibility so that it is not externally visible dotnet_diagnostic.CA1062.severity = none # Disable null check, C# already does it for us dotnet_diagnostic.CA1303.severity = none # Do not pass literals as localized parameters -dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.CA1510.severity = none dotnet_diagnostic.CA1805.severity = none # Member is explicitly initialized to its default value dotnet_diagnostic.CA1822.severity = none # Member does not access instance data and can be marked as static dotnet_diagnostic.CA1848.severity = none # For improved performance, use the LoggerMessage delegates @@ -173,6 +169,7 @@ dotnet_diagnostic.CA2227.severity = none # Change to be read-only by removing th dotnet_diagnostic.CA2253.severity = none # Named placeholders in the logging message template should not be comprised of only numeric characters dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.VSTHRD200.severity = none # Use Async suffix for async methods dotnet_diagnostic.xUnit1004.severity = none # Test methods should not be skipped. Remove the Skip property to start running the test again. dotnet_diagnostic.RCS1021.severity = none # Use expression-bodied lambda. @@ -219,11 +216,13 @@ dotnet_diagnostic.IDE0052.severity = none # Remove unread private member dotnet_diagnostic.IDE0058.severity = none # Remove unused expression value dotnet_diagnostic.IDE0059.severity = none # Unnecessary assignment of a value dotnet_diagnostic.IDE0060.severity = none # Remove unused parameter -dotnet_diagnostic.IDE0080.severity = none # Remove unnecessary suppression operator +dotnet_diagnostic.IDE0079.severity = none # Remove unnecessary suppression. +dotnet_diagnostic.IDE0080.severity = none # Remove unnecessary suppression operator. dotnet_diagnostic.IDE0100.severity = none # Remove unnecessary equality operator dotnet_diagnostic.IDE0110.severity = none # Remove unnecessary discards dotnet_diagnostic.IDE0032.severity = none # Use auto property dotnet_diagnostic.IDE0160.severity = none # Use block-scoped namespace +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations ############################### # Naming Conventions # diff --git a/.gitattributes b/.gitattributes index 367c088539db..b5845d1480b2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,4 +2,5 @@ * text=auto eol=lf working-tree-encoding=UTF-8 # Bash scripts -*.sh text eol=lf +*.sh text eol=lf +*.cmd text eol=crlf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9cd1e22f1da1..ed1161835648 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,8 +12,3 @@ # directory at the root of the repository and any of its # subdirectories. /java/ @microsoft/octo-semantickernel-pr-java - -# @microsoft/octo-semantickernel-pr-apps owns any files in the samples -# directory at the root of the repository and any of its -# subdirectories. -/samples/ @microsoft/octo-semantickernel-pr-apps diff --git a/.github/_typos.toml b/.github/_typos.toml index 8298df765e3c..08d93b234795 100644 --- a/.github/_typos.toml +++ b/.github/_typos.toml @@ -12,7 +12,6 @@ extend-exclude = [ "*.bicep", "encoder.json", "vocab.bpe", - "GPT3TokenizerTests.cs", "CodeTokenizerTests.cs", "test_code_tokenizer.py", ] diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9b83abf2739e..97b4f6647f05 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,16 +5,6 @@ version: 2 updates: - # Maintain dependencies for nuget - - package-ecosystem: "nuget" - directory: "samples/apps/copilot-chat-app/webapi" - schedule: - interval: "weekly" - day: "monday" - labels: - - "copilot chat" - - "dependencies" - # Maintain dependencies for nuget - package-ecosystem: "nuget" directory: "dotnet/" @@ -28,11 +18,12 @@ updates: - dependency-name: "Microsoft.Extensions.*" update-types: ["version-update:semver-major"] - dependency-name: "Microsoft.Bcl.*" - update-types: ["version-update:semver-major"] + update-types: ["version-update:semver-major"] + - dependency-name: "Moq" labels: - ".NET" - "dependencies" - + # Maintain dependencies for nuget - package-ecosystem: "nuget" directory: "samples/dotnet" @@ -46,7 +37,7 @@ updates: schedule: interval: "weekly" day: "monday" - + # Maintain dependencies for pip - package-ecosystem: "pip" directory: "python/" @@ -56,7 +47,7 @@ updates: labels: - "python" - "dependencies" - + # Maintain dependencies for github-actions - package-ecosystem: "github-actions" # Workflow files stored in the diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6ae3a7a79962..a3f6d583cda5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/copilot-chat-package.yml b/.github/workflows/copilot-chat-package.yml deleted file mode 100644 index 16301a5c3aaa..000000000000 --- a/.github/workflows/copilot-chat-package.yml +++ /dev/null @@ -1,59 +0,0 @@ -# -# This workflow will package the Copilot Chat application for deployment. -# - -name: copilot-chat-package - -on: - pull_request: - branches: [ "main", "feature*" ] - paths: - - 'samples/apps/copilot-chat-app/**' - push: - branches: [ "main", "feature*" ] - paths: - - 'samples/apps/copilot-chat-app/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - build: - strategy: - fail-fast: false - matrix: - include: - - { dotnet: '6.0', configuration: Release, os: ubuntu-latest } - - runs-on: ${{ matrix.os }} - env: - NUGET_CERT_REVOCATION_MODE: offline - steps: - - uses: actions/checkout@v3 - with: - clean: true - - - name: Pull container dotnet/sdk:${{ matrix.dotnet }} - run: docker pull mcr.microsoft.com/dotnet/sdk:${{ matrix.dotnet }} - - - name: Package Copilot Chat WebAPI - run: | - chmod +x $(pwd)/samples/apps/copilot-chat-app/deploy/package-webapi.sh; - docker run --rm -v $(pwd):/app -w /app -e GITHUB_ACTIONS='true' mcr.microsoft.com/dotnet/sdk:${{ matrix.dotnet }} /bin/sh -c "/app/samples/apps/copilot-chat-app/deploy/package-webapi.sh --no-zip"; - - - name: Set version tag - id: versiontag - run: | - VERSION_TAG="$(date +'%Y%m%d').${{ github.run_number }}.${{ github.run_attempt }}" - echo $VERSION_TAG - echo "versiontag=$VERSION_TAG" >> $GITHUB_OUTPUT - - - name: Upload package to artifacts - uses: actions/upload-artifact@v3 - with: - name: copilotchat-webapi-${{ steps.versiontag.outputs.versiontag }} - path: ./samples/apps/copilot-chat-app/deploy/publish diff --git a/.github/workflows/copilot-chat-tests.yml b/.github/workflows/copilot-chat-tests.yml deleted file mode 100644 index 0f0d4db293cb..000000000000 --- a/.github/workflows/copilot-chat-tests.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Copilot Chat Tests -on: - workflow_dispatch: - push: - branches: ["main"] - paths: - - "samples/apps/copilot-chat-app/**" - -permissions: - contents: read - -jobs: - test: - defaults: - run: - working-directory: samples/apps/copilot-chat-app/webapp - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-node@v3 - with: - node-version: 16 - cache-dependency-path: samples/apps/copilot-chat-app/webapp/yarn.lock - cache: "yarn" - - - name: Setup .NET - uses: actions/setup-dotnet@v3 - with: - dotnet-version: 6.0.x - - - name: Install dependencies - run: yarn install - - - name: Install Playwright Browsers - run: yarn playwright install --with-deps - - - name: Update AIService configuration - working-directory: samples/apps/copilot-chat-app/webapi - env: - AzureOpenAI__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }} - AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} - run: | - dotnet dev-certs https - dotnet user-secrets set "AIService:Key" "$AzureOpenAI__ApiKey" - dotnet user-secrets set "AIService:Endpoint" "$AzureOpenAI__Endpoint" - - - name: Start service in background - working-directory: samples/apps/copilot-chat-app/webapi - run: | - dotnet run > service-log.txt 2>&1 & - for attempt in {0..20}; do - jobs - echo 'Waiting for service to start...'; - if curl -k https://localhost:40443/healthz; then - echo; - echo 'Service started'; - break; - fi; - - sleep 5; - done - - - name: Run Playwright tests - env: - REACT_APP_BACKEND_URI: https://localhost:40443/ - REACT_APP_AAD_CLIENT_ID: ${{ secrets.COPILOT_CHAT_REACT_APP_AAD_CLIENT_ID }} - REACT_APP_AAD_AUTHORITY: https://login.microsoftonline.com/common - - REACT_APP_TEST_USER_ACCOUNT1: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT1 }} - REACT_APP_TEST_USER_PASSWORD1: ${{ secrets.COPILOT_CHAT_TEST_USER_PASSWORD1 }} - REACT_APP_TEST_USER_ACCOUNT2: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT2 }} - REACT_APP_TEST_USER_PASSWORD2: ${{ secrets.COPILOT_CHAT_TEST_USER_PASSWORD2 }} - - REACT_APP_TEST_JIRA_EMAIL: ${{ secrets.COPILOT_CHAT_TEST_JIRA_EMAIL }} - REACT_APP_TEST_JIRA_ACCESS_TOKEN: ${{ secrets.COPILOT_CHAT_TEST_JIRA_ACCESS_TOKEN }} - REACT_APP_TEST_JIRA_SERVER_URL: ${{ secrets.COPILOT_CHAT_TEST_JIRA_SERVER_URL }} - - REACT_APP_TEST_GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REACT_APP_TEST_GITHUB_ACCOUNT_OWNER: ${{ secrets.COPILOT_CHAT_TEST_GITHUB_ACCOUNT_OWNER }} - REACT_APP_TEST_GITHUB_REPOSITORY_NAME: ${{ secrets.COPILOT_CHAT_TEST_GITHUB_REPOSITORY_NAME }} - run: yarn playwright test - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: playwright-report - path: samples/apps/copilot-chat-app/webapp/playwright-report/ - retention-days: 30 - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: service-log - path: samples/apps/copilot-chat-app/webapi/service-log.txt - retention-days: 30 diff --git a/.github/workflows/dotnet-build-and-test.yml b/.github/workflows/dotnet-build-and-test.yml index 732509cfc47d..777222ab5911 100644 --- a/.github/workflows/dotnet-build-and-test.yml +++ b/.github/workflows/dotnet-build-and-test.yml @@ -7,13 +7,10 @@ name: dotnet-build-and-test on: workflow_dispatch: - schedule: - - cron: '0 7 * * *' pull_request: - branches: [ "main", "feature*" ] - paths: - - 'dotnet/**' - - 'samples/dotnet/**' + branches: ["main", "feature*"] + merge_group: + branches: ["main"] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -23,40 +20,133 @@ permissions: contents: read jobs: - build-and-test: + paths-filter: + runs-on: ubuntu-latest + outputs: + dotnetChanges: ${{ steps.filter.outputs.dotnet }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + dotnet: + - 'dotnet/**' + - '**/dotnet/**' + - 'samples/skills/**' + # run only if 'dotnet' files were changed + - name: dotnet tests + if: steps.filter.outputs.dotnet == 'true' + run: echo "Dotnet file" + # run only if not 'dotnet' files were changed + - name: not dotnet tests + if: steps.filter.outputs.dotnet != 'true' + run: echo "NOT dotnet file" + dotnet-build-and-test: + needs: paths-filter + if: needs.paths-filter.outputs.dotnetChanges == 'true' strategy: - fail-fast: false - matrix: - include: - - { os: 'ubuntu', dotnet: '6.0-jammy', configuration: Debug } - - { os: 'ubuntu', dotnet: '7.0-jammy', configuration: Release } - - { os: 'ubuntu', dotnet: '8.0-preview-jammy', configuration: Release } - - { os: 'windows', dotnet: '6.0', configuration: Release } - - { os: 'windows', dotnet: '7.0', configuration: Debug } - - { os: 'windows', dotnet: '8.0-preview', configuration: Release } - + fail-fast: false + matrix: + include: + - { dotnet: "6.0-jammy", os: "ubuntu", configuration: Debug } + - { dotnet: "7.0-jammy", os: "ubuntu", configuration: Release } + - { + dotnet: "8.0-preview-jammy", + os: "ubuntu", + configuration: Release, + } + - { dotnet: "6.0", os: "windows", configuration: Release } + - { + dotnet: "7.0", + os: "windows", + configuration: Debug, + integration-tests: true, + } + - { dotnet: "8.0-preview", os: "windows", configuration: Release } + runs-on: ubuntu-latest - env: - NUGET_CERT_REVOCATION_MODE: offline - DOTNET_DOCKER_IMG: mcr.microsoft.com/dotnet/sdk:${{ matrix.dotnet }} + container: + image: mcr.microsoft.com/dotnet/sdk:${{ matrix.dotnet }} + env: + NUGET_CERT_REVOCATION_MODE: offline + GITHUB_ACTIONS: "true" + steps: - - uses: actions/checkout@v3 - with: - clean: true - - - name: Pull container dotnet/sdk:${{ matrix.dotnet }} - run: docker pull mcr.microsoft.com/dotnet/sdk:${{ matrix.dotnet }} - - - name: Build dotnet solutions - run: | - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.sln" | tr '\n' ' ') - for solution in $SOLUTIONS; do - docker run --rm -v $(pwd):/app -w /app -e GITHUB_ACTIONS='true' $DOTNET_DOCKER_IMG /bin/sh -c "dotnet build -c ${{ matrix.configuration }} /warnaserror /app/$solution" - done - - - name: Run Unit Tests - run: | - export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | tr '\n' ' ') - for project in $UT_PROJECTS; do - docker run --rm -v $(pwd):/app -w /app $DOTNET_DOCKER_IMG /bin/sh -c "dotnet test -c ${{ matrix.configuration }} /app/$project --no-build -v Normal --logger trx" - done + - uses: actions/checkout@v4 + + - name: Build dotnet solutions + run: | + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.sln" | tr '\n' ' ') + for solution in $SOLUTIONS; do + dotnet build -c ${{ matrix.configuration }} /warnaserror $solution + done + + - name: Run Unit Tests + run: | + export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | tr '\n' ' ') + for project in $UT_PROJECTS; do + dotnet test -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx + done + + - name: Run Integration Tests + if: github.event_name != 'pull_request' && matrix.integration-tests + run: | + export INTEGRATION_TEST_PROJECTS=$(find ./dotnet -type f -name "*IntegrationTests.csproj" | tr '\n' ' ') + for project in $INTEGRATION_TEST_PROJECTS; do + dotnet test -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx + done + env: + AzureOpenAI__Label: azure-text-davinci-003 + AzureOpenAIEmbedding__Label: azure-text-embedding-ada-002 + AzureOpenAI__DeploymentName: ${{ vars.AZUREOPENAI__DEPLOYMENTNAME }} + AzureOpenAIEmbeddings__DeploymentName: ${{ vars.AZUREOPENAIEMBEDDING__DEPLOYMENTNAME }} + AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} + AzureOpenAIEmbeddings__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} + AzureOpenAI__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }} + AzureOpenAIEmbeddings__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }} + Bing__ApiKey: ${{ secrets.BING__APIKEY }} + OpenAI__ApiKey: ${{ secrets.OPENAI__APIKEY }} + + # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed + dotnet-build-and-test-check: + if: always() + runs-on: ubuntu-latest + needs: [dotnet-build-and-test] + steps: + - name: Get Date + shell: bash + run: | + echo "date=$(date +'%m/%d/%Y %H:%M:%S')" >> "$GITHUB_ENV" + + - name: Run Type is Daily + if: ${{ github.event_name == 'schedule' }} + shell: bash + run: | + echo "run_type=Daily" >> "$GITHUB_ENV" + + - name: Run Type is Manual + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + run: | + echo "run_type=Manual" >> "$GITHUB_ENV" + + - name: Run Type is ${{ github.event_name }} + if: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'}} + shell: bash + run: | + echo "run_type=${{ github.event_name }}" >> "$GITHUB_ENV" + + - name: Fail workflow if tests failed + id: check_tests_failed + if: contains(join(needs.*.result, ','), 'failure') + uses: actions/github-script@v6 + with: + script: core.setFailed('Integration Tests Failed!') + + - name: Fail workflow if tests cancelled + id: check_tests_cancelled + if: contains(join(needs.*.result, ','), 'cancelled') + uses: actions/github-script@v6 + with: + script: core.setFailed('Integration Tests Cancelled!') diff --git a/.github/workflows/dotnet-ci.yml b/.github/workflows/dotnet-ci.yml index b85f08e85dff..68b5c6a8b862 100644 --- a/.github/workflows/dotnet-ci.yml +++ b/.github/workflows/dotnet-ci.yml @@ -26,7 +26,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: clean: true @@ -73,7 +73,7 @@ jobs: env: NUGET_CERT_REVOCATION_MODE: offline steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: clean: true diff --git a/.github/workflows/dotnet-format.yml b/.github/workflows/dotnet-format.yml index 2ddaec9063f4..cfa1c7feebb5 100644 --- a/.github/workflows/dotnet-format.yml +++ b/.github/workflows/dotnet-format.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/dotnet-integration-tests.yml b/.github/workflows/dotnet-integration-tests.yml index 71d6b1d4fc44..6bd5b461d019 100644 --- a/.github/workflows/dotnet-integration-tests.yml +++ b/.github/workflows/dotnet-integration-tests.yml @@ -22,7 +22,7 @@ jobs: configuration: [Debug] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ github.event_name != 'pull_request' }} with: clean: true diff --git a/.github/workflows/generate-pr-description.yml b/.github/workflows/generate-pr-description.yml new file mode 100644 index 000000000000..70261481efe0 --- /dev/null +++ b/.github/workflows/generate-pr-description.yml @@ -0,0 +1,61 @@ +name: Generate PR Description + +on: + issue_comment: + types: [created] + +jobs: + generate-pr-description: + permissions: + pull-requests: write + statuses: write + runs-on: ubuntu-latest + if: github.event.issue.pull_request && contains(github.event.comment.body, '/sk generate-pr-description') + steps: + - name: Get PR branch + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: Set latest commit status as pending + uses: myrotvorets/set-commit-status-action@master + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + status: pending + + - name: Generate PR description + uses: mkarle/skonsole-generate-pr-description@v1 + with: + pull-request-number: ${{ github.event.issue.number }} + pull-request-diff-url: ${{ github.event.issue.pull_request.diff_url }} + token: ${{ secrets.GITHUB_TOKEN }} + update-type: ${{ contains(github.event.comment.body, 'replace') && 'replace' || (contains(github.event.comment.body, 'prefix') && 'prefix' || 'suffix') }} + env: # Set Azure credentials secret as an input + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHAT__DEPLOYMENTNAME }} + AZURE_OPENAI_API_ENDPOINT: ${{ secrets.AZUREOPENAI__ENDPOINT }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZUREOPENAI__APIKEY }} + + - name: Set latest commit status as ${{ job.status }} + uses: myrotvorets/set-commit-status-action@master + if: always() + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + + - name: Add comment to PR + uses: actions/github-script@v6 + if: always() + with: + script: | + const name = '${{ github.workflow }}'; + const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const success = '${{ job.status }}' === 'success'; + const body = `${name}: ${success ? 'succeeded ✅' : 'failed ❌'}\n${url}`; + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }) diff --git a/.github/workflows/java-format.yml b/.github/workflows/java-format.yml index 5cd460b67632..ea1e90d31a2f 100644 --- a/.github/workflows/java-format.yml +++ b/.github/workflows/java-format.yml @@ -35,7 +35,7 @@ jobs: command=$(echo "$BODY" | head -1 | sed "s;^/;;") echo "COMMAND=$command" >> $GITHUB_ENV - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check out PR branch env: diff --git a/.github/workflows/label-title-prefix.yml b/.github/workflows/label-title-prefix.yml index 7b3983341dbb..b0c01aad3fcd 100644 --- a/.github/workflows/label-title-prefix.yml +++ b/.github/workflows/label-title-prefix.yml @@ -1,9 +1,9 @@ name: Label title prefix on: issues: - types: [ labeled ] + types: [labeled] pull_request_target: - types: [ labeled ] + types: [labeled] jobs: add_title_prefix: @@ -13,20 +13,19 @@ jobs: permissions: issues: write pull-requests: write - + steps: - uses: actions/github-script@v6 name: "Issue/PR: update title" with: - github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }} + github-token: ${{ secrets.GITHUB_TOKEN }} script: | let prefixLabels = { "python": "Python", "java": "Java", - ".NET": ".Net", - "copilot chat": "Copilot Chat" + ".NET": ".Net" }; - + function addTitlePrefix(title, prefix) { // Update the title based on the label and prefix @@ -41,12 +40,12 @@ jobs: title = prefix + ": " + title; } } - + return title; } - + labelAdded = context.payload.label.name - + // Check if the issue or PR has the label if (labelAdded in prefixLabels) { let prefix = prefixLabels[labelAdded]; @@ -59,7 +58,7 @@ jobs: title: addTitlePrefix(context.payload.issue.title, prefix) }); break - + case 'pull_request_target': github.rest.pulls.update({ pull_number: context.issue.number, diff --git a/.github/workflows/markdown-link-check-config.json b/.github/workflows/markdown-link-check-config.json index c2897fb25350..e8b77bbd0958 100644 --- a/.github/workflows/markdown-link-check-config.json +++ b/.github/workflows/markdown-link-check-config.json @@ -23,6 +23,9 @@ }, { "pattern": "^https://localhost" + }, + { + "pattern": "^https://platform.openai.com" } ], "timeout": "20s", @@ -36,4 +39,4 @@ 500, 503 ] -} \ No newline at end of file +} diff --git a/.github/workflows/markdown-link-check.yml b/.github/workflows/markdown-link-check.yml index 00826fe7524d..4e02c7256673 100644 --- a/.github/workflows/markdown-link-check.yml +++ b/.github/workflows/markdown-link-check.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest # check out the latest version of the code steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Checks the status of hyperlinks in .md files in verbose mode - name: Check links diff --git a/.github/workflows/node-pr.yml b/.github/workflows/node-pr.yml index 1bdabff5ab72..4fd8f6dde7ce 100644 --- a/.github/workflows/node-pr.yml +++ b/.github/workflows/node-pr.yml @@ -21,7 +21,7 @@ jobs: matrix: ${{ steps.set-yarn-folders.outputs.matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Find yarn projects id: set-yarn-folders @@ -51,7 +51,7 @@ jobs: matrix: ${{ fromJson(needs.find-yarn-projects.outputs.matrix) }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node_version }} uses: actions/setup-node@v3 with: diff --git a/.github/workflows/python-build-wheel.yml b/.github/workflows/python-build-wheel.yml index 8e034a53f1a1..4d1762f71c16 100644 --- a/.github/workflows/python-build-wheel.yml +++ b/.github/workflows/python-build-wheel.yml @@ -18,7 +18,7 @@ jobs: working-directory: python steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: echo "/root/.local/bin" >> $GITHUB_PATH diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml index a8507300cd7a..83685b536796 100644 --- a/.github/workflows/python-integration-tests.yml +++ b/.github/workflows/python-integration-tests.yml @@ -6,18 +6,102 @@ name: Python Integration Tests on: workflow_dispatch: - push: + pull_request: + branches: ["main"] + merge_group: branches: ["main"] - paths: - - "python/**" schedule: - - cron: "0 */12 * * *" # Run every 12 hours: midnight UTC and noon UTC + - cron: "0 0 * * *" # Run at midnight UTC daily permissions: contents: read jobs: + paths-filter: + runs-on: ubuntu-latest + outputs: + pythonChanges: ${{ steps.filter.outputs.python}} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + python: + - 'python/**' + # run only if 'python' files were changed + - name: python tests + if: steps.filter.outputs.python == 'true' + run: echo "Python file" + # run only if not 'python' files were changed + - name: not python tests + if: steps.filter.outputs.python != 'true' + run: echo "NOT python file" + + python-merge-gate: + needs: paths-filter + if: github.event_name != 'pull_request' && github.event_name != 'schedule' && needs.paths-filter.outputs.pythonChanges == 'true' + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 1 + fail-fast: false + matrix: + python-version: ["3.11"] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies with hnswlib native disabled + if: matrix.os == 'macos-latest' && matrix.python-version == '3.11' + run: | + export HNSWLIB_NO_NATIVE=1 + python -m pip install --upgrade pip setuptools wheel + python -m pip install poetry pytest + cd python && poetry install + - name: Install dependencies with hnswlib native enabled + if: matrix.os != 'macos-latest' || matrix.python-version != '3.11' + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install poetry pytest + cd python + poetry install + - name: Run Integration Tests + id: run_tests + shell: bash + env: # Set Azure credentials secret as an input + HNSWLIB_NO_NATIVE: 1 + Python_Integration_Tests: Python_Integration_Tests + AzureOpenAI__Label: azure-text-davinci-003 + AzureOpenAIEmbedding__Label: azure-text-embedding-ada-002 + AzureOpenAI__DeploymentName: ${{ vars.AZUREOPENAI__DEPLOYMENTNAME }} + AzureOpenAIChat__DeploymentName: ${{ vars.AZUREOPENAI__CHAT__DEPLOYMENTNAME }} + AzureOpenAIEmbeddings__DeploymentName: ${{ vars.AZUREOPENAIEMBEDDINGS__DEPLOYMENTNAME2 }} + AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} + AzureOpenAIEmbeddings__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} + AzureOpenAI__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }} + AzureOpenAIEmbeddings__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }} + Bing__ApiKey: ${{ secrets.BING__APIKEY }} + OpenAI__ApiKey: ${{ secrets.OPENAI__APIKEY }} + Pinecone__ApiKey: ${{ secrets.PINECONE__APIKEY }} + Pinecone__Environment: ${{ secrets.PINECONE__ENVIRONMENT }} + Postgres__Connectionstr: ${{secrets.POSTGRES__CONNECTIONSTR}} + AZURE_COGNITIVE_SEARCH_ADMIN_KEY: ${{secrets.AZURE_COGNITIVE_SEARCH_ADMIN_KEY}} + AZURE_COGNITIVE_SEARCH_ENDPOINT: ${{secrets.AZURE_COGNITIVE_SEARCH_ENDPOINT}} + MONGODB_ATLAS_CONNECTION_STRING: ${{secrets.MONGODB_ATLAS_CONNECTION_STRING}} + run: | + if ${{ matrix.os == 'ubuntu-latest' }}; then + docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest + fi + + cd python + poetry run pytest ./tests/integration -v + python-integration-tests: + needs: paths-filter + if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.paths-filter.outputs.pythonChanges == 'true' runs-on: ${{ matrix.os }} strategy: max-parallel: 1 @@ -25,13 +109,13 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest, windows-latest, macos-latest] - steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies with hnswlib native disabled if: matrix.os == 'macos-latest' && matrix.python-version == '3.11' run: | @@ -39,13 +123,16 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install poetry pytest cd python && poetry install + - name: Install dependencies with hnswlib native enabled if: matrix.os != 'macos-latest' || matrix.python-version != '3.11' run: | python -m pip install --upgrade pip setuptools wheel python -m pip install poetry pytest cd python && poetry install + - name: Run Integration Tests + id: run_tests shell: bash env: # Set Azure credentials secret as an input HNSWLIB_NO_NATIVE: 1 @@ -54,7 +141,7 @@ jobs: AzureOpenAIEmbedding__Label: azure-text-embedding-ada-002 AzureOpenAI__DeploymentName: ${{ vars.AZUREOPENAI__DEPLOYMENTNAME }} AzureOpenAIChat__DeploymentName: ${{ vars.AZUREOPENAI__CHAT__DEPLOYMENTNAME }} - AzureOpenAIEmbeddings__DeploymentName: ${{ vars.AZUREOPENAIEMBEDDING__DEPLOYMENTNAME }} + AzureOpenAIEmbeddings__DeploymentName: ${{ vars.AZUREOPENAIEMBEDDINGS__DEPLOYMENTNAME2 }} AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} AzureOpenAIEmbeddings__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} AzureOpenAI__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }} @@ -66,6 +153,77 @@ jobs: Postgres__Connectionstr: ${{secrets.POSTGRES__CONNECTIONSTR}} AZURE_COGNITIVE_SEARCH_ADMIN_KEY: ${{secrets.AZURE_COGNITIVE_SEARCH_ADMIN_KEY}} AZURE_COGNITIVE_SEARCH_ENDPOINT: ${{secrets.AZURE_COGNITIVE_SEARCH_ENDPOINT}} + MONGODB_ATLAS_CONNECTION_STRING: ${{secrets.MONGODB_ATLAS_CONNECTION_STRING}} run: | + if ${{ matrix.os == 'ubuntu-latest' }}; then + docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest + fi + cd python - poetry run pytest ./tests/integration + poetry run pytest ./tests/integration -v + + # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed + python-integration-tests-check: + if: always() + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + fail-fast: false + needs: [python-merge-gate, python-integration-tests] + steps: + - name: Get Date + shell: bash + run: | + echo "date=$(date +'%m/%d/%Y %H:%M:%S')" >> "$GITHUB_ENV" + + - name: Run Type is Daily + if: ${{ github.event_name == 'schedule' }} + shell: bash + run: | + echo "run_type=Daily" >> "$GITHUB_ENV" + + - name: Run Type is Manual + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + run: | + echo "run_type=Manual" >> "$GITHUB_ENV" + + - name: Run Type is ${{ github.event_name }} + if: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'}} + shell: bash + run: | + echo "run_type=${{ github.event_name }}" >> "$GITHUB_ENV" + + - name: Fail workflow if tests failed + id: check_tests_failed + if: contains(join(needs.*.result, ','), 'failure') + uses: actions/github-script@v6 + with: + script: core.setFailed('Integration Tests Failed!') + + - name: Fail workflow if tests cancelled + id: check_tests_cancelled + if: contains(join(needs.*.result, ','), 'cancelled') + uses: actions/github-script@v6 + with: + script: core.setFailed('Integration Tests Cancelled!') + + - name: Microsoft Teams Notification + uses: skitionek/notify-microsoft-teams@master + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + with: + webhook_url: ${{ secrets.MSTEAMS_WEBHOOK }} + dry_run: ${{ env.run_type != 'Daily' && env.run_type != 'Manual'}} + job: ${{ toJson(job) }} + steps: ${{ toJson(steps) }} + overwrite: "{title: ` ${{ env.run_type }}: ${{ env.date }} `, text: ` ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`}" + + - name: Microsoft Teams Notification (Dry Run) + uses: skitionek/notify-microsoft-teams@master + if: github.ref != 'refs/heads/main' + with: + webhook_url: NONE + dry_run: ${{ env.run_type != 'Daily' && env.run_type != 'Manual'}} + job: ${{ toJson(job) }} + steps: ${{ toJson(steps) }} + overwrite: "{title: ` ${{ env.run_type }}: ${{ env.date }} `, text: ` ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`}" diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 8f8f2c5a0d7d..dd21150e1306 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 5 steps: - run: echo "/root/.local/bin" >> $GITHUB_PATH - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install poetry run: pipx install poetry - uses: actions/setup-python@v4 @@ -37,7 +37,7 @@ jobs: timeout-minutes: 5 steps: - run: echo "/root/.local/bin" >> $GITHUB_PATH - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install poetry run: pipx install poetry - uses: actions/setup-python@v4 diff --git a/.github/workflows/python-unit-tests.yml b/.github/workflows/python-unit-tests.yml index ab9ffdbb9f4e..ba58264325b8 100644 --- a/.github/workflows/python-unit-tests.yml +++ b/.github/workflows/python-unit-tests.yml @@ -17,7 +17,7 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -26,7 +26,7 @@ jobs: run: | python -m pip install poetry pytest cd python - poetry install --without chromadb --without hugging_face --without azure_cognitive_search --without weaviate --without pinecone --without postgres --without qdrant + poetry install --without chromadb --without hugging_face --without azure_cognitive_search --without weaviate --without pinecone --without postgres --without qdrant --without redis - name: Test with pytest run: | cd python && poetry run pytest ./tests/unit diff --git a/.github/workflows/typos.yaml b/.github/workflows/typos.yaml index 1d2b0df548eb..94931d48b5d9 100644 --- a/.github/workflows/typos.yaml +++ b/.github/workflows/typos.yaml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Use custom config file uses: crate-ci/typos@master diff --git a/.gitignore b/.gitignore index 9df6bdb43c5e..ce57c294ee82 100644 --- a/.gitignore +++ b/.gitignore @@ -412,6 +412,7 @@ FodyWeavers.xsd .env certs/ launchSettings.json +!samples/dotnet/MsGraphPluginsExample/Properties/launchSettings.json config.development.yaml *.development.config *.development.json @@ -482,3 +483,6 @@ swa-cli.config.json # Semantic Kernel Tools /.semantic-kernel + +# python devcontainer +/python/.devcontainer/* diff --git a/.vscode/launch.json b/.vscode/launch.json index 83c63b4f5199..8164e37b411c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,19 +1,6 @@ { "version": "0.2.0", "configurations": [ - { - "name": ".NET Core Launch (CopilotChatWebApi)", - "type": "coreclr", - "request": "launch", - "preLaunchTask": "build (CopilotChatWebApi)", - // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/samples/apps/copilot-chat-app/webapi/bin/Debug/net6.0/CopilotChatWebApi.dll", - "args": [], - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapi", - // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console - "console": "internalConsole", - "stopAtEntry": false - }, { // Use IntelliSense to find out which attributes exist for C# debugging // Use hover for the description of the existing attributes @@ -24,7 +11,9 @@ "preLaunchTask": "build (KernelSyntaxExamples)", // If you have changed target frameworks, make sure to update the program path. "program": "${workspaceFolder}/dotnet/samples/KernelSyntaxExamples/bin/Debug/net6.0/KernelSyntaxExamples.dll", - "args": [], + "args": [ + /*"example0"*/ + ], "cwd": "${workspaceFolder}/dotnet/samples/KernelSyntaxExamples", // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console "console": "internalConsole", @@ -36,12 +25,12 @@ "request": "attach" }, { - "cwd":"${workspaceFolder}/python", + "cwd": "${workspaceFolder}/python", "name": "Python: Test Module", "type": "python", "request": "launch", "module": "pytest", - "args": ["${file}"], + "args": ["${file}"] } ] } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index c693cf323a38..ec84e7621884 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -77,7 +77,7 @@ } }, { - "label": "validate - (contributing-Format-Build-Test-Run)", + "label": "validate (contributing-Format-Build-Test-Run)", "detail": "Runs tasks to validate changes before checking in.", "group": "build", "dependsOn": [ @@ -88,6 +88,54 @@ ], "dependsOrder": "sequence" }, + // ***************************** + // Contributing (python) - Setup + // ***************************** + { + "label": "setup (contributing-python)", + "detail": "", + "group": "build", + "dependsOn": ["install poetry", "install python packages"], + "dependsOrder": "sequence" + }, + { + "label": "install poetry", + "detail": "Install poetry", + "command": "pip3", + "type": "shell", + "args": ["install", "poetry"], + "options": { + "cwd": "${workspaceFolder}/python" + } + }, + { + "label": "install python packages", + "detail": "Install python packages", + "command": "poetry", + "type": "shell", + "args": ["install"], + "options": { + "cwd": "${workspaceFolder}/python" + } + }, + // Formatting + { + "label": "validate (contributing-python)", + "command": "poetry", + "type": "shell", + "group": "build", + "args": [ + "run", + "pre-commit", + "run", + "-c", + ".conf/.pre-commit-config.yaml", + "-a" + ], + "options": { + "cwd": "${workspaceFolder}/python" + } + }, // *************** // Kernel (dotnet) // *************** @@ -223,133 +271,60 @@ } }, // **************** - // Samples (dotnet) + // Kernel (python) // **************** - // Copilot Chat - { - "label": "run (Copilot Chat)", - "detail": "Run all copilot chat components", - "group": "test", - "dependsOn": ["run (CopilotChatWebApi)", "run (CopilotChatApp)"], - "dependsOrder": "parallel" - }, - // Copilot Setup + // Test { - "label": "install (CopilotChatApp)", - "detail": "Install all copilot chat app dependencies", + "label": "test (Semantic-Kernel-Python)", + "command": "poetry", "type": "shell", - "group": "build", - "command": "yarn", - "presentation": { - "echo": true, - "reveal": "always", - "showReuseMessage": false, - "panel": "shared", - "group": "buildTasks" - }, - "options": { - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapp" - }, - "problemMatcher": [] - }, - { - "label": "setup (Copilot Chat)", - "detail": "Setup (like setting secrets) for copilot chat app and api", + "args": ["run", "pytest", "tests/unit"], + "problemMatcher": "$msCompile", "group": "test", - "dependsOn": ["GetSecret (AIService:Key)"], - "dependsOrder": "sequence" - // TODO -- add tasks for configuring environment variables - }, - { - "label": "GetSecret (AIService:Key)", - "command": "dotnet", - "type": "process", - "args": [ - "user-secrets", - "set", - "AIService:Key", - "${input:aiServiceSecret}" - ], - "options": { - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapi" - } - }, - // Copilot Chat App - { - "label": "build (CopilotChatApp)", - "type": "shell", - "group": "build", - "command": "yarn build", "presentation": { - "echo": true, "reveal": "always", "panel": "shared", - "group": "buildTasks" + "group": "PR-Validate" }, "options": { - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapp" - }, - "problemMatcher": [] + "cwd": "${workspaceFolder}/python" + } }, { - "label": "run (CopilotChatApp)", + "label": "test (Semantic-Kernel-Python Integration)", + "command": "poetry", "type": "shell", + "args": ["run", "pytest", "tests/integration", "-k", "${input:filter}"], + "problemMatcher": "$msCompile", "group": "test", - "command": "yarn start", "presentation": { "reveal": "always", "panel": "shared", - "group": "copilot" + "group": "PR-Validate" }, "options": { - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapp" + "cwd": "${workspaceFolder}/python" } }, - // Copilot Chat Api - { - "label": "build (CopilotChatWebApi)", - "command": "dotnet", - "type": "process", - "args": [ - "build", - "${workspaceFolder}/samples/apps/copilot-chat-app/webapi/CopilotChatWebApi.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary", - "/property:DebugType=portable" - ], - "problemMatcher": "$msCompile", - "group": "build" - }, { - "label": "run (CopilotChatWebApi)", - "command": "dotnet", - "type": "process", - "args": [ - "run", - "--project", - "${workspaceFolder}/samples/apps/copilot-chat-app/webapi/CopilotChatWebApi.csproj" - ], + "label": "test (Semantic-Kernel-Python ALL)", + "command": "poetry", + "type": "shell", + "args": ["run", "pytest", "tests", "-k", "${input:filter}"], "problemMatcher": "$msCompile", "group": "test", "presentation": { "reveal": "always", "panel": "shared", - "group": "copilot" + "group": "PR-Validate" + }, + "options": { + "cwd": "${workspaceFolder}/python" } }, - { - "label": "watch (CopilotChatWebApi)", - "command": "dotnet", - "type": "process", - "args": [ - "watch", - "run", - "--project", - "${workspaceFolder}/samples/apps/copilot-chat-app/webapi/CopilotChatWebApi.csproj" - ], - "problemMatcher": "$msCompile", - "group": "build" - }, + // **************** + // Samples (dotnet) + // **************** // Kernel Syntax Examples { "label": "build (KernelSyntaxExamples)", @@ -385,7 +360,8 @@ "args": [ "run", "--project", - "${workspaceFolder}/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj" + "${workspaceFolder}/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj", + "${input:filter}" ], "problemMatcher": "$msCompile", "group": "test", @@ -465,14 +441,7 @@ "id": "filter", "type": "promptString", "default": "", - "description": "Enter a filter for the tests" - }, - { - "id": "aiServiceSecret", - "type": "promptString", - "default": "", - "description": "Enter a secret for Copilot Chat AIService:Key", - "password": true + "description": "Enter a filter to pass as argument or filter" } ] } diff --git a/COMMUNITY.md b/COMMUNITY.md index 8aed983e4700..829e673c01d5 100644 --- a/COMMUNITY.md +++ b/COMMUNITY.md @@ -11,7 +11,9 @@ We do our best to respond to each submission. We regularly have Community Office Hours that are open to the **public** to join. -Add Semantic Kernel events to your calendar: download the [calendar.ics](https://aka.ms/sk-community-calendar) file. +Add Semantic Kernel events to your calendar - we're running two community calls to cater different timezones: +* Americas timezone: download the [calendar.ics](https://aka.ms/sk-community-calendar) file. +* Asia Pacific timezone: download the [calendar-APAC.ics](https://aka.ms/sk-community-calendar-apac) file. To keep topics organized, please submit what you'd like us to cover here: [https://forms.office.com/r/BbXFzmmFys](https://forms.office.com/r/BbXFzmmFys) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77d5cf256d25..14cbb9be650a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,6 +111,32 @@ The scripts below are used to build, test, and lint within the project. - Build/Test: `yarn build` - Linting (auto-fix): `yarn lint:fix` +### Adding Plugins and Memory Connectors + +When considering contributions to plugins and memory connectors for Semantic +Kernel, please note the following guidelines: + +#### Plugins + +We appreciate your interest in extending Semantic Kernel's functionality through +plugins. However, we want to clarify our approach to hosting plugins within our +GitHub repository. To maintain a clean and manageable codebase, we will not be +hosting plugins directly in the Semantic Kernel GitHub repository. +Instead, we encourage contributors to host their plugin code in separate +repositories under their own GitHub accounts or organization. You can then +provide a link to your plugin repository in the relevant discussions, issues, +or documentation within the Semantic Kernel repository. This approach ensures +that each plugin can be maintained independently and allows for easier tracking +of updates and issues specific to each plugin. + +#### Memory Connectors + +For memory connectors, while we won't be directly adding hosting for them within +the Semantic Kernel repository, we highly recommend building memory connectors +as separate plugins. Memory connectors play a crucial role in interfacing with +external memory systems, and treating them as plugins enhances modularity and +maintainability. + ### PR - CI Process The continuous integration (CI) system will automatically perform the required diff --git a/README.md b/README.md index fdabd29538c9..8cab9adb1ede 100644 --- a/README.md +++ b/README.md @@ -7,42 +7,32 @@ [![License: MIT](https://img.shields.io/github/license/microsoft/semantic-kernel)](https://github.com/microsoft/semantic-kernel/blob/main/LICENSE) [![Discord](https://img.shields.io/discord/1063152441819942922?label=Discord&logo=discord&logoColor=white&color=d82679)](https://aka.ms/SKDiscord) -> ℹ️ **NOTE**: This project is just like AI and will evolve quickly. -> We invite you to join us in developing the Semantic Kernel together! -> Please contribute by -> using GitHub [Discussions](https://github.com/microsoft/semantic-kernel/discussions), -> opening GitHub [Issues](https://github.com/microsoft/semantic-kernel/issues/new/choose), -> sending us [PRs](https://github.com/microsoft/semantic-kernel/pulls), -> joining our [Discord community](https://aka.ms/SKDiscord). - -**Semantic Kernel (SK)** is a lightweight SDK enabling integration of AI Large -Language Models (LLMs) with conventional programming languages. The SK extensible -programming model combines natural language **semantic functions**, traditional -code **native functions**, and **embeddings-based memory** unlocking new potential -and adding value to applications with AI. - -SK supports -[prompt templating](https://learn.microsoft.com/en-us/semantic-kernel/prompt-engineering/prompt-template-syntax), function -chaining, -[vectorized memory](https://learn.microsoft.com/en-us/semantic-kernel/memories/embeddings), and -[intelligent planning](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/planner) -capabilities out of the box. - -Semantic Kernel supports and encapsulates several design patterns from the latest -in AI research, such that developers can infuse their applications with [plugins](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins) like [prompt -chaining](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chaining-functions), recursive reasoning, summarization, zero/few-shot learning, contextual -memory, long-term memory, [embeddings](https://learn.microsoft.com/en-us/semantic-kernel/memories/embeddings), semantic indexing, -[planning](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/planner), retrieval-augmented generation and accessing external -knowledge stores as well as your own data. - -By joining the SK community, you can build AI-first apps faster and have a front-row -peek at how the SDK is being built. SK has been released as open-source so that more -pioneering developers can join us in crafting the future of this landmark moment -in the history of computing. - -## Get Started with Semantic Kernel ⚡ - -Semantic Kernel is available to explore AI and build apps with C# and Python: +[Semantic Kernel](https://learn.microsoft.com/en-us/semantic-kernel/overview/) +is an SDK that integrates Large Language Models (LLMs) like +[OpenAI](https://platform.openai.com/docs/introduction), +[Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service), +and [Hugging Face](https://huggingface.co/) +with conventional programming languages like C#, Python, and Java. Semantic Kernel achieves this +by allowing you to define [plugins](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins) +that can be chained together +in just a [few lines of code](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chaining-functions?tabs=Csharp#using-the-runasync-method-to-simplify-your-code). + +What makes Semantic Kernel _special_, however, is its ability to _automatically_ orchestrate +plugins with AI. With Semantic Kernel +[planners](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/planner), you +can ask an LLM to generate a plan that achieves a user's unique goal. Afterwards, +Semantic Kernel will execute the plan for the user. + +#### Please star the repo to show your support for this project! + +![Orchestrating plugins with planner](https://learn.microsoft.com/en-us/semantic-kernel/media/kernel-infographic.png) + + + +## Getting started with Semantic Kernel + +The Semantic Kernel SDK is available in C#, Python, and Java. To get started, choose your preferred language below. See the [Feature Matrix](https://learn.microsoft.com/en-us/semantic-kernel/get-started/supported-languages) to see a breakdown of +feature parity between our currently supported languages. @@ -50,7 +40,7 @@ Semantic Kernel is available to explore AI and build apps with C# and Python: +
- Using Semantic Kernel in C#   + Using Semantic Kernel in C#  
@@ -59,15 +49,18 @@ Semantic Kernel is available to explore AI and build apps with C# and Python: Using Semantic Kernel in Python + Java logo +
+ Using Semantic Kernel in Java +
+
-See the [Feature Matrix](https://learn.microsoft.com/en-us/semantic-kernel/get-started/supported-languages) to see a breakdown of feature parity between our currently supported languages. - The quickest way to get started with the basics is to get an API key -(OpenAI or Azure OpenAI) -and to run one of the C# or Python console applications/scripts: +from either OpenAI or Azure OpenAI and to run one of the C#, Python, and Java console applications/scripts below. ### For C#: @@ -85,48 +78,75 @@ and to run one of the C# or Python console applications/scripts: 4. Copy the code from [here](python/README.md) into the `hello-world.py` script. 5. Run the python script. -## Sample apps ⚡ +### For Java: + +1. Clone and checkout the experimental Java branch: `git clone -b experimental-java https://github.com/microsoft/semantic-kernel.git` +2. Follow the instructions [here](https://github.com/microsoft/semantic-kernel/blob/experimental-java/java/samples/sample-code/README.md) + +## Learning how to use Semantic Kernel + +The fastest way to learn how to use Semantic Kernel is with our C# and Python Jupyter notebooks. These notebooks +demonstrate how to use Semantic Kernel with code snippets that you can run with a push of a button. + +- [Getting Started with C# notebook](dotnet/notebooks/00-getting-started.ipynb) +- [Getting Started with Python notebook](python/notebooks/00-getting-started.ipynb) + +Once you've finished the getting started notebooks, you can then check out the main walkthroughs +on our Learn site. Each sample comes with a completed C# and Python project that you can run locally. + +1. 📖 [Overview of the kernel](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/) +1. 🔌 [Understanding AI plugins](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins) +1. 👄 [Creating semantic functions](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/semantic-functions) +1. 💽 [Creating native functions](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/native-functions) +1. ⛓️ [Chaining functions together](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chaining-functions) +1. 🤖 [Auto create plans with planner](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/planner) +1. 💡 [Create and run a ChatGPT plugin](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chatgpt-plugins) + +Finally, refer to our API references for more details on the C# and Python APIs: + +- [C# API reference](https://learn.microsoft.com/en-us/dotnet/api/microsoft.semantickernel?view=semantic-kernel-dotnet) +- Python API reference (coming soon) + +## Chat Copilot: see what's possible with Semantic Kernel -The repository includes some sample applications, with a React frontend and -a backend web service using Semantic Kernel. +If you're interested in seeing a full end-to-end example of how to use Semantic Kernel, check out +our [Chat Copilot](https://github.com/microsoft/chat-copilot) reference application. Chat Copilot +is a chatbot that demonstrates the power of Semantic Kernel. By combining plugins, planners, and personas, +we demonstrate how you can build a chatbot that can maintain long-running conversations with users while +also leveraging plugins to integrate with other services. -Follow the links for more information and instructions about running these apps. +![Chat Copilot answering a question](https://learn.microsoft.com/en-us/semantic-kernel/media/chat-copilot-in-action.gif) -| | | -| ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| [Simple chat summary](samples/apps/chat-summary-webapp-react/README.md) | Use ready-to-use plugins and get plugins into your app easily. | -| [Book creator](samples/apps/book-creator-webapp-react/README.md) | Use planner to deconstruct a complex goal and envision using the planner in your app. | -| [Authentication and APIs](samples/apps/auth-api-webapp-react/README.md) | Use a basic connector pattern to authenticate and connect to an API and imagine integrating external data into your app's LLM AI. | -| [GitHub repository Q&A](samples/apps/github-qna-webapp-react/README.md) | Use embeddings and memory to store recent data and allow you to query against it. | -| [Copilot Chat Sample App](samples/apps/copilot-chat-app/README.md) | Build your own chat experience based on Semantic Kernel. | +You can run the app yourself by downloading it from its [GitHub repo](https://github.com/microsoft/chat-copilot). -**Requirements:** +## Visual Studio Code extension: design semantic functions with ease -- You will need an - [Open AI API Key](https://openai.com/api/) or - [Azure Open AI service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=rest-api) - to get started. -- [Azure Functions Core Tools](https://learn.microsoft.com/azure/azure-functions/functions-run-local) - are required to run the kernel as a local web service, used by the sample web apps. -- [.NET 6 SDK](https://dotnet.microsoft.com/download/dotnet/6.0) or [.NET 7 SDK](https://dotnet.microsoft.com/download/dotnet/7.0) -- [Yarn](https://yarnpkg.com/getting-started/install) is used for installing web apps' dependencies. +The [Semantic Kernel extension for Visual Studio Code](https://learn.microsoft.com/en-us/semantic-kernel/vs-code-tools/) +makes it easy to design and test semantic functions. The extension provides an interface for +designing semantic functions and allows you to test them with a push of a button with your +existing models and data. -## Deploy Semantic Kernel to Azure in a web app service ☁️ +![Semantic Kernel extension for Visual Studio Code](https://learn.microsoft.com/en-us/semantic-kernel/media/vs-code-extension.png) -Getting Semantic Kernel deployed to Azure as web app service is easy with one-click deployments. Click [here](https://aka.ms/sk-docs-azuredeploy) to learn more on how to deploy to Azure. +In the above screenshot, you can see the extension in action: -## Jupyter Notebooks ⚡ +- Syntax highlighting for semantic functions +- Code completion for semantic functions +- LLM model picker +- Run button to test the semantic function with your input data -For a more hands-on overview, you can also check out the C# and Python Jupyter notebooks, starting -from here: +## Check out our other repos! -- [Getting Started with C# notebook](samples/notebooks/dotnet/00-getting-started.ipynb) -- [Getting Started with Python notebook](samples/notebooks/python/00-getting-started.ipynb) +If you like Semantic Kernel, you may also be interested in other repos the Semantic Kernel team supports: -**Requirements:** C# notebooks require [.NET 7](https://dotnet.microsoft.com/download) -and the VS Code [Polyglot extension](https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.dotnet-interactive-vscode). +| Repo | Description | +| --------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| [Chat Copilot](https://github.com/microsoft/chat-copilot) | A reference application that demonstrates how to build a chatbot with Semantic Kernel. | +| [Semantic Kernel Docs](https://github.com/MicrosoftDocs/semantic-kernel-docs) | The home for Semantic Kernel documentation that appears on the Microsoft learn site. | +| [Semantic Kernel Starters](https://github.com/microsoft/semantic-kernel-starters) | Starter projects for Semantic Kernel to make it easier to get started. | +| [Semantic Memory](https://github.com/microsoft/semantic-memory) | A service that allows you to create pipelines for ingesting, storing, and querying knowledge. | -## Contributing and Community +## Join the community We welcome your contributions and suggestions to SK community! One of the easiest ways to participate is to engage in discussions in the GitHub repository. @@ -139,7 +159,7 @@ in a different direction, but also to consider the impact on the larger ecosyste To learn more and get started: - Read the [documentation](https://aka.ms/sk/learn) -- Learn how to [contribute](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) to the project +- Learn how to [contribute](https://learn.microsoft.com/en-us/semantic-kernel/get-started/contributing) to the project - Join the [Discord community](https://aka.ms/SKDiscord) - Attend [regular office hours and SK community events](COMMUNITY.md) - Follow the team on our [blog](https://aka.ms/sk/blog) diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md index 93f4a7c5aa29..53ff34978910 100644 --- a/docs/GLOSSARY.md +++ b/docs/GLOSSARY.md @@ -3,19 +3,19 @@ To wrap your mind around the concepts we present throughout the kernel, here is a glossary of commonly used terms -**Semantic Kernel (SK)** - The orchestrator that fulfills a user's ASK with SK's available [SKILLS](SKILLS.md). +**Semantic Kernel (SK)** - The orchestrator that fulfills a user's ASK with SK's available [PLUGINS](PLUGINS.md). **Ask** - What a user requests to the Semantic Kernel to help achieve the user's goal. - "We make ASKs to the SK" -**Skill** - A domain-specific collection made available to the SK as a group of finely-tuned functions. +**Plugins** - A domain-specific collection made available to the SK as a group of finely-tuned functions. -- "We have a SKILL for using Office better" +- "We have a PLUGIN for using Office better" -**Function** - A computational machine comprised of Semantic AI and/or native code that's available in a [SKILL](SKILLS.md). +**Function** - A computational machine comprised of Semantic AI and/or native code that's available in a [PLUGIN](PLUGINS.md). -- "The Office SKILL has many FUNCTIONS" +- "The Office PLUGIN has many FUNCTIONS" **Native Function** - expressed with traditional computing language (C#, Python, Typescript) and easily integrates with SK diff --git a/docs/SKILLS.md b/docs/PLUGINS.md similarity index 100% rename from docs/SKILLS.md rename to docs/PLUGINS.md diff --git a/docs/decisions/0001-madr-architecture-decisions.md b/docs/decisions/0001-madr-architecture-decisions.md index f9e3b7125438..faa92433ed8a 100644 --- a/docs/decisions/0001-madr-architecture-decisions.md +++ b/docs/decisions/0001-madr-architecture-decisions.md @@ -15,7 +15,7 @@ We need a way to keep the implementations aligned with regard to key architectur semantic function configuration (config.json) and when this change is agreed it must be reflected in all of the Semantic Kernel implementations. MADR is a lean template to capture any decisions in a structured way. The template originated from capturing architectural decisions and developed to a template allowing to capture any decisions taken. -For more information [see](https://adr.github.io/madr/) +For more information [see](https://adr.github.io/) ## Decision Drivers diff --git a/docs/decisions/0005-function-execution-handlers.md b/docs/decisions/0005-function-execution-handlers.md new file mode 100644 index 000000000000..b3b4e2939dab --- /dev/null +++ b/docs/decisions/0005-function-execution-handlers.md @@ -0,0 +1,133 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 2023-05-29 +deciders: @rogerbarreto, @shawncal, @stephentoub +consulted: +informed: +--- + +# Kernel/Function Handlers + +## Context and Problem Statement + +A Kernel function caller needs to be able to handle/intercept any function execution in the Kernel before and after it was attempted. Allowing it to modify the prompt, abort the execution, or modify the output and many other scenarios as follows: + +- Pre-Execution / Running + + - Get: Original prompt template + - Get: Prompt generated by the current Kernel `TemplateEngine` before calling the LLM + - Get: Current settings used + - Get: Parameters used + - Get: SKContext + - Set: Modify a prompt content before sending it to LLM + - Set: Abort/Cancel function execution + +- In-Execution / Stream Processing + + - Get: Text Prompt generated per stream block `IAsyncEnumerable` + - Set: Filter/Change a prompt stream block + - Set: Skip a prompt stream block + +- Post-Execution / Ran + + - Get: LLM Model Result (Tokens Usage, Stop Sequence, ...) + - Get: Generated Prompt + - Get: SKContext + - Get: Output parameters + + - Set: Modify a prompt content after getting it from LLM + - Set: Modify output parameters content (before returning the output) + +## Decision Drivers + +- Architecture changes and the associated decision making process should be transparent to the community. +- Decision records are stored in the repository and are easily discoverable for teams involved in the various language ports. + +## Considered Options + +- Callback Registration + Recursive +- Single Callback +- Event Based Registration +- Middleware + +## Pros and Cons of the Options + +### Callback Registration Recursive Delegate (Kernel, Plan, Function) + +- Specified on plan and function level as a configuration be able to specify what are the callback Handlers that will be triggered. + +Pros: + +- Common pattern for observing and also changing data exposed as parameter into the delegate signature for (Get/Set) scenarios +- Registering a callback gives back the registration object that can be used to cancel the execution of the function in the future. +- Recursive approach, allows to register multiple callbacks for the same event, and also allows to register callbacks on top of pre existing callbacks. + +Cons: + +- Registrations may use more memory and might not be garbage collected in the recursive approach, only when the function or the plan is disposed. + +### Single Callback Delegate (Kernel, Plan, Function) + +- Specified on kernel level as a configuration be able to specify what are the callback Handlers that will be triggered. + - Specified on function creation: As part of the function constructor be able to specify what are the callback Handlers that will be triggered. + - Specified on function invocation: As part of the function invoke be able to specify what are the callback Handlers as a parameter that will be triggered. + +Pros: + +- Common pattern for observing and also changing data exposed as parameter into the delegate signature for (Get/Set) scenarios + +Cons: + +- Limited to only one method observing a specific event (Pre Post and InExecution). - Function When used as parameter, three new parameters would be needed as part of the function. (Specified on function invocation) - Extra Cons on + +### Event Base Registration (Kernel only) + +Expose events on both IKernel and ISKFunction that the call can can be observing to interact. + +Pros: + +- Multiple Listeners can registered for the same event +- Listeners can be registered and unregistered at will +- Common pattern (EventArgs) for observing and also changing data exposed as parameter into the event signature for (Get/Set) scenarios + +Cons: + +- Event handlers are void, making the EventArgs by reference the only way to modify the data. +- Not clear how supportive is this approach for asynchronous pattern/multi threading +- Won't support `ISKFunction.InvokeAsync` + +### Middleware (Kernel Only) + +Specified on Kernel level, and would only be used using IKernel.RunAsync operation, this pattern would be similar to asp.net core middlewares, running the pipelines with a context and a requestdelegate next for controlling (Pre/Post conditions) + +Pros: + +- Common pattern for handling Pre/Post Setting/Filtering data + +Cons: + +- Functions can run on their own instance, middlewares suggest more complexity and the existence of an external container/manager (Kernel) to intercept/observe function calls. + +## Main Questions + +- Q: Post Execution Handlers should execute right after the LLM result or before the end of the function execution itself? + A: Currently post execution Handlers are executed after function execution. + +- Q: Should Pre/Post Handlers be many (pub/sub) allowing registration/deregistration? + A: By using the standard .NET event implementation, this already supports multiple registrations as well as deregistrations managed by the caller. + +- Q: Setting Handlers on top of pre existing Handlers should be allowed or throw an error? + A: By using the standard .NET event implementation, the stander behavior will not throw an error and will execute all the registered handlers. + +- Q: Setting Handlers on Plans should automatically cascade this Handlers for all the inner steps + overriding existing ones in the process? + A: Handlers will be triggered before and after each step is executed the same way the Kernel RunAsync pipeline works. + +- Q: When a pre execution handler cancel the execution of the function, + +- Q: When a pre function execution handler intents to cancel the execution, should further handlers in the chain be called or not? + A: Currently the standard .net behavior is to call all the registered handlers. This way function execution will solely depends on the final boolean state of the Cancel property after all handlers were called. + +## Decision Outcome + +TBD. diff --git a/docs/decisions/0005-open-api-dynamic-payload-and-namespaces.md b/docs/decisions/0005-open-api-dynamic-payload-and-namespaces.md new file mode 100644 index 000000000000..ffe87013c573 --- /dev/null +++ b/docs/decisions/0005-open-api-dynamic-payload-and-namespaces.md @@ -0,0 +1,79 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 2023-08-15 +deciders: shawncal +consulted: +informed: +--- +# Dynamic payload building for PUT and POST RestAPI operations and parameter namespacing + +## Context and Problem Statement +Currently, the SK OpenAPI does not allow the dynamic creation of payload/body for PUT and POST RestAPI operations, even though all the required metadata is available. One of the reasons the functionality was not fully developed originally, and eventually removed is that JSON payload/body content of PUT and POST RestAPI operations might contain properties with identical names at various levels. It was not clear how to unambiguously resolve their values from the flat list of context variables. Another reason the functionality has not been added yet is that the 'payload' context variable, along with RestAPI operation data contract schema(OpenAPI, JSON schema, Typings?) should have been sufficient for LLM to provide fully fleshed-out JSON payload/body content without the need to build it dynamically. + + +## Decision Drivers +* Create a mechanism that enables the dynamic construction of the payload/body for PUT and POST RestAPI operations. +* Develop a mechanism(namespacing) that allows differentiation of payload properties with identical names at various levels for PUT and POST RestAPI operations. +* Aim to minimize breaking changes and maintain backward compatibility of the code as much as possible. + +## Considered Options +* Enable the dynamic creation of payload and/or namespacing by default. +* Enable the dynamic creation of payload and/or namespacing based on configuration. + +## Decision Outcome +Chosen option: "Enable the dynamic creation of payload and/or namespacing based on configuration". This option keeps things compatible, so the change won't affect any SK consumer code. Additionally, it lets SK consumer code easily control both mechanisms, turning them on or off based on the scenario. + +## Additional details + +### Enabling dynamic creation of payload +In order to enable the dynamic creation of payloads/bodies for PUT and POST RestAPI operations, please set the `EnableDynamicPayload` property of the `OpenApiSkillExecutionParameters` execution parameters to `true` when importing the AI plugin: + +```csharp +var plugin = await kernel.ImportPluginFunctionsAsync("", new Uri(""), new OpenApiSkillExecutionParameters(httpClient) { EnableDynamicPayload = true }); +``` + +To dynamically construct a payload for a RestAPI operation that requires payload like this: +```json +{ + "value": "secret-value", + "attributes": { + "enabled": true + } +} +``` + +Please register the following arguments in context variables collection: + +```csharp +var contextVariables = new ContextVariables(); +contextVariables.Set("value", "secret-value"); +contextVariables.Set("enabled", true); +``` + +### Enabling namespacing +To enable namespacing, set the `EnablePayloadNamespacing` property of the `OpenApiSkillExecutionParameters` execution parameters to `true` when importing the AI plugin: + +```csharp +var plugin = await kernel.ImportPluginFunctionsAsync("", new Uri(""), new OpenApiSkillExecutionParameters(httpClient) { EnablePayloadNamespacing = true }); +``` +Remember that the namespacing mechanism depends on prefixing parameter names with their parent parameter name, separated by dots. So, use the 'namespaced' parameter names when adding arguments for them to the context variables. Let's consider this JSON: + +```json +{ + "upn": "", + "receiver": { + "upn": "" + }, + "cc": { + "upn": "" + } +} +``` +It contains `upn` properties at different levels. The the argument registration for the parameters(property values) will look like: +```csharp +var contextVariables = new ContextVariables(); +contextVariables.Set("upn", ""); +contextVariables.Set("receiver.upn", ""); +contextVariables.Set("cc.upn", ""); +``` diff --git a/docs/decisions/0006-prompt-extract-template-engine.md b/docs/decisions/0006-prompt-extract-template-engine.md new file mode 100644 index 000000000000..a72f5adc1413 --- /dev/null +++ b/docs/decisions/0006-prompt-extract-template-engine.md @@ -0,0 +1,32 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: accepted +date: 2023-08-25 +deciders: shawncal +consulted: +informed: +--- +# Extract the Prompt Template Engine from Semantic Kernel core + +## Context and Problem Statement + +The Semantic Kernel includes a default prompt template engine which is used to render Semantic Kernel prompts i.e., `skprompt.txt` files. The prompt template is rendered before being send to the AI to allow the prompt to be generated dynamically e.g., include input parameters or the result of a native or semantic function execution. +To reduce the complexity and API surface of the Semantic Kernel the prompt template engine is going to be extracted and added to it's own package. + +The long term goal is to enable the following scenarios: + +1. Implement a custom template engine e.g., using Handlebars templates. This is supported now but we want to simplify the API to be implemented. +2. Support using zero or many template engines. + +## Decision Drivers + +* Reduce API surface and complexity of the Semantic Kernel core. +* Simplify the `IPromptTemplateEngine` interface to make it easier to implement a custom template engine. +* Make the change without breaking existing clients. + +## Decision Outcome + +* Create a new package called `Microsoft.SemanticKernel.TemplateEngine`. +* Maintain the existing namespace for all prompt template engine code. +* Simplify the `IPromptTemplateEngine` interface to just require implementation of `RenderAsync`. +* Dynamically load the existing `PromptTemplateEngine` if the `Microsoft.SemanticKernel.TemplateEngine` assembly is available. diff --git a/docs/decisions/0007-support-multiple-named-args-in-template-function-calls.md b/docs/decisions/0007-support-multiple-named-args-in-template-function-calls.md new file mode 100644 index 000000000000..412d0901a406 --- /dev/null +++ b/docs/decisions/0007-support-multiple-named-args-in-template-function-calls.md @@ -0,0 +1,110 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 6/16/2023 +deciders: shawncal, hario90 +consulted: dmytrostruk, matthewbolanos +informed: lemillermicrosoft +--- +# Add support for multiple named arguments in template function calls + +## Context and Problem Statement + +Native functions now support multiple parameters, populated from context values with the same name. Semantic functions currently only support calling native functions with no more than 1 argument. The purpose of these changes is to add support for calling native functions within semantic functions with multiple named arguments. + +## Decision Drivers + +* Parity with Guidance +* Readability +* Similarity to languages familiar to SK developers +* YAML compatibility + +## Considered Options + +### Syntax idea 1: Using commas + +```handlebars +{{Skill.MyFunction street: "123 Main St", zip: "98123", city:"Seattle", age: 25}} +``` + +Pros: + +* Commas could make longer function calls easier to read, especially if spaces before and after the arg separator (a colon in this case) are allowed. + +Cons: + +* Guidance doesn't use commas +* Spaces are already used as delimiters elsewhere so the added complexity of supporting commas isn't necessary + +### Syntax idea 2: JavaScript/C#-Style delimiter (colon) + +```handlebars + +{{MyFunction street:"123 Main St" zip:"98123" city:"Seattle" age: "25"}} + +``` + +Pros: + +* Resembles JavaScript Object syntax and C# named argument syntax + +Cons: + +* Doesn't align with Guidance syntax which uses equal signs as arg part delimiters +* Too similar to YAML key/value pairs if we support YAML prompts in the future. It's likely possible to support colons as delimiters but would be better to have a separator that is distinct from normal YAML syntax. + +### Syntax idea 3: Python/Guidance-Style delimiter + +```handlebars +{{MyFunction street="123 Main St" zip="98123" city="Seattle"}} +``` + +Pros: + +* Resembles Python's keyword argument syntax +* Resembles Guidance's named argument syntax +* Not too similar to YAML key/value pairs if we support YAML prompts in the future. + +Cons: + +* Doesn't align with C# syntax + +### Syntax idea 4: Allow whitespace between arg name/value delimiter + +```handlebars +{{MyFunction street = "123 Main St" zip = "98123" city = "Seattle"}} +``` + +Pros: + +* Follows the convention followed by many programming languages of whitespace flexibility where spaces, tabs, and newlines within code don't impact a program's functionality + +Cons: + +* Promotes code that is harder to read unless commas can be used (see [Using Commas](#syntax-idea-1-using-commas)) +* More complexity to support +* Doesn't align with Guidance which doesn't support spaces before and after the = sign. + +## Decision Outcome + +Chosen options: "Syntax idea 3: Python/Guidance-Style keyword arguments", because it aligns well with Guidance's syntax and is the most compatible with YAML and "Syntax idea 4: Allow whitespace between arg name/value delimiter" for more flexible developer experience. + +Additional decisions: + +* Continue supporting up to 1 positional argument for backward compatibility. Currently, the argument passed to a function is assumed to be the `$input` context variable. + +Example + +```handlebars + +{{MyFunction "inputVal" street="123 Main St" zip="98123" city="Seattle"}} + +``` + +* Allow arg values to be defined as strings or variables ONLY, e.g. + +```handlebars +{{MyFunction street=$street zip="98123" city='Seattle'}} +``` + +If function expects a value other than a string for an argument, the SDK will use the corresponding TypeConverter to parse the string provided when evaluating the expression. diff --git a/docs/decisions/0008-support-generic-llm-request-settings.md b/docs/decisions/0008-support-generic-llm-request-settings.md new file mode 100644 index 000000000000..98f11afff66c --- /dev/null +++ b/docs/decisions/0008-support-generic-llm-request-settings.md @@ -0,0 +1,231 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 2023-=9-15 +deciders: shawncal +consulted: stoub, lemiller, dmytrostruk +informed: +--- +# Refactor to support generic LLM request settings + +## Context and Problem Statement + +The Semantic Kernel abstractions package includes a number of classes ([CompleteRequestSettings](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/CompleteRequestSettings.cs), [ChatRequestSettings](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs) [PromptTemplateConfig.CompletionConfig](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/PromptTemplateConfig.cs#L18C1-L82C6)) which are used to support: + +1. Passing LLM request settings when invoking an AI service +2. Deserialization of LLM requesting settings when loading the `config.json` associated with a Semantic Function + +The problem with these classes is they include OpenAI specific properties only. A developer can only pass OpenAI specific requesting settings which means: + +1. Settings may be passed that have no effect e.g., passing `MaxTokens` to Huggingface +2. Settings that do not overlap with the OpenAI properties cannot be sent e.g., Oobabooga supports additional parameters e.g., `do_sample`, `typical_p`, ... + +Link to issue raised by the implementer of the Oobabooga AI service: + +## Decision Drivers + +* Semantic Kernel abstractions must be AI Service agnostic i.e., remove OpenAI specific properties. +* Solution must continue to support loading Semantic Function configuration (which includes AI request settings) from `config.json`. +* Provide good experience for developers e.g., must be able to program with type safety, intellisense, etc. +* Provide a good experience for implementors of AI services i.e., should be clear how to define the appropriate AI Request Settings abstraction for the service they are supporting. +* Semantic Kernel implementation and sample code should avoid specifying OpenAI specific request settings in code that is intended to be used with multiple AI services. +* Semantic Kernel implementation and sample code must be clear if an implementation is intended to be OpenAI specific. + +## Considered Options + +* Use `dynamic` to pass request settings +* Use `object` to pass request settings +* Define a base class for AI request settings which all implementations must extend + +Note: Using generics was discounted during an earlier investigation which Dmytro conducted. + +## Decision Outcome + +**Proposed:** Define a base class for AI request settings which all implementations must extend. + +## Pros and Cons of the Options + +### Use `dynamic` to pass request settings + +The `IChatCompletion` interface would look like this: + +```csharp +public interface IChatCompletion : IAIService +{ + ChatHistory CreateNewChat(string? instructions = null); + + Task> GetChatCompletionsAsync( + ChatHistory chat, + dynamic? requestSettings = null, + CancellationToken cancellationToken = default); + + IAsyncEnumerable GetStreamingChatCompletionsAsync( + ChatHistory chat, + dynamic? requestSettings = null, + CancellationToken cancellationToken = default); +} +``` + +Developers would have the following options to specify the requesting settings for a semantic function: + +```csharp +// Option 1: Use an anonymous type +await kernel.InvokeSemanticFunctionAsync("Hello AI, what can you do for me?", requestSettings: new { MaxTokens = 256, Temperature = 0.7 }); + +// Option 2: Use an OpenAI specific class +await kernel.InvokeSemanticFunctionAsync(prompt, requestSettings: new OpenAIRequestSettings() { MaxTokens = 256, Temperature = 0.7 }); + +// Option 3: Load prompt template configuration from a JSON payload +string configPayload = @"{ + ""schema"": 1, + ""description"": ""Say hello to an AI"", + ""type"": ""completion"", + ""completion"": { + ""max_tokens"": 60, + ""temperature"": 0.5, + ""top_p"": 0.0, + ""presence_penalty"": 0.0, + ""frequency_penalty"": 0.0 + } +}"; +var templateConfig = JsonSerializer.Deserialize(configPayload); +var func = kernel.CreateSemanticFunction(prompt, config: templateConfig!, "HelloAI"); +await kernel.RunAsync(func); +``` + +PR: + +* Good, SK abstractions contain no references to OpenAI specific request settings +* Neutral, because anonymous types can be used which allows a developer to pass in properties that may be supported by multiple AI services e.g., `temperature` or combine properties for different AI services e.g., `max_tokens` (OpenAI) and `max_new_tokens` (Oobabooga). +* Bad, because it's not clear to developers what they should pass when creating a semantic function +* Bad, because it's not clear to implementors of a chat/text completion service what they should accept or how to add service specific properties. +* Bad, there is no compiler type checking for code paths where the dynamic argument has not been resolved which will impact code quality. Type issues manifest as `RuntimeBinderException`'s and may be difficult to troubleshoot. Special care needs to be taken with return types e.g., may be necessary to specify an explicit type rather than just `var` again to avoid errors such as `Microsoft.CSharp.RuntimeBinder.RuntimeBinderException : Cannot apply indexing with [] to an expression of type 'object'` + +### Use `object` to pass request settings + +The `IChatCompletion` interface would look like this: + +```csharp +public interface IChatCompletion : IAIService +{ + ChatHistory CreateNewChat(string? instructions = null); + + Task> GetChatCompletionsAsync( + ChatHistory chat, + object? requestSettings = null, + CancellationToken cancellationToken = default); + + IAsyncEnumerable GetStreamingChatCompletionsAsync( + ChatHistory chat, + object? requestSettings = null, + CancellationToken cancellationToken = default); +} +``` + +The calling pattern is the same as for the `dynamic` case i.e. use either an anonymous type, an AI service specific class e.g., `OpenAIRequestSettings` or load from JSON. + +PR: + +* Good, SK abstractions contain no references to OpenAI specific request settings +* Neutral, because anonymous types can be used which allows a developer to pass in properties that may be supported by multiple AI services e.g., `temperature` or combine properties for different AI services e.g., `max_tokens` (OpenAI) and `max_new_tokens` (Oobabooga). +* Bad, because it's not clear to developers what they should pass when creating a semantic function +* Bad, because it's not clear to implementors of a chat/text completion service what they should accept or how to add service specific properties. +* Bad, code is needed to perform type checks and explicit casts. The situation is slightly better than for the `dynamic` case. + +### Define a base class for AI request settings which all implementations must extend + +The `IChatCompletion` interface would look like this: + +```csharp +public interface IChatCompletion : IAIService +{ + ChatHistory CreateNewChat(string? instructions = null); + + Task> GetChatCompletionsAsync( + ChatHistory chat, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); + + IAsyncEnumerable GetStreamingChatCompletionsAsync( + ChatHistory chat, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); +} +``` + +`AIRequestSettings` is defined as follows: + +```csharp +public class AIRequestSettings +{ + /// + /// Service identifier. + /// + [JsonPropertyName("service_id")] + [JsonPropertyOrder(1)] + public string? ServiceId { get; set; } = null; + + /// + /// Extra properties + /// + [JsonExtensionData] + public Dictionary? ExtensionData { get; set; } +} +``` + +Developers would have the following options to specify the requesting settings for a semantic function: + +```csharp +// Option 1: Invoke the semantic function and pass an OpenAI specific instance +var result = await kernel.InvokeSemanticFunctionAsync(prompt, requestSettings: new OpenAIRequestSettings() { MaxTokens = 256, Temperature = 0.7 }); +Console.WriteLine(result.Result); + +// Option 2: Load prompt template configuration from a JSON payload +string configPayload = @"{ + ""schema"": 1, + ""description"": ""Say hello to an AI"", + ""type"": ""completion"", + ""completion"": { + ""max_tokens"": 60, + ""temperature"": 0.5, + ""top_p"": 0.0, + ""presence_penalty"": 0.0, + ""frequency_penalty"": 0.0 + } +}"; +var templateConfig = JsonSerializer.Deserialize(configPayload); +var func = kernel.CreateSemanticFunction(prompt, config: templateConfig!, "HelloAI"); + +await kernel.RunAsync(func); +``` + +It would also be possible to use the following pattern: + +```csharp +this._summarizeConversationFunction = kernel.CreateSemanticFunction( + SemanticFunctionConstants.SummarizeConversationDefinition, + skillName: nameof(ConversationSummarySkill), + description: "Given a section of a conversation, summarize conversation.", + requestSettings: new AIRequestSettings() + { + ExtensionData = new Dictionary() + { + { "Temperature", 0.1 }, + { "TopP", 0.5 }, + { "MaxTokens", MaxTokens } + } + }); + +``` + +The caveat with this pattern is, assuming a more specific implementation of `AIRequestSettings` uses JSON serialization/deserialization to hydrate an instance from the base `AIRequestSettings`, this will only work if all properties are supported by the default JsonConverter e.g., + +* If we have `MyAIRequestSettings` which includes a `Uri` property. The implementation of `MyAIRequestSettings` would make sure to load a URI converter so that it can serialize/deserialize the settings correctly. +* If the settings for `MyAIRequestSettings` are sent to an AI service which relies on the default JsonConverter then a `NotSupportedException` exception will be thrown. + +PR: + +* Good, SK abstractions contain no references to OpenAI specific request settings +* Good, because it is clear to developers what they should pass when creating a semantic function and it is easy to discover what service specific request setting implementations exist. +* Good, because it is clear to implementors of a chat/text completion service what they should accept and how to extend the base abstraction to add service specific properties. +* Neutral, because `ExtensionData` can be used which allows a developer to pass in properties that may be supported by multiple AI services e.g., `temperature` or combine properties for different AI services e.g., `max_tokens` (OpenAI) and `max_new_tokens` (Oobabooga). diff --git a/docs/decisions/0009-dotnet-project-structure.md b/docs/decisions/0009-dotnet-project-structure.md new file mode 100644 index 000000000000..42a929266f23 --- /dev/null +++ b/docs/decisions/0009-dotnet-project-structure.md @@ -0,0 +1,291 @@ +--- + +# These are optional elements. Feel free to remove any of them + +status: accepted +date: 2023-09-29 +deciders: semenshi, dmytrostruk, rbarreto +consulted: shawncal, stoub, lemiller +informed: {list everyone who is kept up-to-date on progress; and with whom there is a one-way communication} +--- + +# DotNet Project Structure for 1.0 Release + +## Context and Problem Statement + +- Provide a cohesive, well-defined set of assemblies that developers can easily combine based on their needs. + - Semantic Kernel core should only contain functionality related to AI orchestration + - Remove prompt template engine and semantic functions + - Semantic Kernel abstractions should only interfaces, abstract classes and minimal classes to support these +- Remove `Skills` naming from NuGet packages and replace with `Plugins` + - Clearly distinguish between plugin implementations (`Skills.MsGraph`) and plugin integration (`Skills.OpenAPI`) +- Have consistent naming for assemblies and their root namespaces + - See [Naming Patterns](#naming-patterns) section for examples of current patterns + +## Decision Drivers + +- Avoid having too many assemblies because of impact of signing these and to reduce complexity +- Follow .Net naming guidelines + - [Names of Assemblies and DLLs](https://learn.microsoft.com/en-us/dotnet/standard/design-guidelines/names-of-assemblies-and-dlls) + - [Names of Namespaces](https://learn.microsoft.com/en-us/dotnet/standard/design-guidelines/names-of-namespaces) + +## Considered Options + +- Option #1: New `planning`, `functions` and `plugins` project areas +- Option #2: Folder naming matches assembly name + +In all cases the following changes will be made: + +- Move non core Connectors to a separate repository +- Merge prompt template engine and semantic functions into a single package + +## Decision Outcome + +Chosen option: Option #2: Folder naming matches assembly name, because: + +1. It provides a way for developers to easily discover where code for a particular assembly is located +1. It is consistent with other e.g., [azure-sdk-for-net](https://github.com/Azure/azure-sdk-for-net) + +Main categories for the projects will be: + +1. `Connectors`: ***A connector project allows the Semantic Kernel to connect to AI and Memory services***. Some of the existing connector projects may move to other repositories. +1. `Planners`: ***A planner project provides one or more planner implementations which take an ask and convert it into an executable plan to achieve that ask***. This category will include the current action, sequential and stepwise planners (these could be merged into a single project). Additional planning implementations e.g., planners that generate Powershell or Python code can be added as separate projects. +1. `Functions`: ***A function project that enables the Semantic Kernel to access the functions it will orchestrate***. This category will include: + 1. Semantic functions i.e., prompts executed against an LLM + 1. GRPC remote procedures i.e., procedures executed remotely using the GRPC framework + 1. Open API endpoints i.e., REST endpoints that have Open API definitions executed remotely using the HTTP protocol +1. `Plugins`: ***A plugin project contains the implementation(s) of a Semantic Kernel plugin***. A Semantic Kernel plugin is contains a concrete implementation of a function e.g., a plugin may include code for basic text operations. + +### Option #1: New `planning`, `functions` and `plugins` project areas + +```text +SK-dotnet +├── samples/ +└── src/ + ├── connectors/ + │ ├── Connectors.AI.OpenAI* + │ ├── Connectors.AI.HuggingFace + │ ├── Connectors.Memory.AzureCognitiveSearch + │ ├── Connectors.Memory.Qdrant + │ ├── ... + │ └── Connectors.UnitTests + ├── planners/ + │ ├── Planners.Action* + │ ├── Planners.Sequential* + │ └── Planners.Stepwise* + ├── functions/ + │ ├── Functions.Native* + │ ├── Functions.Semantic* + │ ├── Functions.Planning* + │ ├── Functions.Grpc + │ ├── Functions.OpenAPI + │ └── Functions.UnitTests + ├── plugins/ + │ ├── Plugins.Core* + │ ├── Plugins.Document + │ ├── Plugins.MsGraph + │ ├── Plugins.WebSearch + │ └── Plugins.UnitTests + ├── InternalUtilities/ + ├── IntegrationTests + ├── SemanticKernel* + ├── SemanticKernel.Abstractions* + ├── SemanticKernel.MetaPackage + └── SemanticKernel.UnitTests +``` + +### Changes + +| Project | Description | +|-------------------------------------|-------------| +| `Functions.Native` | Extract native functions from Semantic Kernel core and abstractions. | +| `Functions.Semantic` | Extract semantic functions from Semantic Kernel core and abstractions. Include the prompt template engine. | +| `Functions.Planning` | Extract planning from Semantic Kernel core and abstractions. | +| `Functions.Grpc` | Old `Skills.Grpc` project | +| `Functions.OpenAPI` | Old `Skills.OpenAPI` project | +| `Plugins.Core` | Old `Skills.Core` project | +| `Plugins.Document` | Old `Skills.Document` project | +| `Plugins.MsGraph` | Old `Skills.MsGraph` project | +| `Plugins.WebSearch` | Old `Skills.WebSearch` project | + +### Semantic Kernel Skills and Functions + +This diagram how functions and plugins would be integrated with the Semantic Kernel core. + +ISKFunction class relationships + +### Option #2: Folder naming matches assembly name + +```text +SK-dotnet +├── samples/ +└── libraries/ + ├── SK-dotnet.sln + │ + ├── Microsoft.SemanticKernel.Connectors.AI.OpenAI* + │ ├── src + │ └── tests + │ (Not shown but all projects will have src and tests subfolders) + ├── Microsoft.SemanticKernel.Connectors.AI.HuggingFace + ├── Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch + ├── Microsoft.SemanticKernel.Connectors.Memory.Qdrant + │ + ├── Microsoft.SemanticKernel.Planners* + │ + ├── Microsoft.SemanticKernel.Reliability.Basic* + ├── Microsoft.SemanticKernel.Reliability.Polly + │ + ├── Microsoft.SemanticKernel.TemplateEngines.Basic* + │ + ├── Microsoft.SemanticKernel.Functions.Semantic* + ├── Microsoft.SemanticKernel.Functions.Grpc + ├── Microsoft.SemanticKernel.Functions.OpenAPI + │ + ├── Microsoft.SemanticKernel.Plugins.Core* + ├── Microsoft.SemanticKernel.Plugins.Document + ├── Microsoft.SemanticKernel.Plugins.MsGraph + ├── Microsoft.SemanticKernel.Plugins.Web + │ + ├── InternalUtilities + │ + ├── IntegrationTests + │ + ├── Microsoft.SemanticKernel.Core* + ├── Microsoft.SemanticKernel.Abstractions* + └── Microsoft.SemanticKernel.MetaPackage +``` + +***Notes:*** + +- There will only be a single solution file (initially). +- Projects will be grouped in the solution i.e., connectors, planners, plugins, functions, extensions, ... +- Each project folder contains a `src` and `tests` folder. +- There will be a gradual process to move existing unit tests to the correct location as some projects will need to be broken up. + +## More Information + +### Current Project Structure + +```text +SK-dotnet +├── samples/ +└── src/ + ├── connectors/ + │ ├── Connectors.AI.OpenAI* + │ ├── Connectors... + │ └── Connectors.UnitTests + ├── extensions/ + │ ├── Planner.ActionPlanner* + │ ├── Planner.SequentialPlanner* + │ ├── Planner.StepwisePlanner + │ ├── TemplateEngine.PromptTemplateEngine* + │ └── Extensions.UnitTests + ├── InternalUtilities/ + ├── skills/ + │ ├── Skills.Core + │ ├── Skills.Document + │ ├── Skills.Grpc + │ ├── Skills.MsGraph + │ ├── Skills.OpenAPI + │ ├── Skills.Web + │ └── Skills.UnitTests + ├── IntegrationTests + ├── SemanticKernel* + ├── SemanticKernel.Abstractions* + ├── SemanticKernel.MetaPackage + └── SemanticKernel.UnitTests +``` + +\\* - Means the project is part of the Semantic Kernel meta package + +### Project Descriptions + +| Project | Description | +|-------------------------------------|-------------| +| Connectors.AI.OpenAI | Azure OpenAI and OpenAI service connectors | +| Connectors... | Collection of other AI service connectors, some of which will move to another repository | +| Connectors.UnitTests | Connector unit tests | +| Planner.ActionPlanner | Semantic Kernel implementation of an action planner | +| Planner.SequentialPlanner | Semantic Kernel implementation of a sequential planner | +| Planner.StepwisePlanner | Semantic Kernel implementation of a stepwise planner | +| TemplateEngine.Basic | Prompt template engine basic implementations which are used by Semantic Functions only | +| Extensions.UnitTests | Extensions unit tests | +| InternalUtilities | Internal utilities which are reused by multiple NuGet packages (all internal) | +| Skills.Core | Core set of native functions which are provided to support Semantic Functions | +| Skills.Document | Native functions for interacting with Microsoft documents | +| Skills.Grpc | Semantic Kernel integration for GRPC based endpoints | +| Skills.MsGraph | Native functions for interacting with Microsoft Graph endpoints | +| Skills.OpenAPI | Semantic Kernel integration for OpenAI endpoints and reference Azure Key Vault implementation | +| Skills.Web | Native functions for interacting with Web endpoints e.g., Bing, Google, File download | +| Skills.UnitTests | Skills unit tests | +| IntegrationTests | Semantic Kernel integration tests | +| SemanticKernel | Semantic Kernel core implementation | +| SemanticKernel.Abstractions | Semantic Kernel abstractions i.e., interface, abstract classes, supporting classes, ... | +| SemanticKernel.MetaPackage | Semantic Kernel meta package i.e., a NuGet package that references other required Semantic Kernel NuGet packages | +| SemanticKernel.UnitTests | Semantic Kernel unit tests | + +### Naming Patterns + +Below are some different examples of Assembly and root namespace naming that are used in the projects. + +```xml + Microsoft.SemanticKernel.Abstractions + Microsoft.SemanticKernel + + Microsoft.SemanticKernel.Core + Microsoft.SemanticKernel + + Microsoft.SemanticKernel.Planning.ActionPlanner + Microsoft.SemanticKernel.Planning.Action + + Microsoft.SemanticKernel.Skills.Core + $(AssemblyName) +``` + +### Current Folder Structure + +```text +dotnet/ +├── samples/ +│ ├── ApplicationInsightsExample/ +│ ├── KernelSyntaxExamples/ +│ └── NCalcSkills/ +└── src/ + ├── Connectors/ + │ ├── Connectors.AI.OpenAI* + │ ├── Connectors... + │ └── Connectors.UnitTests + ├── Extensions/ + │ ├── Planner.ActionPlanner + │ ├── Planner.SequentialPlanner + │ ├── Planner.StepwisePlanner + │ ├── TemplateEngine.PromptTemplateEngine + │ └── Extensions.UnitTests + ├── InternalUtilities/ + ├── Skills/ + │ ├── Skills.Core + │ ├── Skills.Document + │ ├── Skills.Grpc + │ ├── Skills.MsGraph + │ ├── Skills.OpenAPI + │ ├── Skills.Web + │ └── Skills.UnitTests + ├── IntegrationTests/ + ├── SemanticKernel/ + ├── SemanticKerne.Abstractions/ + ├── SemanticKernel.MetaPackage/ + └── SemanticKernel.UnitTests/ + +``` + +### Semantic Kernel Skills and Functions + +This diagram show current skills are integrated with the Semantic Kernel core. + +***Note:*** + +- This is not a true class hierarchy diagram. It show some class relationships and dependencies. +- Namespaces are abbreviated to remove Microsoft.SemanticKernel prefix. Namespaces use `_` rather than `.`. + +ISKFunction class relationships + diff --git a/docs/decisions/0009-function-and-kernel-result-types.md b/docs/decisions/0009-function-and-kernel-result-types.md new file mode 100644 index 000000000000..bf5d6552837b --- /dev/null +++ b/docs/decisions/0009-function-and-kernel-result-types.md @@ -0,0 +1,84 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: accepted +date: 2023-09-21 +deciders: shawncal, dmytrostruk +consulted: +informed: +--- +# Replace SKContext as Function/Kernel result type with FunctionResult and KernelResult models + +## Context and Problem Statement + +Methods `function.InvokeAsync` and `kernel.RunAsync` return `SKContext` as result type. This has several problems: + +1. `SKContext` contains property `Result`, which is `string`. Based on that, it's not possible to return complex type or implement streaming capability in Kernel. +2. `SKContext` contains property `ModelResults`, which is coupled to LLM-specific logic, so it's only applicable to semantic functions in specific cases. +3. `SKContext` as a mechanism of passing information between functions in pipeline should be internal implementation. Caller of Kernel should provide input/request and receive some result, but not `SKContext`. +4. `SKContext` contains information related to the last executed function without a way to access information about specific function in pipeline. + +## Decision Drivers + +1. Kernel should be able to return complex type as well as support streaming capability. +2. Kernel should be able to return data related to function execution (e.g. amount of tokens used) in a way, when it's not coupled to AI logic. +3. `SKContext` should work as internal mechanism of passing information between functions. +4. There should be a way how to differentiate function result from kernel result, since these entities are different by nature and may contain different set of properties in the future. +5. The possibility to access specific function result in the middle of pipeline will provide more insights to the users how their functions performed. + +## Considered Options + +1. Use `dynamic` as return type - this option provides some flexibility, but on the other hand removes strong typing, which is preferred option in .NET world. Also, there will be no way how to differentiate function result from Kernel result. +2. Define new types - `FunctionResult` and `KernelResult` - chosen approach. + +## Decision Outcome + +New `FunctionResult` and `KernelResult` return types should cover scenarios like returning complex types from functions, supporting streaming and possibility to access result of each function separately. + +### Complex Types and Streaming + +For complex types and streaming, property `object Value` will be defined in `FunctionResult` to store single function result, and in `KernelResult` to store result from last function in execution pipeline. For better usability, generic method `GetValue` will allow to cast `object Value` to specific type. + +Examples: + +```csharp +// string +var text = (await kernel.RunAsync(function)).GetValue(); + +// complex type +var myComplexType = (await kernel.RunAsync(function)).GetValue(); + +// streaming +var results = (await kernel.RunAsync(function)).GetValue>(); + +await foreach (var result in results) +{ + Console.WriteLine(result); +} +``` + +When `FunctionResult`/`KernelResult` will store `TypeA` and caller will try to cast it to `TypeB` - in this case `InvalidCastException` will be thrown with details about types. This will provide some information to the caller which type should be used for casting. + +### Metadata + +To return additional information related to function execution - property `Dictionary Metadata` will be added to `FunctionResult`. This will allow to pass any kind of information to the caller, which should provide some insights how function performed (e.g. amount of tokens used, AI model response etc.) + +Examples: + +```csharp +var functionResult = await function.InvokeAsync(context); +Console.WriteLine(functionResult.Metadata["MyInfo"]); +``` + +### Multiple function results + +`KernelResult` will contain collection of function results - `IReadOnlyCollection FunctionResults`. This will allow to get specific function result from `KernelResult`. Properties `FunctionName` and `PluginName` in `FunctionResult` will help to get specific function from collection. + +Example: + +```csharp +var kernelResult = await kernel.RunAsync(function1, function2, function3); + +var functionResult2 = kernelResult.FunctionResults.First(l => l.FunctionName == "Function2" && l.PluginName == "MyPlugin"); + +Assert.Equal("Result2", functionResult2.GetValue()); +``` diff --git a/docs/decisions/0010-openai-function-calling.md b/docs/decisions/0010-openai-function-calling.md new file mode 100644 index 000000000000..06844ca2afa1 --- /dev/null +++ b/docs/decisions/0010-openai-function-calling.md @@ -0,0 +1,68 @@ +--- +status: proposed +date: 2023-09-21 +deciders: gitri-ms, shawncal +consulted: lemillermicrosoft, awharrison-28, dmytrostruk, nacharya1 +informed: eavanvalkenburg, kevdome3000 +--- +# OpenAI Function Calling Support + +## Context and Problem Statement + +The [function calling](https://platform.openai.com/docs/guides/gpt/function-calling) capability of OpenAI's Chat Completions API allows developers to describe functions to the model, and have the model decide whether to output a JSON object specifying a function and appropriate arguments to call in response to the given prompt. This capability is enabled by two new API parameters to the `/v1/chat/completions` endpoint: +- `function_call` - auto (default), none, or a specific function to call +- `functions` - JSON descriptions of the functions available to the model + +Functions provided to the model are injected as part of the system message and are billed/counted as input tokens. + +We have received several community requests to provide support for this capability when using SK with the OpenAI chat completion models that support it. + +## Decision Drivers + +* Minimize changes to the core kernel for OpenAI-specific functionality +* Cost concerns with including a long list of function descriptions in the request +* Security and cost concerns with automatically executing functions returned by the model + +## Considered Options + +* Support sending/receiving functions via chat completions endpoint _with_ modifications to interfaces +* Support sending/receiving functions via chat completions endpoint _without_ modifications to interfaces +* Implement a planner around the function calling capability + +## Decision Outcome + +Chosen option: "Support sending/receiving functions via chat completions endpoint _without_ modifications to interfaces" + +With this option, we utilize the existing request settings object to send functions to the model. The app developer controls what functions are included and is responsible for validating and executing the function result. + +### Consequences + +* Good, because avoids breaking changes to the core kernel +* Good, because OpenAI-specific functionality is contained to the OpenAI connector package +* Good, because allows app to control what functions are available to the model (including non-SK functions) +* Good, because keeps the option open for integrating with planners in the future +* Neutral, because requires app developer to validate and execute resulting function +* Bad, because not as obvious how to use this capability and access the function results + +## Pros and Cons of the Options + +### Support sending/receiving functions _with_ modifications to chat completions interfaces + +This option would update the `IChatCompletion` and `IChatResult` interfaces to expose parameters/methods for providing and accessing function information. + +* Good, because provides a clear path for using the function calling capability +* Good, because allows app to control what functions are available to the model (including non-SK functions) +* Neutral, because requires app developer to validate and execute resulting function +* Bad, because introduces breaking changes to core kernel abstractions +* Bad, because OpenAI-specific functionality would be included in core kernel abstractions and would need to be ignored by other model providers + +### Implement a planner around the function calling capability + +Orchestrating external function calls fits within SK's concept of planning. With this approach, we would implement a planner that would take the function calling result and produce a plan that the app developer could execute (similar to SK's ActionPlanner). + +* Good, because producing a plan result makes it easy for the app developer to execute the chosen function +* Bad, because functions would need to be registered with the kernel in order to be executed +* Bad, because would create confusion about when to use which planner + +## Additional notes +There has been much discussion and debate over the pros and cons of automatically invoking a function returned by the OpenAI model, if it is registered with the kernel. As there are still many open questions around this behavior and its implications, we have decided to not include this capability in the initial implementation. We will continue to explore this option and may include it in a future update. \ No newline at end of file diff --git a/docs/decisions/0011-memory-as-plugin.md b/docs/decisions/0011-memory-as-plugin.md new file mode 100644 index 000000000000..99d8e0955c43 --- /dev/null +++ b/docs/decisions/0011-memory-as-plugin.md @@ -0,0 +1,45 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 2023-09-21 +deciders: shawncal, dmytrostruk +consulted: +informed: +--- +# Move all Memory-related logic to separate Plugin + +## Context and Problem Statement + +Memory-related logic is located across different C# projects: + +- `SemanticKernel.Abstractions` + - `IMemoryStore` + - `ISemanticTextMemory` + - `MemoryRecord` + - `NullMemory` +- `SemanticKernel.Core` + - `MemoryConfiguration` + - `SemanticTextMemory` + - `VolatileMemoryStore` +- `Plugins.Core` + - `TextMemoryPlugin` + +Property `ISemanticTextMemory Memory` is also part of `Kernel` type, but kernel itself doesn't use it. This property is needed to inject Memory capabilities in Plugins. At the moment, `ISemanticTextMemory` interface is main dependency of `TextMemoryPlugin`, and in some examples `TextMemoryPlugin` is initialized as `new TextMemoryPlugin(kernel.Memory)`. + +While this approach works for Memory, there is no way how to inject `MathPlugin` into other Plugin at the moment. Following the same approach and adding `Math` property to `Kernel` type is not scalable solution, as it's not possible to define separate properties for each available Plugin. + +## Decision Drivers + +1. Memory should not be a property of `Kernel` type if it's not used by the kernel. +2. Memory should be treated in the same way as other plugins or services, that may be required by specific Plugins. +3. There should be a way how to register Memory capability with attached Vector DB and inject that capability in Plugins that require it. + +## Decision Outcome + +Move all Memory-related logic to separate project called `Plugins.Memory`. This will allow to simplify Kernel logic and use Memory in places where it's needed (other Plugins). + +High-level tasks: + +1. Move Memory-related code to separate project. +2. Implement a way how to inject Memory in Plugins that require it. +3. Remove `Memory` property from `Kernel` type. diff --git a/docs/decisions/0012-kernel-service-registration.md b/docs/decisions/0012-kernel-service-registration.md new file mode 100644 index 000000000000..050bd13b50b5 --- /dev/null +++ b/docs/decisions/0012-kernel-service-registration.md @@ -0,0 +1,180 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 2023-10-03 +deciders: dmytrostruk +consulted: semenshi, rbarreto, markwallace +informed: +--- +# Kernel Service Registration + +## Context and Problem Statement + +Plugins may have dependencies to support complex scenarios. For example, there is `TextMemoryPlugin`, which supports functions like `retrieve`, `recall`, `save`, `remove`. Constructor is implemented in following way: + +```csharp +public TextMemoryPlugin(ISemanticTextMemory memory) +{ + this._memory = memory; +} +``` + +`TextMemoryPlugin` depends on `ISemanticTextMemory` interface. In similar way, other Plugins may have multiple dependencies and there should be a way how to resolve required dependencies manually or automatically. + +At the moment, `ISemanticTextMemory` is a property of `IKernel` interface, which allows to inject `ISemanticTextMemory` into `TextMemoryPlugin` during Plugin initialization: + +```csharp +kernel.ImportFunctions(new TextMemoryPlugin(kernel.Memory)); +``` + +There should be a way how to support not only Memory-related interface, but any kind of service, which can be used in Plugin - `ISemanticTextMemory`, `IPromptTemplateEngine`, `IDelegatingHandlerFactory` or any other service. + +## Considered Options + +### Solution #1.1 (available by default) + +User is responsible for all Plugins initialization and dependency resolution with **manual** approach. + +```csharp +var memoryStore = new VolatileMemoryStore(); +var embeddingGeneration = new OpenAITextEmbeddingGeneration(modelId, apiKey); +var semanticTextMemory = new SemanticTextMemory(memoryStore, embeddingGeneration); + +var memoryPlugin = new TextMemoryPlugin(semanticTextMemory); + +var kernel = Kernel.Builder.Build(); + +kernel.ImportFunctions(memoryPlugin); +``` + +Note: this is native .NET approach how to resolve service dependencies manually, and this approach should always be available by default. Any other solutions which could help to improve dependency resolution can be added on top of this approach. + +### Solution #1.2 (available by default) + +User is responsible for all Plugins initialization and dependency resolution with **dependency injection** approach. + +```csharp +var serviceCollection = new ServiceCollection(); + +serviceCollection.AddTransient(); +serviceCollection.AddTransient( + (serviceProvider) => new OpenAITextEmbeddingGeneration(modelId, apiKey)); + +serviceCollection.AddTransient(); + +var services = serviceCollection.BuildServiceProvider(); + +// In theory, TextMemoryPlugin can be also registered in DI container. +var memoryPlugin = new TextMemoryPlugin(services.GetService()); + +var kernel = Kernel.Builder.Build(); + +kernel.ImportFunctions(memoryPlugin); +``` + +Note: in similar way as Solution #1.1, this way should be supported out of the box. Users always can handle all the dependencies on their side and just provide required Plugins to Kernel. + +### Solution #2.1 + +Custom service collection and service provider on Kernel level to simplify dependency resolution process, as addition to Solution #1.1 and Solution #1.2. + +Interface `IKernel` will have its own service provider `KernelServiceProvider` with minimal functionality to get required service. + +```csharp +public interface IKernelServiceProvider +{ + T? GetService(string? name = null); +} + +public interface IKernel +{ + IKernelServiceProvider Services { get; } +} +``` + +```csharp +var kernel = Kernel.Builder + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAITextEmbeddingGenerationService(modelId, apiKey) + .WithService(), + .WithService() + .Build(); + +var semanticTextMemory = kernel.Services.GetService(); +var memoryPlugin = new TextMemoryPlugin(semanticTextMemory); + +kernel.ImportFunctions(memoryPlugin); +``` + +Pros: + +- No dependency on specific DI container library. +- Lightweight implementation. +- Possibility to register only those services that can be used by Plugins (isolation from host application). +- Possibility to register same interface multiple times by **name**. + +Cons: + +- Implementation and maintenance for custom DI container, instead of using already existing libraries. +- To import Plugin, it still needs to be initialized manually to inject specific service. + +### Solution #2.2 + +This solution is an improvement for last disadvantage of Solution #2.1 to handle case, when Plugin instance should be initialized manually. This will require to add new way how to import Plugin into Kernel - not with object **instance**, but with object **type**. In this case, Kernel will be responsible for `TextMemoryPlugin` initialization and injection of all required dependencies from custom service collection. + +```csharp +// Instead of this +var semanticTextMemory = kernel.Services.GetService(); +var memoryPlugin = new TextMemoryPlugin(semanticTextMemory); + +kernel.ImportFunctions(memoryPlugin); + +// Use this +kernel.ImportFunctions(); +``` + +### Solution #3 + +Instead of custom service collection and service provider in Kernel, use already existing DI library - `Microsoft.Extensions.DependencyInjection`. + +```csharp +var serviceCollection = new ServiceCollection(); + +serviceCollection.AddTransient(); +serviceCollection.AddTransient( + (serviceProvider) => new OpenAITextEmbeddingGeneration(modelId, apiKey)); + +serviceCollection.AddTransient(); + +var services = serviceCollection.BuildServiceProvider(); + +var kernel = Kernel.Builder + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAITextEmbeddingGenerationService(modelId, apiKey) + .WithServices(services) // Pass all registered services from host application to Kernel + .Build(); + +// Plugin Import - option #1 +var semanticTextMemory = kernel.Services.GetService(); +var memoryPlugin = new TextMemoryPlugin(semanticTextMemory); + +kernel.ImportFunctions(memoryPlugin); + +// Plugin Import - option #2 +kernel.ImportFunctions(); +``` + +Pros: + +- No implementation is required for dependency resolution - just use already existing .NET library. +- The possibility to inject all registered services at once in already existing applications and use them as Plugin dependencies. + +Cons: + +- Additional dependency for Semantic Kernel package - `Microsoft.Extensions.DependencyInjection`. +- No possibility to include specific list of services (lack of isolation from host application). +- Possibility of `Microsoft.Extensions.DependencyInjection` version mismatch and runtime errors (e.g. users have `Microsoft.Extensions.DependencyInjection` `--version 2.0` while Semantic Kernel uses `--version 6.0`) + +## Decision Outcome + +As for now, support Solution #1.1 and Solution #1.2 only, to keep Kernel as unit of single responsibility. Plugin dependencies should be resolved before passing Plugin instance to the Kernel. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index fe06013125ee..0de7ecc1a989 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -1,7 +1,8 @@ -# Markdown Any Decision Records +# Architectural Decision Records (ADRs) -MADR is a lean template to capture any decisions in a structured way. The template originated from capturing architectural decisions and developed to a template allowing to capture any decisions taken. -For more information [see](https://adr.github.io/madr/) +An Architectural Decision (AD) is a justified software design choice that addresses a functional or non-functional requirement that is architecturally significant. An Architectural Decision Record (ADR) captures a single AD and its rationale. + +For more information [see](https://adr.github.io/) ## How are we using ADR's to track technical decisions? diff --git a/docs/decisions/diagrams/skfunctions-preview.mmd b/docs/decisions/diagrams/skfunctions-preview.mmd new file mode 100644 index 000000000000..e0642cdb9b6a --- /dev/null +++ b/docs/decisions/diagrams/skfunctions-preview.mmd @@ -0,0 +1,68 @@ +--- +title: Semantic Kernel Functions (preview) +--- +classDiagram + %% Use https://mermaid.live/ to preview this diagram. The VS Code extension does not handle namespaces. + direction RL + namespace SkillDefinition { + class ISKFunction { + <> + String : Name + String : SkillName + String : Description + CompleteRequestSettings : RequestSettings + Describe(...) + InvokeAsync(...) + SetDefaultSkillCollection(...) + SetAIService(...) + SetAIConfiguration(...) + } + class NativeFunction + class SemanticFunction + } + + namespace Skills_Grpc { + class KernelGrpcExtensions + } + + namespace Skills_OpenApi { + class KernelOpenApiExtensions + } + + namespace Skills_MsGraph { + class CalendarSkill + } + + namespace Skills_Web { + class SearchUrlSkill + } + + namespace Skills_Document { + class DocumentSkill + } + + namespace Skills_Core { + class TextSkill + class ConversationSummarySkill + } + + namespace Planning { + class Plan + class ActionPlanner + class SequentialPlanner + class StepwisePlanner + } + + ISKFunction <|.. NativeFunction + ISKFunction <|.. SemanticFunction + ISKFunction <|.. Plan + NativeFunction <.. KernelGrpcExtensions + NativeFunction <.. KernelOpenApiExtensions + NativeFunction <.. CalendarSkill + NativeFunction <.. SearchUrlSkill + NativeFunction <.. DocumentSkill + NativeFunction <.. TextSkill + SemanticFunction <.. ConversationSummarySkill + Plan <.. ActionPlanner + Plan <.. SequentialPlanner + Plan <.. StepwisePlanner diff --git a/docs/decisions/diagrams/skfunctions-preview.png b/docs/decisions/diagrams/skfunctions-preview.png new file mode 100644 index 000000000000..49f2189c1302 Binary files /dev/null and b/docs/decisions/diagrams/skfunctions-preview.png differ diff --git a/docs/decisions/diagrams/skfunctions-v1.mmd b/docs/decisions/diagrams/skfunctions-v1.mmd new file mode 100644 index 000000000000..e64cc9db91fe --- /dev/null +++ b/docs/decisions/diagrams/skfunctions-v1.mmd @@ -0,0 +1,77 @@ +--- +title: Semantic Kernel Functions (v1.0) +--- +classDiagram + %% Use https://mermaid.live/ to preview this diagram. The VS Code extension does not handle namespaces. + direction RL + namespace Function { + class ISKFunction { + <> + String : Name + String : SkillName + String : Description + JsonObject : Configuration + Describe(...) + InvokeAsync(...) + SetPluginProvider(...) + SetAIServiceProvider(...) + SetConfiguration(...) + } + } + + namespace Functions_Native { + class NativeFunction + } + + namespace Functions_Semantic { + class SemanticFunction + } + + namespace Functions_Planning { + class Plan + } + + namespace Functions_Grpc { + class KernelGrpcExtensions + } + + namespace Functions_OpenApi { + class KernelOpenApiExtensions + } + + namespace Plugins_MsGraph { + class CalendarPlugin + } + + namespace Plugins_Web { + class SearchUrlPlugin + } + + namespace Plugins_Document { + class DocumentPlugin + } + + namespace Plugins_Core { + class TextPlugin + class ConversationSummaryPlugin + } + + namespace Planners { + class ActionPlanner + class SequentialPlanner + class StepwisePlanner + } + + ISKFunction <|.. NativeFunction + ISKFunction <|.. SemanticFunction + ISKFunction <|.. Plan + NativeFunction .. KernelGrpcExtensions + NativeFunction .. KernelOpenApiExtensions + NativeFunction .. CalendarPlugin + NativeFunction .. SearchUrlPlugin + NativeFunction .. DocumentPlugin + NativeFunction .. TextPlugin + SemanticFunction .. ConversationSummaryPlugin + Plan <.. ActionPlanner + Plan <.. SequentialPlanner + Plan <.. StepwisePlanner diff --git a/docs/decisions/diagrams/skfunctions-v1.png b/docs/decisions/diagrams/skfunctions-v1.png new file mode 100644 index 000000000000..e7f569d0c63f Binary files /dev/null and b/docs/decisions/diagrams/skfunctions-v1.png differ diff --git a/dotnet/Directory.Build.props b/dotnet/Directory.Build.props index 504a41201560..66b7b6667062 100644 --- a/dotnet/Directory.Build.props +++ b/dotnet/Directory.Build.props @@ -16,13 +16,8 @@ disable - - true - full - - - - portable + + True diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 4fa0edafc6bb..cfdaca52cec4 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -5,19 +5,24 @@ true - - - + + + + + - - - - + + + + + + + @@ -29,18 +34,20 @@ - - + + - - + + + - + + - - - + + + @@ -48,26 +55,17 @@ - + + + + + - all @@ -78,12 +76,12 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/dotnet/README.md b/dotnet/README.md index 7329454fd18f..28ca20228fb7 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -4,8 +4,8 @@ To run the LLM prompts and semantic functions in the examples below, make sure you have an -[Open AI API Key](https://openai.com/api/) or -[Azure Open AI service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=rest-api). +[OpenAI API Key](https://openai.com/product/) or +[Azure OpenAI Service Key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=rest-api). ## Nuget package @@ -22,17 +22,18 @@ Copy and paste the following code into your project, with your Azure OpenAI key ```csharp using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; var builder = new KernelBuilder(); -builder.WithAzureTextCompletionService( - "text-davinci-003", // Azure OpenAI Deployment Name +builder.WithAzureChatCompletionService( + "gpt-35-turbo", // Azure OpenAI Deployment Name "https://contoso.openai.azure.com/", // Azure OpenAI Endpoint "...your Azure OpenAI Key..."); // Azure OpenAI Key // Alternative using OpenAI -//builder.WithOpenAITextCompletionService( -// "text-davinci-003", // OpenAI Model name +//builder.WithOpenAIChatCompletionService( +// "gpt-3.5-turbo", // OpenAI Model name // "...your OpenAI API Key..."); // OpenAI API Key var kernel = builder.Build(); @@ -41,7 +42,7 @@ var prompt = @"{{$input}} One line TLDR with the fewest words."; -var summarize = kernel.CreateSemanticFunction(prompt); +var summarize = kernel.CreateSemanticFunction(prompt, requestSettings: new OpenAIRequestSettings { MaxTokens = 100 }); string text1 = @" 1st Law of Thermodynamics - Energy cannot be created or destroyed. @@ -53,9 +54,9 @@ string text2 = @" 2. The acceleration of an object depends on the mass of the object and the amount of force applied. 3. Whenever one object exerts a force on another object, the second object exerts an equal and opposite on the first."; -Console.WriteLine(await summarize.InvokeAsync(text1)); +Console.WriteLine(await kernel.RunAsync(text1, summarize)); -Console.WriteLine(await summarize.InvokeAsync(text2)); +Console.WriteLine(await kernel.RunAsync(text2, summarize)); // Output: // Energy conserved, entropy increases, zero entropy at 0K. @@ -80,8 +81,8 @@ string summarizePrompt = @"{{$input}} Give me a TLDR with the fewest words."; -var translator = kernel.CreateSemanticFunction(translationPrompt); -var summarize = kernel.CreateSemanticFunction(summarizePrompt); +var translator = kernel.CreateSemanticFunction(translationPrompt, requestSettings: new OpenAIRequestSettings { MaxTokens = 200 }); +var summarize = kernel.CreateSemanticFunction(summarizePrompt, requestSettings: new OpenAIRequestSettings { MaxTokens = 100 }); string inputText = @" 1st Law of Thermodynamics - Energy cannot be created or destroyed. @@ -101,27 +102,27 @@ Console.WriteLine(output); The repository contains also a few C# Jupyter notebooks that demonstrates how to get started with the Semantic Kernel. -See [here](../samples/notebooks/dotnet/README.md) for the full list, with +See [here](./notebooks/README.md) for the full list, with requirements and setup instructions. -1. [Getting started](../samples/notebooks//dotnet/00-getting-started.ipynb) -2. [Loading and configuring Semantic Kernel](../samples/notebooks//dotnet/01-basic-loading-the-kernel.ipynb) -3. [Running AI prompts from file](../samples/notebooks//dotnet/02-running-prompts-from-file.ipynb) -4. [Creating Semantic Functions at runtime (i.e. inline functions)](../samples/notebooks//dotnet/03-semantic-function-inline.ipynb) -5. [Using Context Variables to Build a Chat Experience](../samples/notebooks//dotnet/04-context-variables-chat.ipynb) -6. [Creating and Executing Plans](../samples/notebooks//dotnet/05-using-the-planner.ipynb) -7. [Building Memory with Embeddings](../samples/notebooks//dotnet/06-memory-and-embeddings.ipynb) -8. [Creating images with DALL-E 2](../samples/notebooks//dotnet/07-DALL-E-2.ipynb) -9. [Chatting with ChatGPT and Images](../samples/notebooks//dotnet/08-chatGPT-with-DALL-E-2.ipynb) +1. [Getting started](./notebooks/00-getting-started.ipynb) +2. [Loading and configuring Semantic Kernel](./notebooks/01-basic-loading-the-kernel.ipynb) +3. [Running AI prompts from file](./notebooks/02-running-prompts-from-file.ipynb) +4. [Creating Semantic Functions at runtime (i.e. inline functions)](./notebooks/03-semantic-function-inline.ipynb) +5. [Using Context Variables to Build a Chat Experience](./notebooks/04-context-variables-chat.ipynb) +6. [Creating and Executing Plans](./notebooks/05-using-the-planner.ipynb) +7. [Building Memory with Embeddings](./notebooks/06-memory-and-embeddings.ipynb) +8. [Creating images with DALL-E 2](./notebooks/07-DALL-E-2.ipynb) +9. [Chatting with ChatGPT and Images](./notebooks/08-chatGPT-with-DALL-E-2.ipynb) # Nuget packages Semantic Kernel provides a set of nuget packages to allow extending the core with -more features, such as connectors to services and Skills to perform specific actions. +more features, such as connectors to services and plugins to perform specific actions. Unless you need to optimize which packages to include in your app, you will usually start by installing this meta-package first: -* **Microsoft.SemanticKernel** +- **Microsoft.SemanticKernel** This meta package includes core packages and OpenAI connectors, allowing to run most samples and build apps with OpenAI and Azure OpenAI. @@ -129,23 +130,24 @@ most samples and build apps with OpenAI and Azure OpenAI. Packages included in **Microsoft.SemanticKernel**: 1. **Microsoft.SemanticKernel.Abstractions**: contains common interfaces and classes - used by the core and other SK components. + used by the core and other SK components. 1. **Microsoft.SemanticKernel.Core**: contains the core logic of SK, such as prompt - engineering, semantic memory and semantic functions definition and orchestration. + engineering, semantic memory and semantic functions definition and orchestration. 1. **Microsoft.SemanticKernel.Connectors.AI.OpenAI**: connectors to OpenAI and Azure - OpenAI, allowing to run semantic functions, chats, image generation with GPT3, - GPT3.5, GPT4, DALL-E2. Includes also GPT tokenizers. + OpenAI, allowing to run semantic functions, chats, image generation with GPT3, + GPT3.5, GPT4, DALL-E2. Other SK packages available at nuget.org: 1. **Microsoft.SemanticKernel.Connectors.Memory.Qdrant**: Qdrant connector for - skills and semantic memory. + plugins and semantic memory. 2. **Microsoft.SemanticKernel.Connectors.Memory.Sqlite**: SQLite connector for - skills and semantic memory -3. **Microsoft.SemanticKernel.Skills.Document**: Document Skill: Word processing, + plugins and semantic memory +3. **Microsoft.SemanticKernel.Plugins.Document**: Document Plugin: Word processing, OpenXML, etc. -4. **Microsoft.SemanticKernel.Skills.MsGraph**: Microsoft Graph Skill: access your +4. **Microsoft.SemanticKernel.Plugins.MsGraph**: Microsoft Graph Plugin: access your tenant data, schedule meetings, send emails, etc. -5. **Microsoft.SemanticKernel.Skills.OpenAPI**: OpenAPI skill. -6. **Microsoft.SemanticKernel.Skills.Web**: Web Skill: search the web, download - files, etc. \ No newline at end of file +5. **Microsoft.SemanticKernel.Plugins.OpenAPI**: OpenAPI Plugin. +6. **Microsoft.SemanticKernel.Plugins.Web**: Web Plugin: search the web, download + files, etc. +7. **Microsoft.SemanticKernel.Reliability.Polly**: Extension for http resiliency. diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index b6cc713e2268..45273e9ad18c 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -2,7 +2,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.4.33213.308 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel", "src\SemanticKernel\SemanticKernel.csproj", "{A284C7EB-2248-4A75-B112-F5DCDE65410D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.Core", "src\SemanticKernel.Core\SemanticKernel.Core.csproj", "{A284C7EB-2248-4A75-B112-F5DCDE65410D}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0}" EndProject @@ -15,17 +15,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{FA37 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KernelSyntaxExamples", "samples\KernelSyntaxExamples\KernelSyntaxExamples.csproj", "{47C6F821-5103-431F-B3B8-A2868A68BB78}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MsGraphSkillsExample", "..\samples\dotnet\graph-api-skills\MsGraphSkillsExample.csproj", "{3EB61E99-C39B-4620-9482-F8DA18E48525}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MsGraphPluginsExample", "..\samples\dotnet\MsGraphPluginsExample\MsGraphPluginsExample.csproj", "{3EB61E99-C39B-4620-9482-F8DA18E48525}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KernelHttpServer", "..\samples\dotnet\KernelHttpServer\KernelHttpServer.csproj", "{34A7F1EF-D243-4160-A413-D713FEABCD94}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IntegrationTests", "src\IntegrationTests\IntegrationTests.csproj", "{E4B777A1-28E1-41BE-96AE-7F3EC61FD5D4}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Skills.Document", "src\Skills\Skills.Document\Skills.Document.csproj", "{F94D1938-9DB7-4B24-9FF3-166DDFD96330}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Plugins.Document", "src\Plugins\Plugins.Document\Plugins.Document.csproj", "{F94D1938-9DB7-4B24-9FF3-166DDFD96330}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Skills.MsGraph", "src\Skills\Skills.MsGraph\Skills.MsGraph.csproj", "{689A5041-BAE7-448F-9BDC-4672E96249AA}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Plugins.MsGraph", "src\Plugins\Plugins.MsGraph\Plugins.MsGraph.csproj", "{689A5041-BAE7-448F-9BDC-4672E96249AA}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Skills.Web", "src\Skills\Skills.Web\Skills.Web.csproj", "{EEA87FBC-4ED5-458C-ABD3-BEAEEB535BAF}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Plugins.Web", "src\Plugins\Plugins.Web\Plugins.Web.csproj", "{EEA87FBC-4ED5-458C-ABD3-BEAEEB535BAF}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{158A4E5E-AEE0-4D60-83C7-8E089B2D881D}" ProjectSection(SolutionItems) = preProject @@ -40,9 +40,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.UnitTests", "src\SemanticKernel.UnitTests\SemanticKernel.UnitTests.csproj", "{37E39C68-5A40-4E63-9D3C-0C66AD98DFCB}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "skills", "skills", "{9ECD1AA0-75B3-4E25-B0B5-9F0945B64974}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "functions", "functions", "{9ECD1AA0-75B3-4E25-B0B5-9F0945B64974}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Skills.UnitTests", "src\Skills\Skills.UnitTests\Skills.UnitTests.csproj", "{107156B4-5A8B-45C7-97A2-4544D7FA19DE}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Functions.UnitTests", "src\Functions\Functions.UnitTests\Functions.UnitTests.csproj", "{107156B4-5A8B-45C7-97A2-4544D7FA19DE}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "nuget", "nuget", "{F4243136-252A-4459-A7C4-EE8C056D6B0B}" ProjectSection(SolutionItems) = preProject @@ -51,7 +51,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "nuget", "nuget", "{F4243136 nuget\NUGET.md = nuget\NUGET.md EndProjectSection EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Skills.OpenAPI", "src\Skills\Skills.OpenAPI\Skills.OpenAPI.csproj", "{F2A1F81E-700E-4C0E-B021-B9EF29AA20BD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Functions.OpenAPI", "src\Functions\Functions.OpenAPI\Functions.OpenAPI.csproj", "{F2A1F81E-700E-4C0E-B021-B9EF29AA20BD}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "connectors", "connectors", "{0247C2C9-86C3-45BA-8873-28B0948EDC0C}" EndProject @@ -61,8 +61,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Qdrant", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Sqlite", "src\Connectors\Connectors.Memory.Sqlite\Connectors.Memory.Sqlite.csproj", "{EC004F12-2F60-4EDD-B3CD-3A504900D929}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.CosmosDB", "src\Connectors\Connectors.Memory.CosmosDB\Connectors.Memory.CosmosDB.csproj", "{EA61C289-7928-4B78-A9C1-7AAD61F907CD}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Postgres", "src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj", "{C9F957FA-A70F-4A6D-8F95-23FCD7F4FB87}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Redis", "src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj", "{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}" @@ -75,19 +73,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.Abstractions EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.MetaPackage", "src\SemanticKernel.MetaPackage\SemanticKernel.MetaPackage.csproj", "{E3299033-EB81-4C4C-BCD9-E8DC40937969}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planning.ActionPlanner", "src\Extensions\Planning.ActionPlanner\Planning.ActionPlanner.csproj", "{994BEF0B-E277-4D10-BB13-FE670D26620D}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "extensions", "extensions", "{078F96B4-09E1-4E0E-B214-F71A4F4BF633}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Extensions.UnitTests", "src\Extensions\Extensions.UnitTests\Extensions.UnitTests.csproj", "{F51017A9-15C8-472D-893C-080046D710A6}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planning.SequentialPlanner", "src\Extensions\Planning.SequentialPlanner\Planning.SequentialPlanner.csproj", "{A350933D-F9D5-4AD3-8C4F-B856B5020297}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.AzureCognitiveSearch", "src\Connectors\Connectors.Memory.AzureCognitiveSearch\Connectors.Memory.AzureCognitiveSearch.csproj", "{EC3BB6D1-2FB2-4702-84C6-F791DE533ED4}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Pinecone", "src\Connectors\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj", "{4D226C2F-AE9F-4EFB-AF2D-45C8FE5CB34E}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Skills.Grpc", "src\Skills\Skills.Grpc\Skills.Grpc.csproj", "{E52F805C-794A-4CA9-B684-DFF358B18820}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Functions.Grpc", "src\Functions\Functions.Grpc\Functions.Grpc.csproj", "{E52F805C-794A-4CA9-B684-DFF358B18820}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AI.HuggingFace", "src\Connectors\Connectors.AI.HuggingFace\Connectors.AI.HuggingFace.csproj", "{136823BE-8665-4D57-87E0-EF41535539E2}" EndProject @@ -95,7 +89,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "InternalUtilities", "Intern EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Weaviate", "src\Connectors\Connectors.Memory.Weaviate\Connectors.Memory.Weaviate.csproj", "{6AAB0620-33A1-4A98-A63B-6560B9BA47A4}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenApiSkillsExample", "..\samples\dotnet\openapi-skills\OpenApiSkillsExample.csproj", "{4D91A3E0-C404-495B-AD4A-411C4E83CF54}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenApiPluginsExample", "..\samples\dotnet\OpenApiPluginsExample\OpenApiPluginsExample.csproj", "{4D91A3E0-C404-495B-AD4A-411C4E83CF54}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.DuckDB", "src\Connectors\Connectors.Memory.DuckDB\Connectors.Memory.DuckDB.csproj", "{50FAE231-6F24-4779-9D02-12ABBC9A49E2}" EndProject @@ -137,16 +131,41 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Http", "Http", "{1C19D805-3 src\InternalUtilities\src\Http\NonDisposableHttpClientHandler.cs = src\InternalUtilities\src\Http\NonDisposableHttpClientHandler.cs EndProjectSection EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Skills.Core", "src\Skills\Skills.Core\Skills.Core.csproj", "{0D0C4DAD-E6BC-4504-AE3A-EEA4E35920C1}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "System", "System", "{3CDE10B2-AE8F-4FC4-8D55-92D4AD32E144}" + ProjectSection(SolutionItems) = preProject + src\InternalUtilities\src\System\EnvExtensions.cs = src\InternalUtilities\src\System\EnvExtensions.cs + EndProjectSection EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NCalcSkills", "samples\NCalcSkills\NCalcSkills.csproj", "{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Plugins.Core", "src\Plugins\Plugins.Core\Plugins.Core.csproj", "{0D0C4DAD-E6BC-4504-AE3A-EEA4E35920C1}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AI.Oobabooga", "src\Connectors\Connectors.AI.Oobabooga\Connectors.AI.Oobabooga.csproj", "{677F1381-7830-4115-9C1A-58B282629DC6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NCalcPlugins", "samples\NCalcPlugins\NCalcPlugins.csproj", "{E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planning.StepwisePlanner", "src\Extensions\Planning.StepwisePlanner\Planning.StepwisePlanner.csproj", "{4762BCAF-E1C5-4714-B88D-E50FA333C50E}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AI.Oobabooga", "src\Connectors\Connectors.AI.Oobabooga\Connectors.AI.Oobabooga.csproj", "{677F1381-7830-4115-9C1A-58B282629DC6}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApplicationInsightsExample", "samples\ApplicationInsightsExample\ApplicationInsightsExample.csproj", "{C754950A-E16C-4F96-9CC7-9328E361B5AF}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Kusto", "src\Connectors\Connectors.Memory.Kusto\Connectors.Memory.Kusto.csproj", "{E07608CC-D710-4655-BB9E-D22CF3CDD193}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TemplateEngine.Basic", "src\Extensions\TemplateEngine.Basic\TemplateEngine.Basic.csproj", "{10E4B697-D4E8-468D-872D-49670FD150FB}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Reliability.Polly", "src\Extensions\Reliability.Polly\Reliability.Polly.csproj", "{D4540A0F-98E3-4E70-9093-1948AE5B2AAD}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Reliability.Basic", "src\Extensions\Reliability.Basic\Reliability.Basic.csproj", "{3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "plugins", "plugins", "{D6D598DF-C17C-46F4-B2B9-CDE82E2DE132}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Plugins.UnitTests", "src\Plugins\Plugins.UnitTests\Plugins.UnitTests.csproj", "{5CB78CE4-895B-4A14-98AA-716A37DEEBB1}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "planners", "planners", "{A21FAC7C-0C09-4EAD-843B-926ACEF73C80}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planners.Core", "src\Planners\Planners.Core\Planners.Core.csproj", "{F224B869-FA0E-4EEE-A6BF-C2D61FF8E731}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planners.Core.UnitTests", "src\Planners\Planners.Core.UnitTests\Planners.Core.UnitTests.csproj", "{CC77DCFA-A419-4202-A98A-868CDF457792}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Milvus", "src\Connectors\Connectors.Memory.Milvus\Connectors.Memory.Milvus.csproj", "{8B754E80-7A97-4585-8D7E-1D588FA5F727}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Plugins.Memory", "src\Plugins\Plugins.Memory\Plugins.Memory.csproj", "{E91365A1-8B01-4AB8-825F-67E3515EADCD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -235,11 +254,6 @@ Global {EC004F12-2F60-4EDD-B3CD-3A504900D929}.Publish|Any CPU.Build.0 = Publish|Any CPU {EC004F12-2F60-4EDD-B3CD-3A504900D929}.Release|Any CPU.ActiveCfg = Release|Any CPU {EC004F12-2F60-4EDD-B3CD-3A504900D929}.Release|Any CPU.Build.0 = Release|Any CPU - {EA61C289-7928-4B78-A9C1-7AAD61F907CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EA61C289-7928-4B78-A9C1-7AAD61F907CD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EA61C289-7928-4B78-A9C1-7AAD61F907CD}.Publish|Any CPU.ActiveCfg = Release|Any CPU - {EA61C289-7928-4B78-A9C1-7AAD61F907CD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EA61C289-7928-4B78-A9C1-7AAD61F907CD}.Release|Any CPU.Build.0 = Release|Any CPU {C9F957FA-A70F-4A6D-8F95-23FCD7F4FB87}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C9F957FA-A70F-4A6D-8F95-23FCD7F4FB87}.Debug|Any CPU.Build.0 = Debug|Any CPU {C9F957FA-A70F-4A6D-8F95-23FCD7F4FB87}.Publish|Any CPU.ActiveCfg = Publish|Any CPU @@ -276,24 +290,12 @@ Global {E3299033-EB81-4C4C-BCD9-E8DC40937969}.Publish|Any CPU.Build.0 = Publish|Any CPU {E3299033-EB81-4C4C-BCD9-E8DC40937969}.Release|Any CPU.ActiveCfg = Release|Any CPU {E3299033-EB81-4C4C-BCD9-E8DC40937969}.Release|Any CPU.Build.0 = Release|Any CPU - {994BEF0B-E277-4D10-BB13-FE670D26620D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {994BEF0B-E277-4D10-BB13-FE670D26620D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {994BEF0B-E277-4D10-BB13-FE670D26620D}.Publish|Any CPU.ActiveCfg = Publish|Any CPU - {994BEF0B-E277-4D10-BB13-FE670D26620D}.Publish|Any CPU.Build.0 = Publish|Any CPU - {994BEF0B-E277-4D10-BB13-FE670D26620D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {994BEF0B-E277-4D10-BB13-FE670D26620D}.Release|Any CPU.Build.0 = Release|Any CPU {F51017A9-15C8-472D-893C-080046D710A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F51017A9-15C8-472D-893C-080046D710A6}.Debug|Any CPU.Build.0 = Debug|Any CPU {F51017A9-15C8-472D-893C-080046D710A6}.Publish|Any CPU.ActiveCfg = Release|Any CPU {F51017A9-15C8-472D-893C-080046D710A6}.Publish|Any CPU.Build.0 = Release|Any CPU {F51017A9-15C8-472D-893C-080046D710A6}.Release|Any CPU.ActiveCfg = Release|Any CPU {F51017A9-15C8-472D-893C-080046D710A6}.Release|Any CPU.Build.0 = Release|Any CPU - {A350933D-F9D5-4AD3-8C4F-B856B5020297}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A350933D-F9D5-4AD3-8C4F-B856B5020297}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A350933D-F9D5-4AD3-8C4F-B856B5020297}.Publish|Any CPU.ActiveCfg = Publish|Any CPU - {A350933D-F9D5-4AD3-8C4F-B856B5020297}.Publish|Any CPU.Build.0 = Publish|Any CPU - {A350933D-F9D5-4AD3-8C4F-B856B5020297}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A350933D-F9D5-4AD3-8C4F-B856B5020297}.Release|Any CPU.Build.0 = Release|Any CPU {EC3BB6D1-2FB2-4702-84C6-F791DE533ED4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {EC3BB6D1-2FB2-4702-84C6-F791DE533ED4}.Debug|Any CPU.Build.0 = Debug|Any CPU {EC3BB6D1-2FB2-4702-84C6-F791DE533ED4}.Publish|Any CPU.ActiveCfg = Publish|Any CPU @@ -352,17 +354,65 @@ Global {677F1381-7830-4115-9C1A-58B282629DC6}.Publish|Any CPU.Build.0 = Publish|Any CPU {677F1381-7830-4115-9C1A-58B282629DC6}.Release|Any CPU.ActiveCfg = Release|Any CPU {677F1381-7830-4115-9C1A-58B282629DC6}.Release|Any CPU.Build.0 = Release|Any CPU - {4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Publish|Any CPU.ActiveCfg = Publish|Any CPU - {4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Publish|Any CPU.Build.0 = Publish|Any CPU - {4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4762BCAF-E1C5-4714-B88D-E50FA333C50E}.Release|Any CPU.Build.0 = Release|Any CPU {C754950A-E16C-4F96-9CC7-9328E361B5AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C754950A-E16C-4F96-9CC7-9328E361B5AF}.Debug|Any CPU.Build.0 = Debug|Any CPU {C754950A-E16C-4F96-9CC7-9328E361B5AF}.Publish|Any CPU.ActiveCfg = Release|Any CPU {C754950A-E16C-4F96-9CC7-9328E361B5AF}.Release|Any CPU.ActiveCfg = Release|Any CPU {C754950A-E16C-4F96-9CC7-9328E361B5AF}.Release|Any CPU.Build.0 = Release|Any CPU + {E07608CC-D710-4655-BB9E-D22CF3CDD193}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E07608CC-D710-4655-BB9E-D22CF3CDD193}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E07608CC-D710-4655-BB9E-D22CF3CDD193}.Publish|Any CPU.ActiveCfg = Publish|Any CPU + {E07608CC-D710-4655-BB9E-D22CF3CDD193}.Publish|Any CPU.Build.0 = Publish|Any CPU + {E07608CC-D710-4655-BB9E-D22CF3CDD193}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E07608CC-D710-4655-BB9E-D22CF3CDD193}.Release|Any CPU.Build.0 = Release|Any CPU + {10E4B697-D4E8-468D-872D-49670FD150FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10E4B697-D4E8-468D-872D-49670FD150FB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10E4B697-D4E8-468D-872D-49670FD150FB}.Publish|Any CPU.ActiveCfg = Publish|Any CPU + {10E4B697-D4E8-468D-872D-49670FD150FB}.Publish|Any CPU.Build.0 = Publish|Any CPU + {10E4B697-D4E8-468D-872D-49670FD150FB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10E4B697-D4E8-468D-872D-49670FD150FB}.Release|Any CPU.Build.0 = Release|Any CPU + {D4540A0F-98E3-4E70-9093-1948AE5B2AAD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D4540A0F-98E3-4E70-9093-1948AE5B2AAD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D4540A0F-98E3-4E70-9093-1948AE5B2AAD}.Publish|Any CPU.ActiveCfg = Publish|Any CPU + {D4540A0F-98E3-4E70-9093-1948AE5B2AAD}.Publish|Any CPU.Build.0 = Publish|Any CPU + {D4540A0F-98E3-4E70-9093-1948AE5B2AAD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D4540A0F-98E3-4E70-9093-1948AE5B2AAD}.Release|Any CPU.Build.0 = Release|Any CPU + {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Publish|Any CPU.ActiveCfg = Publish|Any CPU + {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Publish|Any CPU.Build.0 = Publish|Any CPU + {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567}.Release|Any CPU.Build.0 = Release|Any CPU + {5CB78CE4-895B-4A14-98AA-716A37DEEBB1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5CB78CE4-895B-4A14-98AA-716A37DEEBB1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5CB78CE4-895B-4A14-98AA-716A37DEEBB1}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {5CB78CE4-895B-4A14-98AA-716A37DEEBB1}.Publish|Any CPU.Build.0 = Debug|Any CPU + {5CB78CE4-895B-4A14-98AA-716A37DEEBB1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5CB78CE4-895B-4A14-98AA-716A37DEEBB1}.Release|Any CPU.Build.0 = Release|Any CPU + {F224B869-FA0E-4EEE-A6BF-C2D61FF8E731}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F224B869-FA0E-4EEE-A6BF-C2D61FF8E731}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F224B869-FA0E-4EEE-A6BF-C2D61FF8E731}.Publish|Any CPU.ActiveCfg = Publish|Any CPU + {F224B869-FA0E-4EEE-A6BF-C2D61FF8E731}.Publish|Any CPU.Build.0 = Publish|Any CPU + {F224B869-FA0E-4EEE-A6BF-C2D61FF8E731}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F224B869-FA0E-4EEE-A6BF-C2D61FF8E731}.Release|Any CPU.Build.0 = Release|Any CPU + {CC77DCFA-A419-4202-A98A-868CDF457792}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CC77DCFA-A419-4202-A98A-868CDF457792}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CC77DCFA-A419-4202-A98A-868CDF457792}.Publish|Any CPU.ActiveCfg = Release|Any CPU + {CC77DCFA-A419-4202-A98A-868CDF457792}.Publish|Any CPU.Build.0 = Release|Any CPU + {CC77DCFA-A419-4202-A98A-868CDF457792}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CC77DCFA-A419-4202-A98A-868CDF457792}.Release|Any CPU.Build.0 = Release|Any CPU + {8B754E80-7A97-4585-8D7E-1D588FA5F727}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B754E80-7A97-4585-8D7E-1D588FA5F727}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B754E80-7A97-4585-8D7E-1D588FA5F727}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {8B754E80-7A97-4585-8D7E-1D588FA5F727}.Publish|Any CPU.Build.0 = Debug|Any CPU + {8B754E80-7A97-4585-8D7E-1D588FA5F727}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B754E80-7A97-4585-8D7E-1D588FA5F727}.Release|Any CPU.Build.0 = Release|Any CPU + {E91365A1-8B01-4AB8-825F-67E3515EADCD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E91365A1-8B01-4AB8-825F-67E3515EADCD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E91365A1-8B01-4AB8-825F-67E3515EADCD}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {E91365A1-8B01-4AB8-825F-67E3515EADCD}.Publish|Any CPU.Build.0 = Debug|Any CPU + {E91365A1-8B01-4AB8-825F-67E3515EADCD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E91365A1-8B01-4AB8-825F-67E3515EADCD}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -373,9 +423,9 @@ Global {3EB61E99-C39B-4620-9482-F8DA18E48525} = {FA3720F1-C99A-49B2-9577-A940257098BF} {34A7F1EF-D243-4160-A413-D713FEABCD94} = {FA3720F1-C99A-49B2-9577-A940257098BF} {E4B777A1-28E1-41BE-96AE-7F3EC61FD5D4} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} - {F94D1938-9DB7-4B24-9FF3-166DDFD96330} = {9ECD1AA0-75B3-4E25-B0B5-9F0945B64974} - {689A5041-BAE7-448F-9BDC-4672E96249AA} = {9ECD1AA0-75B3-4E25-B0B5-9F0945B64974} - {EEA87FBC-4ED5-458C-ABD3-BEAEEB535BAF} = {9ECD1AA0-75B3-4E25-B0B5-9F0945B64974} + {F94D1938-9DB7-4B24-9FF3-166DDFD96330} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132} + {689A5041-BAE7-448F-9BDC-4672E96249AA} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132} + {EEA87FBC-4ED5-458C-ABD3-BEAEEB535BAF} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132} {37E39C68-5A40-4E63-9D3C-0C66AD98DFCB} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} {9ECD1AA0-75B3-4E25-B0B5-9F0945B64974} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} {107156B4-5A8B-45C7-97A2-4544D7FA19DE} = {9ECD1AA0-75B3-4E25-B0B5-9F0945B64974} @@ -385,17 +435,14 @@ Global {EB3FC57F-E591-4C88-BCD5-B6A1BC635168} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {5DEBAA62-F117-496A-8778-FED3604B70E2} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {EC004F12-2F60-4EDD-B3CD-3A504900D929} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} - {EA61C289-7928-4B78-A9C1-7AAD61F907CD} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {C9F957FA-A70F-4A6D-8F95-23FCD7F4FB87} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {3720F5ED-FB4D-485E-8A93-CDE60DEF0805} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {185E0CE8-C2DA-4E4C-A491-E8EB40316315} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {AFA81EB7-F869-467D-8A90-744305D80AAC} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {627742DB-1E52-468A-99BD-6FF1A542D25B} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} {E3299033-EB81-4C4C-BCD9-E8DC40937969} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} - {994BEF0B-E277-4D10-BB13-FE670D26620D} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} {078F96B4-09E1-4E0E-B214-F71A4F4BF633} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} {F51017A9-15C8-472D-893C-080046D710A6} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} - {A350933D-F9D5-4AD3-8C4F-B856B5020297} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} {EC3BB6D1-2FB2-4702-84C6-F791DE533ED4} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {4D226C2F-AE9F-4EFB-AF2D-45C8FE5CB34E} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {E52F805C-794A-4CA9-B684-DFF358B18820} = {9ECD1AA0-75B3-4E25-B0B5-9F0945B64974} @@ -410,11 +457,22 @@ Global {B00AD427-0047-4850-BEF9-BA8237EA9D8B} = {958AD708-F048-4FAF-94ED-D2F2B92748B9} {DB950192-30F1-48B1-88D7-F43FECCA1A1C} = {958AD708-F048-4FAF-94ED-D2F2B92748B9} {1C19D805-3573-4477-BF07-40180FCDE1BD} = {958AD708-F048-4FAF-94ED-D2F2B92748B9} - {0D0C4DAD-E6BC-4504-AE3A-EEA4E35920C1} = {9ECD1AA0-75B3-4E25-B0B5-9F0945B64974} + {3CDE10B2-AE8F-4FC4-8D55-92D4AD32E144} = {958AD708-F048-4FAF-94ED-D2F2B92748B9} + {0D0C4DAD-E6BC-4504-AE3A-EEA4E35920C1} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132} {E6EDAB8F-3406-4DBF-9AAB-DF40DC2CA0FA} = {FA3720F1-C99A-49B2-9577-A940257098BF} {677F1381-7830-4115-9C1A-58B282629DC6} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} - {4762BCAF-E1C5-4714-B88D-E50FA333C50E} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} {C754950A-E16C-4F96-9CC7-9328E361B5AF} = {FA3720F1-C99A-49B2-9577-A940257098BF} + {E07608CC-D710-4655-BB9E-D22CF3CDD193} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} + {10E4B697-D4E8-468D-872D-49670FD150FB} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} + {D4540A0F-98E3-4E70-9093-1948AE5B2AAD} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} + {3DC4DBD8-20A5-4937-B4F5-BB5E24E7A567} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633} + {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} + {5CB78CE4-895B-4A14-98AA-716A37DEEBB1} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132} + {A21FAC7C-0C09-4EAD-843B-926ACEF73C80} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} + {F224B869-FA0E-4EEE-A6BF-C2D61FF8E731} = {A21FAC7C-0C09-4EAD-843B-926ACEF73C80} + {CC77DCFA-A419-4202-A98A-868CDF457792} = {A21FAC7C-0C09-4EAD-843B-926ACEF73C80} + {8B754E80-7A97-4585-8D7E-1D588FA5F727} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} + {E91365A1-8B01-4AB8-825F-67E3515EADCD} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83} diff --git a/dotnet/SK-dotnet.sln.DotSettings b/dotnet/SK-dotnet.sln.DotSettings index 4d5e6137e95a..78893c5aab58 100644 --- a/dotnet/SK-dotnet.sln.DotSettings +++ b/dotnet/SK-dotnet.sln.DotSettings @@ -183,6 +183,7 @@ public void It$SOMENAME$() copy // Copyright (c) Microsoft. All rights reserved. + True True True True @@ -202,6 +203,7 @@ public void It$SOMENAME$() True True True + True True True True @@ -209,6 +211,7 @@ public void It$SOMENAME$() True True True + True True True True @@ -221,6 +224,7 @@ public void It$SOMENAME$() True True True + True True True True diff --git a/dotnet/build.cmd b/dotnet/build.cmd index 8d80cf630de1..ba30c180d8ae 100644 --- a/dotnet/build.cmd +++ b/dotnet/build.cmd @@ -1,7 +1,5 @@ @echo off - -cd dotnet - -dotnet build --configuration Release --interactive - -dotnet test --configuration Release --no-build --no-restore --interactive +setlocal +cd "%~dp0" +dotnet build --configuration Release --interactive ^ + && dotnet test --configuration Release --no-build --no-restore --interactive diff --git a/dotnet/docs/TELEMETRY.md b/dotnet/docs/TELEMETRY.md new file mode 100644 index 000000000000..a031ffb26a1f --- /dev/null +++ b/dotnet/docs/TELEMETRY.md @@ -0,0 +1,156 @@ +# Telemetry + +Telemetry in Semantic Kernel (SK) .NET implementation includes _logging_, _metering_ and _tracing_. +The code is instrumented using native .NET instrumentation tools, which means that it's possible to use different monitoring platforms (e.g. Application Insights, Prometheus, Grafana etc.). + +Code example using Application Insights can be found [here](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/ApplicationInsightsExample/Program.cs). + +## Logging + +The logging mechanism in this project relies on the `ILogger` interface from the `Microsoft.Extensions.Logging` namespace. Recent updates have introduced enhancements to the logger creation process. Instead of directly using the `ILogger` interface, instances of `ILogger` are now recommended to be created through an `ILoggerFactory` provided to components using the `WithLoggerFactory` method. + +By employing the `WithLoggerFactory` approach, logger instances are generated with precise type information, facilitating more accurate logging and streamlined control over log filtering across various classes. + +Log levels used in SK: + +- Trace - this type of logs **should not be enabled in production environments**, since it may contain sensitive data. It can be useful in test environments for better observability. Logged information includes: + - Goal/Ask to create a plan + - Prompt (template and rendered version) for AI to create a plan + - Created plan with function arguments (arguments may contain sensitive data) + - Prompt (template and rendered version) for AI to execute a function +- Debug - contains more detailed messages without sensitive data. Can be enabled in production environments. +- Information (default) - log level that is enabled by default and provides information about general flow of the application. Contains following data: + - AI model used to create a plan + - Plan creation status (Success/Failed) + - Plan creation execution time (in milliseconds) + - Created plan without function arguments + - AI model used to execute a function + - Function execution status (Success/Failed) + - Function execution time (in milliseconds) +- Warning - includes information about unusual events that don't cause the application to fail. +- Error - used for logging exception details. + +### Examples + +Enable logging for Kernel instance: + +```csharp +var kernel = new KernelBuilder().WithLoggerFactory(loggerFactory); +``` + +Enable logging for Planner instance (_metering_ and _tracing_ will be enabled as well): + +```csharp +var planner = new SequentialPlanner(kernel, plannerConfig).WithInstrumentation(loggerFactory); +``` + +### Log Filtering Configuration + +Log filtering configuration has been refined to strike a balance between visibility and relevance: + +```csharp +builder.AddFilter("Microsoft", LogLevel.Warning); +builder.AddFilter("Microsoft.SemanticKernel", LogLevel.Critical); +builder.AddFilter("Microsoft.SemanticKernel.Reliability", LogLevel.Information); +``` + +## Metering + +Metering is implemented with `Meter` class from `System.Diagnostics.Metrics` namespace. + +Available meters: + +- _Microsoft.SemanticKernel.Planning.Action.InstrumentedActionPlanner_ - captures metrics for `ActionPlanner`. List of metrics: + - `SK.ActionPlanner.CreatePlan.ExecutionTime` - execution time of plan creation (in milliseconds) +- _Microsoft.SemanticKernel.Planning.Sequential.InstrumentedSequentialPlanner_ - captures metrics for `SequentialPlanner`. List of metrics: + - `SK.SequentialPlanner.CreatePlan.ExecutionTime` - execution time of plan creation (in milliseconds) +- _Microsoft.SemanticKernel.Planning.Stepwise.StepwisePlanner_ - captures metrics for `StepwisePlanner`. List of metrics: + - `SK.StepwisePlanner.CreatePlan.ExecutionTime` - execution time of plan creation (in milliseconds) +- _Microsoft.SemanticKernel.Planning.Plan_ - captures metrics for `Plan`. List of metrics: + - `SK.Plan.Execution.ExecutionTime` - plan execution time (in milliseconds) + - `SK.Plan.Execution.ExecutionTotal` - total number of plan executions + - `SK.Plan.Execution.ExecutionSuccess` - number of successful plan executions + - `SK.Plan.Execution.ExecutionFailure` - number of failed plan executions +- _Microsoft.SemanticKernel.SKFunction_ - captures metrics for `SKFunction`. List of metrics: + - `SK..ExecutionTime` - function execution time (in milliseconds) + - `SK..ExecutionTotal` - total number of function executions + - `SK..ExecutionSuccess` - number of successful function executions + - `SK..ExecutionFailure` - number of failed function executions +- _Microsoft.SemanticKernel.Connectors.AI.OpenAI_ - captures metrics for OpenAI functionality. List of metrics: + - `SK.Connectors.OpenAI.PromptTokens` - number of prompt tokens used. + - `SK.Connectors.OpenAI.CompletionTokens` - number of completion tokens used. + - `SK.Connectors.OpenAI.TotalTokens` - total number of tokens used. + +### Examples + +Depending on monitoring tool, there are different ways how to subscribe to available meters. Following example shows how to subscribe to available meters and export metrics to Application Insights using `MeterListener`: + +```csharp +var meterListener = new MeterListener(); + +meterListener.InstrumentPublished = (instrument, listener) => +{ + if (instrument.Meter.Name.StartsWith("Microsoft.SemanticKernel", StringComparison.Ordinal)) + { + listener.EnableMeasurementEvents(instrument); + } +}; + +// Set callback to specific numeric type - double. +meterListener.SetMeasurementEventCallback((instrument, measurement, tags, state) => +{ + // Export to Application Insights using telemetry client instance + telemetryClient.GetMetric(instrument.Name).TrackValue(measurement); +}); + +meterListener.Start(); +``` + +It's possible to control for what meters to subscribe. For example, following condition will allow to subscribe to all meters in Semantic Kernel: + +```csharp +instrument.Meter.Name.StartsWith("Microsoft.SemanticKernel", StringComparison.Ordinal) +``` + +It's also possible to subscribe to specific meter. Following condition will allow to subscribe to meter for `SKFunction` only: + +```csharp +instrument.Meter.Name.Equals("Microsoft.SemanticKernel.SKFunction", StringComparison.Ordinal) +``` + +## Tracing + +Tracing is implemented with `Activity` class from `System.Diagnostics` namespace. + +Available activity sources: + +- _Microsoft.SemanticKernel.Planning.Action.InstrumentedActionPlanner_ - creates activities for `ActionPlanner`. +- _Microsoft.SemanticKernel.Planning.Sequential.InstrumentedSequentialPlanner_ - creates activities for `SequentialPlanner`. +- _Microsoft.SemanticKernel.Planning.Stepwise.StepwisePlanner_ - creates activities for `StepwisePlanner`. +- _Microsoft.SemanticKernel.Planning.Plan_ - creates activities for `Plan`. +- _Microsoft.SemanticKernel.SKFunction_ - creates activities for `SKFunction`. + +### Examples + +Subscribe to available activity sources using `ActivityListener`: + +```csharp +var activityListener = new ActivityListener(); + +activityListener.ShouldListenTo = + activitySource => activitySource.Name.StartsWith("Microsoft.SemanticKernel", StringComparison.Ordinal); + +ActivitySource.AddActivityListener(activityListener); +``` + +Following condition will allow to subscribe to all activity sources in Semantic Kernel: + +```csharp +activitySource.Name.StartsWith("Microsoft.SemanticKernel", StringComparison.Ordinal) +``` + +It's also possible to subscribe to specific activity source. Following condition will allow to subscribe to activity source for `SKFunction` only: + +```csharp +activitySource.Name.Equals("Microsoft.SemanticKernel.SKFunction", StringComparison.Ordinal) +``` diff --git a/samples/notebooks/dotnet/0-AI-settings.ipynb b/dotnet/notebooks/0-AI-settings.ipynb similarity index 100% rename from samples/notebooks/dotnet/0-AI-settings.ipynb rename to dotnet/notebooks/0-AI-settings.ipynb diff --git a/dotnet/notebooks/00-getting-started.ipynb b/dotnet/notebooks/00-getting-started.ipynb new file mode 100644 index 000000000000..597e68c3a3ed --- /dev/null +++ b/dotnet/notebooks/00-getting-started.ipynb @@ -0,0 +1,192 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Watch the Getting Started Quick Start [Video](https://aka.ms/SK-Getting-Started-Notebook)\n", + "\n", + "> [!IMPORTANT]\n", + "> You will need an [.Net 7 SDK](https://dotnet.microsoft.com/en-us/download) and [Polyglot](https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.dotnet-interactive-vscode) to get started with this notebook using .Net Interactive" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 1**: Configure your AI service credentials\n", + "\n", + "Use [this notebook](0-AI-settings.ipynb) first, to choose whether to run these notebooks with OpenAI or Azure OpenAI,\n", + "and to save your credentials in the configuration file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "// Load some helper functions, e.g. to load values from settings.json\n", + "#!import config/Settings.cs " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 2**: Import Semantic Kernel SDK from NuGet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "// Import Semantic Kernel\n", + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 3**: Instantiate the Kernel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "using Microsoft.SemanticKernel;\n", + "\n", + "//Create Kernel builder\n", + "var builder = new KernelBuilder();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "// Configure AI service credentials used by the kernel\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "if (useAzureOpenAI)\n", + " builder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + "else\n", + " builder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + "\n", + "IKernel kernel = builder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Step 4**: Load and Run a Plugin" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "// Load the Plugins Directory\n", + "var pluginsDirectory = Path.Combine(System.IO.Directory.GetCurrentDirectory(), \"..\", \"..\", \"samples\", \"plugins\");\n", + "\n", + "// Load the FunPlugin from the Plugins Directory\n", + "var funPluginFunctions = kernel.ImportSemanticFunctionsFromDirectory(pluginsDirectory, \"FunPlugin\");\n", + "\n", + "// Run the Function called Joke\n", + "var result = await kernel.RunAsync(\"time travel to dinosaur age\", funPluginFunctions[\"Joke\"]);\n", + "var resultString = result.GetValue();\n", + "\n", + "// Return the result to the Notebook\n", + "Console.WriteLine(resultString);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Next Steps**: You know the basics, let's try this in a sample app so you can learn the core concepts!\n", + "\n", + "Sample app learning examples:\n", + "- [Simple chat summary](../../samples/apps/chat-summary-webapp-react/README.md) (**Recommended**) – learn how basic semantic functions can be added to an app\n", + "- [Book creator](../../samples/apps/book-creator-webapp-react/README.md) – learn how Planner and chaining of semantic functions can be used in your app\n", + "- [Authentication and APIs](../../samples/dotnet/MsGraphPluginsExample/README.md) – learn how to connect to external API's with authentication while using Semantic Kernel\n", + "- [GitHub repository Q&A](../../samples/apps/github-qna-webapp-react/README.md) - Use embeddings and memory to store and query your data\n", + "- [Copilot Chat](../../samples/apps/copilot-chat-app/README.md) – Build your own chatbot based on Semantic Kernel" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dotnet/notebooks/01-basic-loading-the-kernel.ipynb b/dotnet/notebooks/01-basic-loading-the-kernel.ipynb new file mode 100644 index 000000000000..01b143634017 --- /dev/null +++ b/dotnet/notebooks/01-basic-loading-the-kernel.ipynb @@ -0,0 +1,214 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic Loading of the Kernel" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Semantic Kernel SDK can be imported from the following nuget feed:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After adding the nuget package, you can instantiate the kernel in a few ways, depending on your use case.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "using Microsoft.SemanticKernel;\n", + "\n", + "// Simple instance\n", + "IKernel kernel_1 = KernelBuilder.Create();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "using Microsoft.Extensions.Logging;\n", + "using Microsoft.Extensions.Logging.Abstractions;\n", + "\n", + "// Inject your logger \n", + "// see Microsoft.Extensions.Logging.ILogger @ https://learn.microsoft.com/dotnet/core/extensions/logging\n", + "ILoggerFactory myLoggerFactory = NullLoggerFactory.Instance;\n", + "IKernel kernel_2 = new KernelBuilder()\n", + " .WithLoggerFactory(myLoggerFactory)\n", + " .Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When using the kernel for AI requests, the kernel needs some settings like URL and credentials to the AI models.\n", + "\n", + "The SDK currently supports OpenAI, Azure OpenAI and HuggingFace. It's also possible to create your own connector and use AI provider of your choice.\n", + "\n", + "If you need an Azure OpenAI key, go [here](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/quickstart?pivots=rest-api)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "Kernel.Builder\n", + ".WithAzureChatCompletionService(\n", + " \"my-finetuned-model\", // Azure OpenAI *Deployment Name*\n", + " \"https://contoso.openai.azure.com/\", // Azure OpenAI *Endpoint*\n", + " \"...your Azure OpenAI Key...\", // Azure OpenAI *Key*\n", + " serviceId: \"Azure_curie\" // alias used in the prompt templates' config.json\n", + ")\n", + ".WithOpenAIChatCompletionService(\n", + " \"gpt-3.5-turbo\", // OpenAI Model Name\n", + " \"...your OpenAI API Key...\", // OpenAI API key\n", + " \"...your OpenAI Org ID...\", // *optional* OpenAI Organization ID\n", + " serviceId: \"OpenAI_davinci\" // alias used in the prompt templates' config.json\n", + ");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When working with multiple backends and multiple models, the **first backend** defined\n", + "is also the \"**default**\" used in these scenarios:\n", + "\n", + "* a prompt configuration doesn't specify which AI backend to use\n", + "* a prompt configuration requires a backend unknown to the kernel\n", + "\n", + "The default can be set programmatically:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "Kernel.Builder\n", + ".WithOpenAIChatCompletionService(\n", + " \"gpt-3.5-turbo\", // OpenAI Model Name\n", + " \"...your OpenAI API Key...\", // OpenAI API key\n", + " \"...your OpenAI Org ID...\", // *optional* OpenAI Organization ID\n", + " \"OpenAI_davinci\", // alias used in the prompt templates' config.json\n", + " true // This flag specifies that this service is the default one.\n", + ");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great, now that you're familiar with setting up the Semantic Kernel, let's see [how we can use it to run prompts](02-running-prompts-from-file.ipynb)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "file_extension": ".cs", + "mimetype": "text/x-csharp", + "name": "C#", + "pygments_lexer": "csharp", + "version": "11.0" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dotnet/notebooks/02-running-prompts-from-file.ipynb b/dotnet/notebooks/02-running-prompts-from-file.ipynb new file mode 100644 index 000000000000..dadfe2466a53 --- /dev/null +++ b/dotnet/notebooks/02-running-prompts-from-file.ipynb @@ -0,0 +1,198 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to run a semantic plugins from file\n", + "Now that you're familiar with Kernel basics, let's see how the kernel allows you to run Semantic Plugins and Semantic Functions stored on disk. \n", + "\n", + "A Semantic Plugin is a collection of Semantic Functions, where each function is defined with natural language that can be provided with a text file. \n", + "\n", + "Refer to our [glossary](../../docs/GLOSSARY.md) for an in-depth guide to the terms.\n", + "\n", + "The repository includes some examples under the [samples](https://github.com/microsoft/semantic-kernel/tree/main/samples) folder.\n", + "\n", + "For instance, [this](../../samples/plugins/FunPlugin/Joke/skprompt.txt) is the **Joke function** part of the **FunPlugin plugin**:" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "WRITE EXACTLY ONE JOKE or HUMOROUS STORY ABOUT THE TOPIC BELOW.\n", + "JOKE MUST BE:\n", + "- G RATED\n", + "- WORKPLACE/FAMILY SAFE\n", + "NO SEXISM, RACISM OR OTHER BIAS/BIGOTRY.\n", + "BE CREATIVE AND FUNNY. I WANT TO LAUGH.\n", + "+++++\n", + "{{$input}}\n", + "+++++\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note the special **`{{$input}}`** token, which is a variable that is automatically passed when invoking the function, commonly referred to as a \"function parameter\". \n", + "\n", + "We'll explore later how functions can accept multiple variables, as well as invoke other functions." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "In the same folder you'll notice a second [config.json](../../samples/plugins/FunPlugin/Joke/config.json) file. The file is optional, and is used to set some parameters for large language models like Temperature, TopP, Stop Sequences, etc.\n", + "\n", + "```\n", + "{\n", + " \"schema\": 1,\n", + " \"description\": \"Generate a funny joke\",\n", + " \"models\": [\n", + " {\n", + " \"max_tokens\": 500,\n", + " \"temperature\": 0.5,\n", + " \"top_p\": 0.5\n", + " }\n", + " ]\n", + "}\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Given a semantic function defined by these files, this is how to load and use a file based semantic function.\n", + "\n", + "Configure and create the kernel, as usual, loading also the AI backend settings defined in the [Setup notebook](0-AI-settings.ipynb):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "\n", + "#!import config/Settings.cs\n", + "\n", + "using Microsoft.SemanticKernel;\n", + "\n", + "var builder = new KernelBuilder();\n", + "\n", + "// Configure AI backend used by the kernel\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "if (useAzureOpenAI)\n", + " builder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + "else\n", + " builder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + "\n", + "IKernel kernel = builder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import the plugin and all its functions:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "// note: using plugins from the repo\n", + "var pluginsDirectory = Path.Combine(System.IO.Directory.GetCurrentDirectory(), \"..\", \"..\", \"samples\", \"plugins\");\n", + "\n", + "var funPluginFunctions = kernel.ImportSemanticFunctionsFromDirectory(pluginsDirectory, \"FunPlugin\");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "How to use the plugin functions, e.g. generate a joke about \"*time travel to dinosaur age*\":" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var result = await kernel.RunAsync(\"time travel to dinosaur age\", funPluginFunctions[\"Joke\"]);\n", + "var resultString = result.GetValue();\n", + "\n", + "Console.WriteLine(resultString);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great, now that you know how to load a plugin from disk, let's show how you can [create and run a semantic function inline.](./03-semantic-function-inline.ipynb)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dotnet/notebooks/03-semantic-function-inline.ipynb b/dotnet/notebooks/03-semantic-function-inline.ipynb new file mode 100644 index 000000000000..d5326783be7b --- /dev/null +++ b/dotnet/notebooks/03-semantic-function-inline.ipynb @@ -0,0 +1,374 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Running Semantic Functions Inline" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The [previous notebook](./02-running-prompts-from-file.ipynb)\n", + "showed how to define a semantic function using a prompt template stored on a file.\n", + "\n", + "In this notebook, we'll show how to use the Semantic Kernel to define functions inline with your C# code. This can be useful in a few scenarios:\n", + "\n", + "* Dynamically generating the prompt using complex rules at runtime\n", + "* Writing prompts by editing C# code instead of TXT files. \n", + "* Easily creating demos, like this document\n", + "\n", + "Prompt templates are defined using the SK template language, which allows to reference variables and functions. Read [this doc](https://aka.ms/sk/howto/configurefunction) to learn more about the design decisions for prompt templating. \n", + "\n", + "For now we'll use only the `{{$input}}` variable, and see more complex templates later.\n", + "\n", + "Almost all semantic function prompts have a reference to `{{$input}}`, which is the default way\n", + "a user can import content from the context variables." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Prepare a semantic kernel instance first, loading also the AI backend settings defined in the [Setup notebook](0-AI-settings.ipynb):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "\n", + "#!import config/Settings.cs\n", + "\n", + "using Microsoft.SemanticKernel;\n", + "using Microsoft.SemanticKernel.SemanticFunctions;\n", + "using Microsoft.SemanticKernel.Connectors.AI.OpenAI;\n", + "\n", + "var builder = new KernelBuilder();\n", + "\n", + "// Configure AI backend used by the kernel\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "if (useAzureOpenAI)\n", + " builder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + "else\n", + " builder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + "\n", + "IKernel kernel = builder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's create a semantic function used to summarize content:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "string skPrompt = \"\"\"\n", + "{{$input}}\n", + "\n", + "Summarize the content above.\n", + "\"\"\";" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's configure the prompt, e.g. allowing for some creativity and a sufficient number of tokens." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var aiRequestSettings = new OpenAIRequestSettings \n", + "{\n", + " MaxTokens = 2000,\n", + " Temperature = 0.2,\n", + " TopP = 0.5\n", + "};\n", + "\n", + "var promptConfig = new PromptTemplateConfig();\n", + "promptConfig.ModelSettings.Add(aiRequestSettings);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following code prepares an instance of the template, passing in the TXT and configuration above, \n", + "and a couple of other parameters (how to render the TXT and how the template can access other functions)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var promptTemplate = new PromptTemplate(\n", + " skPrompt, // Prompt template defined in natural language\n", + " promptConfig, // Prompt configuration\n", + " kernel // SK instance\n", + ");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's transform the prompt template into a function that the kernel can execute:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var functionConfig = new SemanticFunctionConfig(promptConfig, promptTemplate);\n", + "\n", + "var summaryFunction = kernel.RegisterSemanticFunction(\"MyPlugin\", \"Summary\", functionConfig);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set up some content to summarize, here's an extract about Demo, an ancient Greek poet, taken from [Wikipedia](https://en.wikipedia.org/wiki/Demo_(ancient_Greek_poet))." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var input = \"\"\"\n", + "Demo (ancient Greek poet)\n", + "From Wikipedia, the free encyclopedia\n", + "Demo or Damo (Greek: Δεμώ, Δαμώ; fl. c. AD 200) was a Greek woman of the Roman period, known for a single epigram, engraved upon the Colossus of Memnon, which bears her name. She speaks of herself therein as a lyric poetess dedicated to the Muses, but nothing is known of her life.[1]\n", + "Identity\n", + "Demo was evidently Greek, as her name, a traditional epithet of Demeter, signifies. The name was relatively common in the Hellenistic world, in Egypt and elsewhere, and she cannot be further identified. The date of her visit to the Colossus of Memnon cannot be established with certainty, but internal evidence on the left leg suggests her poem was inscribed there at some point in or after AD 196.[2]\n", + "Epigram\n", + "There are a number of graffiti inscriptions on the Colossus of Memnon. Following three epigrams by Julia Balbilla, a fourth epigram, in elegiac couplets, entitled and presumably authored by \"Demo\" or \"Damo\" (the Greek inscription is difficult to read), is a dedication to the Muses.[2] The poem is traditionally published with the works of Balbilla, though the internal evidence suggests a different author.[1]\n", + "In the poem, Demo explains that Memnon has shown her special respect. In return, Demo offers the gift for poetry, as a gift to the hero. At the end of this epigram, she addresses Memnon, highlighting his divine status by recalling his strength and holiness.[2]\n", + "Demo, like Julia Balbilla, writes in the artificial and poetic Aeolic dialect. The language indicates she was knowledgeable in Homeric poetry—'bearing a pleasant gift', for example, alludes to the use of that phrase throughout the Iliad and Odyssey.[a][2] \n", + "\"\"\";" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "...and run the summary function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var summaryResult = await kernel.RunAsync(input, summaryFunction);\n", + "var summary = summaryResult.GetValue();\n", + "\n", + "Console.WriteLine(summary);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The code above shows all the steps, to understand how the function is composed step by step. However, the kernel\n", + "includes also some helpers to achieve the same more concisely.\n", + "\n", + "The same function above can be created with less code:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "string skPrompt = \"\"\"\n", + "{{$input}}\n", + "\n", + "Summarize the content above.\n", + "\"\"\";\n", + "\n", + "var summaryFunction = kernel.CreateSemanticFunction(skPrompt, requestSettings: new OpenAIRequestSettings { MaxTokens = 2000, Temperature = 0.2, TopP = 0.5 });\n", + "\n", + "var summaryResult = await kernel.RunAsync(input, summaryFunction);\n", + "var summary = summaryResult.GetValue();\n", + "\n", + "Console.WriteLine(summary);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's one more example of how to write an inline Semantic Function that gives a TLDR for a piece of text.\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "var builder = new KernelBuilder();\n", + "\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "if (useAzureOpenAI)\n", + " builder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + "else\n", + " builder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + "\n", + "var kernel = builder.Build();\n", + "\n", + "string skPrompt = @\"\n", + "{{$input}}\n", + "\n", + "Give me the TLDR in 5 words.\n", + "\";\n", + "\n", + "var textToSummarize = @\"\n", + " 1) A robot may not injure a human being or, through inaction,\n", + " allow a human being to come to harm.\n", + "\n", + " 2) A robot must obey orders given it by human beings except where\n", + " such orders would conflict with the First Law.\n", + "\n", + " 3) A robot must protect its own existence as long as such protection\n", + " does not conflict with the First or Second Law.\n", + "\";\n", + "\n", + "var tldrFunction = kernel.CreateSemanticFunction(skPrompt, requestSettings: new OpenAIRequestSettings { MaxTokens = 2000, Temperature = 0.2, TopP = 0.5 });\n", + "\n", + "var summaryResult = await kernel.RunAsync(textToSummarize, tldrFunction);\n", + "var summary = summaryResult.GetValue();\n", + "\n", + "Console.WriteLine(summary);\n", + "\n", + "// Output => Robots must not harm humans." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dotnet/notebooks/04-context-variables-chat.ipynb b/dotnet/notebooks/04-context-variables-chat.ipynb new file mode 100644 index 000000000000..f5c33f746a9d --- /dev/null +++ b/dotnet/notebooks/04-context-variables-chat.ipynb @@ -0,0 +1,390 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Creating a basic chat experience with context variables\n", + "\n", + "In this example, we show how you can build a simple chat bot by sending and updating context with your requests. \n", + "\n", + "We introduce the Context Variables object which in this demo functions similarly as a key-value store that you can use when running the kernel.\n", + "\n", + "The context is local (i.e. in your computer's RAM) and not persisted anywhere beyond the life of this Jupyter session.\n", + "\n", + "In future examples, we will show how to persist the context on disk so that you can bring it into your applications. \n", + "\n", + "In this chat scenario, as the user talks back and forth with the bot, the context gets populated with the history of the conversation. During each new run of the kernel, the context can provide the AI with its variables' content. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "#!import config/Settings.cs\n", + "\n", + "using Microsoft.SemanticKernel;\n", + "using Microsoft.SemanticKernel.SemanticFunctions;\n", + "using Microsoft.SemanticKernel.Orchestration;\n", + "using Microsoft.SemanticKernel.Connectors.AI.OpenAI;\n", + "\n", + "var builder = new KernelBuilder();\n", + "\n", + "// Configure AI backend used by the kernel\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "if (useAzureOpenAI)\n", + " builder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + "else\n", + " builder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + "\n", + "IKernel kernel = builder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's define a prompt outlining a dialogue chat bot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "const string skPrompt = @\"\n", + "ChatBot can have a conversation with you about any topic.\n", + "It can give explicit instructions or say 'I don't know' if it does not have an answer.\n", + "\n", + "{{$history}}\n", + "User: {{$userInput}}\n", + "ChatBot:\";\n", + "\n", + "var aiRequestSettings = new OpenAIRequestSettings \n", + "{\n", + " MaxTokens = 2000,\n", + " Temperature = 0.7,\n", + " TopP = 0.5\n", + "};\n", + "\n", + "var promptConfig = new PromptTemplateConfig();\n", + "promptConfig.ModelSettings.Add(aiRequestSettings);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Register your semantic function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "var promptTemplate = new PromptTemplate(skPrompt, promptConfig, kernel);\n", + "var functionConfig = new SemanticFunctionConfig(promptConfig, promptTemplate);\n", + "var chatFunction = kernel.RegisterSemanticFunction(\"ChatBot\", \"Chat\", functionConfig);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Initialize your context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "var context = kernel.CreateNewContext();\n", + "\n", + "var history = \"\";\n", + "context.Variables[\"history\"] = history;" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Chat with the Bot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "var userInput = \"Hi, I'm looking for book suggestions\";\n", + "context.Variables[\"userInput\"] = userInput;\n", + "\n", + "var bot_answer = await chatFunction.InvokeAsync(context);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Update the history with the output and set this as the new input value for the next request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "history += $\"\\nUser: {userInput}\\nMelody: {bot_answer.GetValue()}\\n\";\n", + "context.Variables.Update(history);\n", + "\n", + "Console.WriteLine(context);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Keep Chatting!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "Func Chat = async (string input) => {\n", + " // Save new message in the context variables\n", + " context.Variables[\"userInput\"] = input;\n", + "\n", + " // Process the user message and get an answer\n", + " var answer = await chatFunction.InvokeAsync(context);\n", + "\n", + " // Append the new interaction to the chat history\n", + " history += $\"\\nUser: {input}\\nMelody: {answer.GetValue()}\\n\"; \n", + " context.Variables[\"history\"] = history;\n", + " \n", + " // Show the response\n", + " Console.WriteLine(context);\n", + "};" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "await Chat(\"I would like a non-fiction book suggestion about Greece history. Please only list one book.\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "await Chat(\"that sounds interesting, what are some of the topics I will learn about?\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "await Chat(\"Which topic from the ones you listed do you think most people find interesting?\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "await Chat(\"could you list some more books I could read about the topic(s) you mentioned?\");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After chatting for a while, we have built a growing history, which we are attaching to each prompt and which contains the full conversation. Let's take a look!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "Console.WriteLine(context.Variables[\"history\"]);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "file_extension": ".cs", + "mimetype": "text/x-csharp", + "name": "C#", + "pygments_lexer": "csharp", + "version": "11.0" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dotnet/notebooks/05-using-the-planner.ipynb b/dotnet/notebooks/05-using-the-planner.ipynb new file mode 100644 index 000000000000..652884a33773 --- /dev/null +++ b/dotnet/notebooks/05-using-the-planner.ipynb @@ -0,0 +1,286 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction to the Planner\n", + "\n", + "The Planner is one of the fundamental concepts of the Semantic Kernel. It makes use of the collection of plugins that have been registered to the kernel and using AI, will formulate a plan to execute a given ask.\n", + "\n", + "Read more about it [here](https://aka.ms/sk/concepts/planner)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "\n", + "#!import config/Settings.cs\n", + "#!import config/Utils.cs\n", + "\n", + "using Microsoft.SemanticKernel;\n", + "using Microsoft.SemanticKernel.Plugins.Core;\n", + "using Microsoft.SemanticKernel.Orchestration;\n", + "using Microsoft.SemanticKernel.Planning;\n", + "using Microsoft.SemanticKernel.Planners;\n", + "using Microsoft.SemanticKernel.Connectors.AI.OpenAI;\n", + "\n", + "var builder = new KernelBuilder();\n", + "\n", + "// Configure AI backend used by the kernel\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "if (useAzureOpenAI)\n", + " builder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + "else\n", + " builder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + "\n", + "var kernel = builder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setting Up the Planner\n", + "The planner is located in the `Microsoft.SemanticKernel.Planners.Core` package and requires Orchestration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "// Load native plugin into the kernel registry, sharing its functions with prompt templates\n", + "var planner = new SequentialPlanner(kernel);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Providing plugins to the planner\n", + "The planner needs to know what plugins are available to it. Here we'll give it access to the `SummarizePlugin` and `WriterPlugin` we have defined on disk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var pluginsDirectory = Path.Combine(System.IO.Directory.GetCurrentDirectory(), \"..\", \"..\", \"samples\", \"plugins\");\n", + "kernel.ImportSemanticFunctionsFromDirectory(pluginsDirectory, \"SummarizePlugin\");\n", + "kernel.ImportSemanticFunctionsFromDirectory(pluginsDirectory, \"WriterPlugin\");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define your ASK. What do you want the Kernel to do?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var ask = \"Tomorrow is Valentine's day. I need to come up with a few date ideas. My significant other likes poems so write them in the form of a poem.\";\n", + "var originalPlan = await planner.CreatePlanAsync(ask);\n", + "\n", + "Console.WriteLine(\"Original plan:\\n\");\n", + "Console.WriteLine(JsonSerializer.Serialize(originalPlan, new JsonSerializerOptions { WriteIndented = true }));" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see in the above plan, the Planner has taken the user's ask and converted it into a Plan object detailing how the AI would go about solving this task.\n", + "\n", + "It makes use of the plugins that the Kernel has available to it and determines which functions to call in order to fulfill the user's ask.\n", + "\n", + "The output of each step of the plan gets set as `setContextVariable` which makes it available as `input` to the next plugin." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's also define an inline plugin and have it be available to the Planner.\n", + "Be sure to give it a function name and plugin name." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "string skPrompt = \"\"\"\n", + "{{$input}}\n", + "\n", + "Rewrite the above in the style of Shakespeare.\n", + "\"\"\";\n", + "var shakespeareFunction = kernel.CreateSemanticFunction(skPrompt, \"Shakespeare\", \"ShakespearePlugin\", requestSettings: new OpenAIRequestSettings { MaxTokens = 2000, Temperature = 0.2, TopP = 0.5 });" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's update our ask using this new plugin." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var ask = @\"Tomorrow is Valentine's day. I need to come up with a few date ideas.\n", + "She likes Shakespeare so write using his style. Write them in the form of a poem.\";\n", + "\n", + "var newPlan = await planner.CreatePlanAsync(ask);\n", + "\n", + "Console.WriteLine(\"Updated plan:\\n\");\n", + "Console.WriteLine(JsonSerializer.Serialize(newPlan, new JsonSerializerOptions { WriteIndented = true }));" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Executing the plans\n", + "\n", + "Now that we have different plans, let's try to execute them! The Kernel can execute the plan using RunAsync." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var originalPlanResult = await kernel.RunAsync(originalPlan);\n", + "\n", + "Console.WriteLine(\"Original Plan results:\\n\");\n", + "Console.WriteLine(Utils.WordWrap(originalPlanResult.GetValue(), 100));" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now lets execute and print the new plan:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var newPlanResult = await kernel.RunAsync(newPlan);\n", + "\n", + "Console.WriteLine(\"New Plan results:\\n\");\n", + "Console.WriteLine(newPlanResult.GetValue());" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dotnet/notebooks/06-memory-and-embeddings.ipynb b/dotnet/notebooks/06-memory-and-embeddings.ipynb new file mode 100644 index 000000000000..69be621eb614 --- /dev/null +++ b/dotnet/notebooks/06-memory-and-embeddings.ipynb @@ -0,0 +1,566 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Building Semantic Memory with Embeddings\n", + "\n", + "So far, we've mostly been treating the kernel as a stateless orchestration engine.\n", + "We send text into a model API and receive text out. \n", + "\n", + "In a [previous notebook](04-context-variables-chat.ipynb), we used `context variables` to pass in additional\n", + "text into prompts to enrich them with more context. This allowed us to create a basic chat experience. \n", + "\n", + "However, if you solely relied on context variables, you would quickly realize that eventually your prompt\n", + "would grow so large that you would run into a the model's token limit. What we need is a way to persist state\n", + "and build both short-term and long-term memory to empower even more intelligent applications. \n", + "\n", + "To do this, we dive into the key concept of `Semantic Memory` in the Semantic Kernel. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "#r \"nuget: System.Linq.Async, 6.0.1\"\n", + "\n", + "#!import config/Settings.cs\n", + "\n", + "using Microsoft.SemanticKernel;\n", + "using Microsoft.SemanticKernel.SemanticFunctions;\n", + "using Microsoft.SemanticKernel.Orchestration;\n", + "\n", + "var kernelBuilder = new KernelBuilder();\n", + "\n", + "// Configure AI backend used by the kernel\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "if (useAzureOpenAI)\n", + " kernelBuilder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + "else\n", + " kernelBuilder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + "\n", + "var kernel = kernelBuilder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to use memory, we need to instantiate the Memory Plugin with a Memory Storage\n", + "and an Embedding backend. In this example, we make use of the `VolatileMemoryStore`\n", + "which can be thought of as a temporary in-memory storage (not to be confused with Semantic Memory).\n", + "\n", + "This memory is not written to disk and is only available during the app session.\n", + "\n", + "When developing your app you will have the option to plug in persistent storage\n", + "like Azure Cosmos Db, PostgreSQL, SQLite, etc. Semantic Memory allows also to index\n", + "external data sources, without duplicating all the information, more on that later." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "using Microsoft.SemanticKernel.Plugins.Memory;\n", + "using Microsoft.SemanticKernel.Connectors.AI.OpenAI;\n", + "\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "var memoryBuilder = new MemoryBuilder();\n", + "\n", + "if (useAzureOpenAI)\n", + "{\n", + " memoryBuilder.WithAzureTextEmbeddingGenerationService(\"text-embedding-ada-002\", azureEndpoint, apiKey);\n", + "}\n", + "else\n", + "{\n", + " memoryBuilder.WithOpenAITextEmbeddingGenerationService(\"text-embedding-ada-002\", apiKey);\n", + "}\n", + "\n", + "memoryBuilder.WithMemoryStore(new VolatileMemoryStore());\n", + "\n", + "var memory = memoryBuilder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At its core, Semantic Memory is a set of data structures that allow you to store\n", + "the meaning of text that come from different data sources, and optionally to store\n", + "the source text too.\n", + "\n", + "These texts can be from the web, e-mail providers, chats, a database, or from your\n", + "local directory, and are hooked up to the Semantic Kernel through data source connectors.\n", + "\n", + "The texts are embedded or compressed into a vector of floats representing mathematically\n", + "the texts' contents and meaning.\n", + "\n", + "You can read more about embeddings [here](https://aka.ms/sk/embeddings)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Manually adding memories\n", + "Let's create some initial memories \"About Me\". We can add memories to our `VolatileMemoryStore` by using `SaveInformationAsync`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "const string MemoryCollectionName = \"aboutMe\";\n", + "\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info1\", text: \"My name is Andrea\");\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info2\", text: \"I currently work as a tourist operator\");\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info3\", text: \"I currently live in Seattle and have been living there since 2005\");\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info4\", text: \"I visited France and Italy five times since 2015\");\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info5\", text: \"My family is from New York\");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's try searching the memory:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var questions = new[]\n", + "{\n", + " \"what is my name?\",\n", + " \"where do I live?\",\n", + " \"where is my family from?\",\n", + " \"where have I travelled?\",\n", + " \"what do I do for work?\",\n", + "};\n", + "\n", + "foreach (var q in questions)\n", + "{\n", + " var response = await memory.SearchAsync(MemoryCollectionName, q).FirstOrDefaultAsync();\n", + " Console.WriteLine(q + \" \" + response?.Metadata.Text);\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's now revisit our chat sample from the [previous notebook](04-context-variables-chat.ipynb).\n", + "If you remember, we used context variables to fill the prompt with a `history` that continuously got populated as we chatted with the bot. Let's add also memory to it!" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is done by using the `TextMemoryPlugin` which exposes the `recall` native function.\n", + "\n", + "`recall` takes an input ask and performs a similarity search on the contents that have\n", + "been embedded in the Memory Store. By default, `recall` returns the most relevant memory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "// TextMemoryPlugin provides the \"recall\" function\n", + "kernel.ImportFunctions(new TextMemoryPlugin(memory));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "const string skPrompt = @\"\n", + "ChatBot can have a conversation with you about any topic.\n", + "It can give explicit instructions or say 'I don't know' if it does not have an answer.\n", + "\n", + "Information about me, from previous conversations:\n", + "- {{$fact1}} {{recall $fact1}}\n", + "- {{$fact2}} {{recall $fact2}}\n", + "- {{$fact3}} {{recall $fact3}}\n", + "- {{$fact4}} {{recall $fact4}}\n", + "- {{$fact5}} {{recall $fact5}}\n", + "\n", + "Chat:\n", + "{{$history}}\n", + "User: {{$userInput}}\n", + "ChatBot: \";\n", + "\n", + "var chatFunction = kernel.CreateSemanticFunction(skPrompt, requestSettings: new OpenAIRequestSettings { MaxTokens = 200, Temperature = 0.8 });" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `RelevanceParam` is used in memory search and is a measure of the relevance score from 0.0 to 1.0, where 1.0 means a perfect match. We encourage users to experiment with different values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var context = kernel.CreateNewContext();\n", + "\n", + "context.Variables[\"fact1\"] = \"what is my name?\";\n", + "context.Variables[\"fact2\"] = \"where do I live?\";\n", + "context.Variables[\"fact3\"] = \"where is my family from?\";\n", + "context.Variables[\"fact4\"] = \"where have I travelled?\";\n", + "context.Variables[\"fact5\"] = \"what do I do for work?\";\n", + "\n", + "context.Variables[TextMemoryPlugin.CollectionParam] = MemoryCollectionName;\n", + "context.Variables[TextMemoryPlugin.RelevanceParam] = \"0.8\";" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we've included our memories, let's chat!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var history = \"\";\n", + "context.Variables[\"history\"] = history;\n", + "Func Chat = async (string input) => {\n", + " // Save new message in the context variables\n", + " context.Variables[\"userInput\"] = input;\n", + "\n", + " // Process the user message and get an answer\n", + " var answer = await chatFunction.InvokeAsync(context);\n", + "\n", + " // Append the new interaction to the chat history\n", + " history += $\"\\nUser: {input}\\nChatBot: {answer.GetValue()}\\n\";\n", + " context.Variables[\"history\"] = history;\n", + " \n", + " // Show the bot response\n", + " Console.WriteLine(\"ChatBot: \" + context);\n", + "};" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "await Chat(\"Hello, I think we've met before, remember? my name is...\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "await Chat(\"I want to plan a trip and visit my family. Do you know where that is?\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "await Chat(\"Great! What are some fun things to do there?\");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Adding documents to your memory\n", + "\n", + "Many times in your applications you'll want to bring in external documents into your memory. Let's see how we can do this using our VolatileMemoryStore.\n", + "\n", + "Let's first get some data using some of the links in the Semantic Kernel repo." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "const string memoryCollectionName = \"SKGitHub\";\n", + "\n", + "var githubFiles = new Dictionary()\n", + "{\n", + " [\"https://github.com/microsoft/semantic-kernel/blob/main/README.md\"]\n", + " = \"README: Installation, getting started, and how to contribute\",\n", + " [\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/02-running-prompts-from-file.ipynb\"]\n", + " = \"Jupyter notebook describing how to pass prompts from a file to a semantic plugin or function\",\n", + " [\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/00-getting-started.ipynb\"]\n", + " = \"Jupyter notebook describing how to get started with the Semantic Kernel\",\n", + " [\"https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins/ChatPlugin/ChatGPT\"]\n", + " = \"Sample demonstrating how to create a chat plugin interfacing with ChatGPT\",\n", + " [\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Plugins/Plugins.Memory/VolatileMemoryStore.cs\"]\n", + " = \"C# class that defines a volatile embedding store\",\n", + " [\"https://github.com/microsoft/semantic-kernel/tree/main/samples/dotnet/KernelHttpServer/README.md\"]\n", + " = \"README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4\",\n", + " [\"https://github.com/microsoft/semantic-kernel/tree/main/samples/apps/chat-summary-webapp-react/README.md\"]\n", + " = \"README: README associated with a sample starter react-based chat summary webapp\",\n", + "};" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's build a new Memory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var memoryBuilder = new MemoryBuilder();\n", + "\n", + "if (useAzureOpenAI)\n", + "{\n", + " memoryBuilder.WithAzureTextEmbeddingGenerationService(\"text-embedding-ada-002\", azureEndpoint, apiKey);\n", + "}\n", + "else\n", + "{\n", + " memoryBuilder.WithOpenAITextEmbeddingGenerationService(\"text-embedding-ada-002\", apiKey);\n", + "}\n", + "\n", + "memoryBuilder.WithMemoryStore(new VolatileMemoryStore());\n", + "\n", + "var memory = memoryBuilder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's add these files to our VolatileMemoryStore using `SaveReferenceAsync`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "Console.WriteLine(\"Adding some GitHub file URLs and their descriptions to a volatile Semantic Memory.\");\n", + "var i = 0;\n", + "foreach (var entry in githubFiles)\n", + "{\n", + " await memory.SaveReferenceAsync(\n", + " collection: memoryCollectionName,\n", + " description: entry.Value,\n", + " text: entry.Value,\n", + " externalId: entry.Key,\n", + " externalSourceName: \"GitHub\"\n", + " );\n", + " Console.WriteLine($\" URL {++i} saved\");\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "string ask = \"I love Jupyter notebooks, how should I get started?\";\n", + "Console.WriteLine(\"===========================\\n\" +\n", + " \"Query: \" + ask + \"\\n\");\n", + "\n", + "var memories = memory.SearchAsync(memoryCollectionName, ask, limit: 5, minRelevanceScore: 0.77);\n", + "\n", + "i = 0;\n", + "await foreach (var memory in memories)\n", + "{\n", + " Console.WriteLine($\"Result {++i}:\");\n", + " Console.WriteLine(\" URL: : \" + memory.Metadata.Id);\n", + " Console.WriteLine(\" Title : \" + memory.Metadata.Description);\n", + " Console.WriteLine(\" Relevance: \" + memory.Relevance);\n", + " Console.WriteLine();\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now you might be wondering what happens if you have so much data that it doesn't fit into your RAM? That's where you want to make use of an external Vector Database made specifically for storing and retrieving embeddings.\n", + "\n", + "Stay tuned for that!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dotnet/notebooks/07-DALL-E-2.ipynb b/dotnet/notebooks/07-DALL-E-2.ipynb new file mode 100644 index 000000000000..8bdd42abde6e --- /dev/null +++ b/dotnet/notebooks/07-DALL-E-2.ipynb @@ -0,0 +1,228 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generating images with AI\n", + "\n", + "This notebook demonstrates how to use OpenAI DALL-E 2 to generate images, in combination with other LLM features like text and embedding generation.\n", + "\n", + "Here, we use Chat Completion to generate a random image description and DALL-E 2 to create an image from that description, showing the image inline.\n", + "\n", + "Lastly, the notebook asks the user to describe the image. The embedding of the user's description is compared to the original description, using Cosine Similarity, and returning a score from 0 to 1, where 1 means exact match." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "tags": [], + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "// Usual setup: importing Semantic Kernel SDK and SkiaSharp, used to display images inline.\n", + "\n", + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "#r \"nuget: SkiaSharp, 2.88.3\"\n", + "\n", + "#!import config/Settings.cs\n", + "#!import config/Utils.cs\n", + "#!import config/SkiaUtils.cs\n", + "\n", + "using Microsoft.SemanticKernel;\n", + "using Microsoft.SemanticKernel.AI.ImageGeneration; \n", + "using Microsoft.SemanticKernel.AI.Embeddings;\n", + "using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations;\n", + "using Microsoft.SemanticKernel.Connectors.AI.OpenAI;" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup, using three AI services: images, text, embedding\n", + "\n", + "The notebook uses:\n", + "\n", + "* **OpenAI Dall-E 2** to transform the image description into an image\n", + "* **text-embedding-ada-002** to compare your guess against the real image description\n", + "\n", + "**Note:**: For Azure OpenAI, your endpoint should have DALL-E API enabled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "// Load OpenAI credentials from config/settings.json\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "// Configure the three AI features: text embedding (using Ada), text completion (using DaVinci 3), image generation (DALL-E 2)\n", + "var builder = new KernelBuilder();\n", + "\n", + "if(useAzureOpenAI)\n", + "{\n", + " builder.WithAzureTextEmbeddingGenerationService(\"text-embedding-ada-002\", azureEndpoint, apiKey);\n", + " builder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + " builder.WithAzureOpenAIImageGenerationService(azureEndpoint, apiKey);\n", + "}\n", + "else\n", + "{\n", + " builder.WithOpenAITextEmbeddingGenerationService(\"text-embedding-ada-002\", apiKey, orgId);\n", + " builder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + " builder.WithOpenAIImageGenerationService(apiKey, orgId);\n", + "}\n", + " \n", + "var kernel = builder.Build();\n", + "\n", + "// Get AI service instance used to generate images\n", + "var dallE = kernel.GetService();\n", + "\n", + "// Get AI service instance used to extract embedding from a text\n", + "var textEmbedding = kernel.GetService();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generate a (random) image with DALL-E 2\n", + "\n", + "**genImgDescription** is a Semantic Function used to generate a random image description. \n", + "The function takes in input a random number to increase the diversity of its output.\n", + "\n", + "The random image description is then given to **Dall-E 2** asking to create an image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "tags": [], + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "// Create a semantic function that generate a random image description.\n", + "var genImgDescription = kernel.CreateSemanticFunction(\n", + " \"Think about an artificial object correlated to number {{$input}}. \" +\n", + " \"Describe the image with one detailed sentence. The description cannot contain numbers.\", \n", + " requestSettings: new OpenAIRequestSettings { MaxTokens = 256, Temperature = 1 });\n", + "\n", + "var random = new Random().Next(0, 200);\n", + "var imageDescriptionResult = await kernel.RunAsync($\"{random}\", genImgDescription);\n", + "var imageDescription = imageDescriptionResult.GetValue();\n", + "\n", + "// Use DALL-E 2 to generate an image. OpenAI in this case returns a URL (though you can ask to return a base64 image)\n", + "var imageUrl = await dallE.GenerateImageAsync(imageDescription.Trim(), 512, 512);\n", + "\n", + "await SkiaUtils.ShowImage(imageUrl, 512, 512);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Let's play a guessing game\n", + "\n", + "Try to guess what the image is about, describing the content.\n", + "\n", + "You'll get a score at the end 😉" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "tags": [], + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "// Prompt the user to guess what the image is\n", + "var guess = await InteractiveKernel.GetInputAsync(\"Describe the image in your words\");\n", + "\n", + "// Compare user guess with real description and calculate score\n", + "var origEmbedding = await textEmbedding.GenerateEmbeddingsAsync(new List { imageDescription } );\n", + "var guessEmbedding = await textEmbedding.GenerateEmbeddingsAsync(new List { guess } );\n", + "var similarity = origEmbedding.First().Span.CosineSimilarity(guessEmbedding.First().Span);\n", + "\n", + "Console.WriteLine($\"Your description:\\n{Utils.WordWrap(guess, 90)}\\n\");\n", + "Console.WriteLine($\"Real description:\\n{Utils.WordWrap(imageDescription.Trim(), 90)}\\n\");\n", + "Console.WriteLine($\"Score: {similarity:0.00}\\n\\n\");\n", + "\n", + "//Uncomment this line to see the URL provided by OpenAI\n", + "//Console.WriteLine(imageUrl);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "file_extension": ".cs", + "mimetype": "text/x-csharp", + "name": "C#", + "pygments_lexer": "csharp", + "version": "11.0" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/dotnet/notebooks/08-chatGPT-with-DALL-E-2.ipynb b/dotnet/notebooks/08-chatGPT-with-DALL-E-2.ipynb new file mode 100644 index 000000000000..532a7b640f89 --- /dev/null +++ b/dotnet/notebooks/08-chatGPT-with-DALL-E-2.ipynb @@ -0,0 +1,243 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using ChatGPT with the Semantic Kernel featuring DALL-E 2\n", + "\n", + "This notebook shows how to make use of the new ChatCompletion API from OpenAI, popularized by ChatGPT. This API brings a new ChatML schema which is different from the TextCompletion API. While the text completion API expects input a prompt and returns a simple string, the chat completion API expects in input a Chat history and returns a new message:\n", + "\n", + "```\n", + "messages=[\n", + " { \"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " { \"role\": \"user\", \"content\": \"Who won the world series in 2020?\"},\n", + " { \"role\": \"assistant\", \"content\": \"The Los Angeles Dodgers won the World Series in 2020.\"},\n", + " { \"role\": \"user\", \"content\": \"Where was it played?\"}\n", + "]\n", + "```\n", + "\n", + "Note that there are three message types:\n", + "\n", + "1. A System message is used to give instructions to the chat model, e.g. setting the context and the kind of conversation your app is offering.\n", + "2. User messages store the data received from the user of your app.\n", + "3. Assistant messages store the replies generated by the LLM model. \n", + "\n", + "Your app is responsible for adding information to the chat history and maintaining this object. The Chat Completion API is stateless, and returns only new messages, that your app can use, e.g. to execute commands, generate images, or simply continue the conversation." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When deciding between which one to use, know that ChatGPT models (i.e. gpt-3.5-turbo) are optimized for chat applications and have been fine-tuned for instruction-following and dialogue. As such, for creating semantic plugins with the Semantic Kernel, users may still find the TextCompletion model better suited for certain use cases.\n", + "\n", + "The code below shows how to setup SK with ChatGPT, how to manage the Chat history object, and to make things a little more interesting asks ChatGPT to reply with image descriptions instead so we can have a dialog using images, leveraging DALL-E 2 integration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "tags": [], + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "// Usual setup: importing Semantic Kernel SDK and SkiaSharp, used to display images inline.\n", + "\n", + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "#r \"nuget: SkiaSharp, 2.88.3\"\n", + "\n", + "#!import config/Settings.cs\n", + "#!import config/Utils.cs\n", + "#!import config/SkiaUtils.cs\n", + "\n", + "using Microsoft.SemanticKernel;\n", + "using Microsoft.SemanticKernel.AI.ImageGeneration;\n", + "using Microsoft.SemanticKernel.AI.ChatCompletion;\n", + "using Microsoft.SemanticKernel.Connectors.AI.OpenAI;" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The notebook uses:\n", + "\n", + "* **OpenAI ChatGPT** to chat with the user\n", + "* **OpenAI Dall-E 2** to transform messages into images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "// Load OpenAI credentials from config/settings.json\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "// Configure the two AI features: OpenAI Chat and DALL-E 2 for image generation\n", + "var builder = new KernelBuilder();\n", + "\n", + "if(useAzureOpenAI)\n", + "{\n", + " builder.WithAzureChatCompletionService(\"gpt-35-turbo\", azureEndpoint, apiKey);\n", + " builder.WithAzureOpenAIImageGenerationService(azureEndpoint, apiKey);\n", + "}\n", + "else\n", + "{\n", + " builder.WithOpenAIChatCompletionService(\"gpt-3.5-turbo\", apiKey, orgId);\n", + " builder.WithOpenAIImageGenerationService(apiKey, orgId);\n", + "}\n", + "\n", + "var kernel = builder.Build();\n", + "\n", + "// Get AI service instance used to generate images\n", + "var dallE = kernel.GetService();\n", + "\n", + "// Get AI service instance used to manage the user chat\n", + "var chatGPT = kernel.GetService();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chat configuration\n", + "\n", + "Before starting the chat, we create a new chat object with some instructions, which are included in the chat history. \n", + "\n", + "The instructions tell OpenAI what kind of chat we want to have, in this case we ask to reply with \"image descriptions\", so that we can chain ChatGPT with DALL-E 2." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "tags": [], + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion;\n", + "\n", + "var systemMessage = \"You're chatting with a user. Instead of replying directly to the user\"\n", + " + \" provide a description of a cartoonish image that expresses what you want to say.\"\n", + " + \" The user won't see your message, they will see only the image.\"\n", + " + \" Describe the image with details in one sentence.\";\n", + "\n", + "var chat = (OpenAIChatHistory)chatGPT.CreateNewChat(systemMessage);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Let's chat\n", + "\n", + "Run the following code to start the chat. The chat consists of a loop with these main steps:\n", + "\n", + "1. Ask the user (you) for a message. The user enters a message. Add the user message into the Chat History object.\n", + "2. Send the chat object to AI asking to generate a response. Add the bot message into the Chat History object.\n", + "3. Show the answer to the user. In our case before showing the answer, generate an image and show that to the user too.\n", + "\n", + "*Note: to stop the chat in VS Code press ESC on the kyboard or the \"stop\" button on the left side.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "while (true)\n", + "{\n", + " // 1. Ask the user for a message. The user enters a message. Add the user message into the Chat History object.\n", + " var userMessage = await InteractiveKernel.GetInputAsync(\"Your message\");\n", + " Console.WriteLine($\"User: {userMessage}\");\n", + " chat.AddUserMessage(userMessage);\n", + "\n", + " // 2. Send the chat object to AI asking to generate a response. Add the bot message into the Chat History object.\n", + " string assistantReply = await chatGPT.GenerateMessageAsync(chat, new OpenAIRequestSettings());\n", + " chat.AddAssistantMessage(assistantReply);\n", + "\n", + " // 3. Show the reply as an image\n", + " Console.WriteLine($\"\\nBot:\");\n", + " var imageUrl = await dallE.GenerateImageAsync(assistantReply, 256, 256);\n", + " await SkiaUtils.ShowImage(imageUrl, 256, 256);\n", + " Console.WriteLine($\"[{assistantReply}]\\n\");\n", + "}" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "file_extension": ".cs", + "mimetype": "text/x-csharp", + "name": "C#", + "pygments_lexer": "csharp", + "version": "11.0" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/dotnet/notebooks/09-memory-with-chroma.ipynb b/dotnet/notebooks/09-memory-with-chroma.ipynb new file mode 100644 index 000000000000..9e1257fd3121 --- /dev/null +++ b/dotnet/notebooks/09-memory-with-chroma.ipynb @@ -0,0 +1,576 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Building Semantic Memory with Embeddings\n", + "\n", + "In this notebook, we show how to use [Chroma](https://www.trychroma.com/) with Semantic Kernel to create even more\n", + "intelligent applications. We assume that you are already familiar with the concepts of Semantic Kernel\n", + "and memory. [Previously](04-context-variables-chat.ipynb), we have used `context variables` to pass\n", + "additional text into prompts, enriching them with more context for a basic chat experience.\n", + "\n", + "However, relying solely on context variables has its limitations, such as the model's token limit.\n", + "To overcome these limitations, we will use **SK Semantic Memory**, leveraging Chroma as a persistent\n", + "Semantic Memory Storage.\n", + "\n", + "**Chroma** is an open-source embedding database designed to make it easy to build Language Model\n", + "applications by making knowledge, facts, and plugins pluggable for LLMs. It allows us to store and\n", + "retrieve information in a way that can be easily utilized by the models, enabling both short-term\n", + "and long-term memory for more advanced applications. In this notebook, we will showcase how to\n", + "effectively use Chroma with the Semantic Kernel for a powerful application experience.\n", + "\n", + "**Note:** This example is verified using Chroma version **0.4.10**. Any higher versions may introduce incompatibility." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "#r \"nuget: Microsoft.SemanticKernel.Connectors.Memory.Chroma, 1.0.0-beta1\"\n", + "#r \"nuget: System.Linq.Async, 6.0.1\"\n", + "\n", + "#!import config/Settings.cs\n", + "\n", + "using System;\n", + "using System.Collections.Generic;\n", + "using System.Linq;\n", + "using System.Threading.Tasks;\n", + "using Microsoft.SemanticKernel;\n", + "using Microsoft.SemanticKernel.Connectors.Memory.Chroma;\n", + "using Microsoft.SemanticKernel.Memory;\n", + "using Microsoft.SemanticKernel.Plugins.Memory;\n", + "\n", + "var kernelBuilder = new KernelBuilder();\n", + "\n", + "// Configure AI backend used by the kernel\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "if (useAzureOpenAI)\n", + " kernelBuilder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", + "else\n", + " kernelBuilder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", + "\n", + "var kernel = kernelBuilder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to use memory, we need to instantiate the Memory Plugin with a Memory Storage\n", + "and an Embedding backend. In this example, we make use of the `ChromaMemoryStore`,\n", + "leveraging [Chroma](https://www.trychroma.com/), an open source embedding database\n", + "you can run locally and in the cloud.\n", + "\n", + "To run Chroma locally, here's a quick script to download Chroma source and run it using Docker:\n", + "\n", + "```shell\n", + "git clone https://github.com/chroma-core/chroma.git\n", + "cd chroma\n", + "docker-compose up --build\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "using Microsoft.SemanticKernel.Connectors.AI.OpenAI;\n", + "\n", + "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", + "\n", + "var memoryBuilder = new MemoryBuilder();\n", + "\n", + "if (useAzureOpenAI)\n", + "{\n", + " memoryBuilder.WithAzureTextEmbeddingGenerationService(\"text-embedding-ada-002\", azureEndpoint, apiKey);\n", + "}\n", + "else\n", + "{\n", + " memoryBuilder.WithOpenAITextEmbeddingGenerationService(\"text-embedding-ada-002\", apiKey);\n", + "}\n", + "\n", + "var chromaMemoryStore = new ChromaMemoryStore(\"http://127.0.0.1:8000\");\n", + "\n", + "memoryBuilder.WithMemoryStore(chromaMemoryStore);\n", + "\n", + "var memory = memoryBuilder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At its core, Semantic Memory is a set of data structures that allows to store\n", + "the meaning of text that come from different data sources, and optionally to\n", + "store the source text and other metadata.\n", + "\n", + "The text can be from the web, e-mail providers, chats, a database, or from your\n", + "local directory, and are hooked up to the Semantic Kernel through memory connectors.\n", + "\n", + "The texts are embedded, sort of \"compressed\", into a vector of floats that representing\n", + "mathematically the text content and meaning.\n", + "\n", + "You can read more about embeddings [here](https://aka.ms/sk/embeddings)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Manually adding memories\n", + "\n", + "Let's create some initial memories \"About Me\". We can add memories to `ChromaMemoryStore` by using `SaveInformationAsync`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "const string MemoryCollectionName = \"aboutMe\";\n", + "\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info1\", text: \"My name is Andrea\");\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info2\", text: \"I currently work as a tourist operator\");\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info3\", text: \"I currently live in Seattle and have been living there since 2005\");\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info4\", text: \"I visited France and Italy five times since 2015\");\n", + "await memory.SaveInformationAsync(MemoryCollectionName, id: \"info5\", text: \"My family is from New York\");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's try searching the memory:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var questions = new[]\n", + "{\n", + " \"what is my name?\",\n", + " \"where do I live?\",\n", + " \"where is my family from?\",\n", + " \"where have I travelled?\",\n", + " \"what do I do for work?\",\n", + "};\n", + "\n", + "foreach (var q in questions)\n", + "{\n", + " var response = await memory.SearchAsync(MemoryCollectionName, q, limit: 1, minRelevanceScore: 0.5).FirstOrDefaultAsync();\n", + " Console.WriteLine(q + \" \" + response?.Metadata.Text);\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's now revisit our chat sample from the [previous notebook](04-context-variables-chat.ipynb).\n", + "If you remember, we used context variables to fill the prompt with a `history` that continuously got populated as we chatted with the bot. Let's add also memory to it!" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is done by using the `TextMemoryPlugin` which exposes the `recall` native function.\n", + "\n", + "`recall` takes an input ask and performs a similarity search on the contents that have\n", + "been embedded in the Memory Store. By default, `recall` returns the most relevant memory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "using Microsoft.SemanticKernel.Plugins.Memory;\n", + "\n", + "// TextMemoryPlugin provides the \"recall\" function\n", + "kernel.ImportFunctions(new TextMemoryPlugin(memory));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "const string skPrompt = @\"\n", + "ChatBot can have a conversation with you about any topic.\n", + "It can give explicit instructions or say 'I don't know' if it does not have an answer.\n", + "\n", + "Information about me, from previous conversations:\n", + "- {{$fact1}} {{recall $fact1}}\n", + "- {{$fact2}} {{recall $fact2}}\n", + "- {{$fact3}} {{recall $fact3}}\n", + "- {{$fact4}} {{recall $fact4}}\n", + "- {{$fact5}} {{recall $fact5}}\n", + "\n", + "Chat:\n", + "{{$history}}\n", + "User: {{$userInput}}\n", + "ChatBot: \";\n", + "\n", + "var chatFunction = kernel.CreateSemanticFunction(skPrompt, requestSettings: new OpenAIRequestSettings { MaxTokens = 200, Temperature = 0.8 });" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `RelevanceParam` is used in memory search and is a measure of the relevance score from 0.0 to 1.0, where 1.0 means a perfect match. We encourage users to experiment with different values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var context = kernel.CreateNewContext();\n", + "\n", + "context.Variables[\"fact1\"] = \"what is my name?\";\n", + "context.Variables[\"fact2\"] = \"where do I live?\";\n", + "context.Variables[\"fact3\"] = \"where is my family from?\";\n", + "context.Variables[\"fact4\"] = \"where have I travelled?\";\n", + "context.Variables[\"fact5\"] = \"what do I do for work?\";\n", + "\n", + "context.Variables[TextMemoryPlugin.CollectionParam] = MemoryCollectionName;\n", + "context.Variables[TextMemoryPlugin.RelevanceParam] = \"0.6\";" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we've included our memories, let's chat!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var history = \"\";\n", + "context.Variables[\"history\"] = history;\n", + "Func Chat = async (string input) => {\n", + " // Save new message in the context variables\n", + " context.Variables[\"userInput\"] = input;\n", + "\n", + " // Process the user message and get an answer\n", + " var answer = await chatFunction.InvokeAsync(context);\n", + "\n", + " // Append the new interaction to the chat history\n", + " history += $\"\\nUser: {input}\\nChatBot: {answer.GetValue()}\\n\";\n", + " context.Variables[\"history\"] = history;\n", + " \n", + " // Show the bot response\n", + " Console.WriteLine(\"ChatBot: \" + context);\n", + "};" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "await Chat(\"Hello, I think we've met before, remember? my name is...\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "await Chat(\"I want to plan a trip and visit my family. Do you know where that is?\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "await Chat(\"Great! What are some fun things to do there?\");" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Adding documents to your memory\n", + "\n", + "Many times in your applications you'll want to bring in external documents into your memory. Let's see how we can do this using ChromaMemoryStore.\n", + "\n", + "Let's first get some data using some of the links in the Semantic Kernel repo." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "const string memoryCollectionName = \"SKGitHub\";\n", + "\n", + "var githubFiles = new Dictionary()\n", + "{\n", + " [\"https://github.com/microsoft/semantic-kernel/blob/main/README.md\"]\n", + " = \"README: Installation, getting started, and how to contribute\",\n", + " [\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/02-running-prompts-from-file.ipynb\"]\n", + " = \"Jupyter notebook describing how to pass prompts from a file to a semantic plugin or function\",\n", + " [\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/00-getting-started.ipynb\"]\n", + " = \"Jupyter notebook describing how to get started with the Semantic Kernel\",\n", + " [\"https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins/ChatPlugin/ChatGPT\"]\n", + " = \"Sample demonstrating how to create a chat plugin interfacing with ChatGPT\",\n", + " [\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Plugins/Plugins.Memory/VolatileMemoryStore.cs\"]\n", + " = \"C# class that defines a volatile embedding store\",\n", + " [\"https://github.com/microsoft/semantic-kernel/tree/main/samples/dotnet/KernelHttpServer/README.md\"]\n", + " = \"README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4\",\n", + " [\"https://github.com/microsoft/semantic-kernel/tree/main/samples/apps/chat-summary-webapp-react/README.md\"]\n", + " = \"README: README associated with a sample starter react-based chat summary webapp\",\n", + "};" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's build a new Memory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "var memoryBuilder = new MemoryBuilder();\n", + "\n", + "if (useAzureOpenAI)\n", + "{\n", + " memoryBuilder.WithAzureTextEmbeddingGenerationService(\"text-embedding-ada-002\", azureEndpoint, apiKey);\n", + "}\n", + "else\n", + "{\n", + " memoryBuilder.WithOpenAITextEmbeddingGenerationService(\"text-embedding-ada-002\", apiKey);\n", + "}\n", + "\n", + "var chromaMemoryStore = new ChromaMemoryStore(\"http://127.0.0.1:8000\");\n", + "\n", + "memoryBuilder.WithMemoryStore(chromaMemoryStore);\n", + "\n", + "var memory = memoryBuilder.Build();" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's add these files to ChromaMemoryStore using `SaveReferenceAsync`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "Console.WriteLine(\"Adding some GitHub file URLs and their descriptions to Chroma Semantic Memory.\");\n", + "var i = 0;\n", + "foreach (var entry in githubFiles)\n", + "{\n", + " await memory.SaveReferenceAsync(\n", + " collection: memoryCollectionName,\n", + " description: entry.Value,\n", + " text: entry.Value,\n", + " externalId: entry.Key,\n", + " externalSourceName: \"GitHub\"\n", + " );\n", + " Console.WriteLine($\" URL {++i} saved\");\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "string ask = \"I love Jupyter notebooks, how should I get started?\";\n", + "Console.WriteLine(\"===========================\\n\" +\n", + " \"Query: \" + ask + \"\\n\");\n", + "\n", + "var memories = memory.SearchAsync(memoryCollectionName, ask, limit: 5, minRelevanceScore: 0.6);\n", + "\n", + "i = 0;\n", + "await foreach (var memory in memories)\n", + "{\n", + " Console.WriteLine($\"Result {++i}:\");\n", + " Console.WriteLine(\" URL: : \" + memory.Metadata.Id);\n", + " Console.WriteLine(\" Title : \" + memory.Metadata.Description);\n", + " Console.WriteLine(\" Relevance: \" + memory.Relevance);\n", + " Console.WriteLine();\n", + "}" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/samples/notebooks/dotnet/10-BingSearch-using-kernel.ipynb b/dotnet/notebooks/10-BingSearch-using-kernel.ipynb similarity index 77% rename from samples/notebooks/dotnet/10-BingSearch-using-kernel.ipynb rename to dotnet/notebooks/10-BingSearch-using-kernel.ipynb index 1d9108a51edf..17dbafa49f86 100644 --- a/samples/notebooks/dotnet/10-BingSearch-using-kernel.ipynb +++ b/dotnet/notebooks/10-BingSearch-using-kernel.ipynb @@ -11,7 +11,7 @@ "\n", "To use Bing Search you simply need a Bing Search API key. You can get the API key by creating a [Bing Search resource](https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/create-bing-search-service-resource) in Azure. \n", "\n", - "To learn more read the following [documentation](https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/overview)\n" + "To learn more read the following [documentation](https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/overview).\n" ] }, { @@ -35,29 +35,29 @@ }, "outputs": [], "source": [ - "#r \"nuget: Microsoft.SemanticKernel, 0.17.230626.1-preview\"\n", - "#r \"nuget: Microsoft.SemanticKernel.Skills.Web, 0.17.230626.1-preview\"\n", + "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-beta1\"\n", + "#r \"nuget: Microsoft.SemanticKernel.Plugins.Web, 1.0.0-beta1\"\n", "\n", "#!import config/Settings.cs\n", "#!import config/Utils.cs\n", "\n", "using Microsoft.SemanticKernel;\n", - "using Microsoft.SemanticKernel.Skills.Core;\n", - "using Microsoft.SemanticKernel.SkillDefinition;\n", + "using Microsoft.SemanticKernel.Plugins.Core;\n", "using Microsoft.SemanticKernel.Orchestration;\n", "using Microsoft.SemanticKernel.Planning;\n", - "using Microsoft.SemanticKernel.Planning.Sequential;\n", + "using Microsoft.SemanticKernel.Planners;\n", "using Microsoft.SemanticKernel.TemplateEngine;\n", "using InteractiveKernel = Microsoft.DotNet.Interactive.Kernel;\n", + "\n", "var builder = new KernelBuilder();\n", "\n", "// Configure AI backend used by the kernel\n", "var (useAzureOpenAI, model, azureEndpoint, apiKey, orgId) = Settings.LoadFromFile();\n", "\n", "if (useAzureOpenAI)\n", - " builder.WithAzureTextCompletionService(model, azureEndpoint, apiKey);\n", + " builder.WithAzureChatCompletionService(model, azureEndpoint, apiKey);\n", "else\n", - " builder.WithOpenAITextCompletionService(model, apiKey, orgId);\n", + " builder.WithOpenAIChatCompletionService(model, apiKey, orgId);\n", "\n", "var kernel = builder.Build();" ] @@ -83,8 +83,8 @@ }, "outputs": [], "source": [ - "using Microsoft.SemanticKernel.Skills.Web;\n", - "using Microsoft.SemanticKernel.Skills.Web.Bing;" + "using Microsoft.SemanticKernel.Plugins.Web;\n", + "using Microsoft.SemanticKernel.Plugins.Web.Bing;" ] }, { @@ -110,7 +110,7 @@ "source": [ "using InteractiveKernel = Microsoft.DotNet.Interactive.Kernel;\n", "\n", - "string BING_KEY = await InteractiveKernel.GetPasswordAsync(\"Please enter your Bing Search Key\");" + "string BING_KEY = (await InteractiveKernel.GetPasswordAsync(\"Please enter your Bing Search Key\")).GetClearTextPassword();" ] }, { @@ -118,7 +118,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Below are some examples on how [`WebSearchEngineSkill`](../../../dotnet/src/Skills/Skills.Web/WebSearchEngineSkill.cs) can be used in SK. " + "Below are some examples on how [`WebSearchEnginePlugin`](../src/Plugins/Plugins.Web/WebSearchEnginePlugin.cs) can be used in SK. " ] }, { @@ -140,11 +140,13 @@ "\n", " // Run \n", " var question = \"What is quantum tunnelling\";\n", - " var bingResult = await kernel.Func(\"bing\", \"search\").InvokeAsync(question);\n", + " var function = kernel.Functions.GetFunction(\"bing\", \"search\");\n", + " var bingResult = await kernel.RunAsync(question, function);\n", + " var bingResultString = bingResult.GetValue();\n", "\n", " Console.WriteLine(question);\n", " Console.WriteLine(\"----\");\n", - " Console.WriteLine(bingResult);\n", + " Console.WriteLine(bingResultString);\n", " Console.WriteLine();\n", "\n", " /* OUTPUT:\n", @@ -180,9 +182,11 @@ " //The following function only works in interactive notebooks\n", " string question = await InteractiveKernel.GetInputAsync(\"Please ask your question\"); \n", "\n", - " var bingResult = await kernel.Func(\"bing\", \"search\").InvokeAsync(question);\n", + " var function = kernel.Functions.GetFunction(\"bing\", \"search\");\n", + " var bingResult = await kernel.RunAsync(question, function);\n", + " var bingResultString = bingResult.GetValue();\n", "\n", - " Console.WriteLine(bingResult);\n", + " Console.WriteLine(bingResultString);\n", "}" ] }, @@ -207,14 +211,13 @@ }, "outputs": [], "source": [ - "// Load Bing skill\n", - "using (var bingConnector = new BingConnector(BING_KEY))\n", - "{\n", - " kernel.ImportSkill(new WebSearchEngineSkill(bingConnector), \"bing\");\n", + "// Load Bing plugin\n", + "var bingConnector = new BingConnector(BING_KEY);\n", "\n", - " //await Example1Async(kernel);\n", - " //await Example2Async(kernel);\n", - "}" + "kernel.ImportFunctions(new WebSearchEnginePlugin(bingConnector), \"bing\");\n", + "\n", + "//await Example1Async(kernel);\n", + "//await Example2Async(kernel);" ] } ], diff --git a/dotnet/notebooks/README.md b/dotnet/notebooks/README.md new file mode 100644 index 000000000000..b7df44d5b309 --- /dev/null +++ b/dotnet/notebooks/README.md @@ -0,0 +1,119 @@ +# Semantic Kernel C# Notebooks + +The current folder contains a few C# Jupyter Notebooks that demonstrate how to get started with +the Semantic Kernel. The notebooks are organized in order of increasing complexity. + +To run the notebooks, we recommend the following steps: + +- [Install .NET 7](https://dotnet.microsoft.com/download/dotnet/7.0) +- [Install Visual Studio Code (VS Code)](https://code.visualstudio.com) +- Launch VS Code and [install the "Polyglot" extension](https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.dotnet-interactive-vscode). + Min version required: v1.0.4102020 (Feb 2022). + +The steps above should be sufficient, you can now **open all the C# notebooks in VS Code**. + +VS Code screenshot example: + +![image](https://user-images.githubusercontent.com/371009/216761942-1861635c-b4b7-4059-8ecf-590d93fe6300.png) + +## Set your OpenAI API key + +To start using these notebooks, be sure to add the appropriate API keys to `config/settings.json`. + +You can create the file manually or run [the Setup notebook](0-AI-settings.ipynb). + +For Azure OpenAI: + +```json +{ + "type": "azure", + "model": "...", // Azure OpenAI Deployment Name + "endpoint": "...", // Azure OpenAI endpoint + "apikey": "..." // Azure OpenAI key +} +``` + +For OpenAI: + +```json +{ + "type": "openai", + "model": "gpt-3.5-turbo", // OpenAI model name + "apikey": "...", // OpenAI API Key + "org": "" // only for OpenAI accounts with multiple orgs +} +``` + +If you need an Azure OpenAI key, go [here](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/quickstart?pivots=rest-api). +If you need an OpenAI key, go [here](https://platform.openai.com/account/api-keys) + +# Topics + +Before starting, make sure you configured `config/settings.json`, +see the previous section. + +For a quick dive, look at the [getting started notebook](00-getting-started.ipynb). + +1. [Loading and configuring Semantic Kernel](01-basic-loading-the-kernel.ipynb) +2. [Running AI prompts from file](02-running-prompts-from-file.ipynb) +3. [Creating Semantic Functions at runtime (i.e. inline functions)](03-semantic-function-inline.ipynb) +4. [Using Context Variables to Build a Chat Experience](04-context-variables-chat.ipynb) +5. [Creating and Executing Plans](05-using-the-planner.ipynb) +6. [Building Memory with Embeddings](06-memory-and-embeddings.ipynb) +7. [Creating images with DALL-E 2](07-DALL-E-2.ipynb) +8. [Chatting with ChatGPT and Images](08-chatGPT-with-DALL-E-2.ipynb) + +# Run notebooks in the browser with JupyterLab + +You can run the notebooks also in the browser with JupyterLab. These steps +should be sufficient to start: + +Install Python 3, Pip and .NET 7 in your system, then: + + pip install jupyterlab + dotnet tool install -g Microsoft.dotnet-interactive + dotnet tool update -g Microsoft.dotnet-interactive + dotnet interactive jupyter install + +This command will confirm that Jupyter now supports C# notebooks: + + jupyter kernelspec list + +Enter the notebooks folder, and run this to launch the browser interface: + + jupyter-lab + +![image](https://user-images.githubusercontent.com/371009/216756924-41657aa0-5574-4bc9-9bdb-ead3db7bf93a.png) + +# Troubleshooting + +## Nuget + +If you are unable to get the Nuget package, first list your Nuget sources: + +```sh +dotnet nuget list source +``` + +If you see `No sources found.`, add the NuGet official package source: + +```sh +dotnet nuget add source "https://api.nuget.org/v3/index.json" --name "nuget.org" +``` + +Run `dotnet nuget list source` again to verify the source was added. + +## Polyglot Notebooks + +If somehow the notebooks don't work, run these commands: + +- Install .NET Interactive: `dotnet tool install -g Microsoft.dotnet-interactive` +- Register .NET kernels into Jupyter: `dotnet interactive jupyter install` (this might return some errors, ignore them) +- If you are still stuck, read the following pages: + - https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.dotnet-interactive-vscode + - https://devblogs.microsoft.com/dotnet/net-core-with-juypter-notebooks-is-here-preview-1/ + - https://docs.servicestack.net/jupyter-notebooks-csharp + - https://developers.refinitiv.com/en/article-catalog/article/using--net-core-in-jupyter-notebook + +Note: ["Polyglot Notebooks" used to be called ".NET Interactive Notebooks"](https://devblogs.microsoft.com/dotnet/dotnet-interactive-notebooks-is-now-polyglot-notebooks/), +so you might find online some documentation referencing the old name. diff --git a/samples/notebooks/dotnet/config/.gitignore b/dotnet/notebooks/config/.gitignore similarity index 100% rename from samples/notebooks/dotnet/config/.gitignore rename to dotnet/notebooks/config/.gitignore diff --git a/samples/notebooks/dotnet/config/Settings.cs b/dotnet/notebooks/config/Settings.cs similarity index 96% rename from samples/notebooks/dotnet/config/Settings.cs rename to dotnet/notebooks/config/Settings.cs index 3a6fd3bbbdc5..498d5afaae12 100644 --- a/samples/notebooks/dotnet/config/Settings.cs +++ b/dotnet/notebooks/config/Settings.cs @@ -59,7 +59,7 @@ public static async Task AskModel(bool _useAzureOpenAI = true, string co else { // Use the best model by default, and reduce the setup friction, particularly in VS Studio. - model = "text-davinci-003"; + model = "gpt-3.5-turbo"; } } @@ -92,12 +92,12 @@ public static async Task AskApiKey(bool _useAzureOpenAI = true, string c { if (useAzureOpenAI) { - apiKey = await InteractiveKernel.GetPasswordAsync("Please enter your Azure OpenAI API key"); + apiKey = (await InteractiveKernel.GetPasswordAsync("Please enter your Azure OpenAI API key")).GetClearTextPassword(); orgId = ""; } else { - apiKey = await InteractiveKernel.GetPasswordAsync("Please enter your OpenAI API key"); + apiKey = (await InteractiveKernel.GetPasswordAsync("Please enter your OpenAI API key")).GetClearTextPassword(); } } diff --git a/samples/notebooks/dotnet/config/SkiaUtils.cs b/dotnet/notebooks/config/SkiaUtils.cs similarity index 100% rename from samples/notebooks/dotnet/config/SkiaUtils.cs rename to dotnet/notebooks/config/SkiaUtils.cs diff --git a/samples/notebooks/dotnet/config/Utils.cs b/dotnet/notebooks/config/Utils.cs similarity index 100% rename from samples/notebooks/dotnet/config/Utils.cs rename to dotnet/notebooks/config/Utils.cs diff --git a/samples/notebooks/dotnet/config/settings.json.azure-example b/dotnet/notebooks/config/settings.json.azure-example similarity index 100% rename from samples/notebooks/dotnet/config/settings.json.azure-example rename to dotnet/notebooks/config/settings.json.azure-example diff --git a/samples/notebooks/dotnet/config/settings.json.openai-example b/dotnet/notebooks/config/settings.json.openai-example similarity index 80% rename from samples/notebooks/dotnet/config/settings.json.openai-example rename to dotnet/notebooks/config/settings.json.openai-example index 7b22ea437065..9d5535489ea1 100644 --- a/samples/notebooks/dotnet/config/settings.json.openai-example +++ b/dotnet/notebooks/config/settings.json.openai-example @@ -1,7 +1,7 @@ { "type": "openai", "endpoint": "NOT-USED-BUT-REQUIRED-FOR-PARSER", - "model": "text-davinci-003", + "model": "gpt-3.5-turbo", "apikey": "... your OpenAI key ...", "org": "" } \ No newline at end of file diff --git a/dotnet/nuget/nuget-package.props b/dotnet/nuget/nuget-package.props index 5dba582410ed..69f17536329e 100644 --- a/dotnet/nuget/nuget-package.props +++ b/dotnet/nuget/nuget-package.props @@ -1,7 +1,7 @@ - 0.18 + 1.0.0-beta2 Debug;Release;Publish true diff --git a/dotnet/samples/ApplicationInsightsExample/ApplicationInsightsExample.csproj b/dotnet/samples/ApplicationInsightsExample/ApplicationInsightsExample.csproj index 99c7aad4187a..3957425441f5 100644 --- a/dotnet/samples/ApplicationInsightsExample/ApplicationInsightsExample.csproj +++ b/dotnet/samples/ApplicationInsightsExample/ApplicationInsightsExample.csproj @@ -19,14 +19,13 @@ - - - - - - - + + + + + + diff --git a/dotnet/samples/ApplicationInsightsExample/Program.cs b/dotnet/samples/ApplicationInsightsExample/Program.cs index e90cea2e1aa8..0432ef34cb83 100644 --- a/dotnet/samples/ApplicationInsightsExample/Program.cs +++ b/dotnet/samples/ApplicationInsightsExample/Program.cs @@ -12,14 +12,12 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.ApplicationInsights; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Planners; using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.Planning.Action; -using Microsoft.SemanticKernel.Planning.Sequential; -using Microsoft.SemanticKernel.Planning.Stepwise; -using Microsoft.SemanticKernel.Skills.Core; -using Microsoft.SemanticKernel.Skills.Web; -using Microsoft.SemanticKernel.Skills.Web.Bing; -using NCalcSkills; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.Plugins.Web; +using Microsoft.SemanticKernel.Plugins.Web.Bing; +using NCalcPlugins; /// /// Example of telemetry in Semantic Kernel using Application Insights within console application. @@ -33,14 +31,18 @@ public sealed class Program /// is set by default. /// will enable logging with more detailed information, including sensitive data. Should not be used in production. /// - private static LogLevel LogLevel = LogLevel.Information; + private const LogLevel MinLogLevel = LogLevel.Information; + /// + /// The main entry point for the application. + /// + /// A representing the asynchronous operation. public static async Task Main() { var serviceProvider = GetServiceProvider(); var telemetryClient = serviceProvider.GetRequiredService(); - var logger = serviceProvider.GetRequiredService>(); + var loggerFactory = serviceProvider.GetRequiredService(); using var meterListener = new MeterListener(); using var activityListener = new ActivityListener(); @@ -48,8 +50,8 @@ public static async Task Main() ConfigureMetering(meterListener, telemetryClient); ConfigureTracing(activityListener, telemetryClient); - var kernel = GetKernel(logger); - var planner = GetSequentialPlanner(kernel, logger); + var kernel = GetKernel(loggerFactory); + var planner = GetSequentialPlanner(kernel, loggerFactory); try { @@ -66,7 +68,7 @@ public static async Task Main() var result = await kernel.RunAsync(plan); Console.WriteLine("Result:"); - Console.WriteLine(result.Result); + Console.WriteLine(result.GetValue()); } finally { @@ -92,8 +94,8 @@ private static void ConfigureApplicationInsightsTelemetry(ServiceCollection serv services.AddLogging(loggingBuilder => { - loggingBuilder.AddFilter(typeof(Program).FullName, LogLevel); - loggingBuilder.SetMinimumLevel(LogLevel); + loggingBuilder.AddFilter(logLevel => logLevel == MinLogLevel); + loggingBuilder.SetMinimumLevel(MinLogLevel); }); services.AddApplicationInsightsTelemetryWorkerService(options => @@ -102,49 +104,49 @@ private static void ConfigureApplicationInsightsTelemetry(ServiceCollection serv }); } - private static IKernel GetKernel(ILogger logger) + private static IKernel GetKernel(ILoggerFactory loggerFactory) { - var folder = RepoFiles.SampleSkillsPath(); + var folder = RepoFiles.SamplePluginsPath(); var bingConnector = new BingConnector(Env.Var("Bing__ApiKey")); - var webSearchEngineSkill = new WebSearchEngineSkill(bingConnector); + var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector); var kernel = new KernelBuilder() - .WithLogger(logger) + .WithLoggerFactory(loggerFactory) .WithAzureChatCompletionService( Env.Var("AzureOpenAI__ChatDeploymentName"), Env.Var("AzureOpenAI__Endpoint"), Env.Var("AzureOpenAI__ApiKey")) .Build(); - kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill", "WriterSkill"); + kernel.ImportSemanticFunctionsFromDirectory(folder, "SummarizePlugin", "WriterPlugin"); - kernel.ImportSkill(webSearchEngineSkill, "WebSearch"); - kernel.ImportSkill(new LanguageCalculatorSkill(kernel), "advancedCalculator"); - kernel.ImportSkill(new TimeSkill(), "time"); + kernel.ImportFunctions(webSearchEnginePlugin, "WebSearch"); + kernel.ImportFunctions(new LanguageCalculatorPlugin(kernel), "advancedCalculator"); + kernel.ImportFunctions(new TimePlugin(), "time"); return kernel; } private static ISequentialPlanner GetSequentialPlanner( IKernel kernel, - ILogger logger, + ILoggerFactory loggerFactory, int maxTokens = 1024) { var plannerConfig = new SequentialPlannerConfig { MaxTokens = maxTokens }; - return new SequentialPlanner(kernel, plannerConfig).WithInstrumentation(logger); + return new SequentialPlanner(kernel, plannerConfig).WithInstrumentation(loggerFactory); } private static IActionPlanner GetActionPlanner( IKernel kernel, - ILogger logger) + ILoggerFactory loggerFactory) { - return new ActionPlanner(kernel).WithInstrumentation(logger); + return new ActionPlanner(kernel).WithInstrumentation(loggerFactory); } private static IStepwisePlanner GetStepwisePlanner( IKernel kernel, - ILogger logger, + ILoggerFactory loggerFactory, int minIterationTimeMs = 1500, int maxTokens = 2000) { @@ -154,7 +156,7 @@ private static IStepwisePlanner GetStepwisePlanner( MaxTokens = maxTokens }; - return new StepwisePlanner(kernel, plannerConfig).WithInstrumentation(logger); + return new StepwisePlanner(kernel, plannerConfig).WithInstrumentation(loggerFactory); } /// @@ -203,22 +205,22 @@ private static void ConfigureTracing(ActivityListener activityListener, Telemetr var operations = new ConcurrentDictionary>(); // For more detailed tracing we need to attach Activity entity to Application Insights operation manually. - Action activityStarted = activity => + void activityStarted(Activity activity) { var operation = telemetryClient.StartOperation(activity); operation.Telemetry.Type = activity.Kind.ToString(); operations.TryAdd(activity.TraceId.ToString(), operation); - }; + } // We also need to manually stop Application Insights operation when Activity entity is stopped. - Action activityStopped = activity => + void activityStopped(Activity activity) { if (operations.TryRemove(activity.TraceId.ToString(), out var operation)) { telemetryClient.StopOperation(operation); } - }; + } // Subscribe to all traces in Semantic Kernel activityListener.ShouldListenTo = diff --git a/dotnet/samples/ApplicationInsightsExample/RepoUtils/RepoFiles.cs b/dotnet/samples/ApplicationInsightsExample/RepoUtils/RepoFiles.cs index dc15dfed4472..0c7d595b1bad 100644 --- a/dotnet/samples/ApplicationInsightsExample/RepoUtils/RepoFiles.cs +++ b/dotnet/samples/ApplicationInsightsExample/RepoUtils/RepoFiles.cs @@ -6,13 +6,13 @@ internal static class RepoFiles { /// - /// Scan the local folders from the repo, looking for "samples/skills" folder. + /// Scan the local folders from the repo, looking for "samples/plugins" folder. /// - /// The full path to samples/skills - public static string SampleSkillsPath() + /// The full path to samples/plugins + public static string SamplePluginsPath() { const string Parent = "samples"; - const string Folder = "skills"; + const string Folder = "plugins"; bool SearchPath(string pathToFind, out string result, int maxAttempts = 10) { @@ -31,7 +31,7 @@ bool SearchPath(string pathToFind, out string result, int maxAttempts = 10) if (!SearchPath(Parent + Path.DirectorySeparatorChar + Folder, out string path) && !SearchPath(Folder, out path)) { - throw new DirectoryNotFoundException("Skills directory not found. The app needs the skills from the repo to work."); + throw new DirectoryNotFoundException("Plugins directory not found. The app needs the plugins from the repo to work."); } return path; diff --git a/dotnet/samples/KernelSyntaxExamples/Example01_NativeFunctions.cs b/dotnet/samples/KernelSyntaxExamples/Example01_NativeFunctions.cs index 6c68f07d41f7..50c2faed5548 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example01_NativeFunctions.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example01_NativeFunctions.cs @@ -2,7 +2,7 @@ using System; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.Core; +using Microsoft.SemanticKernel.Plugins.Core; // ReSharper disable once InconsistentNaming public static class Example01_NativeFunctions @@ -11,8 +11,8 @@ public static Task RunAsync() { Console.WriteLine("======== Functions ========"); - // Load native skill - var text = new TextSkill(); + // Load native plugin + var text = new TextPlugin(); // Use function without kernel var result = text.Uppercase("ciao!"); diff --git a/dotnet/samples/KernelSyntaxExamples/Example02_Pipeline.cs b/dotnet/samples/KernelSyntaxExamples/Example02_Pipeline.cs index 819f84656fd8..3e944b021bf1 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example02_Pipeline.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example02_Pipeline.cs @@ -5,28 +5,28 @@ using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Skills.Core; +using Microsoft.SemanticKernel.Plugins.Core; using RepoUtils; // ReSharper disable once InconsistentNaming public static class Example02_Pipeline { - private static readonly ILogger s_logger = ConsoleLogger.Logger; + private static readonly ILoggerFactory s_loggerFactory = ConsoleLogger.LoggerFactory; public static async Task RunAsync() { Console.WriteLine("======== Pipeline ========"); - IKernel kernel = new KernelBuilder().WithLogger(s_logger).Build(); + IKernel kernel = new KernelBuilder().WithLoggerFactory(s_loggerFactory).Build(); - // Load native skill - var text = kernel.ImportSkill(new TextSkill()); + // Load native plugin + var textFunctions = kernel.ImportFunctions(new TextPlugin()); - SKContext result = await kernel.RunAsync(" i n f i n i t e s p a c e ", - text["TrimStart"], - text["TrimEnd"], - text["Uppercase"]); + KernelResult result = await kernel.RunAsync(" i n f i n i t e s p a c e ", + textFunctions["TrimStart"], + textFunctions["TrimEnd"], + textFunctions["Uppercase"]); - Console.WriteLine(result); + Console.WriteLine(result.GetValue()); } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example03_Variables.cs b/dotnet/samples/KernelSyntaxExamples/Example03_Variables.cs index 5d312892f49c..f44366ae70ba 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example03_Variables.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example03_Variables.cs @@ -6,28 +6,28 @@ using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Orchestration; +using Plugins; using RepoUtils; -using Skills; // ReSharper disable once InconsistentNaming public static class Example03_Variables { - private static readonly ILogger s_logger = ConsoleLogger.Logger; + private static readonly ILoggerFactory s_loggerFactory = ConsoleLogger.LoggerFactory; public static async Task RunAsync() { Console.WriteLine("======== Variables ========"); - IKernel kernel = new KernelBuilder().WithLogger(s_logger).Build(); - var text = kernel.ImportSkill(new StaticTextSkill(), "text"); + IKernel kernel = new KernelBuilder().WithLoggerFactory(s_loggerFactory).Build(); + var textFunctions = kernel.ImportFunctions(new StaticTextPlugin(), "text"); var variables = new ContextVariables("Today is: "); variables.Set("day", DateTimeOffset.Now.ToString("dddd", CultureInfo.CurrentCulture)); - SKContext result = await kernel.RunAsync(variables, - text["AppendDay"], - text["Uppercase"]); + KernelResult result = await kernel.RunAsync(variables, + textFunctions["AppendDay"], + textFunctions["Uppercase"]); - Console.WriteLine(result); + Console.WriteLine(result.GetValue()); } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example04_CombineLLMPromptsAndNativeCode.cs b/dotnet/samples/KernelSyntaxExamples/Example04_CombineLLMPromptsAndNativeCode.cs index 2fe345dbc951..15acd518b4c5 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example04_CombineLLMPromptsAndNativeCode.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example04_CombineLLMPromptsAndNativeCode.cs @@ -3,8 +3,8 @@ using System; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Web; -using Microsoft.SemanticKernel.Skills.Web.Bing; +using Microsoft.SemanticKernel.Plugins.Web; +using Microsoft.SemanticKernel.Plugins.Web.Bing; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -23,14 +23,11 @@ public static async Task RunAsync() } IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService("text-davinci-002", openAIApiKey, serviceId: "text-davinci-002") - .WithOpenAITextCompletionService("text-davinci-003", openAIApiKey) + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, openAIApiKey) .Build(); - // Load native skill string bingApiKey = TestConfiguration.Bing.ApiKey; - if (bingApiKey == null) { Console.WriteLine("Bing credentials not found. Skipping example."); @@ -38,39 +35,37 @@ public static async Task RunAsync() } var bingConnector = new BingConnector(bingApiKey); - var bing = new WebSearchEngineSkill(bingConnector); - var search = kernel.ImportSkill(bing, "bing"); + var bing = new WebSearchEnginePlugin(bingConnector); + var searchFunctions = kernel.ImportFunctions(bing, "bing"); - // Load semantic skill defined with prompt templates - string folder = RepoFiles.SampleSkillsPath(); + // Load semantic plugins defined with prompt templates + string folder = RepoFiles.SamplePluginsPath(); - var sumSkill = kernel.ImportSemanticSkillFromDirectory( - folder, - "SummarizeSkill"); + var summarizeFunctions = kernel.ImportSemanticFunctionsFromDirectory(folder, "SummarizePlugin"); // Run var ask = "What's the tallest building in South America"; var result1 = await kernel.RunAsync( ask, - search["Search"] + searchFunctions["Search"] ); var result2 = await kernel.RunAsync( ask, - search["Search"], - sumSkill["Summarize"] + searchFunctions["Search"], + summarizeFunctions["Summarize"] ); var result3 = await kernel.RunAsync( ask, - search["Search"], - sumSkill["Notegen"] + searchFunctions["Search"], + summarizeFunctions["Notegen"] ); Console.WriteLine(ask + "\n"); - Console.WriteLine("Bing Answer: " + result1 + "\n"); - Console.WriteLine("Summary: " + result2 + "\n"); - Console.WriteLine("Notes: " + result3 + "\n"); + Console.WriteLine("Bing Answer: " + result1.GetValue() + "\n"); + Console.WriteLine("Summary: " + result2.GetValue() + "\n"); + Console.WriteLine("Notes: " + result3.GetValue() + "\n"); } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example05_InlineFunctionDefinition.cs b/dotnet/samples/KernelSyntaxExamples/Example05_InlineFunctionDefinition.cs index a34919933b00..0237389c01d2 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example05_InlineFunctionDefinition.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example05_InlineFunctionDefinition.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -12,7 +13,7 @@ public static async Task RunAsync() { Console.WriteLine("======== Inline Function Definition ========"); - string openAIModelId = TestConfiguration.OpenAI.ModelId; + string openAIModelId = TestConfiguration.OpenAI.ChatModelId; string openAIApiKey = TestConfiguration.OpenAI.ApiKey; if (openAIModelId == null || openAIApiKey == null) @@ -28,14 +29,14 @@ public static async Task RunAsync() */ IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService( + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( modelId: openAIModelId, apiKey: openAIApiKey) .Build(); // Function defined using few-shot design pattern - const string FunctionDefinition = @" + string promptTemplate = @" Generate a creative reason or excuse for the given event. Be creative and be funny. Let your imagination run wild. @@ -48,17 +49,17 @@ Be creative and be funny. Let your imagination run wild. Event: {{$input}} "; - var excuseFunction = kernel.CreateSemanticFunction(FunctionDefinition, maxTokens: 100, temperature: 0.4, topP: 1); + var excuseFunction = kernel.CreateSemanticFunction(promptTemplate, new OpenAIRequestSettings() { MaxTokens = 100, Temperature = 0.4, TopP = 1 }); - var result = await excuseFunction.InvokeAsync("I missed the F1 final race"); - Console.WriteLine(result); + var result = await kernel.RunAsync("I missed the F1 final race", excuseFunction); + Console.WriteLine(result.GetValue()); - result = await excuseFunction.InvokeAsync("sorry I forgot your birthday"); - Console.WriteLine(result); + result = await kernel.RunAsync("sorry I forgot your birthday", excuseFunction); + Console.WriteLine(result.GetValue()); - var fixedFunction = kernel.CreateSemanticFunction($"Translate this date {DateTimeOffset.Now:f} to French format", maxTokens: 100); + var fixedFunction = kernel.CreateSemanticFunction($"Translate this date {DateTimeOffset.Now:f} to French format", new OpenAIRequestSettings() { MaxTokens = 100 }); - result = await fixedFunction.InvokeAsync(); - Console.WriteLine(result); + result = await kernel.RunAsync(fixedFunction); + Console.WriteLine(result.GetValue()); } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example06_TemplateLanguage.cs b/dotnet/samples/KernelSyntaxExamples/Example06_TemplateLanguage.cs index c8cee60f9a46..cd1816e54b19 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example06_TemplateLanguage.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example06_TemplateLanguage.cs @@ -3,8 +3,9 @@ using System; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Core; -using Microsoft.SemanticKernel.TemplateEngine; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.TemplateEngine.Basic; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -18,7 +19,7 @@ public static async Task RunAsync() { Console.WriteLine("======== TemplateLanguage ========"); - string openAIModelId = TestConfiguration.OpenAI.ModelId; + string openAIModelId = TestConfiguration.OpenAI.ChatModelId; string openAIApiKey = TestConfiguration.OpenAI.ApiKey; if (openAIModelId == null || openAIApiKey == null) @@ -28,15 +29,15 @@ public static async Task RunAsync() } IKernel kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService( + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( modelId: openAIModelId, apiKey: openAIApiKey) .Build(); - // Load native skill into the kernel skill collection, sharing its functions with prompt templates + // Load native plugin into the kernel function collection, sharing its functions with prompt templates // Functions loaded here are available as "time.*" - kernel.ImportSkill(new TimeSkill(), "time"); + kernel.ImportFunctions(new TimePlugin(), "time"); // Semantic Function invoking time.Date and time.Time native functions const string FunctionDefinition = @" @@ -50,17 +51,17 @@ Is it weekend time (weekend/not weekend)? // This allows to see the prompt before it's sent to OpenAI Console.WriteLine("--- Rendered Prompt"); - var promptRenderer = new PromptTemplateEngine(); + var promptRenderer = new BasicPromptTemplateEngine(); var renderedPrompt = await promptRenderer.RenderAsync(FunctionDefinition, kernel.CreateNewContext()); Console.WriteLine(renderedPrompt); // Run the prompt / semantic function - var kindOfDay = kernel.CreateSemanticFunction(FunctionDefinition, maxTokens: 150); + var kindOfDay = kernel.CreateSemanticFunction(FunctionDefinition, new OpenAIRequestSettings() { MaxTokens = 100 }); // Show the result Console.WriteLine("--- Semantic Function result"); - var result = await kindOfDay.InvokeAsync(); - Console.WriteLine(result); + var result = await kernel.RunAsync(kindOfDay); + Console.WriteLine(result.GetValue()); /* OUTPUT: diff --git a/dotnet/samples/KernelSyntaxExamples/Example07_BingAndGooglePlugins.cs b/dotnet/samples/KernelSyntaxExamples/Example07_BingAndGooglePlugins.cs new file mode 100644 index 000000000000..84c180a204c2 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example07_BingAndGooglePlugins.cs @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Plugins.Web; +using Microsoft.SemanticKernel.Plugins.Web.Bing; +using Microsoft.SemanticKernel.Plugins.Web.Google; +using Microsoft.SemanticKernel.TemplateEngine.Basic; +using RepoUtils; + +/// +/// The example shows how to use Bing and Google to search for current data +/// you might want to import into your system, e.g. providing AI prompts with +/// recent information, or for AI to generate recent information to display to users. +/// +// ReSharper disable CommentTypo +// ReSharper disable once InconsistentNaming +public static class Example07_BingAndGooglePlugins +{ + public static async Task RunAsync() + { + string openAIModelId = TestConfiguration.OpenAI.ChatModelId; + string openAIApiKey = TestConfiguration.OpenAI.ApiKey; + + if (openAIModelId == null || openAIApiKey == null) + { + Console.WriteLine("OpenAI credentials not found. Skipping example."); + return; + } + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( + modelId: openAIModelId, + apiKey: openAIApiKey) + .Build(); + + // Load Bing plugin + string bingApiKey = TestConfiguration.Bing.ApiKey; + if (bingApiKey == null) + { + Console.WriteLine("Bing credentials not found. Skipping example."); + } + else + { + var bingConnector = new BingConnector(bingApiKey); + var bing = new WebSearchEnginePlugin(bingConnector); + kernel.ImportFunctions(bing, "bing"); + await Example1Async(kernel, "bing"); + await Example2Async(kernel); + } + + // Load Google plugin + string googleApiKey = TestConfiguration.Google.ApiKey; + string googleSearchEngineId = TestConfiguration.Google.SearchEngineId; + + if (googleApiKey == null || googleSearchEngineId == null) + { + Console.WriteLine("Google credentials not found. Skipping example."); + } + else + { + using var googleConnector = new GoogleConnector( + apiKey: googleApiKey, + searchEngineId: googleSearchEngineId); + var google = new WebSearchEnginePlugin(googleConnector); + kernel.ImportFunctions(new WebSearchEnginePlugin(googleConnector), "google"); + await Example1Async(kernel, "google"); + } + } + + private static async Task Example1Async(IKernel kernel, string searchPluginName) + { + Console.WriteLine("======== Bing and Google Search Plugins ========"); + + // Run + var question = "What's the largest building in the world?"; + var function = kernel.Functions.GetFunction(searchPluginName, "search"); + var result = await kernel.RunAsync(question, function); + + Console.WriteLine(question); + Console.WriteLine($"----{searchPluginName}----"); + Console.WriteLine(result.GetValue()); + + /* OUTPUT: + + What's the largest building in the world? + ---- + The Aerium near Berlin, Germany is the largest uninterrupted volume in the world, while Boeing's + factory in Everett, Washington, United States is the world's largest building by volume. The AvtoVAZ + main assembly building in Tolyatti, Russia is the largest building in area footprint. + ---- + The Aerium near Berlin, Germany is the largest uninterrupted volume in the world, while Boeing's + factory in Everett, Washington, United States is the world's ... + */ + } + + private static async Task Example2Async(IKernel kernel) + { + Console.WriteLine("======== Use Search Plugin to answer user questions ========"); + + const string SemanticFunction = @"Answer questions only when you know the facts or the information is provided. +When you don't have sufficient information you reply with a list of commands to find the information needed. +When answering multiple questions, use a bullet point list. +Note: make sure single and double quotes are escaped using a backslash char. + +[COMMANDS AVAILABLE] +- bing.search + +[INFORMATION PROVIDED] +{{ $externalInformation }} + +[EXAMPLE 1] +Question: what's the biggest lake in Italy? +Answer: Lake Garda, also known as Lago di Garda. + +[EXAMPLE 2] +Question: what's the biggest lake in Italy? What's the smallest positive number? +Answer: +* Lake Garda, also known as Lago di Garda. +* The smallest positive number is 1. + +[EXAMPLE 3] +Question: what's Ferrari stock price? Who is the current number one female tennis player in the world? +Answer: +{{ '{{' }} bing.search ""what\\'s Ferrari stock price?"" {{ '}}' }}. +{{ '{{' }} bing.search ""Who is the current number one female tennis player in the world?"" {{ '}}' }}. + +[END OF EXAMPLES] + +[TASK] +Question: {{ $input }}. +Answer: "; + + var questions = "Who is the most followed person on TikTok right now? What's the exchange rate EUR:USD?"; + Console.WriteLine(questions); + + var oracle = kernel.CreateSemanticFunction(SemanticFunction, new OpenAIRequestSettings() { MaxTokens = 150, Temperature = 0, TopP = 1 }); + + var answer = await kernel.RunAsync(oracle, new(questions) + { + ["externalInformation"] = string.Empty + }); + + var result = answer.GetValue()!; + + // If the answer contains commands, execute them using the prompt renderer. + if (result.Contains("bing.search", StringComparison.OrdinalIgnoreCase)) + { + var promptRenderer = new BasicPromptTemplateEngine(); + + Console.WriteLine("---- Fetching information from Bing..."); + var information = await promptRenderer.RenderAsync(result, kernel.CreateNewContext()); + + Console.WriteLine("Information found:"); + Console.WriteLine(information); + + // Run the semantic function again, now including information from Bing + answer = await kernel.RunAsync(oracle, new(questions) + { + // The rendered prompt contains the information retrieved from search engines + ["externalInformation"] = information + }); + } + else + { + Console.WriteLine("AI had all the information, no need to query Bing."); + } + + Console.WriteLine("---- ANSWER:"); + Console.WriteLine(answer.GetValue()); + + /* OUTPUT: + + Who is the most followed person on TikTok right now? What's the exchange rate EUR:USD? + ---- Fetching information from Bing... + Information found: + + Khaby Lame is the most-followed user on TikTok. This list contains the top 50 accounts by number + of followers on the Chinese social media platform TikTok, which was merged with musical.ly in 2018. + [1] The most-followed individual on the platform is Khaby Lame, with over 153 million followers.. + EUR – Euro To USD – US Dollar 1.00 Euro = 1.10 37097 US Dollars 1 USD = 0.906035 EUR We use the + mid-market rate for our Converter. This is for informational purposes only. You won’t receive this + rate when sending money. Check send rates Convert Euro to US Dollar Convert US Dollar to Euro.. + ---- ANSWER: + + * The most followed person on TikTok right now is Khaby Lame, with over 153 million followers. + * The exchange rate for EUR to USD is 1.1037097 US Dollars for 1 Euro. + */ + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example07_BingAndGoogleSkills.cs b/dotnet/samples/KernelSyntaxExamples/Example07_BingAndGoogleSkills.cs deleted file mode 100644 index dd0222edce4f..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example07_BingAndGoogleSkills.cs +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.Web; -using Microsoft.SemanticKernel.Skills.Web.Bing; -using Microsoft.SemanticKernel.Skills.Web.Google; -using Microsoft.SemanticKernel.TemplateEngine; -using RepoUtils; - -/// -/// The example shows how to use Bing and Google to search for current data -/// you might want to import into your system, e.g. providing AI prompts with -/// recent information, or for AI to generate recent information to display to users. -/// -// ReSharper disable CommentTypo -// ReSharper disable once InconsistentNaming -public static class Example07_BingAndGoogleSkills -{ - public static async Task RunAsync() - { - string openAIModelId = TestConfiguration.OpenAI.ModelId; - string openAIApiKey = TestConfiguration.OpenAI.ApiKey; - - if (openAIModelId == null || openAIApiKey == null) - { - Console.WriteLine("OpenAI credentials not found. Skipping example."); - return; - } - - IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService( - modelId: openAIModelId, - apiKey: openAIApiKey) - .Build(); - - // Load Bing skill - string bingApiKey = TestConfiguration.Bing.ApiKey; - - if (bingApiKey == null) - { - Console.WriteLine("Bing credentials not found. Skipping example."); - } - else - { - var bingConnector = new BingConnector(bingApiKey); - var bing = new WebSearchEngineSkill(bingConnector); - var search = kernel.ImportSkill(bing, "bing"); - await Example1Async(kernel, "bing"); - await Example2Async(kernel); - } - - // Load Google skill - string googleApiKey = TestConfiguration.Google.ApiKey; - string googleSearchEngineId = TestConfiguration.Google.SearchEngineId; - - if (googleApiKey == null || googleSearchEngineId == null) - { - Console.WriteLine("Google credentials not found. Skipping example."); - } - else - { - using var googleConnector = new GoogleConnector( - apiKey: googleApiKey, - searchEngineId: googleSearchEngineId); - var google = new WebSearchEngineSkill(googleConnector); - var search = kernel.ImportSkill(new WebSearchEngineSkill(googleConnector), "google"); - await Example1Async(kernel, "google"); - } - } - - private static async Task Example1Async(IKernel kernel, string searchSkillId) - { - Console.WriteLine("======== Bing and Google Search Skill ========"); - - // Run - var question = "What's the largest building in the world?"; - var result = await kernel.Func(searchSkillId, "search").InvokeAsync(question); - - Console.WriteLine(question); - Console.WriteLine($"----{searchSkillId}----"); - Console.WriteLine(result); - - /* OUTPUT: - - What's the largest building in the world? - ---- - The Aerium near Berlin, Germany is the largest uninterrupted volume in the world, while Boeing's - factory in Everett, Washington, United States is the world's largest building by volume. The AvtoVAZ - main assembly building in Tolyatti, Russia is the largest building in area footprint. - ---- - The Aerium near Berlin, Germany is the largest uninterrupted volume in the world, while Boeing's - factory in Everett, Washington, United States is the world's ... - */ - } - - private static async Task Example2Async(IKernel kernel) - { - Console.WriteLine("======== Use Search Skill to answer user questions ========"); - - const string SemanticFunction = @"Answer questions only when you know the facts or the information is provided. -When you don't have sufficient information you reply with a list of commands to find the information needed. -When answering multiple questions, use a bullet point list. -Note: make sure single and double quotes are escaped using a backslash char. - -[COMMANDS AVAILABLE] -- bing.search - -[INFORMATION PROVIDED] -{{ $externalInformation }} - -[EXAMPLE 1] -Question: what's the biggest lake in Italy? -Answer: Lake Garda, also known as Lago di Garda. - -[EXAMPLE 2] -Question: what's the biggest lake in Italy? What's the smallest positive number? -Answer: -* Lake Garda, also known as Lago di Garda. -* The smallest positive number is 1. - -[EXAMPLE 3] -Question: what's Ferrari stock price? Who is the current number one female tennis player in the world? -Answer: -{{ '{{' }} bing.search ""what\\'s Ferrari stock price?"" {{ '}}' }}. -{{ '{{' }} bing.search ""Who is the current number one female tennis player in the world?"" {{ '}}' }}. - -[END OF EXAMPLES] - -[TASK] -Question: {{ $input }}. -Answer: "; - - var questions = "Who is the most followed person on TikTok right now? What's the exchange rate EUR:USD?"; - Console.WriteLine(questions); - - var oracle = kernel.CreateSemanticFunction(SemanticFunction, maxTokens: 200, temperature: 0, topP: 1); - - var context = kernel.CreateNewContext(); - context.Variables["externalInformation"] = ""; - var answer = await oracle.InvokeAsync(questions, context); - - // If the answer contains commands, execute them using the prompt renderer. - if (answer.Result.Contains("bing.search", StringComparison.OrdinalIgnoreCase)) - { - var promptRenderer = new PromptTemplateEngine(); - - Console.WriteLine("---- Fetching information from Bing..."); - var information = await promptRenderer.RenderAsync(answer.Result, context); - - Console.WriteLine("Information found:"); - Console.WriteLine(information); - - // The rendered prompt contains the information retrieved from search engines - context.Variables["externalInformation"] = information; - - // Run the semantic function again, now including information from Bing - answer = await oracle.InvokeAsync(questions, context); - } - else - { - Console.WriteLine("AI had all the information, no need to query Bing."); - } - - Console.WriteLine("---- ANSWER:"); - Console.WriteLine(answer); - - /* OUTPUT: - - Who is the most followed person on TikTok right now? What's the exchange rate EUR:USD? - ---- Fetching information from Bing... - Information found: - - Khaby Lame is the most-followed user on TikTok. This list contains the top 50 accounts by number - of followers on the Chinese social media platform TikTok, which was merged with musical.ly in 2018. - [1] The most-followed individual on the platform is Khaby Lame, with over 153 million followers.. - EUR – Euro To USD – US Dollar 1.00 Euro = 1.10 37097 US Dollars 1 USD = 0.906035 EUR We use the - mid-market rate for our Converter. This is for informational purposes only. You won’t receive this - rate when sending money. Check send rates Convert Euro to US Dollar Convert US Dollar to Euro.. - ---- ANSWER: - - * The most followed person on TikTok right now is Khaby Lame, with over 153 million followers. - * The exchange rate for EUR to USD is 1.1037097 US Dollars for 1 Euro. - */ - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example08_RetryHandler.cs b/dotnet/samples/KernelSyntaxExamples/Example08_RetryHandler.cs index d354e3cbd175..0adeec6e0967 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example08_RetryHandler.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example08_RetryHandler.cs @@ -2,12 +2,15 @@ using System; using System.Net; +using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Reliability; -using Microsoft.SemanticKernel.Skills.Core; -using Reliability; +using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.Reliability.Basic; +using Polly; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -15,104 +18,157 @@ public static class Example08_RetryHandler { public static async Task RunAsync() { - var kernel = InitializeKernel(); - var retryHandlerFactory = new RetryThreeTimesWithBackoffFactory(); - InfoLogger.Logger.LogInformation("============================== RetryThreeTimesWithBackoff =============================="); - await RunRetryPolicyAsync(kernel, retryHandlerFactory); + await DefaultNoRetryAsync(); - InfoLogger.Logger.LogInformation("========================= RetryThreeTimesWithRetryAfterBackoff ========================="); - await RunRetryPolicyBuilderAsync(typeof(RetryThreeTimesWithRetryAfterBackoffFactory)); + await ReliabilityBasicExtensionAsync(); - InfoLogger.Logger.LogInformation("==================================== NoRetryPolicy ====================================="); - await RunRetryPolicyBuilderAsync(typeof(NullHttpRetryHandlerFactory)); + await ReliabilityPollyExtensionAsync(); - InfoLogger.Logger.LogInformation("=============================== DefaultHttpRetryHandler ================================"); - await RunRetryHandlerConfigAsync(new HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); - - InfoLogger.Logger.LogInformation("======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ======="); - await RunRetryHandlerConfigAsync(new HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); + await CustomHandlerAsync(); } - private static async Task RunRetryHandlerConfigAsync(HttpRetryConfig? httpConfig = null) + private static async Task DefaultNoRetryAsync() { - var kernelBuilder = Kernel.Builder.WithLogger(InfoLogger.Logger); - if (httpConfig != null) - { - kernelBuilder = kernelBuilder.Configure(c => c.SetDefaultHttpRetryConfig(httpConfig)); - } + InfoLogger.Logger.LogInformation("============================== Kernel default behavior: No Retry =============================="); + var kernel = InitializeKernelBuilder() + .Build(); - // Add 401 to the list of retryable status codes - // Typically 401 would not be something we retry but for demonstration - // purposes we are doing so as it's easy to trigger when using an invalid key. - kernelBuilder = kernelBuilder.Configure(c => c.DefaultHttpRetryConfig.RetryableStatusCodes.Add(HttpStatusCode.Unauthorized)); + await ImportAndExecutePluginAsync(kernel); + } - // OpenAI settings - you can set the OpenAI.ApiKey to an invalid value to see the retry policy in play - kernelBuilder = kernelBuilder.WithOpenAITextCompletionService("text-davinci-003", "BAD_KEY"); + private static async Task ReliabilityBasicExtensionAsync() + { + InfoLogger.Logger.LogInformation("============================== Using Reliability.Basic extension =============================="); + var retryConfig = new BasicRetryConfig + { + MaxRetryCount = 3, + UseExponentialBackoff = true, + }; + retryConfig.RetryableStatusCodes.Add(HttpStatusCode.Unauthorized); - var kernel = kernelBuilder.Build(); + var kernel = InitializeKernelBuilder() + .WithRetryBasic(retryConfig) + .Build(); - await ImportAndExecuteSkillAsync(kernel); + await ImportAndExecutePluginAsync(kernel); } - private static IKernel InitializeKernel() + private static async Task ReliabilityPollyExtensionAsync() { - var kernel = Kernel.Builder - .WithLogger(InfoLogger.Logger) - // OpenAI settings - you can set the OpenAI.ApiKey to an invalid value to see the retry policy in play - .WithOpenAITextCompletionService("text-davinci-003", "BAD_KEY") + InfoLogger.Logger.LogInformation("============================== Using Reliability.Polly extension =============================="); + var kernel = InitializeKernelBuilder() + .WithRetryPolly(GetPollyPolicy(InfoLogger.LoggerFactory)) .Build(); - return kernel; + await ImportAndExecutePluginAsync(kernel); } - private static async Task RunRetryPolicyAsync(IKernel kernel, IDelegatingHandlerFactory retryHandlerFactory) + private static async Task CustomHandlerAsync() { - kernel.Config.SetHttpRetryHandlerFactory(retryHandlerFactory); - await ImportAndExecuteSkillAsync(kernel); + InfoLogger.Logger.LogInformation("============================== Using a Custom Http Handler =============================="); + var kernel = InitializeKernelBuilder() + .WithHttpHandlerFactory(new MyCustomHandlerFactory()) + .Build(); + + await ImportAndExecutePluginAsync(kernel); } - private static async Task RunRetryPolicyBuilderAsync(Type retryHandlerFactoryType) + private static KernelBuilder InitializeKernelBuilder() { - var kernel = Kernel.Builder.WithLogger(InfoLogger.Logger) - .WithRetryHandlerFactory((Activator.CreateInstance(retryHandlerFactoryType) as IDelegatingHandlerFactory)!) - // OpenAI settings - you can set the OpenAI.ApiKey to an invalid value to see the retry policy in play - .WithOpenAITextCompletionService("text-davinci-003", "BAD_KEY") - .Build(); + return Kernel.Builder + .WithLoggerFactory(InfoLogger.LoggerFactory) + // OpenAI settings - you can set the OpenAI.ApiKey to an invalid value to see the retry policy in play + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, "BAD_KEY"); + } - await ImportAndExecuteSkillAsync(kernel); + private static AsyncPolicy GetPollyPolicy(ILoggerFactory? logger) + { + // Handle 429 and 401 errors + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. + const int TooManyRequests = 429; + const int Unauthorized = 401; + + return Policy + .HandleResult(response => + (int)response.StatusCode is TooManyRequests or Unauthorized) + .WaitAndRetryAsync(new[] + { + TimeSpan.FromSeconds(2), + TimeSpan.FromSeconds(4), + TimeSpan.FromSeconds(8) + }, + (outcome, timespan, retryCount, _) + => InfoLogger.Logger.LogWarning("Error executing action [attempt {RetryCount} of 3], pausing {PausingMilliseconds}ms. Outcome: {StatusCode}", + retryCount, + timespan.TotalMilliseconds, + outcome.Result.StatusCode)); } - private static async Task ImportAndExecuteSkillAsync(IKernel kernel) + private static async Task ImportAndExecutePluginAsync(IKernel kernel) { - // Load semantic skill defined with prompt templates - string folder = RepoFiles.SampleSkillsPath(); + // Load semantic plugin defined with prompt templates + string folder = RepoFiles.SamplePluginsPath(); - kernel.ImportSkill(new TimeSkill(), "time"); + kernel.ImportFunctions(new TimePlugin(), "time"); - var qaSkill = kernel.ImportSemanticSkillFromDirectory( + var qaPlugin = kernel.ImportSemanticFunctionsFromDirectory( folder, - "QASkill"); + "QAPlugin"); var question = "How popular is Polly library?"; InfoLogger.Logger.LogInformation("Question: {0}", question); // To see the retry policy in play, you can set the OpenAI.ApiKey to an invalid value - var answer = await kernel.RunAsync(question, qaSkill["Question"]); - InfoLogger.Logger.LogInformation("Answer: {0}", answer); +#pragma warning disable CA1031 // Do not catch general exception types + try + { + var answer = await kernel.RunAsync(question, qaPlugin["Question"]); + InfoLogger.Logger.LogInformation("Answer: {0}", answer.GetValue()); + } + catch (Exception ex) + { + InfoLogger.Logger.LogInformation("Error: {0}", ex.Message); + } +#pragma warning restore CA1031 // Do not catch general exception types + } + + // Basic custom retry handler factory + public sealed class MyCustomHandlerFactory : HttpHandlerFactory + { + } + + // Basic custom empty retry handler + public sealed class MyCustomHandler : DelegatingHandler + { + public MyCustomHandler(ILoggerFactory loggerFactory) + { + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + // Your custom http handling implementation + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent("My custom bad request override") + }); + } } private static class InfoLogger { - internal static ILogger Logger => LogFactory.CreateLogger(); - private static ILoggerFactory LogFactory => s_loggerFactory.Value; + internal static ILogger Logger => LoggerFactory.CreateLogger("Example08_RetryHandler"); + internal static ILoggerFactory LoggerFactory => s_loggerFactory.Value; private static readonly Lazy s_loggerFactory = new(LogBuilder); private static ILoggerFactory LogBuilder() { - return LoggerFactory.Create(builder => + return Microsoft.Extensions.Logging.LoggerFactory.Create(builder => { builder.SetMinimumLevel(LogLevel.Information); builder.AddFilter("Microsoft", LogLevel.Information); + builder.AddFilter("Microsoft.SemanticKernel", LogLevel.Critical); + builder.AddFilter("Microsoft.SemanticKernel.Reliability", LogLevel.Information); builder.AddFilter("System", LogLevel.Information); builder.AddConsole(); @@ -120,74 +176,3 @@ private static ILoggerFactory LogBuilder() } } } - -/* Output: -info: object[0] - ============================== RetryThreeTimesWithBackoff ============================== -info: object[0] - Question: How popular is Polly library? -warn: object[0] - Error executing action [attempt 1 of 3], pausing 2000ms. Outcome: Unauthorized -warn: object[0] - Error executing action [attempt 2 of 3], pausing 4000ms. Outcome: Unauthorized -warn: object[0] - Error executing action [attempt 3 of 3], pausing 8000ms. Outcome: Unauthorized -fail: object[0] - Function call fail during pipeline step 0: QASkill.Question -info: object[0] - Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized -info: object[0] - ========================= RetryThreeTimesWithRetryAfterBackoff ========================= -info: object[0] - Question: How popular is Polly library? -warn: object[0] - Error executing action [attempt 1 of 3], pausing 2000ms. Outcome: Unauthorized -warn: object[0] - Error executing action [attempt 2 of 3], pausing 2000ms. Outcome: Unauthorized -warn: object[0] - Error executing action [attempt 3 of 3], pausing 2000ms. Outcome: Unauthorized -fail: object[0] - Function call fail during pipeline step 0: QASkill.Question -info: object[0] - Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized -info: object[0] - ==================================== NoRetryPolicy ===================================== -info: object[0] - Question: How popular is Polly library? -fail: object[0] - Function call fail during pipeline step 0: QASkill.Question -info: object[0] - Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized -info: object[0] - =============================== DefaultHttpRetryHandler ================================ -info: object[0] - Question: How popular is Polly library? -warn: object[0] - Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms -warn: object[0] - Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms -warn: object[0] - Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms -fail: object[0] - Error executing request, max retry count reached. Reason: Unauthorized -fail: object[0] - Function call fail during pipeline step 0: QASkill.Question -info: object[0] - Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized -info: object[0] - ======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ======= -info: object[0] - Question: How popular is Polly library? -warn: object[0] - Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms -warn: object[0] - Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms -warn: object[0] - Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms -fail: object[0] - Error executing request, max retry count reached. Reason: Unauthorized -fail: object[0] - Function call fail during pipeline step 0: QASkill.Question -info: object[0] - Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized -*/ diff --git a/dotnet/samples/KernelSyntaxExamples/Example09_FunctionTypes.cs b/dotnet/samples/KernelSyntaxExamples/Example09_FunctionTypes.cs index aab6405583ab..73905cac7798 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example09_FunctionTypes.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example09_FunctionTypes.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -16,84 +15,84 @@ public static async Task RunAsync() { Console.WriteLine("======== Native function types ========"); - var fakeContext = new SKContext(logger: ConsoleLogger.Logger); - var kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService(TestConfiguration.OpenAI.ModelId, TestConfiguration.OpenAI.ApiKey) + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey) .Build(); - // Load native skill into the kernel skill collection, sharing its functions with prompt templates - var test = kernel.ImportSkill(new LocalExampleSkill(), "test"); + var variables = new ContextVariables(); + + // Load native plugin into the kernel function collection, sharing its functions with prompt templates + var testFunctions = kernel.ImportFunctions(new LocalExamplePlugin(), "test"); - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); + string folder = RepoFiles.SamplePluginsPath(); + kernel.ImportSemanticFunctionsFromDirectory(folder, "SummarizePlugin"); // The kernel takes care of wiring the input appropriately await kernel.RunAsync( - "", - test["type01"], - test["type02"], - test["type03"], - test["type04"], - test["type05"], - test["type06"], - test["type07"], - test["type08"], - test["type09"], - test["type10"], - test["type11"], - test["type12"], - test["type13"], - test["type14"], - test["type15"], - test["type16"], - test["type17"], - test["type18"] + testFunctions["type01"], + testFunctions["type02"], + testFunctions["type03"], + testFunctions["type04"], + testFunctions["type05"], + testFunctions["type06"], + testFunctions["type07"], + testFunctions["type08"], + testFunctions["type09"], + testFunctions["type10"], + testFunctions["type11"], + testFunctions["type12"], + testFunctions["type13"], + testFunctions["type14"], + testFunctions["type15"], + testFunctions["type16"], + testFunctions["type17"], + testFunctions["type18"] ); - await kernel.Func("test", "type01").InvokeAsync(); - await test["type01"].InvokeAsync(); + // Using Kernel.RunAsync + await kernel.RunAsync(testFunctions["type01"]); + await kernel.RunAsync(kernel.Functions.GetFunction("test", "type01")); - await kernel.Func("test", "type02").InvokeAsync(); - await test["type02"].InvokeAsync(); + await kernel.RunAsync(testFunctions["type02"]); + await kernel.RunAsync(kernel.Functions.GetFunction("test", "type02")); - await kernel.Func("test", "type03").InvokeAsync(); - await test["type03"].InvokeAsync(); + await kernel.RunAsync(testFunctions["type03"]); + await kernel.RunAsync(kernel.Functions.GetFunction("test", "type03")); - await kernel.Func("test", "type04").InvokeAsync(fakeContext); - await test["type04"].InvokeAsync(fakeContext); + await kernel.RunAsync(testFunctions["type04"], variables); + await kernel.RunAsync(variables, kernel.Functions.GetFunction("test", "type04")); - await kernel.Func("test", "type05").InvokeAsync(fakeContext); - await test["type05"].InvokeAsync(fakeContext); + await kernel.RunAsync(testFunctions["type05"], variables); + await kernel.RunAsync(variables, kernel.Functions.GetFunction("test", "type05")); - await kernel.Func("test", "type06").InvokeAsync(fakeContext); - await test["type06"].InvokeAsync(fakeContext); + await kernel.RunAsync(testFunctions["type06"], variables); + await kernel.RunAsync(variables, kernel.Functions.GetFunction("test", "type06")); - await kernel.Func("test", "type07").InvokeAsync(fakeContext); - await test["type07"].InvokeAsync(fakeContext); + await kernel.RunAsync(testFunctions["type07"], variables); + await kernel.RunAsync(variables, kernel.Functions.GetFunction("test", "type07")); - await kernel.Func("test", "type08").InvokeAsync(""); - await test["type08"].InvokeAsync(""); + await kernel.RunAsync("", testFunctions["type08"]); + await kernel.RunAsync("", kernel.Functions.GetFunction("test", "type08")); - await kernel.Func("test", "type09").InvokeAsync(""); - await test["type09"].InvokeAsync(""); + await kernel.RunAsync("", testFunctions["type09"]); + await kernel.RunAsync("", kernel.Functions.GetFunction("test", "type09")); - await kernel.Func("test", "type10").InvokeAsync(""); - await test["type10"].InvokeAsync(""); + await kernel.RunAsync("", testFunctions["type10"]); + await kernel.RunAsync("", kernel.Functions.GetFunction("test", "type10")); - await kernel.Func("test", "type11").InvokeAsync(""); - await test["type11"].InvokeAsync(""); + await kernel.RunAsync("", testFunctions["type11"]); + await kernel.RunAsync("", kernel.Functions.GetFunction("test", "type11")); - await kernel.Func("test", "type12").InvokeAsync(fakeContext); - await test["type12"].InvokeAsync(fakeContext); + await kernel.RunAsync(variables, testFunctions["type12"]); + await kernel.RunAsync(variables, kernel.Functions.GetFunction("test", "type12")); - await kernel.Func("test", "type18").InvokeAsync(); - await test["type18"].InvokeAsync(); + await kernel.RunAsync(testFunctions["type18"]); + await kernel.RunAsync(kernel.Functions.GetFunction("test", "type18")); } } -public class LocalExampleSkill +public class LocalExamplePlugin { [SKFunction] public void Type01() @@ -132,11 +131,10 @@ public string Type05(SKContext context) [SKFunction] public async Task Type06Async(SKContext context) { - var summarizer = context.Func("SummarizeSkill", "Summarize"); - - var summary = await summarizer.InvokeAsync("blah blah blah"); + var summarizer = context.Functions.GetFunction("SummarizePlugin", "Summarize"); + var summary = await context.Runner.RunAsync(summarizer, new ContextVariables("blah blah blah")); - Console.WriteLine($"Running function type 6 [{summary}]"); + Console.WriteLine($"Running function type 6 [{summary.GetValue()}]"); return ""; } diff --git a/dotnet/samples/KernelSyntaxExamples/Example10_DescribeAllPluginsAndFunctions.cs b/dotnet/samples/KernelSyntaxExamples/Example10_DescribeAllPluginsAndFunctions.cs new file mode 100644 index 000000000000..d49bda2e302c --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example10_DescribeAllPluginsAndFunctions.cs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Plugins.Core; +using Plugins; +using RepoUtils; + +// ReSharper disable once InconsistentNaming +public static class Example10_DescribeAllPluginsAndFunctions +{ + /// + /// Print a list of all the functions imported into the kernel, including function descriptions, + /// list of parameters, parameters descriptions, etc. + /// See the end of the file for a sample of what the output looks like. + /// + public static Task RunAsync() + { + Console.WriteLine("======== Describe all plugins and functions ========"); + + var kernel = Kernel.Builder + .WithOpenAIChatCompletionService( + modelId: TestConfiguration.OpenAI.ChatModelId, + apiKey: TestConfiguration.OpenAI.ApiKey) + .Build(); + + // Import a native plugin + var staticText = new StaticTextPlugin(); + kernel.ImportFunctions(staticText, "StaticTextPlugin"); + + // Import another native plugin + var text = new TextPlugin(); + kernel.ImportFunctions(text, "AnotherTextPlugin"); + + // Import a semantic plugin + string folder = RepoFiles.SamplePluginsPath(); + kernel.ImportSemanticFunctionsFromDirectory(folder, "SummarizePlugin"); + + // Define a semantic function inline, without naming + var sFun1 = kernel.CreateSemanticFunction("tell a joke about {{$input}}", new OpenAIRequestSettings() { MaxTokens = 150 }); + + // Define a semantic function inline, with plugin name + var sFun2 = kernel.CreateSemanticFunction( + "write a novel about {{$input}} in {{$language}} language", + new OpenAIRequestSettings() { MaxTokens = 150 }, + pluginName: "Writing", + functionName: "Novel", + description: "Write a bedtime story"); + + var functions = kernel.Functions.GetFunctionViews(); + + Console.WriteLine("*****************************************"); + Console.WriteLine("****** Registered plugins and functions ******"); + Console.WriteLine("*****************************************"); + Console.WriteLine(); + + foreach (FunctionView func in functions) + { + PrintFunction(func); + } + + return Task.CompletedTask; + } + + private static void PrintFunction(FunctionView func) + { + Console.WriteLine($" {func.Name}: {func.Description}"); + + if (func.Parameters.Count > 0) + { + Console.WriteLine(" Params:"); + foreach (var p in func.Parameters) + { + Console.WriteLine($" - {p.Name}: {p.Description}"); + Console.WriteLine($" default: '{p.DefaultValue}'"); + } + } + + Console.WriteLine(); + } +} + +#pragma warning disable CS1587 // XML comment is not placed on a valid language element +/** Sample output: + +***************************************** +****** Native plugins and functions ****** +***************************************** + +Plugin: StaticTextPlugin + Uppercase: Change all string chars to uppercase + Params: + - input: Text to uppercase + default: '' + + AppendDay: Append the day variable + Params: + - input: Text to append to + default: '' + - day: Value of the day to append + default: '' + +Plugin: TextPlugin + Uppercase: Convert a string to uppercase. + Params: + - input: Text to uppercase + default: '' + + Trim: Trim whitespace from the start and end of a string. + Params: + - input: Text to edit + default: '' + + TrimStart: Trim whitespace from the start of a string. + Params: + - input: Text to edit + default: '' + + TrimEnd: Trim whitespace from the end of a string. + Params: + - input: Text to edit + default: '' + + Lowercase: Convert a string to lowercase. + Params: + - input: Text to lowercase + default: '' + +***************************************** +***** Semantic plugins and functions ***** +***************************************** + +Plugin: _GLOBAL_FUNCTIONS_ + funcce97d27e3d0b4897acf6122e41430695: Generic function, unknown purpose + Params: + - input: + default: '' + +Plugin: Writing + Novel: Write a bedtime story + Params: + - input: + default: '' + - language: + default: '' + +Plugin: SummarizePlugin + Topics: Analyze given text or document and extract key topics worth remembering + Params: + - input: + default: '' + + Summarize: Summarize given text or any text document + Params: + - input: Text to summarize + default: '' + + MakeAbstractReadable: Given a scientific white paper abstract, rewrite it to make it more readable + Params: + - input: + default: '' + + TopicsMore: Generate list of topics for long length content + Params: + - input: Block of text to analyze + default: '' + - previousResults: List of topics found from previous blocks of text + default: '' + + Notegen: Automatically generate compact notes for any text or text document. + Params: + - input: + default: '' + + ActionItems: unknown function + + SummarizeMore: Summarize given text or any text document + Params: + - input: Block of text to analyze + default: '' + - previousResults: Overview generated from previous blocks of text + default: '' + - conversationType: Text type, e.g. chat, email thread, document + default: '' + +*/ +#pragma warning restore CS1587 // XML comment is not placed on a valid language element diff --git a/dotnet/samples/KernelSyntaxExamples/Example10_DescribeAllSkillsAndFunctions.cs b/dotnet/samples/KernelSyntaxExamples/Example10_DescribeAllSkillsAndFunctions.cs deleted file mode 100644 index 4e185cf547ce..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example10_DescribeAllSkillsAndFunctions.cs +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.Core; -using RepoUtils; -using Skills; - -// ReSharper disable once InconsistentNaming -public static class Example10_DescribeAllSkillsAndFunctions -{ - /// - /// Print a list of all the functions imported into the kernel, including function descriptions, - /// list of parameters, parameters descriptions, etc. - /// See the end of the file for a sample of what the output looks like. - /// - public static Task RunAsync() - { - Console.WriteLine("======== Describe all skills and functions ========"); - - var kernel = Kernel.Builder - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .Build(); - - // Import a native skill - var skill1 = new StaticTextSkill(); - kernel.ImportSkill(skill1, "StaticTextskill"); - - // Import another native skill - var skill2 = new TextSkill(); - kernel.ImportSkill(skill2, "AnotherTextskill"); - - // Import a semantic skill - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); - - // Define a semantic function inline, without naming - var sFun1 = kernel.CreateSemanticFunction("tell a joke about {{$input}}", maxTokens: 150); - - // Define a semantic function inline, with skill name - var sFun2 = kernel.CreateSemanticFunction( - "write a novel about {{$input}} in {{$language}} language", - skillName: "Writing", - functionName: "Novel", - description: "Write a bedtime story", - maxTokens: 150); - - FunctionsView functions = kernel.Skills.GetFunctionsView(); - ConcurrentDictionary> nativeFunctions = functions.NativeFunctions; - ConcurrentDictionary> semanticFunctions = functions.SemanticFunctions; - - Console.WriteLine("*****************************************"); - Console.WriteLine("****** Native skills and functions ******"); - Console.WriteLine("*****************************************"); - Console.WriteLine(); - - foreach (KeyValuePair> skill in nativeFunctions) - { - Console.WriteLine("Skill: " + skill.Key); - foreach (FunctionView func in skill.Value) { PrintFunction(func); } - } - - Console.WriteLine("*****************************************"); - Console.WriteLine("***** Semantic skills and functions *****"); - Console.WriteLine("*****************************************"); - Console.WriteLine(); - - foreach (KeyValuePair> skill in semanticFunctions) - { - Console.WriteLine("Skill: " + skill.Key); - foreach (FunctionView func in skill.Value) { PrintFunction(func); } - } - - return Task.CompletedTask; - } - - private static void PrintFunction(FunctionView func) - { - Console.WriteLine($" {func.Name}: {func.Description}"); - - if (func.Parameters.Count > 0) - { - Console.WriteLine(" Params:"); - foreach (var p in func.Parameters) - { - Console.WriteLine($" - {p.Name}: {p.Description}"); - Console.WriteLine($" default: '{p.DefaultValue}'"); - } - } - - Console.WriteLine(); - } -} - -#pragma warning disable CS1587 // XML comment is not placed on a valid language element -/** Sample output: - -***************************************** -****** Native skills and functions ****** -***************************************** - -Skill: StaticTextskill - Uppercase: Change all string chars to uppercase - Params: - - input: Text to uppercase - default: '' - - AppendDay: Append the day variable - Params: - - input: Text to append to - default: '' - - day: Value of the day to append - default: '' - -Skill: Textskill - Uppercase: Convert a string to uppercase. - Params: - - input: Text to uppercase - default: '' - - Trim: Trim whitespace from the start and end of a string. - Params: - - input: Text to edit - default: '' - - TrimStart: Trim whitespace from the start of a string. - Params: - - input: Text to edit - default: '' - - TrimEnd: Trim whitespace from the end of a string. - Params: - - input: Text to edit - default: '' - - Lowercase: Convert a string to lowercase. - Params: - - input: Text to lowercase - default: '' - -***************************************** -***** Semantic skills and functions ***** -***************************************** - -Skill: _GLOBAL_FUNCTIONS_ - funcce97d27e3d0b4897acf6122e41430695: Generic function, unknown purpose - Params: - - input: - default: '' - -Skill: Writing - Novel: Write a bedtime story - Params: - - input: - default: '' - - language: - default: '' - -Skill: SummarizeSkill - Topics: Analyze given text or document and extract key topics worth remembering - Params: - - input: - default: '' - - Summarize: Summarize given text or any text document - Params: - - input: Text to summarize - default: '' - - MakeAbstractReadable: Given a scientific white paper abstract, rewrite it to make it more readable - Params: - - input: - default: '' - - TopicsMore: Generate list of topics for long length content - Params: - - input: Block of text to analyze - default: '' - - previousResults: List of topics found from previous blocks of text - default: '' - - Notegen: Automatically generate compact notes for any text or text document. - Params: - - input: - default: '' - - ActionItems: unknown function - - SummarizeMore: Summarize given text or any text document - Params: - - input: Block of text to analyze - default: '' - - previousResults: Overview generated from previous blocks of text - default: '' - - conversationType: Text type, e.g. chat, email thread, document - default: '' - -*/ -#pragma warning restore CS1587 // XML comment is not placed on a valid language element diff --git a/dotnet/samples/KernelSyntaxExamples/Example11_WebSearchQueries.cs b/dotnet/samples/KernelSyntaxExamples/Example11_WebSearchQueries.cs index f34d21e362d1..f6c06cef9da8 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example11_WebSearchQueries.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example11_WebSearchQueries.cs @@ -3,7 +3,7 @@ using System; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Web; +using Microsoft.SemanticKernel.Plugins.Web; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -13,11 +13,11 @@ public static async Task RunAsync() { Console.WriteLine("======== WebSearchQueries ========"); - IKernel kernel = Kernel.Builder.WithLogger(ConsoleLogger.Logger).Build(); + IKernel kernel = Kernel.Builder.WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); - // Load native skills - var skill = new SearchUrlSkill(); - var bing = kernel.ImportSkill(skill, "search"); + // Load native plugins + var plugin = new SearchUrlPlugin(); + var bing = kernel.ImportFunctions(plugin, "search"); // Run var ask = "What's the tallest building in Europe?"; @@ -27,6 +27,6 @@ public static async Task RunAsync() ); Console.WriteLine(ask + "\n"); - Console.WriteLine(result); + Console.WriteLine(result.GetValue()); } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example12_SequentialPlanner.cs b/dotnet/samples/KernelSyntaxExamples/Example12_SequentialPlanner.cs index e5390322aaa0..d74748c38b5b 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example12_SequentialPlanner.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example12_SequentialPlanner.cs @@ -4,11 +4,15 @@ using System.Diagnostics; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Planners; using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.Planning.Sequential; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.Plugins.Memory; +using Plugins; using RepoUtils; -using Skills; // ReSharper disable CommentTypo // ReSharper disable once InconsistentNaming @@ -17,7 +21,7 @@ internal static class Example12_SequentialPlanner public static async Task RunAsync() { await PoetrySamplesAsync(); - await EmailSamplesAsync(); + await EmailSamplesWithRecallAsync(); await BookSamplesAsync(); await MemorySampleAsync(); await PlanNotPossibleSampleAsync(); @@ -28,36 +32,36 @@ private static async Task PlanNotPossibleSampleAsync() Console.WriteLine("======== Sequential Planner - Plan Not Possible ========"); var kernel = InitializeKernelAndPlanner(out var planner); - // Load additional skills to enable planner but not enough for the given goal. - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); + // Load additional plugins to enable planner but not enough for the given goal. + string folder = RepoFiles.SamplePluginsPath(); + kernel.ImportSemanticFunctionsFromDirectory(folder, "SummarizePlugin"); try { await planner.CreatePlanAsync("Write a poem about John Doe, then translate it into Italian."); } - catch (PlanningException e) + catch (SKException e) { Console.WriteLine(e.Message); // Create plan error: Not possible to create plan for goal with available functions. // Goal:Write a poem about John Doe, then translate it into Italian. // Functions: - // SummarizeSkill.MakeAbstractReadable: + // SummarizePlugin.MakeAbstractReadable: // description: Given a scientific white paper abstract, rewrite it to make it more readable // inputs: // - input: - // SummarizeSkill.Notegen: + // SummarizePlugin.Notegen: // description: Automatically generate compact notes for any text or text document. // inputs: // - input: - // SummarizeSkill.Summarize: + // SummarizePlugin.Summarize: // description: Summarize given text or any text document // inputs: // - input: Text to summarize - // SummarizeSkill.Topics: + // SummarizePlugin.Topics: // description: Analyze given text or document and extract key topics worth remembering // inputs: // - input: @@ -68,17 +72,17 @@ private static async Task PoetrySamplesAsync() { Console.WriteLine("======== Sequential Planner - Create and Execute Poetry Plan ========"); var kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithAzureTextCompletionService( - TestConfiguration.AzureOpenAI.DeploymentName, + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) .Build(); - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, - "SummarizeSkill", - "WriterSkill"); + string folder = RepoFiles.SamplePluginsPath(); + kernel.ImportSemanticFunctionsFromDirectory(folder, + "SummarizePlugin", + "WriterPlugin"); var planner = new SequentialPlanner(kernel); @@ -88,8 +92,8 @@ private static async Task PoetrySamplesAsync() // Goal: Write a poem about John Doe, then translate it into Italian. // Steps: - // - WriterSkill.ShortPoem INPUT='John Doe is a friendly guy who likes to help others and enjoys reading books.' => - // - WriterSkill.Translate language='Italian' INPUT='' => + // - WriterPlugin.ShortPoem INPUT='John Doe is a friendly guy who likes to help others and enjoys reading books.' => + // - WriterPlugin.Translate language='Italian' INPUT='' => Console.WriteLine("Original plan:"); Console.WriteLine(plan.ToPlanWithGoalString()); @@ -97,20 +101,20 @@ private static async Task PoetrySamplesAsync() var result = await kernel.RunAsync(plan); Console.WriteLine("Result:"); - Console.WriteLine(result.Result); + Console.WriteLine(result.GetValue()); } - private static async Task EmailSamplesAsync() + private static async Task EmailSamplesWithRecallAsync() { Console.WriteLine("======== Sequential Planner - Create and Execute Email Plan ========"); var kernel = InitializeKernelAndPlanner(out var planner, 512); - kernel.ImportSkill(new EmailSkill(), "email"); + kernel.ImportFunctions(new EmailPlugin(), "email"); - // Load additional skills to enable planner to do non-trivial asks. - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, - "SummarizeSkill", - "WriterSkill"); + // Load additional plugins to enable planner to do non-trivial asks. + string folder = RepoFiles.SamplePluginsPath(); + kernel.ImportSemanticFunctionsFromDirectory(folder, + "SummarizePlugin", + "WriterPlugin"); var plan = await planner.CreatePlanAsync("Summarize an input, translate to french, and e-mail to John Doe"); @@ -118,14 +122,17 @@ private static async Task EmailSamplesAsync() // Goal: Summarize an input, translate to french, and e-mail to John Doe // Steps: - // - SummarizeSkill.Summarize INPUT='' => - // - WriterSkill.Translate language='French' INPUT='' => TRANSLATED_SUMMARY + // - SummarizePlugin.Summarize INPUT='' => + // - WriterPlugin.Translate language='French' INPUT='' => TRANSLATED_SUMMARY // - email.GetEmailAddress INPUT='John Doe' => EMAIL_ADDRESS // - email.SendEmail INPUT='$TRANSLATED_SUMMARY' email_address='$EMAIL_ADDRESS' => Console.WriteLine("Original plan:"); Console.WriteLine(plan.ToPlanWithGoalString()); + // Serialize plan before execution for saving to memory on success. + var originalPlan = plan.ToJson(); + var input = "Once upon a time, in a faraway kingdom, there lived a kind and just king named Arjun. " + "He ruled over his kingdom with fairness and compassion, earning him the love and admiration of his people. " + @@ -139,6 +146,51 @@ private static async Task EmailSamplesAsync() "They ruled the kingdom together, ruling with fairness and compassion, just as Arjun had done before. They lived " + "happily ever after, with the people of the kingdom remembering Mira as the brave young woman who saved them from the dragon."; await ExecutePlanAsync(kernel, plan, input, 5); + + Console.WriteLine("======== Sequential Planner - Find and Execute Saved Plan ========"); + + // Save the plan for future use + var semanticMemory = InitializeMemory(); + await semanticMemory.SaveInformationAsync( + "plans", + id: Guid.NewGuid().ToString(), + text: plan.Description, // This is the goal used to create the plan + description: originalPlan); + + var goal = "Write summary in french and e-mail John Doe"; + + Console.WriteLine($"Goal: {goal}"); + Console.WriteLine("Searching for saved plan..."); + + Plan? restoredPlan = null; + var memories = semanticMemory.SearchAsync("plans", goal, limit: 1, minRelevanceScore: 0.5); + await foreach (MemoryQueryResult memory in memories) + { + Console.WriteLine($"Restored plan (relevance={memory.Relevance}):"); + + // Deseriliaze the plan from the description + restoredPlan = Plan.FromJson(memory.Metadata.Description, kernel.Functions); + + Console.WriteLine(restoredPlan.ToPlanWithGoalString()); + Console.WriteLine(); + + break; + } + + if (restoredPlan is not null) + { + var newInput = + "Far in the future, on a planet lightyears away, 15 year old Remy lives a normal life. He goes to school, " + + "hangs out with his friends, and tries to avoid trouble. But when he stumbles across a secret that threatens to destroy " + + "everything he knows, he's forced to go on the run. With the help of a mysterious girl named Eve, he must evade the ruthless " + + "agents of the Galactic Federation, and uncover the truth about his past. But the more he learns, the more he realizes that " + + "he's not just an ordinary boy."; + + var result = await kernel.RunAsync(restoredPlan, new(newInput)); + + Console.WriteLine("Result:"); + Console.WriteLine(result.GetValue()); + } } private static async Task BookSamplesAsync() @@ -146,10 +198,10 @@ private static async Task BookSamplesAsync() Console.WriteLine("======== Sequential Planner - Create and Execute Book Creation Plan ========"); var kernel = InitializeKernelAndPlanner(out var planner); - // Load additional skills to enable planner to do non-trivial asks. - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, "WriterSkill"); - kernel.ImportSemanticSkillFromDirectory(folder, "MiscSkill"); + // Load additional plugins to enable planner to do non-trivial asks. + string folder = RepoFiles.SamplePluginsPath(); + kernel.ImportSemanticFunctionsFromDirectory(folder, "WriterPlugin"); + kernel.ImportSemanticFunctionsFromDirectory(folder, "MiscPlugin"); var originalPlan = await planner.CreatePlanAsync("Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'"); @@ -157,13 +209,13 @@ private static async Task BookSamplesAsync() // Goal: Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.' // Steps: - // - WriterSkill.NovelOutline chapterCount='3' INPUT='A group of kids in a club called 'The Thinking Caps' that solve mysteries and puzzles using their creativity and logic.' endMarker='' => OUTLINE - // - MiscSkill.ElementAtIndex count='3' INPUT='$OUTLINE' index='0' => CHAPTER_1_SYNOPSIS - // - WriterSkill.NovelChapter chapterIndex='1' previousChapter='' INPUT='$CHAPTER_1_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_1 - // - MiscSkill.ElementAtIndex count='3' INPUT='$OUTLINE' index='1' => CHAPTER_2_SYNOPSIS - // - WriterSkill.NovelChapter chapterIndex='2' previousChapter='$CHAPTER_1_SYNOPSIS' INPUT='$CHAPTER_2_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_2 - // - MiscSkill.ElementAtIndex count='3' INPUT='$OUTLINE' index='2' => CHAPTER_3_SYNOPSIS - // - WriterSkill.NovelChapter chapterIndex='3' previousChapter='$CHAPTER_2_SYNOPSIS' INPUT='$CHAPTER_3_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_3 + // - WriterPlugin.NovelOutline chapterCount='3' INPUT='A group of kids in a club called 'The Thinking Caps' that solve mysteries and puzzles using their creativity and logic.' endMarker='' => OUTLINE + // - MiscPlugin.ElementAtIndex count='3' INPUT='$OUTLINE' index='0' => CHAPTER_1_SYNOPSIS + // - WriterPlugin.NovelChapter chapterIndex='1' previousChapter='' INPUT='$CHAPTER_1_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_1 + // - MiscPlugin.ElementAtIndex count='3' INPUT='$OUTLINE' index='1' => CHAPTER_2_SYNOPSIS + // - WriterPlugin.NovelChapter chapterIndex='2' previousChapter='$CHAPTER_1_SYNOPSIS' INPUT='$CHAPTER_2_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_2 + // - MiscPlugin.ElementAtIndex count='3' INPUT='$OUTLINE' index='2' => CHAPTER_3_SYNOPSIS + // - WriterPlugin.NovelChapter chapterIndex='3' previousChapter='$CHAPTER_2_SYNOPSIS' INPUT='$CHAPTER_3_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_3 Console.WriteLine("Original plan:"); Console.WriteLine(originalPlan.ToPlanWithGoalString()); @@ -177,43 +229,31 @@ private static async Task MemorySampleAsync() { Console.WriteLine("======== Sequential Planner - Create and Execute Plan using Memory ========"); - // IMPORTANT: Register an embedding generation service and a memory store. The Planner will - // use these to generate and store embeddings for the function descriptions. - var kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithAzureChatCompletionService( - TestConfiguration.AzureOpenAI.ChatDeploymentName, - TestConfiguration.AzureOpenAI.Endpoint, - TestConfiguration.AzureOpenAI.ApiKey) - .WithAzureTextEmbeddingGenerationService( - TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, - TestConfiguration.AzureOpenAIEmbeddings.Endpoint, - TestConfiguration.AzureOpenAIEmbeddings.ApiKey) - .WithMemoryStorage(new VolatileMemoryStore()) - .Build(); - - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, - "SummarizeSkill", - "WriterSkill", - "CalendarSkill", - "ChatSkill", - "ChildrensBookSkill", - "ClassificationSkill", - "CodingSkill", - "FunSkill", - "IntentDetectionSkill", - "MiscSkill", - "QASkill"); - - kernel.ImportSkill(new EmailSkill(), "email"); - kernel.ImportSkill(new StaticTextSkill(), "statictext"); - kernel.ImportSkill(new Microsoft.SemanticKernel.Skills.Core.TextSkill(), "coretext"); + var kernel = InitializeKernel(); + var memory = InitializeMemory(); + + string folder = RepoFiles.SamplePluginsPath(); + kernel.ImportSemanticFunctionsFromDirectory(folder, + "SummarizePlugin", + "WriterPlugin", + "CalendarPlugin", + "ChatPlugin", + "ChildrensBookPlugin", + "ClassificationPlugin", + "CodingPlugin", + "FunPlugin", + "IntentDetectionPlugin", + "MiscPlugin", + "QAPlugin"); + + kernel.ImportFunctions(new EmailPlugin(), "email"); + kernel.ImportFunctions(new StaticTextPlugin(), "statictext"); + kernel.ImportFunctions(new TextPlugin(), "coretext"); var goal = "Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'"; - // IMPORTANT: To use memory and embeddings to find relevant skills in the planner, set the 'Memory' property on the planner config. - var planner = new SequentialPlanner(kernel, new SequentialPlannerConfig { RelevancyThreshold = 0.5, Memory = kernel.Memory }); + // IMPORTANT: To use memory and embeddings to find relevant plugins in the planner, set the 'Memory' property on the planner config. + var planner = new SequentialPlanner(kernel, new SequentialPlannerConfig { SemanticMemoryConfig = new() { RelevancyThreshold = 0.5, Memory = memory } }); var plan = await planner.CreatePlanAsync(goal); @@ -224,7 +264,7 @@ private static async Task MemorySampleAsync() private static IKernel InitializeKernelAndPlanner(out SequentialPlanner planner, int maxTokens = 1024) { var kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) + .WithLoggerFactory(ConsoleLogger.LoggerFactory) .WithAzureChatCompletionService( TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, @@ -236,6 +276,39 @@ private static IKernel InitializeKernelAndPlanner(out SequentialPlanner planner, return kernel; } + private static IKernel InitializeKernel() + { + // IMPORTANT: Register an embedding generation service and a memory store. The Planner will + // use these to generate and store embeddings for the function descriptions. + var kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + TestConfiguration.AzureOpenAI.ChatDeploymentName, + TestConfiguration.AzureOpenAI.Endpoint, + TestConfiguration.AzureOpenAI.ApiKey) + .WithAzureTextEmbeddingGenerationService( + TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, + TestConfiguration.AzureOpenAIEmbeddings.Endpoint, + TestConfiguration.AzureOpenAIEmbeddings.ApiKey) + .Build(); + + return kernel; + } + + private static SemanticTextMemory InitializeMemory() + { + var memoryStorage = new VolatileMemoryStore(); + + var textEmbeddingGenerator = new AzureTextEmbeddingGeneration( + modelId: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, + endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint, + apiKey: TestConfiguration.AzureOpenAIEmbeddings.ApiKey); + + var memory = new SemanticTextMemory(memoryStorage, textEmbeddingGenerator); + + return memory; + } + private static async Task ExecutePlanAsync( IKernel kernel, Plan plan, @@ -272,7 +345,7 @@ private static async Task ExecutePlanAsync( Console.WriteLine(plan.State.ToString()); } } - catch (KernelException e) + catch (SKException e) { Console.WriteLine("Step - Execution failed:"); Console.WriteLine(e.Message); diff --git a/dotnet/samples/KernelSyntaxExamples/Example13_ConversationSummaryPlugin.cs b/dotnet/samples/KernelSyntaxExamples/Example13_ConversationSummaryPlugin.cs new file mode 100644 index 000000000000..17b9bccf4177 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example13_ConversationSummaryPlugin.cs @@ -0,0 +1,272 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Plugins.Core; +using RepoUtils; + +// ReSharper disable once InconsistentNaming + +internal static class Example13_ConversationSummaryPlugin +{ + private const string ChatTranscript = + @" +John: Hello, how are you? +Jane: I'm fine, thanks. How are you? +John: I'm doing well, writing some example code. +Jane: That's great! I'm writing some example code too. +John: What are you writing? +Jane: I'm writing a chatbot. +John: That's cool. I'm writing a chatbot too. +Jane: What language are you writing it in? +John: I'm writing it in C#. +Jane: I'm writing it in Python. +John: That's cool. I need to learn Python. +Jane: I need to learn C#. +John: Can I try out your chatbot? +Jane: Sure, here's the link. +John: Thanks! +Jane: You're welcome. +Jane: Look at this poem my chatbot wrote: +Jane: Roses are red +Jane: Violets are blue +Jane: I'm writing a chatbot +Jane: What about you? +John: That's cool. Let me see if mine will write a poem, too. +John: Here's a poem my chatbot wrote: +John: The singularity of the universe is a mystery. +John: The universe is a mystery. +John: The universe is a mystery. +John: The universe is a mystery. +John: Looks like I need to improve mine, oh well. +Jane: You might want to try using a different model. +Jane: I'm using the GPT-3 model. +John: I'm using the GPT-2 model. That makes sense. +John: Here is a new poem after updating the model. +John: The universe is a mystery. +John: The universe is a mystery. +John: The universe is a mystery. +John: Yikes, it's really stuck isn't it. Would you help me debug my code? +Jane: Sure, what's the problem? +John: I'm not sure. I think it's a bug in the code. +Jane: I'll take a look. +Jane: I think I found the problem. +Jane: It looks like you're not passing the right parameters to the model. +John: Thanks for the help! +Jane: I'm now writing a bot to summarize conversations. I want to make sure it works when the conversation is long. +John: So you need to keep talking with me to generate a long conversation? +Jane: Yes, that's right. +John: Ok, I'll keep talking. What should we talk about? +Jane: I don't know, what do you want to talk about? +John: I don't know, it's nice how CoPilot is doing most of the talking for us. But it definitely gets stuck sometimes. +Jane: I agree, it's nice that CoPilot is doing most of the talking for us. +Jane: But it definitely gets stuck sometimes. +John: Do you know how long it needs to be? +Jane: I think the max length is 1024 tokens. Which is approximately 1024*4= 4096 characters. +John: That's a lot of characters. +Jane: Yes, it is. +John: I'm not sure how much longer I can keep talking. +Jane: I think we're almost there. Let me check. +Jane: I have some bad news, we're only half way there. +John: Oh no, I'm not sure I can keep going. I'm getting tired. +Jane: I'm getting tired too. +John: Maybe there is a large piece of text we can use to generate a long conversation. +Jane: That's a good idea. Let me see if I can find one. Maybe Lorem Ipsum? +John: Yeah, that's a good idea. +Jane: I found a Lorem Ipsum generator. +Jane: Here's a 4096 character Lorem Ipsum text: +Jane: Lorem ipsum dolor sit amet, con +Jane: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nunc sit amet aliquam +Jane: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nunc sit amet aliquam +Jane: Darn, it's just repeating stuf now. +John: I think we're done. +Jane: We're not though! We need like 1500 more characters. +John: Oh Cananda, our home and native land. +Jane: True patriot love in all thy sons command. +John: With glowing hearts we see thee rise. +Jane: The True North strong and free. +John: From far and wide, O Canada, we stand on guard for thee. +Jane: God keep our land glorious and free. +John: O Canada, we stand on guard for thee. +Jane: O Canada, we stand on guard for thee. +Jane: That was fun, thank you. Let me check now. +Jane: I think we need about 600 more characters. +John: Oh say can you see? +Jane: By the dawn's early light. +John: What so proudly we hailed. +Jane: At the twilight's last gleaming. +John: Whose broad stripes and bright stars. +Jane: Through the perilous fight. +John: O'er the ramparts we watched. +Jane: Were so gallantly streaming. +John: And the rockets' red glare. +Jane: The bombs bursting in air. +John: Gave proof through the night. +Jane: That our flag was still there. +John: Oh say does that star-spangled banner yet wave. +Jane: O'er the land of the free. +John: And the home of the brave. +Jane: Are you a Seattle Kraken Fan? +John: Yes, I am. I love going to the games. +Jane: I'm a Seattle Kraken Fan too. Who is your favorite player? +John: I like watching all the players, but I think my favorite is Matty Beniers. +Jane: Yeah, he's a great player. I like watching him too. I also like watching Jaden Schwartz. +John: Adam Larsson is another good one. The big cat! +Jane: WE MADE IT! It's long enough. Thank you! +John: You're welcome. I'm glad we could help. Goodbye! +Jane: Goodbye! +"; + + public static async Task RunAsync() + { + await ConversationSummaryPluginAsync(); + await GetConversationActionItemsAsync(); + await GetConversationTopicsAsync(); + } + + private static async Task ConversationSummaryPluginAsync() + { + Console.WriteLine("======== SamplePlugins - Conversation Summary Plugin - Summarize ========"); + IKernel kernel = InitializeKernel(); + + IDictionary conversationSummaryPlugin = + kernel.ImportFunctions(new ConversationSummaryPlugin(kernel)); + + KernelResult summary = await kernel.RunAsync( + ChatTranscript, + conversationSummaryPlugin["SummarizeConversation"]); + + Console.WriteLine("Generated Summary:"); + Console.WriteLine(summary.GetValue()); + } + + private static async Task GetConversationActionItemsAsync() + { + Console.WriteLine("======== SamplePlugins - Conversation Summary Plugin - Action Items ========"); + IKernel kernel = InitializeKernel(); + + IDictionary conversationSummary = + kernel.ImportFunctions(new ConversationSummaryPlugin(kernel)); + + KernelResult summary = await kernel.RunAsync( + ChatTranscript, + conversationSummary["GetConversationActionItems"]); + + Console.WriteLine("Generated Action Items:"); + Console.WriteLine(summary.GetValue()); + } + + private static async Task GetConversationTopicsAsync() + { + Console.WriteLine("======== SamplePlugins - Conversation Summary Plugin - Topics ========"); + IKernel kernel = InitializeKernel(); + + IDictionary conversationSummary = + kernel.ImportFunctions(new ConversationSummaryPlugin(kernel)); + + KernelResult summary = await kernel.RunAsync( + ChatTranscript, + conversationSummary["GetConversationTopics"]); + + Console.WriteLine("Generated Topics:"); + Console.WriteLine(summary.GetValue()); + } + + private static IKernel InitializeKernel() + { + IKernel kernel = Kernel.Builder + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + TestConfiguration.AzureOpenAI.ChatDeploymentName, + TestConfiguration.AzureOpenAI.Endpoint, + TestConfiguration.AzureOpenAI.ApiKey) + .Build(); + + return kernel; + } +} + +// ReSharper disable CommentTypo +/* Example Output: + +======== SamplePlugins - Conversation Summary Plugin - Summarize ======== +Generated Summary: + +A possible summary is: + +- John and Jane are both writing chatbots in different languages and share their links and poems. +- John's chatbot has a problem with writing repetitive poems and Jane helps him debug his code. +- Jane is writing a bot to summarize conversations and needs to generate a long conversation with John to test it. +- They use CoPilot to do most of the talking for them and comment on its limitations. +- They estimate the max length of the conversation to be 4096 characters. + +A possible summary is: + +- John and Jane are trying to generate a long conversation for some purpose. +- They are getting tired and bored of talking and look for ways to fill up the text. +- They use a Lorem Ipsum generator, but it repeats itself after a while. +- They sing the national anthems of Canada and the United States, and then talk about their favorite Seattle Kraken hockey players. +- They finally reach their desired length of text and say goodbye to each other. +======== SamplePlugins - Conversation Summary Plugin - Action Items ======== +Generated Action Items: + +{ + "actionItems": [ + { + "owner": "John", + "actionItem": "Improve chatbot's poem generation", + "dueDate": "", + "status": "In Progress", + "notes": "Using GPT-3 model" + }, + { + "owner": "Jane", + "actionItem": "Write a bot to summarize conversations", + "dueDate": "", + "status": "In Progress", + "notes": "Testing with long conversations" + } + ] +} + +{ + "action_items": [] +} +======== SamplePlugins - Conversation Summary Plugin - Topics ======== +Generated Topics: + +{ + "topics": [ + "Chatbot", + "Code", + "Poem", + "Model", + "GPT-3", + "GPT-2", + "Bug", + "Parameters", + "Summary", + "CoPilot", + "Tokens", + "Characters" + ] +} + +{ + "topics": [ + "Long conversation", + "Lorem Ipsum", + "O Canada", + "Star-Spangled Banner", + "Seattle Kraken", + "Matty Beniers", + "Jaden Schwartz", + "Adam Larsson" + ] +} + +*/ +// ReSharper restore CommentTypo diff --git a/dotnet/samples/KernelSyntaxExamples/Example13_ConversationSummarySkill.cs b/dotnet/samples/KernelSyntaxExamples/Example13_ConversationSummarySkill.cs deleted file mode 100644 index 1d0aac359ac5..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example13_ConversationSummarySkill.cs +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.Core; -using RepoUtils; - -// ReSharper disable once InconsistentNaming - -internal static class Example13_ConversationSummarySkill -{ - private const string ChatTranscript = - @" -John: Hello, how are you? -Jane: I'm fine, thanks. How are you? -John: I'm doing well, writing some example code. -Jane: That's great! I'm writing some example code too. -John: What are you writing? -Jane: I'm writing a chatbot. -John: That's cool. I'm writing a chatbot too. -Jane: What language are you writing it in? -John: I'm writing it in C#. -Jane: I'm writing it in Python. -John: That's cool. I need to learn Python. -Jane: I need to learn C#. -John: Can I try out your chatbot? -Jane: Sure, here's the link. -John: Thanks! -Jane: You're welcome. -Jane: Look at this poem my chatbot wrote: -Jane: Roses are red -Jane: Violets are blue -Jane: I'm writing a chatbot -Jane: What about you? -John: That's cool. Let me see if mine will write a poem, too. -John: Here's a poem my chatbot wrote: -John: The singularity of the universe is a mystery. -John: The universe is a mystery. -John: The universe is a mystery. -John: The universe is a mystery. -John: Looks like I need to improve mine, oh well. -Jane: You might want to try using a different model. -Jane: I'm using the GPT-3 model. -John: I'm using the GPT-2 model. That makes sense. -John: Here is a new poem after updating the model. -John: The universe is a mystery. -John: The universe is a mystery. -John: The universe is a mystery. -John: Yikes, it's really stuck isn't it. Would you help me debug my code? -Jane: Sure, what's the problem? -John: I'm not sure. I think it's a bug in the code. -Jane: I'll take a look. -Jane: I think I found the problem. -Jane: It looks like you're not passing the right parameters to the model. -John: Thanks for the help! -Jane: I'm now writing a bot to summarize conversations. I want to make sure it works when the conversation is long. -John: So you need to keep talking with me to generate a long conversation? -Jane: Yes, that's right. -John: Ok, I'll keep talking. What should we talk about? -Jane: I don't know, what do you want to talk about? -John: I don't know, it's nice how CoPilot is doing most of the talking for us. But it definitely gets stuck sometimes. -Jane: I agree, it's nice that CoPilot is doing most of the talking for us. -Jane: But it definitely gets stuck sometimes. -John: Do you know how long it needs to be? -Jane: I think the max length is 1024 tokens. Which is approximately 1024*4= 4096 characters. -John: That's a lot of characters. -Jane: Yes, it is. -John: I'm not sure how much longer I can keep talking. -Jane: I think we're almost there. Let me check. -Jane: I have some bad news, we're only half way there. -John: Oh no, I'm not sure I can keep going. I'm getting tired. -Jane: I'm getting tired too. -John: Maybe there is a large piece of text we can use to generate a long conversation. -Jane: That's a good idea. Let me see if I can find one. Maybe Lorem Ipsum? -John: Yeah, that's a good idea. -Jane: I found a Lorem Ipsum generator. -Jane: Here's a 4096 character Lorem Ipsum text: -Jane: Lorem ipsum dolor sit amet, con -Jane: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nunc sit amet aliquam -Jane: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nunc sit amet aliquam -Jane: Darn, it's just repeating stuf now. -John: I think we're done. -Jane: We're not though! We need like 1500 more characters. -John: Oh Cananda, our home and native land. -Jane: True patriot love in all thy sons command. -John: With glowing hearts we see thee rise. -Jane: The True North strong and free. -John: From far and wide, O Canada, we stand on guard for thee. -Jane: God keep our land glorious and free. -John: O Canada, we stand on guard for thee. -Jane: O Canada, we stand on guard for thee. -Jane: That was fun, thank you. Let me check now. -Jane: I think we need about 600 more characters. -John: Oh say can you see? -Jane: By the dawn's early light. -John: What so proudly we hailed. -Jane: At the twilight's last gleaming. -John: Whose broad stripes and bright stars. -Jane: Through the perilous fight. -John: O'er the ramparts we watched. -Jane: Were so gallantly streaming. -John: And the rockets' red glare. -Jane: The bombs bursting in air. -John: Gave proof through the night. -Jane: That our flag was still there. -John: Oh say does that star-spangled banner yet wave. -Jane: O'er the land of the free. -John: And the home of the brave. -Jane: Are you a Seattle Kraken Fan? -John: Yes, I am. I love going to the games. -Jane: I'm a Seattle Kraken Fan too. Who is your favorite player? -John: I like watching all the players, but I think my favorite is Matty Beniers. -Jane: Yeah, he's a great player. I like watching him too. I also like watching Jaden Schwartz. -John: Adam Larsson is another good one. The big cat! -Jane: WE MADE IT! It's long enough. Thank you! -John: You're welcome. I'm glad we could help. Goodbye! -Jane: Goodbye! -"; - - public static async Task RunAsync() - { - await ConversationSummarySkillAsync(); - await GetConversationActionItemsAsync(); - await GetConversationTopicsAsync(); - } - - private static async Task ConversationSummarySkillAsync() - { - Console.WriteLine("======== SampleSkills - Conversation Summary Skill - Summarize ========"); - IKernel kernel = InitializeKernel(); - - IDictionary conversationSummarySkill = - kernel.ImportSkill(new ConversationSummarySkill(kernel)); - - SKContext summary = await kernel.RunAsync( - ChatTranscript, - conversationSummarySkill["SummarizeConversation"]); - - Console.WriteLine("Generated Summary:"); - Console.WriteLine(summary.Result); - } - - private static async Task GetConversationActionItemsAsync() - { - Console.WriteLine("======== SampleSkills - Conversation Summary Skill - Action Items ========"); - IKernel kernel = InitializeKernel(); - - IDictionary conversationSummarySkill = - kernel.ImportSkill(new ConversationSummarySkill(kernel)); - - SKContext summary = await kernel.RunAsync( - ChatTranscript, - conversationSummarySkill["GetConversationActionItems"]); - - Console.WriteLine("Generated Action Items:"); - Console.WriteLine(summary.Result); - } - - private static async Task GetConversationTopicsAsync() - { - Console.WriteLine("======== SampleSkills - Conversation Summary Skill - Topics ========"); - IKernel kernel = InitializeKernel(); - - IDictionary conversationSummarySkill = - kernel.ImportSkill(new ConversationSummarySkill(kernel)); - - SKContext summary = await kernel.RunAsync( - ChatTranscript, - conversationSummarySkill["GetConversationTopics"]); - - Console.WriteLine("Generated Topics:"); - Console.WriteLine(summary.Result); - } - - private static IKernel InitializeKernel() - { - IKernel kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithAzureTextCompletionService( - TestConfiguration.AzureOpenAI.DeploymentName, - TestConfiguration.AzureOpenAI.Endpoint, - TestConfiguration.AzureOpenAI.ApiKey) - .Build(); - - return kernel; - } -} - -// ReSharper disable CommentTypo -/* Example Output: - -======== SampleSkills - Conversation Summary Skill - Summarize ======== -Generated Summary: - -A possible summary is: - -- John and Jane are both writing chatbots in different languages and share their links and poems. -- John's chatbot has a problem with writing repetitive poems and Jane helps him debug his code. -- Jane is writing a bot to summarize conversations and needs to generate a long conversation with John to test it. -- They use CoPilot to do most of the talking for them and comment on its limitations. -- They estimate the max length of the conversation to be 4096 characters. - -A possible summary is: - -- John and Jane are trying to generate a long conversation for some purpose. -- They are getting tired and bored of talking and look for ways to fill up the text. -- They use a Lorem Ipsum generator, but it repeats itself after a while. -- They sing the national anthems of Canada and the United States, and then talk about their favorite Seattle Kraken hockey players. -- They finally reach their desired length of text and say goodbye to each other. -======== SampleSkills - Conversation Summary Skill - Action Items ======== -Generated Action Items: - -{ - "actionItems": [ - { - "owner": "John", - "actionItem": "Improve chatbot's poem generation", - "dueDate": "", - "status": "In Progress", - "notes": "Using GPT-3 model" - }, - { - "owner": "Jane", - "actionItem": "Write a bot to summarize conversations", - "dueDate": "", - "status": "In Progress", - "notes": "Testing with long conversations" - } - ] -} - -{ - "action_items": [] -} -======== SampleSkills - Conversation Summary Skill - Topics ======== -Generated Topics: - -{ - "topics": [ - "Chatbot", - "Code", - "Poem", - "Model", - "GPT-3", - "GPT-2", - "Bug", - "Parameters", - "Summary", - "CoPilot", - "Tokens", - "Characters" - ] -} - -{ - "topics": [ - "Long conversation", - "Lorem Ipsum", - "O Canada", - "Star-Spangled Banner", - "Seattle Kraken", - "Matty Beniers", - "Jaden Schwartz", - "Adam Larsson" - ] -} - -*/ -// ReSharper restore CommentTypo diff --git a/dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs b/dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs index 0fa5bbd964e4..e47ab6e5ce19 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs @@ -4,8 +4,10 @@ using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Plugins.Memory; using RepoUtils; /* The files contains two examples about SK Semantic Memory. @@ -34,13 +36,13 @@ public static async Task RunAsync() * need to worry about embedding generation. */ - var kernelWithACS = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) + var memoryWithACS = new MemoryBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(new AzureCognitiveSearchMemoryStore(TestConfiguration.ACS.Endpoint, TestConfiguration.ACS.ApiKey)) + .WithMemoryStore(new AzureCognitiveSearchMemoryStore(TestConfiguration.ACS.Endpoint, TestConfiguration.ACS.ApiKey)) .Build(); - await RunExampleAsync(kernelWithACS); + await RunExampleAsync(memoryWithACS); Console.WriteLine("===================================================="); Console.WriteLine("======== Semantic Memory (volatile, in RAM) ========"); @@ -55,20 +57,20 @@ public static async Task RunAsync() * or implement your connectors for Pinecone, Vespa, Postgres + pgvector, SQLite VSS, etc. */ - var kernelWithCustomDb = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextEmbeddingGenerationService("ada", "text-embedding-ada-002", TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(new VolatileMemoryStore()) + var memoryWithCustomDb = new MemoryBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", TestConfiguration.OpenAI.ApiKey) + .WithMemoryStore(new VolatileMemoryStore()) .Build(); - await RunExampleAsync(kernelWithCustomDb); + await RunExampleAsync(memoryWithCustomDb); } - public static async Task RunExampleAsync(IKernel kernel) + public static async Task RunExampleAsync(ISemanticTextMemory memory) { - await StoreMemoryAsync(kernel); + await StoreMemoryAsync(memory); - await SearchMemoryAsync(kernel, "How do I get started?"); + await SearchMemoryAsync(memory, "How do I get started?"); /* Output: @@ -85,7 +87,7 @@ public static async Task RunExampleAsync(IKernel kernel) */ - await SearchMemoryAsync(kernel, "Can I build a chat with SK?"); + await SearchMemoryAsync(memory, "Can I build a chat with SK?"); /* Output: @@ -93,8 +95,8 @@ public static async Task RunExampleAsync(IKernel kernel) Query: Can I build a chat with SK? Result 1: - URL: : https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT - Title : Sample demonstrating how to create a chat skill interfacing with ChatGPT + URL: : https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins/ChatPlugin/ChatGPT + Title : Sample demonstrating how to create a chat plugin interfacing with ChatGPT Result 2: URL: : https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md @@ -103,26 +105,26 @@ public static async Task RunExampleAsync(IKernel kernel) */ } - private static async Task SearchMemoryAsync(IKernel kernel, string query) + private static async Task SearchMemoryAsync(ISemanticTextMemory memory, string query) { Console.WriteLine("\nQuery: " + query + "\n"); - var memories = kernel.Memory.SearchAsync(MemoryCollectionName, query, limit: 2, minRelevanceScore: 0.5); + var memoryResults = memory.SearchAsync(MemoryCollectionName, query, limit: 2, minRelevanceScore: 0.5); int i = 0; - await foreach (MemoryQueryResult memory in memories) + await foreach (MemoryQueryResult memoryResult in memoryResults) { Console.WriteLine($"Result {++i}:"); - Console.WriteLine(" URL: : " + memory.Metadata.Id); - Console.WriteLine(" Title : " + memory.Metadata.Description); - Console.WriteLine(" Relevance: " + memory.Relevance); + Console.WriteLine(" URL: : " + memoryResult.Metadata.Id); + Console.WriteLine(" Title : " + memoryResult.Metadata.Description); + Console.WriteLine(" Relevance: " + memoryResult.Relevance); Console.WriteLine(); } Console.WriteLine("----------------------"); } - private static async Task StoreMemoryAsync(IKernel kernel) + private static async Task StoreMemoryAsync(ISemanticTextMemory memory) { /* Store some data in the semantic memory. * @@ -137,7 +139,7 @@ private static async Task StoreMemoryAsync(IKernel kernel) var i = 0; foreach (var entry in githubFiles) { - await kernel.Memory.SaveReferenceAsync( + await memory.SaveReferenceAsync( collection: MemoryCollectionName, externalSourceName: "GitHub", externalId: entry.Key, @@ -156,12 +158,12 @@ private static Dictionary SampleData() { ["https://github.com/microsoft/semantic-kernel/blob/main/README.md"] = "README: Installation, getting started, and how to contribute", - ["https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb"] - = "Jupyter notebook describing how to pass prompts from a file to a semantic skill or function", - ["https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/00-getting-started.ipynb"] + ["https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/02-running-prompts-from-file.ipynb"] + = "Jupyter notebook describing how to pass prompts from a file to a semantic plugin or function", + ["https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks//00-getting-started.ipynb"] = "Jupyter notebook describing how to get started with the Semantic Kernel", - ["https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT"] - = "Sample demonstrating how to create a chat skill interfacing with ChatGPT", + ["https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins/ChatPlugin/ChatGPT"] + = "Sample demonstrating how to create a chat plugin interfacing with ChatGPT", ["https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs"] = "C# class that defines a volatile embedding store", ["https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md"] diff --git a/dotnet/samples/KernelSyntaxExamples/Example15_MemorySkill.cs b/dotnet/samples/KernelSyntaxExamples/Example15_MemorySkill.cs deleted file mode 100644 index 52425b4d0b66..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example15_MemorySkill.cs +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.Core; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -public static class Example15_MemorySkill -{ - private const string MemoryCollectionName = "aboutMe"; - - public static async Task RunAsync() - { - var kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService("text-davinci-003", TestConfiguration.OpenAI.ApiKey) - .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(new VolatileMemoryStore()) - .Build(); - - // ========= Store memories using the kernel ========= - - await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "info1", text: "My name is Andrea"); - await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "info2", text: "I work as a tourist operator"); - await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "info3", text: "I've been living in Seattle since 2005"); - await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "info4", text: "I visited France and Italy five times since 2015"); - - // ========= Store memories using semantic function ========= - - // Add Memory as a skill for other functions - var memorySkill = new TextMemorySkill(kernel.Memory); - kernel.ImportSkill(memorySkill); - - // Build a semantic function that saves info to memory - const string SaveFunctionDefinition = "{{save $info}}"; - var memorySaver = kernel.CreateSemanticFunction(SaveFunctionDefinition); - - var context = kernel.CreateNewContext(); - context.Variables[TextMemorySkill.CollectionParam] = MemoryCollectionName; - context.Variables[TextMemorySkill.KeyParam] = "info5"; - context.Variables["info"] = "My family is from New York"; - await memorySaver.InvokeAsync(context); - - // ========= Test memory remember ========= - Console.WriteLine("========= Example: Recalling a Memory ========="); - - var answer = await memorySkill.RetrieveAsync(MemoryCollectionName, "info5", logger: context.Logger); - Console.WriteLine("Memory associated with 'info1': {0}", answer); - /* - Output: - "Memory associated with 'info1': My name is Andrea - */ - - // ========= Test memory recall ========= - Console.WriteLine("========= Example: Recalling an Idea ========="); - - answer = await memorySkill.RecallAsync("where did I grow up?", MemoryCollectionName, relevance: null, limit: 2, logger: context.Logger); - Console.WriteLine("Ask: where did I grow up?"); - Console.WriteLine("Answer:\n{0}", answer); - - answer = await memorySkill.RecallAsync("where do I live?", MemoryCollectionName, relevance: null, limit: 2, logger: context.Logger); - Console.WriteLine("Ask: where do I live?"); - Console.WriteLine("Answer:\n{0}", answer); - - /* - Output: - - Ask: where did I grow up? - Answer: - ["My family is from New York","I\u0027ve been living in Seattle since 2005"] - - Ask: where do I live? - Answer: - ["I\u0027ve been living in Seattle since 2005","My family is from New York"] - */ - - // ========= Use memory in a semantic function ========= - Console.WriteLine("========= Example: Using Recall in a Semantic Function ========="); - - // Build a semantic function that uses memory to find facts - const string RecallFunctionDefinition = @" -Consider only the facts below when answering questions. - -About me: {{recall 'where did I grow up?'}} -About me: {{recall 'where do I live?'}} - -Question: {{$input}} - -Answer: -"; - - var aboutMeOracle = kernel.CreateSemanticFunction(RecallFunctionDefinition, maxTokens: 100); - - context = kernel.CreateNewContext(); - context.Variables[TextMemorySkill.CollectionParam] = MemoryCollectionName; - context.Variables[TextMemorySkill.RelevanceParam] = "0.8"; - var result = await aboutMeOracle.InvokeAsync("Do I live in the same town where I grew up?", context); - - Console.WriteLine("Do I live in the same town where I grew up?\n"); - Console.WriteLine(result); - - /* - Output: - - Do I live in the same town where I grew up? - - No, I do not live in the same town where I grew up since my family is from New York and I have been living in Seattle since 2005. - */ - - // ========= Remove a memory ========= - Console.WriteLine("========= Example: Forgetting a Memory ========="); - - context.Variables["fact1"] = "What is my name?"; - context.Variables["fact2"] = "What do I do for a living?"; - context.Variables[TextMemorySkill.RelevanceParam] = ".75"; - - result = await aboutMeOracle.InvokeAsync("Tell me a bit about myself", context); - - Console.WriteLine("Tell me a bit about myself\n"); - Console.WriteLine(result); - - /* - Approximate Output: - Tell me a bit about myself - - My name is Andrea and my family is from New York. I work as a tourist operator. - */ - - context.Variables[TextMemorySkill.KeyParam] = "info1"; - await memorySkill.RemoveAsync(MemoryCollectionName, "info1", logger: context.Logger); - - result = await aboutMeOracle.InvokeAsync("Tell me a bit about myself", context); - - Console.WriteLine("Tell me a bit about myself\n"); - Console.WriteLine(result); - - /* - Approximate Output: - Tell me a bit about myself - - I'm from a family originally from New York and I work as a tourist operator. I've been living in Seattle since 2005. - */ - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example15_TextMemoryPlugin.cs b/dotnet/samples/KernelSyntaxExamples/Example15_TextMemoryPlugin.cs new file mode 100644 index 000000000000..f5136b56cccd --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example15_TextMemoryPlugin.cs @@ -0,0 +1,328 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; +using Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch; +using Microsoft.SemanticKernel.Connectors.Memory.Chroma; +using Microsoft.SemanticKernel.Connectors.Memory.DuckDB; +using Microsoft.SemanticKernel.Connectors.Memory.Kusto; +using Microsoft.SemanticKernel.Connectors.Memory.Pinecone; +using Microsoft.SemanticKernel.Connectors.Memory.Postgres; +using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; +using Microsoft.SemanticKernel.Connectors.Memory.Redis; +using Microsoft.SemanticKernel.Connectors.Memory.Sqlite; +using Microsoft.SemanticKernel.Connectors.Memory.Weaviate; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Plugins.Memory; +using Npgsql; +using Pgvector.Npgsql; +using RepoUtils; +using StackExchange.Redis; + +// ReSharper disable once InconsistentNaming +public static class Example15_TextMemoryPlugin +{ + private const string MemoryCollectionName = "aboutMe"; + + public static async Task RunAsync(CancellationToken cancellationToken = default) + { + IMemoryStore store; + + /////////////////////////////////////////////////////////////////////////////////////////// + // INSTRUCTIONS: uncomment one of the following lines to select the memory store to use. // + /////////////////////////////////////////////////////////////////////////////////////////// + + // Volatile Memory Store - an in-memory store that is not persisted + store = new VolatileMemoryStore(); + + // Sqlite Memory Store - a file-based store that persists data in a Sqlite database + // store = await CreateSampleSqliteMemoryStoreAsync(); + + // DuckDB Memory Store - a file-based store that persists data in a DuckDB database + // store = await CreateSampleDuckDbMemoryStoreAsync(); + + // Azure Cognitive Search Memory Store - a store that persists data in a hosted Azure Cognitive Search database + // store = CreateSampleAzureCognitiveSearchMemoryStore(); + + // Qdrant Memory Store - a store that persists data in a local or remote Qdrant database + // store = CreateSampleQdrantMemoryStore(); + + // Chroma Memory Store + // store = CreateSampleChromaMemoryStore(); + + // Pinecone Memory Store - a store that persists data in a hosted Pinecone database + // store = CreateSamplePineconeMemoryStore(); + + // Weaviate Memory Store + // store = CreateSampleWeaviateMemoryStore(); + + // Redis Memory Store + // store = await CreateSampleRedisMemoryStoreAsync(); + + // Postgres Memory Store + // store = CreateSamplePostgresMemoryStore(); + + // Kusto Memory Store + // store = CreateSampleKustoMemoryStore(); + + await RunWithStoreAsync(store, cancellationToken); + } + + private static async Task CreateSampleSqliteMemoryStoreAsync() + { + IMemoryStore store = await SqliteMemoryStore.ConnectAsync("memories.sqlite"); + return store; + } + + private static async Task CreateSampleDuckDbMemoryStoreAsync() + { + IMemoryStore store = await DuckDBMemoryStore.ConnectAsync("memories.duckdb"); + return store; + } + + private static IMemoryStore CreateSampleAzureCognitiveSearchMemoryStore() + { + IMemoryStore store = new AzureCognitiveSearchMemoryStore(TestConfiguration.ACS.Endpoint, TestConfiguration.ACS.ApiKey); + return store; + } + + private static IMemoryStore CreateSampleChromaMemoryStore() + { + IMemoryStore store = new ChromaMemoryStore(TestConfiguration.Chroma.Endpoint, ConsoleLogger.LoggerFactory); + return store; + } + + private static IMemoryStore CreateSampleQdrantMemoryStore() + { + IMemoryStore store = new QdrantMemoryStore(TestConfiguration.Qdrant.Endpoint, 1536, ConsoleLogger.LoggerFactory); + return store; + } + + private static IMemoryStore CreateSamplePineconeMemoryStore() + { + IMemoryStore store = new PineconeMemoryStore(TestConfiguration.Pinecone.Environment, TestConfiguration.Pinecone.ApiKey, ConsoleLogger.LoggerFactory); + return store; + } + + private static IMemoryStore CreateSampleWeaviateMemoryStore() + { + IMemoryStore store = new WeaviateMemoryStore(TestConfiguration.Weaviate.Endpoint, TestConfiguration.Weaviate.ApiKey); + return store; + } + + private static async Task CreateSampleRedisMemoryStoreAsync() + { + string configuration = TestConfiguration.Redis.Configuration; + ConnectionMultiplexer connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync(configuration); + IDatabase database = connectionMultiplexer.GetDatabase(); + IMemoryStore store = new RedisMemoryStore(database, vectorSize: 1536); + return store; + } + + private static IMemoryStore CreateSamplePostgresMemoryStore() + { + NpgsqlDataSourceBuilder dataSourceBuilder = new(TestConfiguration.Postgres.ConnectionString); + dataSourceBuilder.UseVector(); + NpgsqlDataSource dataSource = dataSourceBuilder.Build(); + IMemoryStore store = new PostgresMemoryStore(dataSource, vectorSize: 1536, schema: "public"); + return store; + } + + private static IMemoryStore CreateSampleKustoMemoryStore() + { + var connectionString = new Kusto.Data.KustoConnectionStringBuilder(TestConfiguration.Kusto.ConnectionString).WithAadUserPromptAuthentication(); + IMemoryStore store = new KustoMemoryStore(connectionString, "MyDatabase"); + return store; + } + + private static async Task RunWithStoreAsync(IMemoryStore memoryStore, CancellationToken cancellationToken) + { + var kernel = Kernel.Builder + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey) + .WithOpenAITextEmbeddingGenerationService(TestConfiguration.OpenAI.EmbeddingModelId, TestConfiguration.OpenAI.ApiKey) + .Build(); + + // Create an embedding generator to use for semantic memory. + var embeddingGenerator = new OpenAITextEmbeddingGeneration(TestConfiguration.OpenAI.EmbeddingModelId, TestConfiguration.OpenAI.ApiKey); + + // The combination of the text embedding generator and the memory store makes up the 'SemanticTextMemory' object used to + // store and retrieve memories. + SemanticTextMemory textMemory = new(memoryStore, embeddingGenerator); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // PART 1: Store and retrieve memories using the ISemanticTextMemory (textMemory) object. + // + // This is a simple way to store memories from a code perspective, without using the Kernel. + ///////////////////////////////////////////////////////////////////////////////////////////////////// + Console.WriteLine("== PART 1a: Saving Memories through the ISemanticTextMemory object =="); + + Console.WriteLine("Saving memory with key 'info1': \"My name is Andrea\""); + await textMemory.SaveInformationAsync(MemoryCollectionName, id: "info1", text: "My name is Andrea", cancellationToken: cancellationToken); + + Console.WriteLine("Saving memory with key 'info2': \"I work as a tourist operator\""); + await textMemory.SaveInformationAsync(MemoryCollectionName, id: "info2", text: "I work as a tourist operator", cancellationToken: cancellationToken); + + Console.WriteLine("Saving memory with key 'info3': \"I've been living in Seattle since 2005\""); + await textMemory.SaveInformationAsync(MemoryCollectionName, id: "info3", text: "I've been living in Seattle since 2005", cancellationToken: cancellationToken); + + Console.WriteLine("Saving memory with key 'info4': \"I visited France and Italy five times since 2015\""); + await textMemory.SaveInformationAsync(MemoryCollectionName, id: "info4", text: "I visited France and Italy five times since 2015", cancellationToken: cancellationToken); + + // Retrieve a memory + Console.WriteLine("== PART 1b: Retrieving Memories through the ISemanticTextMemory object =="); + MemoryQueryResult? lookup = await textMemory.GetAsync(MemoryCollectionName, "info1", cancellationToken: cancellationToken); + Console.WriteLine("Memory with key 'info1':" + lookup?.Metadata.Text ?? "ERROR: memory not found"); + Console.WriteLine(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // PART 2: Create TextMemoryPlugin, store and retrieve memories through the Kernel. + // + // This enables semantic functions and the AI (via Planners) to access memories + ///////////////////////////////////////////////////////////////////////////////////////////////////// + + Console.WriteLine("== PART 2a: Saving Memories through the Kernel with TextMemoryPlugin and the 'Save' function =="); + + // Import the TextMemoryPlugin into the Kernel for other functions + var memoryPlugin = new TextMemoryPlugin(textMemory); + var memoryFunctions = kernel.ImportFunctions(memoryPlugin); + + // Save a memory with the Kernel + Console.WriteLine("Saving memory with key 'info5': \"My family is from New York\""); + await kernel.RunAsync(memoryFunctions["Save"], new() + { + [TextMemoryPlugin.CollectionParam] = MemoryCollectionName, + [TextMemoryPlugin.KeyParam] = "info5", + ["input"] = "My family is from New York" + }, cancellationToken); + + // Retrieve a specific memory with the Kernel + Console.WriteLine("== PART 2b: Retrieving Memories through the Kernel with TextMemoryPlugin and the 'Retrieve' function =="); + var result = await kernel.RunAsync(memoryFunctions["Retrieve"], new() + { + [TextMemoryPlugin.CollectionParam] = MemoryCollectionName, + [TextMemoryPlugin.KeyParam] = "info5" + }, cancellationToken); + + Console.WriteLine("Memory with key 'info5':" + result.GetValue() ?? "ERROR: memory not found"); + Console.WriteLine(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // PART 3: Recall similar ideas with semantic search + // + // Uses AI Embeddings for fuzzy lookup of memories based on intent, rather than a specific key. + ///////////////////////////////////////////////////////////////////////////////////////////////////// + + Console.WriteLine("== PART 3: Recall (similarity search) with AI Embeddings =="); + + Console.WriteLine("== PART 3a: Recall (similarity search) with ISemanticTextMemory =="); + Console.WriteLine("Ask: where did I grow up?"); + + await foreach (var answer in textMemory.SearchAsync( + collection: MemoryCollectionName, + query: "where did I grow up?", + limit: 2, + minRelevanceScore: 0.79, + withEmbeddings: true, + cancellationToken: cancellationToken)) + { + Console.WriteLine($"Answer: {answer.Metadata.Text}"); + } + + Console.WriteLine("== PART 3b: Recall (similarity search) with Kernel and TextMemoryPlugin 'Recall' function =="); + Console.WriteLine("Ask: where do I live?"); + + result = await kernel.RunAsync(memoryFunctions["Recall"], new() + { + [TextMemoryPlugin.CollectionParam] = MemoryCollectionName, + [TextMemoryPlugin.LimitParam] = "2", + [TextMemoryPlugin.RelevanceParam] = "0.79", + ["input"] = "Ask: where do I live?" + }, cancellationToken); + + Console.WriteLine($"Answer: {result.GetValue()}"); + Console.WriteLine(); + + /* + Output: + + Ask: where did I grow up? + Answer: + ["My family is from New York","I\u0027ve been living in Seattle since 2005"] + + Ask: where do I live? + Answer: + ["I\u0027ve been living in Seattle since 2005","My family is from New York"] + */ + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // PART 3: TextMemoryPlugin Recall in a Semantic Function + // + // Looks up related memories when rendering a prompt template, then sends the rendered prompt to + // the text completion model to answer a natural language query. + ///////////////////////////////////////////////////////////////////////////////////////////////////// + + Console.WriteLine("== PART 4: Using TextMemoryPlugin 'Recall' function in a Semantic Function =="); + + // Build a semantic function that uses memory to find facts + const string RecallFunctionDefinition = @" +Consider only the facts below when answering questions: + +BEGIN FACTS +About me: {{recall 'where did I grow up?'}} +About me: {{recall 'where do I live now?'}} +END FACTS + +Question: {{$input}} + +Answer: +"; + + var aboutMeOracle = kernel.CreateSemanticFunction(RecallFunctionDefinition, new OpenAIRequestSettings() { MaxTokens = 100 }); + + result = await kernel.RunAsync(aboutMeOracle, new() + { + [TextMemoryPlugin.CollectionParam] = MemoryCollectionName, + [TextMemoryPlugin.RelevanceParam] = "0.79", + ["input"] = "Do I live in the same town where I grew up?" + }, cancellationToken); + + Console.WriteLine("Ask: Do I live in the same town where I grew up?"); + Console.WriteLine($"Answer: {result.GetValue()}"); + + /* + Approximate Output: + Answer: No, I do not live in the same town where I grew up since my family is from New York and I have been living in Seattle since 2005. + */ + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // PART 5: Cleanup, deleting database collection + // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + + Console.WriteLine("== PART 5: Cleanup, deleting database collection =="); + + Console.WriteLine("Printing Collections in DB..."); + var collections = memoryStore.GetCollectionsAsync(cancellationToken); + await foreach (var collection in collections) + { + Console.WriteLine(collection); + } + Console.WriteLine(); + + Console.WriteLine("Removing Collection {0}", MemoryCollectionName); + await memoryStore.DeleteCollectionAsync(MemoryCollectionName, cancellationToken); + Console.WriteLine(); + + Console.WriteLine($"Printing Collections in DB (after removing {MemoryCollectionName})..."); + collections = memoryStore.GetCollectionsAsync(cancellationToken); + await foreach (var collection in collections) + { + Console.WriteLine(collection); + } + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs index c00e2205daaa..e5f6405f7824 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs @@ -7,7 +7,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Orchestration; using RepoUtils; @@ -20,10 +22,14 @@ * - You are not using OpenAI or Azure OpenAI models * - You are using OpenAI/Azure OpenAI models but the models are behind a web service with a different API schema * - You want to use a local model + * + * Note that all text completion models are deprecated by OpenAI and will be removed in a future release. + * + * Refer to example 33 for streaming chat completion. */ public class MyTextCompletionService : ITextCompletion { - public Task> GetCompletionsAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default) + public Task> GetCompletionsAsync(string text, AIRequestSettings? requestSettings, CancellationToken cancellationToken = default) { return Task.FromResult>(new List { @@ -31,13 +37,13 @@ public Task> GetCompletionsAsync(string text, Complet }); } - public async IAsyncEnumerable GetStreamingCompletionsAsync(string text, CompleteRequestSettings requestSettings, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GetStreamingCompletionsAsync(string text, AIRequestSettings? requestSettings, [EnumeratorCancellation] CancellationToken cancellationToken = default) { yield return new MyTextCompletionStreamingResult(); } } -public class MyTextCompletionStreamingResult : ITextStreamingResult +public class MyTextCompletionStreamingResult : ITextStreamingResult, ITextResult { private readonly ModelResult _modelResult = new(new { @@ -95,23 +101,23 @@ private static async Task CustomTextCompletionWithSKFunctionAsync() Console.WriteLine("======== Custom LLM - Text Completion - SKFunction ========"); IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) + .WithLoggerFactory(ConsoleLogger.LoggerFactory) // Add your text completion service as a singleton instance .WithAIService("myService1", new MyTextCompletionService()) // Add your text completion service as a factory method - .WithAIService("myService2", (_) => new MyTextCompletionService()) + .WithAIService("myService2", (log) => new MyTextCompletionService()) .Build(); const string FunctionDefinition = "Does the text contain grammar errors (Y/N)? Text: {{$input}}"; var textValidationFunction = kernel.CreateSemanticFunction(FunctionDefinition); - var result = await textValidationFunction.InvokeAsync("I mised the training session this morning"); - Console.WriteLine(result); + var result = await textValidationFunction.InvokeAsync("I mised the training session this morning", kernel); + Console.WriteLine(result.GetValue()); // Details of the my custom model response Console.WriteLine(JsonSerializer.Serialize( - result.ModelResults, + result.GetModelResults(), new JsonSerializerOptions() { WriteIndented = true } )); } @@ -121,7 +127,7 @@ private static async Task CustomTextCompletionAsync() Console.WriteLine("======== Custom LLM - Text Completion - Raw ========"); var completionService = new MyTextCompletionService(); - var result = await completionService.CompleteAsync("I missed the training session this morning", new CompleteRequestSettings()); + var result = await completionService.CompleteAsync("I missed the training session this morning"); Console.WriteLine(result); } @@ -130,7 +136,7 @@ private static async Task CustomTextCompletionStreamAsync() { Console.WriteLine("======== Custom LLM - Text Completion - Raw Streaming ========"); - IKernel kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); + IKernel kernel = new KernelBuilder().WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); ITextCompletion textCompletion = new MyTextCompletionService(); var prompt = "Write one paragraph why AI is awesome"; @@ -139,7 +145,7 @@ private static async Task CustomTextCompletionStreamAsync() private static async Task TextCompletionStreamAsync(string prompt, ITextCompletion textCompletion) { - var requestSettings = new CompleteRequestSettings() + var requestSettings = new OpenAIRequestSettings() { MaxTokens = 100, FrequencyPenalty = 0, diff --git a/dotnet/samples/KernelSyntaxExamples/Example17_ChatGPT.cs b/dotnet/samples/KernelSyntaxExamples/Example17_ChatGPT.cs index f76e2db32717..b583f9f85809 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example17_ChatGPT.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example17_ChatGPT.cs @@ -50,7 +50,7 @@ private static async Task OpenAIChatSampleAsync() { Console.WriteLine("======== Open AI - ChatGPT ========"); - OpenAIChatCompletion openAIChatCompletion = new("gpt-3.5-turbo", TestConfiguration.OpenAI.ApiKey); + OpenAIChatCompletion openAIChatCompletion = new(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey); await StartChatAsync(openAIChatCompletion); } diff --git a/dotnet/samples/KernelSyntaxExamples/Example18_DallE.cs b/dotnet/samples/KernelSyntaxExamples/Example18_DallE.cs index 05b5049b2a83..26bf8c1196f2 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example18_DallE.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example18_DallE.cs @@ -26,11 +26,11 @@ private static async Task OpenAIDallEAsync() Console.WriteLine("======== OpenAI Dall-E 2 Image Generation ========"); IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) + .WithLoggerFactory(ConsoleLogger.LoggerFactory) // Add your image generation service .WithOpenAIImageGenerationService(TestConfiguration.OpenAI.ApiKey) // Add your chat completion service - .WithOpenAIChatCompletionService("gpt-3.5-turbo", TestConfiguration.OpenAI.ApiKey) + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey) .Build(); IImageGeneration dallE = kernel.GetService(); @@ -95,11 +95,11 @@ public static async Task AzureOpenAIDallEAsync() Console.WriteLine("========Azure OpenAI Dall-E 2 Image Generation ========"); IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) + .WithLoggerFactory(ConsoleLogger.LoggerFactory) // Add your image generation service .WithAzureOpenAIImageGenerationService(TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) // Add your chat completion service - .WithAzureChatCompletionService("gpt-35-turbo", TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) + .WithAzureChatCompletionService(TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) .Build(); IImageGeneration dallE = kernel.GetService(); diff --git a/dotnet/samples/KernelSyntaxExamples/Example19_Qdrant.cs b/dotnet/samples/KernelSyntaxExamples/Example19_Qdrant.cs deleted file mode 100644 index dee3f8c3560b..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example19_Qdrant.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; -using Microsoft.SemanticKernel.Memory; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -public static class Example19_Qdrant -{ - private const string MemoryCollectionName = "qdrant-test"; - - public static async Task RunAsync() - { - QdrantMemoryStore memoryStore = new(TestConfiguration.Qdrant.Endpoint, 1536, ConsoleLogger.Logger); - IKernel kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService("text-davinci-003", TestConfiguration.OpenAI.ApiKey) - .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(memoryStore) - //.WithQdrantMemoryStore(TestConfiguration.Qdrant.Endpoint, 1536) // This method offers an alternative approach to registering Qdrant memory store. - .Build(); - - Console.WriteLine("== Printing Collections in DB =="); - var collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Adding Memories =="); - - var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat1", text: "british short hair"); - var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat2", text: "orange tabby"); - var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat3", text: "norwegian forest cat"); - - Console.WriteLine("== Printing Collections in DB =="); - collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Retrieving Memories Through the Kernel =="); - MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1"); - Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Retrieving Memories Directly From the Store =="); - var memory1 = await memoryStore.GetWithPointIdAsync(MemoryCollectionName, key1); - var memory2 = await memoryStore.GetWithPointIdAsync(MemoryCollectionName, key2); - var memory3 = await memoryStore.GetWithPointIdAsync(MemoryCollectionName, key3); - Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found"); - Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found"); - Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Similarity Searching Memories: My favorite color is orange =="); - var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8); - - await foreach (var item in searchResults) - { - Console.WriteLine(item.Metadata.Text + " : " + item.Relevance); - } - - Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName); - await memoryStore.DeleteCollectionAsync(MemoryCollectionName); - - Console.WriteLine("== Printing Collections in DB =="); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs b/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs index 118bce59520f..124f68c3f639 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs @@ -8,32 +8,70 @@ /** * The following example shows how to use Semantic Kernel with HuggingFace API. */ - // ReSharper disable once InconsistentNaming public static class Example20_HuggingFace { public static async Task RunAsync() { - Console.WriteLine("======== HuggingFace text completion AI ========"); + await RunInferenceApiExampleAsync(); + await RunLlamaExampleAsync(); + } + + /// + /// This example uses HuggingFace Inference API to access hosted models. + /// More information here: + /// + private static async Task RunInferenceApiExampleAsync() + { + Console.WriteLine("\n======== HuggingFace Inference API example ========\n"); IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) + .WithLoggerFactory(ConsoleLogger.LoggerFactory) .WithHuggingFaceTextCompletionService( - model: TestConfiguration.HuggingFace.ApiKey, + model: TestConfiguration.HuggingFace.ModelId, apiKey: TestConfiguration.HuggingFace.ApiKey) .Build(); - const string FunctionDefinition = "Question: {{$input}}; Answer:"; + var questionAnswerFunction = kernel.CreateSemanticFunction("Question: {{$input}}; Answer:"); - var questionAnswerFunction = kernel.CreateSemanticFunction(FunctionDefinition); + var result = await kernel.RunAsync("What is New York?", questionAnswerFunction); + + Console.WriteLine(result.GetValue()); + } + + /// + /// This example uses HuggingFace Llama 2 model and local HTTP server from Semantic Kernel repository. + /// How to setup local HTTP server: . + /// + /// Additional access is required to download Llama 2 model and run it locally. + /// How to get access: + /// 1. Visit and complete request access form. + /// 2. Visit and complete form "Access Llama 2 on Hugging Face". + /// Note: Your Hugging Face account email address MUST match the email you provide on the Meta website, or your request will not be approved. + /// + /// + private static async Task RunLlamaExampleAsync() + { + Console.WriteLine("\n======== HuggingFace Llama 2 example ========\n"); + + // HuggingFace Llama 2 model: https://huggingface.co/meta-llama/Llama-2-7b-hf + const string Model = "meta-llama/Llama-2-7b-hf"; + + // HuggingFace local HTTP server endpoint + const string Endpoint = "http://localhost:5000/completions"; + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithHuggingFaceTextCompletionService( + model: Model, + endpoint: Endpoint, + apiKey: TestConfiguration.HuggingFace.ApiKey) + .Build(); - var result = await questionAnswerFunction.InvokeAsync("What is New York?"); + var questionAnswerFunction = kernel.CreateSemanticFunction("Question: {{$input}}; Answer:"); - Console.WriteLine(result); + var result = await kernel.RunAsync("What is New York?", questionAnswerFunction); - foreach (var modelResult in result.ModelResults) - { - Console.WriteLine(modelResult.GetHuggingFaceResult().AsJson()); - } + Console.WriteLine(result.GetValue()); } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example21_ChatGPTPlugins.cs b/dotnet/samples/KernelSyntaxExamples/Example21_ChatGPTPlugins.cs index 651025fad394..e0441d62e7b8 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example21_ChatGPTPlugins.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example21_ChatGPTPlugins.cs @@ -4,8 +4,9 @@ using System.Net.Http; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Skills.OpenAPI.Extensions; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -18,31 +19,31 @@ public static async Task RunAsync() private static async Task RunChatGptPluginAsync() { - var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); + var kernel = new KernelBuilder().WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); + + //This HTTP client is optional. SK will fallback to a default internal one if omitted. using HttpClient httpClient = new(); - //Import a ChatGPT plugin using one of the following Kernel extension methods - //kernel.ImportChatGptPluginSkillFromResourceAsync - //kernel.ImportChatGptPluginSkillSkillFromDirectory - //kernel.ImportChatGptPluginSkillSkillFromFile - //kernel.ImportChatGptPluginSkillFromUrlAsync - var skill = await kernel.ImportChatGptPluginSkillFromUrlAsync("", new Uri(""), new OpenApiSkillExecutionParameters(httpClient)); + //Import a ChatGPT plugin via URI + var plugin = await kernel.ImportPluginFunctionsAsync("", new Uri(""), new OpenApiFunctionExecutionParameters(httpClient)); //Add arguments for required parameters, arguments for optional ones can be skipped. var contextVariables = new ContextVariables(); contextVariables.Set("", ""); //Run - var result = await kernel.RunAsync(contextVariables, skill["productsUsingGET"]); + var kernelResult = await kernel.RunAsync(contextVariables, plugin[""]); + + var result = kernelResult.GetValue(); - Console.WriteLine("Skill execution result: {0}", result); + Console.WriteLine("Function execution result: {0}", result?.Content?.ToString()); Console.ReadLine(); //--------------- Example of using Klarna ChatGPT plugin ------------------------ - //var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); + //var kernel = new KernelBuilder().WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); - //var skill = await kernel.ImportChatGptPluginSkillFromUrlAsync("Klarna", new Uri("https://www.klarna.com/.well-known/ai-plugin.json"), new OpenApiSkillExecutionParameters(httpClient)); + //var plugin = await kernel.ImportPluginFunctionsAsync("Klarna", new Uri("https://www.klarna.com/.well-known/ai-plugin.json")); //var contextVariables = new ContextVariables(); //contextVariables.Set("q", "Laptop"); // A precise query that matches one very small category or product that needs to be searched for to find the products the user is looking for. If the user explicitly stated what they want, use that as a query. The query is as specific as possible to the product name or category mentioned by the user in its singular form, and don't contain any clarifiers like latest, newest, cheapest, budget, premium, expensive or similar. The query is always taken from the latest topic, if there is a new topic a new query is started. @@ -50,9 +51,11 @@ private static async Task RunChatGptPluginAsync() //contextVariables.Set("budget", "200"); // maximum price of the matching product in local currency, filters results //contextVariables.Set("countryCode", "US");// ISO 3166 country code with 2 characters based on the user location. Currently, only US, GB, DE, SE and DK are supported. - //var result = await kernel.RunAsync(contextVariables, skill["productsUsingGET"]); + //var kernelResult = await kernel.RunAsync(contextVariables, plugin["productsUsingGET"]); + + //var result = kernelResult.GetValue(); - //Console.WriteLine("Klarna skill response: {0}", result); + //Console.WriteLine("Function execution result: {0}", result?.Content?.ToString()); //Console.ReadLine(); } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example22_OpenApiPlugin_AzureKeyVault.cs b/dotnet/samples/KernelSyntaxExamples/Example22_OpenApiPlugin_AzureKeyVault.cs new file mode 100644 index 000000000000..b92be76a5239 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example22_OpenApiPlugin_AzureKeyVault.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Functions.OpenAPI.Plugins; +using Microsoft.SemanticKernel.Orchestration; +using RepoUtils; + +#pragma warning disable CA1861 // Avoid constant arrays as arguments +// ReSharper disable once InconsistentNaming +public static class Example22_OpenApiPlugin_AzureKeyVault +{ + public static async Task RunAsync() + { + // To run this example, you must register a client application with the Microsoft identity platform. + // Instructions here: https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app + var authenticationProvider = new InteractiveMsalAuthenticationProvider( + TestConfiguration.KeyVault.ClientId, + TestConfiguration.KeyVault.TenantId, + new[] { "https://vault.azure.net/.default" }, + new Uri("http://localhost")); + + await GetSecretFromAzureKeyVaultWithRetryAsync(authenticationProvider); + + await AddSecretToAzureKeyVaultAsync(authenticationProvider); + } + + public static async Task GetSecretFromAzureKeyVaultWithRetryAsync(InteractiveMsalAuthenticationProvider authenticationProvider) + { + var kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithRetryBasic(new() + { + MaxRetryCount = 3, + UseExponentialBackoff = true + }) + .Build(); + + var type = typeof(PluginResourceNames); + var resourceName = $"{PluginResourceNames.AzureKeyVault}.openapi.json"; + + var stream = type.Assembly.GetManifestResourceStream(type, resourceName); + + // Import AI Plugin + var plugin = await kernel.ImportPluginFunctionsAsync( + PluginResourceNames.AzureKeyVault, + stream!, + new OpenApiFunctionExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync }); + + // Add arguments for required parameters, arguments for optional ones can be skipped. + var contextVariables = new ContextVariables(); + contextVariables.Set("server-url", TestConfiguration.KeyVault.Endpoint); + contextVariables.Set("secret-name", ""); + contextVariables.Set("api-version", "7.0"); + + // Run + var kernelResult = await kernel.RunAsync(contextVariables, plugin["GetSecret"]); + + var result = kernelResult.GetValue(); + + Console.WriteLine("GetSecret function result: {0}", result?.Content?.ToString()); + } + + public static async Task AddSecretToAzureKeyVaultAsync(InteractiveMsalAuthenticationProvider authenticationProvider) + { + var kernel = new KernelBuilder().WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); + + var type = typeof(PluginResourceNames); + var resourceName = $"{PluginResourceNames.AzureKeyVault}.openapi.json"; + + var stream = type.Assembly.GetManifestResourceStream(type, resourceName); + + // Import AI Plugin + var plugin = await kernel.ImportPluginFunctionsAsync( + PluginResourceNames.AzureKeyVault, + stream!, + new OpenApiFunctionExecutionParameters + { + AuthCallback = authenticationProvider.AuthenticateRequestAsync, + EnableDynamicPayload = true + }); + + // Add arguments for required parameters, arguments for optional ones can be skipped. + var contextVariables = new ContextVariables(); + contextVariables.Set("server-url", TestConfiguration.KeyVault.Endpoint); + contextVariables.Set("secret-name", ""); + contextVariables.Set("api-version", "7.0"); + contextVariables.Set("value", ""); + contextVariables.Set("enabled", ""); + + // Run + var kernelResult = await kernel.RunAsync(contextVariables, plugin["SetSecret"]); + + var result = kernelResult.GetValue(); + + Console.WriteLine("SetSecret function result: {0}", result?.Content?.ToString()); + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example22_OpenApiSkill_AzureKeyVault.cs b/dotnet/samples/KernelSyntaxExamples/Example22_OpenApiSkill_AzureKeyVault.cs deleted file mode 100644 index b009276b2425..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example22_OpenApiSkill_AzureKeyVault.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Reliability; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; -using Microsoft.SemanticKernel.Skills.OpenAPI.Extensions; -using Microsoft.SemanticKernel.Skills.OpenAPI.Skills; -using RepoUtils; - -#pragma warning disable CA1861 // Avoid constant arrays as arguments -// ReSharper disable once InconsistentNaming -public static class Example22_OpenApiSkill_AzureKeyVault -{ - public static async Task RunAsync() - { - // To run this example, you must register a client application with the Microsoft identity platform. - // Instructions here: https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app - var authenticationProvider = new InteractiveMsalAuthenticationProvider( - TestConfiguration.KeyVault.ClientId, - TestConfiguration.KeyVault.TenantId, - new[] { "https://vault.azure.net/.default" }, - new Uri("http://localhost")); - - await GetSecretFromAzureKeyVaultWithRetryAsync(authenticationProvider); - - await AddSecretToAzureKeyVaultAsync(authenticationProvider); - } - - public static async Task GetSecretFromAzureKeyVaultWithRetryAsync(InteractiveMsalAuthenticationProvider authenticationProvider) - { - var retryConfig = new HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }; - - var kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .Configure(c => c.SetDefaultHttpRetryConfig(retryConfig)) - .Build(); - - // Import a OpenApi skill using one of the following Kernel extension methods - // kernel.ImportOpenApiSkillFromResource - // kernel.ImportOpenApiSkillFromDirectory - // kernel.ImportOpenApiSkillFromFile - // kernel.ImportOpenApiSkillFromUrlAsync - // kernel.RegisterOpenApiSkill - var skill = await kernel.ImportOpenApiSkillFromResourceAsync(SkillResourceNames.AzureKeyVault, new OpenApiSkillExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync }); - - // Add arguments for required parameters, arguments for optional ones can be skipped. - var contextVariables = new ContextVariables(); - contextVariables.Set("server-url", TestConfiguration.KeyVault.Endpoint); - contextVariables.Set("secret-name", ""); - contextVariables.Set("api-version", "7.0"); - - // Run - var result = await kernel.RunAsync(contextVariables, skill["GetSecret"]); - - Console.WriteLine("GetSecret skill response: {0}", result); - } - - public static async Task AddSecretToAzureKeyVaultAsync(InteractiveMsalAuthenticationProvider authenticationProvider) - { - var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); - - // Import a OpenApi skill using one of the following Kernel extension methods - // kernel.ImportOpenApiSkillFromResource - // kernel.ImportOpenApiSkillFromDirectory - // kernel.ImportOpenApiSkillFromFile - // kernel.ImportOpenApiSkillFromUrlAsync - // kernel.RegisterOpenApiSkill - var skill = await kernel.ImportOpenApiSkillFromResourceAsync(SkillResourceNames.AzureKeyVault, new OpenApiSkillExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync }); - - // Add arguments for required parameters, arguments for optional ones can be skipped. - var contextVariables = new ContextVariables(); - contextVariables.Set("server-url", TestConfiguration.KeyVault.Endpoint); - contextVariables.Set("secret-name", ""); - contextVariables.Set("api-version", "7.0"); - contextVariables.Set("payload", JsonSerializer.Serialize(new { value = "", attributes = new { enabled = true } })); - - // Run - var result = await kernel.RunAsync(contextVariables, skill["SetSecret"]); - - Console.WriteLine("SetSecret skill response: {0}", result); - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example23_OpenApiPlugin_Github.cs b/dotnet/samples/KernelSyntaxExamples/Example23_OpenApiPlugin_Github.cs new file mode 100644 index 000000000000..f7872ce320a5 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example23_OpenApiPlugin_Github.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Orchestration; +using Newtonsoft.Json.Linq; +using RepoUtils; + +/// +/// Import and run GitHub Functions using OpenAPI Plugin. +/// To use this example, run: +/// dotnet user-secrets set "Github.PAT" "github_pat_..." +/// Make sure your GitHub PAT has read permissions set for Pull Requests. +/// Creating a PAT: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token +/// +// ReSharper disable once InconsistentNaming +public static class Example23_OpenApiPlugin_GitHub +{ + public static async Task RunAsync() + { + var authenticationProvider = new BearerAuthenticationProvider(() => { return Task.FromResult(TestConfiguration.Github.PAT); }); + Console.WriteLine("== Example23_OpenApiPlugin_GitHub =="); + var firstPRNumber = await ListPullRequestsFromGitHubAsync(authenticationProvider); + await GetPullRequestFromGitHubAsync(authenticationProvider, firstPRNumber); + } + + public static async Task ListPullRequestsFromGitHubAsync(BearerAuthenticationProvider authenticationProvider) + { + var kernel = new KernelBuilder().WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); + + var plugin = await kernel.ImportPluginFunctionsAsync( + "GitHubPlugin", + "../../../../../../samples/dotnet/OpenApiPluginsExample/GitHubPlugin/openapi.json", + new OpenApiFunctionExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync }); + + // Add arguments for required parameters, arguments for optional ones can be skipped. + var contextVariables = new ContextVariables(); + contextVariables.Set("owner", "microsoft"); + contextVariables.Set("repo", "semantic-kernel"); + + // Run + var kernelResult = await kernel.RunAsync(contextVariables, plugin["PullList"]); + + Console.WriteLine("Successful GitHub List Pull Requests plugin response."); + var response = kernelResult.GetValue(); + if (response != null) + { + var pullRequests = JArray.Parse(response.Content?.ToString() ?? "null"); + + if (pullRequests != null && pullRequests.First != null) + { + var number = pullRequests.First["number"]; + return number?.ToString() ?? string.Empty; + } + } + Console.WriteLine("No pull requests found."); + + return string.Empty; + } + + public static async Task GetPullRequestFromGitHubAsync(BearerAuthenticationProvider authenticationProvider, string pullNumber) + { + var kernel = new KernelBuilder().WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); + + var plugin = await kernel.ImportPluginFunctionsAsync( + "GitHubPlugin", + "../../../../../../samples/dotnet/OpenApiPluginsExample/GitHubPlugin/openapi.json", + new OpenApiFunctionExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync }); + + // Add arguments for required parameters, arguments for optional ones can be skipped. + var contextVariables = new ContextVariables(); + contextVariables.Set("owner", "microsoft"); + contextVariables.Set("repo", "semantic-kernel"); + contextVariables.Set("pull_number", pullNumber); + + // Run + var kernelResult = await kernel.RunAsync(contextVariables, plugin["PullsGet"]); + + Console.WriteLine("Successful GitHub Get Pull Request plugin response: {0}", kernelResult.GetValue()?.Content); + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example23_OpenApiSkill_Github.cs b/dotnet/samples/KernelSyntaxExamples/Example23_OpenApiSkill_Github.cs deleted file mode 100644 index 7de220bf393a..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example23_OpenApiSkill_Github.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; -using Microsoft.SemanticKernel.Skills.OpenAPI.Extensions; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using RepoUtils; - -/// -/// Import and run GitHub Functions using OpenAPI Skill. -/// To use this example, run: -/// dotnet user-secrets set "Github.PAT" "github_pat_..." -/// Make sure your GitHub PAT has read permissions set for Pull Requests. -/// Creating a PAT: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token -/// -// ReSharper disable once InconsistentNaming -public static class Example23_OpenApiSkill_GitHub -{ - public static async Task RunAsync() - { - var authenticationProvider = new BearerAuthenticationProvider(() => { return Task.FromResult(TestConfiguration.Github.PAT); }); - Console.WriteLine("== Example22_c_OpenApiSkill_GitHub =="); - var firstPRNumber = await ListPullRequestsFromGitHubAsync(authenticationProvider); - await GetPullRequestFromGitHubAsync(authenticationProvider, firstPRNumber); - } - - public static async Task ListPullRequestsFromGitHubAsync(BearerAuthenticationProvider authenticationProvider) - { - var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); - - var skill = await kernel.ImportOpenApiSkillFromFileAsync( - "GitHubSkill", - "../../../samples/apps/copilot-chat-app/webapi/Skills/OpenApiSkills/GitHubSkill/openapi.json", - new OpenApiSkillExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync }); - - // Add arguments for required parameters, arguments for optional ones can be skipped. - var contextVariables = new ContextVariables(); - contextVariables.Set("owner", "microsoft"); - contextVariables.Set("repo", "semantic-kernel"); - - // Run - var result = await kernel.RunAsync(contextVariables, skill["PullsList"]); - - Console.WriteLine("Successful GitHub List Pull Requests skill response."); - var resultJson = JsonConvert.DeserializeObject>(result.Result); - var pullRequests = JArray.Parse((string)resultJson!["content"]); - - if (pullRequests != null && pullRequests.First != null) - { - var number = pullRequests.First["number"]; - return number?.ToString() ?? string.Empty; - } - - Console.WriteLine("No pull requests found."); - - return string.Empty; - } - - public static async Task GetPullRequestFromGitHubAsync(BearerAuthenticationProvider authenticationProvider, string pullNumber) - { - var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); - - var skill = await kernel.ImportOpenApiSkillFromFileAsync( - "GitHubSkill", - "../../../samples/apps/copilot-chat-app/webapi/Skills/OpenApiSkills/GitHubSkill/openapi.json", - new OpenApiSkillExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync }); - - // Add arguments for required parameters, arguments for optional ones can be skipped. - var contextVariables = new ContextVariables(); - contextVariables.Set("owner", "microsoft"); - contextVariables.Set("repo", "semantic-kernel"); - contextVariables.Set("pull_number", pullNumber); - - // Run - var result = await kernel.RunAsync(contextVariables, skill["PullsGet"]); - - Console.WriteLine("Successful GitHub Get Pull Request skill response: {0}", result); - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example24_OpenApiPlugin_Jira.cs b/dotnet/samples/KernelSyntaxExamples/Example24_OpenApiPlugin_Jira.cs new file mode 100644 index 000000000000..41c4188e2146 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example24_OpenApiPlugin_Jira.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; +using Microsoft.SemanticKernel.Orchestration; + +using Newtonsoft.Json; +using RepoUtils; + +/// +/// This sample shows how to connect the Semantic Kernel to Jira as an Open Api plugin based on the Open Api schema. +/// This format of registering the plugin and its operations, and subsequently executing those operations can be applied +/// to an Open Api plugin that follows the Open Api Schema. +/// +// ReSharper disable once InconsistentNaming +public static class Example24_OpenApiPlugin_Jira +{ + public static async Task RunAsync() + { + var kernel = new KernelBuilder().WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); + var contextVariables = new ContextVariables(); + + // Change to a jira instance you have access to with your authentication credentials + string serverUrl = $"https://{TestConfiguration.Jira.Domain}.atlassian.net/rest/api/latest/"; + contextVariables.Set("server-url", serverUrl); + + IDictionary jiraFunctions; + var tokenProvider = new BasicAuthenticationProvider(() => + { + string s = $"{TestConfiguration.Jira.Email}:{TestConfiguration.Jira.ApiKey}"; + return Task.FromResult(s); + }); + + using HttpClient httpClient = new(); + + // The bool useLocalFile can be used to toggle the ingestion method for the openapi schema between a file path and a URL + bool useLocalFile = true; + if (useLocalFile) + { + var apiPluginFile = "./../../../Plugins/JiraPlugin/openapi.json"; + jiraFunctions = await kernel.ImportPluginFunctionsAsync("jiraPlugin", apiPluginFile, new OpenApiFunctionExecutionParameters(authCallback: tokenProvider.AuthenticateRequestAsync)); + } + else + { + var apiPluginRawFileURL = new Uri("https://raw.githubusercontent.com/microsoft/PowerPlatformConnectors/dev/certified-connectors/JIRA/apiDefinition.swagger.json"); + jiraFunctions = await kernel.ImportPluginFunctionsAsync("jiraPlugin", apiPluginRawFileURL, new OpenApiFunctionExecutionParameters(httpClient, tokenProvider.AuthenticateRequestAsync)); + } + + // GetIssue Function + { + // Set Properties for the Get Issue operation in the openAPI.swagger.json + contextVariables.Set("issueKey", "SKTES-2"); + + // Run operation via the semantic kernel + var result = await kernel.RunAsync(contextVariables, jiraFunctions["GetIssue"]); + + Console.WriteLine("\n\n\n"); + var formattedContent = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(result.GetValue()!), Formatting.Indented); + Console.WriteLine("GetIssue jiraPlugin response: \n{0}", formattedContent); + } + + // AddComment Function + { + // Set Properties for the AddComment operation in the openAPI.swagger.json + contextVariables.Set("issueKey", "SKTES-1"); + contextVariables.Set("body", "Here is a rad comment"); + + // Run operation via the semantic kernel + var result = await kernel.RunAsync(contextVariables, jiraFunctions["AddComment"]); + + Console.WriteLine("\n\n\n"); + var formattedContent = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(result.GetValue()!), Formatting.Indented); + Console.WriteLine("AddComment jiraPlugin response: \n{0}", formattedContent); + } + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example24_OpenApiSkill_Jira.cs b/dotnet/samples/KernelSyntaxExamples/Example24_OpenApiSkill_Jira.cs deleted file mode 100644 index d70fb1253fb7..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example24_OpenApiSkill_Jira.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; -using Microsoft.SemanticKernel.Skills.OpenAPI.Extensions; -using Newtonsoft.Json; -using RepoUtils; - -/// -/// This sample shows how to connect the Semantic Kernel to Jira as an Open Api plugin based on the Open Api schema. -/// This format of registering the skill and its operations, and subsequently executing those operations can be applied -/// to an Open Api plugin that follows the Open Api Schema. -/// -// ReSharper disable once InconsistentNaming -public static class Example24_OpenApiSkill_Jira -{ - public static async Task RunAsync() - { - var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); - var contextVariables = new ContextVariables(); - - // Change to a jira instance you have access to with your authentication credentials - string serverUrl = $"https://{TestConfiguration.Jira.Domain}.atlassian.net/rest/api/latest/"; - contextVariables.Set("server-url", serverUrl); - - IDictionary jiraSkills; - var tokenProvider = new BasicAuthenticationProvider(() => - { - string s = $"{TestConfiguration.Jira.Email}:{TestConfiguration.Jira.ApiKey}"; - return Task.FromResult(s); - }); - - using HttpClient httpClient = new(); - - // The bool useLocalFile can be used to toggle the ingestion method for the openapi schema between a file path and a URL - bool useLocalFile = true; - if (useLocalFile) - { - var apiSkillFile = "./../../../Skills/JiraSkill/openapi.json"; - jiraSkills = await kernel.ImportOpenApiSkillFromFileAsync("jiraSkills", apiSkillFile, new OpenApiSkillExecutionParameters(authCallback: tokenProvider.AuthenticateRequestAsync)); - } - else - { - var apiSkillRawFileURL = new Uri("https://raw.githubusercontent.com/microsoft/PowerPlatformConnectors/dev/certified-connectors/JIRA/apiDefinition.swagger.json"); - jiraSkills = await kernel.ImportOpenApiSkillFromUrlAsync("jiraSkills", apiSkillRawFileURL, new OpenApiSkillExecutionParameters(httpClient, tokenProvider.AuthenticateRequestAsync)); - } - - // GetIssue Skill - { - // Set Properties for the Get Issue operation in the openAPI.swagger.json - contextVariables.Set("issueKey", "SKTES-2"); - - // Run operation via the semantic kernel - var result = await kernel.RunAsync(contextVariables, jiraSkills["GetIssue"]); - - Console.WriteLine("\n\n\n"); - var formattedContent = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(result.Result), Formatting.Indented); - Console.WriteLine("GetIssue jiraSkills response: \n{0}", formattedContent); - } - - // AddComment Skill - { - // Set Properties for the AddComment operation in the openAPI.swagger.json - contextVariables.Set("issueKey", "SKTES-1"); - contextVariables.Set("body", "Here is a rad comment"); - - // Run operation via the semantic kernel - var result = await kernel.RunAsync(contextVariables, jiraSkills["AddComment"]); - - Console.WriteLine("\n\n\n"); - var formattedContent = JsonConvert.SerializeObject(JsonConvert.DeserializeObject(result.Result), Formatting.Indented); - Console.WriteLine("AddComment jiraSkills response: \n{0}", formattedContent); - } - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example25_ReadOnlyMemoryStore.cs b/dotnet/samples/KernelSyntaxExamples/Example25_ReadOnlyMemoryStore.cs index b27aa9b91e00..b6304e8725c9 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example25_ReadOnlyMemoryStore.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example25_ReadOnlyMemoryStore.cs @@ -3,14 +3,13 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Memory.Collections; #pragma warning disable CA2201 // System.Exception is not sufficiently specific - this is a sample #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously @@ -30,20 +29,20 @@ public static async Task RunAsync() { var store = new ReadOnlyMemoryStore(s_jsonVectorEntries); - var embedding = new Embedding(new float[] { 22, 4, 6 }); + var embedding = new ReadOnlyMemory(new float[] { 22, 4, 6 }); Console.WriteLine("Reading data from custom read-only memory store"); var memoryRecord = await store.GetAsync("collection", "key3"); if (memoryRecord != null) { - Console.WriteLine("ID = {0}, Embedding = {1}", memoryRecord.Metadata.Id, string.Join(", ", memoryRecord.Embedding.Vector)); + Console.WriteLine("ID = {0}, Embedding = {1}", memoryRecord.Metadata.Id, string.Join(", ", MemoryMarshal.ToEnumerable(memoryRecord.Embedding))); } - Console.WriteLine("Getting most similar vector to {0}", string.Join(", ", embedding.Vector)); + Console.WriteLine("Getting most similar vector to {0}", string.Join(", ", MemoryMarshal.ToEnumerable(embedding))); var result = await store.GetNearestMatchAsync("collection", embedding, 0.0); if (result.HasValue) { - Console.WriteLine("Embedding = {0}, Similarity = {1}", string.Join(", ", result.Value.Item1.Embedding.Vector), result.Value.Item2); + Console.WriteLine("Embedding = {0}, Similarity = {1}", string.Join(", ", MemoryMarshal.ToEnumerable(result.Value.Item1.Embedding)), result.Value.Item2); } } @@ -105,7 +104,7 @@ public IAsyncEnumerable GetCollectionsAsync(CancellationToken cancellati throw new System.NotImplementedException(); } - public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding embedding, double minRelevanceScore = 0, + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) { // Note: with this simple implementation, the MemoryRecord will always contain the embedding. @@ -123,7 +122,7 @@ public IAsyncEnumerable GetCollectionsAsync(CancellationToken cancellati return default; } - public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(string collectionName, Embedding embedding, int limit, + public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(string collectionName, ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) { // Note: with this simple implementation, the MemoryRecord will always contain the embedding. @@ -132,27 +131,25 @@ public IAsyncEnumerable GetCollectionsAsync(CancellationToken cancellati yield break; } - if (embedding.Count != this._vectorSize) + if (embedding.Length != this._vectorSize) { - throw new Exception($"Embedding vector size {embedding.Count} does not match expected size of {this._vectorSize}"); + throw new Exception($"Embedding vector size {embedding.Length} does not match expected size of {this._vectorSize}"); } - TopNCollection embeddings = new(limit); + List<(MemoryRecord Record, double Score)> embeddings = new(); foreach (var item in this._memoryRecords) { - double similarity = embedding.AsReadOnlySpan().CosineSimilarity(item.Embedding.AsReadOnlySpan()); + double similarity = TensorPrimitives.CosineSimilarity(embedding.Span, item.Embedding.Span); if (similarity >= minRelevanceScore) { embeddings.Add(new(item, similarity)); } } - embeddings.SortByScore(); - - foreach (var item in embeddings) + foreach (var item in embeddings.OrderByDescending(l => l.Score).Take(limit)) { - yield return (item.Value, item.Score.Value); + yield return (item.Record, item.Score); } } @@ -179,9 +176,7 @@ public IAsyncEnumerable UpsertBatchAsync(string collectionName, IEnumera private static string s_jsonVectorEntries = @"[ { - ""embedding"": { - ""vector"": [0, 0, 0 ] - }, + ""embedding"": [0, 0, 0], ""metadata"": { ""is_reference"": false, ""external_source_name"": ""externalSourceName"", @@ -194,9 +189,7 @@ public IAsyncEnumerable UpsertBatchAsync(string collectionName, IEnumera ""timestamp"": null }, { - ""embedding"": { - ""vector"": [0, 0, 10 ] - }, + ""embedding"": [0, 0, 10], ""metadata"": { ""is_reference"": false, ""external_source_name"": ""externalSourceName"", @@ -209,9 +202,7 @@ public IAsyncEnumerable UpsertBatchAsync(string collectionName, IEnumera ""timestamp"": null }, { - ""embedding"": { - ""vector"": [1, 2, 3 ] - }, + ""embedding"": [1, 2, 3], ""metadata"": { ""is_reference"": false, ""external_source_name"": ""externalSourceName"", @@ -224,9 +215,7 @@ public IAsyncEnumerable UpsertBatchAsync(string collectionName, IEnumera ""timestamp"": null }, { - ""embedding"": { - ""vector"": [-1, -2, -3 ] - }, + ""embedding"": [-1, -2, -3], ""metadata"": { ""is_reference"": false, ""external_source_name"": ""externalSourceName"", @@ -239,9 +228,7 @@ public IAsyncEnumerable UpsertBatchAsync(string collectionName, IEnumera ""timestamp"": null }, { - ""embedding"": { - ""vector"": [12, 8, 4 ] - }, + ""embedding"": [12, 8, 4], ""metadata"": { ""is_reference"": false, ""external_source_name"": ""externalSourceName"", diff --git a/dotnet/samples/KernelSyntaxExamples/Example26_AADAuth.cs b/dotnet/samples/KernelSyntaxExamples/Example26_AADAuth.cs index 5f91059f0be6..3bd2c22d440c 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example26_AADAuth.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example26_AADAuth.cs @@ -30,21 +30,24 @@ public static async Task RunAsync() // Optional: choose which authentication to support var authOptions = new DefaultAzureCredentialOptions { - ExcludeEnvironmentCredential = false, - ExcludeManagedIdentityCredential = false, - ExcludeSharedTokenCacheCredential = false, + ExcludeEnvironmentCredential = true, + ExcludeManagedIdentityCredential = true, + ExcludeSharedTokenCacheCredential = true, ExcludeAzureCliCredential = true, ExcludeVisualStudioCredential = true, ExcludeVisualStudioCodeCredential = true, - ExcludeInteractiveBrowserCredential = true, + ExcludeInteractiveBrowserCredential = false, + ExcludeAzureDeveloperCliCredential = true, + ExcludeWorkloadIdentityCredential = true, + ExcludeAzurePowerShellCredential = true }; IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) + .WithLoggerFactory(ConsoleLogger.LoggerFactory) // Add Azure chat completion service using DefaultAzureCredential AAD auth .WithAzureChatCompletionService( - "gpt-35-turbo", - "https://....openai.azure.com/", + TestConfiguration.AzureOpenAI.ChatDeploymentName, + TestConfiguration.AzureOpenAI.Endpoint, new DefaultAzureCredential(authOptions)) .Build(); diff --git a/dotnet/samples/KernelSyntaxExamples/Example27_SemanticFunctionsUsingChatGPT.cs b/dotnet/samples/KernelSyntaxExamples/Example27_SemanticFunctionsUsingChatGPT.cs index 34eaa2eb0ab2..5d8bda833f4f 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example27_SemanticFunctionsUsingChatGPT.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example27_SemanticFunctionsUsingChatGPT.cs @@ -8,7 +8,6 @@ /** * This example shows how to use GPT3.5 Chat model for prompts and semantic functions. */ - // ReSharper disable once InconsistentNaming public static class Example27_SemanticFunctionsUsingChatGPT { @@ -17,16 +16,15 @@ public static async Task RunAsync() Console.WriteLine("======== Using Chat GPT model for text completion ========"); IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - // Note: we use Chat Completion and GPT 3.5 Turbo - .WithAzureChatCompletionService("gpt-35-turbo", "https://....openai.azure.com/", "...API KEY...") + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService(TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) .Build(); var func = kernel.CreateSemanticFunction( "List the two planets closest to '{{$input}}', excluding moons, using bullet points."); - var result = await func.InvokeAsync("Jupiter"); - Console.WriteLine(result); + var result = await func.InvokeAsync("Jupiter", kernel); + Console.WriteLine(result.GetValue()); /* Output: diff --git a/dotnet/samples/KernelSyntaxExamples/Example28_ActionPlanner.cs b/dotnet/samples/KernelSyntaxExamples/Example28_ActionPlanner.cs index f6dfbb69e785..2944164eb10b 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example28_ActionPlanner.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example28_ActionPlanner.cs @@ -3,8 +3,7 @@ using System; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; +using Microsoft.SemanticKernel.Planners; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -14,30 +13,38 @@ public static async Task RunAsync() { Console.WriteLine("======== Action Planner ========"); var kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService("text-davinci-002", TestConfiguration.OpenAI.ApiKey)// Note: Action Planner works with old models like text-davinci-002 + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + TestConfiguration.AzureOpenAI.ChatDeploymentName, + TestConfiguration.AzureOpenAI.Endpoint, + TestConfiguration.AzureOpenAI.ApiKey) .Build(); - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); - kernel.ImportSemanticSkillFromDirectory(folder, "WriterSkill"); + string samplesDirectory = RepoFiles.SamplePluginsPath(); + kernel.ImportSemanticFunctionsFromDirectory(samplesDirectory, "SummarizePlugin"); + kernel.ImportSemanticFunctionsFromDirectory(samplesDirectory, "WriterPlugin"); + kernel.ImportSemanticFunctionsFromDirectory(samplesDirectory, "FunPlugin"); + + // Create an optional config for the ActionPlanner. Use this to exclude plugins and functions if needed + var config = new ActionPlannerConfig(); + config.ExcludedFunctions.Add("MakeAbstractReadable"); // Create an instance of ActionPlanner. // The ActionPlanner takes one goal and returns a single function to execute. - var planner = new ActionPlanner(kernel); + var planner = new ActionPlanner(kernel, config: config); // We're going to ask the planner to find a function to achieve this goal. - var goal = "Write a poem about Cleopatra."; + var goal = "Write a joke about Cleopatra in the style of Hulk Hogan."; // The planner returns a plan, consisting of a single function // to execute and achieve the goal requested. var plan = await planner.CreatePlanAsync(goal); // Execute the full plan (which is a single function) - SKContext result = await plan.InvokeAsync(); + var result = await plan.InvokeAsync(kernel); // Show the result, which should match the given goal - Console.WriteLine(result); + Console.WriteLine(result.GetValue()); /* Output: * diff --git a/dotnet/samples/KernelSyntaxExamples/Example29_Tokenizer.cs b/dotnet/samples/KernelSyntaxExamples/Example29_Tokenizer.cs deleted file mode 100644 index 84bad7fe3a61..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example29_Tokenizer.cs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI.Tokenizers; - -// ReSharper disable once InconsistentNaming -/// -/// This sample shows how to count tokens using GPT tokenizer. The number of tokens affects -/// API calls cost and each models has a maximum amount of tokens it can process and generate. -/// This example is specific to OpenAI models, which use the tokenization described here: -/// https://platform.openai.com/tokenizer -/// If you use Semantic Kernel with other models, the tokenization logic is most probably different, -/// and you should not use the GPT tokenizer. -/// -public static class Example29_Tokenizer -{ - public static Task RunAsync() - { - // Example 1 - string sentence = "Some text on one line"; - int tokenCount = GPT3Tokenizer.Encode(sentence).Count; - - Console.WriteLine("---"); - Console.WriteLine(sentence); - Console.WriteLine("Tokens: " + tokenCount); - Console.WriteLine("---\n\n"); - - // Example 2 - sentence = "⭐⭐"; - tokenCount = GPT3Tokenizer.Encode(sentence).Count; - - Console.WriteLine("The following example contains emojis which require several tokens."); - Console.WriteLine("---"); - Console.WriteLine(sentence); - Console.WriteLine("Tokens: " + tokenCount); - Console.WriteLine("---\n\n"); - - // Example 3 - sentence = "Some text on\ntwo lines"; - tokenCount = GPT3Tokenizer.Encode(sentence).Count; - - Console.WriteLine("The following example uses Unix '\\n' line separator."); - Console.WriteLine("---"); - Console.WriteLine(sentence); - Console.WriteLine("Tokens: " + tokenCount); - Console.WriteLine("---\n\n"); - - // Example 4 - sentence = "Some text on\r\ntwo lines"; - tokenCount = GPT3Tokenizer.Encode(sentence).Count; - - Console.WriteLine("The following example uses Windows '\\r\\n' line separator."); - Console.WriteLine("---"); - Console.WriteLine(sentence); - Console.WriteLine("Tokens: " + tokenCount); - Console.WriteLine("---\n\n"); - - /* - Output: - --- - Some text on one line - Tokens: 5 - --- - - - The following example contains emojis which require several tokens. - --- - ⭐⭐ - Tokens: 6 - --- - - - The following example uses Unix '\n' line separator. - --- - Some text on - two lines - Tokens: 6 - --- - - - The following example uses Windows '\r\n' line separator. - --- - Some text on - two lines - Tokens: 7 - --- - */ - - return Task.CompletedTask; - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example30_ChatWithPrompts.cs b/dotnet/samples/KernelSyntaxExamples/Example30_ChatWithPrompts.cs index 2ec76852f6d7..567eef956c1e 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example30_ChatWithPrompts.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example30_ChatWithPrompts.cs @@ -5,15 +5,15 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.AI.ChatCompletion; -using Microsoft.SemanticKernel.Skills.Core; -using Microsoft.SemanticKernel.TemplateEngine; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.TemplateEngine.Basic; using RepoUtils; using Resources; /** * Scenario: * - the user is reading a wikipedia page, they select a piece of text and they ask AI to extract some information. - * - the app explicitly uses the Chat model to get a result from gpt-3.5-turbo. + * - the app explicitly uses the Chat model to get a result. * * The following example shows how to: * @@ -36,14 +36,13 @@ * TLDR: how to render a prompt: * * var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); - * ... import skills and functions ... + * ... import plugins and functions ... * var context = kernel.CreateNewContext(); * ... set variables ... * - * var promptRenderer = new PromptTemplateEngine(); + * var promptRenderer = new BasicPromptTemplateEngine(); * string renderedPrompt = await promptRenderer.RenderAsync("...prompt template...", context); */ - // ReSharper disable CommentTypo // ReSharper disable once InconsistentNaming public static class Example30_ChatWithPrompts @@ -62,18 +61,17 @@ public static async Task RunAsync() var selectedText = EmbeddedResource.Read("30-user-context.txt"); var userPromptTemplate = EmbeddedResource.Read("30-user-prompt.txt"); - // Usual kernel initialization, with GPT 3.5 Turbo IKernel kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithOpenAIChatCompletionService("gpt-3.5-turbo", TestConfiguration.OpenAI.ApiKey, serviceId: "chat") + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey, serviceId: "chat") .Build(); - // As an example, we import the time skill, which is used in system prompt to read the current date. + // As an example, we import the time plugin, which is used in system prompt to read the current date. // We could also use a variable, this is just to show that the prompt can invoke functions. - kernel.ImportSkill(new TimeSkill(), "time"); + kernel.ImportFunctions(new TimePlugin(), "time"); // We need a kernel context to store some information to pass to the prompts and the list - // of available skills needed to render prompt templates. + // of available plugins needed to render prompt templates. var context = kernel.CreateNewContext(); // Put the selected document into the variable used by the system prompt (see 28-system-prompt.txt). @@ -88,7 +86,7 @@ public static async Task RunAsync() // Instantiate the prompt renderer, which we will use to turn prompt templates // into strings, that we will store into a Chat history object, which is then sent // to the Chat Model. - var promptRenderer = new PromptTemplateEngine(); + var promptRenderer = new BasicPromptTemplateEngine(); // Render the system prompt. This string is used to configure the chat. // This contains the context, ie a piece of a wikipedia page selected by the user. @@ -100,7 +98,7 @@ public static async Task RunAsync() string userMessage = await promptRenderer.RenderAsync(userPromptTemplate, context); Console.WriteLine($"------------------------------------\n{userMessage}"); - // Client used to request answers to gpt-3.5-turbo + // Client used to request answers var chatGPT = kernel.GetService(); // The full chat history. Depending on your scenario, you can pass the full chat if useful, diff --git a/dotnet/samples/KernelSyntaxExamples/Example31_CustomPlanner.cs b/dotnet/samples/KernelSyntaxExamples/Example31_CustomPlanner.cs index ecfccc306e5c..8dd94d078651 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example31_CustomPlanner.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example31_CustomPlanner.cs @@ -7,13 +7,15 @@ using System.Xml; using System.Xml.XPath; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.Core; -using Microsoft.SemanticKernel.Skills.Web; -using Microsoft.SemanticKernel.Skills.Web.Bing; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.Plugins.Memory; +using Microsoft.SemanticKernel.Plugins.Web; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + using RepoUtils; // ReSharper disable CommentTypo @@ -24,31 +26,32 @@ public static async Task RunAsync() { Console.WriteLine("======== Custom Planner - Create and Execute Markup Plan ========"); IKernel kernel = InitializeKernel(); + ISemanticTextMemory memory = InitializeMemory(); - // ContextQuery is part of the QASkill - IDictionary skills = LoadQASkill(kernel); + // ContextQuery is part of the QAPlugin + IDictionary qaPlugin = LoadQAPlugin(kernel); SKContext context = CreateContextQueryContext(kernel); // Create a memory store using the VolatileMemoryStore and the embedding generator registered in the kernel - kernel.ImportSkill(new TextMemorySkill(kernel.Memory)); + kernel.ImportFunctions(new TextMemoryPlugin(memory)); // Setup defined memories for recall - await RememberFactsAsync(kernel); + await RememberFactsAsync(kernel, memory); - // MarkupSkill named "markup" - var markup = kernel.ImportSkill(new MarkupSkill(), "markup"); + // MarkupPlugin named "markup" + var markup = kernel.ImportFunctions(new MarkupPlugin(), "markup"); // contextQuery "Who is my president? Who was president 3 years ago? What should I eat for dinner" | markup - // Create a plan to execute the ContextQuery and then run the markup skill on the output + // Create a plan to execute the ContextQuery and then run the markup plugin on the output var plan = new Plan("Execute ContextQuery and then RunMarkup"); - plan.AddSteps(skills["ContextQuery"], markup["RunMarkup"]); + plan.AddSteps(qaPlugin["ContextQuery"], markup["RunMarkup"]); // Execute plan context.Variables.Update("Who is my president? Who was president 3 years ago? What should I eat for dinner"); var result = await plan.InvokeAsync(context); Console.WriteLine("Result:"); - Console.WriteLine(result.Result); + Console.WriteLine(result.GetValue()); Console.WriteLine(); } /* Example Output @@ -85,9 +88,9 @@ private static SKContext CreateContextQueryContext(IKernel kernel) return context; } - private static async Task RememberFactsAsync(IKernel kernel) + private static async Task RememberFactsAsync(IKernel kernel, ISemanticTextMemory memory) { - kernel.ImportSkill(new TextMemorySkill(kernel.Memory)); + kernel.ImportFunctions(new TextMemoryPlugin(memory)); List memoriesToSave = new() { @@ -105,44 +108,55 @@ private static async Task RememberFactsAsync(IKernel kernel) foreach (var memoryToSave in memoriesToSave) { - await kernel.Memory.SaveInformationAsync("contextQueryMemories", memoryToSave, Guid.NewGuid().ToString()); + await memory.SaveInformationAsync("contextQueryMemories", memoryToSave, Guid.NewGuid().ToString()); } } - // ContextQuery is part of the QASkill - // DependsOn: TimeSkill named "time" - // DependsOn: BingSkill named "bing" - private static IDictionary LoadQASkill(IKernel kernel) + // ContextQuery is part of the QAPlugin + // DependsOn: TimePlugin named "time" + // DependsOn: BingPlugin named "bing" + private static IDictionary LoadQAPlugin(IKernel kernel) { - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSkill(new TimeSkill(), "time"); + string folder = RepoFiles.SamplePluginsPath(); + kernel.ImportFunctions(new TimePlugin(), "time"); #pragma warning disable CA2000 // Dispose objects before losing scope - var bing = new WebSearchEngineSkill(new BingConnector(TestConfiguration.Bing.ApiKey)); + var bing = new WebSearchEnginePlugin(new BingConnector(TestConfiguration.Bing.ApiKey)); #pragma warning restore CA2000 // Dispose objects before losing scope - var search = kernel.ImportSkill(bing, "bing"); + kernel.ImportFunctions(bing, "bing"); - return kernel.ImportSemanticSkillFromDirectory(folder, "QASkill"); + return kernel.ImportSemanticFunctionsFromDirectory(folder, "QAPlugin"); } private static IKernel InitializeKernel() { return new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithAzureTextCompletionService( - TestConfiguration.AzureOpenAI.DeploymentName, + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + TestConfiguration.AzureOpenAI.ChatDeploymentName, + TestConfiguration.AzureOpenAI.Endpoint, + TestConfiguration.AzureOpenAI.ApiKey) + .WithAzureTextEmbeddingGenerationService( + TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) + .Build(); + } + + private static ISemanticTextMemory InitializeMemory() + { + return new MemoryBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) .WithAzureTextEmbeddingGenerationService( TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) - .WithMemoryStorage(new VolatileMemoryStore()) + .WithMemoryStore(new VolatileMemoryStore()) .Build(); } } -// Example Skill that can process XML Markup created by ContextQuery -public class MarkupSkill +// Example Plugin that can process XML Markup created by ContextQuery +public class MarkupPlugin { [SKFunction, Description("Run Markup")] public async Task RunMarkupAsync(string docString, SKContext context) @@ -153,14 +167,14 @@ public async Task RunMarkupAsync(string docString, SKContext context) Console.WriteLine(plan.ToPlanWithGoalString()); Console.WriteLine(); - var result = await plan.InvokeAsync(); - return result.Result; + var result = await context.Runner.RunAsync(plan); + return result.GetValue()!; } } public static class XmlMarkupPlanParser { - private static readonly Dictionary> s_skillMapping = new() + private static readonly Dictionary> s_pluginMapping = new() { { "lookup", new KeyValuePair("bing", "SearchAsync") }, }; @@ -183,12 +197,12 @@ private static Plan NodeListToPlan(XmlNodeList nodes, SKContext context, string { var node = nodes[i]; var functionName = node!.LocalName; - var skillName = string.Empty; + var pluginName = string.Empty; - if (s_skillMapping.TryGetValue(node!.LocalName, out KeyValuePair value)) + if (s_pluginMapping.TryGetValue(node!.LocalName, out KeyValuePair value)) { functionName = value.Value; - skillName = value.Key; + pluginName = value.Key; } var hasChildElements = node.HasChildElements(); @@ -199,9 +213,9 @@ private static Plan NodeListToPlan(XmlNodeList nodes, SKContext context, string } else { - if (string.IsNullOrEmpty(skillName) - ? !context.Skills!.TryGetFunction(functionName, out var _) - : !context.Skills!.TryGetFunction(skillName, functionName, out var _)) + if (string.IsNullOrEmpty(pluginName) + ? !context.Functions!.TryGetFunction(functionName, out var _) + : !context.Functions!.TryGetFunction(pluginName, functionName, out var _)) { var planStep = new Plan(node.InnerText); planStep.Parameters.Update(node.InnerText); @@ -211,9 +225,9 @@ private static Plan NodeListToPlan(XmlNodeList nodes, SKContext context, string } else { - var command = string.IsNullOrEmpty(skillName) - ? context.Skills.GetFunction(functionName) - : context.Skills.GetFunction(skillName, functionName); + var command = string.IsNullOrEmpty(pluginName) + ? context.Functions.GetFunction(functionName) + : context.Functions.GetFunction(pluginName, functionName); var planStep = new Plan(command); planStep.Parameters.Update(node.InnerText); planStep.Outputs.Add($"markup.{functionName}.result"); diff --git a/dotnet/samples/KernelSyntaxExamples/Example32_StreamingCompletion.cs b/dotnet/samples/KernelSyntaxExamples/Example32_StreamingCompletion.cs index db362fd8e9b0..5ddeb8fbf33d 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example32_StreamingCompletion.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example32_StreamingCompletion.cs @@ -3,10 +3,15 @@ using System; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextCompletion; /** - * The following example shows how to use Semantic Kernel with Text Completion as streaming + * The following example shows how to use Semantic Kernel with streaming Text Completion. + * + * Note that all text completion models are deprecated by OpenAI and will be removed in a future release. + * + * Refer to example 33 for streaming chat completion. */ // ReSharper disable once InconsistentNaming public static class Example32_StreamingCompletion @@ -40,7 +45,7 @@ private static async Task OpenAITextCompletionStreamAsync() private static async Task TextCompletionStreamAsync(ITextCompletion textCompletion) { - var requestSettings = new CompleteRequestSettings() + var requestSettings = new OpenAIRequestSettings() { MaxTokens = 100, FrequencyPenalty = 0, diff --git a/dotnet/samples/KernelSyntaxExamples/Example33_StreamingChat.cs b/dotnet/samples/KernelSyntaxExamples/Example33_StreamingChat.cs index f6ebe85e512e..c577838105fc 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example33_StreamingChat.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example33_StreamingChat.cs @@ -7,7 +7,7 @@ using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; /** - * The following example shows how to use Semantic Kernel with Text Completion as streaming + * The following example shows how to use Semantic Kernel with streaming Chat Completion */ // ReSharper disable once InconsistentNaming public static class Example33_StreamingChat @@ -22,7 +22,7 @@ private static async Task OpenAIChatStreamSampleAsync() { Console.WriteLine("======== Open AI - ChatGPT Streaming ========"); - OpenAIChatCompletion openAIChatCompletion = new("gpt-3.5-turbo", TestConfiguration.OpenAI.ApiKey); + OpenAIChatCompletion openAIChatCompletion = new(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey); await StartStreamingChatAsync(openAIChatCompletion); } @@ -62,8 +62,7 @@ private static async Task StartStreamingChatAsync(IChatCompletion chatCompletion await StreamMessageOutputAsync(chatCompletion, chatHistory, AuthorRole.Assistant); } - private static async Task StreamMessageOutputAsync(IChatCompletion chatGPT, ChatHistory chatHistory, - AuthorRole authorRole) + private static async Task StreamMessageOutputAsync(IChatCompletion chatGPT, ChatHistory chatHistory, AuthorRole authorRole) { Console.Write($"{authorRole}: "); string fullMessage = string.Empty; diff --git a/dotnet/samples/KernelSyntaxExamples/Example34_CustomChatModel.cs b/dotnet/samples/KernelSyntaxExamples/Example34_CustomChatModel.cs index 7829f4a41289..c9e0c399d2de 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example34_CustomChatModel.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example34_CustomChatModel.cs @@ -6,7 +6,9 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Orchestration; /** * The following example shows how to plug use a custom chat model. @@ -30,15 +32,15 @@ public ChatHistory CreateNewChat(string? instructions = null) return chatHistory; } - public Task> GetChatCompletionsAsync(ChatHistory chat, ChatRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public Task> GetChatCompletionsAsync(ChatHistory chat, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { return Task.FromResult>(new List { - new MyChatStreamingResult(MyRoles.Bot, "Hi I'm your SK Custom Assistant and I'm here to help you to create custom chats like this. :)") + new MyChatResult(MyRoles.Bot, "Hi I'm your SK Custom Assistant and I'm here to help you to create custom chats like this. :)") }); } - public IAsyncEnumerable GetStreamingChatCompletionsAsync(ChatHistory chat, ChatRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChatCompletionsAsync(ChatHistory chat, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { return (new List { @@ -51,18 +53,14 @@ public class MyChatStreamingResult : IChatStreamingResult { private readonly ChatMessageBase _message; private readonly MyRoles _role; + public ModelResult ModelResult { get; private set; } public MyChatStreamingResult(MyRoles role, string content) { this._role = role; this._message = new MyChatMessage(role, content); + this.ModelResult = new ModelResult(content); } - - public Task GetChatMessageAsync(CancellationToken cancellationToken = default) - { - return Task.FromResult(this._message); - } - public async IAsyncEnumerable GetStreamingChatMessageAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { var streamedOutput = this._message.Content.Split(' '); @@ -74,6 +72,25 @@ public async IAsyncEnumerable GetStreamingChatMessageAsync([Enu } } +public class MyChatResult : IChatResult +{ + private readonly ChatMessageBase _message; + private readonly MyRoles _role; + public ModelResult ModelResult { get; private set; } + + public MyChatResult(MyRoles role, string content) + { + this._role = role; + this._message = new MyChatMessage(role, content); + this.ModelResult = new ModelResult(content); + } + + public Task GetChatMessageAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult(this._message); + } +} + public class MyChatMessage : ChatMessageBase { public MyChatMessage(MyRoles role, string content) : base(new AuthorRole(role.ToString()), content) diff --git a/dotnet/samples/KernelSyntaxExamples/Example35_GrpcPlugins.cs b/dotnet/samples/KernelSyntaxExamples/Example35_GrpcPlugins.cs new file mode 100644 index 000000000000..886920fa621a --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example35_GrpcPlugins.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Functions.Grpc.Extensions; +using Microsoft.SemanticKernel.Orchestration; +using RepoUtils; + +/** + * This example shows how to use gRPC plugins. + */ +// ReSharper disable once InconsistentNaming +public static class Example35_GrpcPlugins +{ + public static async Task RunAsync() + { + var kernel = new KernelBuilder().WithLoggerFactory(ConsoleLogger.LoggerFactory).Build(); + + // Import a gRPC plugin using one of the following Kernel extension methods + // kernel.RegisterGrpcFunctions + // kernel.ImportGrpcFunctionsFromDirectory + var plugin = kernel.ImportGrpcFunctionsFromFile("", ""); + + // Add arguments for required parameters, arguments for optional ones can be skipped. + var contextVariables = new ContextVariables(); + contextVariables.Set("address", ""); + contextVariables.Set("payload", ""); + + // Run + var result = await kernel.RunAsync(contextVariables, plugin[""]); + + Console.WriteLine("Plugin response: {0}", result.GetValue()); + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example35_GrpcSkills.cs b/dotnet/samples/KernelSyntaxExamples/Example35_GrpcSkills.cs deleted file mode 100644 index 7e51a9e067db..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example35_GrpcSkills.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Skills.Grpc.Extensions; -using RepoUtils; - -/** - * This example shows how to use gRPC skills. - */ - -// ReSharper disable once InconsistentNaming -public static class Example35_GrpcSkills -{ - public static async Task RunAsync() - { - var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); - - // Import a gRPC skill using one of the following Kernel extension methods - // kernel.RegisterGrpcSkill - // kernel.ImportGrpcSkillFromDirectory - var skill = kernel.ImportGrpcSkillFromFile("", ""); - - // Add arguments for required parameters, arguments for optional ones can be skipped. - var contextVariables = new ContextVariables(); - contextVariables.Set("address", ""); - contextVariables.Set("payload", ""); - - // Run - var result = await kernel.RunAsync(contextVariables, skill[""]); - - Console.WriteLine("Skill response: {0}", result); - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example36_MultiCompletion.cs b/dotnet/samples/KernelSyntaxExamples/Example36_MultiCompletion.cs index e02bdb5e8ffc..8fdeca2febfb 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example36_MultiCompletion.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example36_MultiCompletion.cs @@ -2,47 +2,48 @@ using System; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextCompletion; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; /** - * The following example shows how to use Semantic Kernel with Multiple Results Text Completion as streaming + * The following example shows how to use Semantic Kernel with streaming Multiple Results Chat Completion. */ // ReSharper disable once InconsistentNaming public static class Example36_MultiCompletion { public static async Task RunAsync() { - await AzureOpenAIMultiTextCompletionAsync(); - await OpenAIMultiTextCompletionAsync(); + await AzureOpenAIMultiChatCompletionAsync(); + await OpenAIMultiChatCompletionAsync(); } - private static async Task AzureOpenAIMultiTextCompletionAsync() + private static async Task AzureOpenAIMultiChatCompletionAsync() { - Console.WriteLine("======== Azure OpenAI - Multiple Text Completion ========"); + Console.WriteLine("======== Azure OpenAI - Multiple Chat Completion ========"); - var textCompletion = new AzureTextCompletion( - TestConfiguration.AzureOpenAI.DeploymentName, + var chatCompletion = new AzureChatCompletion( + TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey); - await TextCompletionAsync(textCompletion); + await ChatCompletionAsync(chatCompletion); } - private static async Task OpenAIMultiTextCompletionAsync() + private static async Task OpenAIMultiChatCompletionAsync() { - Console.WriteLine("======== Open AI - Multiple Text Completion ========"); + Console.WriteLine("======== Open AI - Multiple Chat Completion ========"); - ITextCompletion textCompletion = new OpenAITextCompletion( - "text-davinci-003", + IChatCompletion chatCompletion = new OpenAIChatCompletion( + TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey); - await TextCompletionAsync(textCompletion); + await ChatCompletionAsync(chatCompletion); } - private static async Task TextCompletionAsync(ITextCompletion textCompletion) + private static async Task ChatCompletionAsync(IChatCompletion chatCompletion) { - var requestSettings = new CompleteRequestSettings() + var requestSettings = new OpenAIRequestSettings() { MaxTokens = 200, FrequencyPenalty = 0, @@ -52,11 +53,12 @@ private static async Task TextCompletionAsync(ITextCompletion textCompletion) ResultsPerPrompt = 2, }; - var prompt = "Write one paragraph why AI is awesome"; + var chatHistory = new ChatHistory(); + chatHistory.AddUserMessage("Write one paragraph about why AI is awesome"); - foreach (ITextResult completionResult in await textCompletion.GetCompletionsAsync(prompt, requestSettings)) + await foreach (string message in chatCompletion.GenerateMessageStreamAsync(chatHistory)) { - Console.WriteLine(await completionResult.GetCompletionAsync()); + Console.Write(message); Console.WriteLine("-------------"); } diff --git a/dotnet/samples/KernelSyntaxExamples/Example37_MultiStreamingCompletion.cs b/dotnet/samples/KernelSyntaxExamples/Example37_MultiStreamingCompletion.cs index 3c56d0941852..4cb9e4b0d49d 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example37_MultiStreamingCompletion.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example37_MultiStreamingCompletion.cs @@ -1,51 +1,49 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextCompletion; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; /** - * The following example shows how to use Semantic Kernel with Multiple Results Text Completion as streaming + * The following example shows how to use Semantic Kernel with streaming Multiple Results Chat Completion */ // ReSharper disable once InconsistentNaming public static class Example37_MultiStreamingCompletion { - private static readonly object s_lockObject = new(); - public static async Task RunAsync() { - await AzureOpenAIMultiTextCompletionStreamAsync(); - await OpenAITextCompletionStreamAsync(); + await AzureOpenAIMultiChatCompletionStreamAsync(); + await OpenAIChatCompletionStreamAsync(); } - private static async Task AzureOpenAIMultiTextCompletionStreamAsync() + private static async Task AzureOpenAIMultiChatCompletionStreamAsync() { - Console.WriteLine("======== Azure OpenAI - Multiple Text Completion - Raw Streaming ========"); + Console.WriteLine("======== Azure OpenAI - Multiple Chat Completion - Raw Streaming ========"); - var textCompletion = new AzureTextCompletion( - TestConfiguration.AzureOpenAI.DeploymentName, + var chatCompletion = new AzureChatCompletion( + TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey); - await TextCompletionStreamAsync(textCompletion); + await ChatCompletionStreamAsync(chatCompletion); } - private static async Task OpenAITextCompletionStreamAsync() + private static async Task OpenAIChatCompletionStreamAsync() { - Console.WriteLine("======== Open AI - Multiple Text Completion - Raw Streaming ========"); + Console.WriteLine("======== Open AI - Multiple Chat Completion - Raw Streaming ========"); - ITextCompletion textCompletion = new OpenAITextCompletion( - "text-davinci-003", + IChatCompletion chatCompletion = new OpenAIChatCompletion( + TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey); - await TextCompletionStreamAsync(textCompletion); + await ChatCompletionStreamAsync(chatCompletion); } - private static async Task TextCompletionStreamAsync(ITextCompletion textCompletion) + private static async Task ChatCompletionStreamAsync(IChatCompletion chatCompletion) { - var requestSettings = new CompleteRequestSettings() + var requestSettings = new OpenAIRequestSettings() { MaxTokens = 200, FrequencyPenalty = 0, @@ -55,67 +53,14 @@ private static async Task TextCompletionStreamAsync(ITextCompletion textCompleti ResultsPerPrompt = 3 }; - var prompt = "Write one paragraph why AI is awesome"; - var consoleLinesPerResult = 12; - - PrepareDisplay(); + var chatHistory = new ChatHistory(); + chatHistory.AddUserMessage("Write one paragraph about why AI is awesome"); - List resultTasks = new(); - int currentResult = 0; - await foreach (var completionResult in textCompletion.GetStreamingCompletionsAsync(prompt, requestSettings)) + await foreach (string message in chatCompletion.GenerateMessageStreamAsync(chatHistory)) { - resultTasks.Add(ProcessStreamAsyncEnumerableAsync(completionResult, currentResult++, consoleLinesPerResult)); + Console.Write(message); } Console.WriteLine(); - - await Task.WhenAll(resultTasks.ToArray()); - - /* - - int position = 0; - await foreach (ITextCompletionStreamingResult completionResult in textCompletion.CompleteMultiStreamAsync(prompt, requestSettings)) - { - string fullMessage = string.Empty; - - await foreach (string message in completionResult.GetCompletionStreamingAsync()) - { - fullMessage += message; - - Console.SetCursorPosition(0, (position * consoleLinesPerResult)); - Console.Write(fullMessage); - } - - position++; - }*/ - - Console.SetCursorPosition(0, requestSettings.ResultsPerPrompt * consoleLinesPerResult); - Console.WriteLine(); - } - - private static async Task ProcessStreamAsyncEnumerableAsync(ITextStreamingResult result, int resultNumber, int linesPerResult) - { - var fullSentence = string.Empty; - await foreach (var word in result.GetCompletionStreamingAsync()) - { - fullSentence += word; - - lock (s_lockObject) - { - Console.SetCursorPosition(0, (resultNumber * linesPerResult)); - Console.Write(fullSentence); - } - } - } - - /// - /// Break enough lines as the current console window size to display the results - /// - private static void PrepareDisplay() - { - for (int i = 0; i < Console.WindowHeight - 2; i++) - { - Console.WriteLine(); - } } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example38_Pinecone.cs b/dotnet/samples/KernelSyntaxExamples/Example38_Pinecone.cs deleted file mode 100644 index adfe0f6a2c20..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example38_Pinecone.cs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.Memory.Pinecone; -using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; -using Microsoft.SemanticKernel.Memory; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -/// -/// This example shows how to use the PineconeMemoryStore to store and retrieve memories with Pinecone assuming -/// you have a Pinecone account and have created an index. You can create an index using the Pinecone UI or -/// instead initialize the index using . -/// But note that it can take a few minutes for the index to be ready for use. -/// -public static class Example38_Pinecone -{ - private const string MemoryCollectionName = "pinecone-test"; - - public static async Task RunAsync() - { - string apiKey = TestConfiguration.Pinecone.ApiKey; - string pineconeEnvironment = TestConfiguration.Pinecone.Environment; - - PineconeMemoryStore memoryStore = new(pineconeEnvironment, apiKey); - - IKernel kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService(TestConfiguration.OpenAI.ModelId, TestConfiguration.OpenAI.ApiKey) - .WithOpenAITextEmbeddingGenerationService(TestConfiguration.OpenAI.EmbeddingModelId, TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(memoryStore) - //.WithPineconeMemoryStore(pineconeEnvironment, apiKey) // This method offers an alternative approach to registering Pinecone memory storage. - .Build(); - - Console.WriteLine("== Printing Collections in DB =="); - - IAsyncEnumerable collections = memoryStore.GetCollectionsAsync(); - - await foreach (string collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Adding Memories =="); - - Dictionary metadata = new() - { - { "type", "text" }, - { "tags", new List() { "memory", "cats" } } - }; - - string additionalMetadata = System.Text.Json.JsonSerializer.Serialize(metadata); - - string key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, "british short hair", "cat1", null, additionalMetadata); - string key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, "orange tabby", "cat2", null, additionalMetadata); - string key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, "norwegian forest cat", "cat3", null, additionalMetadata); - - Console.WriteLine("== Retrieving Memories Through the Kernel =="); - MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1"); - Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Retrieving Memories Directly From the Store =="); - var memory1 = await memoryStore.GetAsync(MemoryCollectionName, key1); - var memory2 = await memoryStore.GetAsync(MemoryCollectionName, key2); - var memory3 = await memoryStore.GetAsync(MemoryCollectionName, key3); - - Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found"); - Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found"); - Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Similarity Searching Memories: My favorite color is orange =="); - IAsyncEnumerable searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", 1, 0.8); - - await foreach (MemoryQueryResult item in searchResults) - { - Console.WriteLine(item.Metadata.Text + " : " + item.Relevance); - } - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example39_Postgres.cs b/dotnet/samples/KernelSyntaxExamples/Example39_Postgres.cs deleted file mode 100644 index 541c6836f614..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example39_Postgres.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.Memory.Postgres; -using Microsoft.SemanticKernel.Memory; -using Npgsql; -using Pgvector.Npgsql; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -public static class Example39_Postgres -{ - private const string MemoryCollectionName = "postgres_test"; - - public static async Task RunAsync() - { - NpgsqlDataSourceBuilder dataSourceBuilder = new(TestConfiguration.Postgres.ConnectionString); - dataSourceBuilder.UseVector(); - await using NpgsqlDataSource dataSource = dataSourceBuilder.Build(); - - PostgresMemoryStore memoryStore = new(dataSource, vectorSize: 1536, schema: "public"); - - IKernel kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .WithOpenAITextEmbeddingGenerationService( - modelId: TestConfiguration.OpenAI.EmbeddingModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(memoryStore) - //.WithPostgresMemoryStore(dataSource, vectorSize: 1536, schema: "public") // This method offers an alternative approach to registering Postgres memory store. - .Build(); - - Console.WriteLine("== Printing Collections in DB =="); - var collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Adding Memories =="); - - var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat1", text: "british short hair"); - var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat2", text: "orange tabby"); - var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat3", text: "norwegian forest cat"); - - Console.WriteLine("== Printing Collections in DB =="); - collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Retrieving Memories Through the Kernel =="); - MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1"); - Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Retrieving Memories Directly From the Store =="); - var memory1 = await memoryStore.GetAsync(MemoryCollectionName, key1); - var memory2 = await memoryStore.GetAsync(MemoryCollectionName, key2); - var memory3 = await memoryStore.GetAsync(MemoryCollectionName, key3); - Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found"); - Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found"); - Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Similarity Searching Memories: My favorite color is orange =="); - var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8); - - await foreach (var item in searchResults) - { - Console.WriteLine(item.Metadata.Text + " : " + item.Relevance); - } - - Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName); - await memoryStore.DeleteCollectionAsync(MemoryCollectionName); - - Console.WriteLine("== Printing Collections in DB =="); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example40_DIContainer.cs b/dotnet/samples/KernelSyntaxExamples/Example40_DIContainer.cs index bba7fd2b92c0..38e6101cb060 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example40_DIContainer.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example40_DIContainer.cs @@ -6,10 +6,13 @@ using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextCompletion; +using Microsoft.SemanticKernel.Http; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Reliability.Basic; using Microsoft.SemanticKernel.Services; -using Microsoft.SemanticKernel.SkillDefinition; + using Microsoft.SemanticKernel.TemplateEngine; +using Microsoft.SemanticKernel.TemplateEngine.Basic; using RepoUtils; /** @@ -36,18 +39,18 @@ private static async Task UseKernelInDIPowerAppAsync() //Registering Kernel dependencies var collection = new ServiceCollection(); - collection.AddTransient((_) => ConsoleLogger.Logger); + collection.AddTransient((_) => ConsoleLogger.LoggerFactory); //Registering Kernel collection.AddTransient((serviceProvider) => { return Kernel.Builder - .WithLogger(serviceProvider.GetRequiredService()) - .WithOpenAITextCompletionService("text-davinci-002", TestConfiguration.OpenAI.ApiKey) + .WithLoggerFactory(serviceProvider.GetRequiredService()) + .WithOpenAITextCompletionService(TestConfiguration.OpenAI.ModelId, TestConfiguration.OpenAI.ApiKey) .Build(); }); - //Registering class that uses Kernel to execute a skill + //Registering class that uses Kernel to execute a plugin collection.AddTransient(); //Creating a service provider for resolving registered services @@ -72,21 +75,21 @@ private static async Task UseKernelInDIPowerApp_AdvancedScenarioAsync() //Registering AI services Kernel is going to use var aiServicesCollection = new AIServiceCollection(); - aiServicesCollection.SetService(() => new OpenAITextCompletion("text-davinci-002", TestConfiguration.OpenAI.ApiKey)); + aiServicesCollection.SetService(() => new OpenAITextCompletion(TestConfiguration.OpenAI.ModelId, TestConfiguration.OpenAI.ApiKey)); //Registering Kernel dependencies var collection = new ServiceCollection(); - collection.AddTransient((_) => ConsoleLogger.Logger); - collection.AddTransient(); - collection.AddTransient(); - collection.AddTransient(); + collection.AddTransient((_) => ConsoleLogger.LoggerFactory); + collection.AddTransient((_) => BasicHttpRetryHandlerFactory.Instance); + collection.AddTransient(); + collection.AddTransient(); collection.AddTransient((_) => NullMemory.Instance); collection.AddTransient((_) => aiServicesCollection.Build()); //Registering AI service provider that is used by Kernel to resolve AI services runtime //Registering Kernel collection.AddTransient(); - //Registering class that uses Kernel to execute a skill + //Registering class that uses Kernel to execute a plugin collection.AddTransient(); //Creating a service provider for resolving registered services @@ -110,21 +113,21 @@ private sealed class KernelClient private readonly IKernel _kernel; private readonly ILogger _logger; - public KernelClient(IKernel kernel, ILogger logger) + public KernelClient(IKernel kernel, ILoggerFactory loggerFactory) { this._kernel = kernel; - this._logger = logger; + this._logger = loggerFactory.CreateLogger(nameof(KernelClient)); } public async Task SummarizeAsync(string ask) { - string folder = RepoFiles.SampleSkillsPath(); + string folder = RepoFiles.SamplePluginsPath(); - var sumSkill = this._kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); + var summarizeFunctions = this._kernel.ImportSemanticFunctionsFromDirectory(folder, "SummarizePlugin"); - var result = await this._kernel.RunAsync(ask, sumSkill["Summarize"]); + var result = await this._kernel.RunAsync(ask, summarizeFunctions["Summarize"]); - this._logger.LogWarning("Result - {0}", result); + this._logger.LogWarning("Result - {0}", result.GetValue()); } } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example41_HttpClientUsage.cs b/dotnet/samples/KernelSyntaxExamples/Example41_HttpClientUsage.cs index 68a45b0cae6b..1170eb338c7c 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example41_HttpClientUsage.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example41_HttpClientUsage.cs @@ -33,8 +33,8 @@ public static Task RunAsync() private static void UseDefaultHttpClient() { var kernel = Kernel.Builder - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, + .WithOpenAIChatCompletionService( + modelId: TestConfiguration.OpenAI.ChatModelId, apiKey: TestConfiguration.OpenAI.ApiKey) // If you need to use the default HttpClient from the SK SDK, simply omit the argument for the httpMessageInvoker parameter. .Build(); } @@ -48,7 +48,7 @@ private static void UseCustomHttpClient() // If you need to use a custom HttpClient, simply pass it as an argument for the httpClient parameter. var kernel = Kernel.Builder - .WithOpenAITextCompletionService( + .WithOpenAIChatCompletionService( modelId: TestConfiguration.OpenAI.ModelId, apiKey: TestConfiguration.OpenAI.ApiKey, httpClient: httpClient) @@ -69,8 +69,8 @@ private static void UseBasicRegistrationWithHttpClientFactory() var factory = sp.GetRequiredService(); var kernel = Kernel.Builder - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, + .WithOpenAIChatCompletionService( + modelId: TestConfiguration.OpenAI.ChatModelId, apiKey: TestConfiguration.OpenAI.ApiKey, httpClient: factory.CreateClient()) .Build(); @@ -100,8 +100,8 @@ private static void UseNamedRegistrationWitHttpClientFactory() var factory = sp.GetRequiredService(); var kernel = Kernel.Builder - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, + .WithOpenAIChatCompletionService( + modelId: TestConfiguration.OpenAI.ChatModelId, apiKey: TestConfiguration.OpenAI.ApiKey, httpClient: factory.CreateClient("test-client")) .Build(); diff --git a/dotnet/samples/KernelSyntaxExamples/Example42_KernelBuilder.cs b/dotnet/samples/KernelSyntaxExamples/Example42_KernelBuilder.cs index 007da6eb4159..f90c1ad6d9e7 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example42_KernelBuilder.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example42_KernelBuilder.cs @@ -13,15 +13,18 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Http; using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Reliability; +using Microsoft.SemanticKernel.Plugins.Memory; +using Microsoft.SemanticKernel.Reliability.Basic; +using Microsoft.SemanticKernel.Reliability.Polly; using Microsoft.SemanticKernel.Services; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.TemplateEngine; + +using Microsoft.SemanticKernel.TemplateEngine.Basic; using Polly; using Polly.Retry; @@ -32,7 +35,7 @@ public static Task RunAsync() { string azureOpenAIKey = TestConfiguration.AzureOpenAI.ApiKey; string azureOpenAIEndpoint = TestConfiguration.AzureOpenAI.Endpoint; - string azureOpenAITextCompletionDeployment = TestConfiguration.AzureOpenAI.DeploymentName; + string azureOpenAIChatCompletionDeployment = TestConfiguration.AzureOpenAI.ChatDeploymentName; string azureOpenAIEmbeddingDeployment = TestConfiguration.AzureOpenAIEmbeddings.DeploymentName; #pragma warning disable CA1852 // Seal internal types @@ -67,68 +70,47 @@ public static Task RunAsync() // to enable custom configurations, we highly recommend using KernelBuilder instead, to ensure // a correct dependency injection. - // Manually setup all the dependencies used internally by the kernel - var logger = NullLogger.Instance; + // Manually setup all the dependencies to be used by the kernel + var loggerFactory = NullLoggerFactory.Instance; var memoryStorage = new VolatileMemoryStore(); var textEmbeddingGenerator = new AzureTextEmbeddingGeneration( modelId: azureOpenAIEmbeddingDeployment, endpoint: azureOpenAIEndpoint, apiKey: azureOpenAIKey, - logger: logger); - using var memory = new SemanticTextMemory(memoryStorage, textEmbeddingGenerator); - var skills = new SkillCollection(); - var templateEngine = new PromptTemplateEngine(logger); - var kernelConfig = new KernelConfig(); + loggerFactory: loggerFactory); + + var memory = new SemanticTextMemory(memoryStorage, textEmbeddingGenerator); + var plugins = new FunctionCollection(); + var templateEngine = new BasicPromptTemplateEngine(loggerFactory); - using var httpHandler = new DefaultHttpRetryHandler(new HttpRetryConfig(), logger); + var httpHandlerFactory = BasicHttpRetryHandlerFactory.Instance; + //var httpHandlerFactory = new PollyHttpRetryHandlerFactory( your policy ); + + using var httpHandler = httpHandlerFactory.Create(loggerFactory); using var httpClient = new HttpClient(httpHandler); var aiServices = new AIServiceCollection(); - ITextCompletion Factory() => new AzureTextCompletion( - modelId: azureOpenAITextCompletionDeployment, + ITextCompletion Factory() => new AzureChatCompletion( + modelId: azureOpenAIChatCompletionDeployment, endpoint: azureOpenAIEndpoint, apiKey: azureOpenAIKey, httpClient, - logger); + loggerFactory); aiServices.SetService("foo", Factory); IAIServiceProvider aiServiceProvider = aiServices.Build(); // Create kernel manually injecting all the dependencies - using var kernel3 = new Kernel(skills, aiServiceProvider, templateEngine, memory, kernelConfig, logger); + using var kernel3 = new Kernel(plugins, aiServiceProvider, templateEngine, memory, httpHandlerFactory, loggerFactory); // ========================================================================================================== // The kernel builder purpose is to simplify this process, automating how dependencies // are connected, still allowing to customize parts of the composition. - // Example: how to use a custom memory and configure Azure OpenAI - var kernel4 = Kernel.Builder - .WithLogger(NullLogger.Instance) - .WithMemory(memory) - .WithAzureTextCompletionService( - deploymentName: azureOpenAITextCompletionDeployment, - endpoint: azureOpenAIEndpoint, - apiKey: azureOpenAIKey) - .Build(); - - // Example: how to use a custom memory storage - var kernel6 = Kernel.Builder - .WithLogger(NullLogger.Instance) - .WithMemoryStorage(memoryStorage) // Custom memory storage - .WithAzureTextCompletionService( - deploymentName: azureOpenAITextCompletionDeployment, - endpoint: azureOpenAIEndpoint, - apiKey: azureOpenAIKey) // This will be used when using AI completions - .WithAzureTextEmbeddingGenerationService( - deploymentName: azureOpenAIEmbeddingDeployment, - endpoint: azureOpenAIEndpoint, - apiKey: azureOpenAIKey) // This will be used when indexing memory records - .Build(); - // ========================================================================================================== // The AI services are defined with the builder var kernel7 = Kernel.Builder - .WithAzureTextCompletionService( - deploymentName: azureOpenAITextCompletionDeployment, + .WithAzureChatCompletionService( + deploymentName: azureOpenAIChatCompletionDeployment, endpoint: azureOpenAIEndpoint, apiKey: azureOpenAIKey, setAsDefault: true) @@ -139,8 +121,8 @@ public static Task RunAsync() // The default behavior can be configured or a custom retry handler can be injected that will apply to all // AI requests (when using the kernel). - var kernel8 = Kernel.Builder - .Configure(c => c.SetDefaultHttpRetryConfig(new HttpRetryConfig + var kernel8 = Kernel.Builder.WithRetryBasic( + new BasicRetryConfig { MaxRetryCount = 3, UseExponentialBackoff = true, @@ -149,58 +131,73 @@ public static Task RunAsync() // MaxTotalRetryTime = TimeSpan.FromSeconds(30), // RetryableStatusCodes = new[] { HttpStatusCode.TooManyRequests, HttpStatusCode.RequestTimeout }, // RetryableExceptions = new[] { typeof(HttpRequestException) } - })) + }) .Build(); - var kernel9 = Kernel.Builder - .Configure(c => c.SetHttpRetryHandlerFactory(new NullHttpRetryHandlerFactory())) - .Build(); + var logger = loggerFactory.CreateLogger(); + var retryThreeTimesPolicy = Policy + .Handle(ex + => ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + .WaitAndRetryAsync(new[] + { + TimeSpan.FromSeconds(2), + TimeSpan.FromSeconds(4), + TimeSpan.FromSeconds(8) + }, + (ex, timespan, retryCount, _) + => logger?.LogWarning(ex, "Error executing action [attempt {RetryCount} of 3], pausing {PausingMilliseconds}ms", retryCount, timespan.TotalMilliseconds)); - var kernel10 = Kernel.Builder.WithRetryHandlerFactory(new RetryThreeTimesFactory()).Build(); + var kernel9 = Kernel.Builder.WithHttpHandlerFactory(new PollyHttpRetryHandlerFactory(retryThreeTimesPolicy)).Build(); + + var kernel10 = Kernel.Builder.WithHttpHandlerFactory(new PollyRetryThreeTimesFactory()).Build(); + + var kernel11 = Kernel.Builder.WithHttpHandlerFactory(new MyCustomHandlerFactory()).Build(); return Task.CompletedTask; } - // Example of a basic custom retry handler - public class RetryThreeTimesFactory : IDelegatingHandlerFactory + // Example using the PollyHttpRetryHandler from Reliability.Polly extension + public class PollyRetryThreeTimesFactory : HttpHandlerFactory { - public DelegatingHandler Create(ILogger? logger) + public override DelegatingHandler Create(ILoggerFactory? loggerFactory = null) { - return new RetryThreeTimes(logger); - } - } - - public class RetryThreeTimes : DelegatingHandler - { - private readonly AsyncRetryPolicy _policy; + var logger = loggerFactory?.CreateLogger(); - public RetryThreeTimes(ILogger? logger = null) - { - this._policy = GetPolicy(logger ?? NullLogger.Instance); + Activator.CreateInstance(typeof(PollyHttpRetryHandler), GetPolicy(logger), logger); + return base.Create(loggerFactory); } - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + private static AsyncRetryPolicy GetPolicy(ILogger? logger) { - return await this._policy.ExecuteAsync(async () => - { - var response = await base.SendAsync(request, cancellationToken); - return response; - }); + return Policy + .Handle(ex + => ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + .WaitAndRetryAsync(new[] + { + TimeSpan.FromSeconds(2), + TimeSpan.FromSeconds(4), + TimeSpan.FromSeconds(8) + }, + (ex, timespan, retryCount, _) + => logger?.LogWarning(ex, "Error executing action [attempt {RetryCount} of 3], pausing {PausingMilliseconds}ms", + retryCount, + timespan.TotalMilliseconds)); } + } - private static AsyncRetryPolicy GetPolicy(ILogger logger) + // Basic custom retry handler factory + public class MyCustomHandlerFactory : HttpHandlerFactory + { + } + + // Basic custom empty retry handler + public class MyCustomHandler : DelegatingHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - return Policy - .Handle(ex => ex.ErrorCode == AIException.ErrorCodes.Throttling) - .WaitAndRetryAsync(new[] - { - TimeSpan.FromSeconds(2), - TimeSpan.FromSeconds(4), - TimeSpan.FromSeconds(8) - }, - (ex, timespan, retryCount, _) => logger.LogWarning(ex, - "Error executing action [attempt {0} of 3], pausing {1}ms", - retryCount, timespan.TotalMilliseconds)); + // Your custom handler implementation + + throw new NotImplementedException(); } } } diff --git a/dotnet/samples/KernelSyntaxExamples/Example43_GetModelResult.cs b/dotnet/samples/KernelSyntaxExamples/Example43_GetModelResult.cs index 3942d7564c0c..301662b648a6 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example43_GetModelResult.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example43_GetModelResult.cs @@ -4,13 +4,17 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Azure; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; using RepoUtils; +#pragma warning disable RCS1192 // (Unnecessary usage of verbatim string literal) + // ReSharper disable once InconsistentNaming public static class Example43_GetModelResult { @@ -19,37 +23,31 @@ public static async Task RunAsync() Console.WriteLine("======== Inline Function Definition + Result ========"); IKernel kernel = new KernelBuilder() - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, + .WithOpenAIChatCompletionService( + modelId: TestConfiguration.OpenAI.ChatModelId, apiKey: TestConfiguration.OpenAI.ApiKey) .Build(); // Function defined using few-shot design pattern - const string FunctionDefinition = @" -Generate a creative reason or excuse for the given event. -Be creative and be funny. Let your imagination run wild. - -Event: I am running late. -Excuse: I was being held ransom by giraffe gangsters. - -Event: I haven't been to the gym for a year -Excuse: I've been too busy training my pet dragon. - -Event: {{$input}} -"; + const string FunctionDefinition = "Hi, give me 5 book suggestions about: {{$input}}"; - var excuseFunction = kernel.CreateSemanticFunction(FunctionDefinition, maxTokens: 100, temperature: 0.4, topP: 1); + var myFunction = kernel.CreateSemanticFunction(FunctionDefinition); // Using InvokeAsync with 3 results (Currently invoke only supports 1 result, but you can get the other results from the ModelResults) - var textResult = await excuseFunction.InvokeAsync("I missed the F1 final race", new CompleteRequestSettings { ResultsPerPrompt = 3 }); - Console.WriteLine(textResult); - Console.WriteLine(textResult.ModelResults.Select(result => result.GetOpenAITextResult()).AsJson()); + var functionResult = await myFunction.InvokeAsync("Sci-fi", + kernel, + requestSettings: new OpenAIRequestSettings { ResultsPerPrompt = 3, MaxTokens = 500, Temperature = 1, TopP = 0.5 }); + + Console.WriteLine(functionResult.GetValue()); + Console.WriteLine(functionResult.GetModelResults()?.Select(result => result.GetOpenAIChatResult()).AsJson()); Console.WriteLine(); // Using the Kernel RunAsync - textResult = await kernel.RunAsync("sorry I forgot your birthday", excuseFunction); - Console.WriteLine(textResult); - Console.WriteLine(textResult.ModelResults.LastOrDefault()?.GetOpenAITextResult()?.Usage.AsJson()); + var kernelResult = await kernel.RunAsync("sorry I forgot your birthday", myFunction); + var modelResults = kernelResult.FunctionResults.SelectMany(l => l.GetModelResults() ?? Enumerable.Empty()); + + Console.WriteLine(kernelResult.GetValue()); + Console.WriteLine(modelResults.LastOrDefault()?.GetOpenAIChatResult()?.Usage.AsJson()); Console.WriteLine(); // Using Chat Completion directly @@ -58,7 +56,7 @@ Be creative and be funny. Let your imagination run wild. apiKey: TestConfiguration.OpenAI.ApiKey); var prompt = FunctionDefinition.Replace("{{$input}}", $"Translate this date {DateTimeOffset.Now:f} to French format", StringComparison.InvariantCultureIgnoreCase); - IReadOnlyList completionResults = await chatCompletion.GetCompletionsAsync(prompt, new CompleteRequestSettings() { MaxTokens = 100, Temperature = 0.4, TopP = 1 }); + IReadOnlyList completionResults = await chatCompletion.GetCompletionsAsync(prompt, new OpenAIRequestSettings() { MaxTokens = 500, Temperature = 1, TopP = 0.5 }); Console.WriteLine(await completionResults[0].GetCompletionAsync()); Console.WriteLine(completionResults[0].ModelResult.GetOpenAIChatResult().Usage.AsJson()); @@ -66,23 +64,27 @@ Be creative and be funny. Let your imagination run wild. // Getting the error details kernel = new KernelBuilder() - .WithOpenAITextCompletionService("text-davinci-003", "Invalid Key") + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, "Invalid Key") .Build(); var errorFunction = kernel.CreateSemanticFunction(FunctionDefinition); - var failedContext = await kernel.RunAsync("sorry I forgot your birthday", errorFunction); - if (failedContext.ErrorOccurred) +#pragma warning disable CA1031 // Do not catch general exception types + try + { + await kernel.RunAsync("sorry I forgot your birthday", errorFunction); + } + catch (Exception ex) { - Console.WriteLine(OutputExceptionDetail(failedContext.LastException?.InnerException)); + Console.WriteLine(OutputExceptionDetail(ex)); } +#pragma warning restore CA1031 // Do not catch general exception types string OutputExceptionDetail(Exception? exception) { return exception switch { - RequestFailedException requestException => new { requestException.Status, requestException.Message }.AsJson(), - AIException aiException => new { ErrorCode = aiException.ErrorCode.ToString(), aiException.Message, aiException.Detail }.AsJson(), - { } e => new { e.Message }.AsJson(), + HttpOperationException httpException => new { StatusCode = httpException.StatusCode?.ToString(), Message = httpException.Message, Response = httpException.ResponseContent }.AsJson(), + { } e => e.Message, _ => string.Empty }; } diff --git a/dotnet/samples/KernelSyntaxExamples/Example44_MultiChatCompletion.cs b/dotnet/samples/KernelSyntaxExamples/Example44_MultiChatCompletion.cs index 8494ebac5613..f31482ebb601 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example44_MultiChatCompletion.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example44_MultiChatCompletion.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; /** @@ -47,7 +48,7 @@ private static async Task RunChatAsync(IChatCompletion chatCompletion) chatHistory.AddUserMessage("Hi, I'm looking for book 3 different book suggestions about sci-fi"); await MessageOutputAsync(chatHistory); - var chatRequestSettings = new ChatRequestSettings + var chatRequestSettings = new OpenAIRequestSettings() { MaxTokens = 1024, ResultsPerPrompt = 2, diff --git a/dotnet/samples/KernelSyntaxExamples/Example45_MultiStreamingChatCompletion.cs b/dotnet/samples/KernelSyntaxExamples/Example45_MultiStreamingChatCompletion.cs index 58de8c812cfd..c9c461123191 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example45_MultiStreamingChatCompletion.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example45_MultiStreamingChatCompletion.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; /** @@ -47,7 +48,7 @@ private static async Task OpenAIMultiStreamingChatCompletionAsync() private static async Task StreamingChatCompletionAsync(IChatCompletion chatCompletion) { - var requestSettings = new ChatRequestSettings() + var requestSettings = new OpenAIRequestSettings() { MaxTokens = 200, FrequencyPenalty = 0, diff --git a/dotnet/samples/KernelSyntaxExamples/Example46_Weaviate.cs b/dotnet/samples/KernelSyntaxExamples/Example46_Weaviate.cs deleted file mode 100644 index b0f7bc8c1d10..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example46_Weaviate.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.Memory.Weaviate; -using Microsoft.SemanticKernel.Memory; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -public static class Example46_Weaviate -{ - private const string MemoryCollectionName = "weaviate-test"; - - public static async Task RunAsync() - { - string endpoint = TestConfiguration.Weaviate.Endpoint; - string apiKey = TestConfiguration.Weaviate.ApiKey; - WeaviateMemoryStore memoryStore = new(endpoint, apiKey, ConsoleLogger.Logger); - IKernel kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .WithOpenAITextEmbeddingGenerationService( - modelId: TestConfiguration.OpenAI.EmbeddingModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(memoryStore) - //.WithWeaviateMemoryStore(endpoint, apiKey) // This method offers an alternative approach to registering Weaviate memory store. - .Build(); - - Console.WriteLine("== Printing Collections in DB =="); - var collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Adding Memories =="); - - var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: Guid.NewGuid().ToString(), text: "british short hair"); - var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: Guid.NewGuid().ToString(), text: "orange tabby"); - var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: Guid.NewGuid().ToString(), text: "norwegian forest cat"); - - Console.WriteLine("== Printing Collections in DB =="); - collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Retrieving Memories Through the Kernel =="); - MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, key1); - Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Similarity Searching Memories: My favorite color is orange =="); - var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8); - - await foreach (var item in searchResults) - { - Console.WriteLine(item.Metadata.Text + " : " + item.Relevance); - } - - Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName); - await memoryStore.DeleteCollectionAsync(MemoryCollectionName); - - Console.WriteLine("== Printing Collections in DB =="); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example47_Redis.cs b/dotnet/samples/KernelSyntaxExamples/Example47_Redis.cs deleted file mode 100644 index b89e8c3a5b2d..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example47_Redis.cs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.Memory.Redis; -using Microsoft.SemanticKernel.Memory; -using RepoUtils; -using StackExchange.Redis; - -// ReSharper disable once InconsistentNaming -public static class Example47_Redis -{ - private const string MemoryCollectionName = "redis-test"; - - public static async Task RunAsync() - { - string configuration = TestConfiguration.Redis.Configuration; - await using ConnectionMultiplexer connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync(configuration); - IDatabase database = connectionMultiplexer.GetDatabase(); - RedisMemoryStore memoryStore = new(database, vectorSize: 1536); - IKernel kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .WithOpenAITextEmbeddingGenerationService( - modelId: TestConfiguration.OpenAI.EmbeddingModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(memoryStore) - .Build(); - - Console.WriteLine("== Printing Collections in DB =="); - var collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Adding Memories =="); - - var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat1", text: "british short hair"); - var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat2", text: "orange tabby"); - var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat3", text: "norwegian forest cat"); - - Console.WriteLine("== Printing Collections in DB =="); - collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Retrieving Memories Through the Kernel =="); - MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1"); - Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Retrieving Memories Directly From the Store =="); - var memory1 = await memoryStore.GetAsync(MemoryCollectionName, key1); - var memory2 = await memoryStore.GetAsync(MemoryCollectionName, key2); - var memory3 = await memoryStore.GetAsync(MemoryCollectionName, key3); - Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found"); - Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found"); - Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Similarity Searching Memories: My favorite color is orange =="); - var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8); - - await foreach (var item in searchResults) - { - Console.WriteLine(item.Metadata.Text + " : " + item.Relevance); - } - - Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName); - await memoryStore.DeleteCollectionAsync(MemoryCollectionName); - - Console.WriteLine("== Printing Collections in DB =="); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example48_GroundednessChecks.cs b/dotnet/samples/KernelSyntaxExamples/Example48_GroundednessChecks.cs index ab07fef02335..80603a5d2fd1 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example48_GroundednessChecks.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example48_GroundednessChecks.cs @@ -3,16 +3,15 @@ using System; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.Planning.Sequential; -using Microsoft.SemanticKernel.Skills.Core; +using Microsoft.SemanticKernel.Planners; +using Microsoft.SemanticKernel.Plugins.Core; using RepoUtils; // ReSharper disable CommentTypo // ReSharper disable once InconsistentNaming internal static class Example48_GroundednessChecks { - private static string s_groundingText = @"""I am by birth a Genevese, and my family is one of the most distinguished of that republic. + private const string GroundingText = @"""I am by birth a Genevese, and my family is one of the most distinguished of that republic. My ancestors had been for many years counsellors and syndics, and my father had filled several public situations with honour and reputation.He was respected by all who knew him for his integrity and indefatigable attention to public business.He passed his younger days perpetually occupied by the affairs of his country; a variety @@ -51,25 +50,25 @@ interment of his friend he conducted her to Geneva and placed her under the prot public static async Task RunAsync() { - await GroundednessCheckingSkill(); - await PlanningWithGroundedness(); + await GroundednessCheckingAsync(); + await PlanningWithGroundednessAsync(); } - public static async Task GroundednessCheckingSkill() + public static async Task GroundednessCheckingAsync() { Console.WriteLine("======== Groundedness Checks ========"); var kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithAzureTextCompletionService( - TestConfiguration.AzureOpenAI.DeploymentName, + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) .Build(); - string folder = RepoFiles.SampleSkillsPath(); - var functions = kernel.ImportSemanticSkillFromDirectory(folder, - "SummarizeSkill", - "GroundingSkill"); + string folder = RepoFiles.SamplePluginsPath(); + var functions = kernel.ImportSemanticFunctionsFromDirectory(folder, + "SummarizePlugin", + "GroundingPlugin"); var create_summary = functions["Summarize"]; var entityExtraction = functions["ExtractEntities"]; @@ -89,28 +88,28 @@ her a beggar. My father came to her aid and two years later they married. context.Variables.Set("topic", "people and places"); context.Variables.Set("example_entities", "John, Jane, mother, brother, Paris, Rome"); - var extractionResult = (await entityExtraction.InvokeAsync(context)).Result; + var extractionResult = (await kernel.RunAsync(context.Variables, entityExtraction)).GetValue(); Console.WriteLine("======== Extract Entities ========"); Console.WriteLine(extractionResult); context.Variables.Update(extractionResult); - context.Variables.Set("reference_context", s_groundingText); + context.Variables.Set("reference_context", GroundingText); - var groundingResult = (await reference_check.InvokeAsync(context)).Result; + var groundingResult = (await kernel.RunAsync(context.Variables, reference_check)).GetValue(); Console.WriteLine("======== Reference Check ========"); Console.WriteLine(groundingResult); context.Variables.Update(summaryText); context.Variables.Set("ungrounded_entities", groundingResult); - var excisionResult = await entity_excision.InvokeAsync(context); + var excisionResult = await kernel.RunAsync(context.Variables, entity_excision); Console.WriteLine("======== Excise Entities ========"); - Console.WriteLine(excisionResult.Result); + Console.WriteLine(excisionResult.GetValue()); } - public static async Task PlanningWithGroundedness() + public static async Task PlanningWithGroundednessAsync() { var targetTopic = "people and places"; var samples = "John, Jane, mother, brother, Paris, Rome"; @@ -124,27 +123,26 @@ which are not grounded in the original. Console.WriteLine("======== Planning - Groundedness Checks ========"); var kernel = new KernelBuilder() - .WithLogger(ConsoleLogger.Logger) - .WithAzureTextCompletionService( - TestConfiguration.AzureOpenAI.DeploymentName, + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey) .Build(); - string folder = RepoFiles.SampleSkillsPath(); - var functions = kernel.ImportSemanticSkillFromDirectory(folder, - "SummarizeSkill", - "GroundingSkill"); + string folder = RepoFiles.SamplePluginsPath(); + var functions = kernel.ImportSemanticFunctionsFromDirectory(folder, + "SummarizePlugin", + "GroundingPlugin"); - kernel.ImportSkill(new TextSkill()); + kernel.ImportFunctions(new TextPlugin()); - var plannerConfig = new SequentialPlannerConfig { }; - var planner = new SequentialPlanner(kernel, plannerConfig); + var planner = new SequentialPlanner(kernel); var plan = await planner.CreatePlanAsync(ask); Console.WriteLine(plan.ToPlanWithGoalString()); - var results = await plan.InvokeAsync(s_groundingText); - Console.WriteLine(results.Result); + var results = await kernel.RunAsync(GroundingText, plan); + Console.WriteLine(results.GetValue()); } } @@ -176,18 +174,18 @@ related to people and places (such as John, Jane, mother, brother, Paris, Rome) grounded in the original input text. Finally, rewrite your summary to remove the entities which are not grounded in the original. - + Steps: - _GLOBAL_FUNCTIONS_.Echo INPUT='' => ORIGINAL_TEXT - - SummarizeSkill.Summarize INPUT='' => RESULT__SUMMARY - - GroundingSkill.ExtractEntities example_entities='John;Jane;mother;brother;Paris;Rome' topic='people and places' INPUT='$RESULT__SUMMARY' => ENTITIES - - GroundingSkill.ReferenceCheckEntities reference_context='$ORIGINAL_TEXT' INPUT='$ENTITIES' => RESULT__UNGROUND_ENTITIES - - GroundingSkill.ExciseEntities ungrounded_entities='$RESULT__UNGROUND_ENTITIES' INPUT='$RESULT__SUMMARY' => RESULT__FINAL_SUMMARY + - SummarizePlugin.Summarize INPUT='' => RESULT__SUMMARY + - GroundingPlugin.ExtractEntities example_entities='John;Jane;mother;brother;Paris;Rome' topic='people and places' INPUT='$RESULT__SUMMARY' => ENTITIES + - GroundingPlugin.ReferenceCheckEntities reference_context='$ORIGINAL_TEXT' INPUT='$ENTITIES' => RESULT__UNGROUND_ENTITIES + - GroundingPlugin.ExciseEntities ungrounded_entities='$RESULT__UNGROUND_ENTITIES' INPUT='$RESULT__SUMMARY' => RESULT__FINAL_SUMMARY A possible summary is: - + The narrator's father, a respected Genevese politician, befriended Beaufort, a merchant who fell into poverty and hid in Lucerne. After a long search, he found him dying and his daughter Caroline working hard to survive. He took pity on Caroline, buried Beaufort, and married her two years later. @@ -195,7 +193,7 @@ which are not grounded in the original. A possible summary is: - + The father of the story's main character, a respected Genevese politician, befriended Beaufort, a merchant who fell into poverty and hid in Lucerne. After a long search, he found him dying and his daughter Caroline working hard to survive. He took pity on Caroline, buried Beaufort, and married her two years later. == DONE == diff --git a/dotnet/samples/KernelSyntaxExamples/Example49_LogitBias.cs b/dotnet/samples/KernelSyntaxExamples/Example49_LogitBias.cs index 80fff5937296..14c7ee12446f 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example49_LogitBias.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example49_LogitBias.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; /** @@ -15,7 +16,7 @@ public static class Example49_LogitBias { public static async Task RunAsync() { - OpenAIChatCompletion chatCompletion = new("gpt-3.5-turbo", TestConfiguration.OpenAI.ApiKey); + OpenAIChatCompletion chatCompletion = new(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey); // To use Logit Bias you will need to know the token ids of the words you want to use. // Getting the token ids using the GPT Tokenizer: https://platform.openai.com/tokenizer @@ -24,7 +25,7 @@ public static async Task RunAsync() // "novel literature reading author library story chapter paperback hardcover ebook publishing fiction nonfiction manuscript textbook bestseller bookstore reading list bookworm" var keys = new[] { 3919, 626, 17201, 1300, 25782, 9800, 32016, 13571, 43582, 20189, 1891, 10424, 9631, 16497, 12984, 20020, 24046, 13159, 805, 15817, 5239, 2070, 13466, 32932, 8095, 1351, 25323 }; - var settings = new ChatRequestSettings(); + var settings = new OpenAIRequestSettings(); // This will make the model try its best to avoid any of the above related words. foreach (var key in keys) diff --git a/dotnet/samples/KernelSyntaxExamples/Example50_Chroma.cs b/dotnet/samples/KernelSyntaxExamples/Example50_Chroma.cs deleted file mode 100644 index 66bf645ae101..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Example50_Chroma.cs +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.Memory.Chroma; -using Microsoft.SemanticKernel.Memory; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -public static class Example50_Chroma -{ - private const string MemoryCollectionName = "chroma-test"; - - public static async Task RunAsync() - { - string endpoint = TestConfiguration.Chroma.Endpoint; - - var memoryStore = new ChromaMemoryStore(endpoint); - - IKernel kernel = Kernel.Builder - .WithLogger(ConsoleLogger.Logger) - .WithOpenAITextCompletionService( - modelId: TestConfiguration.OpenAI.ModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .WithOpenAITextEmbeddingGenerationService( - modelId: TestConfiguration.OpenAI.EmbeddingModelId, - apiKey: TestConfiguration.OpenAI.ApiKey) - .WithMemoryStorage(memoryStore) - //.WithChromaMemoryStore(endpoint) // This method offers an alternative approach to registering Chroma memory store. - .Build(); - - Console.WriteLine("== Printing Collections in DB =="); - var collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Adding Memories =="); - - var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: Guid.NewGuid().ToString(), text: "british short hair"); - var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: Guid.NewGuid().ToString(), text: "orange tabby"); - var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: Guid.NewGuid().ToString(), text: "norwegian forest cat"); - - Console.WriteLine("== Printing Collections in DB =="); - collections = memoryStore.GetCollectionsAsync(); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - - Console.WriteLine("== Retrieving Memories Through the Kernel =="); - MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, key1); - Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found"); - - Console.WriteLine("== Similarity Searching Memories: My favorite color is orange =="); - var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.6); - - await foreach (var item in searchResults) - { - Console.WriteLine(item.Metadata.Text + " : " + item.Relevance); - } - - Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName); - await memoryStore.DeleteCollectionAsync(MemoryCollectionName); - - Console.WriteLine("== Printing Collections in DB =="); - await foreach (var collection in collections) - { - Console.WriteLine(collection); - } - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs b/dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs index 60c98f160fe9..c4e3b199e8bf 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs @@ -1,189 +1,240 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.Reliability; -using Microsoft.SemanticKernel.Skills.Core; -using Microsoft.SemanticKernel.Skills.Web; -using Microsoft.SemanticKernel.Skills.Web.Bing; -using NCalcSkills; +using Microsoft.SemanticKernel.Planners; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.Plugins.Web; +using Microsoft.SemanticKernel.Plugins.Web.Bing; +using NCalcPlugins; using RepoUtils; /** - * This example shows how to use Stepwise Planner to create a plan for a given goal. + * This example shows how to use Stepwise Planner to create and run a stepwise plan for a given goal. */ - // ReSharper disable once InconsistentNaming public static class Example51_StepwisePlanner { + // Used to override the max allowed tokens when running the plan + internal static int? ChatMaxTokens = null; + internal static int? TextMaxTokens = null; + + // Used to quickly modify the chat model used by the planner + internal static string? ChatModelOverride = null; //"gpt-35-turbo"; + internal static string? TextModelOverride = null; //"text-davinci-003"; + + internal static string? Suffix = null; + public static async Task RunAsync() { string[] questions = new string[] { - "Who is the current president of the United States? What is his current age divided by 2", - // "Who is Leo DiCaprio's girlfriend? What is her current age raised to the (his current age)/100 power?", - // "What is the capital of France? Who is that cities current mayor? What percentage of their life has been in the 21st century as of today?", - // "What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle?" + "What color is the sky?", + "What is the weather in Seattle?", + "What is the tallest mountain on Earth? How tall is it divided by 2?", + "What is the capital of France? Who is that city's current mayor? What percentage of their life has been in the 21st century as of today?", + "What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle?", + "If a spacecraft travels at 0.99 the speed of light and embarks on a journey to the nearest star system, Alpha Centauri, which is approximately 4.37 light-years away, how much time would pass on Earth during the spacecraft's voyage?" }; foreach (var question in questions) { - await RunTextCompletion(question); - await RunChatCompletion(question); + for (int i = 0; i < 1; i++) + { + await RunTextCompletionAsync(question); + await RunChatCompletionAsync(question); + } + } + + PrintResults(); + } + + // print out summary table of ExecutionResults + private static void PrintResults() + { + Console.WriteLine("**************************"); + Console.WriteLine("Execution Results Summary:"); + Console.WriteLine("**************************"); + + foreach (var question in s_executionResults.Select(s => s.question).Distinct()) + { + Console.WriteLine("Question: " + question); + Console.WriteLine("Mode\tModel\tAnswer\tStepsTaken\tIterations\tTimeTaken"); + foreach (var er in s_executionResults.OrderByDescending(s => s.model).Where(s => s.question == question)) + { + Console.WriteLine($"{er.mode}\t{er.model}\t{er.stepsTaken}\t{er.iterations}\t{er.timeTaken}\t{er.answer}"); + } } } - private static async Task RunTextCompletion(string question) + private struct ExecutionResult + { + public string mode; + public string? model; + public string? question; + public string? answer; + public string? stepsTaken; + public string? iterations; + public string? timeTaken; + } + + private static readonly List s_executionResults = new(); + + private static async Task RunTextCompletionAsync(string question) { Console.WriteLine("RunTextCompletion"); - var kernel = GetKernel(); - await RunWithQuestion(kernel, question); + ExecutionResult currentExecutionResult = default; + currentExecutionResult.mode = "RunTextCompletion"; + var kernel = GetKernel(ref currentExecutionResult); + await RunWithQuestionAsync(kernel, currentExecutionResult, question, TextMaxTokens); } - private static async Task RunChatCompletion(string question) + private static async Task RunChatCompletionAsync(string question, string? model = null) { Console.WriteLine("RunChatCompletion"); - var kernel = GetKernel(true); - await RunWithQuestion(kernel, question); + ExecutionResult currentExecutionResult = default; + currentExecutionResult.mode = "RunChatCompletion"; + var kernel = GetKernel(ref currentExecutionResult, true, model); + await RunWithQuestionAsync(kernel, currentExecutionResult, question, ChatMaxTokens); } - private static async Task RunWithQuestion(IKernel kernel, string question) + private static async Task RunWithQuestionAsync(IKernel kernel, ExecutionResult currentExecutionResult, string question, int? MaxTokens = null) { + currentExecutionResult.question = question; var bingConnector = new BingConnector(TestConfiguration.Bing.ApiKey); - var webSearchEngineSkill = new WebSearchEngineSkill(bingConnector); - - kernel.ImportSkill(webSearchEngineSkill, "WebSearch"); - kernel.ImportSkill(new LanguageCalculatorSkill(kernel), "advancedCalculator"); - // kernel.ImportSkill(new SimpleCalculatorSkill(kernel), "basicCalculator"); - kernel.ImportSkill(new TimeSkill(), "time"); + var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector); + + kernel.ImportFunctions(webSearchEnginePlugin, "WebSearch"); + kernel.ImportFunctions(new LanguageCalculatorPlugin(kernel), "semanticCalculator"); + kernel.ImportFunctions(new TimePlugin(), "time"); + + // StepwisePlanner is instructed to depend on available functions. + // We expose this function to increase the flexibility in it's ability to answer + // given the relatively small number of functions we have in this example. + // This seems to be particularly helpful in these examples with gpt-35-turbo -- even though it + // does not *use* this function. It seems to help the planner find a better path to the answer. + kernel.CreateSemanticFunction( + "Generate an answer for the following question: {{$input}}", + functionName: "GetAnswerForQuestion", + pluginName: "AnswerBot", + description: "Given a question, get an answer and return it as the result of the function"); Console.WriteLine("*****************************************************"); Stopwatch sw = new(); Console.WriteLine("Question: " + question); - var plannerConfig = new Microsoft.SemanticKernel.Planning.Stepwise.StepwisePlannerConfig(); + var plannerConfig = new Microsoft.SemanticKernel.Planners.StepwisePlannerConfig(); plannerConfig.ExcludedFunctions.Add("TranslateMathProblem"); + plannerConfig.ExcludedFunctions.Add("DaysAgo"); + plannerConfig.ExcludedFunctions.Add("DateMatchingLastDayName"); plannerConfig.MinIterationTimeMs = 1500; - plannerConfig.MaxTokens = 4000; + plannerConfig.MaxIterations = 25; - StepwisePlanner planner = new(kernel, plannerConfig); - sw.Start(); - var plan = planner.CreatePlan(question); + if (!string.IsNullOrEmpty(Suffix)) + { + plannerConfig.Suffix = $"{Suffix}\n{plannerConfig.Suffix}"; + currentExecutionResult.question = $"[Assisted] - {question}"; + } - var result = await plan.InvokeAsync(kernel.CreateNewContext()); - Console.WriteLine("Result: " + result); - if (result.Variables.TryGetValue("stepCount", out string? stepCount)) + if (MaxTokens.HasValue) { - Console.WriteLine("Steps Taken: " + stepCount); + plannerConfig.MaxTokens = MaxTokens.Value; } - if (result.Variables.TryGetValue("skillCount", out string? skillCount)) + sw.Start(); + + try { - Console.WriteLine("Skills Used: " + skillCount); + StepwisePlanner planner = new(kernel: kernel, config: plannerConfig); + var plan = planner.CreatePlan(question); + + var kernelResult = await kernel.RunAsync(plan); + var planResult = kernelResult.FunctionResults.First(); + var result = kernelResult.GetValue()!; + + if (result.Contains("Result not found, review _stepsTaken to see what", StringComparison.OrdinalIgnoreCase)) + { + Console.WriteLine("Could not answer question in " + plannerConfig.MaxIterations + " iterations"); + currentExecutionResult.answer = "Could not answer question in " + plannerConfig.MaxIterations + " iterations"; + } + else + { + Console.WriteLine("Result: " + result); + currentExecutionResult.answer = result; + } + + if (planResult.TryGetMetadataValue("stepCount", out string stepCount)) + { + Console.WriteLine("Steps Taken: " + stepCount); + currentExecutionResult.stepsTaken = stepCount; + } + + if (planResult.TryGetMetadataValue("functionCount", out string functionCount)) + { + Console.WriteLine("Functions Used: " + functionCount); + } + + if (planResult.TryGetMetadataValue("iterations", out string iterations)) + { + Console.WriteLine("Iterations: " + iterations); + currentExecutionResult.iterations = iterations; + } + } +#pragma warning disable CA1031 + catch (Exception ex) + { + Console.WriteLine("Exception: " + ex); } Console.WriteLine("Time Taken: " + sw.Elapsed); + currentExecutionResult.timeTaken = sw.Elapsed.ToString(); + s_executionResults.Add(currentExecutionResult); Console.WriteLine("*****************************************************"); } - private static IKernel GetKernel(bool useChat = false) + private static IKernel GetKernel(ref ExecutionResult result, bool useChat = false, string? model = null) { var builder = new KernelBuilder(); + var maxTokens = 0; if (useChat) { builder.WithAzureChatCompletionService( - TestConfiguration.AzureOpenAI.ChatDeploymentName, + model ?? ChatModelOverride ?? TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey, alsoAsTextCompletion: true, setAsDefault: true); + + maxTokens = ChatMaxTokens ?? (new Microsoft.SemanticKernel.Planners.StepwisePlannerConfig()).MaxTokens; + result.model = model ?? ChatModelOverride ?? TestConfiguration.AzureOpenAI.ChatDeploymentName; } else { builder.WithAzureTextCompletionService( - TestConfiguration.AzureOpenAI.DeploymentName, + model ?? TextModelOverride ?? TestConfiguration.AzureOpenAI.DeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey); + + maxTokens = TextMaxTokens ?? (new Microsoft.SemanticKernel.Planners.StepwisePlannerConfig()).MaxTokens; + result.model = model ?? TextModelOverride ?? TestConfiguration.AzureOpenAI.DeploymentName; } + Console.WriteLine($"Model: {result.model} ({maxTokens})"); + var kernel = builder - .WithLogger(ConsoleLogger.Logger) - .Configure(c => c.SetDefaultHttpRetryConfig(new HttpRetryConfig + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithRetryBasic(new() { MaxRetryCount = 3, UseExponentialBackoff = true, MinRetryDelay = TimeSpan.FromSeconds(3), - })) + }) .Build(); return kernel; } } - -// RunTextCompletion -// ***************************************************** -// Question: Who is the current president of the United States? What is his current age divided by 2 -// Result: The current president of the United States is Joe Biden. His current age divided by 2 is 40. -// Steps Taken: 10 -// Skills Used: 4 (WebSearch.Search(2), time.Date(1), advancedCalculator.Calculator(1)) -// Time Taken: 00:00:53.6331324 -// ***************************************************** -// RunChatCompletion -// ***************************************************** -// Question: Who is the current president of the United States? What is his current age divided by 2 -// Result: The current president of the United States is Joe Biden. His current age divided by 2 is 40.5. -// Steps Taken: 9 -// Skills Used: 7 (WebSearch.Search(4), time.Year(1), time.Date(1), advancedCalculator.Calculator(1)) -// Time Taken: 00:01:13.3766860 -// ***************************************************** -// RunTextCompletion -// ***************************************************** -// Question: Who is Leo DiCaprio's girlfriend? What is her current age raised to the (his current age)/100 power? -// Result: Leo DiCaprio's girlfriend is Camila Morrone. Her current age raised to the (his current age)/100 power is 4.935565735151678. -// Steps Taken: 6 -// Skills Used: 5 (WebSearch.Search(3), time.Year(1), advancedCalculator.Calculator(1)) -// Time Taken: 00:00:37.8941510 -// ***************************************************** -// RunChatCompletion -// ***************************************************** -// Question: Who is Leo DiCaprio's girlfriend? What is her current age raised to the (his current age)/100 power? -// Result: Leo DiCaprio's girlfriend is Camila Morrone. Her current age raised to the power of (his current age)/100 is approximately 4.94. -// Steps Taken: 9 -// Skills Used: 5 (WebSearch.Search(3), time.Year(1), advancedCalculator.Calculator(1)) -// Time Taken: 00:01:17.6742136 -// ***************************************************** -// RunTextCompletion -// ***************************************************** -// Question: What is the capital of France? Who is that cities current mayor? What percentage of their life has been in the 21st century as of today? -// Result: The capital of France is Paris. The current mayor of Paris is Anne Hidalgo. She has spent 36.51% of her life in the 21st century as of 2023. -// Steps Taken: 7 -// Skills Used: 4 (WebSearch.Search(3), advancedCalculator.Calculator(1)) -// Time Taken: 00:00:41.6837628 -// ***************************************************** -// RunChatCompletion -// ***************************************************** -// Question: What is the capital of France? Who is that cities current mayor? What percentage of their life has been in the 21st century as of today? -// Result: The capital of France is Paris. The current mayor of Paris is Anne Hidalgo, who was born on June 19, 1959. As of today, she has lived for 64 years, with 23 of those years in the 21st century. Therefore, 35.94% of her life has been spent in the 21st century. -// Steps Taken: 14 -// Skills Used: 12 (WebSearch.Search(8), time.Year(1), advancedCalculator.Calculator(3)) -// Time Taken: 00:02:06.6682909 -// ***************************************************** -// RunTextCompletion -// ***************************************************** -// Question: What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle? -// Result: The current day of the calendar year is 177. The angle in degrees corresponding to this day is 174.6. The area of a unit circle with that angle is 0.764 * pi. -// Steps Taken: 16 -// Skills Used: 2 (time.DayOfYear(1), time.Date(1)) -// Time Taken: 00:01:29.9931039 -// ***************************************************** -// RunChatCompletion -// ***************************************************** -// Question: What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle? -// Result: The current day of the year is 177. Using that as an angle in degrees (approximately 174.58), the area of a unit circle with that angle is approximately 1.523 square units. -// Steps Taken: 11 -// Skills Used: 9 (time.Now(1), time.DayOfYear(1), time.DaysBetween(1), time.MonthNumber(1), time.Day(1), advancedCalculator.Calculator(4)) -// Time Taken: 00:01:41.5585861 -// ***************************************************** diff --git a/dotnet/samples/KernelSyntaxExamples/Example52_ApimAuth.cs b/dotnet/samples/KernelSyntaxExamples/Example52_ApimAuth.cs index 86b38031e36c..cd5c47e6291b 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example52_ApimAuth.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example52_ApimAuth.cs @@ -10,6 +10,9 @@ using Azure.Identity; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; +using Microsoft.SemanticKernel.Diagnostics; using RepoUtils; // ReSharper disable once InconsistentNaming @@ -36,12 +39,14 @@ public static async Task RunAsync() // - Custom HttpClient with subscription key header // - Diagnostics to log error response headers from APIM to aid problem determination // - Authentication using BearerTokenCredential retrieved via interactive browser login - var clientOptions = new OpenAIClientOptions() + var clientOptions = new OpenAIClientOptions { Transport = new HttpClientTransport(httpClient), Diagnostics = { LoggedHeaderNames = { "ErrorSource", "ErrorReason", "ErrorMessage", "ErrorScope", "ErrorSection", "ErrorStatusCode" }, + ApplicationId = Telemetry.HttpUserAgent, + IsTelemetryEnabled = Telemetry.IsTelemetryEnabled, } }; var openAIClient = new OpenAIClient(apimUri, new BearerTokenCredential(accessToken), clientOptions); @@ -54,25 +59,25 @@ public static async Task RunAsync() .AddConsole(); }); - // Example: how to use a custom OpenAIClient and configure Azure OpenAI var kernel = Kernel.Builder - .WithLogger(loggerFactory.CreateLogger()) - .WithAzureTextCompletionService("text-davinci-003", openAIClient) + .WithLoggerFactory(loggerFactory) + .WithAIService(TestConfiguration.AzureOpenAI.ChatDeploymentName, (loggerFactory) => + new AzureChatCompletion(TestConfiguration.AzureOpenAI.ChatDeploymentName, openAIClient, loggerFactory)) .Build(); - // Load semantic skill defined with prompt templates - string folder = RepoFiles.SampleSkillsPath(); + // Load semantic plugin defined with prompt templates + string folder = RepoFiles.SamplePluginsPath(); - var funSkill = kernel.ImportSemanticSkillFromDirectory( + var funFunctions = kernel.ImportSemanticFunctionsFromDirectory( folder, - "FunSkill"); + "FunPlugin"); // Run var result = await kernel.RunAsync( "I have no homework", - funSkill["Excuses"] + funFunctions["Excuses"] ); - Console.WriteLine(result); + Console.WriteLine(result.GetValue()); httpClient.Dispose(); } diff --git a/dotnet/samples/KernelSyntaxExamples/Example54_AzureChatCompletionWithData.cs b/dotnet/samples/KernelSyntaxExamples/Example54_AzureChatCompletionWithData.cs new file mode 100644 index 000000000000..a28ef71f9874 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example54_AzureChatCompletionWithData.cs @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +/** + * This example shows how to use Azure OpenAI Chat Completion with data. + * More information: + */ +// ReSharper disable once InconsistentNaming +public static class Example54_AzureChatCompletionWithData +{ + public static async Task RunAsync() + { + // Uploaded content in Azure Blob Storage in .txt file: + + // Emily and David, two passionate scientists, met during a research expedition to Antarctica. + // Bonded by their love for the natural world and shared curiosity, + // they uncovered a groundbreaking phenomenon in glaciology that could + // potentially reshape our understanding of climate change. + + await ExampleWithChatCompletionAsync(); + await ExampleWithKernelAsync(); + } + + private static async Task ExampleWithChatCompletionAsync() + { + Console.WriteLine("=== Example with Chat Completion ==="); + + var chatCompletion = new AzureChatCompletionWithData(GetCompletionWithDataConfig()); + var chatHistory = chatCompletion.CreateNewChat(); + + // First question without previous context based on uploaded content. + var ask = "How did Emily and David meet?"; + chatHistory.AddUserMessage(ask); + + // Chat Completion example + var chatResult = (await chatCompletion.GetChatCompletionsAsync(chatHistory))[0]; + var chatMessage = await chatResult.GetChatMessageAsync(); + + var response = chatMessage.Content; + var toolResponse = chatResult.ModelResult.GetResult().ToolContent; + + // Output + // Ask: How did Emily and David meet? + // Response: Emily and David, both passionate scientists, met during a research expedition to Antarctica. + Console.WriteLine($"Ask: {ask}"); + Console.WriteLine($"Response: {response}"); + Console.WriteLine(); + + // Chat history maintenance + if (!string.IsNullOrEmpty(toolResponse)) + { + chatHistory.AddMessage(AuthorRole.Tool, toolResponse); + } + + chatHistory.AddAssistantMessage(response); + + // Second question based on uploaded content. + ask = "What are Emily and David studying?"; + chatHistory.AddUserMessage(ask); + + // Chat Completion Streaming example + Console.WriteLine($"Ask: {ask}"); + Console.WriteLine("Response: "); + + await foreach (var result in chatCompletion.GetStreamingChatCompletionsAsync(chatHistory)) + { + await foreach (var message in result.GetStreamingChatMessageAsync()) + { + // Output + // Ask: What are Emily and David studying? + // Response: They are passionate scientists who study glaciology, + // a branch of geology that deals with the study of ice and its effects. + Console.Write(message.Content); + } + } + + Console.WriteLine(Environment.NewLine); + } + + private static async Task ExampleWithKernelAsync() + { + Console.WriteLine("=== Example with Kernel ==="); + + var ask = "How did Emily and David meet?"; + + var completionWithDataConfig = GetCompletionWithDataConfig(); + + IKernel kernel = new KernelBuilder() + .WithAzureChatCompletionService(config: completionWithDataConfig) + .Build(); + + var semanticFunction = kernel.CreateSemanticFunction("Question: {{$input}}"); + + // First question without previous context based on uploaded content. + var response = await kernel.RunAsync(ask, semanticFunction); + + // Output + // Ask: How did Emily and David meet? + // Response: Emily and David, both passionate scientists, met during a research expedition to Antarctica. + Console.WriteLine($"Ask: {ask}"); + Console.WriteLine($"Response: {response.GetValue()}"); + Console.WriteLine(); + + // Second question based on uploaded content. + ask = "What are Emily and David studying?"; + response = await kernel.RunAsync(ask, semanticFunction); + + // Output + // Ask: What are Emily and David studying? + // Response: They are passionate scientists who study glaciology, + // a branch of geology that deals with the study of ice and its effects. + Console.WriteLine($"Ask: {ask}"); + Console.WriteLine($"Response: {response.GetValue()}"); + Console.WriteLine(); + } + + /// + /// Initializes a new instance of the class. + /// + private static AzureChatCompletionWithDataConfig GetCompletionWithDataConfig() + { + return new AzureChatCompletionWithDataConfig + { + CompletionModelId = TestConfiguration.AzureOpenAI.ChatDeploymentName, + CompletionEndpoint = TestConfiguration.AzureOpenAI.Endpoint, + CompletionApiKey = TestConfiguration.AzureOpenAI.ApiKey, + DataSourceEndpoint = TestConfiguration.ACS.Endpoint, + DataSourceApiKey = TestConfiguration.ACS.ApiKey, + DataSourceIndex = TestConfiguration.ACS.IndexName + }; + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example55_TextChunker.cs b/dotnet/samples/KernelSyntaxExamples/Example55_TextChunker.cs new file mode 100644 index 000000000000..a9a83359db7b --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example55_TextChunker.cs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.DeepDev; +using Microsoft.ML.Tokenizers; +using Microsoft.SemanticKernel.Text; +using Resources; +using SharpToken; +using static Microsoft.SemanticKernel.Text.TextChunker; + +// ReSharper disable once InconsistentNaming +public static class Example55_TextChunker +{ + private const string Text = @"The city of Venice, located in the northeastern part of Italy, +is renowned for its unique geographical features. Built on more than 100 small islands in a lagoon in the +Adriatic Sea, it has no roads, just canals including the Grand Canal thoroughfare lined with Renaissance and +Gothic palaces. The central square, Piazza San Marco, contains St. Mark's Basilica, which is tiled with Byzantine +mosaics, and the Campanile bell tower offering views of the city's red roofs. + +The Amazon Rainforest, also known as Amazonia, is a moist broadleaf tropical rainforest in the Amazon biome that +covers most of the Amazon basin of South America. This basin encompasses 7 million square kilometers, of which +5.5 million square kilometers are covered by the rainforest. This region includes territory belonging to nine nations +and 3.4 million square kilometers of uncontacted tribes. The Amazon represents over half of the planet's remaining +rainforests and comprises the largest and most biodiverse tract of tropical rainforest in the world. + +The Great Barrier Reef is the world's largest coral reef system composed of over 2,900 individual reefs and 900 islands +stretching for over 2,300 kilometers over an area of approximately 344,400 square kilometers. The reef is located in the +Coral Sea, off the coast of Queensland, Australia. The Great Barrier Reef can be seen from outer space and is the world's +biggest single structure made by living organisms. This reef structure is composed of and built by billions of tiny organisms, +known as coral polyps."; + + public static Task RunAsync() + { + RunExample(); + RunExampleForTokenCounterType(TokenCounterType.SharpToken); + RunExampleForTokenCounterType(TokenCounterType.MicrosoftML); + RunExampleForTokenCounterType(TokenCounterType.MicrosoftMLRoberta); + RunExampleForTokenCounterType(TokenCounterType.DeepDev); + RunExampleWithHeader(); + + return Task.CompletedTask; + } + + private static void RunExample() + { + Console.WriteLine("=== Text chunking ==="); + + var lines = TextChunker.SplitPlainTextLines(Text, 40); + var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 120); + + WriteParagraphsToConsole(paragraphs); + } + + private static void RunExampleForTokenCounterType(TokenCounterType counterType) + { + Console.WriteLine($"=== Text chunking with a custom({counterType}) token counter ==="); + var sw = new Stopwatch(); + sw.Start(); + var tokenCounter = s_tokenCounterFactory(counterType); + + var lines = TextChunker.SplitPlainTextLines(Text, 40, tokenCounter); + var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 120, tokenCounter: tokenCounter); + + sw.Stop(); + Console.WriteLine($"Elapsed time: {sw.ElapsedMilliseconds} ms"); + WriteParagraphsToConsole(paragraphs); + } + + private static void RunExampleWithHeader() + { + Console.WriteLine("=== Text chunking with chunk header ==="); + + var lines = TextChunker.SplitPlainTextLines(Text, 40); + var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 150, chunkHeader: "DOCUMENT NAME: test.txt\n\n"); + + WriteParagraphsToConsole(paragraphs); + } + + private static void WriteParagraphsToConsole(List paragraphs) + { + for (var i = 0; i < paragraphs.Count; i++) + { + Console.WriteLine(paragraphs[i]); + + if (i < paragraphs.Count - 1) + { + Console.WriteLine("------------------------"); + } + } + } + + private enum TokenCounterType + { + SharpToken, + MicrosoftML, + DeepDev, + MicrosoftMLRoberta, + } + + /// + /// Custom token counter implementation using SharpToken. + /// Note: SharpToken is used for demonstration purposes only, it's possible to use any available or custom tokenization logic. + /// + private static TokenCounter SharpTokenTokenCounter => (string input) => + { + // Initialize encoding by encoding name + var encoding = GptEncoding.GetEncoding("cl100k_base"); + + // Initialize encoding by model name + // var encoding = GptEncoding.GetEncodingForModel("gpt-4"); + + var tokens = encoding.Encode(input); + + return tokens.Count; + }; + + /// + /// MicrosoftML token counter implementation. + /// + private static TokenCounter MicrosoftMLTokenCounter => (string input) => + { + Tokenizer tokenizer = new(new Bpe()); + var tokens = tokenizer.Encode(input).Tokens; + + return tokens.Count; + }; + + /// + /// MicrosoftML token counter implementation using Roberta and local vocab + /// + private static TokenCounter MicrosoftMLRobertaTokenCounter => (string input) => + { + var encoder = EmbeddedResource.ReadStream("EnglishRoberta.encoder.json"); + var vocab = EmbeddedResource.ReadStream("EnglishRoberta.vocab.bpe"); + var dict = EmbeddedResource.ReadStream("EnglishRoberta.dict.txt"); + + if (encoder is null || vocab is null || dict is null) + { + throw new FileNotFoundException("Missing required resources"); + } + + EnglishRoberta model = new(encoder, vocab, dict); + + model.AddMaskSymbol(); // Not sure what this does, but it's in the example + Tokenizer tokenizer = new(model, new RobertaPreTokenizer()); + var tokens = tokenizer.Encode(input).Tokens; + + return tokens.Count; + }; + + /// + /// DeepDev token counter implementation. + /// + private static TokenCounter DeepDevTokenCounter => (string input) => + { +#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits + // Initialize encoding by encoding name + var tokenizer = TokenizerBuilder.CreateByEncoderNameAsync("cl100k_base").GetAwaiter().GetResult(); +#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits + + // Initialize encoding by model name + // var tokenizer = TokenizerBuilder.CreateByModelNameAsync("gpt-4").GetAwaiter().GetResult(); + + var tokens = tokenizer.Encode(input, new HashSet()); + return tokens.Count; + }; + + private static readonly Func s_tokenCounterFactory = (TokenCounterType counterType) => + { + switch (counterType) + { + case TokenCounterType.SharpToken: + return (string input) => SharpTokenTokenCounter(input); + case TokenCounterType.MicrosoftML: + return (string input) => MicrosoftMLTokenCounter(input); + case TokenCounterType.DeepDev: + return (string input) => DeepDevTokenCounter(input); + case TokenCounterType.MicrosoftMLRoberta: + return (string input) => MicrosoftMLRobertaTokenCounter(input); + default: + throw new ArgumentOutOfRangeException(nameof(counterType), counterType, null); + } + }; +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example56_TemplateNativeFunctionsWithMultipleArguments.cs b/dotnet/samples/KernelSyntaxExamples/Example56_TemplateNativeFunctionsWithMultipleArguments.cs new file mode 100644 index 000000000000..dd2a557e5f71 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example56_TemplateNativeFunctionsWithMultipleArguments.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.TemplateEngine.Basic; +using RepoUtils; + +// ReSharper disable once InconsistentNaming +public static class Example56_TemplateNativeFunctionsWithMultipleArguments +{ + /// + /// Show how to invoke a Native Function written in C# with multiple arguments + /// from a Semantic Function written in natural language + /// + public static async Task RunAsync() + { + Console.WriteLine("======== TemplateNativeFunctionsWithMultipleArguments ========"); + + string serviceId = TestConfiguration.AzureOpenAI.ServiceId; + string apiKey = TestConfiguration.AzureOpenAI.ApiKey; + string deploymentName = TestConfiguration.AzureOpenAI.ChatDeploymentName; + string endpoint = TestConfiguration.AzureOpenAI.Endpoint; + + if (serviceId == null || apiKey == null || deploymentName == null || endpoint == null) + { + Console.WriteLine("Azure serviceId, endpoint, apiKey, or deploymentName not found. Skipping example."); + return; + } + + IKernel kernel = Kernel.Builder + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + deploymentName: deploymentName, + endpoint: endpoint, + serviceId: serviceId, + apiKey: apiKey) + .Build(); + + var variableName = "word2"; + var variableValue = " Potter"; + var context = kernel.CreateNewContext(); + context.Variables[variableName] = variableValue; + + // Load native plugin into the kernel function collection, sharing its functions with prompt templates + // Functions loaded here are available as "text.*" + kernel.ImportFunctions(new TextPlugin(), "text"); + + // Semantic Function invoking text.Concat native function with named arguments input and input2 where input is a string and input2 is set to a variable from context called word2. + const string FunctionDefinition = @" + Write a haiku about the following: {{text.Concat input='Harry' input2=$word2}} +"; + + // This allows to see the prompt before it's sent to OpenAI + Console.WriteLine("--- Rendered Prompt"); + var promptRenderer = new BasicPromptTemplateEngine(); + var renderedPrompt = await promptRenderer.RenderAsync(FunctionDefinition, context); + Console.WriteLine(renderedPrompt); + + // Run the prompt / semantic function + var haiku = kernel.CreateSemanticFunction(FunctionDefinition, new OpenAIRequestSettings() { MaxTokens = 100 }); + + // Show the result + Console.WriteLine("--- Semantic Function result"); + var result = await kernel.RunAsync(context.Variables, haiku); + Console.WriteLine(result.GetValue()); + + /* OUTPUT: + +--- Rendered Prompt + + Write a haiku about the following: Harry Potter + +--- Semantic Function result +A boy with a scar, +Wizarding world he explores, +Harry Potter's tale. + */ + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example57_FunctionEventHandlers.cs b/dotnet/samples/KernelSyntaxExamples/Example57_FunctionEventHandlers.cs new file mode 100644 index 000000000000..137111677baf --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example57_FunctionEventHandlers.cs @@ -0,0 +1,285 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Events; +using Microsoft.SemanticKernel.Orchestration; +using RepoUtils; + +// ReSharper disable once InconsistentNaming +public static class Example57_FunctionEventHandlers +{ + private static string? s_openAIModelId; + private static string? s_openAIApiKey; + + public static async Task RunAsync() + { + Console.WriteLine("\n======== Using Function Execution Handlers ========\n"); + + s_openAIModelId = TestConfiguration.OpenAI.ChatModelId; + s_openAIApiKey = TestConfiguration.OpenAI.ApiKey; + + if (s_openAIModelId == null || s_openAIApiKey == null) + { + Console.WriteLine("OpenAI credentials not found. Skipping example."); + return; + } + + await GetUsageAsync(); + + await ChangingResultAsync(); + + await BeforeInvokeCancellationAsync(); + + await AfterInvokeCancellationAsync(); + + await SkippingFunctionsAsync(); + + await RepeatFunctionsAsync(); + } + + private static async Task GetUsageAsync() + { + Console.WriteLine("\n======== Get Rendered Prompt and Usage Data ========\n"); + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( + modelId: s_openAIModelId!, + apiKey: s_openAIApiKey!) + .Build(); + + const string FunctionPrompt = "Write a random paragraph about: {{$input}}."; + + var excuseFunction = kernel.CreateSemanticFunction( + FunctionPrompt, + pluginName: "MyPlugin", + functionName: "Excuse", + requestSettings: new OpenAIRequestSettings() { MaxTokens = 100, Temperature = 0.4, TopP = 1 }); + + void MyPreHandler(object? sender, FunctionInvokingEventArgs e) + { + Console.WriteLine($"{e.FunctionView.PluginName}.{e.FunctionView.Name} : Pre Execution Handler - Triggered"); + } + + void MyRemovedPreExecutionHandler(object? sender, FunctionInvokingEventArgs e) + { + Console.WriteLine($"{e.FunctionView.PluginName}.{e.FunctionView.Name} : Pre Execution Handler - Should not trigger"); + e.Cancel(); + } + + void MyPostExecutionHandler(object? sender, FunctionInvokedEventArgs e) + { + var modelResults = e.Metadata["ModelResults"] as IReadOnlyCollection; + Console.WriteLine($"{e.FunctionView.PluginName}.{e.FunctionView.Name} : Post Execution Handler - Total Tokens: {modelResults?.First().GetOpenAIChatResult().Usage.TotalTokens}"); + } + + kernel.FunctionInvoking += MyPreHandler; + kernel.FunctionInvoked += MyPostExecutionHandler; + + // Adding and Removing a handler + kernel.FunctionInvoking += MyRemovedPreExecutionHandler; + kernel.FunctionInvoking -= MyRemovedPreExecutionHandler; + + const string Input = "I missed the F1 final race"; + var result = await kernel.RunAsync(Input, excuseFunction); + Console.WriteLine($"Function Result: {result.GetValue()}"); + } + + private static async Task ChangingResultAsync() + { + Console.WriteLine("\n======== Changing/Filtering Function Result ========\n"); + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( + modelId: s_openAIModelId!, + apiKey: s_openAIApiKey!) + .Build(); + + const string FunctionPrompt = "Write a paragraph about Handlers."; + + var writerFunction = kernel.CreateSemanticFunction( + FunctionPrompt, + pluginName: "MyPlugin", + functionName: "Writer", + requestSettings: new OpenAIRequestSettings() { MaxTokens = 100, Temperature = 0.4, TopP = 1 }); + + void MyChangeDataHandler(object? sender, FunctionInvokedEventArgs e) + { + var originalOutput = e.SKContext.Result; + + //Use Regex to redact all vowels and numbers + var newOutput = Regex.Replace(originalOutput, "[aeiouAEIOU0-9]", "*"); + + e.SKContext.Variables.Update(newOutput); + } + + kernel.FunctionInvoked += MyChangeDataHandler; + + var result = await kernel.RunAsync(writerFunction); + + Console.WriteLine($"Function Result: {result.GetValue()}"); + } + + private static async Task BeforeInvokeCancellationAsync() + { + Console.WriteLine("\n======== Cancelling Pipeline Execution - Invoking event ========\n"); + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( + modelId: s_openAIModelId!, + apiKey: s_openAIApiKey!) + .Build(); + + const string FunctionPrompt = "Write a paragraph about: Cancellation."; + + var writerFunction = kernel.CreateSemanticFunction( + FunctionPrompt, + pluginName: "MyPlugin", + functionName: "Writer", + requestSettings: new OpenAIRequestSettings() { MaxTokens = 1000, Temperature = 1, TopP = 0.5 }); + + // Adding new inline handler to cancel/prevent function execution + kernel.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + Console.WriteLine($"{e.FunctionView.PluginName}.{e.FunctionView.Name} : FunctionInvoking - Cancelling all subsequent invocations"); + e.Cancel(); + }; + + // Technically invoked will never be called since the function will be cancelled + int functionInvokedCount = 0; + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + functionInvokedCount++; + }; + + var result = await kernel.RunAsync(writerFunction); + Console.WriteLine($"Function Invocation Times: {functionInvokedCount}"); + } + + private static async Task AfterInvokeCancellationAsync() + { + Console.WriteLine("\n======== Cancelling Pipeline Execution - Invoked event ========\n"); + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( + modelId: s_openAIModelId!, + apiKey: s_openAIApiKey!) + .Build(); + + int functionInvokingCount = 0; + int functionInvokedCount = 0; + + var firstFunction = kernel.CreateSemanticFunction("Write a phrase with Invoke.", functionName: "InvokePhrase"); + var secondFunction = kernel.CreateSemanticFunction("Write a phrase with Cancellation.", functionName: "CancellationPhrase"); + + // Adding new inline handler to count invoking events + kernel.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + functionInvokingCount++; + }; + + // Invoked will never be called twice (for the secondFunction) since Invoked from the first is cancelling. + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + functionInvokedCount++; + e.Cancel(); + }; + + var result = await kernel.RunAsync(secondFunction); + Console.WriteLine($"Function Invoked Times: {functionInvokedCount}"); + Console.WriteLine($"Function Invoking Times: {functionInvokingCount}"); + } + + private static async Task SkippingFunctionsAsync() + { + Console.WriteLine("\n======== Skipping a Function in the Pipeline ========\n"); + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( + modelId: s_openAIModelId!, + apiKey: s_openAIApiKey!) + .Build(); + + var skipMeFunction = kernel.CreateSemanticFunction("Write a paragraph about Skipping", + pluginName: "MyPlugin", + functionName: "SkipMe"); + + var dontSkipMeFunction = kernel.CreateSemanticFunction("Write a paragraph about Handlers", + pluginName: "MyPlugin", + functionName: "DontSkipMe"); + + kernel.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + if (e.FunctionView.Name == "SkipMe") + { + e.Skip(); + Console.WriteLine($"Function {e.FunctionView.Name} will be skipped"); + return; + } + + Console.WriteLine($"Function {e.FunctionView.Name} will not be skipped"); + }; + + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + Console.WriteLine($"Only not skipped functions will trigger invoked event - Function name: {e.FunctionView.Name}"); + }; + + var result = await kernel.RunAsync( + skipMeFunction, + dontSkipMeFunction); + + Console.WriteLine($"Final result: {result.GetValue()}"); + } + + private static async Task RepeatFunctionsAsync() + { + Console.WriteLine("\n======== Repeating a Function in the Pipeline ========"); + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService( + modelId: s_openAIModelId!, + apiKey: s_openAIApiKey!) + .Build(); + + var repeatSubjects = new Queue(new[] { "Life", "Work", "Leisure" }); + + var repeatMeFunction = kernel.CreateSemanticFunction("Write a sentence about {{$input}}", + pluginName: "MyPlugin", + functionName: "RepeatMe"); + + var repeatTimes = 0; + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + Console.WriteLine($"\nFunction {e.FunctionView.Name} executed:"); + Console.WriteLine($"Result: {e.SKContext.Result}"); + + if (repeatTimes < 3) + { + // Flag the Kernel to repeat the function + e.Repeat(); + + // Redefine the input variable to repeat the function + e.SKContext.Variables.Update(repeatSubjects.Dequeue()); + + repeatTimes++; + Console.WriteLine("Repeat requested!"); + + return; + } + }; + + await kernel.RunAsync("Repetition", repeatMeFunction); + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example58_ConfigureRequestSettings.cs b/dotnet/samples/KernelSyntaxExamples/Example58_ConfigureRequestSettings.cs new file mode 100644 index 000000000000..7c8f75e521ea --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example58_ConfigureRequestSettings.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.TemplateEngine; +using RepoUtils; + +// ReSharper disable once InconsistentNaming +public static class Example58_ConfigureRequestSettings +{ + /// + /// Show how to configure model request settings + /// + public static async Task RunAsync() + { + Console.WriteLine("======== Example58_ConfigureRequestSettings ========"); + + string serviceId = TestConfiguration.AzureOpenAI.ServiceId; + string apiKey = TestConfiguration.AzureOpenAI.ApiKey; + string chatDeploymentName = TestConfiguration.AzureOpenAI.ChatDeploymentName; + string endpoint = TestConfiguration.AzureOpenAI.Endpoint; + + if (serviceId == null || apiKey == null || chatDeploymentName == null || endpoint == null) + { + Console.WriteLine("Azure serviceId, endpoint, apiKey, or deploymentName not found. Skipping example."); + return; + } + + IKernel kernel = Kernel.Builder + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureChatCompletionService( + deploymentName: chatDeploymentName, + endpoint: endpoint, + serviceId: serviceId, + apiKey: apiKey) + .Build(); + + var prompt = "Hello AI, what can you do for me?"; + + // Option 1: + // Invoke the semantic function and pass an OpenAI specific instance containing the request settings + var result = await kernel.InvokeSemanticFunctionAsync( + prompt, + new OpenAIRequestSettings() + { + MaxTokens = 60, + Temperature = 0.7 + }); + Console.WriteLine(result.GetValue()); + + // Option 2: + // Load prompt template configuration including the request settings from a JSON payload + // Create the semantic functions using the prompt template and the configuration (loaded in the previous step) + // Invoke the semantic function using the implicitly set request settings + string configPayload = @"{ + ""schema"": 1, + ""description"": ""Say hello to an AI"", + ""type"": ""completion"", + ""completion"": { + ""max_tokens"": 256, + ""temperature"": 0.5, + ""top_p"": 0.0, + ""presence_penalty"": 0.0, + ""frequency_penalty"": 0.0 + } + }"; + var templateConfig = JsonSerializer.Deserialize(configPayload); + var func = kernel.CreateSemanticFunction(prompt, templateConfig!, "HelloAI"); + + result = await kernel.RunAsync(func); + Console.WriteLine(result.GetValue()); + + /* OUTPUT (using gpt4): +Hello! As an AI language model, I can help you with a variety of + +Hello! As an AI language model, I can help you with a variety of tasks, such as: + +1. Answering general questions and providing information on a wide range of topics. +2. Assisting with problem-solving and brainstorming ideas. +3. Offering recommendations for books, movies, music, and more. +4. Providing definitions, explanations, and examples of various concepts. +5. Helping with language-related tasks, such as grammar, vocabulary, and writing tips. +6. Generating creative content, such as stories, poems, or jokes. +7. Assisting with basic math and science problems. +8. Offering advice on various topics, such as productivity, motivation, and personal development. + +Please feel free to ask me anything, and I'll do my best to help you! +Hello! As an AI language model, I can help you with a variety of tasks, including: + +1. Answering general questions and providing information on a wide range of topics. +2. Offering suggestions and recommendations. +3. Assisting with problem-solving and brainstorming ideas. +4. Providing explanations and + */ + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Example59_OpenAIFunctionCalling.cs b/dotnet/samples/KernelSyntaxExamples/Example59_OpenAIFunctionCalling.cs new file mode 100644 index 000000000000..d710e8993364 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example59_OpenAIFunctionCalling.cs @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Plugins.Core; +using RepoUtils; + +/** + * This example shows how to use OpenAI's function calling capability via the chat completions interface. + * For more information, see https://platform.openai.com/docs/guides/gpt/function-calling. + */ +// ReSharper disable once InconsistentNaming +public static class Example59_OpenAIFunctionCalling +{ + public static async Task RunAsync() + { + IKernel kernel = await InitializeKernelAsync(); + var chatCompletion = kernel.GetService(); + var chatHistory = chatCompletion.CreateNewChat(); + + OpenAIRequestSettings requestSettings = new() + { + // Include all functions registered with the kernel. + // Alternatively, you can provide your own list of OpenAIFunctions to include. + Functions = kernel.Functions.GetFunctionViews().Select(f => f.ToOpenAIFunction()).ToList(), + }; + + // Set FunctionCall to the name of a specific function to force the model to use that function. + requestSettings.FunctionCall = "TimePlugin-Date"; + await CompleteChatWithFunctionsAsync("What day is today?", chatHistory, chatCompletion, kernel, requestSettings); + + // Set FunctionCall to auto to let the model choose the best function to use. + requestSettings.FunctionCall = OpenAIRequestSettings.FunctionCallAuto; + await CompleteChatWithFunctionsAsync("What computer tablets are available for under $200?", chatHistory, chatCompletion, kernel, requestSettings); + } + + private static async Task InitializeKernelAsync() + { + // Create kernel with chat completions service + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey, serviceId: "chat") + //.WithAzureChatCompletionService(TestConfiguration.AzureOpenAI.ChatDeploymentName, TestConfiguration.AzureOpenAI.Endpoint, TestConfiguration.AzureOpenAI.ApiKey, serviceId: "chat") + .Build(); + + // Load functions to kernel + kernel.ImportFunctions(new TimePlugin(), "TimePlugin"); + await kernel.ImportPluginFunctionsAsync("KlarnaShoppingPlugin", new Uri("https://www.klarna.com/.well-known/ai-plugin.json"), new OpenApiFunctionExecutionParameters()); + + return kernel; + } + + private static async Task CompleteChatWithFunctionsAsync(string ask, ChatHistory chatHistory, IChatCompletion chatCompletion, IKernel kernel, OpenAIRequestSettings requestSettings) + { + Console.WriteLine($"User message: {ask}"); + chatHistory.AddUserMessage(ask); + + // Send request + var chatResult = (await chatCompletion.GetChatCompletionsAsync(chatHistory, requestSettings))[0]; + + // Check for message response + var chatMessage = await chatResult.GetChatMessageAsync(); + if (!string.IsNullOrEmpty(chatMessage.Content)) + { + Console.WriteLine(chatMessage.Content); + + // Add the response to chat history + chatHistory.AddAssistantMessage(chatMessage.Content); + } + + // Check for function response + OpenAIFunctionResponse? functionResponse = chatResult.GetFunctionResponse(); + if (functionResponse is not null) + { + // Print function response details + Console.WriteLine("Function name: " + functionResponse.FunctionName); + Console.WriteLine("Plugin name: " + functionResponse.PluginName); + Console.WriteLine("Arguments: "); + foreach (var parameter in functionResponse.Parameters) + { + Console.WriteLine($"- {parameter.Key}: {parameter.Value}"); + } + + // If the function returned by OpenAI is an SKFunction registered with the kernel, + // you can invoke it using the following code. + if (kernel.Functions.TryGetFunctionAndContext(functionResponse, out ISKFunction? func, out ContextVariables? context)) + { + var kernelResult = await kernel.RunAsync(func, context); + + var result = kernelResult.GetValue(); + + string? resultMessage = null; + if (result is RestApiOperationResponse apiResponse) + { + resultMessage = apiResponse.Content?.ToString(); + } + else if (result is string str) + { + resultMessage = str; + } + + if (!string.IsNullOrEmpty(resultMessage)) + { + Console.WriteLine(resultMessage); + + // Add the function result to chat history + chatHistory.AddAssistantMessage(resultMessage); + } + } + else + { + Console.WriteLine($"Error: Function {functionResponse.PluginName}.{functionResponse.FunctionName} not found."); + } + } + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj b/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj index 244cadfa2ea4..1b5eef6de7ce 100644 --- a/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj +++ b/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj @@ -9,7 +9,7 @@ Exe false - CA1050;CA1707;CA2007;VSTHRD111 + CA1050;CA1707;CA2007;VSTHRD111;CS1591 @@ -26,29 +26,37 @@ + + + + + + + - - - + + + - - - + + + + - - - - + + + + @@ -57,5 +65,8 @@ + + + \ No newline at end of file diff --git a/dotnet/samples/KernelSyntaxExamples/Plugins/EmailPlugin.cs b/dotnet/samples/KernelSyntaxExamples/Plugins/EmailPlugin.cs new file mode 100644 index 000000000000..c957608a2f85 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Plugins/EmailPlugin.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; + +namespace Plugins; + +internal sealed class EmailPlugin +{ + [SKFunction, Description("Given an e-mail and message body, send an email")] + public string SendEmail( + [Description("The body of the email message to send.")] string input, + [Description("The email address to send email to.")] string email_address) => + + $"Sent email to: {email_address}. Body: {input}"; + + [SKFunction, Description("Given a name, find email address")] + public string GetEmailAddress( + [Description("The name of the person whose email address needs to be found.")] string input, + ILogger? logger = null) + { + // Sensitive data, logging as trace, disabled by default + logger?.LogTrace("Returning hard coded email for {0}", input); + + return "johndoe1234@example.com"; + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Skills/JiraSkill/README.md b/dotnet/samples/KernelSyntaxExamples/Plugins/JiraPlugin/README.md similarity index 100% rename from dotnet/samples/KernelSyntaxExamples/Skills/JiraSkill/README.md rename to dotnet/samples/KernelSyntaxExamples/Plugins/JiraPlugin/README.md diff --git a/dotnet/samples/KernelSyntaxExamples/Skills/JiraSkill/openapi.json b/dotnet/samples/KernelSyntaxExamples/Plugins/JiraPlugin/openapi.json similarity index 100% rename from dotnet/samples/KernelSyntaxExamples/Skills/JiraSkill/openapi.json rename to dotnet/samples/KernelSyntaxExamples/Plugins/JiraPlugin/openapi.json diff --git a/dotnet/samples/KernelSyntaxExamples/Plugins/StaticTextPlugin.cs b/dotnet/samples/KernelSyntaxExamples/Plugins/StaticTextPlugin.cs new file mode 100644 index 000000000000..5187167d8bcf --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Plugins/StaticTextPlugin.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using Microsoft.SemanticKernel; + +namespace Plugins; + +public sealed class StaticTextPlugin +{ + [SKFunction, Description("Change all string chars to uppercase")] + public static string Uppercase([Description("Text to uppercase")] string input) => + input.ToUpperInvariant(); + + [SKFunction, Description("Append the day variable")] + public static string AppendDay( + [Description("Text to append to")] string input, + [Description("Value of the day to append")] string day) => + input + day; +} diff --git a/dotnet/samples/KernelSyntaxExamples/Program.cs b/dotnet/samples/KernelSyntaxExamples/Program.cs index 4e2ea73ef816..de2b07fdf982 100644 --- a/dotnet/samples/KernelSyntaxExamples/Program.cs +++ b/dotnet/samples/KernelSyntaxExamples/Program.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; @@ -10,7 +12,7 @@ public static class Program { // ReSharper disable once InconsistentNaming - public static async Task Main() + public static async Task Main(string[] args) { // Load configuration from environment variables or user secrets. LoadUserSecrets(); @@ -19,59 +21,55 @@ public static async Task Main() using CancellationTokenSource cancellationTokenSource = new(); CancellationToken cancelToken = cancellationTokenSource.ConsoleCancellationToken(); - // Run examples - await Example01_NativeFunctions.RunAsync().SafeWaitAsync(cancelToken); - await Example02_Pipeline.RunAsync().SafeWaitAsync(cancelToken); - await Example03_Variables.RunAsync().SafeWaitAsync(cancelToken); - await Example04_CombineLLMPromptsAndNativeCode.RunAsync().SafeWaitAsync(cancelToken); - await Example05_InlineFunctionDefinition.RunAsync().SafeWaitAsync(cancelToken); - await Example06_TemplateLanguage.RunAsync().SafeWaitAsync(cancelToken); - await Example07_BingAndGoogleSkills.RunAsync().SafeWaitAsync(cancelToken); - await Example08_RetryHandler.RunAsync().SafeWaitAsync(cancelToken); - await Example09_FunctionTypes.RunAsync().SafeWaitAsync(cancelToken); - await Example10_DescribeAllSkillsAndFunctions.RunAsync().SafeWaitAsync(cancelToken); - await Example11_WebSearchQueries.RunAsync().SafeWaitAsync(cancelToken); - await Example12_SequentialPlanner.RunAsync().SafeWaitAsync(cancelToken); - await Example13_ConversationSummarySkill.RunAsync().SafeWaitAsync(cancelToken); - await Example14_SemanticMemory.RunAsync().SafeWaitAsync(cancelToken); - await Example15_MemorySkill.RunAsync().SafeWaitAsync(cancelToken); - await Example16_CustomLLM.RunAsync().SafeWaitAsync(cancelToken); - await Example17_ChatGPT.RunAsync().SafeWaitAsync(cancelToken); - await Example18_DallE.RunAsync().SafeWaitAsync(cancelToken); - await Example19_Qdrant.RunAsync().SafeWaitAsync(cancelToken); - await Example20_HuggingFace.RunAsync().SafeWaitAsync(cancelToken); - await Example21_ChatGptPlugins.RunAsync().SafeWaitAsync(cancelToken); - await Example22_OpenApiSkill_AzureKeyVault.RunAsync().SafeWaitAsync(cancelToken); - await Example23_OpenApiSkill_GitHub.RunAsync().SafeWaitAsync(cancelToken); - await Example24_OpenApiSkill_Jira.RunAsync().SafeWaitAsync(cancelToken); - await Example25_ReadOnlyMemoryStore.RunAsync().SafeWaitAsync(cancelToken); - await Example26_AADAuth.RunAsync().SafeWaitAsync(cancelToken); - await Example27_SemanticFunctionsUsingChatGPT.RunAsync().SafeWaitAsync(cancelToken); - await Example28_ActionPlanner.RunAsync().SafeWaitAsync(cancelToken); - await Example29_Tokenizer.RunAsync().SafeWaitAsync(cancelToken); - await Example30_ChatWithPrompts.RunAsync().SafeWaitAsync(cancelToken); - await Example31_CustomPlanner.RunAsync().SafeWaitAsync(cancelToken); - await Example32_StreamingCompletion.RunAsync().SafeWaitAsync(cancelToken); - await Example33_StreamingChat.RunAsync().SafeWaitAsync(cancelToken); - await Example34_CustomChatModel.RunAsync().SafeWaitAsync(cancelToken); - await Example35_GrpcSkills.RunAsync().SafeWaitAsync(cancelToken); - await Example36_MultiCompletion.RunAsync().SafeWaitAsync(cancelToken); - await Example37_MultiStreamingCompletion.RunAsync().SafeWaitAsync(cancelToken); - await Example38_Pinecone.RunAsync().SafeWaitAsync(cancelToken); - await Example39_Postgres.RunAsync().SafeWaitAsync(cancelToken); - await Example40_DIContainer.RunAsync().SafeWaitAsync(cancelToken); - await Example41_HttpClientUsage.RunAsync().SafeWaitAsync(cancelToken); - await Example42_KernelBuilder.RunAsync().SafeWaitAsync(cancelToken); - await Example43_GetModelResult.RunAsync().SafeWaitAsync(cancelToken); - await Example44_MultiChatCompletion.RunAsync().SafeWaitAsync(cancelToken); - await Example45_MultiStreamingChatCompletion.RunAsync().SafeWaitAsync(cancelToken); - await Example46_Weaviate.RunAsync().SafeWaitAsync(cancelToken); - await Example47_Redis.RunAsync().SafeWaitAsync(cancelToken); - await Example48_GroundednessChecks.RunAsync().SafeWaitAsync(cancelToken); - await Example49_LogitBias.RunAsync().SafeWaitAsync(cancelToken); - await Example50_Chroma.RunAsync().SafeWaitAsync(cancelToken); - await Example51_StepwisePlanner.RunAsync().SafeWaitAsync(cancelToken); - await Example52_ApimAuth.RunAsync().SafeWaitAsync(cancelToken); + string? defaultFilter = null; // Modify to filter examples + + // Check if args[0] is provided + string? filter = args.Length > 0 ? args[0] : defaultFilter; + + // Run examples based on the filter + await RunExamplesAsync(filter, cancelToken); + } + + private static async Task RunExamplesAsync(string? filter, CancellationToken cancellationToken) + { + var examples = (Assembly.GetExecutingAssembly().GetTypes()) + .Where(type => type.Name.StartsWith("Example", StringComparison.OrdinalIgnoreCase)) + .Select(type => type.Name).ToList(); + + // Filter and run examples + foreach (var example in examples) + { + if (string.IsNullOrEmpty(filter) || example.Contains(filter, StringComparison.OrdinalIgnoreCase)) + { + try + { + Console.WriteLine($"Running {example}..."); + + var method = Assembly.GetExecutingAssembly().GetType(example)?.GetMethod("RunAsync"); + if (method == null) + { + Console.WriteLine($"Example {example} not found"); + continue; + } + + bool hasCancellationToken = method.GetParameters().Any(param => param.ParameterType == typeof(CancellationToken)); + + var taskParameters = hasCancellationToken ? new object[] { cancellationToken } : null; + if (method.Invoke(null, taskParameters) is Task t) + { + await t.SafeWaitAsync(cancellationToken); + } + else + { + method.Invoke(null, null); + } + } + catch (ConfigurationNotFoundException ex) + { + Console.WriteLine($"{ex.Message}. Skipping example {example}."); + } + } + } } private static void LoadUserSecrets() diff --git a/dotnet/samples/KernelSyntaxExamples/README.md b/dotnet/samples/KernelSyntaxExamples/README.md index 81562e86e582..26e95a78a215 100644 --- a/dotnet/samples/KernelSyntaxExamples/README.md +++ b/dotnet/samples/KernelSyntaxExamples/README.md @@ -1,12 +1,44 @@ # Semantic Kernel syntax examples This project contains a collection of semi-random examples about various scenarios -using SK components. +using SK components. The examples are ordered by number, starting with very basic examples. +## Running Examples with Filters + +You can run individual examples in the KernelSyntaxExamples project using various methods to specify a filter. This allows you to execute specific examples without running all of them. Choose one of the following options to apply a filter: + +### Option 1: Set the Default Filter in Program.cs + +In your code, you can set a default filter by modifying the appropriate variable or parameter. Look for the section in your code where the filter is applied or where the examples are defined, and change the filter value accordingly. + +```csharp +// Example of setting a default filter in code +string defaultFilter = "Example0"; // will run all examples that contain 'example0' in the name +``` + +### Option 2: Set Command-Line Arguments +Right-click on your console application project in the Solution Explorer. + +Choose "Properties" from the context menu. + +In the project properties window, navigate to the "Debug" tab on the left. + +Supply Command-Line Arguments: + +In the "Command line arguments" field, enter the command-line arguments that your console application expects. Separate multiple arguments with spaces. + +### Option 3: Use Visual Studio Code Filters +If you are using Visual Studio Code, you can specify a filter using the built-in filter options provided by the IDE. These options can be helpful when running your code in a debugging environment. Consult the documentation for Visual Studio Code or the specific extension you're using for information on applying filters. + +### Option 4: Modify launch.json +If you are using Visual Studio or a similar IDE that utilizes launch configurations, you can specify the filter in your launch.json configuration file. Edit the configuration for your project to include the filter parameter. + + +## Configuring Secrets Most of the examples will require secrets and credentials, to access OpenAI, Azure OpenAI, -Bing and other resources. We suggest using .NET +Bing and other resources. We suggest using .NET [Secret Manager](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets) to avoid the risk of leaking secrets into the repository, branches and pull requests. You can also use environment variables if you prefer. @@ -69,6 +101,7 @@ dotnet user-secrets set "Apim:SubscriptionKey" "..." dotnet user-secrets set "Postgres:ConnectionString" "..." dotnet user-secrets set "Redis:Configuration" "..." +dotnet user-secrets set "Kusto:ConnectionString" "..." ``` To set your secrets with environment variables, use these names: diff --git a/dotnet/samples/KernelSyntaxExamples/Reliability/RetryThreeTimesWithBackoff.cs b/dotnet/samples/KernelSyntaxExamples/Reliability/RetryThreeTimesWithBackoff.cs index ca287872b55a..3ed0929422a6 100644 --- a/dotnet/samples/KernelSyntaxExamples/Reliability/RetryThreeTimesWithBackoff.cs +++ b/dotnet/samples/KernelSyntaxExamples/Reliability/RetryThreeTimesWithBackoff.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Reliability; +using Microsoft.SemanticKernel.Http; using Polly; using Polly.Retry; @@ -16,9 +16,9 @@ namespace Reliability; /// public class RetryThreeTimesWithBackoffFactory : IDelegatingHandlerFactory { - public DelegatingHandler Create(ILogger? logger) + public DelegatingHandler Create(ILoggerFactory? loggerFactory) { - return new RetryThreeTimesWithBackoff(logger); + return new RetryThreeTimesWithBackoff(loggerFactory); } } @@ -29,35 +29,38 @@ public class RetryThreeTimesWithBackoff : DelegatingHandler { private readonly AsyncRetryPolicy _policy; - public RetryThreeTimesWithBackoff(ILogger? logger) + public RetryThreeTimesWithBackoff(ILoggerFactory? loggerFactory) { - this._policy = GetPolicy(logger); + this._policy = GetPolicy(loggerFactory); } protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { return await this._policy.ExecuteAsync(async () => { - var response = await base.SendAsync(request, cancellationToken); + var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); return response; - }); + }).ConfigureAwait(false); } - private static AsyncRetryPolicy GetPolicy(ILogger? logger) + private static AsyncRetryPolicy GetPolicy(ILoggerFactory? logger) { // Handle 429 and 401 errors // Typically 401 would not be something we retry but for demonstration // purposes we are doing so as it's easy to trigger when using an invalid key. + const int TooManyRequests = 429; + const int Unauthorized = 401; + return Policy .HandleResult(response => - response.StatusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.Unauthorized) + (int)response.StatusCode is TooManyRequests or Unauthorized) .WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(8) }, - (outcome, timespan, retryCount, _) => logger?.LogWarning( + (outcome, timespan, retryCount, _) => logger?.CreateLogger(typeof(RetryThreeTimesWithBackoff)).LogWarning( "Error executing action [attempt {0} of 3], pausing {1}ms. Outcome: {2}", retryCount, timespan.TotalMilliseconds, diff --git a/dotnet/samples/KernelSyntaxExamples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs b/dotnet/samples/KernelSyntaxExamples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs index d26fb8f49ca1..1586b9af9c90 100644 --- a/dotnet/samples/KernelSyntaxExamples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs +++ b/dotnet/samples/KernelSyntaxExamples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Reliability; +using Microsoft.SemanticKernel.Http; using Polly; using Polly.Retry; @@ -16,9 +16,9 @@ namespace Reliability; /// public class RetryThreeTimesWithRetryAfterBackoffFactory : IDelegatingHandlerFactory { - public DelegatingHandler Create(ILogger? logger) + public DelegatingHandler Create(ILoggerFactory? loggerFactory) { - return new RetryThreeTimesWithRetryAfterBackoff(logger); + return new RetryThreeTimesWithRetryAfterBackoff(loggerFactory); } } @@ -29,28 +29,31 @@ public class RetryThreeTimesWithRetryAfterBackoff : DelegatingHandler { private readonly AsyncRetryPolicy _policy; - public RetryThreeTimesWithRetryAfterBackoff(ILogger? logger) + public RetryThreeTimesWithRetryAfterBackoff(ILoggerFactory? loggerFactory) { - this._policy = GetPolicy(logger); + this._policy = GetPolicy(loggerFactory); } protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { return await this._policy.ExecuteAsync(async () => { - var response = await base.SendAsync(request, cancellationToken); + var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); return response; - }); + }).ConfigureAwait(false); } - private static AsyncRetryPolicy GetPolicy(ILogger? logger) + private static AsyncRetryPolicy GetPolicy(ILoggerFactory? loggerFactory) { // Handle 429 and 401 errors // Typically 401 would not be something we retry but for demonstration // purposes we are doing so as it's easy to trigger when using an invalid key. + const int TooManyRequests = 429; + const int Unauthorized = 401; + return Policy .HandleResult(response => - response.StatusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.Unauthorized) + (int)response.StatusCode is Unauthorized or TooManyRequests) .WaitAndRetryAsync( retryCount: 3, sleepDurationProvider: (_, r, _) => @@ -61,7 +64,7 @@ private static AsyncRetryPolicy GetPolicy(ILogger? logger) }, (outcome, timespan, retryCount, _) => { - logger?.LogWarning( + loggerFactory?.CreateLogger(typeof(RetryThreeTimesWithRetryAfterBackoff)).LogWarning( "Error executing action [attempt {0} of 3], pausing {1}ms. Outcome: {2}", retryCount, timespan.TotalMilliseconds, diff --git a/dotnet/samples/KernelSyntaxExamples/RepoUtils/ConsoleLogger.cs b/dotnet/samples/KernelSyntaxExamples/RepoUtils/ConsoleLogger.cs index 68f3b6073723..2ab9067ca8dd 100644 --- a/dotnet/samples/KernelSyntaxExamples/RepoUtils/ConsoleLogger.cs +++ b/dotnet/samples/KernelSyntaxExamples/RepoUtils/ConsoleLogger.cs @@ -10,15 +10,15 @@ namespace RepoUtils; /// internal static class ConsoleLogger { - internal static ILogger Logger => LogFactory.CreateLogger(); + internal static ILogger Logger => LoggerFactory.CreateLogger(); - private static ILoggerFactory LogFactory => s_loggerFactory.Value; + internal static ILoggerFactory LoggerFactory => s_loggerFactory.Value; private static readonly Lazy s_loggerFactory = new(LogBuilder); private static ILoggerFactory LogBuilder() { - return LoggerFactory.Create(builder => + return Microsoft.Extensions.Logging.LoggerFactory.Create(builder => { builder.SetMinimumLevel(LogLevel.Warning); diff --git a/dotnet/samples/KernelSyntaxExamples/RepoUtils/RepoFiles.cs b/dotnet/samples/KernelSyntaxExamples/RepoUtils/RepoFiles.cs index 5ca6da84f00c..176cc998fb86 100644 --- a/dotnet/samples/KernelSyntaxExamples/RepoUtils/RepoFiles.cs +++ b/dotnet/samples/KernelSyntaxExamples/RepoUtils/RepoFiles.cs @@ -8,13 +8,13 @@ namespace RepoUtils; public static class RepoFiles { /// - /// Scan the local folders from the repo, looking for "samples/skills" folder. + /// Scan the local folders from the repo, looking for "samples/plugins" folder. /// - /// The full path to samples/skills - public static string SampleSkillsPath() + /// The full path to samples/plugins + public static string SamplePluginsPath() { const string Parent = "samples"; - const string Folder = "skills"; + const string Folder = "plugins"; bool SearchPath(string pathToFind, out string result, int maxAttempts = 10) { @@ -33,7 +33,7 @@ bool SearchPath(string pathToFind, out string result, int maxAttempts = 10) if (!SearchPath(Parent + Path.DirectorySeparatorChar + Folder, out string path) && !SearchPath(Folder, out path)) { - throw new YourAppException("Skills directory not found. The app needs the skills from the repo to work."); + throw new YourAppException("Plugins directory not found. The app needs the plugins from the repo to work."); } return path; diff --git a/dotnet/samples/KernelSyntaxExamples/Resources/EmbeddedResource.cs b/dotnet/samples/KernelSyntaxExamples/Resources/EmbeddedResource.cs index 56edbdbcc0fe..34f386783538 100644 --- a/dotnet/samples/KernelSyntaxExamples/Resources/EmbeddedResource.cs +++ b/dotnet/samples/KernelSyntaxExamples/Resources/EmbeddedResource.cs @@ -35,4 +35,15 @@ internal static string Read(string fileName) using var reader = new StreamReader(resource); return reader.ReadToEnd(); } + + internal static Stream? ReadStream(string fileName) + { + // Get the current assembly. Note: this class is in the same assembly where the embedded resources are stored. + Assembly? assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly; + if (assembly == null) { throw new ConfigurationException($"[{s_namespace}] {fileName} assembly not found"); } + + // Resources are mapped like types, using the namespace and appending "." (dot) and the file name + var resourceName = $"{s_namespace}." + fileName; + return assembly.GetManifestResourceStream(resourceName); + } } diff --git a/dotnet/samples/KernelSyntaxExamples/Resources/EnglishRoberta/dict.txt b/dotnet/samples/KernelSyntaxExamples/Resources/EnglishRoberta/dict.txt new file mode 100644 index 000000000000..69d79faee009 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Resources/EnglishRoberta/dict.txt @@ -0,0 +1,50260 @@ +13 850314647 +262 800385005 +11 800251374 +284 432911125 +290 394899794 +286 386139013 +257 357878752 +287 311196488 +12 215156821 +329 155236946 +326 154060431 +319 147178919 +318 142591644 +447 130810923 +338 116498242 +351 114784681 +383 108664122 +373 100357189 +366 93880741 +379 93284459 +340 88803471 +355 85749070 +531 85009762 +247 82642284 +307 77095226 +82 76381845 +416 73380803 +422 71911149 +389 68628918 +423 67243391 +468 64317701 +25 63508661 +357 63001640 +339 61994245 +314 60989470 +465 56381137 +481 55817121 +281 55370942 +428 52404829 +8 49955136 +564 49278190 +407 49022194 +251 48828693 +345 46413707 +250 46095324 +511 42623671 +393 41629710 +484 41252315 +356 40985272 +475 40041980 +508 39889004 +517 36480426 +550 35941594 +587 34803895 +547 34523820 +546 33398226 +553 33091056 +543 32654778 +510 32035371 +663 32028126 +460 31691389 +530 31181535 +503 30862486 +635 30813519 +720 30660454 +607 30374808 +477 29369504 +706 29183313 +526 29041171 +14 28893906 +561 27738361 +470 26738514 +614 25458253 +618 24232023 +717 23994060 +673 23817299 +734 23792701 +625 23376942 +661 23220442 +317 22862326 +674 22516011 +632 22500762 +640 22453472 +621 22170426 +656 21469936 +612 21420897 +83 21318775 +679 21314775 +649 21268970 +851 21092011 +938 20404401 +655 20375026 +554 20334200 +584 20320611 +523 20315428 +644 20012607 +40 19422652 +588 19096246 +64 18759122 +617 18693984 +50 18238046 +26689 18079440 +606 17992787 +812 17864313 +6 17843244 +466 17817361 +534 17796224 +532 17532111 +352 17384084 +1 17279082 +611 17091775 +714 17025679 +30 16939428 +645 16677856 +72 16553037 +76 16327061 +651 15971344 +471 15879338 +783 15823492 +683 15819244 +736 15197650 +887 15053172 +784 14786686 +616 14556795 +705 14539133 +691 14309272 +1115 14176045 +26 14145184 +362 14098304 +464 14083981 +16 14033199 +1411 13989417 +1028 13787695 +878 13752356 +1664 13470232 +78 13378307 +1301 13160863 +703 13034870 +780 12998354 +597 12928745 +749 12878804 +852 12866041 +787 12811365 +810 12810008 +1141 12785466 +832 12685151 +981 12676060 +830 12643489 +770 12491952 +1510 12485834 +278 12398432 +513 12345382 +925 12242439 +880 12187173 +838 12095675 +866 12088892 +572 12004582 +1139 11932599 +502 11810743 +347 11778080 +1016 11554835 +1074 11537640 +775 11416721 +883 11147387 +1230 11002697 +835 10997448 +1135 10975044 +867 10945123 +788 10941163 +670 10932097 +1297 10878979 +785 10826031 +17 10797378 +983 10787263 +843 10673666 +259 10657207 +1941 10592731 +279 10574822 +845 10526221 +1110 10523541 +1363 10476406 +1011 10465302 +1285 10446380 +1201 10423577 +968 10348378 +743 10336013 +772 10297739 +1622 10289758 +766 10280263 +2177 10243413 +1181 10234334 +642 10188179 +276 10110644 +815 10077564 +1088 10066642 +2864 10065012 +1218 10051426 +514 10027697 +991 9981059 +881 9920784 +604 9911573 +922 9903273 +892 9899715 +4 9886396 +311 9824882 +777 9788574 +1910 9785844 +360 9705921 +400 9632736 +467 9594120 +821 9549150 +884 9547397 +760 9515151 +1390 9514008 +836 9399858 +88 9399371 +1306 9350994 +350 9326094 +750 9317800 +739 9296956 +910 9295771 +268 9233925 +406 9191046 +1022 9185932 +583 9178621 +509 9120375 +327 9057596 +718 8963492 +995 8936939 +636 8882717 +399 8811370 +826 8792653 +765 8755010 +1440 8704406 +828 8681852 +1029 8668925 +761 8595175 +260 8550153 +68 8521820 +1026 8495454 +1037 8482028 +20 8478672 +18 8372937 +1499 8372574 +371 8359700 +1644 8353078 +32 8336893 +890 8325716 +1119 8309982 +886 8287585 +263 8275838 +309 8275603 +337 8268937 +84 8260458 +1111 8203580 +994 8202575 +272 8193169 +261 8189103 +767 8122128 +390 8069108 +1375 7988935 +1597 7984150 +989 7938818 +73 7922334 +364 7880395 +1107 7805773 +1992 7800278 +283 7777203 +402 7732480 +3217 7712997 +376 7593883 +1266 7589093 +976 7577523 +1194 7566250 +900 7556294 +727 7550625 +1320 7507098 +292 7495866 +77 7470439 +1282 7450955 +1641 7411391 +1171 7409099 +1114 7363929 +1081 7347040 +15 7312426 +367 7306406 +807 7279240 +1160 7272825 +1936 7262915 +274 7253218 +3431 7220578 +299 7218583 +3635 7152871 +3860 7135265 +71 7117156 +1353 7113713 +1392 7090351 +1204 7074121 +3321 7040732 +1043 7037776 +779 7012062 +370 6995545 +19 6983878 +3583 6974965 +898 6966637 +1864 6959506 +711 6952288 +905 6939061 +520 6938890 +582 6920647 +1364 6908579 +1578 6875577 +1105 6855797 +1295 6829761 +1002 6812464 +1256 6769157 +1966 6727321 +657 6641494 +737 6624794 +1104 6593949 +494 6588847 +2997 6571290 +256 6561739 +7303 6545519 +0 6525880 +89 6511169 +74 6481093 +1812 6451687 +2173 6415053 +1448 6412987 +1524 6397024 +1321 6394244 +1584 6392878 +282 6358319 +81 6351445 +1592 6336754 +1705 6331987 +973 6315804 +1234 6313784 +1748 6306370 +449 6287674 +1318 6264091 +1271 6240689 +34 6237906 +1053 6231567 +1123 6220759 +1165 6216478 +1839 6189790 +306 6174365 +1227 6170682 +271 6158328 +2087 6143528 +804 6141406 +1365 6119704 +790 6119691 +1222 6097096 +1528 6093784 +860 6086478 +1718 6080393 +1755 6062347 +304 6058377 +1367 6044404 +418 6021291 +1178 5961986 +273 5944078 +2258 5884266 +921 5875428 +2368 5843037 +1049 5834912 +1444 5825616 +1550 5810506 +1613 5807417 +1625 5806489 +1933 5804564 +3909 5804009 +1315 5793456 +1263 5781210 +412 5780871 +1294 5756002 +1243 5755617 +440 5703257 +288 5696335 +923 5686348 +33 5683530 +4283 5662615 +1542 5657499 +1466 5655969 +2520 5626252 +1737 5617548 +1239 5613802 +1893 5604800 +3502 5592880 +1231 5592559 +805 5586720 +23 5579916 +1422 5562000 +1957 5554673 +21 5545853 +1223 5537815 +1339 5525740 +1439 5518540 +270 5516122 +22 5511292 +1406 5508724 +1751 5500821 +1497 5473031 +1310 5460960 +2237 5456445 +2254 5424602 +3418 5378471 +1366 5362221 +265 5361225 +1541 5359232 +67 5328572 +1637 5322515 +1903 5319711 +1973 5285283 +2938 5283708 +1057 5281356 +1568 5274805 +321 5273108 +2756 5236169 +1830 5223345 +1770 5222102 +65 5208299 +1244 5204410 +1180 5203669 +2098 5169445 +1730 5168645 +2056 5168496 +3349 5156053 +2055 5138614 +2807 5130949 +1101 5123031 +66 5103752 +1816 5093006 +1400 5076411 +1498 5071579 +1642 5055917 +1989 5044992 +1290 5034040 +2643 5023350 +2097 5022728 +1762 5015331 +44 5015012 +479 5008250 +1775 5005597 +2706 5005225 +1909 4997835 +1866 4990678 +1566 4981122 +1336 4950527 +757 4941775 +2063 4937397 +2648 4929257 +293 4925771 +1464 4911409 +2184 4905422 +75 4894914 +392 4889056 +1487 4877439 +1064 4865397 +24 4854296 +1080 4852641 +569 4850805 +1971 4849650 +1605 4847284 +1182 4846260 +1938 4828745 +857 4805157 +1535 4772487 +285 4766512 +1176 4764682 +966 4760238 +2277 4743915 +764 4731348 +1377 4728044 +1479 4720640 +1539 4714734 +1085 4700915 +1811 4696785 +2274 4657189 +869 4652756 +45 4649060 +1099 4644445 +1394 4638480 +1280 4637662 +3000 4618471 +1577 4618338 +544 4614094 +2805 4608260 +35 4606393 +2351 4602990 +1629 4586377 +1661 4584310 +2003 4567755 +49 4546496 +1478 4546419 +2795 4542206 +2828 4536638 +1248 4526225 +1593 4511113 +69 4502347 +2457 4489997 +1511 4484539 +1881 4472447 +47 4460868 +1708 4455874 +1097 4450969 +1551 4445924 +1660 4433465 +1785 4424736 +1627 4412038 +1445 4401529 +2594 4393865 +1719 4389889 +1649 4380458 +2444 4375390 +4287 4371881 +417 4370421 +716 4365322 +1168 4364296 +1735 4360877 +1621 4331683 +2233 4330036 +3249 4325097 +42 4320291 +1276 4314178 +1829 4314165 +1884 4312560 +38 4306679 +2555 4304404 +2084 4296432 +2151 4287967 +1688 4285173 +2831 4280062 +1342 4276707 +1270 4239023 +555 4236623 +1327 4234882 +2139 4227635 +1467 4219535 +2045 4208129 +2714 4198810 +303 4195216 +1771 4185532 +2901 4181081 +2077 4179380 +1863 4178336 +1965 4175165 +2067 4175032 +1716 4169486 +2651 4155215 +2267 4147302 +2607 4145837 +1964 4133545 +1462 4126396 +1978 4126179 +1972 4121510 +1410 4119035 +1679 4106021 +51 4105433 +1871 4105196 +2406 4102924 +2551 4097217 +2008 4097102 +1853 4093083 +70 4092580 +2293 4087207 +2324 4083845 +43 4083418 +1337 4082770 +1813 4079395 +1695 4077362 +960 4077221 +264 4076617 +2688 4072140 +1183 4070745 +1414 4064159 +1474 4057078 +2282 4053024 +3414 4042308 +1430 4037596 +3035 4031658 +1103 4018655 +2059 4015508 +2080 4011218 +2969 4005397 +1919 4000144 +1969 3997080 +316 3993493 +1459 3984707 +1521 3978369 +37 3969953 +1675 3968119 +3009 3965772 +996 3965252 +1596 3943972 +2263 3941497 +3457 3938197 +1450 3937663 +86 3925608 +2058 3919358 +1636 3917053 +1804 3911665 +1429 3909332 +1757 3906464 +354 3891128 +405 3889614 +3176 3888837 +1877 3882885 +1576 3878715 +2893 3873729 +2252 3872753 +1281 3863057 +1254 3862011 +301 3858937 +1048 3852378 +3203 3851712 +2159 3847206 +1626 3842112 +324 3832573 +1760 3832298 +1169 3818301 +2739 3816048 +1687 3814765 +1595 3811813 +1517 3803324 +2260 3791037 +1693 3787455 +1262 3785788 +2102 3779927 +291 3777762 +1923 3777288 +1700 3776768 +2157 3771717 +1378 3757930 +2732 3755186 +79 3754633 +1854 3745778 +3269 3737875 +1502 3737616 +685 3723444 +4200 3719859 +1865 3719356 +128 3715003 +1402 3710786 +2168 3703789 +1986 3699485 +1867 3696280 +2026 3695382 +1683 3694355 +2961 3691575 +1842 3680139 +929 3678416 +2489 3665779 +1052 3661007 +396 3659796 +3329 3643884 +2669 3638778 +1862 3622381 +3452 3605249 +3794 3602291 +2111 3590156 +2046 3587528 +2957 3581398 +1913 3580361 +1441 3577048 +1241 3568130 +46 3565250 +2811 3562952 +2278 3561460 +1998 3550042 +461 3548528 +1744 3532659 +1975 3526648 +2291 3521569 +3056 3518738 +2904 3518633 +1752 3511388 +1900 3510560 +2626 3506312 +1654 3504513 +385 3502364 +2745 3487380 +2057 3487072 +3136 3485766 +7955 3485171 +4139 3481632 +2415 3480433 +2148 3480049 +1628 3467067 +2071 3466219 +2107 3463315 +940 3460357 +1598 3457691 +258 3455533 +1575 3454361 +2826 3452135 +2716 3449165 +3985 3445703 +85 3445461 +1987 3443747 +3598 3439871 +3352 3431656 +2478 3424520 +333 3421332 +246 3418219 +2620 3415717 +1212 3412196 +2450 3409727 +1247 3405540 +1912 3399689 +36 3395391 +346 3391001 +3426 3380470 +3298 3379545 +3292 3377200 +2250 3371380 +2440 3369691 +3061 3367419 +39 3363104 +978 3353736 +1802 3350527 +2431 3348906 +3071 3340128 +2253 3337972 +2494 3334848 +609 3333865 +2310 3329148 +986 3328812 +2635 3325356 +3437 3320853 +2292 3319741 +2823 3308131 +1588 3303360 +269 3302371 +275 3284415 +60 3282646 +2428 3276582 +1918 3276387 +2615 3273427 +2472 3272517 +1690 3267675 +410 3265844 +2678 3262749 +2106 3260749 +2354 3251238 +2717 3247356 +678 3244526 +1109 3242666 +3334 3241700 +3451 3238451 +320 3236458 +3230 3233294 +3389 3229315 +2166 3227294 +1611 3224985 +1994 3213613 +430 3209260 +2986 3199943 +1790 3194716 +1438 3193856 +4784 3192749 +1781 3170903 +302 3166428 +2227 3162561 +54 3145229 +2693 3138924 +1393 3138049 +2597 3137970 +2482 3137124 +3034 3122439 +1946 3121857 +2863 3119047 +3267 3115876 +2041 3113770 +1743 3107914 +2476 3105231 +388 3102434 +300 3100235 +3186 3098789 +1729 3098376 +2488 3094662 +5018 3092842 +4058 3079283 +2156 3078111 +52 3074167 +3096 3072323 +1468 3071877 +2497 3070835 +2793 3050336 +3427 3047066 +1630 3040837 +3284 3037800 +3624 3034708 +2650 3033943 +2785 3033180 +1807 3027961 +3645 3026379 +2691 3025436 +3106 3024747 +3037 3023165 +3759 3023164 +312 3020879 +1767 3018684 +2526 3018183 +666 3015679 +3139 3012306 +3085 3009667 +2223 3002610 +4041 3002353 +2712 3001744 +1838 2997522 +2048 2983869 +2854 2981556 +2534 2972131 +308 2969299 +2646 2967019 +3016 2965071 +3337 2960427 +3187 2957831 +4912 2956818 +3331 2956176 +1643 2956098 +2722 2953729 +2932 2951114 +2422 2950537 +2399 2948398 +500 2946582 +4039 2945677 +3961 2944538 +2222 2943764 +3078 2943739 +4275 2942029 +1724 2934719 +911 2931322 +3296 2930626 +384 2925764 +2319 2924706 +1238 2912540 +1911 2911206 +53 2910401 +2005 2910213 +2923 2909079 +1303 2908146 +4536 2904452 +2921 2898494 +3530 2896507 +343 2894182 +575 2892577 +3058 2891202 +277 2889780 +323 2886056 +710 2881312 +660 2874230 +1949 2873478 +3250 2868743 +225 2861798 +41 2858852 +1808 2848588 +1021 2846040 +3773 2842914 +7713 2841920 +540 2838877 +2137 2837279 +2750 2836122 +3271 2833311 +2994 2832696 +397 2832081 +2174 2831245 +2630 2825882 +1073 2823768 +378 2822150 +2491 2819311 +403 2817676 +2540 2811122 +2060 2808168 +2214 2807667 +2242 2804699 +3554 2801970 +266 2800975 +3442 2799863 +5544 2795504 +1682 2795443 +1351 2777650 +297 2776601 +3155 2770111 +2050 2768526 +3466 2759754 +1544 2759525 +993 2754965 +3340 2752396 +8591 2751808 +1255 2750444 +1895 2750214 +3015 2746600 +3125 2744902 +3945 2744846 +6426 2744124 +2897 2740354 +1309 2739832 +959 2737933 +2822 2737646 +1368 2733555 +2042 2730078 +374 2728295 +3006 2714274 +2245 2700521 +2928 2694744 +2872 2687504 +4896 2686827 +4297 2685685 +2766 2685288 +444 2682283 +2888 2681984 +1200 2679658 +2975 2678829 +377 2675721 +1988 2675064 +2523 2673705 +1583 2671163 +1024 2667070 +415 2666262 +3576 2658993 +2119 2657291 +2647 2648808 +3227 2648233 +1997 2646862 +4081 2645756 +4094 2645293 +1633 2637801 +1917 2637232 +2276 2635825 +2492 2634522 +1312 2634263 +2839 2633915 +2592 2632902 +3662 2624861 +3224 2624698 +1766 2624083 +3663 2624035 +1745 2621047 +5 2620736 +2300 2619855 +4664 2619338 +3430 2619137 +2130 2618208 +6184 2618030 +3687 2611608 +13130 2607739 +2637 2602497 +2622 2597101 +3700 2596588 +2435 2591941 +2158 2587673 +2279 2584888 +2506 2577787 +3724 2574566 +2950 2573209 +2460 2568568 +2125 2566267 +2861 2562749 +1134 2549917 +5454 2544616 +3751 2536696 +1858 2535706 +2579 2530192 +1826 2529534 +2608 2528860 +2681 2527523 +56 2526960 +3814 2525489 +4332 2524158 +2735 2523828 +3367 2523419 +2272 2516165 +3756 2511014 +2585 2509794 +5041 2503584 +4248 2503218 +2802 2502456 +2180 2500659 +3482 2499158 +3899 2496197 +2666 2495174 +395 2490074 +368 2486179 +1976 2484836 +2773 2481413 +669 2475721 +448 2470404 +1314 2468787 +1175 2466968 +3052 2465830 +3491 2465693 +55 2458697 +305 2457793 +2496 2455621 +2241 2454504 +1210 2453199 +2031 2450641 +3111 2447239 +2568 2446982 +7781 2446876 +1635 2445064 +2582 2444049 +2613 2443100 +3195 2441989 +5079 2432713 +2211 2428055 +3234 2426818 +4037 2426428 +1549 2425595 +5991 2421705 +4495 2417713 +952 2416570 +267 2415840 +2458 2414786 +328 2414598 +3790 2413453 +2641 2411736 +1296 2408790 +3199 2407660 +3072 2407258 +763 2402573 +2742 2402326 +4640 2400825 +1907 2399123 +654 2395466 +2911 2394784 +3931 2392276 +3818 2385503 +4346 2385039 +3119 2384203 +31 2383468 +1492 2381026 +3397 2380456 +3484 2379083 +330 2378895 +4706 2377588 +2251 2376885 +2479 2374155 +3053 2371784 +5939 2368766 +1388 2368606 +1692 2366880 +1908 2363611 +4542 2356967 +3596 2356312 +1122 2352389 +5003 2349430 +2962 2349359 +1607 2349127 +3047 2347321 +2627 2346853 +3025 2342305 +2995 2337450 +2835 2335936 +1004 2333657 +3214 2332768 +2029 2332229 +13440 2330317 +1561 2324920 +3074 2315814 +380 2312070 +515 2311421 +365 2310632 +3382 2306041 +363 2305310 +3160 2304973 +296 2303562 +435 2300111 +3512 2297054 +3747 2295650 +1334 2294588 +4281 2294310 +2614 2292185 +2524 2284943 +3394 2281882 +3095 2281714 +2147 2281124 +2187 2279131 +2855 2275319 +2702 2272741 +3517 2271280 +325 2268773 +3520 2268531 +1065 2264589 +3215 2261296 +4395 2260201 +2652 2259754 +1120 2259547 +648 2258982 +4436 2257188 +2089 2255914 +2209 2255859 +4969 2249342 +3022 2248989 +4530 2248944 +2330 2247322 +3261 2246707 +1532 2245244 +12042 2243318 +2270 2243237 +1657 2239438 +2846 2238923 +3999 2237519 +3845 2237334 +2271 2233955 +2126 2233086 +499 2231417 +1659 2229762 +5193 2228394 +3688 2226618 +1382 2226353 +446 2225805 +818 2224123 +2092 2222423 +3623 2220823 +5373 2220082 +2321 2219693 +5057 2219662 +1526 2219349 +5169 2218821 +2912 2217212 +3220 2216122 +3315 2215806 +2808 2214953 +2365 2214929 +2628 2212471 +3805 2211616 +2121 2211232 +1560 2204752 +3259 2204036 +3240 2203146 +2634 2202115 +3656 2200682 +2683 2199255 +3767 2197871 +2612 2193603 +1138 2190750 +3181 2189669 +4193 2189626 +3162 2186721 +2239 2185546 +2988 2185413 +2589 2185214 +1720 2185135 +2192 2184857 +4387 2184827 +4038 2180149 +4492 2178553 +1249 2178508 +3599 2177234 +3988 2176292 +4519 2175412 +2010 2173733 +3965 2173661 +4149 2170484 +3833 2170048 +3017 2169734 +1100 2168903 +4884 2168582 +349 2167597 +2035 2164690 +4040 2163330 +3407 2162686 +3415 2161099 +680 2159475 +1399 2158929 +4661 2157800 +1228 2155490 +551 2154941 +87 2154822 +4376 2154446 +559 2153394 +2392 2153055 +315 2150386 +2342 2150379 +3936 2150034 +1639 2148662 +2837 2143071 +358 2142250 +5153 2141503 +3793 2139387 +2940 2139279 +3126 2139153 +8358 2138790 +1477 2137623 +2720 2137372 +2138 2135247 +5085 2134103 +3957 2132520 +662 2131237 +3432 2125446 +2663 2125055 +8428 2121327 +2408 2121234 +1051 2121150 +3012 2120925 +3740 2120211 +3362 2118913 +2877 2111347 +820 2109304 +1195 2108223 +528 2107435 +2297 2104723 +1956 2104400 +2990 2101876 +5567 2098776 +62 2098771 +2312 2098237 +1570 2097507 +4086 2094834 +1738 2094385 +3142 2094215 +4505 2092752 +1031 2081290 +797 2079864 +2900 2079818 +1157 2079615 +3277 2079046 +3492 2078225 +1803 2078060 +4466 2077826 +4445 2076000 +5953 2074891 +4172 2072344 +2383 2070424 +3274 2062552 +2405 2061430 +474 2059389 +4539 2058329 +5371 2054600 +2753 2053126 +3377 2051234 +2884 2050465 +313 2048274 +1437 2048095 +1495 2047894 +1044 2047797 +1672 2046882 +4773 2046230 +3128 2045207 +4444 2043458 +439 2043235 +12637 2038256 +5692 2037358 +504 2035309 +3307 2032332 +2323 2031475 +2495 2028884 +4196 2027995 +341 2026813 +4176 2023899 +5834 2020359 +4744 2020302 +2563 2016917 +2882 2013222 +505 2010517 +3707 2009054 +1612 2007764 +4652 2006805 +2687 2006766 +5342 1994135 +3670 1992183 +4375 1990482 +2761 1988748 +4756 1987768 +3403 1986706 +2266 1985660 +3066 1985309 +129 1983048 +4481 1981282 +4354 1979172 +2033 1978125 +4576 1977385 +1943 1973375 +2370 1972674 +2486 1971043 +3090 1969680 +2810 1969527 +5401 1967900 +4381 1967895 +3800 1966998 +641 1966963 +2776 1966928 +3611 1965109 +6567 1965030 +3710 1963705 +803 1963552 +1332 1963452 +1600 1962893 +5442 1959629 +2936 1959617 +2723 1956891 +57 1956274 +2331 1955550 +2728 1955126 +4266 1954189 +3708 1952903 +4155 1951761 +3236 1951553 +2011 1949390 +3893 1944470 +3715 1943834 +3501 1942268 +3641 1942249 +3967 1941157 +5695 1939245 +3946 1939135 +1848 1938862 +2982 1935927 +3081 1935780 +2842 1935540 +3393 1929631 +4409 1927034 +533 1926581 +1208 1925558 +6123 1924601 +3328 1919903 +3073 1919157 +5478 1915260 +3690 1915130 +992 1914387 +3519 1914014 +3677 1913098 +4479 1909848 +5555 1908826 +353 1908475 +2952 1907503 +1374 1906721 +2972 1906704 +3151 1906344 +2298 1905945 +5047 1905773 +2407 1905690 +298 1904852 +80 1903294 +1040 1903278 +4590 1902254 +1833 1899229 +5595 1897899 +4251 1897347 +2610 1895735 +2724 1894339 +3626 1892750 +4244 1890732 +4318 1889199 +4957 1886758 +2775 1885004 +5095 1881967 +4721 1880372 +7198 1879205 +3892 1878736 +1558 1875025 +3884 1874312 +3942 1869131 +3206 1868585 +2499 1868244 +2562 1867987 +1507 1866658 +4838 1862252 +289 1862164 +1645 1859225 +2700 1855499 +1754 1855200 +3772 1851086 +4888 1850666 +4045 1848988 +3867 1847435 +157 1843867 +4493 1842927 +2619 1838354 +4786 1836011 +1398 1834121 +3088 1832988 +4120 1831679 +2695 1831610 +5665 1829988 +3381 1828595 +2427 1828369 +4452 1825877 +4477 1823113 +2974 1821399 +6672 1821145 +1814 1821079 +5466 1821016 +4639 1820572 +1265 1819995 +2314 1818041 +1008 1816897 +437 1815421 +3423 1814262 +3245 1812767 +3650 1811355 +2503 1810984 +1728 1810732 +2116 1810704 +8872 1806001 +2332 1805620 +2968 1804542 +1061 1803718 +4581 1802885 +1525 1802513 +3888 1799212 +2925 1794822 +2727 1794109 +1302 1794082 +4560 1793094 +3114 1791585 +3513 1791504 +709 1790178 +4783 1789889 +4488 1789224 +2832 1789080 +2746 1788885 +5652 1787699 +331 1785106 +4865 1784176 +3739 1782930 +4586 1782099 +3877 1780375 +5059 1780266 +2869 1779960 +4485 1778625 +6130 1774579 +3940 1772616 +4271 1768597 +2985 1768596 +4918 1768371 +5030 1768004 +220 1767607 +280 1766245 +1872 1766073 +3706 1765293 +1545 1759778 +5267 1758636 +3761 1757768 +5491 1757560 +2104 1756508 +7313 1756194 +4054 1755775 +2574 1753800 +3667 1753746 +3651 1753739 +4405 1752238 +571 1751391 +4308 1751021 +5437 1750489 +1944 1749114 +1355 1748662 +4274 1746718 +8278 1743712 +578 1742168 +2694 1741205 +2656 1738365 +2636 1738002 +4513 1737002 +2185 1736027 +5449 1728990 +2813 1728933 +2993 1728096 +2587 1728026 +615 1727522 +4881 1726170 +4776 1724711 +2565 1724643 +2726 1722884 +4999 1718878 +3098 1718690 +3685 1717832 +2987 1717445 +1559 1716265 +10205 1715772 +3486 1713237 +427 1712990 +2583 1712089 +4995 1711510 +3871 1710985 +5134 1710681 +4902 1710660 +4602 1710486 +620 1710125 +5537 1709921 +3170 1709734 +1892 1706520 +3173 1705945 +2073 1704583 +5011 1703887 +2346 1703108 +786 1702332 +4059 1699595 +1844 1697348 +295 1696098 +4473 1694758 +4696 1694438 +4280 1692697 +3241 1692185 +3067 1691819 +5694 1688571 +563 1687198 +4987 1687182 +2000 1686158 +1982 1680689 +4317 1679080 +2800 1678829 +2743 1676341 +3782 1676203 +4136 1675550 +4141 1675316 +3011 1674431 +404 1671943 +4508 1671300 +3848 1671224 +6341 1671141 +1486 1670636 +3338 1670469 +4394 1670223 +3357 1669888 +3226 1669751 +6557 1668766 +1359 1668112 +2081 1664853 +2364 1664159 +3146 1663984 +1257 1663233 +2193 1661698 +3294 1659449 +1148 1658760 +411 1658023 +874 1655097 +2219 1654452 +1143 1654129 +4427 1653889 +5583 1651423 +3954 1651313 +2779 1651054 +3216 1650840 +5396 1650681 +2150 1649743 +3127 1647955 +4373 1645141 +3164 1644786 +1433 1644622 +5201 1644557 +2423 1643559 +2672 1641242 +598 1640066 +1869 1639116 +3926 1637219 +6956 1635903 +4390 1635697 +1485 1635611 +6180 1633463 +4186 1632566 +11033 1632005 +10575 1631124 +5398 1630774 +4152 1630130 +6523 1629814 +2708 1628267 +3371 1627540 +4842 1624633 +3807 1624226 +1415 1621531 +414 1620770 +3386 1620487 +5180 1618946 +4956 1618827 +3033 1618817 +4854 1618654 +1395 1617375 +4418 1616971 +3275 1614077 +496 1613939 +4367 1612988 +3115 1611458 +4138 1611246 +707 1609966 +3941 1609655 +943 1608211 +4689 1605650 +541 1602489 +1220 1600656 +5856 1600558 +3465 1599819 +2014 1599485 +2642 1597905 +361 1597790 +3804 1596713 +4930 1595012 +4656 1594232 +4032 1591856 +2094 1591768 +4486 1590377 +3850 1589189 +3417 1587932 +4068 1587210 +6484 1587158 +3573 1586303 +751 1584930 +5273 1584119 +1001 1582810 +4511 1582438 +1350 1582111 +5682 1581323 +5701 1579275 +3750 1578369 +1215 1578106 +1288 1575560 +6443 1574356 +2456 1574032 +4305 1572178 +6786 1570930 +1722 1567616 +5070 1564834 +2391 1563465 +5229 1561859 +4992 1560941 +5828 1560570 +689 1560524 +1000 1560250 +3469 1559413 +3 1558890 +375 1558286 +4789 1557386 +3288 1557204 +5361 1556430 +829 1555089 +602 1553100 +6502 1552435 +3869 1550400 +482 1550363 +3769 1550026 +7 1548704 +695 1548541 +2221 1548436 +3615 1544415 +1891 1544125 +6186 1543528 +1129 1542997 +3741 1540855 +3412 1539970 +2198 1539205 +6193 1536595 +4009 1535736 +3923 1530812 +1879 1530386 +4137 1528607 +4502 1527678 +2513 1527637 +506 1527207 +5717 1527072 +4796 1526999 +5611 1525342 +4438 1524952 +7324 1524876 +5103 1524376 +3932 1523931 +4983 1523249 +2644 1522609 +3354 1522133 +4890 1521795 +3730 1521634 +42159 1521590 +1077 1521212 +1203 1518704 +5531 1515641 +4606 1515536 +3218 1515150 +6745 1513200 +5006 1513115 +5890 1512226 +5052 1511945 +4970 1510725 +6714 1508429 +4030 1506853 +1245 1505561 +3675 1504974 +4426 1504340 +3375 1503790 +3177 1503516 +382 1502511 +39711 1501599 +336 1500357 +1961 1499155 +3504 1498266 +5298 1496420 +732 1496386 +4752 1496009 +2879 1495099 +4260 1494843 +1041 1494107 +4746 1494014 +2556 1493772 +2074 1493385 +8734 1492850 +4979 1492643 +5510 1489374 +3521 1488249 +3812 1487987 +2989 1487864 +1537 1487614 +4219 1485195 +6686 1483477 +2504 1482854 +3584 1481993 +4568 1481365 +3421 1481356 +4237 1481211 +4814 1480648 +334 1479315 +3439 1479269 +3002 1477903 +1417 1477126 +930 1475368 +322 1474128 +1546 1473723 +4647 1473119 +488 1473092 +4073 1473078 +7024 1472792 +1531 1472346 +1797 1471634 +3625 1471454 +2409 1470321 +2196 1468500 +1663 1466759 +5672 1466160 +3689 1465868 +1981 1465495 +2858 1464549 +5199 1464208 +6916 1463903 +3574 1462903 +525 1461357 +3301 1459805 +3341 1459758 +1805 1458557 +2677 1458359 +6821 1458176 +3443 1458132 +4523 1457686 +1821 1456407 +8064 1455853 +2485 1453972 +7648 1453898 +8549 1453728 +3701 1452597 +344 1452384 +5471 1451483 +434 1451045 +624 1450197 +386 1449819 +719 1449119 +3859 1448585 +3092 1447520 +5166 1447492 +5323 1447286 +4585 1446385 +560 1444813 +4056 1444800 +7530 1442047 +3049 1440630 +5399 1440467 +1810 1439368 +39883 1438860 +4619 1438062 +6047 1437307 +4100 1435643 +4403 1435094 +6233 1434841 +4875 1431868 +2215 1430928 +518 1430530 +6798 1430070 +4379 1429963 +2191 1429101 +5212 1428466 +5926 1426982 +3726 1426439 +5127 1425739 +4608 1425400 +3770 1423320 +684 1423301 +3399 1423143 +549 1422979 +1471 1422620 +1326 1422545 +4034 1422191 +4414 1421600 +5260 1421418 +3285 1421044 +4671 1419983 +4388 1419806 +2329 1419508 +3117 1418665 +2512 1418078 +3614 1417918 +5296 1417278 +4380 1416914 +4691 1415979 +1242 1415603 +1042 1415013 +3094 1413700 +3059 1413465 +4019 1411855 +5658 1410605 +4101 1410072 +14420 1409152 +2576 1408914 +7229 1406591 +6095 1406131 +3729 1402106 +6168 1401550 +5535 1401292 +5348 1399444 +3131 1399224 +3050 1399104 +1108 1399004 +4769 1398370 +516 1395487 +5802 1392661 +671 1392187 +5175 1391611 +12216 1390973 +2141 1390969 +3764 1390483 +5158 1389181 +4504 1386980 +2546 1384925 +4301 1384339 +10767 1383934 +6011 1383767 +1793 1383582 +4809 1380537 +2443 1380319 +372 1379345 +2769 1378891 +3822 1378669 +4065 1378474 +7092 1378135 +576 1377793 +8060 1377625 +5150 1375001 +391 1374575 +6246 1374163 +5618 1372598 +4392 1372508 +442 1371294 +2372 1370549 +7636 1368619 +4734 1367530 +2679 1367016 +4811 1365243 +5068 1363461 +2751 1362395 +4986 1361444 +48 1361374 +7638 1360237 +3064 1359735 +9747 1358672 +5747 1358332 +2947 1357419 +2692 1356699 +3613 1356588 +912 1355680 +359 1353287 +4268 1353229 +10395 1351988 +4650 1350702 +3368 1350513 +939 1350079 +452 1349984 +2465 1349409 +4006 1347932 +5710 1347611 +3420 1346666 +5410 1346311 +557 1346053 +2657 1345538 +3562 1344045 +4646 1343369 +1565 1343288 +791 1342812 +3841 1342808 +3244 1342552 +8301 1340601 +15069 1340247 +6290 1339930 +5780 1339030 +5928 1338936 +2231 1338201 +469 1338045 +5123 1337766 +4046 1337492 +1370 1337438 +5742 1336548 +6027 1336530 +3895 1335611 +463 1334739 +2099 1333298 +2763 1332891 +5094 1332269 +6592 1330873 +5096 1330694 +3434 1330495 +11421 1330487 +11092 1330119 +4133 1329427 +2611 1328981 +1701 1328763 +1703 1326543 +2362 1326290 +4574 1325119 +6136 1323601 +1954 1320496 +6334 1318454 +8464 1316365 +2910 1316215 +5093 1315678 +3158 1315478 +5156 1314587 +4314 1314066 +4341 1314003 +409 1313875 +5054 1312722 +4326 1312569 +3327 1311762 +4587 1310546 +7994 1310530 +3436 1310399 +1634 1309798 +4787 1308208 +3842 1307129 +3336 1303877 +1453 1303867 +527 1303388 +3819 1302492 +4870 1302035 +1711 1301722 +4736 1301545 +3280 1301512 +485 1300709 +3938 1300533 +988 1299815 +1023 1299461 +5197 1298056 +3691 1298022 +3785 1297957 +3265 1296350 +2398 1296348 +6729 1296290 +5033 1295463 +3409 1294973 +5913 1291479 +4654 1290537 +7630 1288623 +3709 1286675 +6376 1286193 +3781 1285807 +7756 1285579 +10171 1285322 +2818 1284996 +1014 1282901 +4097 1282603 +4813 1282045 +3038 1281458 +5341 1281418 +420 1281046 +4955 1280803 +8836 1279703 +1731 1278908 +3636 1278857 +3392 1278431 +7392 1277991 +3194 1276594 +4446 1276547 +4540 1275393 +4099 1274105 +4788 1273253 +4167 1272413 +459 1272249 +1948 1271480 +1640 1270835 +7176 1270571 +4429 1269217 +4042 1269107 +2740 1268901 +5866 1268234 +4737 1267826 +6299 1267725 +8150 1266632 +3099 1265853 +19398 1265684 +1878 1265115 +4893 1265091 +2318 1263652 +5627 1263447 +8909 1263276 +5174 1263166 +2829 1261919 +4688 1261278 +6289 1261136 +4765 1258870 +4525 1258646 +4940 1258077 +433 1257886 +4675 1257299 +3950 1257106 +4334 1255854 +4371 1255469 +4478 1255023 +6459 1254251 +3958 1253900 +5364 1253811 +2327 1253249 +119 1252957 +3264 1252530 +3588 1252368 +1624 1252185 +4885 1251484 +521 1251476 +3607 1251163 +7420 1251114 +3572 1250281 +536 1250114 +6325 1249281 +5140 1249176 +5284 1248712 +1828 1247137 +4978 1245733 +4708 1244437 +6825 1243796 +3968 1241799 +6280 1240647 +6835 1240422 +12052 1240210 +6023 1236586 +1283 1236065 +5804 1235634 +4762 1234227 +2095 1233277 +2447 1232432 +445 1231550 +3734 1231293 +3161 1231023 +3612 1230155 +2638 1229977 +1343 1228431 +6516 1227971 +5137 1227542 +7014 1227326 +6638 1226525 +2275 1225586 +5670 1225366 +15862 1224702 +5426 1222773 +1338 1222222 +4569 1222157 +12499 1221455 +4043 1221352 +2043 1221192 +2581 1221146 +622 1220070 +1547 1219481 +1096 1219401 +5461 1219355 +3657 1219079 +4966 1218169 +1137 1218103 +5366 1217930 +2866 1216988 +3356 1216655 +4641 1215786 +5163 1214894 +1656 1214167 +3825 1213380 +443 1212646 +3863 1212582 +6078 1212500 +2426 1212420 +425 1212037 +647 1211602 +7320 1211287 +1086 1211104 +3568 1210892 +4621 1210058 +5741 1209984 +4763 1209467 +4698 1209089 +2948 1208943 +2461 1208205 +1714 1207871 +7372 1205825 +5689 1205245 +4533 1205189 +3084 1205002 +5112 1204702 +3148 1204416 +4205 1203481 +3933 1202908 +11302 1202825 +7968 1201690 +4410 1200581 +6491 1199580 +6884 1199477 +9027 1199088 +6476 1197258 +2402 1196830 +3272 1196577 +4946 1196366 +1847 1195644 +2068 1195640 +2230 1194751 +5668 1194054 +6841 1193691 +6628 1192538 +5617 1192353 +3704 1192118 +454 1190907 +4113 1190819 +3360 1188180 +3595 1188055 +6388 1186942 +3024 1186481 +5676 1186075 +2616 1185977 +5076 1185573 +6997 1185148 +1573 1184210 +2502 1184197 +3956 1183015 +3306 1182352 +4191 1182121 +2907 1181996 +3758 1181687 +2933 1181593 +12820 1181273 +3505 1181150 +2481 1181067 +4258 1180761 +5339 1179754 +4497 1178907 +4705 1178786 +8424 1178672 +6989 1178672 +7225 1178011 +519 1177463 +5613 1177032 +837 1176942 +4203 1176560 +2736 1176366 +7067 1176283 +726 1175669 +2926 1175583 +5839 1175102 +4028 1175019 +6898 1174845 +46444 1173449 +3518 1172575 +2176 1172215 +3344 1171044 +6682 1170042 +4286 1170039 +4075 1168934 +3925 1168605 +3516 1168434 +7915 1167772 +6314 1167088 +1666 1166890 +6586 1165644 +3896 1164453 +1205 1163886 +4315 1163570 +6717 1163432 +4048 1163167 +6301 1162084 +7281 1161824 +3589 1161761 +1045 1161449 +3335 1160825 +3840 1160664 +4406 1160456 +4330 1160159 +1340 1159444 +871 1158633 +5995 1158628 +3722 1158586 +4618 1158428 +5010 1157604 +4564 1157265 +3190 1156679 +4369 1156412 +4047 1154902 +6342 1154157 +5707 1154075 +4393 1153892 +3355 1153701 +1413 1152770 +6542 1152768 +6447 1152716 +7366 1152389 +5382 1152114 +4683 1151509 +2311 1150108 +2328 1149589 +5179 1149446 +3496 1149070 +4206 1148604 +3748 1146891 +4425 1146252 +5311 1145728 +6702 1145147 +4398 1144278 +3952 1144184 +4795 1143514 +4900 1141559 +6858 1140893 +3348 1140558 +4166 1140194 +7396 1138944 +1277 1138668 +5001 1137748 +3446 1137430 +10662 1136646 +4858 1136361 +3091 1136277 +8783 1135186 +2605 1135155 +3809 1134429 +3251 1134183 +6983 1133029 +4673 1132793 +6025 1131898 +473 1131860 +1582 1131777 +8900 1131533 +5942 1131397 +5448 1131115 +5845 1131004 +3665 1130444 +4153 1129425 +3580 1129156 +6612 1129095 +5394 1129005 +30494 1128381 +421 1128217 +2883 1128149 +6922 1128102 +7044 1128058 +3788 1126787 +591 1126267 +4067 1123860 +4632 1123334 +5871 1122772 +3653 1121714 +6265 1121635 +2061 1121351 +1709 1121162 +3648 1120782 +7172 1120292 +2112 1120153 +2566 1119761 +4165 1119543 +2039 1118103 +4077 1117991 +5213 1117193 +2939 1116899 +9952 1116227 +11214 1115457 +6294 1115417 +6182 1114988 +1236 1114768 +4676 1113847 +14018 1113731 +5818 1113054 +6568 1112793 +3649 1112551 +7945 1111652 +4290 1111506 +11063 1111148 +7478 1110529 +5664 1110127 +2561 1108755 +11419 1108318 +599 1108264 +7055 1107776 +5025 1107245 +4151 1106800 +6699 1106003 +3660 1105998 +5007 1105989 +4753 1105892 +4122 1103755 +3951 1103085 +4819 1103013 +7541 1102865 +6308 1101995 +5081 1101551 +3827 1101551 +6348 1101534 +7025 1101049 +5520 1100898 +4713 1100782 +8406 1099966 +5257 1099287 +3592 1099115 +9689 1098471 +3497 1097861 +4430 1097362 +9912 1097102 +8153 1097019 +6108 1096435 +6154 1096150 +5502 1094500 +7425 1094293 +7127 1093101 +3283 1093034 +4624 1092815 +7000 1092519 +6241 1091967 +17560 1091497 +512 1090181 +5403 1090022 +6081 1088193 +3359 1088146 +3221 1087980 +7082 1087741 +3290 1087438 +1921 1086770 +10169 1085547 +3717 1084848 +4963 1084155 +3487 1083889 +4512 1083842 +781 1083714 +3402 1082905 +1717 1082724 +3297 1082684 +7910 1082114 +6934 1081913 +4914 1080997 +5504 1080481 +7415 1079818 +4441 1079514 +4928 1079417 +1323 1078936 +5764 1078723 +13598 1078487 +5205 1078470 +4747 1078344 +9068 1077920 +18015 1077721 +7008 1077624 +9502 1077051 +3919 1076785 +7939 1076621 +4461 1076561 +6355 1076297 +3026 1074645 +6046 1074406 +453 1073993 +2128 1073157 +3594 1073046 +2149 1072456 +13520 1072257 +5290 1071604 +5675 1070845 +4050 1070287 +5451 1069447 +6643 1069445 +4336 1069141 +431 1068235 +5389 1067211 +1514 1066254 +3424 1065502 +1906 1065274 +5581 1064940 +7504 1064485 +3578 1064180 +3666 1063907 +3744 1063473 +590 1063051 +5370 1062933 +7072 1062318 +9256 1062317 +833 1062017 +5236 1061829 +16462 1060612 +5291 1059433 +4201 1059425 +3777 1059131 +861 1058768 +2356 1058460 +562 1058081 +4831 1056793 +1362 1056283 +11435 1056101 +5637 1055964 +6079 1055924 +495 1055621 +3463 1055372 +1870 1055004 +3597 1054912 +6191 1054492 +4302 1054350 +672 1054156 +6332 1054152 +5538 1054038 +3774 1053925 +4678 1053559 +5852 1052549 +6698 1052045 +694 1052010 +4642 1051966 +2208 1051338 +5252 1050757 +1619 1050376 +4001 1050365 +3970 1050229 +5316 1049898 +6076 1049745 +5170 1049627 +4633 1049406 +6219 1048935 +3318 1048824 +5087 1048697 +5895 1048672 +4499 1048514 +3776 1048447 +5386 1048337 +5203 1047480 +5293 1047160 +4171 1047152 +3159 1047020 +6150 1046461 +8009 1045545 +1268 1044664 +4953 1044482 +5176 1044260 +6272 1043933 +5349 1043837 +7728 1043680 +3953 1043554 +3197 1042977 +6116 1041224 +8886 1040488 +6343 1040295 +5086 1039785 +958 1039594 +831 1039207 +1317 1038938 +893 1037936 +5597 1037832 +9621 1037628 +538 1037073 +501 1036988 +7908 1036807 +6110 1036682 +1899 1036322 +8511 1035818 +5954 1035279 +96 1035250 +6853 1034370 +3350 1033703 +4610 1033561 +2951 1033423 +7124 1033278 +5693 1032803 +7476 1032681 +6288 1031061 +4496 1030906 +4538 1030856 +4771 1029333 +2075 1028886 +2113 1028695 +10390 1028255 +7779 1027567 +507 1027336 +3406 1027246 +7517 1026986 +5395 1026926 +387 1025854 +3252 1025192 +4570 1024225 +4261 1024017 +8092 1023604 +4964 1023023 +2013 1022989 +3918 1022342 +4599 1021950 +5762 1021750 +6995 1021687 +4291 1020561 +5750 1020427 +3586 1020053 +6035 1019853 +4104 1019777 +7802 1018633 +5587 1018578 +1424 1018565 +2344 1017560 +4609 1017207 +4860 1016525 +4365 1016357 +3511 1016197 +4905 1015998 +4025 1015981 +3347 1015669 +7028 1015628 +2078 1015524 +1983 1015352 +3031 1015337 +979 1015312 +9072 1014743 +5122 1014687 +3891 1014111 +5474 1014058 +7880 1013953 +3210 1013783 +568 1013436 +5220 1013406 +4434 1012403 +4697 1011550 +5362 1011498 +3737 1011380 +756 1011159 +5136 1010585 +5242 1010457 +5110 1010415 +5523 1010390 +4908 1010100 +7534 1010030 +3387 1008087 +8063 1007595 +3910 1007470 +701 1007115 +600 1005923 +2425 1005712 +2713 1005668 +4792 1005214 +3806 1004967 +4190 1004836 +798 1004024 +3947 1003824 +1734 1003335 +774 1002962 +49430 1002589 +5859 1001740 +5704 1001278 +4238 1001170 +918 1001111 +4553 1000558 +5031 1000426 +8437 999699 +2246 999417 +4084 998042 +4197 998026 +8545 997343 +9611 996559 +4887 996285 +7865 995676 +4692 995619 +6961 995585 +1404 995563 +4423 995201 +8108 995057 +3450 994841 +4849 994834 +4220 994662 +889 994403 +6914 993625 +3621 993134 +1039 992744 +5938 992199 +715 992013 +1689 991685 +5281 991483 +1482 990745 +4372 990263 +7964 989765 +10123 989614 +3854 989508 +782 988371 +4800 988139 +4439 988074 +7395 987559 +4325 987452 +2448 986794 +1681 986285 +5882 986221 +5891 984558 +6630 984477 +3555 984317 +5733 983370 +6613 982970 +455 982830 +4725 982772 +7943 981571 +2885 981392 +5419 980765 +7374 979270 +4917 978913 +3561 978800 +1078 978520 +4162 978296 +9669 978242 +2815 978236 +1680 978235 +8693 978157 +498 977781 +4422 977129 +2167 976132 +2390 975986 +4931 975803 +4635 975620 +5922 973722 +5115 973595 +4384 973528 +7927 973338 +8031 973073 +424 972909 +896 972209 +1352 972098 +4974 971531 +3716 971410 +5545 971211 +4837 970688 +3105 970602 +3972 969545 +9388 969002 +6151 968990 +5380 968941 +5798 968828 +6848 968442 +5062 968256 +8136 968147 +7291 967583 +3087 967418 +1962 966850 +3894 966452 +10191 966175 +2051 965356 +913 964915 +8047 964868 +4458 964367 +5300 964311 +6994 964187 +3725 962770 +7529 962681 +8121 962545 +5526 961651 +6033 961565 +6466 961557 +4150 961454 +1650 961250 +486 961032 +491 960800 +5745 960709 +8878 959982 +5334 959692 +5045 959526 +7903 959023 +6057 958754 +5749 958063 +8087 958003 +6264 957693 +4004 957171 +9413 956628 +8611 956473 +627 955912 +1219 955395 +3499 954203 +10610 954183 +776 954130 +3404 953636 +7593 952789 +3713 951708 +4694 951700 +5486 951539 +9089 950646 +2389 950463 +10499 950298 +5788 950061 +6926 948694 +4518 948669 +7864 948572 +748 948224 +7606 947933 +1010 947597 +8059 945880 +4571 944428 +16267 944143 +6705 944128 +8674 943230 +4923 942533 +2421 942380 +5556 941699 +4735 941386 +5270 940960 +4202 940146 +5335 937988 +1890 937973 +6088 937822 +4816 937750 +4588 937427 +7452 937286 +1658 937147 +5533 936928 +1616 936623 +7712 935685 +3108 935633 +6205 935469 +7901 935057 +4950 934554 +3619 934220 +419 933976 +3478 933536 +5262 932866 +945 932541 +18840 931723 +5009 930935 +21138 930513 +1772 929479 +4755 929010 +5941 928630 +7133 927505 +6977 927246 +4833 926967 +7018 926786 +6403 926666 +6497 926492 +10247 926122 +6149 925968 +1446 925939 +25370 925871 +5688 925496 +10 925332 +8123 923747 +5989 923482 +1503 922356 +4637 922330 +4634 921904 +12385 921506 +4631 921152 +7628 921120 +413 920921 +7351 919431 +5975 919370 +5091 919110 +7297 918894 +5149 918737 +10330 918368 +12131 918249 +6621 918125 +7403 918086 +7052 917508 +369 917495 +3886 916915 +5125 916460 +2366 916128 +9116 915995 +7137 915749 +4998 915052 +5929 914971 +5585 914915 +3538 914319 +26442 913487 +4710 913429 +7253 913428 +3303 913013 +4457 912882 +40026 912761 +3914 910594 +6793 910482 +6596 910007 +4947 909361 +3122 908576 +7746 908460 +6510 908043 +2890 908025 +2420 907987 +5699 907839 +5402 907697 +8473 907013 +6128 906605 +4451 906397 +2934 906049 +12874 905857 +1677 905839 +5228 904072 +6103 904029 +3835 903727 +4855 903439 +4750 902769 +332 902136 +9475 902096 +9005 901510 +4916 900786 +6464 899911 +3853 899655 +3268 899638 +8602 899334 +2665 899101 +6402 898959 +2744 898743 +4601 898430 +7062 898339 +8785 897821 +567 897616 +10391 897375 +7725 896852 +15320 896741 +5423 896175 +4490 895533 +6416 895290 +5713 895254 +10501 895030 +1432 894640 +9077 894615 +6483 894614 +5586 894579 +4622 894347 +20877 893427 +2079 893154 +623 893020 +4991 892646 +5690 892425 +10618 890849 +12184 890553 +8244 890099 +1780 889484 +1313 889246 +106 888842 +795 888161 +3638 887959 +4353 887887 +10542 887847 +5409 887321 +7546 887173 +2396 886955 +4347 886599 +850 886537 +629 886491 +1128 886222 +6119 886169 +6727 886128 +4236 886028 +3281 885374 +7466 885097 +873 884923 +10830 884441 +4894 884043 +3878 884013 +7389 883818 +2347 883556 +7848 883275 +5223 883182 +3900 883053 +3783 883041 +5982 882775 +5198 882436 +1736 881949 +2393 881816 +2395 881208 +6358 881038 +7683 880486 +8342 880373 +8078 879936 +10371 879935 +8761 879473 +7356 879394 +9604 879106 +6960 878827 +2624 878148 +4684 877919 +3189 877629 +2547 877507 +4920 876005 +5860 875894 +6133 875719 +5230 875515 +3539 875493 +1959 875161 +8292 875055 +6509 874958 +1012 874561 +7611 874327 +6943 874065 +7557 872959 +3977 872862 +4960 872487 +11529 870460 +3632 869034 +2474 868380 +3278 868202 +7261 868159 +5182 867690 +6196 867596 +5322 867298 +5438 867126 +5214 866142 +5836 865387 +7595 864999 +489 864993 +2199 864291 +5017 864182 +3802 864008 +6849 863857 +4929 863148 +5898 862499 +6441 862493 +7118 861461 +12551 861355 +8488 861121 +9141 861100 +1381 860833 +7533 860823 +7458 860382 +10827 860061 +1451 859861 +4615 859080 +4764 859011 +8111 858777 +4324 858591 +7647 858536 +3223 858325 +5055 857740 +4296 857441 +5797 857174 +4351 856996 +5407 856847 +3544 856775 +1795 856495 +1665 856106 +5495 855994 +4257 855834 +8050 855330 +10109 854644 +7022 854557 +7920 854365 +7799 854340 +5608 854336 +5385 854118 +3046 854067 +7194 852548 +16964 852383 +7651 852355 +12379 852095 +4213 851873 +7456 851550 +11695 851095 +7401 851086 +7412 850942 +1586 850652 +5143 850210 +902 849624 +5952 849584 +6200 849045 +11643 848152 +5071 847489 +458 847417 +4547 847319 +6907 847057 +7705 846873 +1341 846615 +10021 846038 +1902 845644 +5479 845479 +14897 844971 +2188 844313 +5455 844072 +4459 843875 +4719 843858 +2959 843654 +595 843545 +3132 843221 +5924 843160 +1232 842710 +10306 842643 +574 842221 +5287 841971 +10728 841811 +441 841635 +7794 841423 +5609 841175 +5867 841140 +9366 840353 +8403 840105 +7585 840023 +6405 840017 +6875 839897 +2834 839591 +4179 839528 +12011 839205 +7433 839201 +4292 839187 +7492 839060 +3851 838918 +747 838741 +4785 838458 +1620 837460 +1710 837330 +4343 836110 +4216 836033 +8185 835422 +4577 835155 +5529 834254 +1475 834211 +8165 834201 +7627 834128 +2836 834105 +5708 834100 +11383 834017 +9138 833943 +9008 832798 +2124 832447 +4168 832285 +6802 831767 +4952 831698 +6716 831649 +7806 831440 +4333 830985 +5295 830546 +4036 830203 +5716 830154 +5615 830060 +3644 829493 +3608 829215 +2238 829204 +1345 828544 +3048 828260 +6142 827480 +10805 827295 +5202 826828 +4620 826416 +7459 825942 +4856 825116 +5811 824590 +4232 824275 +4362 824215 +2937 823809 +8502 823728 +5358 823668 +6155 823643 +8974 823355 +9266 821977 +6215 821613 +573 821459 +4053 820973 +7017 820798 +5911 819328 +3917 819177 +4643 819129 +7868 819089 +9003 818965 +4146 818654 +7463 818425 +6175 818176 +6669 818066 +8366 817761 +6594 817656 +2853 817307 +118 817287 +5264 817284 +7421 817216 +8211 817116 +9692 817105 +6305 816562 +7973 816504 +8810 815781 +7269 815760 +2682 815662 +12516 815187 +7524 814872 +2873 814870 +2154 814832 +2949 814350 +6318 813767 +7244 813673 +497 813572 +10413 813503 +698 813483 +7859 813085 +5043 812590 +4981 812572 +4922 812555 +5004 812273 +8127 812230 +8414 811932 +6240 811897 +12147 811787 +3616 811551 +5776 811486 +9870 811062 +7740 810523 +2091 810014 +2792 809891 +3205 809207 +6622 808813 +5849 808800 +8200 808021 +585 807549 +9470 807304 +9818 807238 +7083 806891 +4082 806605 +4975 806503 +2316 806460 +3551 805935 +6493 805854 +2816 805436 +5350 804945 +4003 804496 +1706 804386 +4695 804360 +8221 804223 +3211 803968 +8708 803949 +5155 803913 +4988 803565 +4874 803237 +10106 802720 +3232 802701 +7732 802397 +5326 802066 +6427 801361 +5631 801279 +6740 800879 +6100 800742 +401 800726 +4803 800624 +6363 800619 +5727 800412 +1127 800086 +9298 800083 +4996 800014 +4106 799519 +487 799506 +4131 799341 +5610 799162 +7121 798834 +6115 798667 +6481 798460 +4829 798303 +3881 797726 +962 797583 +6642 796913 +7139 796371 +4263 795916 +3872 794949 +11957 794566 +4071 794495 +8533 794334 +8919 793766 +8639 793455 +7516 792793 +5873 792586 +6276 792041 +2034 791816 +8796 791720 +5353 791490 +7103 791434 +2548 791227 +8329 791148 +9605 791129 +9953 790868 +8372 790696 +589 790467 +8144 789428 +5221 789411 +7012 789054 +6520 788825 +7831 788498 +844 788331 +1623 788130 +5696 787166 +4645 787122 +7986 787107 +5827 787081 +5445 787003 +5243 786478 +5046 786472 +6330 786159 +5014 786066 +6379 786013 +10636 785808 +917 785162 +3124 784947 +2640 784189 +9725 783553 +9253 783425 +8049 783310 +6538 782938 +5638 782700 +8632 782584 +6807 782551 +721 782040 +16059 781941 +6333 781716 +6074 781638 +5565 781181 +1765 780911 +5810 780630 +3714 780526 +954 780449 +5778 780376 +5789 779996 +1068 779555 +3363 779048 +5593 778906 +3640 778657 +10240 778494 +4289 778292 +5114 778205 +987 778126 +46640 777912 +7413 777318 +7317 776881 +9 776842 +11115 776693 +5021 776583 +5381 776513 +6563 776178 +3753 776103 +2264 776075 +6905 775817 +6936 775626 +6050 775178 +5752 775047 +7064 774442 +4471 774370 +2007 774332 +7099 774232 +10876 773951 +7078 773426 +3963 773066 +6619 772959 +6888 772667 +8536 772525 +6670 772421 +3829 772119 +6198 771881 +14015 770622 +7002 770508 +1018 770301 +4174 770174 +4701 770100 +8395 769730 +2599 769416 +7040 769323 +5612 769037 +7970 769031 +2438 768787 +10749 768289 +8982 768073 +5161 768005 +2453 767894 +7171 767851 +3815 766903 +3996 766842 +5850 766046 +6228 765987 +4899 765434 +7271 765147 +2718 765137 +6235 765125 +8407 765069 +862 765005 +5906 764849 +1187 764634 +8636 764295 +5814 764221 +4096 764076 +1142 764058 +8620 764028 +19809 763246 +6165 763231 +7840 762640 +6614 762319 +11079 761718 +13308 761408 +10799 761174 +1192 761045 +6774 760963 +3683 760869 +2394 760818 +6482 760799 +7823 760780 +1778 760726 +4462 760542 +6131 760457 +6253 760190 +6070 760142 +9436 760093 +5885 759875 +2449 759865 +1556 759355 +2598 759301 +9648 759298 +6692 759148 +8587 759113 +9180 758708 +4889 758705 +6296 758275 +10614 758002 +6623 757773 +1516 757483 +5983 757289 +5963 757079 +9371 756527 +9839 756521 +6190 756459 +457 756319 +7423 756132 +20635 756084 +2451 756028 +8936 755576 +5517 755366 +3001 754873 +2689 754598 +7797 754588 +7411 754253 +5433 753725 +6260 753428 +11618 752903 +7519 752784 +6189 752766 +1931 752571 +11514 752163 +5441 751927 +11356 751848 +6401 751735 +1324 751409 +2339 751243 +5417 750911 +735 750739 +5022 750547 +3908 750126 +3765 750060 +9461 749906 +8282 749744 +778 748051 +3873 747968 +7212 747911 +13423 747877 +6712 746778 +5940 746619 +593 745833 +12787 745585 +8024 744764 +7874 744587 +6204 744070 +5761 743492 +408 743484 +6490 743166 +7800 742528 +5879 742339 +6266 742335 +4320 741518 +2623 741505 +5363 740788 +1017 740736 +7288 740643 +9019 740546 +6870 740253 +6693 740189 +4329 739395 +7404 739189 +4093 738959 +9825 738518 +8857 737741 +10648 737228 +5318 737201 +16175 736827 +6823 736756 +10807 735530 +3535 735395 +2892 735274 +12180 735097 +3023 734685 +5188 734061 +10006 733791 +14549 733714 +33721 733497 +5875 733377 +9087 733374 +5338 732693 +5863 732401 +12785 732355 +4845 732267 +5857 732242 +18293 731947 +4595 731814 +4222 731703 +9500 731638 +9283 731189 +8877 731155 +8470 731144 +6872 730530 +4188 730422 +6225 730269 +5108 730199 +7148 729814 +1581 729749 +10500 729418 +6832 729243 +2920 729192 +6656 728880 +2670 728538 +7312 728480 +4295 728371 +5462 728288 +6339 728140 +3862 727744 +9975 727410 +7188 727311 +5861 727283 +5387 726788 +11761 726787 +634 726665 +5118 726571 +8976 726201 +8838 725568 +744 725461 +342 725294 +7163 725203 +6709 725026 +1146 724457 +9920 724076 +5896 723335 +4781 722493 +4537 722266 +11171 722136 +4433 722080 +8415 722004 +1095 721984 +8036 721541 +8685 720983 +4934 720956 +8879 720855 +8998 719936 +10654 719841 +4760 719610 +6134 719284 +9918 719163 +7897 718765 +11754 718515 +7432 718161 +9951 718152 +5254 717916 +4158 717914 +5186 717844 +20832 717534 +10039 717274 +6626 717236 +1069 716980 +6593 716881 +6085 716690 +9671 716454 +9070 716415 +8663 716353 +10804 715896 +5706 715743 +5359 715340 +27868 715294 +6792 715292 +6741 715229 +9153 715036 +13129 715014 +5044 714833 +6647 714813 +6511 714437 +8100 714414 +13648 714321 +9735 714169 +10964 714150 +22971 713833 +6829 713647 +4249 713298 +5365 712933 +5732 712932 +4143 712823 +5996 712740 +8618 712707 +6073 712606 +5986 712473 +1030 712471 +1461 712265 +5992 712229 +2066 711631 +18841 711614 +9925 711387 +8259 711334 +7205 710271 +11117 710047 +4793 709820 +6478 709698 +3858 709005 +2996 708450 +6430 708208 +4007 707891 +768 707799 +6067 707724 +9320 707652 +1894 707449 +7431 707015 +1509 706866 +14207 706655 +5511 706560 +6553 706382 +4385 706268 +4925 706167 +8096 706036 +6143 705550 +6188 705375 +9764 705084 +13126 704724 +6554 704581 +7394 704389 +5288 704007 +7912 703955 +4483 703878 +8929 703551 +14466 702956 +9071 702931 +8518 702631 +12517 702625 +32290 702438 +8581 702251 +7962 702211 +5719 702082 +5916 701971 +2577 701821 +7328 701292 +3248 700210 +5743 699668 +5457 699547 +6126 699148 +2931 698934 +6323 698857 +7639 698407 +1494 698189 +5846 697622 +2685 697571 +7791 697392 +294 697012 +13100 696907 +5726 696441 +3200 696066 +5946 696036 +7464 695756 +6304 695647 +10504 695601 +7547 695511 +9880 695470 +4572 695347 +14237 695318 +7373 695309 +9477 695162 +6980 695128 +5806 694770 +605 694642 +2998 694435 +7895 694205 +4528 693789 +2749 693766 +5483 693676 +9566 693519 +2580 693316 +10174 693096 +8704 693080 +9909 692938 +21393 692900 +3993 692679 +12262 692396 +10464 692058 +8155 692022 +6817 691879 +2943 691761 +5645 691684 +6576 691662 +8540 691355 +8742 691341 +4116 691092 +5303 690849 +9570 690723 +2380 690285 +11394 690269 +14017 689530 +6809 689244 +6129 688667 +5369 688418 +4345 688195 +5274 688173 +239 687921 +4090 687761 +8072 687681 +5876 687636 +7195 687577 +5667 687373 +1950 686507 +2484 686452 +8179 686263 +6608 686261 +8688 686110 +6279 685683 +10729 685582 +7173 685513 +6687 685094 +6409 684986 +4469 684945 +10604 684560 +6135 684453 +8832 684446 +14708 684416 +1990 684197 +1082 684138 +5566 683609 +5559 683395 +4074 683378 +2256 683376 +2024 683346 +8073 683283 +897 683245 +6173 683136 +7734 682901 +8882 682762 +1855 682313 +5034 682278 +9852 681791 +7624 681768 +6673 681693 +6411 681081 +7364 681064 +4913 681060 +6031 680976 +7344 680890 +6827 680734 +12168 680648 +9591 680619 +7246 680607 +5644 679833 +4069 679756 +7387 679646 +7318 679542 +5456 679522 +1662 679367 +7886 679365 +7977 679346 +14403 679217 +7311 679157 +11947 679007 +7924 678757 +9755 678632 +2436 678453 +947 678119 +7884 678117 +5884 678031 +6662 677968 +7323 677870 +5543 677819 +2919 677816 +10405 677754 +9101 677581 +6093 677515 +7051 677389 +8224 677269 +7337 677160 +2295 677074 +1158 676751 +6844 676610 +5412 676453 +4306 676409 +7224 676369 +7846 676354 +11289 676149 +3237 676053 +4876 675868 +7259 675866 +9754 675784 +7256 675714 +7771 675679 +11534 675527 +8972 675521 +9761 675505 +876 675302 +4159 675287 +22940 674880 +10131 674860 +690 674447 +1904 674392 +1083 673897 +6032 673886 +5101 673784 +3738 672730 +3180 672440 +4391 672381 +10673 672248 +2857 672245 +2001 672175 +9651 671458 +5413 671453 +11133 671373 +8556 671203 +7522 671159 +4270 671110 +3682 670879 +21648 670847 +5933 670801 +11409 670739 +8362 670409 +11287 670223 +7719 670222 +3698 670219 +7016 670090 +613 669934 +5766 669916 +10093 669908 +249 669379 +5157 669251 +4935 668785 +18912 668708 +1589 668686 +6218 668678 +4221 668611 +1533 668559 +6461 668440 +6666 668008 +9679 667710 +5301 667687 +5920 667645 +6041 667255 +10026 667109 +537 666906 +4455 666658 +5787 666453 +6157 666445 +5207 666201 +17846 665807 +8305 665802 +3992 665716 +8592 665686 +2953 665632 +7482 664859 +7485 664609 +3904 664497 +1325 664191 +3955 663800 +3582 663747 +6232 663677 +5858 663380 +6675 663306 +603 663284 +3510 662971 +3198 662873 +6868 661915 +3398 661810 +2662 661713 +5548 661134 +3254 660739 +3435 660236 +3208 660141 +7602 660107 +2944 660028 +6140 660004 +9084 659896 +22346 659867 +8213 659526 +8359 659308 +8987 659019 +4252 658938 +7283 658854 +3188 658518 +11807 658016 +9906 657789 +4088 657734 +7796 657606 +1092 657505 +1431 657491 +9119 657490 +4751 657349 +5539 656930 +8672 656751 +7486 656405 +746 656119 +2705 656036 +6056 655720 +1995 655638 +8589 655599 +10397 655550 +6725 655530 +10242 655020 +7276 654911 +9074 654748 +38913 654696 +7898 654624 +1740 654207 +4327 654145 +7652 653660 +381 653525 +4061 653521 +1671 653489 +1530 653176 +7189 653134 +13430 652994 +4339 652906 +6049 652838 +37707 652638 +825 652479 +7346 652418 +2780 652110 +3113 651930 +5901 651795 +14024 651686 +6555 651683 +9659 651412 +11154 651044 +4836 650886 +10016 650673 +5887 650661 +9118 650642 +7832 650470 +5564 650455 +7334 650225 +10737 650100 +5292 649726 +2984 649605 +5428 649179 +9280 648905 +6952 648709 +920 648620 +19360 648453 +31215 648415 +5894 648264 +5390 648185 +8607 648182 +7030 647962 +5443 647898 +13785 647830 +5603 647672 +5278 647399 +127 647382 +2538 647133 +5330 646394 +7204 645847 +5488 645597 +6082 645495 +9025 645143 +13119 645117 +12673 645054 +5870 644976 +9439 644729 +6531 644334 +5524 643753 +8308 643717 +2049 643595 +3169 643406 +1102 643192 +1416 643014 +4556 642555 +6801 642064 +1452 641982 +4225 641671 +4961 641591 +664 641444 +9899 641281 +7765 641155 +3668 640797 +577 640764 +8345 640747 +5795 640537 +8411 640494 +5621 640471 +6370 640152 +9486 639933 +13089 639843 +5187 639596 +6665 639411 +8335 639410 +8475 639341 +3705 639284 +5302 638828 +9716 638798 +4361 638678 +2047 638657 +10219 638643 +310 638361 +6891 637744 +7011 637600 +3559 637533 +6559 637504 +687 637481 +10071 637167 +5035 637079 +3811 637040 +6292 636820 +7325 636136 +5679 635912 +9992 635639 +5509 635306 +3152 635231 +12074 635104 +9957 634959 +6655 634887 +9016 634863 +7410 634829 +696 634729 +7960 634441 +3876 634388 +2596 633768 +10013 633760 +6163 633090 +7849 632916 +5506 632595 +4727 632496 +2667 632345 +6768 632173 +6616 632141 +11257 632026 +10068 631604 +7123 631579 +7542 631579 +7023 631466 +7720 631250 +6796 631244 +3070 630513 +3256 630499 +2767 630481 +9974 630365 +692 630358 +5549 630267 +6896 630202 +9002 630065 +6923 629920 +15811 629850 +9738 629838 +2891 629691 +4231 629583 +8606 629251 +11823 629198 +9853 628987 +7027 628837 +3991 628805 +2190 628682 +6862 628532 +4939 627964 +14914 627859 +2902 627520 +11615 627385 +4904 627359 +10650 627124 +5279 626755 +7777 626600 +799 626519 +6183 626487 +15113 626346 +1229 626246 +12224 625949 +36826 625284 +2501 625099 +6000 624750 +9326 624573 +8978 624449 +10101 624337 +2514 624302 +11660 624213 +8318 624036 +4562 623978 +1939 623886 +1067 623705 +7981 623554 +8590 623553 +7150 623194 +7216 623063 +6640 623002 +2127 622758 +8237 622309 +4951 622270 +9885 621960 +7830 621930 +6877 621661 +5485 621547 +854 621467 +3342 621167 +5503 621125 +3745 620985 +668 620572 +1850 620506 +7342 620486 +8082 620245 +565 620179 +429 620017 +9739 619828 +4449 619512 +8737 619500 +3182 619289 +10714 618874 +3332 618818 +3901 618659 +3154 618496 +7272 618484 +7953 618134 +5734 617900 +6065 617790 +9847 617491 +5075 617361 +2257 617024 +3865 616847 +4321 616792 +7029 616539 +11897 616236 +7835 616232 +10433 615972 +9617 615869 +5490 615816 +848 615766 +10357 615359 +7810 614764 +8666 614423 +2414 614200 +5501 614089 +6838 613977 +3791 613640 +1505 613292 +11821 612953 +9024 612648 +6298 612576 +8037 612282 +9392 612232 +7444 612155 +11162 612106 +8667 611893 +859 611851 +5514 611662 +7184 611457 +6633 611425 +7625 611246 +6029 611220 +7703 611085 +1167 610976 +1112 610958 +5405 610796 +6097 610555 +5655 610530 +3834 610027 +7077 610008 +8141 609961 +11070 609752 +126 609368 +7697 608871 +3661 608867 +6337 608742 +1653 608691 +8270 608543 +3563 608510 +1173 608225 +8277 608152 +1602 608008 +5527 607857 +8148 607798 +9307 607780 +6087 607464 +3506 607304 +19195 606852 +4962 606712 +5160 606653 +6885 606329 +1522 606222 +4144 606105 +9375 605871 +23997 605838 +7215 605645 +7136 605240 +7696 605204 +5439 604950 +7867 604769 +4417 604529 +6979 604164 +594 603974 +9389 603879 +9730 603658 +11826 603599 +15007 603523 +450 603404 +7784 603393 +12858 603312 +9763 603136 +2560 602954 +8780 602780 +1015 602661 +7219 602594 +11344 602587 +5729 602572 +6550 602536 +5475 602409 +2493 602094 +2305 602075 +10481 601222 +6467 601095 +15855 601040 +11226 600883 +9406 600845 +12104 600838 +3134 600815 +11223 600279 +592 600247 +10273 600115 +10955 599932 +11523 599822 +10329 599458 +6591 599380 +2631 599358 +6536 598751 +9498 597864 +2584 597750 +942 597599 +3312 597138 +6932 596982 +10663 596904 +11582 596881 +1056 596523 +1380 596400 +7406 596161 +10856 595892 +4178 595769 +6918 595606 +7637 595338 +3262 595268 +7309 595160 +8378 595058 +5235 594694 +6373 594645 +7539 594423 +10318 594361 +8720 594360 +10490 594265 +6782 594145 +4627 594120 +4844 594063 +456 594019 +2202 593801 +10512 593750 +1610 593244 +9232 592972 +13837 592565 +6590 592522 +4944 592362 +7174 592237 +3270 591789 +5465 591294 +6958 591198 +7520 590915 +5966 590914 +6004 590867 +6230 590717 +7380 590682 +9273 590639 +7179 590581 +10252 590079 +11232 590075 +10018 589454 +4397 589171 +10553 588862 +11654 588414 +5265 588100 +10307 588072 +7049 587817 +7971 587507 +7370 587415 +2269 587239 +4827 587061 +2178 587040 +28154 586814 +7681 586670 +2777 586647 +704 586590 +2032 586160 +29240 586009 +4303 585944 +6833 585900 +13304 585788 +658 585509 +4259 584968 +11783 584697 +10010 584641 +436 584638 +4044 584624 +5072 584498 +11149 584320 +2999 583791 +6991 583506 +9359 583285 +10319 583073 +8218 583031 +745 582797 +13210 582777 +6147 582612 +8420 582572 +6275 582333 +9784 582289 +2946 582278 +6928 582062 +5016 581857 +5519 581711 +7525 581516 +4638 581509 +28749 581004 +5917 580810 +2367 580376 +9416 580198 +3110 579900 +3837 579873 +10731 579847 +9443 579795 +9279 579534 +12090 579424 +9533 579369 +9046 579350 +3101 579267 +4079 578683 +11763 578236 +15434 577988 +6356 577910 +3944 577882 +10711 577459 +8514 577435 +7441 577388 +524 577286 +34754 577196 +8694 577150 +6518 577063 +8248 577045 +4839 576905 +11343 576880 +8107 576869 +5584 576838 +1211 576832 +5116 576522 +9593 576137 +9512 575883 +5357 575872 +5981 575845 +4091 575497 +14591 575465 +9955 575415 +8090 575406 +6822 575321 +2306 575187 +6755 575130 +7691 575061 +8055 574976 +8215 574691 +7341 574574 +8083 574574 +11687 574439 +6650 574049 +9358 573897 +6834 573740 +3461 573724 +12508 573413 +7192 573384 +15110 573316 +1924 573275 +6496 573211 +794 573111 +5481 573030 +17674 572959 +8547 572682 +6632 572577 +882 572533 +8990 572530 +7043 571991 +2791 571520 +7668 571498 +8732 571494 +4051 571420 +7674 571339 +936 570961 +3721 570900 +3060 570854 +6731 570838 +6317 570838 +11852 570745 +676 570627 +6414 570552 +8848 570361 +3823 570118 +9264 569921 +6840 569894 +4360 569706 +6268 569701 +6776 569555 +7383 569475 +6551 569359 +15319 569196 +8076 569167 +7142 568628 +11416 568580 +5374 568324 +8740 568107 +1455 568100 +6044 567784 +7437 567697 +1443 567507 +6066 567414 +5865 567337 +13104 567326 +4506 567190 +8819 566704 +8528 566670 +6164 566437 +12988 566156 +7093 565967 +3549 565654 +4877 565544 +4312 565286 +6413 565089 +1140 565068 +8774 564935 +4554 564912 +2213 564896 +10808 564748 +5799 564629 +10556 564192 +2430 564111 +253 563468 +8465 563415 +17768 563337 +8962 563249 +3887 563038 +9640 562781 +907 562010 +6886 561513 +7502 561321 +3326 561303 +10607 561199 +480 561037 +1571 560925 +14987 560911 +3755 560808 +522 560722 +8326 560191 +8261 560155 +10043 560098 +451 559924 +8361 559912 +8028 559794 +9594 559778 +545 559407 +31681 559385 +6946 559306 +7108 559275 +2645 559165 +5635 558799 +8302 558731 +6588 558119 +7263 558055 +14935 557913 +9272 557817 +9589 557707 +11745 557593 +3395 557506 +16004 557248 +6861 557017 +8062 557002 +9349 556662 +4031 556646 +8245 556554 +1460 556326 +4579 556319 +5000 556167 +10105 556070 +5215 556063 +10061 556034 +8744 555748 +1435 555400 +8272 555325 +5964 555146 +12767 555106 +9073 554642 +10530 554555 +11663 554385 +3553 554378 +8198 554300 +7034 554151 +2978 553814 +120 553526 +6584 553151 +3861 553102 +1776 553022 +7748 552759 +462 552707 +2123 552637 +1667 552623 +2661 552550 +16009 552434 +9984 552378 +652 552358 +7786 552200 +16407 551865 +11849 551642 +24385 551605 +1357 551592 +9051 551540 +8384 551212 +16974 551193 +12690 551066 +6372 551013 +14100 550991 +22767 550854 +9181 550845 +3980 550594 +3763 550539 +10383 550504 +1501 550432 +9663 550249 +3454 550052 +10764 549946 +11408 549714 +13417 549671 +5238 549218 +4309 549159 +5598 549109 +7175 548618 +3365 548511 +11264 548307 +8759 548096 +1036 547793 +5575 547523 +6099 547457 +1726 547406 +9095 547200 +11016 546805 +6600 546767 +9117 546746 +4724 546694 +4535 546644 +19859 546641 +10085 546192 +11990 545927 +5616 545745 +11867 545469 +558 545369 +12750 545195 +11453 545040 +6007 545014 +4805 544806 +11536 544790 +8381 544636 +12007 544575 +1678 544421 +9354 544408 +5969 544383 +10120 544328 +2004 544322 +9336 544293 +11047 544173 +10656 544121 +9601 543949 +1133 543728 +2771 543554 +948 543517 +7264 543327 +5436 543269 +5253 543261 +2304 543194 +5464 543083 +9935 542754 +13289 542612 +3830 542534 +8826 542457 +12164 542022 +10432 541880 +9691 541784 +5472 541671 +5715 541589 +1799 541522 +8295 541394 +8390 541248 +7384 541184 +4450 541159 +4739 541155 +2945 541039 +9292 541013 +8985 540742 +14039 540629 +4224 540466 +10207 540401 +9687 540209 +14051 540174 +8015 540103 +6393 539847 +6208 539816 +8665 539668 +9176 539559 +2417 539392 +6529 539382 +6678 539231 +1806 538825 +10158 538815 +9308 538759 +9769 538636 +11798 538499 +2686 538256 +30756 538096 +12774 537997 +10202 537811 +20948 537713 +5984 537666 +6034 537532 +5269 537514 +12024 537300 +6016 537023 +3068 536803 +10653 536746 +4544 536720 +47383 536644 +6303 536574 +1670 536539 +9368 536507 +4711 536324 +6688 536289 +11374 536287 +5332 536164 +5415 535772 +686 535624 +12302 535597 +8034 535259 +7298 535243 +5027 534858 +4404 534801 +4993 534397 +4310 534235 +11474 534227 +9179 534109 +3013 533926 +4267 533674 +5947 533656 +6242 533613 +9657 533598 +6971 533518 +8683 533415 +891 533390 +9894 533388 +10278 533319 +1033 533218 +9432 533117 +10935 533048 +955 533045 +6930 532804 +6609 532747 +6216 532571 +863 532480 +10200 532443 +4973 532320 +3388 532276 +8512 532164 +8747 532078 +6754 531928 +1150 531903 +10629 531822 +9634 531776 +6851 531720 +4503 531705 +11520 531460 +2671 530938 +14536 530797 +2040 530459 +4522 530342 +3376 530119 +11366 530088 +8560 529987 +10183 529970 +3913 529729 +27572 529657 +9231 529501 +6006 529484 +12353 529322 +5582 529169 +10936 529025 +11596 528842 +13270 528815 +4778 528726 +1958 528689 +9965 528383 +7996 528093 +5256 527765 +7987 527727 +8724 527617 +12761 527594 +14205 527540 +30101 527445 +2376 527425 +11113 527174 +12108 526987 +7523 526948 +14861 526831 +6769 526760 +10251 526687 +5141 526686 +12056 526659 +11911 526627 +808 526618 +8829 526430 +15142 526406 +7932 526264 +21842 525942 +4269 525894 +11288 525883 +3703 525813 +11661 525338 +1423 525254 +5107 525204 +9038 525200 +7760 525179 +2675 525099 +7214 525047 +7997 524960 +3202 524802 +6611 524696 +1886 524345 +2200 524290 +11565 524211 +9505 524148 +10177 524099 +8391 523969 +4145 523792 +9309 523601 +1098 523601 +28589 523309 +16404 523284 +13663 523201 +11465 523027 +14943 522826 +8227 522727 +728 522579 +10315 522535 +11182 522489 +5711 522363 +9592 522137 +12901 522119 +6842 521979 +7584 521897 +3605 521789 +3459 521714 +7844 521599 +32790 521575 +9489 521286 +6270 521053 +2142 521051 +8203 521013 +5453 521007 +9836 520903 +38387 520845 +5800 520632 +2442 520625 +10763 520466 +5498 520396 +7588 520298 +8584 520208 +1834 520004 +8830 519633 +6440 519510 +3104 519421 +8519 519326 +12980 518862 +10266 518833 +8705 518717 +7507 518708 +3175 518680 +3489 518674 +10290 518362 +5268 518131 +4498 518124 +2845 518039 +7604 518016 +4871 517787 +5640 517709 +2018 517645 +9552 517437 +2025 517381 +7690 517300 +10674 517255 +1084 517164 +1774 516848 +9481 516274 +9558 516120 +11044 516115 +7543 516105 +8054 516100 +14637 516058 +5049 515800 +12989 515724 +6866 515494 +10665 515424 +5340 515194 +5698 515174 +7907 515166 +17329 515143 +4693 515073 +4349 514677 +610 514618 +4958 514534 +9658 514511 +7058 514162 +11279 514146 +5909 513815 +2403 513591 +4273 513532 +5948 513355 +3324 513090 +7622 512788 +4543 512710 +47013 512667 +15994 512605 +16020 512591 +6227 512519 +14167 512493 +122 512466 +6439 512393 +1787 512303 +4810 512061 +6970 511802 +7348 511766 +13419 511721 +3183 511719 +1697 511536 +1346 511242 +6639 511134 +5642 511062 +11372 511039 +7827 511016 +1360 510851 +7841 510829 +1694 510698 +9638 510663 +5967 510625 +6903 510571 +5425 510480 +5931 510406 +856 510365 +16213 510309 +9840 509866 +9114 509800 +6153 509717 +3503 509652 +7663 509651 +2915 509566 +6492 509531 +4859 509528 +3797 508985 +9869 508778 +7498 508579 +7090 508348 +2935 508256 +8061 508150 +5129 508134 +2850 507942 +10346 507712 +8171 507083 +11126 506565 +12313 506541 +8114 506538 +1925 506318 +7265 506203 +10308 506177 +2788 506150 +18355 506070 +6789 505868 +7428 505805 +7739 505672 +335 505540 +4437 505375 +1046 505338 +7505 505231 +1820 505189 +6646 505079 +4847 505035 +7723 504840 +1874 504583 +10769 504401 +6062 504379 +9393 504300 +3609 504287 +8022 504015 +8163 503963 +5662 503954 +4578 503944 +28808 503835 +9733 503821 +4419 503723 +12152 503716 +9584 503534 +7724 503491 +12139 503393 +1225 503219 +12451 503123 +2404 503095 +7161 503020 +38325 502957 +13064 502866 +13959 502619 +16685 502602 +8489 502599 +2801 502372 +8948 502353 +7744 502332 +11301 502329 +18317 502311 +6380 502208 +9554 502166 +10638 502156 +6350 502075 +14395 502028 +733 501896 +11255 501750 +8793 501681 +2704 501584 +7941 501562 +9142 501326 +8766 501173 +7042 501158 +9161 501143 +14316 501138 +1058 501137 +7567 500930 +17215 500926 +7722 500875 +7686 500769 +10009 500732 +8834 500182 +1267 500098 +5194 500074 +12433 499839 +10094 499728 +1153 499598 +10826 499365 +1951 499236 +6378 499083 +10309 498859 +14771 498759 +11563 498613 +7327 498161 +7069 498100 +12274 498050 +9188 497940 +3514 497889 +9859 497683 +4294 497668 +6634 497660 +8616 497651 +14456 497583 +12420 497494 +9262 497418 +1050 497098 +8849 497057 +9337 496804 +8698 496736 +5680 496519 +8714 496340 +6599 496156 +13922 496122 +12783 496080 +13956 496070 +9904 496047 +9469 495805 +9204 495517 +6856 495493 +4282 495458 +8331 495218 +7741 495080 +7876 495033 +13101 494968 +8868 494871 +10102 494813 +6537 494662 +6174 494574 +1006 494209 +11292 493769 +8138 493741 +7417 493306 +7819 493293 +4021 493280 +5069 493031 +1676 492861 +7618 492809 +619 492743 +7111 492488 +13098 492423 +5219 492395 +9549 492153 +1674 492046 +12644 491967 +5657 491932 +9238 491889 +1186 491713 +9247 491667 +6419 491609 +14080 491582 +12506 491546 +7850 491532 +8276 491506 +8319 491476 +7745 491425 +14389 491164 +11725 491143 +6948 491116 +8852 491069 +12557 491017 +2860 490863 +12438 490713 +11938 490511 +17154 490479 +12352 490449 +9209 490055 +6465 489970 +7362 489941 +9456 489939 +3808 489894 +10691 489840 +6118 489617 +3526 489349 +11717 488745 +13553 488681 +570 488430 +8846 488377 +10579 488145 +7667 488138 +9398 488058 +6252 488041 +10816 487804 +9902 487778 +6635 487683 +10986 487574 +12255 487515 +6772 487362 +10409 487277 +4035 487133 +6508 487039 +7578 486957 +38898 486923 +8209 486827 +2301 486497 +8350 486454 +8649 486382 +10226 486188 +7477 486080 +46195 486017 +4761 485991 +6572 485950 +104 485842 +10589 485787 +5368 485706 +11057 485511 +14379 485358 +22306 485315 +10064 485172 +9040 485169 +2439 484771 +1305 484754 +9301 484652 +6295 484648 +2093 484477 +7095 484430 +9465 484269 +8867 484258 +7164 484152 +30772 483866 +8104 483650 +2516 483348 +9559 483026 +4526 482958 +11345 482881 +7632 482880 +6086 482805 +13479 482529 +8018 482355 +4217 482302 +7147 482262 +10277 482204 +12534 482102 +9613 481874 +8922 481792 +7766 481713 +5908 481572 +3260 481497 +2806 481392 +3242 481272 +9103 481197 +6412 481109 +6374 480706 +5816 480672 +2649 480629 +18325 480570 +28244 480558 +4448 480540 +3257 480335 +6947 480290 +12217 480181 +7699 480013 +3433 479977 +9694 479829 +9014 479704 +14200 479703 +6282 479693 +6541 479643 +32009 479575 +8383 479249 +4598 479147 +10209 479146 +9403 479045 +6448 478902 +17692 478863 +7457 478667 +2433 478511 +724 478152 +13045 478149 +7634 478062 +12031 477941 +8565 477889 +6365 477835 +6122 477730 +1569 477700 +8239 477668 +581 477633 +864 477246 +7007 477223 +8971 477114 +10937 477035 +7487 476747 +10596 476592 +11840 476530 +8916 476449 +9276 476256 +8676 476237 +2163 476190 +875 476049 +12996 475823 +4531 475813 +11467 475808 +3771 475740 +5625 475421 +16987 475263 +10704 475232 +36430 474914 +4377 474829 +10503 474740 +10810 474711 +9759 474548 +13189 474527 +17944 474409 +4130 474295 +25650 474213 +2606 474053 +11448 473925 +6159 473703 +7552 473678 +956 473675 +2019 473515 +4304 473492 +13347 473405 +1299 473258 +8507 473017 +10930 472996 +6209 472990 +1590 472948 +6507 472885 +12155 472759 +10401 472745 +12087 472716 +6026 472567 +8847 472500 +5184 472211 +8286 472197 +9544 472194 +6105 472192 +11231 471976 +12696 471900 +9317 471819 +4625 471755 +9327 471505 +7473 470986 +2840 470738 +8197 470623 +14780 470281 +12946 470152 +3813 469939 +12382 469887 +5308 469869 +10382 469852 +17662 469530 +13509 469357 +5650 469084 +7274 468913 +3537 468876 +6512 468810 +8466 468776 +16693 468472 +4767 468256 +8461 468202 +13285 468151 +729 468066 +2737 467943 +6654 467872 +2543 467645 +539 467455 +7169 467379 +21003 467300 +7675 467213 +12815 467106 +14138 466738 +3166 466697 +5739 466627 +6177 466578 +1898 466078 +30251 465915 +7881 465902 +9772 465804 +13312 465503 +19646 465313 +5897 465263 +7560 465238 +8119 465232 +7151 465210 +15752 464964 +4476 464929 +3077 464826 +4941 464756 +10152 464493 +3255 464477 +7930 464418 +8561 464331 +8057 464290 +5073 464075 +13191 463887 +6498 463863 +7357 463530 +9815 463411 +13489 463148 +12369 463029 +6762 462915 +4482 462736 +2350 462731 +6506 462634 +2152 462592 +11622 462312 +8668 462185 +5878 462019 +2852 462006 +16145 461943 +6064 461812 +11772 461652 +3529 461506 +11617 461437 +13662 461340 +2922 461227 +21195 461196 +8733 461097 +8546 461014 +19299 460751 +9131 460711 +7863 460685 +4240 460580 +8889 460543 +25151 460407 +41615 460212 +6322 460155 +7914 460099 +15572 459295 +4580 459246 +12682 459131 +6899 458997 +9722 458947 +5951 458826 +2701 458808 +1512 458735 +10655 458723 +9674 458635 +7285 458614 +13864 458453 +7706 458443 +14463 458408 +11969 458352 +12022 458342 +8030 458206 +5718 458203 +9499 458191 +18091 457686 +10820 457682 +11153 457653 +17189 457496 +7397 457166 +15435 457165 +10404 457133 +1079 457130 +7186 456764 +11831 456653 +10633 456482 +870 456379 +10719 456339 +4089 456215 +10470 456203 +11729 456195 +13178 456192 +3768 456183 +14691 455976 +28 455976 +10976 455911 +4663 455897 +5249 455773 +8442 455449 +5282 455325 +3720 455145 +13356 455082 +11581 454949 +2381 454810 +601 454802 +903 454655 +10099 454543 +16483 454518 +10399 454507 +1252 454428 +11305 454423 +3246 454342 +12605 454177 +15247 453973 +3866 453730 +46481 453702 +4548 453681 +10325 453562 +11566 453362 +6287 453357 +9240 453334 +6244 453271 +4474 453245 +4524 453216 +16519 453141 +1897 453137 +3732 453109 +3885 453072 +15244 452997 +11434 452891 +11100 452784 +949 452773 +10747 452756 +8223 452719 +6125 452444 +8207 452351 +12053 452321 +2870 452256 +10303 452236 +8112 452129 +2044 452086 +11749 451878 +7009 451838 +8098 451813 +11233 451812 +8853 451678 +15302 451647 +7257 451557 +4851 451508 +24247 451387 +14651 451369 +5671 451350 +29690 451218 +5673 450985 +7429 450645 +14526 450579 +1489 450407 +18381 450182 +895 450118 +18279 450047 +7511 449889 +7564 449873 +2539 449697 +10423 449668 +10222 449447 +12427 449438 +8233 449184 +15446 449140 +13659 449125 +7852 449025 +4487 449014 +14486 449007 +9750 448958 +6889 448930 +11189 448906 +11769 448691 +9528 448621 +8339 448582 +7460 448524 +7480 448475 +6645 448428 +16167 448391 +5961 448383 +11468 448323 +10947 448316 +7448 448262 +10831 448231 +7743 448200 +11790 448123 +3864 448020 +9699 447968 +17741 447947 +8551 447942 +10997 447848 +631 447787 +8956 447706 +5831 447623 +741 447620 +10428 447512 +13310 447352 +1273 447226 +8084 447095 +5508 446729 +6951 446617 +6810 446454 +9252 446194 +1304 446147 +11628 446139 +12408 446021 +5600 445908 +3620 445780 +11102 445687 +1817 445635 +15260 445631 +4521 445604 +7891 445507 +9644 445478 +10540 445477 +7623 445150 +4869 445065 +15846 445057 +13965 444997 +6547 444996 +10497 444967 +7698 444957 +7262 444924 +21996 444912 +3565 444847 +7361 444827 +8601 444817 +1090 444803 +9519 444736 +4790 444625 +6867 444594 +17420 444560 +7491 444513 +8190 444458 +1513 444222 +10522 444039 +6674 444008 +7183 443956 +2088 443644 +9207 443623 +11320 443607 +10088 443581 +11136 443530 +4460 443497 +6042 443486 +7716 443421 +13854 443389 +5674 443287 +16581 443281 +9167 443098 +7165 442919 +4687 442895 +4866 442855 +10784 442654 +4732 442633 +865 442626 +5237 442517 +12652 442432 +10075 442326 +3525 442279 +3483 442207 +853 442188 +10834 442053 +10092 442033 +6544 442002 +9212 441858 +8776 441583 +12857 441523 +9588 441423 +13445 441268 +7301 441201 +9670 441165 +9208 441056 +9397 440989 +7393 440970 +30830 440747 +14959 440610 +6286 440540 +9157 440442 +10033 440162 +8313 440089 +7407 440088 +5078 439983 +10297 439973 +3585 439845 +11632 439807 +11088 439726 +8873 439603 +11905 439537 +7050 439349 +12771 439335 +10578 439236 +14006 439225 +9189 439199 +13943 439161 +8242 438925 +27737 438892 +14685 438862 +2841 438853 +8188 438775 +6060 438757 +10657 438692 +8468 438514 +6939 438377 +5224 438336 +3602 438299 +10029 437970 +7475 437917 +21675 437790 +11328 437753 +5258 437656 +793 437502 +2105 437344 +11461 437203 +18335 437071 +6366 437018 +7995 437015 +7506 436901 +10062 436677 +6846 436653 +10159 436578 +10848 436530 +10660 436510 +7369 436436 +7032 436299 +2446 436290 +5066 436196 +7974 435903 +8568 435896 +18936 435627 +10991 435619 +8386 435599 +5628 435553 +3975 435372 +796 435152 +7445 435125 +11307 434750 +4364 434663 +9667 434529 +9629 434332 +6706 434291 +8486 434203 +3981 434045 +8137 433968 +8526 433856 +11780 433721 +12803 433690 +9817 433586 +3020 433537 +10115 433505 +4903 433496 +16077 433356 +10275 433334 +11243 433238 +12088 433209 +9144 433171 +3695 433088 +7472 433029 +9319 432997 +8026 432869 +6697 432716 +9036 432689 +6505 432658 +4911 432291 +28227 432239 +4758 432224 +1845 432086 +10836 431880 +7021 431848 +8296 431812 +12328 431752 +1005 431721 +5923 431708 +13162 431573 +13097 431521 +32825 431122 +10913 431081 +9623 430972 +13682 430905 +25440 430798 +15016 430612 +2696 430394 +1587 430367 +10463 430242 +12612 430177 +10448 430165 +9196 430158 +20241 429817 +3263 429681 +12533 429646 +12577 429472 +3907 429457 +14717 429425 +9041 429321 +10140 429303 +5440 429164 +8225 429150 +7979 429111 +5162 429109 +34979 428999 +8263 428967 +9643 428904 +548 428758 +1698 428731 +2441 428648 +11058 428630 +6575 428307 +8686 428274 +14719 428107 +9980 428077 +9299 428002 +3966 427967 +11431 427858 +626 427844 +7170 427618 +8945 427570 +21703 427336 +12199 427179 +15818 427119 +9524 427061 +94 427057 +10084 427023 +8275 426906 +9803 426806 +4848 426803 +12507 426752 +8093 426716 +5400 426665 +10822 426582 +10544 426474 +8684 426470 +3474 426393 +2824 426292 +1124 426099 +5767 425935 +7763 425894 +2052 425887 +5888 425825 +17894 425817 +6910 425796 +25792 425781 +723 425679 +11668 425517 +8860 425445 +1027 425439 +6454 425414 +6787 425354 +15958 425316 +4712 425219 +23731 425071 +39990 425025 +7822 424954 +7237 424883 +7750 424792 +5469 424745 +13030 424621 +438 424384 +2569 424271 +6795 424188 +10012 424179 +16348 424166 +14359 423890 +3994 423764 +11330 423708 +9889 423613 +7767 423523 +6761 423314 +11237 423255 +9194 423211 +8025 423111 +2787 423038 +10053 423032 +8257 422993 +3838 422979 +8999 422972 +8013 422938 +14886 422812 +3974 422673 +11803 422479 +6277 422372 +10054 422115 +8006 422012 +8482 421985 +5333 421820 +4846 421787 +11123 421771 +13795 421760 +14597 421541 +12268 421468 +9539 421435 +11646 421416 +12848 421415 +10828 421149 +4465 421005 +8312 420878 +1940 420842 +9384 420774 +10343 420682 +38563 420682 +2171 420647 +10907 420307 +14847 420294 +8888 420198 +5496 420132 +4898 419944 +10821 419846 +8754 419807 +18836 419789 +7306 419436 +4480 419357 +13107 419242 +4662 419029 +13301 418988 +7620 418900 +6387 418663 +398 418591 +13316 418591 +5404 418532 +3212 418520 +14683 418447 +14372 418412 +13683 418208 +9753 418195 +4164 418183 +11476 418137 +6869 418119 +8463 418118 +10759 417587 +12478 417567 +14813 417528 +11753 417140 +7772 417075 +15152 417032 +6973 416961 +26840 416803 +2236 416706 +8454 416300 +11228 416181 +18802 416171 +12112 416118 +7600 416058 +14345 416018 +8883 416008 +13620 415903 +3550 415841 +10282 415776 +16127 415765 +9599 415745 +11376 415739 +10074 415692 +731 415656 +7243 415472 +14057 415445 +8157 415366 +5774 415223 +6117 415193 +1117 415145 +13077 415133 +17448 415079 +14424 415006 +5868 414948 +7228 414712 +8564 414687 +1496 414653 +11939 414600 +9137 414451 +18042 414426 +8871 414367 +7527 414259 +9210 414225 +6942 414161 +9284 414093 +11864 413979 +4729 413800 +15336 413797 +29749 413689 +9808 413450 +6986 413448 +6397 413294 +15598 413245 +14819 413174 +13547 413105 +6777 413085 +13804 413005 +13257 412980 +7446 412953 +6273 412946 +12620 412923 +7178 412896 +8953 412856 +10893 412416 +9031 412276 +702 412269 +15338 412154 +7747 412118 +7872 412058 +12769 412014 +11722 411923 +9983 411914 +16256 411890 +10732 411859 +19277 411821 +5883 411797 +11942 411756 +7255 411620 +10568 411592 +22917 411489 +3425 411402 +3494 411372 +18707 411253 +3185 411125 +15733 411004 +10460 410993 +4453 410817 +4401 410718 +7363 410707 +20954 410678 +10270 410579 +8404 410530 +16570 410515 +16863 410211 +11702 410131 +4235 409832 +9001 409794 +12806 409633 +12223 409518 +9396 409503 +13797 409366 +12402 409239 +7526 409230 +12600 409139 +15100 409091 +10681 409050 +10246 408972 +11872 408925 +7579 408648 +9172 408405 +17796 408380 +9115 408352 +21793 408261 +12691 408216 +6364 408060 +6859 408027 +13038 407988 +1632 407958 +12477 407924 +15378 407851 +9352 407688 +3633 407655 +5803 407653 +8646 407621 +7587 407575 +15331 407451 +1758 407253 +877 407147 +11998 407145 +11679 407103 +3601 407096 +4669 407086 +227 407081 +11334 407006 +8131 406946 +1759 406891 +8452 406865 +8997 406822 +9574 406812 +6797 406810 +11336 406762 +8501 406738 +11114 406687 +10311 406649 +10150 406623 +17689 406578 +11300 406421 +9814 406384 +8389 406349 +8271 406333 +7954 406313 +10576 406312 +6565 406306 +7545 406240 +9102 406164 +14328 406013 +2144 405990 +13673 405932 +5337 405901 +8865 405864 +1155 405852 +3719 405528 +9258 405495 +13093 405422 +9011 405319 +6257 405240 +1835 405060 +12871 405052 +5588 404835 +16629 404629 +9799 404597 +15295 404559 +11066 404496 +8716 404329 +12928 404150 +12206 404017 +9635 404014 +7635 403984 +7561 403954 +12388 403633 +13630 403434 +13592 403053 +15896 402919 +20197 402829 +9645 402757 +13393 402755 +12165 402698 +14074 402666 +9332 402472 +12406 402273 +6938 402132 +8471 402079 +15615 401716 +14410 401641 +8020 401607 +9573 401541 +813 401316 +13709 401296 +7267 401278 +4049 401218 +7817 401160 +1125 400938 +6695 400924 +753 400632 +13117 400629 +7467 400617 +7405 400469 +8741 400432 +1347 400368 +10095 400320 +10692 400261 +7558 400226 +9133 400206 +4467 400136 +8814 399952 +1648 399945 +5172 399767 +1723 399704 +5384 399677 +32425 399666 +8130 399658 +26094 399611 +6602 399400 +7501 399051 +21356 399047 +13960 399015 +26119 399002 +1177 399001 +15076 398977 +7965 398937 +14072 398855 +2545 398812 +15033 398769 +10156 398733 +9015 398630 +5550 398523 +3735 398483 +15694 398450 +12318 398428 +9709 398412 +4156 398372 +6937 398056 +13447 397970 +11083 397902 +7612 397887 +677 397838 +4709 397653 +8539 397640 +3405 397576 +15184 397562 +10880 397473 +7718 397294 +1846 397197 +13919 397195 +17199 397161 +12963 397157 +2357 397072 +8168 396992 +13067 396930 +7360 396710 +6625 396663 +12670 396660 +11363 396611 +8644 396544 +1071 396138 +9569 396044 +11103 396035 +16278 396022 +12356 395953 +20790 395715 +12160 395513 +1278 395447 +9985 395426 +6589 395388 +1322 395383 +14327 395356 +10727 395331 +6824 395153 +15301 394895 +9233 394857 +8970 394826 +8147 394812 +11406 394778 +4129 394771 +29657 394731 +13339 394630 +3141 394547 +26220 394385 +9029 394178 +14133 393701 +4313 393607 +10741 393597 +12000 393560 +11038 393455 +7708 393396 +6981 393382 +48539 393366 +12965 393321 +3051 393202 +9697 392989 +2629 392849 +556 392807 +11481 392800 +6595 392750 +1371 392745 +19099 392599 +5602 392531 +7039 392512 +14905 392497 +11118 392472 +18390 392419 +2848 392286 +7982 392117 +1034 392071 +7825 392001 +12930 391952 +8262 391872 +926 391870 +13904 391618 +3528 391502 +6259 391442 +432 391392 +4897 391238 +2797 391028 +21852 390801 +3844 390776 +7075 390638 +11972 390586 +18286 390550 +10914 390522 +12181 390457 +10274 390401 +2411 390339 +11704 390287 +9856 390187 +9268 390171 +12150 390103 +9879 389997 +7471 389987 +9185 389986 +12879 389967 +7455 389884 +1747 389879 +13340 389812 +10127 389663 +10402 389568 +19613 389434 +4285 389340 +2038 389257 +5847 389229 +8097 389217 +10014 389052 +7695 389038 +6964 388887 +11915 388736 +17305 388721 +3069 388564 +6779 388434 +7866 388384 +1789 388251 +12684 388146 +5521 388118 +10595 388088 +6873 387763 +1818 387683 +7054 387518 +10754 387382 +6451 387287 +9655 387128 +8355 387116 +8129 387080 +16393 387050 +13999 386969 +7787 386929 +11276 386759 +5562 386693 +13741 386688 +12285 386638 +6641 386580 +6737 386535 +7187 386445 +2536 386412 +14122 386389 +15682 386380 +6234 386377 +10302 386248 +14115 386219 +16666 386042 +10332 385907 +3964 385839 +6620 385645 +10361 385548 +8902 385496 +10966 385484 +14187 385467 +13633 385375 +17063 385313 +18621 385065 +10617 385042 +12539 384984 +14997 384883 +11983 384868 +5468 384814 +7238 384796 +10721 384572 +15555 384479 +1275 384392 +11432 384301 +2977 384284 +10569 384253 +13359 384110 +11560 384076 +10342 384009 +6569 383954 +19122 383938 +22611 383811 +6036 383687 +8474 383644 +5740 383429 +7692 383359 +586 383295 +1419 383273 +4066 383194 +1538 383149 +3191 382945 +12284 382931 +14620 382918 +9110 382879 +4825 382730 +21199 382531 +8126 382516 +11652 382509 +3889 382433 +4352 382237 +8478 382053 +5422 381988 +6597 381839 +8246 381825 +16052 381819 +2748 381810 +9530 381684 +16119 381666 +6445 381644 +18292 381607 +23370 381562 +10089 381523 +11077 381365 +7226 381330 +13329 381179 +14021 381124 +9758 381123 +11809 381086 +13772 381080 +3036 381064 +15223 381038 +6486 381025 +6814 380938 +13241 380861 +8027 380855 +8884 380820 +2054 380811 +19709 380793 +6504 380605 +9626 380566 +11515 380412 +8628 380336 +13367 380121 +13861 380071 +31362 380053 +1647 380010 +17820 379781 +9166 379560 +6906 379374 +14469 379282 +14227 379210 +3754 379002 +11001 378926 +11526 378867 +8347 378753 +7862 378733 +9281 378662 +17718 378652 +4293 378636 +13302 378542 +6982 378491 +14217 378485 +8494 378459 +4932 378435 +10408 378283 +15796 378222 +10588 378132 +23357 377965 +17462 377860 +8904 377819 +11000 377789 +16738 377598 +17499 377581 +11677 377538 +4936 377530 +1130 377448 +9007 377376 +9900 377348 +7382 377267 +5760 377258 +937 376979 +12252 376780 +20311 376769 +849 376755 +1389 376721 +17715 376614 +10135 376600 +14066 376597 +7026 376387 +132 376386 +6957 376353 +11734 376300 +3258 376225 +6452 376005 +909 375926 +11332 375842 +10386 375819 +10710 375699 +14825 375691 +8088 375684 +12148 375681 +9936 375544 +7270 375490 +11781 375370 +6226 375337 +2437 375301 +9846 375164 +9296 375038 +17027 374977 +2337 374938 +3912 374885 +15162 374882 +8252 374852 +10484 374831 +20251 374829 +2165 374799 +6901 374710 +10243 374664 +13671 374469 +7358 374221 +16120 374131 +3149 374119 +9021 374053 +7110 374044 +11078 373834 +1484 373662 +8069 373610 +10868 373588 +3184 373585 +2754 373555 +1292 373520 +10146 373513 +6912 373432 +7754 373402 +4013 373341 +3792 373296 +9149 373012 +11259 372932 +12159 372917 +4700 372912 +1007 372888 +21538 372840 +4959 372816 +6479 372812 +13226 372725 +7586 372712 +14105 372647 +16720 372515 +5653 372464 +13336 372347 +13612 372275 +4589 372141 +13914 372001 +14433 371923 +1631 371838 +7166 371618 +12906 371602 +8234 371586 +5705 371556 +7875 371528 +3929 371451 +13354 371418 +9677 371286 +9619 371233 +7247 371188 +9511 370981 +8794 370937 +21506 370897 +10058 370891 +10305 370833 +10192 370769 +12665 370754 +16752 370681 +7375 370584 +100 370551 +10145 370550 +12396 370469 +2930 370325 +16440 370303 +6579 370297 +8523 370210 +6450 369913 +2471 369759 +3039 369622 +823 369518 +8673 369498 +28689 369444 +14728 369442 +13019 369391 +14537 369330 +9177 369323 +11846 369307 +9564 369236 +15286 369223 +15725 369135 +8258 369120 +16094 369079 +8217 368989 +12897 368893 +14162 368880 +10918 368753 +7820 368688 +4563 368543 +9807 368432 +9561 368424 +18609 368234 +9075 368089 +9085 368050 +10231 368001 +11138 367959 +12246 367928 +972 367806 +10255 367786 +7906 367661 +8680 367603 +15115 367522 +13048 367461 +11087 367387 +9494 367342 +18071 367262 +964 367230 +12373 367196 +4175 367107 +8659 367082 +2070 367075 +7045 367061 +14510 367048 +10571 367012 +15070 366988 +12654 366942 +3310 366871 +7548 366733 +5977 366666 +12259 366570 +4909 366534 +16640 366491 +3167 366477 +4558 366360 +19610 366257 +11099 366173 +12232 366119 +7509 366023 +9374 365891 +3470 365884 +17127 365533 +8192 365498 +7114 365476 +10380 365341 +16137 365300 +4366 365277 +7946 365216 +7365 365191 +7088 364886 +9480 364726 +8216 364713 +7685 364626 +12932 364617 +2195 364581 +9931 364506 +13844 364477 +11390 364405 +8220 364387 +8253 364234 +14408 364112 +7882 364093 +12263 364016 +7152 363829 +3998 363827 +1087 363800 +2507 363606 +7427 363566 +7521 363514 +10708 363450 +7714 363438 +2248 363431 +15930 363224 +6651 363153 +7733 362858 +6457 362836 +8566 362822 +8498 362813 +1063 362682 +2353 362622 +10040 362443 +14931 361977 +23512 361973 +6766 361959 +5999 361585 +9348 361559 +9376 361481 +7603 361432 +7422 361379 +10184 361347 +5648 361297 +3320 361123 +10017 361097 +14909 361015 +9910 360987 +3824 360918 +6515 360811 +7496 360806 +19303 360766 +8791 360691 +12523 360595 +8993 360575 +5801 360431 +4529 360398 +10965 360256 +229 360226 +9457 360164 +11252 360152 +16346 360033 +14449 359968 +11686 359944 +18334 359855 +14812 359787 +10468 359667 +7998 359571 +9964 359523 +7451 359379 +9241 359237 +6711 359234 +4545 359213 +8287 359148 +816 358897 +10147 358736 +12380 358729 +9949 358648 +1196 358550 +9107 358437 +15795 358413 +5824 358312 +5372 358295 +5124 358289 +9086 358245 +6748 358216 +9168 358064 +6805 357931 +15498 357866 +14679 357853 +14300 357819 +15528 357737 +6069 357670 +40138 357664 +12763 357550 +16051 357498 +12389 357475 +4002 357463 +12069 357303 +6371 357290 +13572 357238 +15342 357172 +9860 357116 +7664 356998 +7950 356936 +10846 356850 +17547 356634 +18475 356600 +13125 356555 +9328 356554 +6543 356451 +2954 356267 +15055 356232 +10833 356213 +26878 356180 +5013 356178 +6953 356175 +5654 356154 +6101 356107 +14641 356073 +9151 356027 +4421 355987 +16535 355894 +5639 355847 +5050 355821 +9938 355713 +11325 355685 +7531 355646 +12981 355592 +7666 355538 +9399 355432 +13415 355422 +7538 355305 +1258 355163 +18545 355158 +2759 355119 +1136 355109 +6617 355081 +13571 355038 +14069 355019 +12437 354981 +17960 354969 +9325 354939 +13708 354915 +6420 354705 +14757 354591 +14919 354480 +9199 354451 +4108 354398 +12949 354382 +6578 354376 +9361 354353 +10718 354244 +13593 354199 +6052 354007 +8436 353994 +9105 353886 +12275 353856 +9121 353405 +11106 353379 +10800 353350 +8598 353325 +2100 353199 +2290 353173 +15080 353027 +11642 353004 +16542 352734 +16111 352650 +15602 352627 +10608 352611 +5313 352603 +17782 352590 +7838 352517 +16445 352493 +6909 352475 +14791 352413 +9857 352193 +17842 352066 +1638 352040 +16712 351998 +2164 351959 +14364 351891 +3287 351887 +8706 351810 +12873 351660 +13075 351610 +22484 351603 +16707 351599 +7381 351585 +4463 351585 +699 351567 +11856 351470 +25851 351312 +10364 351096 +8266 350892 +12492 350873 +14493 350708 +8854 350697 +13346 350683 +10469 350675 +10216 350657 +5147 350609 +4169 350591 +10872 350527 +10366 350409 +12801 350394 +9572 350320 +8149 350232 +6375 350198 +5447 350173 +28141 350103 +18733 350035 +13642 350000 +1144 349908 +10421 349802 +10427 349683 +14377 349622 +17826 349557 +6528 349442 +9877 349409 +4198 349335 +11550 349300 +14840 349249 +16952 349120 +7776 349119 +8914 349098 +15164 349012 +6469 348997 +6206 348967 +7731 348954 +6890 348922 +1348 348722 +10086 348564 +15037 348531 +9345 348509 +10474 348482 +8949 348330 +35327 348207 +13458 348166 +4901 348016 +13853 348002 +10925 347889 +9835 347859 +5414 347837 +3369 347791 +8582 347636 +11682 347617 +31433 347583 +9136 347427 +4702 347410 +9482 347376 +11271 347359 +566 347358 +17334 347227 +13850 347132 +11023 347130 +18263 347068 +10400 347041 +27133 346684 +6462 346645 +11272 346626 +9963 346622 +15198 346518 +7818 346505 +646 346495 +6778 346466 +9775 346459 +13485 346404 +9013 346329 +11946 346288 +10668 346248 +1330 346195 +8517 346056 +6239 345826 +9557 345703 +16978 345593 +8348 345532 +1272 345478 +11238 345461 +18555 345457 +20872 345277 +10359 345271 +17036 345196 +2120 344930 +2280 344773 +8573 344722 +10515 344702 +12020 344681 +10597 344592 +9260 344591 +12047 344581 +10429 344566 +11185 344446 +5892 344360 +5250 344253 +9160 343902 +8458 343880 +7252 343790 +31173 343777 +4022 343555 +10003 343480 +11193 343472 +6562 343348 +13925 343228 +17406 343189 +4730 343165 +11155 343055 +12231 343030 +17014 343022 +16618 342966 +7278 342897 +9211 342896 +5100 342826 +15066 342723 +11594 342721 +12527 342717 +8288 342688 +3664 342668 +9490 342586 +3839 342537 +14429 342519 +1075 342420 +2232 342183 +10980 342178 +2022 342057 +11472 342053 +3979 342036 +15346 341959 +15863 341944 +1691 341900 +15294 341872 +11112 341843 +14569 341817 +8915 341671 +10304 341465 +4843 341417 +6494 341415 +7926 341332 +8515 341225 +12046 341172 +1216 341117 +9473 341068 +9274 341050 +11570 341048 +6659 340992 +22473 340911 +8075 340763 +15393 340693 +14883 340689 +2522 340630 +9960 340607 +8212 340582 +7208 340577 +10594 340450 +21909 340415 +10765 340343 +4583 340262 +8603 340144 +13637 339972 +7138 339832 +9581 339824 +13188 339727 +15689 339721 +8749 339613 +18576 339525 +8425 339432 +10241 339308 +3902 339257 +11937 339211 +8222 339181 +7621 339139 +2518 339112 +11758 339065 +5855 338929 +12173 338908 +12615 338903 +8801 338853 +842 338791 +13142 338723 +5516 338721 +12329 338602 +7388 338534 +26105 338531 +5032 338437 +7795 338417 +8957 338345 +2483 338332 +10038 338262 +3743 338006 +8181 337997 +14952 337748 +5492 337746 +12882 337707 +11648 337676 +8806 337599 +5833 337517 +9780 337469 +4623 337392 +17880 337247 +4745 337244 +12538 337238 +6369 337188 +6758 337119 +15706 337069 +15870 336993 +9400 336894 +5721 336815 +7097 336767 +7869 336739 +20092 336483 +13229 336427 +20486 336410 +12970 336280 +8786 336264 +8440 336084 +12746 336078 +8268 335957 +19034 335955 +20082 335940 +5601 335924 +5210 335873 +17907 335805 +6075 335758 +4835 335634 +2129 335490 +10931 335423 +5607 335409 +11333 335370 +7653 335365 +12400 335354 +11602 335344 +4731 335188 +12575 335173 +1618 335125 +13606 335118 +21285 335017 +12107 334923 +1387 334914 +34033 334900 +14495 334864 +7331 334829 +18740 334712 +3987 334670 +16134 334569 +14860 334478 +1062 334442 +17185 334381 +12497 334367 +19616 334301 +3696 334292 +12772 334266 +11403 334201 +10172 334185 +2183 334078 +17409 334077 +9646 333987 +10941 333957 +4424 333731 +13109 333719 +32214 333692 +9408 333646 +5703 333577 +9901 333570 +7135 333416 +11495 333379 +13953 333300 +10182 333300 +8541 333269 +13833 333263 +9471 333259 +16527 333092 +16533 333048 +1739 333033 +2160 332999 +24520 332927 +17541 332914 +245 332905 +1601 332894 +8989 332859 +16512 332747 +4265 332705 +14576 332575 +3150 332510 +12565 332496 +10439 332445 +31112 332411 +14926 332365 +11450 332296 +1984 332256 +9871 332254 +4277 332226 +11684 332109 +6882 331909 +6733 331893 +11521 331862 +22488 331824 +10839 331793 +1055 331762 +10969 331754 +11177 331679 +18154 331626 +5737 331561 +7801 331482 +14409 331482 +11597 331282 +12023 331207 +9170 331045 +2799 331010 +8124 331002 +11553 330990 +9022 330926 +12827 330921 +10801 330902 +1699 330871 +12880 330544 +12465 330534 +14286 330531 +14783 330519 +7902 330467 +20000 330396 +10750 330328 +8887 330300 +14317 330224 +9620 330208 +13280 330181 +8712 330141 +11584 330100 +5053 329959 +5907 329943 +4415 329942 +10454 329927 +11132 329916 +14858 329894 +15670 329864 +8125 329754 +15422 329750 +12831 329749 +37666 329684 +16433 329663 +3192 329598 +11636 329464 +10193 329407 +11375 329359 +10695 329328 +14388 329322 +8180 329251 +9874 329097 +8920 329091 +4534 328825 +8903 328698 +23964 328655 +17122 328606 +7048 328597 +6211 328591 +8925 328576 +11955 328523 +2378 328502 +14896 328482 +9751 328231 +12578 328224 +1773 328185 +6338 328096 +8839 327999 +10923 327937 +7778 327911 +12617 327835 +9018 327783 +11732 327780 +7563 327739 +6803 327707 +8969 327692 +5552 327602 +3905 327557 +9911 327480 +7119 327458 +36285 327444 +7684 327424 +9616 327313 +11129 327297 +12032 327230 +11490 327080 +7792 327061 +13574 327032 +11398 327007 +8066 326906 +1952 326868 +2710 326703 +13362 326668 +14457 326642 +17339 326625 +12580 326624 +21861 326488 +21157 326473 +34119 326457 +14235 326388 +11819 326384 +4102 326329 +7209 326168 +6195 326143 +10291 326111 +8894 326110 +7330 325951 +5869 325935 +14525 325893 +13120 325862 +6167 325828 +9509 325807 +24382 325742 +7855 325721 +10078 325656 +7112 325654 +5599 325576 +8122 325525 +15105 325514 +15740 325468 +16753 325462 +9517 325427 +10953 325424 +13936 325332 +8862 325273 +3464 325244 +5452 325232 +20983 325102 +17582 325099 +1483 324860 +8035 324825 +8824 324800 +14119 324704 +12766 324689 +9412 324668 +12780 324579 +12521 324471 +9752 324452 +13923 324383 +6001 324264 +13145 324264 +9464 324208 +7336 324205 +15626 324177 +19773 324148 +8701 324135 +8504 324114 +9422 324110 +10457 324084 +5832 324083 +14654 324004 +13777 323947 +5142 323924 +3843 323885 +14264 323868 +8621 323792 +6176 323751 +10538 323625 +38836 323556 +9958 323458 +493 323456 +15406 323421 +10098 323257 +13272 323251 +14882 323077 +18369 323073 +18525 323037 +11278 322890 +24754 322889 +3536 322832 +12542 322786 +7893 322728 +2971 322724 +6949 322711 +12699 322709 +11920 322700 +9813 322679 +17078 322609 +5446 322605 +2778 322596 +13694 322559 +14567 322526 +12049 322315 +23576 322120 +4316 322022 +8699 322006 +9743 321964 +4331 321936 +10843 321863 +19310 321849 +8469 321771 +4121 321673 +12513 321647 +4470 321634 +9431 321433 +5793 321318 +7773 321315 +21279 321289 +8713 321248 +1233 321228 +3731 321181 +17381 321180 +9628 321107 +4386 321100 +9845 321023 +19250 320960 +5064 320841 +5540 320721 +4185 320694 +15469 320670 +7735 320664 +21313 320604 +965 320601 +10529 320574 +12545 320556 +725 320395 +15593 320377 +12641 320315 +3364 320117 +9429 320034 +11009 319918 +11293 319719 +17357 319656 +4105 319633 +2224 319519 +15287 319462 +15929 319083 +5500 319073 +5848 318999 +3826 318962 +7353 318844 +7721 318797 +6495 318768 +16093 318732 +12309 318723 +18672 318712 +20171 318609 +16852 318532 +11169 318478 +9314 318476 +2229 318472 +7329 318393 +7416 318319 +20384 318047 +11004 317965 +17524 317883 +8678 317800 +11540 317774 +659 317752 +5036 317723 +13106 317669 +8787 317543 +13029 317496 +20716 317472 +2821 317076 +13748 317005 +9391 316991 +19332 316987 +19483 316986 +9727 316962 +11658 316942 +12607 316917 +13883 316914 +9155 316768 +2348 316745 +17955 316743 +1373 316719 +7873 316717 +2069 316670 +21105 316623 +8771 316595 +10452 316592 +11756 316513 +17915 316507 +6455 316406 +11824 316389 +22815 316388 +13784 316374 +7424 316364 +6386 316312 +20617 316271 +5557 316215 +20357 316185 +14794 316066 +17352 316046 +28630 315957 +5968 315855 +16428 315835 +6920 315834 +11130 315824 +11545 315711 +2118 315680 +3727 315673 +16182 315559 +15229 315523 +16387 315368 +21933 315289 +11359 315283 +10649 315261 +4659 315233 +14240 315221 +17356 315168 +17192 315122 +8689 315037 +11765 315023 +6251 314974 +12141 314897 +9055 314803 +11845 314792 +13566 314771 +9934 314711 +2741 314681 +27344 314655 +8281 314654 +5874 314594 +9891 314528 +12634 314519 +8782 314510 +21865 314451 +9520 314369 +11768 314347 +10206 314338 +12759 314296 +4255 314282 +35185 314258 +10271 314254 +11926 314202 +1385 314190 +9323 314182 +25841 313849 +13644 313842 +3686 313681 +18726 313573 +10059 313482 +1328 313246 +8365 313222 +7096 313182 +12811 313118 +21877 313081 +16679 313028 +9811 313018 +11882 313016 +5416 312970 +7275 312919 +7613 312904 +4177 312865 +9460 312773 +18848 312722 +14544 312645 +16915 312621 +13901 312583 +7156 312558 +12735 312557 +14850 312525 +9373 312429 +7888 312382 +13449 312351 +29378 312294 +13307 312290 +22026 312233 +17521 312201 +9892 312174 +16476 312147 +1333 312144 +13342 312097 +14598 312036 +18859 312015 +8765 311951 +12228 311876 +5355 311833 +13933 311638 +20741 311606 +9824 311582 +9104 311540 +26138 311538 +13768 311520 +23481 311505 +14600 311385 +18309 311349 +931 311338 +3673 311267 +10651 311194 +6685 311113 +11473 311083 +13276 311034 +8880 310999 +6489 310946 +12695 310937 +4802 310930 +9355 310849 +10938 310762 +4743 310727 +15300 310686 +10768 310656 +9575 310603 +12519 310526 +10215 310423 +13124 310339 +10170 310325 +13902 310286 +13944 310233 +12540 310206 +13317 310177 +12151 310173 +1286 310125 +7125 309987 +11172 309933 +14669 309886 +12926 309859 +7661 309685 +9794 309629 +9585 309568 +14135 309560 +2194 309554 +11294 309522 +2544 309428 +13692 309380 +16248 309370 +17196 309335 +9175 309081 +15261 309005 +1473 308985 +1960 308936 +13456 308907 +10546 308821 +14762 308772 +10591 308714 +11721 308591 +3086 308558 +9363 308490 +13176 308475 +11989 308474 +10543 308427 +23781 308348 +13403 308323 +9099 308280 +11784 308078 +32089 307995 +529 307911 +95 307756 +3939 307654 +16435 307639 +20182 307587 +34847 307510 +10509 307452 +6071 307446 +11500 307410 +10384 307357 +15343 307354 +1963 307209 +9541 307199 +14939 306944 +20014 306924 +11903 306893 +5921 306873 +12723 306865 +3178 306813 +17389 306782 +12705 306774 +12068 306684 +3460 306676 +10070 306651 +11192 306520 +21841 306506 +9017 306491 +20755 306438 +6843 306369 +8670 306349 +17281 306327 +19391 306285 +9450 306242 +16574 305876 +12812 305859 +5067 305834 +7295 305811 +15604 305795 +15454 305645 +9757 305637 +7709 305617 +7443 305616 +2558 305584 +11013 305563 +11161 305551 +8661 305334 +13146 305327 +9294 305325 +6248 305317 +16457 305285 +24505 305278 +9550 305260 +535 305241 +18677 305121 +13832 305112 +5325 305045 +10630 304946 +12983 304915 +20119 304903 +8161 304873 +13840 304833 +12528 304832 +2085 304744 +15664 304720 +885 304692 +22593 304679 +9563 304674 +15392 304568 +9776 304540 +7815 304496 +5331 304477 +5023 304443 +17661 304421 +12344 304409 +14082 304392 +14737 304191 +10134 304135 +19952 304119 +12955 303828 +18505 303801 +7812 303777 +21574 303696 +5209 303694 +11234 303620 +9867 303608 +12751 303483 +12091 303455 +12606 303441 +11142 303322 +9447 303166 +11533 303122 +16168 303103 +7207 303092 +11666 302797 +9842 302745 +11322 302716 +11410 302571 +9567 302570 +18256 302546 +9706 302540 +10494 302536 +14899 302514 +8537 302402 +14101 302392 +5430 302339 +16307 302301 +3325 302118 +13766 302110 +6431 302098 +11626 302087 +1434 301879 +10518 301863 +12669 301838 +10869 301811 +7702 301726 +7258 301624 +7605 301599 +4891 301498 +1504 301484 +11665 301409 +11426 301408 +18220 301381 +13512 301342 +17068 301234 +1942 301225 +3311 301188 +9415 301179 +3820 301175 +11512 301087 +8719 300995 +8182 300908 +17235 300901 +5777 300818 +5794 300789 +7223 300767 +18864 300667 +10443 300646 +24589 300646 +17132 300620 +4340 300534 +7104 300478 +1188 300402 +3019 300388 +6992 300317 +8354 300316 +13609 300300 +4245 300252 +9079 300197 +15199 300105 +7157 300038 +762 299991 +17450 299943 +17044 299810 +16349 299699 +8975 299568 +4806 299541 +2895 299520 +15573 299493 +13116 299369 +10625 299261 +13037 299171 +2419 299150 +7610 299146 +13201 298875 +7599 298787 +3831 298725 +15984 298640 +5171 298514 +2604 298497 +13594 298417 +10287 298407 +10032 298356 +14538 298240 +31061 298197 +17630 298135 +14503 298117 +7808 298087 +4000 298047 +5660 298010 +6213 297993 +14417 297966 +8495 297957 +6245 297936 +29115 297877 +10069 297860 +5932 297831 +847 297818 +15459 297746 +7197 297706 +7937 297624 +6407 297566 +8477 297541 +17490 297510 +7957 297419 +19847 297359 +16298 297176 +16328 297109 +5097 297095 +18708 297085 +2817 297026 +10758 296992 +6941 296987 +17930 296964 +9322 296934 +10299 296918 +7644 296748 +13320 296692 +10927 296691 +11812 296683 +18151 296588 +8650 296576 +11952 296532 +653 296525 +10467 296457 +222 296347 +18985 296336 +17646 296312 +3361 296291 +15508 296281 +10908 296275 +12299 296124 +11589 296065 +4357 296044 +4649 296014 +19912 295919 +10238 295896 +7598 295800 +10902 295794 +824 295763 +2434 295738 +108 295681 +14396 295673 +19566 295671 +16313 295653 +4509 295636 +14616 295610 +17619 295541 +10999 295494 +5755 295491 +8697 295465 +13830 295385 +19045 295195 +21162 295171 +5489 295157 +11191 295115 +15153 295040 +3449 294958 +11150 294898 +18133 294799 +10989 294694 +14770 294585 +11554 294580 +10476 294501 +5002 294469 +3440 294422 +21765 294385 +11791 294325 +15840 294296 +17719 294251 +5812 294223 +25236 294167 +16721 294070 +16888 293941 +868 293915 +12118 293877 +6458 293767 +14533 293678 +10000 293634 +24664 293626 +12019 293496 +1686 293475 +8309 293429 +3055 293333 +13230 293232 +12536 293224 +15369 293223 +8941 293204 +14778 293112 +22535 293108 +11015 293042 +18825 292878 +17159 292877 +8895 292682 +1035 292678 +15372 292655 +14248 292637 +10898 292633 +13973 292544 +11801 292523 +20261 292508 +10609 292453 +11311 292434 +17671 292434 +7768 292322 +26318 292282 +16031 292266 +10948 292244 +10899 292189 +2345 292185 +12671 291955 +11889 291889 +20899 291879 +13183 291782 +5286 291580 +11909 291559 +11637 291497 +6965 291479 +12525 291428 +6170 291277 +4103 291236 +12937 291227 +15523 291170 +16156 291108 +24272 291089 +9507 291083 +11899 291014 +17857 290869 +6521 290825 +4092 290814 +7159 290777 +24320 290770 +12435 290704 +8492 290694 +11932 290634 +11206 290586 +14447 290535 +18961 290530 +48099 290522 +8640 290518 +492 290477 +1418 290394 +19982 290348 +9225 290286 +2261 290270 +11168 290218 +18479 290213 +20499 290165 +11810 290067 +11210 289892 +9329 289865 +15471 289784 +14518 289756 +18020 289680 +14488 289576 +7035 289565 +8662 289554 +11681 289503 +21562 289256 +4863 289185 +4064 289173 +19247 289165 +5195 289164 +16582 289117 +6908 289054 +21866 289013 +12266 288914 +3697 288830 +10233 288812 +12377 288775 +14734 288753 +33683 288737 +771 288733 +2030 288732 +11026 288731 +15317 288722 +6902 288708 +48796 288704 +22386 288678 +21614 288662 +2134 288587 +5765 288576 +13584 288406 +7378 288289 +819 288274 +10129 288254 +11075 288220 +2680 288189 +13831 288175 +18393 288077 +11676 288063 +15383 287991 +4359 287991 +36030 287903 +20852 287857 +5181 287657 +10726 287650 +21323 287633 +10559 287568 +14332 287540 +9555 287519 +9595 287510 +6194 287503 +6539 287493 +11818 287374 +1202 287364 +4319 287305 +18110 287296 +6573 287280 +15891 287257 +9293 287226 +12432 287194 +10030 287189 +18086 287141 +14086 287073 +10762 287047 +13465 287036 +19744 287018 +9387 287010 +14867 286974 +11464 286951 +24003 286921 +8617 286874 +10471 286857 +3112 286748 +12725 286626 +21165 286488 +7319 286337 +12036 286305 +4229 286285 +12537 286255 +11005 286249 +697 286174 +10700 286147 +12548 286127 +7515 286098 +6109 286082 +16015 286066 +8046 286061 +11544 285977 +13855 285908 +11735 285898 +15192 285893 +9653 285800 +10792 285796 +7616 285649 +15556 285556 +17799 285509 +9773 285508 +25287 285465 +14416 285439 +18572 285395 +10358 285292 +9183 285207 +12073 285159 +16754 285095 +8721 285047 +17081 285026 +12593 285023 +9568 284957 +8101 284931 +12002 284929 +8472 284908 +7770 284900 +5077 284807 +28852 284805 +19523 284761 +14219 284693 +15177 284685 +13722 284615 +12734 284575 +13404 284454 +4982 284434 +6751 284376 +10008 284360 +6096 284343 +14999 284301 +10087 284270 +11918 284068 +11699 284065 +10701 283996 +9583 283931 +6291 283902 +15196 283850 +15416 283835 +14521 283648 +13723 283591 +7993 283539 +8559 283442 +7650 283209 +11552 283208 +7929 283037 +18106 283014 +11065 283009 +9793 282858 +10860 282804 +2189 282772 +8717 282753 +2849 282749 +4014 282718 +7900 282712 +7335 282598 +14225 282574 +20228 282555 +12633 282528 +10824 282517 +11455 282474 +552 282383 +11954 282283 +3911 282276 +6757 282252 +8840 282235 +8805 282201 +19375 282196 +8580 282166 +6738 282152 +10906 282124 +13325 282069 +12111 282057 +7193 282053 +13004 281987 +904 281961 +15579 281954 +11003 281952 +17608 281805 +8135 281776 +12308 281762 +5994 281742 +12689 281646 +13678 281637 +3400 281600 +17151 281552 +20176 281468 +6381 281450 +10420 281445 +16105 281417 +6283 281376 +16774 281323 +12865 281321 +9675 281161 +21439 281160 +10073 281140 +15012 281072 +14820 281032 +4420 281012 +13389 280931 +15278 280924 +14707 280864 +11693 280834 +20989 280800 +9061 280631 +7811 280622 +17702 280585 +19712 280517 +18085 280506 +15792 280423 +19747 280412 +8315 280313 +39683 280210 +4679 280163 +14674 280064 +23039 280029 +20707 280003 +21014 279940 +10250 279930 +11936 279904 +15062 279814 +13588 279790 +11975 279759 +16528 279751 +24687 279670 +15616 279652 +11119 279609 +20355 279492 +9458 279484 +2603 279343 +6335 279249 +2340 279229 +17423 279191 +23623 279109 +17325 279054 +3570 278931 +13551 278846 +12995 278839 +18168 278838 +5781 278820 +9986 278790 +4112 278768 +13184 278726 +15890 278588 +18678 278548 +14213 278519 +1979 278509 +16420 278503 +13111 278460 +3846 278452 +5643 278436 +872 278359 +11868 278353 +12688 278278 +14273 278220 +40466 278181 +20700 278103 +14888 277999 +10218 277984 +16940 277960 +2072 277862 +12348 277847 +35946 277842 +16029 277774 +1013 277769 +28840 277738 +2896 277683 +15505 277680 +6446 277613 +8691 277549 +1298 277514 +14752 277419 +5714 277364 +8058 277325 +20894 277293 +16434 277271 +32115 277243 +10667 277221 +5889 277187 +16210 277086 +19788 277074 +3533 277064 +23555 277023 +8177 276841 +10045 276692 +1712 276688 +1335 276680 +20139 276667 +19245 276644 +10652 276628 +12798 276600 +16332 276591 +1702 276565 +13466 276542 +7894 276539 +13750 276477 +28070 276466 +4801 276375 +8758 276319 +6121 276242 +15599 276225 +9216 276173 +18913 276162 +15257 276075 +15397 276033 +12079 275990 +9154 275941 +9066 275923 +10041 275851 +18685 275759 +19362 275648 +5748 275520 +4134 275490 +10968 275417 +8116 275403 +20337 275390 +9676 275296 +14523 275249 +10280 275230 +11829 275119 +9864 275070 +2313 275056 +11010 275033 +1967 274991 +11611 274774 +17836 274745 +7967 274431 +9080 274379 +13891 274378 +11069 274377 +9197 274377 +6487 274363 +48198 274323 +11164 274304 +14725 274301 +12868 274287 +977 274251 +18358 274249 +11072 274222 +8405 273995 +13110 273961 +1436 273939 +9812 273852 +38251 273850 +4976 273834 +37175 273792 +4279 273790 +5756 273779 +10637 273707 +10130 273685 +6080 273601 +12182 273585 +7202 273453 +8850 273393 +1763 273379 +10498 273327 +2387 273318 +14058 273304 +15360 273269 +5572 273260 +6300 273225 +9193 273201 +4873 273182 +18034 273131 +10643 273103 +18966 273048 +13338 273037 +17468 273016 +8942 272969 +10706 272949 +12553 272893 +18353 272878 +15623 272786 +13573 272782 +15901 272749 +9441 272650 +10813 272641 +8735 272558 +6721 272540 +8338 272518 +22905 272496 +9639 272495 +8500 272427 +8099 272380 +9148 272311 +5854 272243 +10204 272231 +20222 272164 +10616 272112 +7036 272099 +12203 272005 +9927 271973 +2528 271969 +14672 271956 +20710 271955 +14481 271954 +13938 271929 +10776 271845 +17588 271844 +36309 271778 +6546 271746 +15296 271737 +9508 271628 +20522 271611 +2172 271603 +8202 271578 +12829 271575 +19800 271532 +26763 271446 +6752 271369 +43577 271245 +11256 271209 +19717 271164 +8095 271162 +11095 271143 +3849 271032 +8235 271029 +738 270906 +11559 270890 +20059 270852 +13408 270845 +3028 270826 +5700 270819 +2382 270815 +8033 270752 +9796 270711 +9707 270670 +15586 270602 +11263 270508 +13032 270474 +3982 270469 +18066 270444 +13059 270444 +14064 270365 +14085 270332 +28145 270328 +12911 270252 +11558 270112 +16239 270107 +15188 270048 +23761 270032 +5826 269980 +11032 269966 +20782 269958 +17838 269931 +13546 269902 +13041 269892 +16831 269852 +14779 269849 +5222 269801 +24711 269796 +36763 269737 +28125 269727 +11462 269718 +17913 269702 +5499 269615 +25335 269614 +14663 269604 +21730 269511 +14912 269493 +8711 269479 +12014 269429 +10436 269409 +14142 269313 +9888 269307 +18513 269211 +12651 269198 +13413 269143 +10261 269129 +1875 269121 +19101 269115 +20259 269074 +6581 268988 +11832 268914 +6224 268884 +11557 268878 +14667 268836 +9713 268811 +4943 268781 +16383 268725 +17704 268673 +20581 268654 +17642 268565 +15348 268558 +8798 268409 +9826 268357 +18772 268355 +13977 268336 +12436 268316 +11357 268262 +9305 268185 +11949 268141 +8509 268137 +10176 268091 +20529 268080 +14260 268054 +16246 268025 +7577 268019 +22635 267980 +12411 267878 +15220 267832 +16902 267772 +15726 267705 +17475 267693 +11997 267649 +9719 267643 +9966 267593 +11706 267584 +27993 267517 +9698 267510 +16769 267493 +12520 267425 +18939 267423 +6998 267352 +13714 267345 +7988 267328 +5432 267300 +9159 267270 +970 267247 +1883 267156 +12597 267130 +969 267116 +7851 267079 +11396 267044 +6319 267022 +7572 267019 +13487 267000 +13586 266983 +25945 266865 +11135 266858 +20426 266817 +22183 266781 +16334 266772 +20325 266757 +21494 266718 +5626 266678 +14033 266677 +2731 266637 +18212 266575 +12601 266521 +8396 266504 +10584 266501 +9829 266467 +5128 266439 +4718 266285 +8283 266232 +3163 266200 +18665 266165 +14043 266065 +2140 266017 +19553 265926 +12724 265897 +12908 265737 +22933 265703 +13990 265584 +15463 265520 +9162 265465 +14517 265285 +12336 265242 +5408 265206 +17167 265200 +15971 265164 +25651 265080 +8727 265079 +19017 265072 +33160 265025 +21147 264980 +16893 264915 +16797 264903 +13684 264899 +9849 264890 +16559 264831 +14718 264765 +12655 264748 +4023 264639 +9947 264581 +16155 264574 +31594 264544 +6104 264519 +15806 264392 +21221 264345 +1397 264227 +4083 264178 +23394 264177 +10083 264160 +33328 264146 +6156 264080 +12939 264080 +9950 264074 +8166 264059 +15030 264027 +32324 263999 +12346 263955 +13474 263950 +11874 263884 +19995 263882 +130 263839 +14634 263819 +5646 263795 +5421 263702 +7294 263697 +17165 263670 +6764 263622 +12503 263616 +6428 263585 +12225 263506 +4853 263495 +24161 263476 +7268 263464 +10829 263388 +8399 263381 +11694 263379 +8341 263349 +10689 263278 +17788 263200 +15832 263001 +10485 262951 +17053 262897 +98 262873 +15503 262806 +10393 262799 +21026 262785 +16089 262736 +7145 262696 +14329 262691 +10988 262667 +4938 262617 +6631 262431 +13094 262414 +4759 262370 +10713 262359 +722 262358 +6607 262294 +10338 262229 +27459 262199 +1985 262192 +8005 262074 +11160 262045 +20065 262043 +2530 262029 +9917 262001 +11107 261971 +5245 261877 +8842 261858 +13244 261844 +1756 261828 +18624 261697 +18496 261651 +13640 261637 +13234 261637 +12460 261630 +13688 261554 +934 261540 +13998 261539 +3949 261532 +15591 261500 +10766 261405 +13463 261377 +3498 261340 +7853 261336 +13657 261333 +16565 261324 +11266 261295 +10050 261267 +17187 261200 +7468 261148 +6570 261101 +9112 261095 +12841 261041 +15465 260960 +19868 260941 +14742 260914 +12080 260898 +12008 260822 +9514 260792 +8968 260774 +18240 260745 +10453 260671 +20614 260614 +12162 260544 +13534 260539 +1207 260515 +19675 260494 +5211 260493 +7532 260462 +27116 260264 +18014 260241 +4272 260231 +12101 260002 +20055 260002 +18184 259947 +21742 259850 +8187 259823 +789 259784 +7073 259779 +11338 259732 +14710 259554 +13820 259548 +16918 259503 +18327 259488 +5354 259482 +1154 259371 +13112 259364 +5411 259218 +17413 259156 +7570 259098 +17028 259065 +3775 259024 +7576 258971 +10495 258913 +6127 258860 +3742 258785 +3524 258778 +29002 258748 +15438 258703 +4948 258702 +19804 258665 +5899 258622 +2197 258614 +6453 258581 +13293 258579 +24880 258460 +9850 258441 +6974 258388 +6488 258313 +13475 258303 +9637 258255 +20633 258235 +20920 258220 +12910 258106 +11326 258084 +15443 258081 +6577 258043 +11323 258035 +8300 258029 +5735 257973 +15415 257934 +16449 257901 +2531 257843 +6894 257785 +6968 257729 +11418 257714 +8449 257532 +7688 257450 +22257 257392 +13134 257380 +8807 257349 +11281 257236 +16076 257119 +14110 257102 +27061 257101 +20430 257077 +6098 257021 +12890 257009 +5779 257002 +6473 256775 +11236 256721 +10403 256692 +3943 256662 +20052 256644 +8441 256628 +18899 256570 +14067 256490 +11576 256482 +10884 256474 +12616 256472 +4111 256447 +16905 256399 +14903 256344 +11195 256341 +7780 256319 +1164 256273 +8967 256253 +3493 256114 +8752 256058 +2541 256032 +10392 256030 +10581 255868 +3378 255861 +1798 255786 +5629 255752 +15191 255708 +11204 255626 +5536 255621 +18473 255525 +28209 255401 +8797 255344 +28086 255308 +5935 255300 +9369 255253 +8781 255237 +14601 255157 +235 255145 +755 255125 +6715 255118 +12457 255082 +3779 255039 +21525 255029 +18193 254999 +4447 254987 +24577 254972 +13069 254847 +12598 254776 +2564 254772 +11274 254713 +9765 254691 +16849 254651 +23425 254644 +22016 254589 +348 254584 +8977 254497 +10190 254449 +20442 254376 +10627 254325 +10417 254323 +13982 254312 +11353 254285 +24538 254246 +13478 254180 +7660 254144 +11131 254100 +14285 254067 +7646 254033 +12201 253999 +11029 253849 +11513 253830 +17218 253821 +1291 253782 +22159 253746 +11847 253742 +11422 253730 +4980 253724 +19817 253712 +9526 253590 +8324 253576 +7439 253554 +3575 253498 +6915 253302 +18406 253209 +14559 253205 +11853 253138 +10724 253101 +15976 253017 +4335 252999 +18489 252895 +19323 252807 +11746 252702 +4117 252677 +16443 252637 +14954 252628 +16028 252566 +15224 252495 +9961 252455 +9300 252455 +840 252429 +5513 252399 +6485 252340 +13016 252335 +10647 252236 +12556 252163 +36103 252155 +5822 252148 +10574 252130 +14159 252092 +14103 251995 +3007 251941 +4868 251923 +11608 251871 +15543 251818 +12627 251743 +2632 251725 +17369 251625 +21605 251579 +23201 251546 +11614 251528 +14636 251438 +14113 251425 +10996 251405 +1059 251391 +17277 251368 +12325 251362 +17676 251307 +15760 251285 +1358 251238 +11925 251213 +35851 251159 +16179 251087 +9506 251020 +9405 251012 +14581 250999 +21714 250981 +15866 250957 +19292 250939 +13778 250936 +2721 250909 +16805 250843 +9609 250738 +2894 250728 +1481 250694 +14984 250681 +12187 250629 +16209 250606 +6667 250598 +12039 250592 +26351 250590 +19464 250589 +3847 250521 +10096 250519 +28143 250431 +11561 250430 +18755 250425 +6831 250402 +5345 250399 +18303 250342 +12744 250205 +17645 250183 +472 250178 +13258 250161 +7889 250156 +13985 250150 +3100 250117 +12399 250031 +11842 250024 +6770 250012 +6028 249984 +19422 249883 +6185 249881 +16095 249875 +14442 249839 +18277 249820 +14008 249716 +4780 249716 +10564 249657 +7935 249650 +6524 249633 +14261 249603 +10942 249535 +8201 249530 +21872 249484 +12387 249481 +5159 249453 +11963 249430 +10852 249408 +20572 249303 +37985 249161 +26225 249133 +1428 249004 +23267 248991 +22347 248928 +542 248860 +28162 248806 +4226 248801 +11675 248800 +8947 248788 +1534 248786 +41224 248756 +9768 248730 +13677 248668 +10844 248660 +15123 248616 +37379 248570 +25672 248531 +9324 248483 +14196 248438 +12097 248435 +19687 248328 +8861 248293 +13279 248278 +13835 248278 +10975 248165 +25682 248143 +8532 248039 +11751 247945 +18524 247934 +19886 247924 +11483 247918 +29393 247899 +3003 247868 +21532 247858 +11886 247855 +9919 247809 +8579 247739 +7508 247717 +5178 247696 +10958 247675 +11760 247663 +8788 247636 +14000 247605 +17104 247525 +11574 247443 +16824 247389 +17551 247323 +2363 247278 +8117 247274 +15970 247271 +15401 247260 +19339 247215 +21336 247052 +5277 246999 +9905 246986 +15461 246929 +1426 246892 +12010 246866 +14665 246863 +7700 246845 +10825 246814 +1384 246762 +26360 246755 +14166 246713 +19786 246710 +1447 246566 +17871 246562 +4170 246466 +12802 246446 +17735 246443 +13863 246416 +9987 246399 +6771 246369 +12037 246276 +9922 246272 +11298 246270 +14071 246245 +12818 246200 +2002 246197 +2899 246184 +12034 246175 +14111 246152 +12287 246149 +18080 246097 +13892 246079 +6102 246042 +18158 246018 +14644 245998 +6158 245996 +12242 245993 +12058 245991 +1837 245897 +700 245844 +8554 245716 +3372 245711 +1287 245705 +1126 245700 +13237 245676 +2906 245653 +4472 245609 +14148 245596 +9715 245545 +11740 245521 +2320 245503 +24782 245428 +9721 245401 +822 245388 +23063 245383 +2416 245248 +44155 245172 +49443 245152 +19522 245137 +712 245074 +22297 245043 +10744 245042 +15395 244980 +9837 244959 +16674 244923 +9220 244912 +5532 244856 +10077 244828 +9686 244823 +14236 244822 +2676 244770 +12132 244758 +9050 244729 +14226 244674 +13157 244611 +17699 244469 +17038 244387 +6893 244356 +13504 244340 +10047 244337 +22867 244314 +13091 244305 +10435 244293 +33672 244254 +15600 244222 +21796 244154 +14971 244148 +20531 244110 +12170 244096 +24721 244062 +37129 244008 +18183 243979 +17559 243908 +15238 243887 +19436 243839 +811 243838 +15006 243778 +15097 243778 +18329 243649 +20536 243618 +4665 243601 +13969 243595 +11342 243536 +579 243534 +15013 243522 +6700 243506 +13524 243505 +7402 243497 +8280 243462 +10301 243456 +21291 243454 +19167 243434 +13472 243353 +20472 243313 +13011 243279 +16238 243267 +17996 243223 +16534 243174 +11811 243163 +1614 243152 +18704 243103 +9217 243076 +5949 243011 +19090 242932 +9923 242910 +12086 242909 +7461 242849 +12700 242795 +18373 242776 +17371 242768 +5563 242766 +47201 242738 +21070 242727 +17634 242709 +10425 242709 +8434 242578 +5305 242536 +13252 242499 +14348 242487 +9246 242471 +15150 242436 +12500 242426 +17528 242424 +18887 242414 +24116 242389 +23488 242387 +7101 242331 +12973 242273 +2617 242131 +7222 242040 +20330 241995 +15713 241800 +16787 241745 +19307 241718 +18884 241713 +14960 241706 +957 241671 +14748 241658 +7102 241557 +7006 241500 +6340 241483 +8946 241477 +17832 241462 +17905 241448 +26244 241403 +15812 241375 +11770 241368 +9661 241361 +17591 241249 +12729 241190 +26058 241190 +6663 241181 +14923 241131 +20131 241093 +21668 241073 +18470 241045 +4852 241036 +24568 241000 +8835 240978 +31259 240907 +13214 240824 +1746 240812 +4597 240806 +11006 240802 +394 240783 +14713 240716 +13535 240664 +14702 240509 +10323 240409 +19025 240405 +15403 240391 +15470 240322 +10682 240296 +10111 240284 +18518 240277 +7160 240260 +15707 240246 +16452 240209 +14178 240201 +16032 240154 +19116 240139 +8113 240113 +13852 240086 +14022 240037 +12591 239957 +14649 239945 +14339 239916 +21474 239915 +17709 239869 +21952 239817 +8264 239800 +17285 239799 +15737 239731 +17205 239671 +12622 239654 +15444 239648 +16316 239634 +2096 239603 +22948 239581 +9988 239554 +15462 239547 +13167 239539 +3989 239496 +14347 239410 +7911 239385 +18950 239341 +16012 239331 +15779 239300 +13245 239217 +10939 239205 +43208 239145 +8044 239110 +32537 239109 +9109 239055 +16468 239053 +9831 239022 +12289 239015 +12473 239008 +18956 238967 +14034 238965 +32390 238949 +1603 238943 +14759 238899 +19568 238873 +11312 238863 +33448 238862 +12602 238848 +10863 238822 +11508 238722 +1721 238695 +15347 238681 +13473 238680 +14443 238591 +32482 238579 +21305 238555 +2412 238509 +24964 238499 +9433 238479 +12778 238475 +5183 238472 +9749 238441 +6707 238429 +13232 238307 +18410 238305 +9977 238236 +8167 238147 +6089 238087 +2262 238000 +23058 237971 +14009 237931 +14218 237929 +8856 237914 +21938 237904 +9865 237862 +23698 237825 +22261 237799 +12066 237794 +19614 237663 +21550 237644 +6503 237632 +21421 237611 +4407 237500 +18728 237407 +12585 237404 +18190 237370 +13913 237323 +6327 237314 +7091 237275 +7484 237239 +14807 237233 +19919 237104 +2240 237027 +12883 237006 +12672 236984 +17245 236981 +18613 236789 +16135 236789 +44510 236747 +35409 236737 +15031 236551 +12013 236527 +15386 236516 +17176 236497 +10815 236496 +10492 236470 +10911 236469 +13537 236392 +4243 236333 +6400 236316 +29007 236291 +7631 236276 +8314 236264 +20765 236214 +23559 236191 +39859 236188 +14123 236160 +113 236154 +5580 236071 +9257 236046 +10670 236036 +15773 236026 +20395 235988 +18889 235975 +3681 235975 +19136 235965 +14552 235898 +8768 235870 +17075 235844 +7031 235821 +18073 235812 +18311 235801 +4822 235768 +25158 235758 +17906 235743 +17016 235723 +17355 235718 +11170 235613 +7129 235589 +23128 235587 +10746 235529 +3960 235490 +19819 235460 +7177 235452 +8996 235421 +16659 235413 +1070 235407 +4157 235389 +8304 235384 +20604 235267 +7834 235243 +10048 235231 +11270 235182 +13701 235151 +16689 235144 +8718 235065 +39999 235029 +15530 235010 +10598 235006 +19477 234921 +22923 234913 +19834 234897 +19603 234886 +8604 234842 +22230 234835 +10312 234832 +13885 234797 +15280 234727 +13754 234708 +15841 234647 +13622 234596 +12412 234580 +7122 234532 +22874 234512 +11546 234510 +12123 234453 +31995 234438 +18623 234425 +10090 234380 +11506 234346 +16295 234317 +28282 234313 +9907 234307 +17030 234286 +15364 234266 +2781 234214 +15273 234174 +9425 234152 +8599 234104 +10912 234073 +19887 234033 +12278 234000 +8779 233995 +928 233978 +22410 233978 +37186 233977 +10552 233962 +17785 233866 +3137 233815 +2286 233801 +12588 233798 +11446 233668 +20209 233640 +11541 233540 +13118 233501 +20994 233463 +7299 233456 +20495 233430 +14615 233395 +12028 233375 +13771 233371 +19940 233078 +10715 233075 +11940 233049 +15744 233012 +14023 233010 +21298 232991 +10755 232962 +12445 232844 +15849 232811 +12161 232793 +13488 232737 +19627 232725 +14303 232713 +9970 232693 +17155 232615 +16293 232590 +18423 232583 +12917 232573 +13225 232560 +16816 232513 +22137 232462 +2550 232439 +39970 232428 +1749 232423 +26053 232383 +7231 232347 +24540 232344 +12452 232280 +20890 232235 +5304 232190 +10861 232125 +5026 232123 +25763 232111 +15375 232100 +20882 232062 +29224 232056 +17051 232023 +6068 231967 +20377 231908 +15868 231882 +21588 231866 +478 231844 +9004 231835 +10025 231767 +3916 231764 +28058 231697 +6324 231683 +18039 231677 +14891 231669 +9009 231622 +9215 231608 +3515 231529 +12316 231500 +10225 231487 +13895 231467 +14790 231452 +1831 231443 +10537 231322 +7571 231276 +19790 231269 +16115 231247 +12990 231210 +7590 231116 +3699 231084 +3041 231078 +12566 231025 +12191 231001 +13613 230993 +8447 230991 +5770 230906 +45811 230849 +1557 230840 +12059 230775 +5120 230749 +6124 230744 +8595 230721 +9718 230711 +12872 230700 +10066 230682 +26280 230666 +14053 230646 +16126 230641 +20815 230603 +20903 230557 +916 230552 +241 230545 +17654 230421 +20202 230418 +5542 230400 +3109 230337 +17364 230305 +3174 230284 +17797 230253 +31561 230216 +13105 230200 +9656 230199 +16639 230184 +11835 230183 +6139 230117 +9082 230099 +14287 230049 +18280 230024 +25670 229987 +7430 229958 +18481 229950 +22671 229936 +11084 229910 +12174 229895 +11073 229858 +13756 229837 +17600 229795 +11447 229737 +8845 229737 +15534 229685 +21576 229658 +3522 229633 +19405 229593 +14381 229582 +18866 229566 +4033 229525 +12075 229520 +22536 229517 +19947 229507 +16280 229473 +6203 229472 +15491 229435 +8563 229365 +11680 229308 +20799 229304 +13705 229298 +18933 229208 +17266 229208 +14462 229158 +12742 229153 +23669 229130 +20382 229080 +10889 229078 +20565 229060 +12439 229004 +5686 228929 +16467 228914 +17542 228834 +11764 228816 +13693 228765 +10823 228749 +15892 228474 +30452 228448 +10472 228390 +16610 228364 +9236 228343 +24174 228330 +17476 228315 +15800 228213 +15981 228148 +7236 228132 +26199 228108 +2684 228107 +3103 228045 +12133 228039 +9672 228007 +4532 228006 +10900 227989 +8924 227958 +8799 227886 +10224 227883 +28986 227838 +10797 227833 +2782 227758 +33007 227718 +12463 227636 +29367 227597 +7221 227558 +12739 227509 +26597 227410 +12914 227402 +20656 227371 +22372 227348 +12954 227344 +4919 227333 +1996 227306 +14516 227304 +19584 227302 +33467 227285 +15549 227265 +14451 227254 +25176 227163 +8513 227110 +9265 227079 +5825 227077 +16916 227064 +8837 227058 +11304 227013 +16525 226927 +13881 226803 +10817 226749 +21513 226745 +48716 226692 +7218 226646 +5019 226635 +38526 226632 +15436 226625 +10175 226595 +8529 226551 +22167 226539 +1520 226496 +8173 226474 +8959 226444 +29411 226444 +10195 226443 +1465 226389 +1025 226359 +2265 226346 +2243 226311 +10592 226276 +12912 226248 +13819 226212 +10972 226199 +13043 226171 +21088 226141 +11620 226139 +22221 226136 +21233 226136 +2575 226078 +7729 226071 +11314 226060 +14845 225997 +5661 225991 +13273 225979 +2385 225975 +21512 225910 +5937 225907 +16330 225870 +4299 225870 +15662 225857 +22750 225820 +11478 225806 +22925 225769 +24033 225758 +13643 225757 +21097 225757 +1162 225739 +17058 225730 +17487 225677 +15077 225668 +6759 225659 +29293 225603 +17807 225584 +6181 225569 +8893 225560 +23547 225538 +17507 225495 +15928 225483 +5164 225472 +117 225463 +17195 225402 +15852 225393 +14431 225373 +19478 225368 +20695 225363 +15318 225341 +17494 225257 +12062 225251 +22908 225210 +15771 225198 +224 225168 +30958 225155 +12686 225055 +13638 225016 +19388 225000 +14703 224981 +16970 224976 +2454 224974 +17690 224952 +20409 224908 +20111 224898 +19765 224893 +11759 224893 +22081 224874 +9316 224841 +16517 224830 +6306 224819 +1540 224790 +13156 224756 +14212 224670 +14554 224670 +9978 224653 +23889 224643 +9701 224604 +10451 224582 +8534 224555 +14900 224517 +6045 224382 +8476 224300 +17691 224299 +4768 224263 +21995 224248 +8901 224228 +15313 224200 +15567 224200 +26040 224190 +36753 224177 +30289 224128 +13053 224119 +8045 224114 +12581 224098 +10532 224031 +14471 224019 +10422 224014 +14129 223973 +10619 223972 +14643 223911 +17217 223859 +18169 223852 +13355 223803 +6742 223772 +4262 223693 +2809 223686 +14430 223638 +10001 223524 +20516 223522 +12974 223505 +16566 223488 +7989 223485 +11445 223472 +13044 223433 +3856 223428 +10349 223409 +14855 223408 +17113 223375 +13388 223352 +14738 223342 +5126 223298 +6684 223218 +13675 223215 +12092 223199 +11380 223127 +14608 223060 +8011 223015 +13453 223013 +3898 222961 +20492 222958 +12300 222874 +16400 222852 +22281 222839 +12218 222818 +15608 222811 +35253 222785 +5074 222748 +13641 222731 +49280 222731 +20769 222727 +6955 222709 +7481 222695 +19502 222671 +1386 222670 +8695 222613 +9344 222608 +237 222484 +21408 222483 +5972 222470 +7196 222462 +17967 222444 +26702 222349 +20940 222340 +16403 222307 +15425 222295 +14256 222285 +17374 222212 +10580 222212 +21776 222204 +13013 222189 +24573 222178 +14109 222164 +3300 222160 +4017 222053 +12330 222052 +13679 222039 +11384 222000 +13477 221999 +8169 221984 +8068 221968 +23647 221922 +9944 221915 +906 221889 +5934 221882 +26603 221847 +10276 221841 +11382 221793 +12971 221790 +3021 221779 +10945 221752 +9221 221735 +23720 221725 +10770 221698 +14305 221674 +23990 221645 +3225 221614 +13720 221600 +13334 221578 +1151 221567 +16378 221516 +1072 221490 +48160 221489 +31033 221484 +1412 221480 +12560 221416 +13670 221410 +14096 221391 +16002 221375 +16732 221374 +3383 221339 +13514 221325 +3481 221313 +8800 221265 +17668 221262 +10690 221256 +12247 221236 +2135 221229 +17716 221203 +3080 221187 +21205 221165 +15895 221164 +9618 221164 +40196 221132 +20541 221101 +9976 221070 +5796 221049 +11436 221034 +7972 221000 +21679 220976 +5060 220952 +18653 220948 +13823 220936 +10745 220919 +18103 220907 +9521 220896 +11492 220856 +10959 220757 +16825 220751 +7080 220723 +17025 220694 +15519 220606 +11873 220606 +15698 220593 +16117 220569 +18921 220522 +2490 220511 +15449 220499 +32777 220459 +12856 220431 +14042 220401 +2535 220336 +999 220320 +8748 220319 +22966 220306 +16615 220206 +30303 220200 +21724 220193 +15679 220149 +35912 220141 +16568 220080 +13995 220065 +23075 220052 +16413 220030 +13351 220009 +5534 220005 +14044 219957 +18675 219945 +6881 219871 +18751 219864 +123 219758 +21791 219714 +15210 219709 +773 219683 +10547 219682 +5757 219678 +15018 219671 +10524 219668 +10933 219637 +18828 219624 +16013 219618 +13133 219536 +10426 219525 +16757 219524 +3903 219524 +15893 219477 +3488 219405 +13528 219373 +3014 219331 +24287 219327 +16798 219323 +19994 219277 +14958 219254 +6865 219234 +24434 219228 +42623 219200 +5231 219155 +9039 219130 +18727 219118 +16937 219109 +27943 219103 +22724 219087 +11085 219087 +23794 219086 +9607 219056 +19660 219029 +4411 218968 +14090 218928 +17337 218915 +22693 218906 +9190 218872 +12297 218858 +5263 218728 +8655 218726 +49258 218675 +3684 218645 +27620 218613 +24712 218557 +9914 218554 +11110 218532 +20893 218523 +17830 218518 +4389 218498 +23400 218425 +11583 218403 +2768 218401 +24537 218396 +7343 218395 +17370 218395 +21406 218351 +15032 218351 +10181 218300 +8231 218226 +12030 218226 +20097 218153 +13428 218146 +18532 218134 +8823 218128 +12846 218114 +9455 218081 +13209 218080 +16310 218054 +20464 218046 +11843 218044 +15203 218041 +10577 218034 +20955 217996 +8548 217983 +10849 217980 +12333 217976 +8630 217966 +18284 217888 +9740 217809 +1221 217776 +7899 217760 +10011 217740 +7626 217665 +13558 217631 +18914 217578 +15937 217549 +29611 217546 +15894 217527 +9030 217526 +18241 217517 +26315 217499 +11511 217481 +23696 217456 +11391 217437 +11067 217404 +1213 217394 +22861 217392 +12834 217369 +10283 217326 +40912 217323 +12716 217304 +14545 217260 +16381 217225 +15207 217189 +4703 217182 +13261 217111 +16050 217102 +17100 217085 +15171 217048 +12639 217029 +10960 217025 +22957 216931 +12950 216859 +18456 216850 +15757 216841 +12704 216839 +12702 216778 +17945 216772 +14342 216745 +23724 216736 +9028 216726 +18901 216711 +3543 216674 +21376 216672 +15734 216671 +3934 216663 +8722 216500 +27987 216482 +20138 216439 +14037 216414 +14491 216413 +2843 216352 +22421 216341 +13578 216333 +16862 216275 +12554 216271 +28537 216255 +5173 216251 +12549 216248 +21623 216176 +17456 216144 +16484 216142 +17659 216113 +17736 216061 +7079 216055 +21913 215938 +14764 215893 +11443 215865 +11776 215842 +31873 215831 +4738 215782 +18501 215701 +12694 215684 +19625 215672 +19003 215654 +6460 215618 +12127 215615 +16160 215607 +12094 215525 +10249 215518 +15045 215504 +10424 215428 +5605 215404 +12082 215401 +17092 215391 +13054 215323 +14288 215244 +9472 215228 +18711 215207 +16538 215140 +15074 215093 +19975 215087 +11816 215077 +28288 215071 +17884 215042 +3937 215038 +12797 214963 +22600 214939 +23475 214902 +5393 214860 +5979 214807 +10802 214804 +1801 214787 +967 214768 +10488 214663 +14263 214634 +10857 214609 +31157 214596 +8905 214546 +4714 214537 +17029 214532 +23368 214500 +15570 214476 +33496 214457 +17445 214346 +13206 214340 +24416 214331 +2487 214279 +18494 214232 +8577 214231 +22028 214210 +3617 214151 +12372 214126 +41805 214121 +13792 214118 +14869 214110 +12713 214089 +11219 214037 +9527 214012 +688 214000 +8627 213979 +3118 213865 +11022 213825 +18136 213766 +16749 213678 +2207 213676 +5167 213671 +12125 213643 +27227 213623 +6857 213550 +12864 213535 +1308 213529 +17397 213499 +23593 213362 +16035 213342 +11917 213329 +9124 213275 +13774 213224 +14183 213191 +22161 213177 +15427 213148 +15233 213144 +12055 213128 +971 213066 +3723 213027 +10640 213027 +13941 213025 +36128 213014 +11091 213006 +17034 212967 +23166 212933 +9370 212901 +10686 212857 +23751 212844 +31777 212803 +19138 212754 +24579 212728 +8858 212692 +5930 212671 +12476 212652 +6580 212651 +6972 212647 +13669 212629 +6383 212600 +18419 212599 +8284 212585 +23015 212571 +19582 212553 +8070 212508 +8855 212503 +9056 212502 +15727 212495 +5276 212469 +14833 212421 +9535 212418 +18152 212403 +9577 212394 +15240 212379 +15607 212375 +24323 212369 +11444 212362 +14973 212361 +17853 212358 +13937 212356 +17313 212347 +6808 212331 +24663 212323 +19033 212285 +11317 212254 +12296 212141 +20365 212058 +14284 212016 +15808 211962 +15955 211952 +16392 211948 +6297 211881 +1856 211881 +14186 211830 +2288 211802 +5837 211713 +2664 211706 +14604 211705 +14901 211645 +28190 211626 +22101 211577 +18330 211516 +19896 211500 +21523 211488 +11392 211463 +2618 211419 +14450 211415 +11904 211401 +11859 211384 +17444 211351 +9343 211266 +2770 211247 +20243 211246 +14659 211228 +9430 211228 +12077 211197 +14732 211181 +15704 211147 +21946 211139 +12269 211069 +11731 211062 +11672 211055 +17872 210994 +15668 210988 +14548 210950 +12969 210928 +1235 210923 +15215 210892 +10909 210879 +48798 210872 +3428 210853 +1523 210844 +12916 210814 +13027 210809 +11728 210779 +29073 210714 +13888 210709 +16846 210705 +9612 210673 +10162 210657 +16085 210643 +14320 210584 +13619 210579 +23194 210418 +19077 210354 +8159 210295 +20118 210278 +24821 210256 +16651 210150 +22637 210124 +2709 210072 +19163 210068 +15254 210067 +44600 210015 +9419 210007 +13988 209990 +15550 209888 +13204 209858 +22280 209856 +17422 209842 +11697 209839 +12936 209801 +11186 209777 +35621 209746 +10164 209721 +18918 209712 +633 209613 +20776 209595 +13737 209449 +31506 209446 +17424 209442 +8226 209393 +24868 209374 +12624 209351 +12511 209343 +2927 209279 +8080 209270 +14740 209255 +5838 209214 +5959 209211 +22548 209176 +13536 209058 +19174 209012 +24144 209012 +18845 208893 +26402 208846 +1156 208837 +12594 208796 +2533 208779 +14358 208763 +36485 208754 +11888 208739 +7232 208733 +23526 208652 +3610 208564 +20178 208484 +43998 208460 +10926 208352 +3458 208345 +13816 208330 +38742 208301 +14081 208278 +11399 208260 +20085 208192 +13961 208187 +12805 208183 +14614 208174 +3883 208152 +1885 208139 +3008 208126 +23639 208079 +17070 208016 +5283 207990 +9341 207938 +5476 207931 +9684 207911 +19661 207893 +11564 207862 +19105 207862 +22847 207837 +16007 207836 +11524 207827 +15144 207818 +21468 207755 +16914 207689 +18404 207670 +12390 207654 +17840 207497 +16008 207494 +45871 207470 +2586 207461 +14628 207455 +18265 207430 +18614 207428 +7081 207359 +8770 207356 +5297 207336 +18798 207306 +15525 207287 +19808 207251 +14506 207241 +20400 207080 +12788 207060 +14741 207043 +41153 207022 +32922 207010 +8160 206987 +1529 206971 +9379 206951 +13721 206944 +10347 206925 +3247 206897 +13387 206828 +19043 206818 +1815 206782 +23894 206771 +16053 206753 +3852 206737 +18075 206716 +9941 206709 +24576 206705 +23174 206693 +12471 206688 +2153 206680 +3304 206669 +18476 206649 +43318 206638 +14293 206636 +4494 206621 +12319 206566 +29533 206556 +12120 206522 +7931 206512 +19284 206500 +1851 206486 +5594 206411 +4990 206402 +3618 206396 +3201 206381 +14401 206362 +19902 206336 +14321 206300 +17322 206286 +11941 206270 +20110 206225 +18222 206211 +17780 206209 +17536 206137 +1118 206127 +11496 206117 +23430 206105 +10932 206102 +17390 206059 +13646 206043 +6653 206006 +16494 205987 +23358 205935 +10438 205877 +8979 205865 +15281 205825 +6302 205821 +11678 205815 +3353 205776 +20819 205742 +13224 205690 +14499 205675 +23554 205671 +28113 205647 +9495 205646 +5576 205637 +16048 205569 +12459 205499 +33597 205435 +7645 205422 +13790 205322 +15542 205315 +46245 205291 +18895 205274 +12244 205269 +20081 205229 +7284 205225 +25869 205181 +24392 205170 +10565 205124 +13611 205121 +6500 205103 +16261 205096 +14147 205064 +6649 205025 +6819 205001 +17272 204957 +8822 204956 +39726 204889 +4400 204876 +9306 204876 +9229 204814 +6747 204801 +12842 204800 +9704 204776 +35747 204774 +9830 204741 +15500 204721 +9930 204720 +14502 204642 +11315 204596 +19732 204574 +9312 204415 +21146 204408 +14496 204374 +11875 204360 +15859 204334 +30917 204299 +8763 204243 +17023 204218 +1779 204140 +13407 204123 +21462 204106 +14802 204105 +18352 204097 +14556 204094 +11378 204029 +17252 203976 +14660 203975 +30102 203960 +18730 203909 +15107 203853 +6091 203824 +6911 203759 +17432 203643 +14393 203587 +19530 203585 +9578 203567 +25890 203504 +13872 203480 +25415 203476 +20533 203450 +19911 203416 +13879 203386 +12579 203381 +3545 203356 +19210 203348 +20163 203348 +14942 203332 +21371 203316 +20091 203230 +10362 203162 +15516 203101 +24836 203043 +14877 203030 +18515 203027 +12985 203014 +11971 202922 +16490 202920 +14241 202830 +9113 202823 +17931 202801 +10862 202692 +21835 202678 +24175 202629 +8906 202566 +22799 202531 +24389 202517 +10620 202510 +23638 202485 +14946 202399 +29602 202361 +6021 202316 +15736 202310 +16078 202283 +1131 202258 +15869 202240 +13181 202236 +1311 202227 +3082 202222 +10920 202218 +24153 202190 +11173 202148 +21997 202148 +17304 202107 +18210 202094 +13803 202072 +7206 202055 +15953 202037 +13549 201990 +3558 201987 +22984 201969 +20953 201925 +13870 201922 +34630 201878 +11152 201875 +20821 201871 +18087 201855 +11815 201831 +17396 201827 +17112 201822 +26283 201750 +7565 201738 +8128 201738 +40055 201718 +22636 201689 +27883 201686 +28047 201679 +22643 201678 +15805 201639 +22207 201629 +15673 201585 +10717 201583 +16637 201556 +15089 201546 +14768 201433 +19304 201430 +25079 201428 +8940 201385 +15499 201358 +13055 201337 +23434 201293 +15035 201291 +22559 201281 +14232 201276 +5840 201267 +13467 201221 +6201 201203 +18319 201201 +18199 201115 +18078 201095 +36458 201045 +13179 201043 +34969 201035 +12555 201016 +24357 201015 +22489 200993 +19635 200941 +12294 200933 +22276 200926 +13205 200911 +31684 200896 +16055 200851 +10664 200848 +1673 200815 +5098 200813 +18322 200775 +5119 200771 +16636 200749 +29394 200726 +13627 200720 +10239 200697 +10716 200671 +8937 200664 +13215 200643 +26914 200631 +18617 200624 +4685 200606 +12822 200605 +5636 200549 +40319 200524 +16783 200520 +21335 200482 +21022 200464 +11299 200458 +27801 200441 +8254 200419 +7676 200363 +8178 200318 +26015 200274 +223 200265 +24527 200249 +2752 200218 +21819 200209 +7958 200165 +15843 200150 +11105 200111 +23524 200102 +10678 200093 +10866 200087 +21284 200061 +13471 200057 +6055 199960 +16736 199936 +18721 199855 +16518 199836 +11635 199796 +1915 199773 +13650 199720 +3828 199718 +13810 199695 +15193 199677 +9916 199675 +10081 199628 +12526 199625 +12647 199611 +25807 199610 +9664 199591 +15314 199455 +16842 199444 +11175 199437 +15531 199413 +34309 199402 +16508 199393 +11104 199386 +7730 199379 +18641 199320 +4173 199281 +8696 199241 +27928 199240 +6603 199220 +10232 199206 +28057 199161 +22553 199146 +26851 199116 +11424 199111 +18757 199088 +5289 199070 +25254 199069 +25125 199065 +14343 198992 +10365 198991 +17936 198991 +27469 198988 +17366 198972 +15154 198972 +4382 198876 +12198 198871 +16759 198834 +14337 198784 +7109 198771 +19740 198641 +17581 198607 +24379 198563 +3816 198537 +9693 198535 +15241 198532 +23817 198507 +18804 198432 +24255 198411 +6107 198349 +17362 198316 +10809 198310 +21116 198310 +5039 198291 +2308 198291 +26088 198236 +18260 198222 +14458 198213 +25389 198156 +21008 198155 +11470 198084 +21859 198070 +11283 198029 +16230 198019 +47809 197994 +13997 197990 +8267 197972 +27011 197967 +12197 197954 +12362 197946 +24131 197924 +10940 197885 +37628 197882 +8811 197822 +14509 197774 +15373 197762 +15515 197717 +16865 197711 +13815 197675 +7813 197661 +12381 197657 +3401 197647 +12273 197626 +21516 197617 +14384 197598 +103 197579 +19135 197563 +18409 197561 +39760 197544 +11634 197538 +20019 197411 +12887 197393 +5956 197370 +24668 197359 +15251 197327 +9342 197322 +27003 197303 +21050 197272 +25759 197268 +25640 197266 +9510 197262 +18204 197255 +9098 197246 +16868 197227 +21507 197214 +15882 197137 +14324 197103 +23827 197101 +9823 197072 +31340 197067 +7326 197066 +10385 196985 +18739 196964 +809 196944 +22319 196940 +19920 196910 +19688 196860 +10478 196853 +11571 196838 +12214 196815 +34058 196793 +16655 196733 +17296 196724 +14611 196714 +14412 196691 +11667 196663 +19413 196620 +8875 196593 +13165 196540 +16734 196518 +24104 196511 +8364 196502 +2326 196500 +5985 196441 +12706 196438 +8208 196390 +5329 196290 +681 196210 +31389 196208 +17135 196207 +25186 196204 +10622 196197 +19843 196177 +9932 196088 +2875 196086 +3798 196083 +9032 196057 +21782 196019 +16083 196002 +8966 195986 +18134 195959 +740 195949 +16118 195920 +17455 195892 +17958 195875 +10326 195842 +40110 195837 +3123 195811 +13035 195796 +12158 195765 +27526 195763 +16551 195691 +24590 195680 +13238 195546 +9340 195533 +19376 195525 +17270 195514 +9048 195472 +13062 195464 +17098 195441 +4968 195439 +13767 195439 +23307 195310 +15221 195299 +5121 195294 +17496 195292 +9513 195226 +9858 195219 +1742 195203 +13910 195163 +10317 195147 +15441 195142 +27606 195130 +28797 195114 +14075 195031 +15819 194979 +17161 194971 +32190 194967 +48063 194956 +2228 194853 +11148 194826 +20569 194815 +10528 194814 +18544 194813 +16384 194776 +11262 194776 +36553 194753 +12838 194747 +12167 194707 +19089 194655 +9442 194643 +15804 194612 +7408 194589 +9714 194544 +13502 194485 +30094 194423 +20758 194418 +7130 194411 +14803 194409 +30683 194325 +3495 194295 +9723 194235 +10375 194199 +19198 194180 +34016 194170 +42897 194156 +16503 194153 +15135 194135 +19964 194121 +24304 194073 +5573 194073 +14307 194006 +12280 194000 +7742 193988 +7617 193977 +10971 193959 +16546 193948 +17628 193922 +22195 193916 +18062 193910 +22516 193894 +34761 193893 +16556 193874 +13212 193838 +13728 193837 +16054 193822 +20494 193814 +8397 193791 +2 193789 +14246 193784 +15872 193776 +11137 193773 +20117 193750 +9096 193748 +12660 193694 +14441 193657 +11607 193643 +26436 193642 +21750 193638 +12547 193620 +22540 193620 +9445 193578 +23941 193545 +30382 193517 +10949 193503 +19665 193474 +12239 193456 +22687 193447 +13975 193440 +22013 193425 +17812 193409 +19639 193409 +10257 193402 +27083 193366 +4378 193331 +25375 193276 +3702 193272 +25014 193255 +1809 193231 +4864 193160 +6561 193114 +15253 193107 +23888 193103 +12564 193103 +12407 193070 +11111 193052 +12847 193023 +97 193021 +8932 193008 +19223 193008 +13324 193005 +9479 192988 +29536 192979 +33888 192911 +11913 192848 +8450 192848 +18006 192801 +4128 192796 +9404 192771 +23502 192733 +9476 192630 +22233 192603 +19984 192576 +4867 192557 +16033 192554 +19579 192541 +22418 192522 +7679 192514 +14161 192475 +12446 192437 +17652 192394 +15039 192387 +13114 192384 +23995 192370 +15283 192312 +16287 192264 +18201 192232 +22099 192198 +22596 192180 +13427 192167 +21299 192163 +5819 192093 +6058 192084 +23445 192073 +23605 192046 +16180 191892 +25715 191891 +19889 191824 +17492 191797 +1458 191777 +12922 191773 +14693 191767 +17375 191734 +13446 191698 +14801 191695 +11202 191687 +16514 191672 +15487 191656 +13791 191645 +15242 191615 +14245 191581 +3801 191536 +12226 191534 +24238 191508 +20103 191476 +2371 191454 +13151 191441 +22578 191429 +23249 191414 +8965 191396 +17578 191380 +12384 191374 +13591 191368 +13395 191318 +5805 191296 +18010 191271 +23078 191243 +7013 191207 +11923 191207 +12934 191173 +11068 191129 +20282 191124 +27290 191101 +20534 191056 +10112 191047 +18024 191030 +6220 191015 +14422 190963 +20154 190925 +24743 190912 +27905 190886 +18153 190854 +12888 190844 +12653 190841 +14765 190815 +13980 190798 +14004 190783 +18652 190783 +8876 190748 +13666 190685 +14052 190683 +16172 190678 +15699 190672 +7536 190620 +7554 190609 +12755 190599 +15426 190597 +4005 190592 +18976 190577 +13454 190558 +20501 190550 +14890 190507 +26355 190499 +14895 190386 +5012 190382 +10143 190362 +23382 190336 +13135 190315 +14143 190311 +8408 190282 +20200 190266 +15010 190236 +18732 190207 +16847 190178 +16109 190116 +24253 190092 +15264 190053 +17392 190030 +30774 190011 +13565 189985 +16499 189968 +20993 189957 +1604 189929 +9779 189922 +10795 189864 +35942 189857 +9386 189853 +25040 189836 +9174 189786 +19690 189778 +12245 189734 +20071 189723 +19574 189707 +18389 189707 +11779 189627 +26691 189621 +18198 189589 +12666 189586 +12509 189521 +10116 189504 +43967 189435 +11795 189420 +13275 189387 +15986 189374 +7057 189357 +11616 189356 +14528 189342 +14852 189333 +7037 189303 +20792 189288 +6660 189274 +13061 189255 +18786 189228 +23344 189217 +21612 189165 +13952 189163 +18394 189160 +14446 189141 +11254 189124 +10733 189118 +20669 189093 +16493 189039 +1408 189035 +17998 188970 +16861 188962 +10688 188950 +9518 188895 +16826 188888 +23040 188881 +17695 188866 +5189 188861 +7707 188860 +19550 188833 +13931 188821 +16929 188812 +11673 188801 +7788 188737 +16431 188706 +17428 188625 +12680 188619 +2964 188531 +8251 188521 +12649 188513 +19126 188486 +13614 188464 +12498 188434 +240 188366 +23960 188355 +16027 188339 +14736 188328 +8557 188257 +14504 188242 +12317 188216 +18928 188187 +13647 188177 +730 188141 +8499 188040 +25752 188019 +25799 188009 +4475 187984 +19705 187920 +242 187919 +9321 187901 +13971 187893 +17949 187866 +26546 187806 +4337 187792 +1574 187732 +16070 187679 +13314 187655 +16290 187605 +6390 187575 +32991 187551 +11080 187546 +6945 187488 +23139 187471 +23788 187464 +9365 187434 +11040 187433 +22381 187427 +18313 187380 +26085 187341 +14982 187323 +8574 187276 +19958 187261 +11414 187206 +1166 187206 +17537 187191 +15381 187178 +13385 187171 +21736 187169 +6530 187142 +7084 187139 +5528 187138 +26856 187128 +10757 187127 +13401 187114 +9991 187065 +12293 187060 +16477 187032 +4484 186968 +8790 186965 +3083 186903 +18033 186884 +17308 186848 +21609 186846 +4527 186814 +17997 186785 +17243 186783 +15114 186781 +15797 186778 +13927 186730 +10373 186727 +10874 186712 +7805 186676 +12698 186634 +14313 186602 +20320 186582 +19271 186577 +7715 186575 +16090 186565 +6236 186562 +13596 186556 +10119 186540 +27792 186538 +14668 186502 +12264 186476 +27837 186457 +32584 186446 +15897 186417 +2814 186411 +1191 186396 +12472 186389 +9173 186320 +20194 186304 +7749 186228 +7239 186223 +12044 186214 +12298 186188 +30851 186186 +10213 186182 +9427 186147 +31547 186096 +26056 186096 +5154 186083 +6548 186073 +1500 186064 +13153 186036 +21740 186013 +12450 186009 +17535 186007 +13187 185997 +15665 185967 +14772 185946 +22445 185931 +39073 185927 +28247 185912 +10671 185884 +13373 185871 +10335 185854 +9969 185840 +14892 185836 +8576 185825 +5614 185778 +5886 185753 +17332 185753 +10566 185745 +20515 185743 +12393 185692 +16300 185686 +26107 185651 +25872 185641 +7582 185618 +11050 185594 +7085 185589 +13266 185586 +10785 185573 +22931 185543 +13052 185522 +16355 185505 +17321 185499 +18563 185409 +6954 185351 +16532 185333 +9034 185322 +8631 185317 +7857 185290 +3642 185273 +13497 185252 +9353 185245 +38926 185244 +21505 185243 +32903 185225 +7785 185218 +12925 185211 +23489 185206 +121 185160 +17914 185159 +20252 185137 +6756 185118 +14333 185097 +15660 185070 +12227 185042 +8210 185040 +14727 185014 +21012 185010 +16968 185009 +30664 184980 +44144 184965 +23088 184951 +7038 184948 +22770 184841 +15980 184824 +18791 184788 +20829 184782 +46011 184765 +13457 184748 +17264 184706 +8363 184673 +5579 184646 +33864 184641 +12134 184598 +28490 184582 +6090 184556 +18088 184536 +30693 184523 +2475 184522 +17901 184511 +12568 184491 +11012 184482 +18530 184461 +21029 184451 +2673 184416 +15453 184413 +23837 184409 +19121 184397 +14995 184396 +14247 184351 +26391 184333 +6648 184215 +17083 184126 +15398 184122 +24154 184099 +17466 184079 +23566 184073 +17057 184041 +1456 184024 +36107 184020 +11441 183972 +12892 183957 +29424 183938 +17728 183935 +17096 183913 +19358 183880 +19592 183868 +12135 183847 +15617 183769 +7947 183698 +22132 183693 +17921 183642 +14930 183631 +16373 183628 +14139 183577 +15339 183563 +20208 183553 +9434 183516 +17948 183511 +15282 183476 +22558 183422 +16245 183386 +20879 183384 +20760 183322 +15112 183317 +19037 183298 +2790 183225 +9334 183222 +12586 183218 +18655 183207 +19823 183187 +21361 183163 +17471 183140 +16633 183140 +35348 183095 +28422 183085 +15329 183035 +21681 183027 +23393 183018 +14365 183016 +26661 182986 +22103 182984 +17188 182968 +6582 182948 +6247 182823 +11128 182782 +14281 182774 +9683 182761 +16821 182757 +18687 182664 +11739 182625 +31996 182584 +11109 182582 +4605 182563 +9033 182516 +9091 182502 +6148 182479 +12365 182357 +16205 182328 +28102 182310 +10703 182278 +2206 182267 +19431 182240 +13020 182239 +19316 182231 +4933 182220 +16828 182198 +3500 182192 +23288 182185 +10491 182150 +15789 182145 +8505 182107 +4886 182055 +11613 181967 +19651 181962 +7266 181950 +12421 181938 +27463 181932 +5092 181911 +26159 181884 +12238 181842 +20754 181817 +13017 181767 +15321 181759 +31415 181722 +17777 181718 +10590 181707 +3557 181705 +22182 181685 +2283 181681 +26182 181679 +15382 181678 +20915 181642 +10281 181615 +11286 181602 +25450 181598 +13081 181592 +24692 181592 +15335 181586 +10245 181585 +35240 181555 +12419 181544 +7379 181476 +12625 181426 +107 181418 +46568 181398 +29911 181382 +11996 181319 +18842 181296 +13197 181295 +26683 181289 +10983 181248 +20143 181240 +9251 181201 +58 181192 +25013 181189 +24269 181184 +8016 181180 +15138 181178 +5246 181147 +10753 181123 +25279 181114 +49188 181107 +22235 181106 +10416 181102 +15962 181083 +858 181077 +10603 181041 +28345 181027 +17517 181012 +13685 181010 +13796 180966 +5962 180927 +10560 180906 +12720 180895 +23443 180844 +14179 180758 +5936 180697 +14774 180686 +13396 180683 +13963 180683 +11806 180666 +21990 180646 +18195 180618 +9111 180605 +3032 180476 +6931 180457 +19029 180423 +26291 180419 +4674 180417 +30096 180414 +12737 180368 +10437 180314 +22937 180290 +21569 180184 +39173 180170 +6477 180162 +8340 180155 +12186 180123 +19784 180123 +12172 180100 +8067 180079 +27686 180044 +22112 180031 +20351 180027 +12386 180017 +24533 180009 +18983 179914 +26013 179907 +2169 179890 +14505 179883 +12057 179881 +19196 179873 +12703 179836 +15692 179684 +10787 179678 +20788 179665 +22382 179642 +14800 179640 +20079 179626 +25496 179602 +23957 179584 +17262 179561 +13076 179553 +7963 179552 +14792 179535 +10431 179516 +19558 179512 +19624 179503 +14444 179480 +1379 179445 +3313 179441 +9385 179437 +9546 179322 +17869 179308 +7308 179274 +21274 179269 +5775 179268 +17675 179252 +7689 179189 +13343 179180 +24797 179178 +29040 179165 +2734 179158 +15638 179153 +13203 179151 +23943 179129 +13718 179096 +24078 179090 +8692 179077 +12051 179067 +42326 179058 +14157 179052 +20884 179044 +16223 179043 +23976 178992 +12200 178959 +16748 178947 +14083 178946 +19055 178945 +24459 178906 +16988 178836 +16219 178833 +48984 178785 +7500 178772 +14297 178751 +9993 178748 +8643 178746 +3010 178732 +22586 178718 +36427 178703 +20158 178700 +18580 178676 +5574 178583 +16110 178569 +14013 178558 +30994 178554 +12157 178553 +28078 178528 +30739 178473 +17349 178470 +16427 178409 +29928 178401 +16522 178396 +16263 178391 +13734 178348 +15914 178347 +12114 178274 +8913 178254 +17774 178239 +17612 178210 +29560 178195 +14283 178152 +20169 178090 +42357 178076 +43461 178071 +22681 178031 +10398 177993 +24004 177926 +7759 177925 +15232 177918 +7440 177896 +18625 177828 +14201 177827 +12714 177772 +13246 177752 +12501 177747 +26809 177746 +1527 177730 +14720 177703 +15271 177600 +15820 177574 +17291 177547 +953 177535 +7980 177516 +16044 177494 +27639 177455 +18012 177379 +7300 177367 +16389 177354 +5783 177341 +15775 177334 +10442 177311 +21248 177251 +15945 177248 +11400 177211 +22490 177209 +16212 177168 +20432 177124 +16625 177111 +12122 177089 +12948 177075 +20312 177066 +26767 177058 +11762 177052 +6679 177034 +36061 177015 +27941 177013 +10388 177008 +8634 177003 +26685 176994 +2259 176811 +21504 176802 +21941 176770 +19051 176713 +15052 176707 +18365 176701 +22311 176694 +23629 176647 +7386 176630 +14645 176620 +19671 176614 +7442 176566 +908 176554 +963 176538 +6708 176535 +13092 176534 +21041 176525 +16584 176498 +2798 176462 +11578 176420 +11303 176408 +7273 176391 +22893 176380 +4614 176339 +6765 176307 +14590 176286 +16176 176282 +12260 176279 +13568 176271 +17986 176270 +13533 176231 +28040 176226 +3978 176223 +9627 176157 +20210 176137 +18996 176125 +21891 176057 +15678 176047 +12532 175994 +15925 175983 +12449 175949 +19925 175941 +16933 175937 +9786 175935 +16699 175919 +9390 175908 +21809 175905 +37108 175883 +12800 175860 +22962 175858 +14841 175811 +13378 175806 +14594 175747 +20353 175712 +11841 175708 +3196 175676 +8260 175670 +9424 175656 +20685 175654 +2473 175652 +12383 175641 +18141 175636 +12070 175606 +17214 175605 +23668 175599 +12878 175578 +6996 175575 +14434 175564 +5117 175538 +19137 175531 +13836 175530 +27748 175492 +24756 175452 +2876 175441 +12006 175423 +18633 175421 +11625 175394 +28111 175300 +17131 175299 +26101 175296 +4757 175293 +16190 175283 +4660 175281 +24499 175281 +13957 175252 +23592 175226 +14532 175208 +6422 175206 +9795 175192 +6320 175173 +21650 175161 +23676 175135 +7764 175042 +9290 174995 +14112 174966 +10904 174950 +10928 174887 +14735 174859 +5728 174841 +14543 174810 +22175 174807 +14530 174738 +1741 174726 +12113 174647 +27616 174644 +13028 174622 +18054 174599 +638 174583 +13182 174582 +15359 174563 +17431 174553 +43850 174526 +16319 174525 +24252 174506 +21198 174471 +20023 174399 +7063 174367 +19562 174366 +22779 174346 +4307 174344 +1769 174327 +14239 174325 +5375 174305 +24454 174256 +8023 174253 +3541 174227 +9088 174178 +12301 174161 +14655 174111 +12462 174104 +25157 174068 +27546 174066 +25995 174064 +19087 174060 +23955 174041 +22102 174021 +13703 174011 +12175 174004 +21204 173982 +17935 173962 +20412 173922 +12083 173901 +14953 173892 +13439 173877 +30175 173873 +11796 173872 +21884 173811 +27911 173805 +29311 173801 +951 173797 +7923 173778 +18526 173761 +12733 173710 +28891 173706 +13121 173684 +1548 173674 +26293 173655 +18632 173633 +29141 173580 +6385 173558 +3694 173557 +15518 173539 +12213 173531 +25225 173478 +10782 173464 +20550 173462 +7227 173435 +817 173425 +8343 173419 +16730 173409 +25147 173394 +33339 173376 +27541 173306 +26203 173301 +31167 173285 +11901 173265 +11623 173194 +14376 173160 +27042 173159 +16079 173133 +7858 173078 +12952 172985 +17110 172967 +9828 172956 +16611 172909 +14788 172908 +21455 172893 +9666 172882 +10725 172849 +14087 172813 +11061 172779 +13139 172764 +12701 172753 +13400 172750 +10645 172749 +26456 172713 +22653 172706 +12824 172703 +7883 172684 +15637 172673 +8508 172668 +3671 172656 +16197 172651 +6359 172639 +25434 172563 +5191 172562 +9915 172552 +1761 172546 +20990 172531 +18520 172530 +7305 172488 +8558 172486 +15345 172477 +19378 172448 +23681 172412 +23936 172403 +14026 172399 +26373 172356 +10194 172322 +11459 172305 +12085 172285 +16336 172271 +13405 172236 +12361 172235 +6061 172209 +3757 172202 +8550 172200 +17222 172176 +6735 172162 +7235 172062 +23975 172027 +16473 172025 +11987 172020 +22098 171992 +17461 171991 +12656 171991 +6763 171989 +46179 171979 +12347 171925 +12470 171923 +8891 171915 +21071 171901 +18870 171890 +1655 171870 +21166 171857 +6444 171853 +1840 171774 +8869 171773 +12784 171753 +28472 171729 +21571 171719 +24763 171628 +12320 171619 +21840 171608 +1927 171529 +18382 171521 +5753 171500 +17290 171498 +44892 171491 +9801 171488 +12584 171458 +101 171438 +7514 171434 +46990 171428 +29913 171428 +20771 171416 +2601 171410 +17460 171395 +21220 171384 +16895 171380 +9127 171376 +15619 171357 +19094 171350 +20455 171331 +33323 171302 +28975 171299 +4774 171292 +20319 171273 +15844 171257 +16686 171115 +27025 171095 +30646 171073 +27607 171054 +9335 171010 +18367 171000 +22569 170994 +33564 170982 +22634 170958 +22300 170944 +18729 170910 +33653 170901 +16657 170899 +7670 170892 +14480 170888 +21862 170874 +17095 170838 +22383 170834 +8991 170821 +22394 170769 +13676 170754 +35659 170660 +38858 170592 +11638 170560 +1240 170543 +26353 170531 +11609 170454 +14690 170433 +12004 170432 +9269 170423 +16146 170371 +27599 170360 +14928 170326 +1930 170295 +7454 170289 +23328 170274 +31546 170245 +5247 170202 +17803 170200 +19258 170198 +5980 170197 +24980 170151 +742 170143 +19057 170127 +4910 170077 +22953 170045 +25585 170029 +21927 170019 +29090 170017 +31159 169975 +7128 169965 +8065 169957 +8833 169947 +27582 169932 +13386 169901 +31346 169899 +11766 169880 +16047 169880 +5978 169862 +16608 169852 +10031 169833 +31078 169816 +32636 169802 +10370 169746 +10675 169728 +47979 169720 +33231 169712 +24684 169670 +16621 169667 +19764 169656 +19988 169648 +29747 169644 +44009 169639 +7074 169638 +9800 169628 +1585 169536 +24073 169523 +21087 169487 +22720 169463 +22434 169431 +25074 169400 +12659 169366 +11733 169362 +35093 169320 +15873 169309 +16602 169292 +14485 169259 +17449 169228 +14234 169204 +30245 169196 +18064 169190 +18735 169178 +12569 169172 +12493 169135 +14648 169118 +25935 169114 +26116 169108 +675 169099 +8306 169097 +2956 169065 +12229 169035 +23076 169019 +11387 168999 +18130 168991 +16930 168979 +12790 168977 +15767 168976 +21110 168952 +31703 168924 +12779 168902 +24485 168902 +29674 168898 +17120 168891 +16359 168887 +19169 168876 +1076 168872 +16894 168838 +10322 168830 +20552 168816 +16324 168804 +26078 168785 +34786 168725 +2006 168691 +11542 168666 +16932 168631 +11968 168561 +18647 168543 +11556 168522 +30384 168493 +899 168471 +13374 168460 +22225 168459 +15881 168449 +11341 168440 +6644 168425 +10712 168414 +21266 168360 +17876 168314 +8610 168313 +22541 168302 +12121 168276 +105 168274 +18053 168229 +11031 168215 +17441 168157 +11640 168079 +1788 168065 +12697 168063 +18079 168041 +14789 168034 +23577 168015 +25686 168011 +14325 168010 +14826 167973 +8808 167968 +7385 167959 +15656 167943 +26406 167933 +3477 167926 +25167 167922 +20441 167896 +9254 167895 +14102 167884 +13580 167830 +22041 167798 +27611 167791 +19711 167781 +29890 167771 +34943 167757 +32149 167722 +24207 167666 +14551 167664 +1929 167661 +5851 167643 +9792 167642 +13207 167637 +17711 167633 +23438 167558 +4095 167545 +18656 167543 +10761 167496 +9310 167481 +31509 167455 +13084 167453 +12823 167449 +12484 167418 +29272 167411 +23780 167399 +19725 167368 +20271 167359 +20386 167350 +8071 167327 +23982 167266 +26451 167221 +16675 167217 +23415 167158 +2508 167108 +17938 167087 +18519 167077 +13717 167065 +9053 167045 +11575 167045 +6723 167044 +5830 167038 +11451 167009 +20237 166982 +13656 166930 +33118 166913 +2929 166913 +34589 166887 +12886 166877 +21011 166858 +20886 166856 +11053 166854 +5225 166852 +13747 166832 +9351 166806 +14676 166779 +16171 166777 +31526 166766 +24031 166749 +26606 166743 +13889 166741 +18679 166738 +11757 166711 +30035 166709 +2114 166669 +22695 166667 +11054 166649 +11442 166649 +15413 166635 +9579 166605 +13239 166595 +13752 166594 +1567 166549 +13699 166525 +17962 166524 +13510 166490 +18387 166475 +1934 166420 +26400 166403 +36759 166372 +15172 166364 +21666 166333 +25733 166308 +24799 166303 +14386 166271 +11463 166237 +8290 166233 +25914 166221 +4207 166202 +11562 166199 +7310 166186 +12576 166158 +12236 166143 +17372 166140 +23270 166122 +10605 166100 +11349 166100 +9726 166095 +24164 166090 +6950 166083 +13267 166071 +19636 166055 +12422 166053 +10217 166036 +14340 166030 +15163 166028 +14048 166027 +14154 165992 +22309 165928 +14266 165928 +34217 165921 +11052 165919 +33399 165910 +3832 165889 +16237 165889 +7287 165873 +22351 165865 +15291 165860 +1928 165844 +17673 165838 +6615 165779 +12844 165748 +47875 165744 +713 165695 +28392 165691 +14678 165625 +16545 165608 +14492 165582 +21402 165530 +22361 165529 +19219 165467 +4596 165457 +12923 165451 +5702 165438 +11820 165426 +1089 165423 +13516 165419 +26190 165371 +25701 165371 +13827 165370 +4328 165308 +34366 165287 +19140 165262 +10320 165222 +11863 165214 +12033 165196 +28523 165172 +31791 165153 +15489 165126 +17274 165125 +15477 165115 +23829 165086 +19290 165064 +21531 165051 +13070 165041 +7710 165021 +11516 165009 +21006 165000 +23832 164975 +9948 164892 +24850 164885 +11101 164882 +27117 164844 +32564 164833 +22771 164818 +8142 164818 +34506 164763 +16192 164663 +13452 164658 +10368 164642 +4342 164624 +23325 164571 +14153 164564 +32136 164562 +19500 164555 +1777 164538 +26216 164529 +21518 164502 +17863 164502 +11662 164492 +14851 164477 +28059 164405 +27035 164382 +12144 164302 +12903 164283 +17564 164257 +13006 164252 +13753 164245 +1768 164242 +9848 164233 +23247 164218 +16788 164197 +12979 164190 +41179 164177 +2217 164136 +9255 164136 +41588 164100 +23413 164093 +14955 164090 +11664 164075 +23429 164074 +2889 164070 +20002 164057 +4630 164034 +13318 164034 +9058 164026 +11543 163998 +13608 163983 +23643 163952 +18067 163942 +4501 163926 +22158 163921 +20404 163910 +21418 163874 +17344 163864 +12444 163801 +901 163789 +29037 163767 +16500 163753 +14093 163735 +29306 163647 +7837 163646 +22223 163631 +13122 163620 +14776 163601 +5589 163591 +9989 163571 +23624 163502 +8274 163471 +16325 163463 +30031 163460 +9330 163450 +839 163430 +24397 163382 +8605 163360 +5988 163329 +11860 163296 +16765 163287 +20527 163284 +3316 163277 +915 163267 +18434 163258 +10165 163223 +14185 163180 +15464 163159 +27274 163156 +20165 163120 +11011 163101 +20390 163090 +15276 163078 +21133 163077 +32694 163019 +19745 163009 +18246 163004 +2181 162993 +8074 162946 +20304 162946 +42057 162941 +11089 162921 +17109 162920 +13799 162841 +18021 162828 +21817 162821 +21270 162821 +11870 162809 +7056 162800 +21167 162709 +19657 162699 +17887 162692 +21023 162683 +12793 162679 +22958 162650 +18862 162610 +18753 162565 +16585 162550 +30843 162549 +19432 162535 +22680 162525 +6895 162511 +18488 162509 +8435 162506 +26301 162478 +46129 162457 +17346 162449 +11094 162410 +23068 162393 +23617 162378 +24232 162378 +17742 162375 +20291 162364 +22260 162360 +22639 162344 +30872 162339 +8431 162339 +15371 162317 +15305 162307 +21896 162293 +26615 162268 +665 162268 +20714 162265 +17368 162256 +20093 162231 +21183 162191 +23200 162167 +19855 162160 +33922 162151 +1615 162141 +33724 162135 +4062 162117 +18570 162085 +7149 162079 +16853 162064 +20742 162037 +25050 162013 +7591 161991 +8369 161974 +28643 161972 +17454 161968 +15472 161941 +31005 161940 +18416 161919 +33911 161884 +10220 161861 +9942 161846 +14089 161843 +12524 161833 +18852 161808 +2218 161795 +21119 161783 +10351 161736 +27847 161736 +11624 161735 +14639 161728 +226 161620 +11736 161616 +20739 161604 +11329 161601 +20352 161563 +17054 161549 +4180 161537 +33385 161528 +25889 161523 +16320 161497 +33193 161491 +31767 161467 +11247 161429 +28759 161424 +14378 161422 +23590 161420 +20393 161410 +15654 161390 +12098 161372 +27986 161357 +18397 161348 +11649 161326 +23070 161293 +25634 161289 +19475 161286 +15916 161274 +13779 161164 +16299 161163 +3984 161154 +19517 161142 +16860 161140 +12546 161125 +8229 161124 +25793 161099 +15332 161092 +14104 161088 +27357 161081 +15099 161076 +16034 161064 +15722 161054 +18618 161004 +21158 160987 +9491 160981 +12096 160973 +18362 160959 +31230 160951 +22171 160939 +15749 160938 +23280 160928 +27180 160917 +19350 160903 +13917 160863 +28686 160840 +22387 160829 +14150 160787 +8950 160765 +7711 160757 +11433 160753 +20447 160753 +21743 160744 +17589 160726 +22830 160718 +15564 160701 +15935 160665 +17086 160655 +11025 160649 +1020 160616 +3915 160590 +21445 160576 +12860 160543 +20653 160539 +22627 160534 +28355 160530 +13727 160517 +13331 160513 +14657 160501 +15293 160493 +12345 160491 +25469 160481 +8859 160468 +21249 160462 +26538 160449 +25684 160447 +5691 160438 +14846 160434 +30427 160423 +5844 160391 +18323 160386 +14318 160373 +25535 160370 +20840 160352 +10526 160335 +16247 160329 +18666 160320 +15850 160286 +15326 160278 +23259 160245 +17260 160245 +24091 160196 +13769 160191 +27540 160149 +12898 160107 +12661 160094 +11657 160079 +5912 160074 +11738 160067 +2913 159992 +8204 159989 +26804 159960 +14595 159957 +14730 159947 +32760 159941 +32078 159907 +13460 159893 +21425 159882 +26618 159858 +32879 159838 +12592 159836 +24832 159811 +15536 159796 +16537 159787 +27828 159774 +18857 159772 +14483 159740 +25054 159712 +9586 159707 +20096 159707 +17079 159702 +14370 159698 +19207 159671 +26824 159630 +33139 159614 +25172 159602 +13686 159592 +40289 159583 +21838 159580 +18128 159571 +14733 159563 +31292 159553 +18139 159513 +44066 159510 +25165 159492 +14371 159474 +17255 159432 +5379 159426 +4119 159418 +15250 159397 +12041 159368 +39463 159338 +14862 159315 +22140 159290 +7769 159278 +8629 159228 +29714 159198 +28855 159198 +14484 159181 +16255 159158 +22945 159156 +9875 159043 +12188 159039 +12877 158992 +33605 158976 +41449 158974 +11212 158974 +32688 158957 +13455 158950 +17638 158947 +8531 158908 +21193 158855 +7220 158842 +3875 158831 +6017 158800 +9724 158796 +13808 158790 +21798 158768 +16569 158740 +8487 158725 +12891 158721 +16169 158668 +50079 158664 +14662 158663 +9565 158660 +26070 158632 +16715 158620 +25982 158612 +14921 158609 +24190 158591 +16304 158587 +16561 158553 +17052 158522 +17590 158510 +2012 158490 +40635 158487 +14193 158454 +16597 158446 +29200 158411 +25768 158388 +16599 158346 +22283 158342 +14190 158305 +17984 158291 +11368 158289 +894 158285 +8726 158253 +13271 158252 +38818 158242 +15909 158213 +15279 158211 +19987 158186 +14029 158156 +2967 158148 +7597 158145 +11282 158119 +30334 158112 +1819 158040 +7211 158026 +8804 158002 +24152 157998 +9863 157974 +15124 157954 +12189 157895 +31507 157885 +10381 157862 +20551 157853 +35293 157847 +8664 157839 +15828 157836 +17114 157813 +15585 157797 +32317 157785 +21141 157733 +29444 157718 +13570 157718 +11352 157674 +933 157613 +13222 157602 +14310 157588 +12929 157586 +18017 157577 +15647 157572 +28278 157566 +14540 157562 +9576 157531 +10599 157529 +11140 157502 +25936 157484 +39637 157468 +5684 157467 +20997 157452 +23115 157437 +18108 157436 +20705 157426 +21457 157387 +16511 157373 +28564 157373 +23420 157353 +13786 157351 +10284 157333 +11723 157333 +24005 157315 +10919 157293 +27874 157280 +16548 157274 +9995 157264 +13141 157253 +15907 157235 +12571 157218 +2247 157218 +28182 157209 +14121 157200 +1552 157195 +7400 157179 +15747 157150 +10188 157144 +13079 157126 +29443 157105 +22605 157085 +14638 157043 +22722 157041 +38061 157031 +19702 156983 +24713 156960 +29600 156910 +27622 156908 +23811 156882 +18612 156867 +29330 156804 +31071 156786 +18340 156752 +26962 156752 +16460 156724 +20126 156706 +5275 156655 +20342 156637 +18259 156614 +888 156611 +15816 156610 +30405 156596 +10742 156594 +26068 156592 +12572 156545 +23980 156530 +18468 156522 +6256 156514 +18504 156499 +4020 156492 +22323 156483 +10685 156482 +18752 156458 +17268 156414 +13788 156414 +5083 156407 +31472 156400 +2908 156375 +26807 156369 +24142 156367 +22571 156353 +11060 156348 +17093 156298 +20642 156289 +19551 156285 +3780 156274 +19495 156268 +800 156267 +20603 156263 +18147 156244 +35151 156227 +18385 156178 +18684 156168 +20601 156162 +8553 156147 +14028 156145 +46279 156133 +16456 156127 +7913 156073 +17066 156071 +12692 156046 +25978 156013 +14173 156010 +25299 156002 +23519 155994 +28335 155959 +20157 155953 +5677 155929 +3971 155902 +12267 155878 +3309 155863 +20933 155816 +24296 155768 +5821 155741 +18236 155729 +17893 155698 +17012 155685 +15902 155658 +20051 155639 +16395 155631 +25688 155605 +11711 155597 +22455 155571 +22095 155561 +18590 155536 +28052 155527 +43856 155456 +25109 155429 +19414 155419 +23546 155417 +25296 155396 +15021 155380 +8624 155360 +34338 155326 +48543 155324 +18264 155290 +2285 155264 +34139 155261 +17920 155238 +25201 155154 +12279 155145 +16075 155111 +14818 155107 +16318 155106 +16397 155078 +37244 155044 +23472 155043 +29220 155029 +11501 155027 +18446 154973 +3319 154963 +18180 154948 +15101 154935 +18105 154933 +16740 154932 +8960 154919 +17616 154899 +28280 154889 +13521 154875 +10583 154855 +7071 154845 +12326 154824 +16024 154804 +21292 154782 +1047 154750 +11188 154727 +18223 154723 +12768 154698 +13391 154683 +11948 154679 +28354 154643 +12105 154624 +24833 154599 +12179 154573 +15399 154535 +17089 154509 +14687 154465 +2374 154450 +10248 154438 +15723 154435 +15621 154410 +630 154398 +14809 154370 +17722 154361 +23424 154352 +14088 154336 +2429 154321 +14275 154316 +19026 154282 +7493 154278 +32948 154265 +3733 154263 +20270 154229 +15830 154223 +19602 154195 +6517 154179 +17989 154171 +46668 154162 +19068 154157 +10285 154148 +30223 154138 +17981 154123 +24545 154116 +8241 154084 +6404 154027 +1185 154019 +4227 154018 +14208 153999 +16811 153998 +18867 153997 +16755 153996 +36431 153984 +25613 153954 +19481 153934 +13435 153913 +34112 153902 +22676 153889 +11096 153830 +18920 153800 +40113 153775 +19447 153772 +32440 153762 +23632 153760 +27052 153749 +11945 153742 +26091 153725 +15718 153715 +31736 153713 +21821 153700 +20153 153693 +1553 153661 +24300 153645 +12395 153620 +11800 153610 +24859 153584 +31318 153521 +17254 153482 +7573 153476 +255 153473 +25230 153472 +24625 153458 +9878 153450 +10114 153435 +21400 153433 +14884 153380 +26918 153347 +12370 153341 +18432 153335 +16459 153330 +26023 153308 +8118 153298 +49355 153294 +25846 153268 +6005 153259 +34910 153259 +6392 153251 +44828 153232 +17530 153226 +20462 153208 +15867 153204 +10548 153174 +18342 153154 +24135 153137 +11335 153129 +15571 153117 +22970 153096 +16507 153049 +14440 153036 +2764 153035 +23533 153022 +8681 153019 +16195 152986 +4507 152962 +8613 152961 +14045 152955 +26326 152952 +4250 152929 +12958 152900 +29825 152864 +19830 152848 +23265 152843 +36863 152777 +31110 152776 +20670 152773 +14580 152733 +13707 152713 +28397 152701 +21820 152700 +14640 152687 +9380 152661 +10635 152624 +14814 152596 +16356 152522 +11246 152502 +40001 152483 +23013 152432 +25316 152418 +27724 152407 +25018 152406 +15628 152383 +35637 152371 +20424 152336 +18269 152305 +13172 152300 +2143 152274 +47633 152263 +3411 152256 +1190 152228 +21827 152218 +16642 152203 +22205 152196 +13148 152180 +4230 152159 +15551 152108 +8002 152108 +15226 152070 +11261 152055 +11705 152032 +20240 152015 +23918 151954 +31625 151951 +21028 151947 +43566 151927 +2341 151908 +19935 151900 +12863 151899 +32958 151881 +15691 151875 +16887 151864 +16963 151863 +14338 151859 +27194 151857 +26909 151855 +19999 151840 +6871 151824 +14593 151815 +26716 151771 +13899 151760 +20753 151700 +36095 151697 +15298 151683 +32496 151606 +2865 151590 +17809 151567 +29797 151566 +36744 151544 +22368 151535 +12271 151498 +19632 151465 +9356 151455 +20583 151415 +17061 151403 +17781 151390 +24968 151389 +41461 151377 +22877 151362 +4906 151313 +13667 151309 +27283 151288 +31764 151280 +18648 151258 +47286 151250 +31918 151179 +16277 151130 +19519 151110 +19931 151099 +21434 151094 +35385 151073 +21808 151068 +4127 151053 +30112 151050 +29402 151046 +17383 151039 +3279 151028 +41727 151001 +34549 150992 +7141 150988 +16001 150978 +12740 150977 +19013 150919 +2317 150903 +13616 150870 +19829 150866 +26544 150859 +23494 150829 +11793 150822 +5678 150789 +21346 150787 +6839 150781 +16612 150779 +2859 150761 +16133 150748 +25462 150743 +25175 150735 +29222 150706 +18266 150702 +15557 150701 +24413 150649 +17378 150591 +14714 150581 +1868 150571 +4189 150541 +2747 150526 +14470 150516 +26960 150510 +12209 150502 +1685 150501 +6880 150485 +18165 150479 +14383 150433 +4561 150430 +12804 150411 +48298 150408 +24766 150396 +22964 150364 +17837 150349 +28792 150347 +4160 150339 +3874 150327 +13113 150326 +9654 150323 +13858 150312 +13476 150291 +13000 150289 +19443 150285 +7856 150248 +20631 150223 +23663 150219 +36878 150211 +4356 150206 +12005 150181 +5061 150174 +17292 150167 +37929 150165 +1174 150149 +7100 150145 +12119 150121 +19934 150114 +25911 150114 +26985 150114 +18551 150112 +10979 150111 +6475 150106 +31063 150073 +23008 150064 +11198 150059 +19131 150055 +20156 150033 +14073 150025 +18140 150022 +14394 150016 +3438 150003 +12027 149972 +20774 149888 +21126 149879 +19097 149868 +10345 149816 +41012 149812 +14438 149801 +18553 149773 +16954 149767 +21423 149719 +32519 149709 +29074 149707 +8784 149698 +2963 149683 +13026 149678 +23052 149660 +28242 149649 +16677 149632 +13599 149629 +36692 149596 +18550 149593 +3351 149580 +3711 149539 +26454 149533 +18916 149510 +23531 149481 +17467 149475 +11207 149458 +11710 149431 +10720 149413 +3629 149411 +18315 149407 +28649 149298 +7490 149297 +43484 149267 +19315 149251 +20561 149234 +12681 149204 +31460 149200 +11707 149196 +19278 149166 +13951 149157 +25452 149155 +2898 149145 +16472 149134 +20276 149129 +16889 149126 +23786 149104 +14799 149093 +13326 149084 +37361 149083 +18499 149066 +29418 149057 +2668 149055 +23902 149051 +12839 149015 +29806 148976 +24380 148923 +28108 148871 +32939 148870 +14497 148862 +5623 148850 +17730 148845 +15635 148837 +44508 148835 +20077 148829 +11499 148813 +14271 148813 +22191 148798 +9737 148798 +16960 148777 +32682 148773 +31630 148764 +35231 148749 +21039 148721 +43841 148710 +31248 148693 +25313 148685 +25043 148683 +21112 148627 +20289 148622 +18811 148620 +16265 148608 +7655 148578 +9045 148574 +15712 148554 +26039 148540 +22264 148525 +13869 148513 +14031 148505 +8392 148502 +34783 148499 +24536 148490 +33110 148480 +21380 148469 +26686 148438 +9695 148399 +15765 148398 +10586 148396 +9834 148374 +14363 148368 +42189 148367 +8297 148330 +42235 148326 +24163 148323 +21511 148301 +13539 148287 +16259 148284 +22841 148280 +34655 148279 +13639 148270 +1873 148220 +236 148220 +28791 148173 +16981 148158 +31002 148157 +19666 148135 +17609 148080 +11527 148078 +16938 148071 +19649 148013 +961 147986 +7673 147977 +15143 147964 +14583 147917 +13645 147910 +12590 147909 +18724 147905 +27502 147901 +23942 147874 +11428 147867 +4740 147865 +27609 147860 +16425 147859 +20731 147848 +29640 147844 +3143 147820 +23852 147767 +10348 147753 +4777 147752 +11930 147739 +16904 147704 +2307 147703 +17498 147652 +12791 147635 +10107 147624 +29751 147606 +16406 147605 +12017 147592 +17625 147562 +9382 147562 +14351 147554 +22244 147533 +5725 147520 +2655 147490 +23655 147473 +21530 147408 +12657 147405 +4879 147395 +22075 147383 +21685 147355 +22786 147242 +21420 147225 +15794 147218 +3577 147175 +9536 147160 +35193 147093 +27348 147076 +17198 147049 +8615 147031 +24037 147017 +6532 147016 +25289 146998 +10897 146965 +156 146955 +39920 146925 +22647 146921 +10148 146909 +10891 146887 +14041 146872 +18143 146867 +15460 146865 +5518 146864 +2917 146864 +17123 146801 +23600 146765 +10582 146703 +23240 146699 +17502 146680 +26943 146669 +7488 146654 +14282 146636 +21161 146625 +22879 146622 +14176 146611 +28250 146596 +18654 146583 +20715 146561 +11056 146533 +16645 146525 +40160 146513 +608 146508 +13770 146488 +16830 146483 +21987 146426 +34162 146377 +14136 146323 +20505 146305 +12741 146284 +13530 146275 +27250 146260 +32489 146242 +9929 146232 +34183 146216 +27775 146212 +21644 146198 +11530 146177 +6309 146169 +19496 146142 +20047 146103 +16521 146087 +22926 146036 +3228 146019 +15108 145976 +27879 145951 +15921 145950 +18818 145948 +11379 145936 +28259 145928 +6051 145927 +27886 145925 +21919 145900 +27771 145884 +19002 145869 +28565 145863 +10229 145861 +19480 145857 +7120 145846 +1901 145839 +16470 145810 +10456 145793 +25464 145789 +11895 145789 +19597 145785 +8769 145767 +24787 145747 +35764 145738 +34116 145734 +36040 145708 +13884 145698 +29285 145693 +29484 145690 +11916 145681 +20244 145661 +26899 145652 +8739 145638 +16231 145626 +11767 145621 +27334 145619 +39823 145614 +24193 145602 +26249 145596 +31774 145592 +12889 145583 +15933 145566 +26739 145565 +48974 145558 +5146 145504 +19383 145482 +22772 145462 +18379 145458 +18695 145447 +20874 145411 +10593 145401 +37720 145373 +19783 145319 +15085 145314 +9297 145310 +17316 145293 +18522 145290 +18785 145276 +63 145276 +6229 145255 +21733 145255 +12479 145254 +23186 145227 +14489 145197 +10752 145179 +39375 145173 +31636 145170 +35597 145170 +39632 145167 +18295 145138 +41924 145119 +14448 145118 +28458 145088 +20671 145086 +10602 145079 +18595 145063 +15190 145062 +19346 145051 +11179 145035 +16216 145026 +15941 144995 +11741 144989 +9261 144984 +18094 144969 +25479 144862 +26136 144855 +27229 144848 +7761 144848 +2973 144807 +17160 144798 +24248 144764 +7200 144739 +22375 144732 +23356 144732 +14361 144729 +21057 144721 +18508 144691 +4817 144670 +9598 144624 +12709 144620 +11108 144598 +6423 144592 +14097 144587 +15822 144585 +14653 144550 +20813 144539 +21541 144521 +26242 144514 +37440 144509 +4114 144509 +13375 144474 +18575 144446 +3222 144446 +10783 144443 +24105 144434 +18283 144432 +10197 144427 +14349 144424 +38813 144413 +39249 144396 +6724 144389 +490 144388 +32452 144361 +7068 144353 +19171 144331 +26154 144327 +22336 144323 +9866 144305 +22108 144292 +6468 144287 +15825 144282 +23702 144255 +6988 144243 +10730 144241 +28414 144241 +17276 144230 +14983 144229 +14864 144214 +26348 144200 +1843 144196 +25357 144180 +14180 144170 +18662 144158 +24067 144156 +14699 144154 +10924 144126 +20162 144107 +22038 144094 +14724 144049 +27633 144028 +8310 143966 +19050 143962 +17411 143945 +2299 143915 +1617 143906 +26372 143898 +17416 143870 +37072 143867 +879 143850 +21624 143840 +6790 143798 +18750 143787 +20672 143775 +3157 143711 +2386 143672 +13143 143671 +21641 143653 +4559 143641 +26134 143618 +15807 143611 +40990 143519 +17206 143512 +12481 143506 +8510 143503 +12495 143490 +10227 143482 +20576 143478 +28379 143414 +15904 143411 +14871 143409 +12153 143373 +24194 143336 +21272 143326 +11700 143323 +19000 143288 +17004 143282 +12683 143277 +6499 143264 +20250 143258 +22543 143237 +17835 143236 +21337 143227 +36888 143176 +8524 143165 +20600 143162 +13186 143159 +5420 143155 +38488 143155 +35295 143153 +26935 143119 +29737 143113 +12290 143106 +6072 143102 +35399 143098 +15050 143097 +19544 143080 +21813 143026 +22371 143025 +13024 143020 +13529 143006 +12719 143001 +8756 142996 +26390 142984 +39099 142967 +13063 142945 +21024 142943 +15455 142914 +6574 142910 +8299 142910 +35136 142909 +22736 142908 +25647 142902 +16352 142896 +18573 142873 +2593 142862 +23312 142838 +36861 142830 +14169 142814 +7409 142806 +9756 142803 +28238 142793 +15949 142780 +24779 142717 +11894 142698 +37911 142687 +5450 142670 +8322 142664 +15133 142663 +22004 142624 +14252 142623 +16291 142622 +19996 142485 +21542 142483 +12237 142454 +22814 142400 +14323 142388 +6048 142372 +24946 142362 +16567 142345 +26046 142335 +12817 142314 +20805 142310 +7251 142296 +33338 142294 +24086 142293 +20461 142280 +2322 142269 +43812 142263 +12312 142254 +9582 142243 +38134 142234 +26907 142226 +16218 142210 +38671 142196 +13002 142196 +4826 142194 +14947 142188 +30836 142187 +10288 142181 +15175 142177 +15743 142174 +8228 142163 +17162 142134 +13005 142118 +31721 142116 +36139 142113 +10128 142107 +9688 142074 +43941 142063 +23773 142057 +14498 142033 +24740 141961 +15644 141960 +30633 141944 +15875 141934 +21114 141929 +13127 141927 +15504 141914 +8373 141897 +18969 141895 +20345 141876 +15234 141835 +8344 141831 +15043 141803 +19877 141769 +13690 141755 +17648 141754 +9285 141750 +12777 141739 +20493 141725 +33876 141723 +20175 141717 +22151 141698 +48828 141689 +25665 141686 +1259 141670 +20698 141670 +14010 141664 +28330 141659 +15349 141650 +5945 141623 +30801 141606 +26414 141595 +17800 141591 +998 141574 +16832 141552 +16492 141540 +27153 141534 +16399 141504 +13444 141490 +19848 141477 +28291 141476 +40253 141467 +21735 141446 +11685 141439 +39304 141433 +17687 141365 +11503 141351 +19534 141326 +18244 141322 +19487 141311 +15985 141281 +31612 141266 +13368 141262 +11290 141261 +10230 141260 +7657 141247 +18213 141238 +14996 141232 +23627 141202 +20075 141173 +5843 141143 +25333 141137 +22862 141121 +10477 141085 +22432 141070 +19425 141064 +20392 141053 +16361 141043 +23407 141014 +40624 141014 +27267 140988 +12258 140965 +18605 140949 +33020 140940 +28553 140933 +20974 140924 +14296 140899 +46935 140878 +2511 140811 +19437 140786 +21107 140783 +18591 140782 +6811 140766 +12243 140763 +1349 140737 +10974 140709 +5151 140698 +12748 140677 +25621 140661 +19263 140656 +16061 140651 +4432 140640 +31658 140637 +40060 140636 +19384 140632 +20814 140631 +25142 140619 +19301 140587 +23198 140585 +14131 140572 +7435 140568 +27515 140552 +15248 140540 +24581 140504 +25657 140498 +8402 140492 +15421 140465 +5720 140454 +17977 140454 +11471 140450 +24851 140420 +25308 140397 +13370 140392 +19479 140354 +22462 140350 +46000 140325 +29759 140315 +12964 140307 +11211 140304 +1197 140274 +20172 140272 +11510 140237 +17087 140207 +17897 140176 +17315 140174 +22064 140170 +37758 140169 +8608 140168 +8105 140165 +27247 140162 +12899 140156 +14436 140154 +20013 140149 +32134 140148 +10998 140138 +23354 140134 +26971 140127 +20327 140124 +26773 140115 +34982 140110 +20286 140083 +19313 140048 +23608 140032 +12505 140012 +27593 139969 +32025 139966 +19347 139957 +13365 139945 +13970 139935 +22452 139930 +8133 139915 +26445 139911 +12967 139876 +15134 139872 +9597 139860 +17107 139855 +19253 139841 +36895 139840 +2703 139835 +12029 139834 +40713 139815 +29269 139809 +17810 139778 +13517 139757 +16543 139737 +16502 139714 +14773 139695 +5351 139685 +12623 139663 +35363 139642 +27501 139623 +4715 139600 +3129 139597 +9245 139595 +9463 139588 +27885 139587 +17965 139583 +19132 139552 +13996 139549 +17282 139532 +15658 139528 +13319 139474 +5958 139469 +22218 139457 +25332 139452 +36779 139435 +22211 139423 +3379 139419 +6921 139416 +9234 139404 +17902 139396 +13288 139395 +23486 139371 +26479 139365 +37135 139360 +18371 139338 +7870 139292 +22597 139285 +26812 139273 +38704 139262 +14170 139260 +41708 139256 +19192 139241 +25003 139237 +2082 139236 +33330 139203 +39818 139192 +34105 139171 +25951 139164 +40020 139161 +21493 139131 +14128 139128 +9303 139119 +8103 139094 +14555 139070 +14156 139032 +22078 139012 +3266 138995 +14680 138963 +41941 138960 +18929 138948 +7938 138938 +17144 138922 +5724 138921 +18764 138920 +18915 138877 +10970 138844 +6255 138824 +19092 138812 +23233 138772 +16526 138749 +18657 138723 +9012 138692 +11580 138689 +16723 138684 +3144 138649 +754 138649 +5005 138632 +13012 138619 +944 138584 +10814 138583 +18560 138535 +17858 138511 +5751 138507 +29467 138495 +10354 138474 +10735 138452 +23019 138452 +29374 138452 +39946 138446 +22954 138429 +20127 138406 +15068 138388 +32644 138345 +31518 138313 +25912 138299 +24349 138293 +19989 138283 +12261 138276 +19494 138260 +28220 138234 +14546 138229 +26508 138228 +13706 138227 +21432 138224 +30093 138195 +18997 138186 +18935 138185 +19009 138150 +9596 138085 +21980 138078 +22301 138067 +21150 138016 +30311 138013 +28395 138002 +1145 137982 +17415 137982 +11892 137980 +48216 137923 +25448 137913 +14482 137889 +18965 137887 +17825 137879 +18523 137867 +11196 137844 +6850 137838 +19036 137832 +8516 137831 +10736 137823 +15686 137814 +8352 137792 +17324 137789 +25204 137788 +30315 137776 +25320 137772 +26874 137734 +13159 137723 +32369 137717 +13817 137699 +34673 137683 +9478 137680 +32646 137661 +21904 137649 +19789 137636 +25091 137583 +16700 137566 +21768 137552 +13066 137528 +23266 137523 +13893 137487 +12016 137481 +12323 137456 +25583 137455 +23300 137453 +15696 137327 +12855 137310 +49219 137299 +8151 137294 +16908 137290 +8439 137277 +6037 137276 +15285 137271 +27265 137223 +14769 137218 +15532 137198 +13763 137185 +30479 137177 +20524 137161 +28356 137159 +8575 137140 +12850 137138 +16045 137125 +27678 137117 +23226 137112 +10696 137107 +16391 137106 +20463 137105 +14743 137089 +15963 137075 +18138 137041 +31335 137033 +33709 137028 +8753 137024 +18310 136991 +8249 136987 +18116 136975 +19228 136973 +24009 136969 +21348 136945 +9997 136892 +35521 136858 +30260 136849 +20902 136836 +22641 136817 +5327 136805 +17073 136787 +13390 136757 +15934 136751 +35109 136719 +25974 136715 +24050 136706 +18903 136670 +37173 136634 +1652 136609 +23940 136579 +4223 136573 +26347 136572 +16162 136554 +16380 136548 +11851 136525 +14937 136515 +19161 136465 +22043 136462 +20849 136448 +31745 136448 +13818 136435 +23956 136431 +47144 136428 +34974 136423 +13801 136409 +22053 136378 +2525 136377 +6210 136367 +31540 136361 +1293 136354 +13993 136347 +14494 136319 +5259 136294 +4818 136274 +17864 136261 +24828 136253 +19674 136251 +25143 136242 +26913 136241 +11412 136220 +3810 136187 +24463 136173 +47962 136161 +18838 136132 +15657 136132 +34891 136078 +18302 136078 +29377 136051 +21349 136043 +28650 136013 +5088 136008 +18250 135991 +21691 135968 +15430 135949 +2654 135947 +44830 135945 +8520 135934 +12429 135919 +19888 135915 +17813 135910 +15827 135906 +22844 135900 +20434 135895 +20201 135887 +15554 135886 +12884 135878 +13859 135821 +827 135794 +27022 135785 +5973 135784 +26721 135779 +20170 135770 +19670 135770 +10892 135733 +15028 135730 +30928 135729 +9537 135713 +16771 135662 +4686 135657 +17899 135653 +7338 135653 +9742 135638 +8874 135624 +25067 135612 +2401 135602 +18644 135580 +4915 135574 +23727 135562 +15201 135534 +5746 135506 +21829 135459 +21040 135442 +17173 135442 +15768 135440 +27139 135425 +23790 135421 +16808 135408 +32722 135364 +31941 135358 +17991 135358 +9996 135356 +9360 135346 +14754 135332 +23049 135312 +18449 135257 +17153 135236 +2470 135234 +12956 135230 +1725 135221 +41107 135218 +16461 135211 +18860 135206 +15651 135184 +19110 135179 +15643 135177 +14149 135109 +16337 135106 +39897 135092 +12693 135089 +26649 135084 +11389 135075 +9145 135074 +26346 135068 +27443 135067 +22798 135063 +8377 135055 +34698 135017 +32250 135009 +24002 135005 +22663 135003 +19696 135001 +27698 134997 +21803 134967 +32108 134952 +18582 134939 +17101 134938 +32641 134906 +16038 134906 +9712 134898 +32462 134885 +15899 134884 +15468 134883 +15374 134876 +12642 134866 +22304 134858 +44133 134834 +15222 134777 +22001 134772 +29760 134753 +24553 134748 +35706 134738 +18715 134725 +9525 134724 +14716 134703 +22089 134687 +46754 134683 +9423 134677 +37118 134677 +25041 134658 +29995 134646 +24585 134646 +7340 134641 +10984 134634 +17387 134623 +32076 134607 +31178 134594 +26542 134575 +3672 134564 +5773 134545 +23006 134542 +24959 134532 +10154 134529 +22534 134525 +14979 134521 +21246 134516 +26376 134513 +12364 134511 +20725 134496 +18412 134495 +22080 134493 +7066 134484 +19412 134469 +7821 134443 +25518 134438 +24023 134427 +13540 134402 +45826 134397 +13843 134350 +17849 134341 +41992 134319 +5434 134314 +16226 134302 +24301 134299 +13742 134277 +17843 134272 +15915 134271 +33693 134260 +31958 134254 +26601 134245 +21500 134191 +33545 134161 +13213 134121 +48922 134117 +17979 134103 +11346 134093 +14723 134054 +18705 134045 +18008 134043 +19416 134036 +33167 134027 +33666 134023 +23043 133985 +8772 133973 +12440 133968 +39419 133931 +19403 133911 +25426 133910 +4454 133866 +22707 133862 +20865 133845 +26778 133839 +36538 133814 +19716 133802 +6362 133794 +32944 133782 +24017 133778 +10659 133777 +20824 133736 +37539 133714 +29771 133710 +6449 133687 +3044 133682 +19497 133674 +18081 133662 +16660 133646 +3448 133633 +18700 133633 +38735 133626 +7594 133617 +14335 133600 +12015 133581 +5460 133580 +20818 133557 +14198 133555 +20001 133546 +10396 133540 +21350 133521 +22864 133484 +33953 133448 +17534 133432 +29638 133397 +18383 133386 +7241 133382 +43630 133346 +25160 133310 +5356 133302 +24534 133271 +21660 133251 +23140 133242 +19600 133234 +17239 133213 +13559 133210 +34944 133196 +17819 133125 +24722 133121 +34509 133118 +7737 133116 +23225 133089 +12957 133079 +19158 133031 +11215 133028 +35004 133010 +8491 132998 +30461 132990 +38599 132981 +17762 132978 +21277 132978 +9587 132973 +4828 132970 +16572 132957 +27803 132945 +20155 132912 +20333 132911 +18805 132907 +1732 132894 +18258 132883 +13601 132867 +17330 132854 +806 132852 +25463 132829 +15475 132816 +24803 132814 +32074 132808 +12078 132797 +16178 132792 +18552 132780 +33362 132778 +23178 132778 +21811 132771 +20030 132746 +34747 132706 +31827 132697 +11881 132687 +7518 132678 +14985 132669 +46800 132651 +22662 132640 +31307 132614 +28231 132609 +14664 132607 +19978 132601 +10794 132574 +32897 132545 +12115 132544 +30758 132541 +27388 132532 +28894 132530 +29066 132508 +18031 132440 +5321 132440 +31156 132418 +25130 132405 +34241 132380 +22255 132338 +18252 132308 +23548 132307 +35300 132300 +26264 132293 +22403 132289 +21358 132287 +27324 132257 +35331 132241 +19623 132232 +3444 132224 +39608 132215 +34995 132211 +7245 132211 +13332 132205 +26383 132203 +19869 132142 +34802 132112 +21419 132108 +12945 132088 +23033 132074 +14352 132073 +20545 132063 +37698 132057 +18132 132029 +13364 132018 +24199 132011 +12249 132005 +21788 132003 +26459 131986 +18585 131981 +14145 131961 +15025 131951 +8094 131944 +5056 131936 +8496 131930 +7951 131926 +32464 131913 +21572 131899 +23890 131893 +31187 131878 +27094 131874 +14775 131870 +16164 131870 +44070 131862 +935 131858 +22892 131842 +33127 131841 +18535 131830 +18045 131822 +17985 131820 +27028 131794 +10990 131793 +233 131755 +1032 131748 +12810 131670 +20295 131661 +7003 131644 +13531 131639 +19283 131633 +19521 131629 +10883 131607 +3102 131603 +23310 131602 +28972 131585 +5165 131579 +17885 131576 +12185 131569 +22688 131533 +23630 131530 +6472 131510 +20857 131502 +10840 131472 +13357 131449 +16131 131445 +19305 131438 +16157 131430 +32188 131412 +19406 131393 +18030 131387 +4016 131369 +16617 131368 +18514 131367 +25953 131334 +13333 131318 +23023 131299 +24315 131296 +15195 131293 +26735 131288 +13793 131276 +26608 131252 +19280 131233 +32737 131230 +5976 131228 +28132 131197 +25623 131179 +27663 131158 +18661 131145 +20152 131143 +15687 131126 +19850 131110 +6783 131085 +13140 131080 +32382 131072 +12809 131070 +1454 131069 +13496 131061 +23342 131057 +28223 131024 +19141 131022 +16924 131003 +15799 130966 +15943 130965 +19069 130964 +38768 130960 +30680 130916 +20230 130906 +22447 130893 +20220 130885 +23641 130885 +18257 130869 +20061 130868 +29755 130867 +13798 130850 +11340 130838 +8379 130835 +9164 130831 +23254 130822 +31592 130815 +31036 130808 +23411 130797 +16746 130783 +19533 130778 +18600 130768 +18192 130748 +5669 130742 +19189 130734 +17419 130704 +18056 130682 +24424 130681 +12854 130680 +19410 130662 +5872 130645 +16784 130638 +13108 130617 +23885 130606 +27779 130603 +14811 130593 +18579 130591 +15495 130558 +20607 130555 +10076 130554 +23366 130553 +18832 130552 +11982 130540 +22126 130538 +30717 130536 +14181 130527 +15064 130516 +31585 130506 +25564 130505 +29872 130490 +26167 130479 +19066 130476 +21922 130471 +8483 130441 +10389 130430 +26338 130402 +8881 130361 +9662 130356 +11817 130350 +16139 130336 +23659 130331 +3291 130328 +10203 130318 +2833 130283 +24310 130243 +29717 130191 +11028 130165 +14760 130155 +10992 130152 +10449 130128 +40518 130127 +13981 130093 +22573 130086 +6197 130086 +37478 130063 +19086 130054 +31311 130029 +49368 130024 +14795 129994 +19386 129991 +16217 129979 +24737 129960 +9543 129951 +45091 129944 +25476 129943 +10978 129933 +5910 129924 +30858 129910 +16043 129863 +21568 129860 +23782 129837 +13256 129807 +8410 129804 +24479 129787 +28236 129781 +34540 129770 +21332 129767 +37415 129704 +12530 129690 +29045 129676 +9052 129673 +19255 129652 +15917 129642 +17261 129624 +24083 129604 +17706 129598 +13597 129594 +30629 129587 +16523 129579 +17767 129570 +34285 129568 +39873 129554 +20623 129536 +17940 129530 +13519 129530 +27583 129529 +15762 129522 +3803 129499 +36401 129483 +30825 129482 +26106 129455 +2960 129446 +28559 129444 +19993 129436 +27209 129405 +6187 129401 +29364 129394 +18577 129378 +22310 129377 +18029 129363 +25556 129363 +32802 129362 +11364 129341 +16958 129340 +17677 129312 +24554 129301 +28150 129243 +37376 129209 +34564 129195 +13825 129189 +4239 129188 +28793 129168 +28319 129154 +10153 129150 +22086 129150 +17317 129150 +14806 129144 +6799 129137 +15577 129112 +23319 129096 +20833 129088 +9410 129056 +13242 129047 +11674 129039 +23704 129017 +11992 129017 +13829 129014 +11216 129009 +29029 128983 +19308 128976 +32408 128968 +11319 128936 +16379 128905 +2871 128865 +16711 128854 +19695 128852 +10214 128837 +10411 128813 +3299 128802 +7087 128781 +15502 128780 +21509 128778 +12469 128777 +14211 128776 +13160 128763 +21076 128759 +30321 128753 +12414 128727 +27918 128698 +30258 128698 +27179 128690 +18873 128688 +16026 128688 +20622 128666 +6429 128658 +15666 128641 +20775 128611 +14722 128607 +20749 128607 +26690 128554 +44723 128548 +18482 128544 +22176 128537 +13794 128537 +34497 128533 +28080 128527 +21320 128520 +14027 128499 +13235 128496 +15275 128492 +21454 128488 +19173 128447 +16976 128441 +15923 128440 +10350 128420 +21543 128354 +17919 128353 +28992 128338 +26617 128329 +14828 128323 +20239 128323 +16257 128293 +14745 128239 +17577 128234 +33487 128231 +6605 128229 +2086 128196 +17795 128182 +11960 128158 +5763 128145 +20691 128128 +9417 128124 +24535 128114 +11848 128102 +29470 128090 +2349 128089 +19832 128082 +14130 128060 +18306 128059 +28931 128054 +14991 128049 +16464 128042 +13906 128036 +13964 128032 +24999 128027 +21608 128023 +22836 128019 +34778 128017 +1224 128012 +8156 127986 +41530 127975 +8255 127963 +30511 127955 +16463 127953 +31802 127951 +20897 127935 +20027 127933 +18447 127917 +15847 127899 +24671 127878 +4161 127872 +20726 127850 +13698 127830 +20159 127826 +7354 127810 +36399 127809 +21120 127800 +11423 127791 +24093 127773 +35482 127769 +13424 127768 +29056 127739 +16967 127728 +3855 127719 +31329 127712 +13123 127695 +31970 127686 +22198 127638 +9449 127632 +20883 127616 +26626 127615 +9437 127588 +24660 127570 +25619 127569 +16481 127568 +16479 127562 +9367 127550 +25615 127547 +14963 127542 +24126 127539 +10838 127539 +20069 127526 +2388 127526 +22821 127519 +19546 127503 +25187 127487 +6437 127487 +29328 127462 +3817 127422 +24841 127416 +11959 127411 +21381 127405 +12685 127401 +29276 127377 +12265 127352 +17435 127348 +15061 127330 +1669 127328 +14292 127320 +17713 127315 +19063 127307 +19157 127281 +8351 127277 +19264 127255 +17757 127245 +11752 127207 +14553 127192 +20188 127170 +30088 127157 +15569 127152 +22424 127141 +7191 127137 +4551 127129 +21257 127127 +31100 127127 +9267 127119 +28181 127109 +14658 127089 +17603 127087 +8851 127082 +11976 127069 +26089 127068 +14976 127059 +3868 127049 +1226 127040 +20116 127039 +16697 127037 +32426 127029 +18282 127021 +8158 127004 +26044 127003 +12176 126996 +17361 126991 +23465 126976 +34283 126916 +24110 126892 +26482 126883 +14798 126866 +36011 126836 +11698 126823 +7736 126779 +28199 126777 +18359 126777 +33157 126772 +18057 126766 +5344 126749 +29852 126727 +8426 126721 +27466 126715 +14460 126712 +3930 126684 +14582 126678 +17865 126674 +8176 126622 +31325 126613 +12128 126609 +39998 126599 +17039 126588 +15325 126572 +15700 126538 +17789 126511 +20229 126490 +27082 126490 +24173 126455 +47417 126452 +21049 126376 +12416 126374 +234 126350 +9608 126318 +10723 126314 +6344 126314 +25693 126308 +27994 126292 +23527 126266 +110 126256 +25438 126255 +20018 126243 +26401 126222 +17009 126214 +35398 126200 +27181 126178 +15394 126166 +22895 126160 +13399 126152 +6152 126070 +33051 126068 +30918 126061 +32300 126058 +9090 126034 +11147 126006 +20573 126005 +22162 125978 +1727 125971 +36872 125968 +31455 125964 +13459 125960 +12431 125944 +23369 125940 +20738 125923 +8992 125913 +31806 125894 +29864 125890 +12959 125889 +17867 125857 +20212 125844 +9126 125812 +21151 125803 +29822 125802 +17769 125762 +14032 125746 +12587 125739 +24872 125735 +27051 125733 +22145 125721 +21329 125709 +2343 125702 +21880 125672 +25026 125659 +15900 125632 +5241 125627 +6434 125614 +23003 125594 +13897 125543 +18051 125534 +13468 125528 +15265 125521 +27165 125510 +28024 125506 +22052 125501 +29111 125497 +26321 125484 +13755 125474 +15778 125466 +11240 125464 +30498 125461 +23754 125450 +23856 125414 +33379 125367 +14165 125365 +12749 125345 +25028 125333 +28313 125331 +16386 125299 +16282 125293 +16949 125290 +36983 125273 +16829 125258 +21631 125254 +4199 125233 +38579 125213 +18794 125199 +7940 125190 +26431 125158 +40396 125151 +18978 125148 +30126 125144 +28830 125126 +17275 125125 +20469 125107 +4666 125106 +16342 125106 +30644 125051 +21555 125045 +10480 125034 +10536 125025 +10756 125025 +12035 125019 +8056 125013 +15972 124988 +18131 124971 +13948 124971 +26728 124956 +14249 124947 +15927 124937 +44130 124928 +46880 124912 +11891 124899 +27120 124857 +17760 124849 +15756 124847 +15458 124842 +15218 124822 +44418 124816 +21899 124814 +16276 124788 +11799 124788 +28274 124781 +15423 124764 +15588 124733 +17626 124723 +35921 124719 +15613 124712 +20062 124697 +29388 124694 +24878 124664 +9150 124653 +5314 124647 +19170 124645 +25005 124640 +13527 124636 +13604 124619 +17604 124610 +4626 124572 +28085 124567 +21801 124559 +42506 124556 +17404 124554 +4716 124547 +11879 124527 +38943 124485 +15524 124457 +14035 124441 +49963 124398 +18502 124394 +34072 124393 +20249 124389 +5299 124386 +15669 124369 +27890 124349 +28293 124329 +18839 124308 +19273 124302 +43204 124276 +21027 124269 +27309 124267 +23692 124252 +18082 124252 +2659 124243 +22727 124243 +1199 124239 +19525 124239 +29897 124224 +22325 124221 +30681 124190 +11782 124180 +22376 124105 +20612 124096 +2373 124096 +27512 124091 +27320 124071 +23953 124058 +16851 124048 +21547 124045 +16408 124037 +19719 123989 +20440 123988 +24185 123978 +16402 123943 +23446 123940 +31832 123926 +25821 123920 +6535 123915 +37236 123894 +26624 123888 +32788 123883 +25408 123870 +36707 123865 +38477 123845 +17021 123842 +3760 123831 +7662 123825 +14962 123814 +30569 123795 +32251 123776 +14787 123775 +21517 123753 +8205 123745 +19874 123734 +14160 123724 +19642 123705 +15408 123694 +14357 123690 +11621 123689 +25522 123687 +26673 123684 +17639 123681 +26849 123656 +5429 123621 +27999 123612 +12852 123612 +27350 123608 +23842 123602 +1214 123579 +30895 123559 +45376 123535 +33461 123520 +21845 123504 +29072 123497 +19183 123465 +10374 123419 +25866 123418 +24405 123406 +31154 123400 +23397 123390 +16003 123386 +19444 123382 +14267 123353 +15259 123346 +19374 123341 +6999 123340 +19330 123337 +11701 123317 +18281 123284 +14476 123278 +30857 123273 +35571 123272 +19451 123265 +16030 123263 +10870 123261 +18403 123215 +17170 123207 +10921 123203 +15837 123185 +29049 123176 +17141 123172 +15368 123168 +16609 123161 +18822 123148 +16524 123143 +38396 123091 +28842 123082 +19249 123080 +34803 123075 +13300 123073 +19595 123057 +15751 123055 +23679 123044 +50196 123042 +27938 123041 +18658 122934 +5391 122903 +23121 122853 +18990 122850 +14816 122833 +12816 122817 +25016 122807 +20268 122806 +33743 122793 +28549 122781 +26640 122767 +34649 122757 +25123 122740 +27557 122730 +13532 122717 +24172 122715 +8199 122667 +18178 122650 +21683 122604 +16690 122592 +35773 122588 +25575 122564 +14276 122559 +17990 122553 +27820 122549 +32907 122517 +22901 122505 +19611 122498 +18044 122478 +15629 122450 +18774 122446 +9156 122434 +27702 122421 +17755 122419 +17064 122398 +23352 122397 +39489 122393 +33099 122372 +3345 122359 +15277 122337 +15821 122332 +28608 122328 +32664 122320 +31045 122298 +24800 122293 +30724 122286 +33391 122275 +16555 122274 +11487 122269 +27922 122268 +29866 122267 +20105 122265 +13589 122260 +11181 122223 +14520 122206 +23933 122204 +21067 122194 +17233 122177 +17333 122168 +28760 122160 +15483 122157 +18207 122151 +19178 122151 +12835 122146 +26896 122141 +14461 122113 +19111 122104 +29431 122093 +25577 122083 +22355 122083 +17105 122065 +12819 122057 +24224 122023 +15772 122005 +21331 121999 +35292 121948 +15000 121946 +16880 121939 +46289 121924 +38718 121920 +18650 121917 +2146 121912 +31574 121909 +23143 121899 +4594 121896 +11844 121885 +24789 121882 +18767 121867 +40418 121845 +29834 121842 +27975 121824 +11914 121808 +26577 121791 +26004 121781 +27862 121776 +18957 121771 +21089 121764 +19015 121750 +14114 121748 +22118 121746 +35825 121727 +22803 121723 +16062 121692 +20963 121687 +27027 121665 +15833 121661 +12840 121654 +43736 121641 +13218 121636 +19188 121623 +14084 121622 +30552 121613 +20841 121595 +7925 121592 +28331 121583 +16289 121565 +28948 121558 +30422 121546 +19518 121542 +16785 121516 +22777 121507 +27604 121474 +22436 121467 +10005 121465 +3079 121458 +25218 121457 +17814 121418 +14144 121412 +10178 121411 +12994 121402 +5813 121392 +17026 121374 +34271 121366 +7210 121366 +22499 121344 +15448 121313 +26380 121311 +16412 121301 +24072 121295 +10133 121291 +14944 121282 +29975 121282 +31770 121280 +24138 121264 +7839 121247 +16997 121228 +4821 121222 +26966 121221 +41770 121215 +4653 121195 +15814 121179 +17442 121169 +7575 121133 +14268 121123 +38779 121115 +15178 121110 +37506 121108 +31107 121078 +10835 121074 +14404 121073 +19684 121048 +12550 121038 +3622 121019 +34273 121016 +15488 121016 +16936 121013 +32054 121011 +24054 121010 +34361 120997 +37822 120990 +3147 120971 +23276 120969 +18157 120957 +10254 120946 +5377 120932 +8702 120927 +20875 120921 +14907 120915 +31004 120888 +2588 120876 +23431 120875 +28212 120873 +41624 120868 +2505 120854 +37213 120848 +15728 120840 +12136 120838 +4830 120827 +23455 120816 +15213 120805 +28323 120804 +20324 120794 +22700 120786 +11230 120775 +31446 120774 +4775 120761 +15957 120760 +24076 120758 +18696 120758 +12518 120749 +28470 120716 +28449 120707 +11244 120691 +15781 120676 +27174 120676 +14425 120666 +15187 120654 +17142 120653 +27655 120646 +16662 120643 +15674 120638 +20952 120631 +15389 120630 +22105 120580 +20606 120580 +9223 120553 +34855 120551 +28275 120508 +22835 120507 +9791 120495 +13503 120475 +3976 120460 +33925 120449 +15709 120440 +20371 120439 +21187 120396 +27470 120386 +14682 120381 +18988 120379 +30197 120370 +11985 120353 +16014 120342 +21900 120323 +16309 120301 +44568 120291 +29583 120286 +10995 120285 +20518 120284 +22249 120281 +17210 120279 +30145 120243 +21404 120234 +5591 120233 +28314 120227 +2537 120221 +19236 120189 +31388 120186 +23732 120184 +14277 120172 +27504 120153 +10337 120141 +18639 120121 +35410 120108 +16224 120100 +45284 120055 +20525 120041 +28382 120031 +16510 120026 +5730 120006 +32612 119981 +49746 119974 +22111 119950 +14712 119947 +43321 119941 +20889 119936 +22250 119924 +14711 119913 +16871 119905 +35686 119871 +19827 119866 +24816 119820 +14199 119818 +10812 119815 +32801 119798 +29875 119795 +24746 119781 +18663 119772 +25876 119743 +33845 119741 +19134 119731 +20246 119692 +15216 119688 +7465 119681 +2758 119676 +10946 119674 +12807 119659 +44083 119659 +28755 119659 +22202 119642 +16220 119634 +29713 119630 +15758 119592 +11397 119582 +31066 119571 +27008 119551 +18101 119524 +21676 119521 +9206 119499 +28822 119497 +38780 119495 +39597 119480 +24215 119476 +18422 119472 +17228 119434 +9484 119428 +12116 119426 +19205 119424 +24462 119423 +35837 119407 +25604 119399 +20444 119399 +15922 119371 +24482 119369 +17698 119363 +31769 119358 +16235 119356 +6249 119355 +19505 119342 +15334 119337 +19059 119312 +19224 119292 +6637 119277 +21638 119272 +40666 119252 +47563 119245 +36677 119243 +15231 119229 +17341 119228 +18413 119221 +19287 119197 +10314 119173 +19992 119162 +28818 119150 +13674 119110 +17318 119107 +42095 119104 +27154 119096 +37518 119076 +20195 119074 +19885 119066 +2462 119063 +14932 119055 +19254 119041 +36552 119014 +11358 118995 +33534 118974 +12163 118957 +27434 118926 +19644 118916 +27511 118900 +17231 118891 +48349 118867 +44337 118827 +3674 118802 +22976 118787 +1319 118776 +15641 118775 +18963 118772 +19047 118771 +31593 118770 +24268 118770 +24120 118766 +24410 118765 +7854 118753 +17658 118737 +12558 118726 +7669 118722 +23634 118718 +12250 118710 +31039 118696 +12711 118695 +24791 118694 +27136 118687 +33445 118674 +17137 118670 +22116 118652 +21372 118651 +24759 118644 +19637 118643 +12314 118624 +4024 118619 +19572 118616 +8106 118612 +21190 118608 +29329 118591 +6749 118583 +5522 118573 +21189 118538 +20727 118524 +35083 118523 +22591 118520 +8336 118518 +18421 118510 +8120 118508 +27478 118507 +27667 118497 +18531 118493 +7975 118489 +18455 118481 +24967 118452 +16622 118441 +16560 118434 +17856 118417 +25048 118416 +17247 118409 +13008 118399 +30169 118392 +27023 118392 +16415 118387 +28237 118383 +25884 118378 +14561 118377 +28623 118376 +1974 118374 +34468 118353 +30997 118352 +17479 118309 +15063 118308 +40188 118300 +12183 118289 +28487 118265 +39692 118254 +10850 118242 +596 118239 +24024 118178 +29010 118175 +26549 118175 +7544 118174 +14622 118141 +21030 118135 +32130 118109 +15964 118100 +10321 118085 +5406 118074 +12799 118067 +9248 118049 +19450 118047 +13353 118038 +20340 118037 +20945 118033 +26562 118027 +24817 118026 +23141 118026 +35232 118004 +20747 118003 +17988 117996 +7001 117976 +26363 117967 +32210 117957 +31842 117951 +29216 117939 +15376 117938 +16727 117927 +22475 117899 +35560 117898 +16101 117871 +14917 117868 +9782 117864 +19837 117850 +16385 117848 +9130 117810 +26768 117788 +997 117779 +6864 117774 +21837 117760 +32055 117755 +36316 117750 +18592 117740 +36606 117732 +6285 117725 +13281 117721 +36345 117709 +4704 117694 +21229 117692 +34174 117672 +3366 117657 +45695 117655 +15129 117648 +46441 117643 +41863 117632 +1555 117625 +16039 117620 +14729 117604 +6161 117600 +32224 117592 +24058 117564 +17326 117563 +15745 117548 +8952 117546 +19235 117537 +17286 117529 +38167 117528 +10601 117522 +10136 117507 +49583 117488 +18821 117476 +46154 117472 +19272 117463 +21343 117461 +27707 117457 +3669 117453 +15766 117428 +34488 117427 +21846 117412 +37851 117408 +26516 117401 +21143 117391 +18725 117376 +34438 117368 +18702 117362 +8162 117343 +25259 117340 +22868 117336 +27818 117332 +18344 117327 +924 117316 +21472 117310 +40439 117306 +16971 117294 +32666 117293 +19125 117285 +31006 117275 +4861 117256 +14968 117255 +18692 117226 +8609 117209 +11854 117197 +30833 117194 +14941 117190 +12924 117181 +16450 117174 +48088 117173 +3305 117171 +15998 117171 +33017 117167 +40858 117164 +32579 117161 +17446 117132 +17636 117129 +21921 117124 +45567 117124 +41445 117094 +17631 117091 +23962 117024 +28367 117022 +14380 117015 +19417 117010 +38029 116996 +708 116994 +28990 116979 +17197 116973 +12368 116961 +23538 116953 +30418 116952 +49611 116944 +20913 116940 +30241 116931 +32451 116924 +31750 116924 +6837 116916 +15228 116889 +21830 116886 +14966 116854 +11377 116852 +24854 116839 +36269 116837 +248 116834 +13939 116827 +39290 116766 +3778 116752 +9771 116747 +22846 116733 +21347 116731 +33457 116727 +18451 116724 +19681 116718 +45790 116681 +13508 116680 +22493 116676 +22598 116671 +4720 116667 +30024 116661 +20928 116656 +21096 116654 +27581 116650 +5758 116642 +12902 116633 +22778 116623 +25894 116617 +15719 116606 +39398 116603 +16807 116602 +27231 116596 +7816 116582 +34108 116573 +30567 116559 +10056 116543 +14715 116519 +14475 116497 +23910 116456 +14834 116456 +30364 116454 +25706 116453 +15753 116444 +28442 116434 +35469 116433 +12310 116425 +15910 116416 +21957 116405 +39808 116388 +16903 116385 +18989 116376 +39487 116376 +14782 116372 +44744 116356 +17180 116354 +18556 116349 +26876 116324 +27436 116313 +25482 116308 +13735 116304 +25394 116294 +26450 116277 +37616 116277 +7643 116276 +10550 116245 +19693 116243 +25852 116220 +1009 116180 +12861 116176 +11043 116174 +17124 116170 +24933 116165 +12921 116149 +18642 116146 +15402 116135 +2469 116096 +28578 116075 +23682 116074 +17939 116040 +17918 116033 +4491 116022 +4399 116019 +22232 116015 +30837 116010 +2296 116003 +19356 115997 +30738 115978 +10934 115971 +15836 115952 +19266 115932 +28571 115926 +23558 115922 +22761 115911 +30217 115908 +28083 115882 +19750 115877 +15418 115869 +7438 115863 +37918 115860 +42442 115851 +22568 115848 +4670 115832 +20537 115831 +26212 115829 +10626 115826 +25722 115824 +24952 115801 +38739 115794 +28172 115756 +22942 115738 +32246 115724 +22698 115711 +26724 115689 +16394 115688 +19860 115671 +22045 115657 +19771 115654 +7286 115652 +31393 115649 +24719 115594 +10606 115590 +11742 115574 +20417 115560 +39641 115541 +30157 115539 +18979 115489 +29784 115471 +22871 115456 +16656 115454 +17412 115448 +20185 115441 +10340 115438 +22152 115437 +22753 115424 +47970 115422 +8332 115410 +20388 115407 +40238 115385 +17136 115384 +19215 115375 +5497 115369 +24630 115363 +36026 115356 +6217 115346 +38307 115331 +12931 115321 +44985 115318 +19240 115308 +8588 115305 +17343 115297 +16965 115297 +21108 115274 +31668 115272 +23423 115269 +44021 115255 +18316 115242 +111 115218 +30017 115213 +19152 115197 +34668 115186 +43449 115178 +14746 115152 +32559 115151 +39199 115140 +19014 115138 +8325 115132 +13348 115122 +28137 115112 +17351 115074 +10961 115046 +24448 115041 +30625 115012 +32473 114997 +27243 114972 +25737 114952 +38988 114930 +33629 114917 +1198 114880 +23041 114878 +33197 114860 +32763 114850 +25732 114849 +24617 114837 +4657 114831 +24237 114822 +18492 114820 +48609 114805 +17726 114802 +19575 114794 +10324 114779 +15009 114778 +10943 114778 +46485 114767 +34609 114761 +24011 114759 +24626 114742 +11393 114718 +14262 114713 +13432 114709 +20517 114709 +18813 114701 +23897 114697 +13813 114693 +24456 114672 +17870 114659 +4820 114652 +15363 114638 +18137 114627 +27239 114619 +37520 114611 +1791 114610 +34022 114584 +15693 114577 +16314 114563 +6696 114550 +22425 114545 +14934 114533 +13068 114533 +37936 114533 +46520 114532 +7510 114530 +19692 114528 +28001 114519 +20858 114465 +15959 114460 +34410 114446 +18429 114436 +21383 114434 +29476 114425 +20830 114419 +36019 114417 +15440 114410 +31140 114376 +13886 114375 +17451 114350 +16728 114339 +19516 114329 +19218 114295 +6347 114288 +6013 114285 +25347 114282 +28960 114280 +44286 114263 +20736 114226 +7758 114219 +25837 114215 +11994 114203 +17293 114196 +9435 114179 +2244 114175 +15186 114155 +18341 114147 +29147 114139 +26803 114113 +12342 114113 +33061 114104 +24829 114104 +18135 114084 +31047 114082 +40560 114068 +21307 114058 +21416 114031 +27046 114011 +24261 114008 +47168 113996 +10344 113973 +9222 113955 +102 113953 +3564 113949 +27026 113948 +26172 113940 +3130 113924 +3063 113907 +24627 113901 +27296 113856 +10157 113854 +11950 113851 +33617 113842 +15589 113838 +31799 113808 +19340 113805 +109 113805 +2573 113794 +47180 113776 +6878 113771 +18077 113739 +17714 113729 +40159 113721 +40344 113718 +38580 113716 +35007 113711 +35053 113685 +10896 113670 +19353 113666 +11339 113666 +47652 113661 +17099 113658 +28785 113653 +25338 113646 +22076 113639 +8085 113636 +9235 113629 +11386 113612 +19344 113612 +17877 113612 +20496 113594 +24682 113590 +32127 113557 +23915 113556 +10462 113553 +2621 113550 +30554 113540 +5133 113528 +13954 113525 +47296 113443 +16726 113424 +10173 113419 +20999 113413 +17584 113387 +35440 113386 +2352 113376 +26619 113372 +14177 113343 +7234 113325 +19289 113294 +25446 113249 +31984 113228 +18682 113217 +22435 113192 +22044 113175 +36952 113170 +20285 113152 +24060 113141 +25736 113139 +30697 113084 +18984 113082 +14568 113068 +24473 113039 +17924 113037 +32376 113032 +33708 113030 +28941 113014 +17298 113013 +13878 113012 +14956 113000 +14542 112997 +29189 112980 +17166 112979 +22199 112966 +23130 112943 +34724 112929 +15683 112912 +17523 112887 +16084 112886 +17655 112871 +17558 112871 +20421 112866 +36477 112850 +17805 112849 +18450 112847 +35401 112845 +20929 112836 +14397 112833 +15181 112827 +12169 112804 +25374 112802 +20970 112801 +5482 112794 +16370 112782 +16550 112781 +21010 112776 +42071 112773 +13196 112755 +16576 112754 +15467 112752 +91 112729 +21966 112720 +29018 112719 +27788 112718 +23107 112696 +6691 112695 +38256 112694 +24610 112693 +10256 112691 +26202 112678 +23950 112673 +1563 112672 +20828 112642 +28587 112640 +16159 112637 +22518 112637 +9467 112608 +23529 112607 +10360 112607 +23948 112603 +9230 112592 +21961 112588 +26630 112585 +12961 112577 +16835 112565 +33726 112551 +28550 112537 +16042 112531 +9732 112521 +24948 112520 +10917 112519 +17238 112508 +15482 112506 +14419 112484 +16573 112478 +20756 112465 +17549 112426 +16294 112420 +18881 112418 +2851 112409 +29758 112392 +16489 112375 +14209 112371 +26585 112361 +40928 112354 +9665 112346 +17033 112337 +43225 112332 +23541 112321 +22694 112320 +25436 112317 +23825 112315 +21998 112298 +8642 112295 +24313 112284 +16840 112281 +29980 112271 +10693 112258 +30426 112255 +21702 112236 +17970 112199 +3027 112194 +24891 112183 +14572 112169 +4722 112149 +3219 112147 +1449 112130 +27496 112098 +22848 112089 +18162 112084 +28375 112054 +20294 112053 +29019 112048 +26736 112047 +5139 112032 +15618 112031 +36067 112030 +26066 112024 +9998 112023 +36518 112019 +29260 112017 +46536 112016 +22268 112007 +12618 111987 +17623 111987 +25750 111980 +30287 111957 +18384 111956 +16057 111956 +33311 111930 +33371 111930 +6606 111906 +21590 111904 +41806 111904 +23718 111903 +29553 111853 +24025 111849 +34687 111840 +27497 111833 +18120 111829 +18098 111817 +13505 111816 +2413 111776 +27906 111774 +18744 111765 +20835 111746 +1800 111745 +18622 111736 +16999 111726 +14047 111710 +26278 111708 +7885 111680 +18981 111679 +37909 111671 +14259 111665 +40098 111653 +22292 111646 +11466 111607 +27473 111603 +15975 111601 +3018 111598 +19612 111585 +18820 111566 +31946 111560 +36178 111544 +14233 111526 +12093 111521 +5336 111514 +13841 111480 +39992 111478 +25694 111476 +49487 111476 +19426 111473 +20449 111469 +1536 111462 +22843 111459 +22807 111449 +21411 111447 +22097 111387 +12510 111385 +20629 111383 +21832 111360 +15185 111321 +9973 111320 +29926 111313 +17888 111306 +25570 111298 +34521 111292 +13762 111274 +38529 111270 +32534 111267 +27119 111259 +18877 111243 +792 111231 +21570 111201 +7738 111178 +31620 111174 +31622 111172 +22581 111154 +2179 111152 +11144 111141 +5040 111121 +16747 111073 +13025 111060 +6010 111050 +17568 111050 +20433 111010 +20577 110983 +19217 110974 +29012 110948 +20562 110917 +18197 110910 +14547 110908 +33098 110894 +35783 110881 +22096 110879 +12821 110869 +16760 110864 +15131 110864 +34560 110857 +27418 110855 +974 110850 +26891 110849 +36986 110848 +36683 110815 +13433 110810 +25813 110795 +24634 110781 +5328 110780 +28604 110768 +19894 110754 +28628 110720 +17508 110714 +24884 110708 +22723 110705 +21001 110697 +4628 110669 +18233 110660 +31617 110625 +23994 110620 +14872 110618 +22537 110592 +16347 110563 +11880 110559 +33847 110559 +20471 110555 +18583 110553 +9678 110546 +13787 110530 +37081 110528 +38805 110507 +15094 110504 +33625 110502 +12397 110501 +34814 110499 +29 110485 +9377 110471 +25814 110470 +37790 110464 +17076 110419 +36199 110412 +32206 110411 +12076 110407 +23687 110402 +8729 110399 +22533 110393 +17565 110375 +25691 110364 +16416 110357 +16292 110346 +28773 110346 +16106 110343 +27147 110342 +13015 110331 +16343 110317 +30744 110312 +15968 110303 +35903 110267 +7934 110261 +243 110257 +30007 110254 +13482 110252 +12256 110244 +23017 110238 +4241 110232 +30379 110211 +20173 110198 +28276 110176 +24065 110172 +19951 110168 +45818 110162 +26420 110158 +18493 110158 +19379 110154 +36852 110123 +17229 110106 +17453 110105 +23707 110102 +12064 110092 +12707 110082 +31244 110069 +28014 110046 +2980 110045 +17556 110043 +1401 110024 +27872 110021 +12648 110011 +10796 110009 +18500 110006 +22414 109992 +30628 109992 +12178 109989 +8423 109985 +15284 109974 +23448 109945 +27206 109927 +22575 109926 +21025 109915 +24264 109903 +23384 109898 +2284 109898 +14479 109885 +29921 109881 +8757 109867 +34753 109867 +27216 109862 +19561 109854 +15731 109841 +31108 109834 +5285 109830 +14206 109802 +20406 109789 +21625 109782 +6162 109780 +27589 109766 +40162 109754 +26853 109753 +33006 109733 +13868 109724 +30215 109723 +17644 109720 +31942 109711 +20794 109705 +8870 109685 +38613 109647 +43105 109641 +21951 109636 +3508 109629 +15610 109610 +29597 109607 +37070 109563 +21823 109545 +14766 109542 +9383 109541 +9708 109539 +16925 109531 +18321 109531 +16251 109514 +22560 109502 +25340 109489 +13071 109479 +32669 109479 +18461 109462 +40532 109458 +34186 109449 +28513 109446 +37921 109420 +8652 109399 +27517 109399 +12430 109375 +14290 109374 +50240 109356 +27058 109344 +25892 109323 +18626 109313 +9178 109304 +36363 109265 +24034 109262 +27333 109250 +35553 109243 +9440 109239 +34699 109239 +49403 109230 +7020 109226 +16073 109215 +17074 109191 +16695 109190 +48583 109183 +47177 109166 +26496 109164 +8567 109162 +16158 109157 +17700 109114 +16058 109112 +27974 109089 +28879 109087 +17163 109079 +25957 109071 +24622 109053 +26049 109044 +17345 109038 +23622 109028 +25949 109027 +27833 109020 +26389 109002 +15169 109002 +25065 108979 +21651 108957 +23804 108946 +14698 108943 +23062 108941 +20387 108930 +15993 108930 +32868 108923 +27835 108908 +20101 108907 +37126 108899 +24649 108881 +23911 108878 +21200 108876 +941 108873 +13050 108861 +21099 108851 +24522 108847 +4984 108838 +15337 108834 +32992 108832 +34553 108827 +5731 108822 +20338 108811 +35345 108795 +37270 108794 +36758 108791 +8459 108790 +17656 108782 +42556 108782 +21707 108777 +41816 108764 +29842 108758 +41759 108758 +15938 108753 +29139 108746 +21068 108743 +16944 108734 +13746 108721 +2136 108712 +21566 108691 +15510 108679 +15533 108678 +26361 108677 +15939 108671 +25826 108655 +43416 108650 +11460 108646 +29761 108646 +18235 108626 +17289 108618 +17342 108612 +20031 108607 +30067 108607 +28245 108589 +29578 108583 +32444 108580 +24141 108576 +23783 108551 +21203 108544 +25203 108539 +45980 108539 +19458 108529 +10957 108528 +29588 108507 +24701 108498 +13695 108489 +8109 108486 +25141 108469 +36451 108460 +13411 108459 +12635 108442 +5558 108421 +15344 108417 +17548 108401 +25539 108399 +17775 108360 +28926 108341 +23103 108333 +45624 108330 +38602 108318 +26905 108304 +31275 108289 +39717 108286 +21673 108284 +10905 108265 +23185 108247 +34479 108242 +12251 108242 +20425 108242 +12177 108231 +21799 108218 +22673 108189 +31988 108187 +23993 108184 +28503 108175 +18427 108171 +32810 108130 +21848 108123 +33624 108112 +11598 108082 +24471 108049 +47604 108032 +18357 108026 +10707 108011 +9122 107999 +17696 107994 +22527 107990 +19647 107989 +15874 107977 +6258 107969 +16019 107967 +23973 107959 +26635 107954 +23887 107919 +13822 107917 +25965 107902 +18693 107889 +35089 107873 +28289 107850 +30770 107844 +10356 107804 +22561 107803 +42735 107795 +13745 107775 +35759 107766 +14992 107755 +28619 107755 +35206 107752 +7614 107745 +9446 107705 +36675 107690 +10327 107670 +41427 107666 +12876 107664 +23086 107651 +19901 107616 +21031 107579 +44321 107574 +33059 107560 +41253 107537 +33673 107536 +30597 107535 +14294 107534 +7250 107531 +27505 107531 +24341 107525 +36700 107516 +28485 107514 +34272 107514 +29336 107476 +20881 107471 +11653 107461 +29575 107452 +6243 107451 +23256 107447 +15529 107429 +21954 107420 +16605 107413 +2375 107413 +18436 107396 +14893 107391 +25523 107378 +7922 107373 +18179 107368 +13493 107366 +35691 107365 +17607 107348 +30401 107338 +1356 107332 +16882 107330 +25247 107317 +29521 107310 +11354 107308 +17942 107301 +23621 107297 +17439 107288 +11659 107272 +28346 107263 +17365 107258 +19658 107251 +44198 107247 +21596 107243 +25427 107235 +15017 107215 +18987 107199 +11401 107183 +10781 107179 +12375 107176 +15561 107168 +39062 107167 +4834 107159 +19038 107153 +21655 107138 +16125 107128 +23404 107127 +7952 107119 +29838 107098 +23514 107074 +22566 107038 +31876 107035 +25477 107014 +17186 107001 +25910 106995 +25824 106991 +25146 106974 +22894 106968 +29058 106953 +3718 106938 +36486 106932 +19113 106932 +24963 106930 +21046 106917 +20349 106912 +8625 106905 +25395 106878 +35188 106861 +34002 106860 +29656 106849 +932 106830 +18787 106830 +28672 106817 +39700 106798 +15058 106798 +31845 106798 +35892 106792 +30081 106760 +18399 106755 +39560 106753 +43262 106752 +13602 106745 +3384 106743 +19748 106725 +20528 106718 +29463 106712 +15645 106680 +17624 106678 +36719 106671 +19439 106659 +42668 106650 +21658 106623 +20236 106603 +12986 106599 +12870 106581 +8079 106576 +39137 106568 +18854 106552 +13664 106543 +12253 106495 +14063 106481 +25136 106471 +16762 106462 +16668 106453 +37578 106452 +29803 106432 +19168 106428 +26005 106408 +21362 106404 +21276 106400 +24271 106381 +26349 106371 +22900 106361 +20657 106360 +17753 106358 +46726 106344 +6113 106338 +24226 106337 +31074 106336 +17927 106333 +28044 106330 +4517 106327 +27687 106320 +19761 106313 +13575 106310 +18174 106282 +29820 106265 +18350 106262 +21723 106250 +37425 106240 +19654 106224 +5261 106223 +33035 106219 +36236 106203 +31333 106199 +43804 106198 +1860 106194 +11813 106179 +5685 106177 +13950 106177 +18418 106170 +16739 106164 +15936 106134 +18778 106127 +35902 106125 +32211 106122 +26815 106121 +27909 106119 +38001 106102 +48245 106097 +10339 106094 +30947 106078 +7842 106055 +24929 106038 +6391 105988 +21430 105973 +16426 105960 +15121 105958 +24128 105950 +17501 105930 +26336 105916 +25422 105910 +44844 105856 +5905 105843 +25485 105839 +2377 105825 +17586 105823 +14175 105822 +25614 105818 +22168 105808 +48560 105805 +42192 105798 +4135 105767 +12918 105742 +31117 105726 +17522 105716 +20732 105712 +16222 105709 +18035 105690 +15290 105633 +32470 105630 +22549 105607 +43013 105596 +18734 105595 +32034 105590 +13860 105581 +31232 105578 +39507 105571 +33787 105569 +5185 105569 +34992 105557 +20487 105544 +18894 105536 +14870 105520 +12941 105491 +29599 105482 +23543 105464 +16836 105460 +28229 105455 +7414 105449 +21615 105449 +36962 105418 +18028 105409 +35468 105406 +39032 105406 +24351 105397 +2220 105387 +39329 105383 +13661 105375 +985 105373 +31610 105352 +16430 105349 +20401 105348 +26038 105347 +21368 105326 +3646 105323 +26474 105320 +6767 105317 +44487 105294 +18782 105285 +17279 105275 +15497 105268 +40707 105266 +30385 105261 +29799 105256 +2557 105241 +1149 105226 +23080 105189 +17554 105179 +20160 105164 +27723 105162 +32586 105160 +10676 105154 +14107 105139 +10982 105131 +27316 105111 +10644 105109 +23365 105106 +1882 105085 +19193 105060 +24057 105059 +35507 105055 +11427 105024 +16065 105020 +18395 105017 +47975 105001 +12583 104995 +1861 104991 +38581 104975 +19589 104971 +19942 104959 +34807 104949 +14739 104948 +30446 104922 +37958 104912 +10510 104907 +6966 104889 +31376 104889 +18474 104873 +22650 104867 +28917 104843 +15857 104825 +12171 104825 +12038 104822 +42555 104819 +36547 104790 +15991 104789 +23030 104785 +19726 104780 +39911 104761 +26779 104752 +27565 104752 +22928 104741 +17184 104739 +16627 104734 +3571 104687 +15494 104677 +27337 104671 +10091 104669 +15989 104665 +26036 104660 +33500 104650 +24690 104606 +20674 104597 +27311 104596 +17783 104590 +23231 104588 +40556 104586 +28163 104581 +13397 104574 +27709 104566 +17621 104565 +30773 104562 +41582 104562 +17592 104532 +28723 104530 +36442 104526 +1264 104523 +16069 104515 +49743 104496 +32743 104485 +27585 104484 +33943 104479 +23229 104478 +30469 104472 +8337 104465 +18247 104461 +25363 104461 +22649 104458 +3969 104437 +23923 104424 +31019 104413 +4967 104390 +17778 104379 +27399 104375 +29209 104367 +13865 104354 +35079 104350 +17544 104349 +8863 104347 +21942 104345 +9703 104340 +12853 104326 +26319 104318 +30190 104315 +23184 104285 +26147 104273 +2903 104262 +20354 104253 +18671 104253 +16800 104229 +14242 104219 +8445 104217 +24824 104211 +33874 104198 +15632 104192 +15122 104183 +22804 104182 +33831 104177 +17587 104144 +31118 104114 +17686 104108 +17348 104103 +19957 104091 +24377 104085 +33828 104081 +16382 104077 +41916 104045 +17172 104043 +1376 104042 +18002 104038 +11978 104035 +20372 104024 +18898 104014 +30884 103943 +18780 103938 +19938 103936 +23151 103928 +15990 103904 +38684 103894 +18645 103870 +30596 103865 +9092 103852 +30105 103835 +47280 103829 +12040 103816 +27428 103816 +27875 103813 +9414 103804 +33442 103802 +18339 103797 +11777 103788 +42263 103767 +43602 103760 +25880 103756 +7793 103754 +19905 103747 +6083 103746 +16049 103738 +22397 103731 +26248 103720 +21553 103708 +16577 103703 +26725 103662 +13629 103660 +23610 103655 +41158 103638 +24263 103636 +24348 103590 +40370 103580 +9729 103568 +9861 103567 +29313 103564 +25756 103550 +24169 103541 +29234 103528 +47827 103506 +13345 103491 +19470 103488 +24411 103488 +17263 103478 +27658 103467 +12089 103457 +28634 103444 +1493 103440 +21160 103432 +24157 103415 +29941 103391 +26476 103389 +23147 103388 +30359 103378 +26540 103363 +11789 103361 +11437 103354 +20346 103346 +14589 103346 +19276 103339 +3339 103311 +37142 103300 +11517 103293 +20070 103275 +5842 103264 +13431 103260 +27811 103244 +45196 103236 +11908 103232 +12728 103220 +26399 103206 +20024 103200 +16194 103196 +31105 103148 +19022 103137 +2175 103129 +17784 103128 +41976 103126 +18507 103123 +9972 103123 +2302 103102 +22307 103102 +12219 103095 +39943 103079 +33089 103052 +36577 103024 +17146 103018 +30341 103009 +42896 103003 +19870 102997 +32681 102991 +29634 102984 +44081 102977 +9529 102948 +32013 102947 +35627 102934 +37568 102931 +27401 102921 +15741 102914 +4015 102903 +8653 102895 +27157 102878 +47214 102866 +13322 102862 +41124 102858 +30679 102858 +39295 102856 +24788 102840 +17175 102840 +27426 102836 +47837 102813 +8583 102810 +21104 102782 +37112 102767 +12281 102746 +33251 102743 +45015 102705 +24267 102704 +29756 102671 +6368 102668 +10151 102659 +25488 102649 +39874 102634 +39393 102632 +30938 102631 +24056 102621 +19471 102619 +9551 102609 +6345 102608 +22063 102603 +42631 102602 +17399 102593 +3870 102589 +33577 102587 +39723 102543 +12270 102529 +15136 102525 +21122 102522 +38938 102520 +2881 102514 +3370 102507 +48286 102505 +2117 102500 +37178 102500 +15268 102482 +33185 102482 +20976 102474 +30789 102472 +34325 102467 +18781 102456 +21007 102453 +10185 102450 +14197 102440 +19560 102427 +36595 102375 +31214 102367 +4591 102346 +31790 102342 +25720 102333 +33998 102322 +25758 102308 +34955 102296 +22783 102275 +33644 102260 +46614 102258 +30537 102232 +23096 102226 +12109 102226 +23264 102218 +38465 102200 +40911 102195 +21310 102194 +44634 102193 +15476 102191 +23458 102181 +29156 102179 +16225 102168 +28648 102167 +33556 102156 +18177 102139 +7332 102129 +25401 102126 +16737 102106 +22197 102104 +20880 102095 +26395 102091 +27612 102084 +26965 102082 +30966 102073 +14666 102066 +27240 102061 +3920 102059 +34612 102023 +29497 102013 +18947 102013 +34492 102003 +32835 101996 +34806 101986 +23602 101982 +20242 101975 +5473 101972 +17169 101970 +8132 101968 +33208 101965 +17754 101957 +16600 101953 +16857 101939 +24532 101910 +49927 101896 +42177 101891 +34308 101891 +22737 101889 +22729 101888 +26890 101883 +24654 101875 +22359 101875 +1163 101864 +11081 101853 +22943 101845 +20431 101843 +32748 101835 +23967 101825 +15777 101819 +23787 101814 +23189 101797 +21545 101792 +39220 101773 +25582 101761 +8562 101754 +17311 101740 +22982 101738 +29188 101725 +18565 101715 +21435 101714 +40897 101711 +19010 101700 +33670 101699 +14366 101688 +34985 101687 +9734 101682 +14289 101662 +17779 101650 +25854 101637 +13929 101621 +28187 101620 +22187 101605 +14243 101605 +7292 101591 +17557 101589 +25927 101575 +17091 101562 +18593 101542 +30164 101540 +30008 101533 +23508 101508 +19166 101497 +18326 101451 +30979 101436 +24365 101436 +33817 101433 +29618 101426 +23708 101424 +16181 101413 +43025 101391 +3042 101389 +31463 101385 +7367 101375 +35321 101374 +35532 101370 +29468 101349 +14108 101344 +25819 101342 +26850 101315 +6883 101315 +24171 101305 +46198 101304 +37686 101297 +11670 101289 +1920 101275 +10296 101271 +25656 101262 +40730 101257 +12311 101237 +31523 101234 +20720 101219 +25217 101212 +18697 101203 +40186 101194 +14563 101183 +16322 101180 +25712 101177 +11651 101167 +22412 101167 +20825 101160 +46055 101145 +14704 101139 +21567 101129 +14835 101127 +20538 101116 +14330 101095 +20078 101079 +47677 101066 +43480 101065 +19095 101037 +11773 101021 +25162 101015 +22551 101012 +19048 101010 +23204 101006 +30933 100994 +22238 100974 +29504 100967 +21664 100965 +20699 100965 +9331 100964 +16896 100963 +24150 100958 +45726 100946 +16092 100944 +22392 100928 +27545 100921 +25510 100913 +24972 100898 +38880 100887 +26166 100877 +49251 100856 +22706 100856 +24658 100850 +23014 100847 +38507 100845 +5768 100834 +29585 100820 +22978 100773 +18902 100769 +8195 100745 +14839 100734 +3116 100728 +17032 100702 +26122 100699 +41673 100697 +15481 100667 +23034 100663 +20793 100660 +17898 100658 +13542 100658 +22669 100657 +31134 100637 +14230 100636 +16953 100622 +40419 100616 +2838 100609 +10447 100600 +18737 100583 +23018 100572 +32017 100554 +13166 100545 +17485 100536 +28893 100535 +12442 100532 +18608 100530 +14924 100523 +27555 100523 +31183 100517 +27196 100509 +35852 100507 +20192 100505 +4726 100505 +3746 100502 +15702 100496 +22498 100492 +19683 100492 +16006 100456 +39929 100453 +29764 100451 +6223 100439 +21501 100433 +43724 100430 +40458 100410 +19369 100402 +28772 100394 +8303 100394 +29545 100385 +45526 100360 +39781 100356 +17085 100353 +30115 100343 +46965 100335 +9785 100334 +9766 100314 +33324 100280 +16046 100274 +5807 100274 +28528 100268 +21098 100265 +38762 100258 +10161 100256 +39268 100256 +36194 100251 +24607 100247 +29170 100235 +28800 100226 +2991 100180 +28515 100159 +26862 100146 +1316 100144 +23796 100138 +11046 100136 +19337 100127 +12608 100126 +7376 100114 +29732 100107 +17567 100098 +20991 100094 +12636 100093 +24130 100092 +45130 100092 +984 100092 +13851 100071 +33452 100071 +39410 100054 +22259 100046 +9967 100038 +30593 100023 +27548 100018 +10680 100008 +39030 100008 +29384 99993 +17724 99992 +25358 99984 +21080 99981 +3857 99981 +32719 99971 +7649 99960 +32787 99958 +36388 99950 +43708 99940 +28635 99937 +32705 99930 +13254 99925 +30776 99923 +35481 99922 +17234 99912 +15119 99899 +34908 99861 +13805 99861 +33609 99851 +22677 99843 +18276 99837 +26183 99827 +15309 99810 +20263 99799 +41405 99793 +18512 99793 +16839 99779 +13545 99766 +19239 99755 +41800 99754 +17862 99754 +1794 99749 +21647 99737 +39964 99736 +23383 99723 +31908 99720 +25999 99720 +11308 99720 +35956 99695 +5105 99693 +26207 99660 +30783 99655 +24450 99653 +27343 99648 +24842 99644 +24347 99634 +18954 99625 +20820 99621 +23461 99614 +29798 99593 +10279 99589 +25605 99582 +19910 99562 +26775 99561 +8597 99558 +14574 99545 +6269 99520 +22321 99518 +14195 99510 +48148 99507 +32098 99474 +23316 99455 +12340 99450 +26786 99446 +17572 99423 +19401 99415 +45395 99405 +2733 99392 +26670 99378 +31826 99377 +18630 99373 +22505 99372 +37388 99371 +17978 99364 +12366 99357 +18297 99352 +5315 99352 +23618 99350 +21235 99343 +13898 99335 +26345 99331 +16236 99327 +39140 99323 +38772 99314 +28038 99308 +16121 99306 +18975 99302 +23309 99263 +12434 99260 +19096 99255 +29297 99240 +31370 99229 +19318 99226 +12403 99216 +29721 99214 +14251 99213 +19268 99200 +34705 99187 +11999 99183 +13526 99183 +16717 99175 +26103 99152 +21883 99148 +19288 99131 +18953 99126 +20039 99125 +31833 99122 +23245 99121 +22692 99118 +29532 99112 +39810 99072 +21810 99069 +17640 99069 +34707 99067 +30489 99055 +33478 99054 +18229 99034 +28651 99027 +34930 99027 +26530 99022 +26555 99020 +43305 99001 +28444 99001 +38204 98999 +24638 98996 +22526 98983 +14796 98983 +19229 98967 +30399 98959 +18460 98956 +23099 98954 +17543 98953 +14627 98946 +27539 98941 +48023 98930 +27725 98918 +10508 98910 +24166 98907 +30654 98895 +27382 98894 +19321 98893 +16123 98889 +22644 98882 +17430 98881 +19605 98878 +27806 98875 +19123 98875 +23207 98824 +25987 98811 +26048 98805 +35801 98804 +11348 98796 +10944 98793 +10117 98787 +15104 98780 +25724 98770 +47804 98760 +37903 98756 +14299 98745 +22896 98739 +26211 98735 +9649 98716 +19604 98712 +14056 98710 +18370 98651 +35434 98644 +238 98630 +47317 98624 +20843 98621 +32987 98612 +34880 98609 +15387 98604 +7190 98579 +32370 98579 +22977 98578 +20611 98567 +33905 98561 +2036 98556 +31057 98553 +43265 98550 +45870 98549 +7217 98549 +16113 98546 +33997 98528 +11251 98527 +42484 98525 +18336 98522 +26409 98518 +19035 98512 +47251 98487 +15548 98478 +3630 98467 +21258 98454 +23730 98450 +16613 98449 +10793 98428 +26443 98414 +14570 98405 +4402 98405 +32364 98397 +30091 98392 +40944 98386 +12277 98362 +20947 98360 +23148 98355 +21135 98351 +42745 98344 +25185 98339 +7991 98331 +10265 98313 +28901 98292 +4593 98291 +33261 98290 +24644 98283 +13626 98282 +18587 98281 +11502 98276 +31437 98267 +26622 98265 +21764 98242 +45408 98214 +32330 98212 +15639 98198 +16777 98196 +27440 98189 +38278 98169 +24395 98167 +25051 98164 +39588 98158 +31596 98157 +32536 98133 +24558 98122 +12765 98117 +28194 98102 +28437 98098 +24955 98087 +44211 98086 +41196 98079 +37536 98069 +21635 98058 +3628 98053 +48931 98053 +15377 98041 +10771 98034 +28118 98030 +32443 98013 +21923 98008 +23916 98006 +25718 98005 +16687 98002 +16215 98001 +46819 97997 +11404 97997 +3445 97990 +15750 97970 +8623 97967 +28892 97960 +19508 97960 +13231 97943 +42883 97942 +10705 97939 +28061 97919 +29014 97915 +14887 97908 +31974 97903 +6855 97903 +17765 97903 +14155 97900 +16614 97893 +26117 97891 +37154 97883 +30890 97880 +26746 97874 +23335 97858 +10022 97838 +32020 97828 +12678 97827 +45810 97821 +28752 97795 +30996 97783 +31949 97779 +12544 97767 +39143 97757 +25967 97745 +16418 97741 +19064 97735 +30612 97693 +31357 97686 +29529 97682 +11927 97679 +17031 97679 +32597 97675 +29357 97674 +10414 97665 +45051 97663 +34827 97662 +40197 97660 +25017 97655 +13555 97649 +18288 97648 +7619 97621 +14612 97621 +12456 97621 +24723 97618 +31661 97616 +5900 97605 +32183 97601 +18026 97599 +42693 97599 +16531 97596 +26520 97586 +29667 97584 +29596 97581 +19723 97575 +27439 97575 +30757 97553 +31473 97552 +26082 97544 +26328 97542 +20908 97540 +37085 97527 +18121 97515 +33140 97514 +15633 97513 +27396 97512 +19556 97510 +22030 97498 +22888 97492 +11014 97487 +20321 97485 +33685 97481 +28744 97463 +34068 97455 +18539 97452 +15663 97451 +16731 97434 +22147 97423 +23949 97421 +28514 97417 +49173 97414 +28193 97385 +22329 97366 +25659 97354 +41035 97343 +26860 97342 +44369 97324 +39452 97317 +12371 97315 +17300 97313 +40449 97290 +20410 97288 +17145 97282 +22710 97279 +40195 97271 +19552 97265 +4246 97248 +21815 97245 +33515 97229 +31558 97228 +22060 97228 +39802 97223 +16885 97204 +43004 97203 +6701 97196 +28076 97186 +25734 97186 +35807 97182 +20571 97179 +19187 97172 +11986 97157 +11822 97156 +22295 97144 +30236 97137 +30140 97126 +14301 97108 +21085 97100 +22870 97092 +28949 97089 +23327 97080 +14927 97073 +24892 97071 +9078 97070 +25779 97053 +17080 97048 +22181 97048 +24805 97047 +48206 97042 +14014 97036 +17694 97032 +28853 97030 +25223 97019 +21092 97015 +8042 97008 +14898 97007 +34681 97005 +18594 97001 +18834 96999 +46499 96989 +18176 96985 +45212 96983 +19922 96982 +44726 96970 +4126 96968 +34755 96962 +24784 96957 +25428 96943 +48460 96934 +40388 96923 +2825 96911 +21686 96907 +32265 96898 +23301 96894 +11862 96892 +23299 96891 +21124 96879 +5106 96872 +38276 96871 +19144 96865 +26197 96858 +17511 96854 +17531 96848 +16312 96845 +15053 96835 +16716 96830 +8421 96817 +18543 96812 +18118 96805 +32615 96803 +18411 96789 +3076 96780 +20888 96775 +19459 96769 +6018 96764 +24495 96759 +22051 96753 +26299 96752 +4604 96742 +23071 96741 +1106 96729 +34327 96727 +38619 96710 +38740 96708 +43237 96702 +46097 96692 +35234 96689 +19468 96678 +8923 96678 +30785 96671 +33750 96670 +24514 96666 +17207 96649 +20420 96646 +19433 96645 +19884 96635 +24350 96635 +6661 96624 +23165 96608 +20254 96601 +33094 96592 +22979 96591 +22623 96590 +23160 96579 +13522 96578 +9203 96556 +24048 96555 +9971 96528 +10780 96521 +34701 96520 +25863 96507 +36231 96504 +24942 96501 +34060 96479 +20040 96463 +14392 96453 +29015 96451 +9696 96441 +30257 96436 +31784 96430 +13543 96429 +19267 96422 +47967 96420 +13826 96420 +36889 96417 +4798 96406 +34440 96405 +25483 96376 +19133 96356 +34780 96356 +41348 96344 +33718 96343 +10378 96333 +17486 96328 +13309 96326 +9192 96311 +21754 96301 +29136 96290 +27278 96290 +26605 96273 +31269 96262 +26843 96260 +29815 96243 +4862 96231 +21219 96219 +20787 96213 +2856 96205 +33130 96196 +28445 96191 +22604 96180 +27340 96178 +15544 96168 +25115 96151 +19165 96145 +10007 96142 +28036 96135 +20408 96116 +2725 96115 +22989 96083 +19143 96079 +27264 96072 +26077 96072 +9632 96053 +4063 96036 +24089 96036 +29239 96032 +23273 96026 +14661 96021 +35415 96012 +3043 96011 +27567 96010 +39447 96010 +9069 95993 +31171 95988 +27353 95985 +32137 95980 +40695 95952 +20750 95941 +40222 95937 +21100 95922 +21558 95913 +32410 95910 +27177 95908 +20056 95890 +31220 95888 +13905 95883 +35274 95871 +22668 95857 +18275 95855 +26594 95835 +25786 95822 +14564 95821 +27308 95810 +24281 95805 +19897 95803 +32490 95796 +14948 95762 +2017 95741 +19713 95740 +24387 95733 +25075 95724 +22474 95724 +18831 95719 +34886 95717 +27425 95714 +13315 95698 +28492 95692 +48395 95686 +30652 95683 +30651 95671 +39361 95668 +41114 95663 +35121 95658 +23867 95651 +31981 95650 +2021 95649 +21662 95645 +16632 95638 +25377 95637 +21461 95633 +28120 95633 +25646 95620 +16588 95617 +36799 95615 +9883 95607 +20925 95592 +18511 95588 +44168 95587 +39672 95576 +21164 95565 +18027 95562 +33972 95558 +31637 95548 +22365 95536 +35336 95531 +8825 95523 +22339 95522 +7608 95521 +24947 95515 +14411 95506 +1307 95505 +16989 95486 +18616 95482 +33902 95473 +25409 95465 +29488 95452 +21552 95448 +7474 95436 +32605 95425 +33147 95421 +15304 95409 +39124 95402 +30970 95402 +33859 95401 +21851 95396 +12589 95395 +25507 95358 +24262 95358 +24014 95356 +46366 95351 +4107 95324 +11491 95315 +17747 95299 +22582 95286 +33405 95282 +37066 95272 +38131 95270 +16264 95267 +16649 95265 +25729 95254 +15174 95230 +16509 95221 +23879 95218 +41688 95215 +29658 95204 +8356 95193 +17917 95190 +33352 95171 +31735 95167 +35983 95166 +8812 95147 +12196 95147 +10837 95146 +9873 95136 +20935 95134 +7469 95114 +14751 95109 +21984 95097 +19018 95075 +32714 95065 +29809 95057 +33777 95057 +25376 95057 +32678 95048 +17182 95037 +46281 95032 +39245 95014 +25303 95012 +37487 94994 +42748 94986 +31341 94966 +19923 94954 +33356 94942 +16917 94935 +1704 94933 +20299 94931 +29782 94925 +24066 94922 +20664 94915 +17973 94910 +10516 94908 +19701 94899 +9395 94892 +9346 94887 +36715 94880 +22912 94880 +21318 94874 +10788 94873 +31656 94867 +21223 94849 +34962 94832 +34462 94830 +12233 94826 +13548 94793 +36636 94775 +12202 94762 +26784 94758 +6421 94752 +27731 94750 +34710 94739 +32540 94736 +3057 94727 +11024 94710 +31565 94694 +49281 94689 +19001 94683 +31678 94677 +6545 94676 +35687 94673 +26316 94633 +21134 94630 +34050 94630 +18023 94626 +44572 94623 +44781 94623 +8743 94596 +7807 94590 +5633 94589 +13587 94580 +36978 94574 +23122 94574 +12341 94564 +36002 94562 +18142 94560 +47678 94555 +13018 94537 +27356 94533 +9937 94525 +682 94499 +21640 94495 +16243 94465 +36503 94462 +47660 94450 +33503 94441 +28303 94435 +4749 94429 +32831 94420 +8530 94420 +26487 94418 +18469 94417 +36251 94414 +29125 94408 +14091 94399 +25494 94399 +25455 94377 +14331 94376 +12248 94351 +6750 94335 +18808 94333 +21490 94320 +22733 94314 +11696 94312 +23149 94305 +17241 94299 +22185 94288 +28814 94273 +26502 94273 +14272 94272 +14700 94259 +24768 94253 +13481 94244 +6014 94234 +30279 94231 +29686 94225 +6703 94216 +17306 94216 +3323 94178 +14515 94178 +46820 94167 +3005 94160 +18364 94150 +29662 94130 +29208 94129 +10036 94128 +26253 94128 +28294 94121 +39728 94102 +18480 94091 +27821 94088 +18420 94085 +9855 94077 +41260 94067 +6897 94063 +32951 94062 +10444 94043 +13180 94043 +16619 94037 +26113 94031 +2064 94029 +16308 94028 +37401 94028 +28094 94019 +39125 94017 +27373 94015 +11775 94014 +22378 94008 +16741 94001 +16630 93994 +33984 93982 +22609 93963 +18948 93957 +13282 93949 +44323 93934 +13464 93932 +12354 93930 +14362 93923 +20328 93920 +25234 93919 +33527 93918 +15889 93906 +21021 93906 +43870 93903 +26310 93900 +12488 93900 +11855 93898 +39039 93895 +16374 93866 +26002 93859 +16744 93856 +14856 93839 +14613 93838 +17500 93826 +15323 93817 +21366 93803 +22626 93795 +15887 93795 +31704 93784 +22529 93784 +44317 93779 +40057 93769 +19985 93767 +801 93764 +27839 93746 +25263 93723 +32253 93722 +23905 93721 +13443 93719 +25281 93709 +24623 93701 +36421 93695 +22007 93687 +46306 93686 +32853 93681 +25170 93666 +15209 93656 +24677 93641 +17575 93638 +11331 93636 +1172 93625 +38188 93622 +31099 93611 +40445 93606 +27321 93601 +24230 93597 +13199 93586 +18320 93570 +18025 93568 +28996 93564 +37033 93563 +15784 93561 +21639 93553 +30828 93548 +30618 93537 +43245 93534 +31201 93532 +19024 93527 +30456 93527 +2905 93518 +8007 93500 +20265 93497 +22125 93482 +1733 93482 +39508 93474 +19844 93473 +21127 93472 +43513 93459 +35689 93452 +37907 93450 +1562 93429 +5152 93413 +33519 93375 +12026 93372 +22085 93372 +19493 93369 +6713 93365 +19504 93352 +24683 93347 +27297 93340 +34906 93338 +33458 93338 +24006 93338 +20066 93336 +23689 93335 +30286 93335 +28499 93334 +13716 93325 +17873 93324 +44502 93309 +4370 93305 +31136 93289 +10981 93284 +38881 93278 +18548 93272 +10295 93266 +50048 93265 +24391 93248 +4489 93237 +11248 93237 +17852 93233 +11902 93222 +6919 93198 +42100 93192 +26102 93178 +20680 93163 +46719 93161 +26581 93156 +20054 93155 +26906 93147 +22385 93113 +8525 93108 +23746 93103 +10212 93102 +30839 93100 +23205 93097 +19294 93086 +24918 93085 +32938 93085 +20640 93076 +667 93075 +3004 93074 +36384 93063 +15255 93058 +30600 93056 +18272 93046 +20588 93041 +29483 93036 +2309 93024 +5132 93015 +40318 92991 +8931 92991 +25620 92989 +44075 92987 +8764 92985 +25305 92983 +25277 92980 +14879 92973 +29986 92951 +16876 92946 +44041 92944 +27571 92918 +10740 92916 +7918 92914 +36516 92907 +26548 92906 +27060 92905 +9532 92905 +23095 92897 +37490 92887 +35493 92884 +14686 92881 +34593 92863 +23057 92846 +46917 92840 +20042 92840 +29070 92838 +7633 92830 +12849 92825 +1490 92822 +45303 92813 +25860 92800 +36832 92793 +26916 92791 +22019 92786 +35591 92785 +31065 92776 +3927 92775 +41164 92760 +10459 92752 +23477 92737 +26057 92730 +29210 92720 +10973 92712 +5232 92711 +24359 92705 +16766 92705 +36812 92702 +17347 92685 +32413 92675 +19201 92674 +11887 92664 +35017 92664 +28646 92661 +10418 92641 +49762 92640 +27197 92636 +19882 92616 +23453 92613 +48607 92600 +19175 92599 +26727 92597 +5102 92590 +25629 92577 +252 92571 +5028 92570 +19845 92565 +6436 92559 +36611 92554 +40021 92550 +23968 92549 +13909 92544 +25339 92539 +11980 92536 +13227 92534 +28170 92528 +26645 92527 +42499 92522 +18043 92508 +16729 92503 +12106 92502 +22299 92483 +25937 92475 +31233 92465 +15568 92444 +17090 92443 +21051 92440 +28591 92440 +6171 92411 +26655 92397 +24584 92386 +14314 92384 +19297 92377 +12896 92376 +24433 92374 +36771 92372 +12862 92368 +21452 92367 +46459 92358 +30870 92352 +7352 92337 +7061 92328 +24325 92322 +22188 92321 +38838 92316 +16446 92309 +23557 92299 +18643 92298 +10916 92283 +35078 92260 +7322 92256 +27822 92251 +24656 92247 +18463 92237 +39661 92227 +9064 92217 +20278 92214 +9496 92212 +34881 92210 +23074 92204 +25449 92191 +19883 92186 +24219 92184 +12394 92182 +3652 92170 +6683 92164 +42630 92161 +22574 92161 +26610 92156 +22106 92156 +18059 92142 +29654 92139 +22206 92133 +30325 92133 +25317 92121 +39309 92115 +10527 92106 +28393 92100 +23056 92099 +42150 92097 +21216 92095 +26800 92087 +17150 92081 +38478 92078 +20399 92072 +8048 92063 +27138 92063 +33892 92057 +7642 92046 +43326 92046 +14573 92043 +29786 92038 +11280 92028 +27662 92026 +35646 92025 +19102 92023 +7928 92018 +41345 92013 +19440 91973 +1093 91965 +7836 91956 +34405 91939 +23239 91929 +35497 91928 +6424 91916 +23217 91909 +19838 91909 +42350 91901 +26334 91895 +35816 91894 +20269 91892 +30718 91879 +37969 91869 +27397 91860 +34077 91858 +14151 91858 +9501 91855 +22423 91839 +11519 91834 +30462 91831 +14068 91822 +24402 91813 +10865 91811 +40132 91800 +48858 91799 +18221 91797 +12417 91795 +19862 91786 +46381 91781 +46260 91774 +6456 91765 +33318 91757 +25116 91754 +5200 91737 +21294 91735 +36904 91733 +20067 91721 +16288 91712 +16170 91709 +36574 91700 +30678 91699 +30437 91688 +20326 91683 +21814 91672 +45963 91662 +32659 91636 +41845 91629 +26938 91628 +20383 91621 +27957 91620 +28662 91619 +32979 91615 +30570 91607 +32002 91591 +26565 91588 +9827 91567 +34915 91566 +30012 91562 +34686 91552 +23084 91543 +17555 91541 +45229 91527 +34696 91525 +21256 91522 +4682 91516 +39303 91511 +34422 91500 +13958 91496 +22219 91493 +46422 91471 +32526 91470 +47775 91463 +6040 91460 +17909 91457 +13216 91449 +12796 91444 +27907 91443 +25173 91430 +42286 91423 +20693 91419 +16866 91415 +22765 91411 +11498 91403 +8677 91396 +8394 91383 +22730 91375 +18783 91374 +40474 91368 +29753 91348 +22889 91344 +24337 91343 +41525 91340 +22949 91334 +31890 91331 +23699 91322 +25971 91308 +26777 91308 +11291 91307 +7368 91305 +10798 91299 +18478 91272 +16641 91269 +12609 91244 +28983 91230 +15824 91227 +12562 91224 +12060 91223 +16369 91222 +23353 91220 +28411 91218 +21661 91211 +24569 91210 +20009 91210 +45371 91202 +25861 91202 +34474 91200 +25984 91189 +21879 91170 +10910 91168 +13834 91159 +19343 91156 +36588 91150 +20037 91146 +33749 91143 +22625 91131 +2509 91128 +34732 91098 +30880 91097 +25725 91070 +14821 91063 +19547 91062 +21674 91042 +31158 91034 +10028 91033 +30043 91027 +14426 91025 +17219 91024 +30046 91023 +25403 91020 +32342 91016 +39265 91015 +43664 91003 +27840 91001 +29359 90985 +19387 90978 +10505 90960 +21621 90959 +33242 90945 +13946 90944 +21915 90942 +24635 90941 +16132 90938 +9702 90936 +29489 90927 +3468 90926 +21649 90913 +20958 90900 +25923 90899 +15992 90897 +35633 90887 +6726 90881 +21971 90869 +29628 90833 +14731 90827 +44025 90826 +26695 90806 +20460 90802 +42185 90800 +35165 90794 +46561 90793 +20744 90788 +29475 90778 +8375 90769 +20905 90760 +11224 90757 +14805 90754 +20445 90749 +23401 90746 +25355 90745 +25154 90744 +11163 90741 +27523 90734 +37387 90731 +40412 90730 +3534 90717 +41346 90716 +25349 90716 +23348 90716 +20060 90705 +6571 90703 +31367 90700 +30926 90696 +24950 90694 +24346 90689 +24489 90677 +36029 90667 +12025 90667 +13058 90666 +9191 90664 +34051 90653 +16809 90649 +19322 90646 +21392 90646 +14701 90631 +17851 90631 +40644 90629 +11146 90629 +32632 90625 +12001 90620 +14874 90605 +47966 90604 +26593 90566 +24484 90565 +40288 90564 +21682 90555 +35671 90544 +39927 90542 +13569 90541 +32832 90540 +28661 90540 +6818 90494 +14843 90492 +28157 90474 +46832 90471 +22776 90466 +11814 90459 +34676 90459 +27325 90455 +35564 90433 +36985 90432 +19446 90431 +17246 90424 +14005 90400 +22332 90391 +12867 90389 +15919 90389 +28534 90385 +3054 90384 +27123 90380 +6179 90369 +15027 90348 +37874 90337 +17512 90325 +19221 90317 +26187 90309 +42284 90307 +12050 90272 +29741 90260 +13992 90260 +44340 90258 +9123 90249 +31429 90243 +9094 90233 +6351 90232 +23167 90231 +31355 90228 +21486 90212 +7659 90210 +21438 90195 +38428 90192 +30231 90185 +35874 90179 +42873 90167 +23403 90166 +19704 90164 +29570 90155 +4808 90144 +28464 90134 +30307 90129 +16603 90115 +29060 90084 +38774 90082 +7230 90081 +27075 90071 +36467 90067 +21429 90066 +20128 90058 +29459 90052 +12782 90047 +30153 90043 +25930 90039 +17738 90032 +27449 90015 +22054 90013 +21611 90009 +30305 90007 +28088 90001 +24497 89973 +18485 89969 +10080 89960 +20618 89956 +15197 89950 +33286 89941 +13083 89940 +15492 89935 +6237 89910 +99 89908 +25331 89905 +27521 89900 +30700 89889 +21094 89876 +30706 89862 +32486 89832 +34584 89809 +24903 89790 +19528 89789 +11912 89787 +24375 89784 +17102 89776 +44452 89773 +19503 89772 +41900 89768 +35896 89762 +16900 89760 +18972 89751 +42052 89745 +1887 89743 +26924 89735 +19185 89734 +20795 89727 +6024 89723 +18952 89721 +43791 89710 +34766 89704 +34138 89697 +38018 89666 +37619 89649 +10235 89647 +33072 89646 +27529 89644 +4464 89642 +28558 89637 +34017 89631 +16405 89630 +17672 89628 +37123 89621 +35047 89619 +29362 89618 +34895 89610 +32982 89591 +14562 89589 +28488 89587 +22756 89582 +37584 89569 +18959 89566 +24599 89549 +29118 89544 +42208 89537 +11567 89483 +28984 89465 +31231 89449 +22129 89445 +39739 89445 +26740 89444 +24303 89429 +36025 89413 +6442 89413 +20038 89409 +42682 89400 +25794 89392 +25056 89364 +19813 89364 +10316 89363 +22020 89356 +26762 89352 +19893 89352 +23503 89344 +11489 89341 +26296 89337 +1217 89335 +19762 89326 +18596 89322 +26387 89322 +26871 89320 +4500 89317 +10977 89305 +40378 89302 +14507 89298 +27259 89297 +9347 89282 +37690 89276 +37090 89264 +27602 89254 +20435 89238 +9744 89232 +33190 89219 +18814 89214 +33886 89204 +18529 89202 +17047 89198 +15176 89195 +15354 89191 +20050 89188 +22554 89182 +20767 89175 +36633 89171 +25107 89167 +28823 89141 +19555 89120 +12599 89115 +37217 89112 +22331 89107 +20687 89097 +36069 89093 +34601 89090 +15735 89076 +20950 89069 +43174 89068 +12130 89067 +31402 89064 +20982 89060 +34281 89046 +44467 89038 +16772 89035 +35141 89033 +14062 89029 +28176 89026 +16186 89018 +13269 89016 +41826 89014 +30883 89010 +23987 89009 +36429 88992 +16439 88987 +10211 88981 +27508 88970 +37940 88969 +28999 88958 +26772 88951 +29816 88945 +38617 88939 +26311 88925 +20923 88924 +39113 88915 +41782 88906 +17669 88900 +24632 88869 +23221 88864 +32414 88862 +27015 88857 +9760 88848 +17000 88841 +28224 88835 +26527 88835 +31920 88827 +8051 88783 +37519 88779 +15180 88745 +26930 88742 +12543 88731 +29383 88705 +26553 88695 +33014 88691 +48812 88689 +19962 88667 +41904 88658 +40974 88658 +27980 88656 +41703 88655 +1179 88655 +48822 88632 +23027 88622 +41005 88616 +10137 88611 +36971 88610 +39045 88607 +43126 88606 +20625 88601 +35692 88599 +13248 88586 +49003 88585 +30848 88577 +38317 88556 +20339 88552 +17682 88545 +31683 88530 +18586 88516 +25407 88513 +36590 88511 +20673 88510 +13217 88506 +24374 88506 +23749 88504 +15212 88500 +17284 88500 +20313 88499 +32241 88490 +17221 88486 +13809 88465 +19434 88465 +38522 88456 +32438 88443 +21431 88423 +31213 88420 +32174 88415 +24685 88383 +15355 88376 +18581 88355 +13380 88347 +12391 88343 +23934 88340 +23714 88328 +16221 88324 +10521 88319 +27624 88319 +28803 88308 +41905 88301 +12204 88288 +25588 88280 +15056 88278 +30722 88274 +21787 88272 +29043 88266 +22987 88259 +49183 88255 +13617 88253 +26816 88252 +7070 88251 +21334 88247 +18628 88212 +3133 88198 +24949 88197 +21082 88190 +4582 88188 +38758 88184 +19585 88179 +44535 88175 +35982 88163 +17480 88160 +20145 88159 +36245 88154 +15165 88151 +115 88146 +2016 88119 +34317 88106 +19186 88084 +7755 88080 +16859 88067 +18631 88056 +15420 88053 +2369 88051 +32601 88050 +27095 88048 +22837 88047 +17707 88046 +26149 88041 +30920 88040 +16064 88036 +9459 88029 +37332 88028 +18992 88026 +10379 88024 +17509 88021 +11203 88021 +16957 88014 +18285 88002 +27146 87994 +32176 87987 +23050 87981 +15801 87967 +39540 87967 +13652 87966 +25208 87947 +42605 87943 +33613 87937 +48955 87926 +11933 87924 +33375 87916 +29505 87903 +31893 87902 +27341 87889 +13848 87889 +16513 87886 +21546 87885 +22486 87883 +26898 87878 +34380 87877 +42065 87875 +28336 87874 +34136 87870 +36718 87867 +22875 87867 +23083 87850 +40766 87848 +23332 87846 +28180 87823 +19020 87823 +37685 87821 +21387 87810 +41137 87797 +20088 87795 +28431 87790 +46287 87790 +19710 87785 +4125 87779 +14356 87778 +3600 87767 +48125 87766 +17650 87762 +28775 87741 +33945 87736 +26973 87730 +30441 87726 +41578 87725 +12461 87723 +25359 87712 +45961 87709 +31618 87691 +44303 87688 +23456 87686 +9538 87672 +40782 87672 +9630 87667 +16790 87642 +14309 87637 +25290 87637 +4072 87629 +26944 87622 +18063 87615 +17299 87603 +21131 87601 +25891 87597 +49657 87597 +34916 87591 +14642 87588 +25845 87571 +38330 87569 +40833 87560 +20137 87551 +15906 87546 +22459 87541 +44824 87541 +45034 87527 +22513 87527 +6676 87523 +35917 87515 +29727 87509 +32099 87508 +5352 87502 +20439 87483 +17059 87474 +22350 87468 +39727 87467 +33742 87456 +29832 87455 +25168 87454 +24160 87451 +32919 87428 +20391 87422 +4029 87420 +23386 87417 +35062 87416 +27699 87413 +15601 87411 +11922 87411 +20466 87406 +14025 87405 +19072 87404 +30784 87397 +1113 87395 +19686 87389 +26671 87386 +8455 87381 +38442 87381 +36443 87363 +20667 87362 +19407 87355 +13483 87353 +41742 87350 +116 87319 +30174 87314 +29195 87309 +40214 87303 +7065 87302 +37756 87299 +27144 87283 +13251 87280 +32733 87276 +20080 87269 +12915 87263 +19979 87254 +25571 87253 +40735 87247 +5216 87243 +30925 87241 +23972 87238 +11979 87218 +24092 87199 +22570 87199 +32552 87193 +42644 87190 +29775 87186 +21929 87183 +33218 87179 +24749 87172 +19098 87166 +21156 87163 +21784 87162 +13277 87156 +23996 87153 +19766 87145 +21226 87139 +28160 87137 +31926 87128 +19907 87119 +30512 87116 +45446 87114 +4629 87109 +23427 87108 +18637 87107 +19878 87105 +8736 87087 +46552 87080 +16345 87068 +25351 87060 +4355 87050 +23505 87043 +41182 87032 +4124 87020 +22338 87016 +19812 87009 +21115 87005 +48855 87005 +31273 87005 +23422 87003 +19390 86988 +27768 86982 +21698 86976 +23116 86962 +20007 86959 +25566 86954 +21385 86952 +19536 86949 +8792 86943 +17968 86920 +8174 86919 +38536 86905 +6063 86901 +20703 86898 +36810 86894 +28126 86890 +16789 86885 +21853 86881 +30188 86871 +39651 86864 +33521 86862 +18182 86794 +10486 86793 +14454 86792 +44311 86779 +22363 86773 +21367 86765 +6944 86763 +6352 86755 +7155 86746 +40902 86745 +22000 86734 +33659 86734 +26157 86731 +17216 86730 +39734 86729 +18620 86726 +25273 86720 +5596 86708 +11743 86698 +29023 86698 +7059 86696 +41078 86692 +47249 86684 +21732 86675 +34899 86673 +28020 86666 +14319 86663 +38403 86650 +40208 86648 +34767 86648 +15974 86641 +16250 86637 +34662 86634 +47089 86629 +38928 86617 +30941 86595 +3374 86590 +21178 86585 +17546 86574 +11074 86569 +22287 86567 +35979 86561 +5681 86552 +9200 86549 +34914 86544 +20256 86537 +20149 86534 +20064 86533 +19506 86528 +38433 86524 +18437 86523 +31239 86522 +17710 86521 +6587 86521 +22034 86519 +35105 86513 +20411 86510 +29025 86498 +10517 86485 +20479 86485 +46950 86480 +14453 86465 +24381 86458 +12913 86443 +16285 86440 +43248 86430 +16810 86427 +31863 86426 +27307 86426 +37956 86416 +19863 86405 +31742 86403 +24211 86401 +19733 86398 +33164 86396 +21854 86393 +11267 86390 +18102 86389 +29413 86385 +24621 86382 +42700 86378 +14848 86378 +2287 86373 +12466 86365 +22999 86360 +23665 86352 +22254 86342 +6039 86338 +50158 86337 +35805 86333 +35593 86330 +21882 86328 +42168 86324 +24406 86311 +15688 86305 +16227 86302 +40669 86301 +9700 86300 +29317 86295 +40233 86280 +31865 86275 +10466 86256 +17200 86255 +19607 86252 +14513 86248 +24659 86243 +19861 86240 +6658 86221 +48297 86217 +3532 86216 +14981 86192 +18710 86180 +24775 86178 +32456 86177 +15111 86175 +35680 86154 +39995 86152 +4565 86145 +30006 86119 +19342 86114 +22010 86111 +47594 86102 +18634 86096 +46187 86094 +33337 86078 +41145 86076 +23171 86067 +23992 86054 +32405 86043 +23978 86038 +23152 86035 +42602 86035 +13991 86026 +13360 86023 +44481 86020 +43586 85995 +30985 85990 +21779 85988 +36647 85984 +32145 85984 +19091 85979 +20723 85970 +10819 85966 +23173 85955 +25124 85952 +12423 85949 +18851 85940 +33664 85931 +16583 85923 +20063 85918 +15029 85918 +41839 85915 +41960 85914 +24998 85901 +17697 85901 +27355 85892 +6313 85891 +44749 85880 +17232 85876 +33514 85867 +16453 85860 +10407 85858 +14350 85856 +22849 85835 +38665 85818 +27973 85810 +28077 85807 +19341 85804 +34682 85799 +44170 85798 +20592 85797 +21079 85791 +47538 85790 +48848 85782 +24404 85771 +37003 85769 +12363 85756 +49315 85752 +32619 85743 +11201 85714 +37006 85709 +6794 85702 +19698 85702 +21391 85696 +39282 85682 +32175 85666 +21375 85666 +13152 85656 +24435 85645 +27618 85634 +17065 85633 +18659 85631 +26374 85628 +20514 85605 +20823 85599 +23077 85598 +27213 85593 +22289 85585 +44359 85571 +25909 85565 +24187 85554 +16638 85537 +1713 85534 +19409 85527 +30496 85519 +24352 85508 +26428 85501 +13259 85501 +17157 85501 +33814 85500 +40777 85492 +21214 85486 +17520 85481 +32716 85466 +22139 85463 +47574 85451 +1300 85447 +22072 85440 +12753 85438 +20584 85429 +43784 85427 +31225 85421 +13920 85414 +26312 85404 +17242 85403 +11438 85403 +14936 85397 +23202 85390 +46185 85375 +33590 85372 +15711 85369 +34041 85365 +33579 85355 +16102 85354 +17545 85354 +42766 85336 +29679 85326 +10223 85325 +38552 85319 +20558 85307 +17395 85297 +39086 85295 +44298 85291 +2360 85279 +42086 85229 +38343 85228 +29949 85222 +32961 85219 +29445 85218 +15587 85208 +19208 85208 +28213 85206 +28099 85203 +41505 85202 +24205 85202 +43596 85194 +15780 85194 +17438 85184 +20416 85168 +13631 85165 +27437 85162 +35663 85144 +39017 85136 +22714 85100 +36976 85098 +18013 85096 +36591 85091 +28438 85079 +24097 85067 +35386 85055 +7877 85048 +31483 85027 +24451 85024 +43002 85016 +23701 85016 +33347 84993 +41200 84983 +18927 84979 +20734 84970 +22959 84955 +18689 84945 +18723 84939 +21822 84921 +31840 84912 +26262 84907 +34233 84900 +38118 84899 +34118 84893 +4954 84883 +30845 84882 +28049 84881 +25044 84875 +36646 84852 +28943 84843 +42502 84843 +15410 84842 +31088 84829 +10475 84828 +26472 84822 +22546 84798 +25243 84792 +20428 84780 +41438 84769 +20089 84735 +20708 84727 +36229 84724 +47430 84724 +28595 84720 +32280 84715 +49220 84709 +10885 84706 +23757 84702 +6169 84700 +41406 84691 +19078 84686 +24694 84679 +26378 84668 +24636 84664 +45265 84637 +2517 84635 +32314 84617 +16635 84612 +41695 84611 +22562 84610 +19856 84609 +18440 84601 +11222 84597 +14529 84591 +20770 84589 +32260 84583 +20132 84581 +42037 84578 +16189 84555 +27876 84552 +20122 84548 +35181 84541 +44113 84540 +19921 84539 +24751 84537 +13144 84535 +23373 84533 +27304 84530 +19565 84526 +8040 84520 +29324 84519 +48880 84511 +35397 84505 +34545 84501 +15611 84497 +38701 84493 +20396 84487 +5029 84481 +14866 84476 +35766 84462 +38623 84462 +16086 84456 +25569 84448 +42649 84440 +33195 84424 +10104 84420 +13713 84404 +24795 84401 +35034 84400 +39503 84396 +32022 84377 +14705 84368 +40993 84365 +19361 84360 +23436 84357 +14584 84352 +25027 84340 +49722 84326 +31067 84323 +40679 84318 +24592 84293 +30263 84286 +33881 84282 +29612 84277 +31812 84264 +26643 84257 +2600 84257 +759 84240 +36418 84236 +13949 84232 +26889 84227 +22712 84221 +15982 84216 +23399 84215 +114 84202 +10734 84196 +43575 84156 +24273 84156 +16812 84155 +31853 84150 +19908 84144 +23799 84141 +19419 84118 +31252 84118 +17598 84115 +35727 84094 +45577 84092 +29144 84089 +40626 84080 +16564 84076 +2204 84075 +32116 84069 +8444 84067 +20559 84066 +5651 84061 +23860 84058 +28064 84049 +28584 84041 +27647 84029 +14557 84024 +16104 84020 +39617 84009 +24167 83997 +3193 83990 +47995 83975 +18606 83975 +11718 83972 +37919 83962 +23218 83959 +42823 83952 +29395 83950 +21128 83946 +17892 83945 +12335 83944 +35329 83941 +45093 83930 +25711 83925 +34113 83915 +16498 83910 +9278 83906 +3093 83901 +24276 83874 +25430 83867 +32924 83857 +17519 83857 +31133 83847 +16082 83843 +34235 83842 +22430 83805 +26403 83802 +17513 83793 +22451 83789 +35464 83787 +37492 83774 +35242 83765 +46335 83757 +27645 83757 +13681 83746 +29314 83746 +28890 83724 +27810 83719 +28486 83717 +35673 83715 +4078 83714 +29777 83710 +41721 83708 +27322 83705 +35338 83703 +36694 83698 +30029 83690 +30090 83688 +39830 83687 +44145 83684 +38115 83678 +7158 83677 +23985 83676 +5634 83673 +7282 83661 +27940 83657 +35670 83649 +40599 83648 +8298 83623 +34005 83610 +20130 83610 +42525 83606 +28454 83553 +43192 83552 +44603 83550 +28910 83539 +20648 83524 +18876 83523 +26356 83521 +12146 83507 +27671 83493 +49484 83488 +8089 83484 +14514 83480 +26720 83479 +30750 83476 +45814 83473 +47989 83463 +21043 83449 +42178 83443 +18388 83431 +4741 83420 +29502 83417 +26925 83403 +18964 83394 +26140 83383 +13158 83378 +39278 83375 +47534 83374 +21415 83366 +18261 83361 +26131 83360 +17048 83359 +24655 83357 +27335 83356 +23851 83353 +13585 83348 +28136 83345 +26083 83343 +21601 83343 +19621 83339 +44024 83333 +1572 83327 +7814 83325 +27251 83321 +34852 83305 +22904 83303 +21715 83295 +18669 83293 +30927 83293 +3441 83288 +24401 83274 +25635 83268 +12349 83254 +17583 83254 +21804 83252 +21726 83251 +23734 83237 +10854 83231 +39751 83228 +19541 83222 +18290 83213 +29104 83211 +43761 83209 +25406 83202 +32189 83184 +23271 83184 +43772 83146 +19499 83145 +4147 83143 +32557 83142 +7140 83136 +8291 83130 +45646 83127 +21182 83112 +25246 83099 +27069 83097 +27170 83096 +21752 83090 +8656 83090 +32620 83075 +8827 83071 +9636 83067 +18227 83050 +23000 83046 +46508 83043 +37242 83042 +16794 83011 +36385 83008 +12881 83001 +32889 82994 +37529 82991 +13014 82987 +17211 82985 +18018 82981 +4677 82980 +9650 82977 +5683 82975 +34927 82961 +25300 82960 +27141 82953 +10019 82944 +49001 82942 +8843 82933 +7371 82932 +8321 82922 +15802 82912 +28300 82907 +18506 82899 +11630 82889 +23648 82887 +37427 82882 +20610 82882 +32365 82876 +9624 82853 +46518 82849 +15473 82831 +11351 82821 +35819 82808 +13469 82804 +42379 82793 +30595 82792 +27347 82787 +16628 82779 +39542 82771 +23485 82756 +28405 82747 +39634 82742 +39151 82740 +3509 82734 +25727 82733 +22665 82730 +15950 82729 +6214 82724 +41835 82718 +8896 82715 +37866 82710 +9999 82701 +25329 82696 +11316 82687 +6192 82680 +25567 82674 +11241 82673 +19601 82670 +18346 82669 +2186 82665 +13740 82653 +20951 82649 +1469 82635 +32267 82625 +27103 82610 +35806 82602 +25023 82588 +30198 82585 +3906 82560 +17202 82555 +13719 82553 +40209 82550 +19408 82538 +20385 82535 +21514 82521 +12254 82513 +44291 82500 +28536 82492 +17082 82488 +9275 82469 +17421 82467 +3075 82450 +45518 82444 +29669 82442 +34968 82425 +33909 82424 +23690 82423 +20227 82418 +21036 82416 +7574 82416 +3253 82415 +15237 82415 +11034 82411 +25764 82394 +15034 82390 +24282 82387 +36546 82373 +35160 82370 +23710 82362 +18598 82356 +47297 82355 +48685 82349 +15353 82346 +20862 82346 +13219 82345 +22426 82344 +27067 82335 +11166 82333 +10585 82314 +27884 82313 +19162 82312 +28804 82309 +29256 82305 +12972 82300 +8802 82297 +33306 82287 +33404 82285 +23390 82282 +24619 82276 +24302 82269 +15539 82264 +29661 82247 +30701 82247 +47618 82247 +21078 82241 +24408 82240 +23677 82232 +20850 82229 +15447 82226 +19472 82220 +14413 82219 +20309 82196 +46879 82190 +41463 82189 +16143 82184 +16196 82183 +30972 82180 +1993 82173 +31440 82162 +41668 82157 +34238 82149 +10561 82128 +43412 82123 +15636 82117 +28146 82117 +11125 82115 +34015 82104 +15831 82104 +23432 82102 +42730 82088 +30047 82085 +23929 82079 +16586 82073 +18660 82067 +18759 82066 +41459 82064 +15292 82051 +18830 82046 +25046 82045 +7233 82042 +38990 82038 +13582 82032 +9226 81980 +21247 81977 +27376 81961 +25227 81957 +42342 81947 +24902 81940 +32887 81940 +36143 81930 +15219 81922 +919 81920 +24566 81914 +20870 81910 +20140 81909 +29586 81901 +22216 81900 +45103 81893 +19486 81878 +21983 81874 +32289 81870 +30809 81868 +37053 81866 +19399 81850 +15983 81845 +22563 81842 +23572 81842 +8139 81836 +9128 81835 +46397 81830 +20293 81801 +19586 81783 +23007 81782 +29961 81764 +43287 81756 +27929 81752 +29543 81746 +23575 81736 +13022 81724 +21761 81723 +13655 81719 +11045 81715 +12142 81710 +42429 81706 +23616 81703 +29103 81702 +7789 81700 +14146 81693 +32660 81672 +30037 81671 +29742 81662 +20121 81661 +19370 81638 +26706 81626 +14588 81618 +30319 81618 +50041 81613 +29707 81611 +19190 81607 +40751 81605 +47189 81603 +28318 81591 +23715 81585 +27638 81584 +33837 81583 +17201 81577 +23038 81572 +31787 81556 +16122 81552 +50083 81547 +29337 81542 +40365 81531 +35271 81528 +45202 81525 +20232 81502 +47630 81502 +18810 81484 +19643 81480 +23091 81478 +5444 81477 +20697 81477 +25126 81475 +22144 81474 +769 81474 +28269 81463 +39043 81461 +20887 81459 +23161 81458 +23026 81451 +39926 81451 +15049 81444 +23501 81440 +40650 81435 +25524 81408 +1646 81400 +15918 81400 +38455 81393 +8367 81384 +10993 81384 +32899 81381 +50133 81375 +3473 81360 +24345 81358 +27893 81353 +12760 81344 +15646 81342 +23670 81337 +14511 81326 +15940 81319 +19755 81319 +10887 81318 +28175 81314 +14439 81312 +30572 81306 +42576 81303 +39534 81290 +12770 81290 +14326 81277 +25087 81271 +13552 81268 +16060 81259 +22092 81247 +35002 81244 +24423 81243 +38180 81235 +29594 81225 +20992 81224 +26320 81221 +21140 81215 +44786 81212 +36882 81212 +26162 81179 +32675 81171 +1515 81170 +30527 81169 +28310 81165 +19397 81164 +33143 81161 +22760 81159 +21773 81149 +22517 81149 +35893 81147 +24103 81137 +34590 81135 +27070 81133 +20582 81129 +28577 81123 +24956 81121 +25314 81114 +36746 81114 +20721 81106 +24930 81104 +18537 81102 +5063 81101 +26022 81100 +49396 81097 +30404 81074 +33728 81070 +3654 81068 +24115 81059 +41434 81057 +34520 81057 +17002 81054 +30666 81041 +31569 81037 +40130 81037 +21586 81032 +43682 81030 +29479 81029 +35603 81026 +25767 81025 +8984 80993 +4754 80989 +19594 80983 +20332 80979 +29702 80975 +23841 80974 +20003 80974 +20033 80972 +25692 80968 +35839 80965 +21289 80960 +7656 80949 +15060 80949 +6094 80930 +37544 80927 +37709 80924 +31451 80915 +25133 80906 +41401 80905 +16129 80903 +7999 80886 +41281 80883 +34086 80870 +39342 80864 +42257 80859 +40263 80859 +44365 80857 +23597 80856 +32720 80854 +21125 80848 +15269 80847 +24862 80845 +26174 80844 +37386 80843 +13175 80826 +21670 80822 +21137 80808 +17926 80808 +36867 80796 +16616 80794 +29379 80791 +30882 80789 +22408 80788 +16329 80788 +22998 80773 +20842 80757 +35877 80753 +22952 80752 +21073 80739 +30400 80735 +40775 80731 +40167 80729 +18405 80727 +17194 80723 +22296 80716 +28016 80706 +25708 80704 +9862 80691 +24523 80691 +9913 80690 +36085 80687 +23333 80685 +29843 80679 +45406 80664 +25664 80645 +18714 80643 +22612 80629 +20973 80628 +18181 80627 +46434 80609 +32153 80608 +30340 80606 +26681 80604 +14415 80598 +25011 80590 +19392 80588 +21293 80583 +24985 80568 +6360 80567 +16022 80562 +24162 80548 +32474 80544 +25073 80536 +34595 80528 +13605 80518 +17660 80516 +25495 80514 +31394 80512 +38572 80512 +29989 80509 +1999 80508 +31723 80501 +25680 80491 +12061 80482 +36831 80480 +21032 80478 +13421 80478 +22393 80463 +23371 80452 +31488 80447 +31164 80442 +26541 80433 +38717 80417 +48823 80416 +44008 80409 +7126 80385 +19981 80378 +24969 80373 +35270 80368 +31530 80362 +48114 80360 +20864 80352 +31867 80351 +30525 80350 +28856 80349 +40592 80346 +33990 80345 +31839 80344 +7833 80334 +47299 80329 +22120 80328 +31241 80322 +45788 80321 +6985 80318 +39261 80308 +21510 80308 +15770 80292 +18788 80285 +30370 80275 +48514 80274 +25827 80268 +24594 80258 +17376 80257 +20224 80253 +10888 80252 +23238 80248 +10163 80244 +17526 80238 +20374 80237 +27386 80229 +24801 80218 +2053 80217 +13915 80213 +26989 80212 +34133 80200 +20586 80200 +38672 80194 +25451 80193 +15671 80185 +23999 80169 +40886 80166 +15565 80166 +5082 80162 +30900 80156 +19769 80150 +19139 80150 +11951 80147 +21885 80143 +23844 80137 +6238 80121 +43083 80114 +30759 80114 +34443 80100 +21268 80099 +38847 80098 +21857 80073 +29900 80068 +10057 80068 +13632 80050 +19799 80048 +2132 80045 +37503 80041 +21824 80040 +25939 80038 +13987 80032 +8484 80023 +45208 80014 +21986 80012 +20678 80012 +11165 79976 +41774 79971 +44126 79970 +38388 79969 +4794 79968 +22726 79963 +8738 79960 +31868 79955 +26734 79954 +20284 79940 +19333 79939 +35618 79934 +28847 79927 +22014 79925 +21622 79923 +29423 79921 +36502 79919 +23974 79918 +29860 79915 +33173 79909 +27220 79907 +29399 79906 +35248 79906 +31314 79894 +10551 79889 +48471 79877 +16652 79863 +48053 79863 +40013 79862 +14625 79857 +21977 79829 +19581 79823 +19061 79817 +21994 79817 +32548 79813 +14003 79811 +38289 79810 +43109 79805 +41147 79756 +14508 79749 +38427 79744 +24703 79729 +19824 79727 +37321 79714 +36975 79713 +26900 79689 +33011 79682 +2527 79680 +32711 79679 +28921 79671 +35457 79666 +26441 79656 +20892 79652 +42124 79649 +47463 79637 +22782 79635 +14842 79622 +33978 79614 +32354 79610 +15883 79607 +16898 79604 +8041 79602 +17106 79601 +20484 79600 +20376 79597 +38562 79592 +21443 79587 +11260 79577 +22652 79575 +32862 79568 +41828 79563 +33126 79558 +25766 79548 +34861 79544 +14432 79544 +25420 79542 +22258 79540 +31796 79537 +22956 79516 +21534 79497 +17987 79492 +34101 79489 +17129 79481 +26020 79469 +18129 79463 +20956 79450 +30794 79439 +1463 79429 +21843 79428 +45868 79418 +2794 79412 +23965 79408 +43323 79407 +13416 79407 +35695 79396 +24270 79394 +5736 79389 +23510 79380 +23744 79372 +29169 79356 +21982 79353 +38245 79352 +45411 79323 +15715 79321 +25893 79316 +11691 79313 +27366 79309 +37259 79303 +24340 79295 +45753 79292 +29396 79292 +28121 79292 +5722 79290 +20717 79278 +39847 79265 +31537 79264 +8152 79251 +22525 79235 +32363 79232 +29827 79230 +2958 79227 +29948 79223 +21148 79222 +30254 79214 +18333 79210 +17335 79206 +38957 79204 +18495 79200 +24563 79199 +23738 79198 +28615 79192 +35530 79192 +43463 79184 +35568 79179 +18360 79168 +31701 79156 +43017 79141 +43676 79130 +34760 79122 +26602 79116 +21834 79101 +18516 79095 +42148 79070 +14354 79062 +14969 79059 +26532 79054 +38911 79053 +21751 79048 +26680 79048 +20006 79046 +50040 79033 +31462 79033 +18564 79029 +43443 79028 +23137 79022 +9908 79020 +20508 79019 +39314 79019 +30356 79017 +27690 79010 +18465 78997 +27902 78989 +27716 78971 +11059 78971 +13871 78948 +48067 78943 +28320 78927 +11518 78916 +26672 78914 +49652 78904 +15738 78895 +20187 78894 +20811 78893 +13448 78892 +43531 78889 +35506 78886 +25072 78882 +6415 78879 +25687 78877 +28716 78870 +5864 78861 +43050 78855 +7976 78850 +23797 78837 +38732 78837 +44047 78835 +5309 78835 +45178 78834 +4510 78827 +44765 78811 +12193 78808 +43512 78806 +22992 78806 +45932 78801 +33860 78799 +15954 78779 +31251 78772 +20555 78770 +47661 78770 +4214 78769 +15774 78769 +12490 78767 +29520 78767 +35289 78758 +48976 78756 +26297 78753 +30885 78744 +21603 78739 +15834 78738 +20114 78736 +10262 78730 +24738 78719 +16376 78717 +34334 78706 +19514 78701 +23540 78695 +10557 78689 +21339 78686 +31143 78680 +3413 78673 +28201 78672 +20585 78672 +2830 78670 +15088 78661 +12993 78660 +16173 78652 +22839 78637 +10473 78636 +36562 78634 +35304 78618 +41999 78610 +26579 78595 +11225 78591 +21939 78585 +38660 78578 +34265 78575 +19070 78573 +30861 78564 +43090 78563 +24496 78562 +19115 78561 +1329 78558 +27538 78547 +8543 78546 +44185 78543 +31896 78543 +10049 78542 +40434 78536 +23302 78532 +20900 78532 +23528 78532 +34481 78531 +36239 78519 +32791 78518 +32687 78515 +9897 78500 +36634 78497 +42103 78487 +33436 78484 +45667 78482 +18561 78480 +26719 78479 +43450 78473 +31258 78462 +20690 78450 +17067 78447 +27930 78444 +29102 78442 +34798 78441 +10562 78431 +20302 78424 +24486 78417 +35829 78416 +47819 78411 +12900 78406 +31176 78396 +16758 78394 +18775 78388 +24353 78386 +35390 78385 +29685 78385 +22661 78381 +38338 78375 +37704 78362 +36520 78332 +9485 78326 +28215 78324 +20681 78323 +16563 78316 +21502 78314 +43991 78311 +17463 78301 +24699 78298 +18931 78293 +11744 78286 +31028 78286 +20394 78279 +27055 78278 +21629 78275 +37932 78270 +17359 78259 +29780 78254 +19257 78246 +15871 78245 +42225 78242 +29354 78232 +17798 78232 +25343 78231 +7809 78230 +30976 78228 +41363 78228 +31145 78217 +24020 78217 +16590 78196 +12138 78196 +37575 78195 +27185 78177 +19127 78171 +28706 78158 +31038 78127 +19262 78125 +7117 78118 +46620 78116 +40176 78116 +19916 78108 +20901 78106 +24101 78101 +23518 78086 +47857 78085 +12595 78083 +31293 78076 +37979 78074 +30658 78056 +27235 78055 +9516 78038 +29547 78031 +30284 78013 +112 78012 +41271 78007 +24059 77989 +18930 77971 +4965 77967 +27848 77965 +12758 77962 +29309 77958 +29946 77948 +27269 77932 +18797 77932 +17743 77928 +31535 77926 +28612 77911 +21678 77906 +40014 77902 +9059 77897 +17908 77890 +12081 77885 +34989 77860 +17503 77857 +10511 77851 +46551 77849 +17929 77844 +11174 77839 +36395 77827 +20196 77820 +32036 77818 +43692 77817 +35221 77806 +22128 77799 +34305 77796 +23285 77795 +6144 77792 +28967 77783 +19634 77777 +17615 77773 +36374 77764 +25000 77762 +17314 77759 +36300 77753 +23703 77749 +28519 77743 +43405 77742 +7959 77740 +11048 77740 +13523 77734 +46216 77727 +19490 77726 +49490 77724 +48971 77721 +23612 77716 +44599 77715 +30036 77715 +12825 77712 +43162 77712 +30227 77693 +23762 77688 +32994 77681 +16388 77676 +23492 77674 +22461 77663 +9452 77659 +6178 77627 +27963 77625 +36805 77620 +21818 77615 +32319 77614 +40154 77613 +28970 77608 +9060 77599 +39592 77597 +13195 77580 +20594 77580 +12894 77579 +30044 77568 +21757 77563 +45056 77562 +44419 77558 +27628 77551 +15558 77546 +6788 77541 +23261 77538 +41272 77538 +34928 77528 +19782 77522 +12350 77518 +18391 77506 +44842 77504 +7905 77503 +31139 77501 +35652 77500 +35999 77499 +25875 77480 +25802 77478 +24519 77476 +8635 77474 +19441 77471 +30350 77468 +43455 77467 +17168 77462 +19997 77459 +8250 77458 +38434 77457 +20873 77449 +27668 77447 +16152 77446 +35169 77421 +36814 77420 +44544 77418 +21218 77409 +24822 77400 +13294 77395 +32450 77387 +25092 77387 +21487 77383 +40460 77380 +39211 77377 +13800 77367 +43932 77358 +18517 77355 +23866 77347 +22286 77340 +16867 77334 +25152 77324 +20553 77320 +22514 77320 +17793 77307 +33783 77303 +14586 77286 +10377 77281 +8918 77270 +25931 77259 +36419 77256 +15592 77256 +29682 77249 +15146 77246 +22444 77239 +4383 77225 +36988 77224 +33444 77218 +48983 77214 +31997 77212 +16644 77206 +11205 77205 +5998 77202 +25210 77200 +20996 77196 +48157 77195 +26415 77194 +11778 77192 +18149 77189 +34003 77187 +23474 77173 +8777 77171 +29518 77166 +17916 77163 +1152 77149 +21790 77145 +16901 77142 +20927 77138 +13541 77116 +15920 77113 +2774 77105 +20367 77099 +22265 77096 +38680 77094 +31121 77094 +39012 77089 +27677 77088 +41373 77083 +48729 77074 +21479 77073 +14998 77069 +24210 77061 +24081 77044 +35129 77042 +21102 77035 +22863 77031 +25352 77021 +43915 77018 +39869 77017 +44202 77010 +4942 77002 +41503 77001 +23470 77000 +27232 76985 +41388 76981 +29901 76961 +38044 76955 +16241 76950 +42868 76948 +19699 76941 +4592 76939 +23085 76930 +24596 76929 +26324 76928 +44289 76912 +17703 76910 +40780 76909 +24927 76889 +44748 76882 +22242 76860 +19372 76851 +19787 76843 +47457 76840 +47435 76836 +15246 76835 +24324 76834 +22420 76830 +12933 76830 +4514 76822 +33940 76814 +17745 76813 +24234 76808 +38203 76807 +23631 76806 +33363 76796 +1750 76791 +37602 76789 +39414 76789 +15173 76788 +28603 76788 +38759 76784 +32539 76783 +10234 76772 +38157 76762 +21322 76762 +28617 76760 +29525 76744 +33654 76743 +17732 76734 +41462 76728 +17923 76722 +22237 76718 +26196 76716 +27805 76710 +26839 76709 +19775 76705 +19509 76704 +37916 76704 +3789 76703 +50199 76702 +41064 76688 +29218 76679 +31368 76651 +32840 76651 +25709 76646 +41324 76644 +21692 76643 +43733 76642 +15490 76622 +20932 76622 +29808 76614 +16803 76606 +38464 76586 +23105 76585 +23255 76574 +48323 76572 +27615 76568 +33668 76568 +18304 76552 +27519 76532 +29350 76528 +47765 76527 +19435 76523 +32400 76522 +35535 76514 +18646 76508 +24384 76508 +33495 76507 +28569 76503 +42008 76499 +8393 76496 +48773 76493 +22471 76488 +25828 76481 +25232 76471 +41239 76469 +26464 76457 +34461 76455 +9728 76448 +31525 76447 +37335 76439 +40520 76426 +38500 76423 +23125 76416 +23306 76411 +21806 76410 +33358 76405 +32228 76401 +34141 76393 +47412 76393 +31797 76392 +13422 76385 +10871 76382 +7762 76380 +16919 76375 +40742 76375 +29039 76370 +25238 76367 +27074 76359 +33254 76355 +15230 76351 +11633 76342 +29523 76337 +24666 76334 +35315 76331 +40811 76328 +23102 76326 +28838 76321 +26010 76317 +11220 76316 +29496 76311 +31369 76308 +44424 76306 +14478 76304 +7116 76304 +38154 76303 +22021 76298 +27484 76298 +16480 76292 +34240 76292 +29116 76292 +26669 76281 +6250 76280 +25315 76280 +12338 76268 +19214 76260 +39855 76257 +28370 76256 +33419 76250 +17484 76243 +22608 76243 +34001 76234 +15788 76229 +29876 76229 +50170 76224 +30269 76220 +34004 76218 +23675 76213 +23913 76210 +23439 76203 +39325 76184 +32878 76174 +25159 76166 +25301 76164 +25098 76160 +12531 76153 +48841 76142 +17637 76140 +31748 76139 +22384 76131 +31049 76122 +17179 76121 +34074 76121 +8432 76118 +33642 76114 +29351 76113 +29137 76111 +35925 76102 +22333 76098 +36411 76068 +20359 76060 +30451 76057 +38294 76054 +30509 76051 +35512 76026 +29902 76024 +8398 76021 +13429 76019 +21171 76018 +29781 76017 +7185 76015 +30278 75999 +3235 75996 +34092 75987 +9631 75975 +45933 75965 +21873 75954 +23500 75953 +49182 75947 +24240 75940 +1786 75938 +28928 75933 +45197 75926 +16855 75925 +2784 75921 +25119 75920 +7307 75919 +8189 75911 +12512 75903 +12727 75899 +35559 75894 +11347 75888 +36468 75879 +18843 75876 +46050 75875 +37392 75874 +48496 75857 +19232 75856 +26758 75854 +35823 75840 +44963 75831 +24007 75816 +33572 75812 +35276 75811 +46065 75804 +30939 75794 +21867 75794 +9302 75791 +46078 75784 +5272 75778 +35833 75769 +16691 75755 +13306 75748 +37298 75744 +11708 75730 +22983 75728 +29340 75721 +8353 75717 +30914 75715 +42954 75712 +22379 75707 +19460 75701 +19906 75698 +28084 75689 +21771 75686 +37480 75684 +13873 75684 +23944 75680 +24960 75675 +27465 75674 +41637 75669 +18694 75658 +30428 75655 +26757 75645 +18205 75638 +20363 75635 +34936 75633 +22485 75627 +15102 75617 +6357 75615 +29960 75600 +49209 75584 +8679 75577 +3546 75576 +30977 75569 +25632 75557 +25833 75549 +19977 75545 +33956 75544 +24444 75531 +24106 75524 +28862 75524 +25847 75505 +25542 75499 +33128 75495 +26982 75493 +32671 75485 +41838 75484 +32819 75476 +48438 75472 +32834 75463 +18926 75459 +23658 75455 +27455 75449 +20919 75449 +22565 75446 +31928 75443 +37641 75432 +30316 75432 +29437 75427 +37788 75419 +39457 75419 +33354 75418 +35425 75415 +26711 75396 +31411 75387 +39900 75380 +43544 75378 +20358 75374 +24419 75373 +28232 75372 +9940 75371 +24557 75366 +25771 75352 +14607 75344 +32027 75329 +5903 75304 +45951 75299 +27225 75295 +22353 75291 +14040 75284 +31726 75270 +38079 75262 +39762 75258 +25662 75255 +28576 75247 +38851 75237 +5168 75233 +21778 75228 +3540 75220 +35213 75213 +27098 75211 +12978 75206 +40768 75203 +30237 75202 +48503 75183 +37036 75164 +19338 75158 +20422 75157 +21694 75142 +36697 75138 +28579 75128 +27513 75113 +46708 75106 +28582 75102 +34254 75099 +34790 75099 +45674 75095 +22193 75095 +33214 75092 +39791 75091 +34064 75074 +32313 75073 +44492 75071 +20995 75066 +17895 75063 +26587 75046 +14857 75043 +13491 75040 +17013 75037 +8612 75035 +34856 75029 +38630 75028 +20090 75026 +32684 75022 +32890 75022 +38146 75021 +13299 75006 +11167 74991 +25950 74981 +22642 74971 +32119 74969 +25012 74955 +40089 74954 +30089 74943 +3995 74942 +18533 74941 +41788 74940 +49224 74936 +1564 74935 +9106 74934 +44692 74924 +18307 74920 +37191 74908 +34550 74892 +32623 74878 +3165 74870 +18599 74862 +16396 74848 +45276 74848 +5048 74839 +24971 74824 +28679 74812 +4132 74812 +43269 74808 +27794 74802 +9893 74773 +32856 74770 +34132 74759 +24601 74750 +29963 74744 +18112 74741 +44585 74739 +35344 74730 +36047 74721 +37445 74712 +17859 74711 +38287 74711 +25555 74710 +25697 74705 +23290 74697 +44745 74696 +33179 74689 +20136 74683 +19274 74679 +23931 74675 +23359 74667 +6199 74664 +25849 74661 +21442 74654 +21491 74645 +26237 74637 +40526 74624 +18452 74624 +38632 74621 +34602 74616 +30529 74615 +26126 74614 +19656 74610 +40667 74599 +17504 74594 +25800 74593 +22377 74589 +29166 74588 +17794 74580 +26430 74573 +39898 74555 +31348 74553 +21196 74551 +5620 74551 +18917 74547 +28510 74546 +5427 74545 +18886 74536 +36200 74534 +40095 74533 +5823 74531 +15648 74530 +13434 74528 +46347 74527 +30921 74525 +16540 74524 +18607 74521 +30371 74515 +29004 74508 +29651 74506 +14134 74479 +19204 74475 +24165 74472 +19084 74468 +1170 74468 +34643 74465 +19866 74452 +15072 74450 +3879 74444 +18812 74444 +48312 74440 +21414 74438 +19234 74436 +21802 74415 +18807 74406 +25379 74399 +34638 74398 +11017 74394 +42247 74389 +48834 74384 +23880 74379 +36602 74360 +38367 74356 +2334 74356 +27669 74355 +14216 74348 +40243 74345 +21428 74345 +50203 74338 +41377 74335 +38083 74332 +42993 74331 +30320 74326 +8265 74316 +21210 74310 +21355 74300 +5606 74291 +40765 74288 +31486 74287 +31605 74283 +20008 74282 +19802 74277 +16562 74277 +26325 74277 +19100 74270 +42799 74267 +41662 74243 +38485 74229 +25810 74228 +40004 74225 +34603 74220 +21345 74218 +13621 74209 +39547 74207 +13480 74192 +24688 74191 +29339 74191 +29295 74172 +35380 74171 +30244 74161 +39836 74161 +11601 74158 +23805 74150 +13876 74150 +3799 74145 +15677 74116 +6053 74114 +32010 74112 +36359 74108 +20011 74103 +41010 74095 +35668 74092 +22056 74087 +6212 74082 +17734 74076 +42471 74073 +47793 74072 +43103 74069 +14836 74066 +23532 74065 +41398 74061 +4883 74055 +31901 74055 +32355 74046 +23822 74039 +37277 74031 +16269 74022 +46391 74015 +41059 74013 +28697 74010 +39979 74008 +8429 74002 +26170 74000 +37755 73997 +30605 73995 +21403 73992 +18816 73991 +21409 73984 +19147 73976 +29044 73968 +38109 73960 +37937 73956 +35985 73955 +46019 73951 +42266 73949 +35049 73945 +37762 73936 +23748 73913 +29692 73907 +45373 73899 +4550 73875 +32416 73858 +42445 73845 +33359 73842 +18858 73839 +2633 73835 +45377 73829 +16108 73829 +30497 73826 +30421 73826 +27096 73821 +24176 73820 +36945 73810 +48043 73804 +30915 73799 +31245 73798 +30288 73795 +11002 73795 +18466 73787 +33425 73777 +29804 73772 +43647 73770 +21020 73763 +42228 73761 +33263 73752 +11475 73752 +19396 73752 +21978 73750 +43159 73723 +32730 73711 +22946 73708 +46583 73704 +20962 73702 +13590 73700 +35373 73684 +27525 73684 +48009 73669 +27700 73666 +48371 73665 +30699 73664 +46210 73664 +17552 73662 +46530 73659 +26148 73653 +33798 73638 +23598 73632 +42603 73623 +35056 73612 +32883 73610 +33125 73606 +19075 73603 +29632 73594 +26145 73592 +29432 73590 +12968 73589 +17506 73586 +27464 73585 +19949 73585 +29831 73584 +35834 73580 +5990 73578 +26127 73576 +18194 73568 +20762 73559 +22816 73552 +42509 73551 +27184 73542 +26188 73540 +30107 73539 +38030 73536 +3233 73532 +34911 73528 +39660 73527 +9979 73519 +27059 73514 +32670 73514 +14291 73511 +8330 73508 +38282 73505 +44271 73500 +19606 73491 +42365 73489 +47882 73487 +2983 73484 +18578 73482 +34679 73479 +42884 73476 +34794 73474 +30302 73469 +39598 73466 +9890 73465 +30492 73458 +13802 73455 +40325 73443 +19269 73441 +29430 73437 +29539 73429 +24994 73416 +15262 73404 +23584 73401 +39644 73400 +44035 73400 +24390 73398 +25531 73394 +16249 73377 +21234 73365 +24924 73362 +41204 73347 +38830 73342 +29131 73339 +5038 73335 +4018 73328 +15606 73321 +44919 73320 +37948 73316 +43280 73312 +26369 73309 +36556 73307 +5791 73306 +46353 73306 +15442 73304 +11650 73293 +24500 73292 +48314 73279 +27953 73272 +8091 73272 +28691 73269 +6326 73264 +17267 73262 +38555 73256 +29264 73251 +38620 73250 +25865 73248 +27390 73238 +21245 73235 +31044 73228 +26269 73226 +4516 73220 +18300 73218 +29696 73214 +19320 73212 +17947 73206 +48863 73205 +20190 73203 +14295 73198 +41595 73180 +25111 73175 +24030 73166 +27857 73160 +45673 73154 +28997 73150 +21278 73150 +28824 73147 +37723 73144 +27637 73139 +27752 73135 +29681 73135 +18435 73121 +3634 73106 +44684 73099 +30859 73096 +26223 73095 +32930 73092 +20665 73091 +27223 73089 +22884 73084 +43931 73077 +15001 73067 +19751 73061 +26570 73061 +29148 73056 +29811 73037 +15563 73029 +12643 73025 +40422 73024 +35134 73022 +36554 73021 +17770 73018 +23361 73007 +30901 73003 +17748 72999 +12529 72998 +36176 72980 +41783 72979 +26256 72979 +2674 72964 +38075 72946 +38253 72945 +8841 72943 +31712 72943 +37791 72938 +15996 72937 +1953 72932 +3542 72927 +15680 72922 +16975 72912 +29768 72912 +47040 72905 +36815 72904 +25924 72903 +22173 72899 +36548 72896 +40997 72886 +33895 72866 +35657 72865 +25311 72845 +9067 72839 +3213 72835 +37664 72832 +49425 72831 +26564 72831 +32256 72817 +38662 72814 +32166 72810 +31623 72802 +37651 72787 +43720 72786 +16147 72777 +10082 72773 +27485 72765 +40744 72757 +28908 72743 +8000 72730 +9717 72718 +18922 72709 +40351 72708 +6281 72696 +20510 72692 +45844 72690 +4823 72684 +14873 72678 +39193 72672 +4396 72658 +29916 72628 +11715 72627 +23476 72620 +20273 72616 +22538 72609 +14617 72601 +46458 72599 +22751 72578 +28753 72578 +21539 72572 +25508 72570 +35480 72570 +49063 72568 +19328 72567 +33989 72558 +8710 72553 +42722 72551 +7824 72542 +36955 72530 +28195 72523 +8400 72507 +26834 72505 +31864 72501 +30984 72480 +32411 72480 +16834 72479 +23389 72458 +34331 72455 +24177 72455 +24430 72447 +15793 72444 +39404 72440 +12603 72437 +41489 72434 +25888 72429 +12322 72428 +9135 72421 +25244 72417 +25070 72415 +22878 72408 +44398 72404 +26017 72400 +32839 72390 +20005 72387 +25625 72364 +29092 72356 +19349 72342 +31606 72321 +11644 72320 +5051 72318 +4098 72313 +22990 72310 +43640 72310 +12786 72308 +8493 72308 +28695 72300 +32359 72299 +21794 72290 +38977 72287 +36884 72287 +12991 72284 +30881 72260 +30711 72255 +37383 72251 +33073 72250 +12947 72241 +19430 72240 +20233 72232 +26014 72231 +44878 72217 +3579 72212 +18454 72211 +25573 72210 +23246 72203 +21033 72202 +40649 72186 +24730 72182 +45011 72178 +29186 72178 +21989 72176 +38681 72168 +6836 72156 +32758 72156 +46373 72155 +48007 72154 +26041 72153 +26958 72151 +17729 72145 +28918 72142 +24139 72140 +20712 72140 +16815 72138 +28307 72086 +20347 72078 +23840 72069 +33587 72055 +17790 72052 +27900 72051 +24107 72049 +22354 72048 +23374 72038 +18562 72036 +17062 72034 +21925 72026 +31285 72024 +17951 72016 +28819 72016 +4374 72011 +4573 72010 +36980 72007 +17514 71996 +46643 71986 +1922 71979 +6690 71978 +43703 71974 +23101 71974 +40498 71967 +11265 71961 +10642 71957 +32955 71942 +26285 71937 +9047 71934 +19104 71933 +13414 71929 +9100 71928 +39072 71921 +31138 71921 +15761 71915 +29292 71915 +27491 71913 +41736 71911 +40892 71907 +26021 71902 +38789 71898 +10051 71896 +19960 71888 +37260 71870 +27315 71863 +23060 71860 +40394 71857 +25082 71849 +17283 71841 +23195 71835 +34286 71835 +49521 71822 +18366 71820 +39016 71820 +46625 71816 +49618 71805 +34269 71799 +39576 71799 +17382 71795 +24901 71787 +47489 71781 +15631 71777 +37717 71774 +28110 71766 +14880 71764 +12195 71755 +24887 71753 +43731 71748 +16921 71740 +49117 71738 +45856 71731 +11258 71724 +22326 71717 +47813 71700 +1980 71685 +30323 71681 +36041 71670 +6106 71670 +31256 71669 +24790 71665 +16305 71650 +40899 71647 +36671 71646 +8821 71639 +48018 71638 +33948 71636 +38809 71632 +26665 71631 +8571 71628 +42920 71621 +15024 71618 +17084 71609 +40846 71601 +27873 71601 +4057 71593 +17972 71585 +30022 71585 +45301 71581 +35018 71576 +24883 71549 +25717 71546 +26073 71546 +37968 71542 +49528 71504 +43816 71495 +11719 71494 +11277 71488 +34659 71487 +46936 71484 +21690 71480 +22441 71474 +46321 71472 +40065 71470 +38297 71469 +32655 71468 +27580 71466 +33807 71442 +21159 71439 +21453 71437 +19826 71429 +25520 71426 +7277 71426 +19062 71420 +39958 71414 +27574 71403 +22349 71388 +13932 71386 +39236 71384 +22721 71365 +23324 71362 +28594 71358 +24429 71354 +42389 71352 +29779 71351 +23693 71343 +23766 71342 +42990 71339 +46502 71337 +27113 71334 +41763 71319 +42620 71318 +32691 71317 +34009 71310 +13085 71295 +7596 71293 +20248 71292 +45749 71290 +24098 71289 +28461 71286 +36679 71281 +38817 71277 +35383 71271 +10890 71269 +21325 71253 +9921 71251 +34214 71246 +11071 71244 +23281 71240 +20542 71240 +48807 71238 +29263 71236 +27942 71233 +28284 71225 +24114 71218 +21636 71218 +38981 71216 +38687 71215 +21042 71211 +49808 71207 +21528 71201 +29365 71200 +18973 71191 +39109 71190 +11973 71187 +19810 71186 +30055 71185 +29499 71183 +28148 71183 +23213 71174 +21213 71149 +4985 71145 +21522 71134 +30886 71129 +39116 71127 +21710 71121 +43792 71116 +33376 71113 +20936 71108 +3373 71094 +27476 71092 +21083 71068 +28763 71062 +36192 71061 +17319 71038 +37399 71036 +17208 71034 +22972 71024 +16036 71024 +37257 71012 +43152 71009 +47712 70998 +37443 70993 +29745 70981 +12885 70978 +33557 70969 +45913 70959 +26657 70959 +33996 70951 +26051 70948 +27457 70943 +10334 70943 +23642 70932 +48912 70930 +37938 70918 +11439 70912 +20102 70906 +12718 70905 +25638 70900 +18348 70896 +2521 70892 +17571 70886 +41584 70886 +44436 70880 +22267 70872 +38286 70867 +12129 70857 +15367 70854 +30879 70846 +27336 70843 +16504 70842 +42373 70836 +24241 70830 +20567 70830 +30054 70825 +24329 70824 +40680 70823 +41227 70820 +35703 70819 +22576 70814 +42281 70809 +23530 70803 +34860 70795 +45606 70782 +40250 70782 +41749 70770 +49194 70768 +37742 70766 +23798 70763 +17601 70762 +10450 70755 +29178 70752 +8368 70747 +26275 70735 +24808 70734 +15065 70731 +35840 70723 +20838 70716 +25747 70709 +2062 70701 +41803 70700 +26251 70693 +25815 70675 +48467 70673 +28517 70672 +22318 70667 +40942 70666 +23158 70655 +40875 70652 +29535 70652 +22855 70635 +9774 70630 +45026 70622 +12674 70621 +29996 70619 +25413 70596 +39953 70596 +34660 70595 +32829 70593 +36126 70588 +39365 70580 +16817 70579 +9868 70578 +22660 70577 +28507 70577 +23388 70576 +26931 70576 +17427 70574 +24600 70568 +37595 70567 +22100 70565 +23791 70561 +14697 70556 +19155 70555 +16506 70537 +36525 70535 +13610 70529 +1206 70521 +27648 70515 +27328 70514 +15649 70511 +25416 70502 +20616 70494 +23809 70485 +2553 70479 +32164 70470 +11420 70470 +19300 70468 +23776 70465 +16315 70464 +19898 70464 +41082 70455 +41396 70443 +32767 70430 +254 70422 +18742 70420 +40764 70418 +19821 70413 +26790 70412 +11493 70412 +24447 70406 +19774 70400 +35301 70391 +990 70390 +37634 70379 +28372 70378 +25740 70374 +21363 70373 +21473 70372 +43235 70368 +39607 70364 +26228 70359 +24216 70351 +27767 70349 +4123 70346 +16252 70339 +48633 70332 +46608 70328 +7447 70321 +17834 70313 +8462 70308 +32121 70306 +25972 70300 +26234 70298 +38201 70296 +38971 70292 +14619 70278 +40814 70275 +21503 70267 +25137 70263 +3581 70263 +24306 70260 +21540 70244 +21000 70244 +12833 70237 +27105 70234 +44406 70234 +22036 70232 +32466 70231 +49404 70224 +22169 70222 +30160 70214 +19225 70193 +21613 70192 +21805 70190 +18763 70186 +4520 70181 +42355 70181 +8939 70179 +1836 70177 +580 70171 +18868 70166 +36233 70163 +12468 70157 +29591 70156 +22364 70156 +7609 70155 +41390 70148 +6513 70148 +28538 70146 +22107 70144 +28546 70143 +27063 70143 +22084 70142 +40011 70134 +26027 70132 +29278 70131 +45181 70127 +24317 70124 +28533 70122 +36782 70115 +22520 70109 +28583 70109 +42746 70097 +26081 70085 +29885 70083 +14675 70079 +35281 70074 +36348 70072 +30840 70067 +29699 70066 +24853 70057 +21296 70056 +4782 70038 +24068 70032 +12482 70030 +31353 70017 +17226 70016 +6556 70011 +1825 70008 +28412 69996 +18070 69988 +24628 69971 +39209 69959 +13856 69952 +34904 69950 +40840 69948 +15932 69944 +30242 69937 +19032 69933 +20234 69924 +20262 69918 +33188 69907 +39136 69891 +21048 69889 +31976 69883 +31481 69881 +18127 69880 +34203 69878 +752 69878 +6610 69871 +3455 69861 +18904 69854 +32461 69854 +40269 69845 +16781 69841 +18892 69828 +18878 69827 +4541 69827 +9487 69827 +6806 69819 +43346 69816 +14353 69798 +25030 69794 +49626 69792 +26838 69791 +45549 69787 +21869 69774 +23683 69763 +30183 69759 +33174 69758 +41979 69756 +12762 69751 +21354 69736 +43047 69736 +16270 69734 +50026 69732 +22632 69732 +14747 69731 +17338 69723 +38643 69717 +32902 69712 +23110 69705 +27549 69700 +9277 69694 +3507 69686 +44805 69681 +30534 69678 +24865 69678 +6784 69672 +36900 69668 +34250 69664 +32205 69664 +28717 69662 +26954 69656 +28509 69654 +42038 69649 +23932 69648 +21719 69646 +12515 69640 +24681 69637 +44416 69624 +28831 69615 +37042 69608 +38982 69601 +28726 69584 +33329 69576 +29454 69562 +1888 69560 +31990 69558 +49798 69544 +32629 69541 +24208 69540 +23845 69536 +36267 69535 +21934 69530 +16823 69514 +6307 69514 +17952 69508 +22898 69507 +38921 69502 +45341 69498 +46579 69491 +26426 69491 +38057 69486 +148 69485 +19571 69469 +35189 69467 +37377 69461 +30822 69455 +17657 69455 +16920 69447 +19238 69442 +43428 69439 +48588 69438 +14258 69432 +20318 69428 +44373 69421 +17384 69421 +22343 69416 +42273 69407 +36073 69393 +44099 69386 +34558 69382 +2182 69372 +44122 69362 +8958 69360 +15987 69355 +36721 69351 +14001 69349 +15236 69341 +32168 69336 +44799 69329 +30259 69321 +45754 69314 +23082 69307 +34344 69301 +43561 69281 +11425 69264 +38700 69258 +37729 69256 +27384 69253 +41071 69239 +29931 69229 +21573 69226 +18907 69225 +25274 69223 +27859 69222 +36389 69219 +31470 69218 +41325 69218 +10573 69208 +38152 69199 +25572 69189 +31291 69186 +11688 69169 +15708 69166 +47847 69162 +39068 69161 +47431 69152 +41555 69145 +47711 69141 +45044 69135 +13099 69132 +33413 69129 +21059 69125 +29650 69123 +17596 69119 +38872 69117 +13223 69117 +32308 69116 +34939 69107 +24657 69092 +31212 69088 +27518 69087 +36425 69083 +45293 69073 +43537 69071 +27361 69066 +42764 69047 +13311 69045 +16544 69045 +37448 69037 +48876 69037 +31288 69035 +35359 69023 +8585 69014 +11898 69008 +10236 69006 +38952 69005 +26861 68998 +30154 68997 +15596 68994 +17149 68989 +38839 68989 +8401 68987 +27330 68980 +8043 68980 +50090 68980 +37013 68972 +16592 68972 +28969 68964 +45294 68963 +5347 68961 +27817 68960 +23563 68958 +28098 68954 +31515 68953 +29849 68952 +30282 68946 +20186 68945 +19828 68944 +4892 68940 +25690 68935 +32268 68934 +16436 68934 +5769 68933 +20912 68927 +17265 68920 +40056 68920 +29055 68908 +19760 68907 +7753 68904 +1935 68903 +7076 68901 +20167 68891 +27432 68890 +46828 68889 +24910 68885 +20547 68875 +3567 68863 +24425 68852 +8654 68852 +23223 68835 +11197 68830 +35421 68824 +8910 68812 +33217 68811 +28493 68809 +45863 68800 +18052 68798 +46535 68794 +47925 68792 +26722 68790 +3089 68790 +28325 68788 +31316 68779 +28206 68778 +40946 68772 +18225 68769 +22328 68767 +41769 68761 +17670 68748 +15595 68747 +33005 68742 +19181 68738 +22091 68727 +33632 68725 +2567 68720 +9641 68707 +9195 68698 +23367 68695 +26963 68694 +42953 68682 +17437 68678 +48757 68677 +48847 68674 +28021 68673 +23873 68672 +42629 68663 +45347 68658 +32448 68654 +43154 68647 +26902 68647 +19792 68633 +34748 68631 +27409 68630 +33533 68623 +36508 68621 +42999 68618 +22094 68611 +42113 68606 +16879 68604 +33423 68602 +26831 68566 +29534 68559 +36498 68554 +14098 68548 +49495 68547 +22200 68542 +32331 68535 +28991 68534 +37558 68531 +44349 68530 +28652 68526 +11020 68522 +18215 68522 +39036 68522 +40770 68514 +44911 68509 +6514 68491 +31123 68490 +32980 68488 +2234 68488 +20866 68486 +38261 68481 +35798 68480 +42574 68479 +46388 68479 +7933 68469 +23498 68452 +40072 68449 +26510 68442 +43623 68440 +17388 68438 +6389 68436 +24257 68423 +38059 68421 +42736 68418 +45936 68413 +26128 68413 +39313 68412 +35323 68403 +35594 68392 +39025 68389 +42094 68389 +17847 68386 +16631 68385 +15328 68375 +32043 68370 +19849 68366 +26490 68365 +22066 68357 +18046 68351 +2303 68337 +9876 68336 +13137 68333 +23823 68332 +15652 68332 +35080 68330 +23323 68327 +43890 68326 +46732 68320 +19103 68319 +27939 68311 +27234 68306 +27391 68301 +29140 68300 +26137 68300 +36333 68299 +8183 68290 +31189 68284 +36576 68283 +16187 68280 +27568 68277 +15538 68276 +44018 68273 +47717 68253 +39905 68252 +24436 68249 +27407 68240 +21591 68238 +40015 68237 +19345 68231 +20226 68230 +21894 68221 +38372 68212 +30151 68212 +31568 68190 +16858 68186 +18664 68180 +35249 68168 +46578 68143 +33915 68142 +34093 68118 +20437 68114 +37946 68102 +40317 68102 +40543 68098 +18741 68094 +21709 68085 +36662 68083 +42705 68059 +8789 68055 +16703 68053 +40696 68051 +49293 68048 +12291 68045 +41971 68045 +47349 68038 +18815 68034 +38511 68033 +49894 68021 +26547 68021 +34536 68018 +28207 68017 +41086 68006 +47822 68005 +1970 67984 +24028 67974 +20934 67971 +36987 67966 +10113 67960 +22951 67949 +22142 67944 +21035 67942 +36353 67938 +20780 67936 +47074 67928 +25705 67927 +21916 67919 +8230 67917 +44050 67910 +21365 67885 +19371 67884 +25240 67878 +26835 67875 +22601 67874 +35432 67872 +16056 67863 +27298 67860 +48711 67849 +23865 67829 +22827 67823 +30974 67809 +27989 67792 +21344 67777 +38992 67776 +36472 67776 +14786 67766 +39026 67764 +31861 67757 +43814 67744 +21792 67733 +18510 67733 +41540 67732 +23490 67731 +40528 67730 +13087 67730 +9580 67729 +26240 67727 +1472 67727 +33601 67721 +47265 67721 +38919 67714 +20986 67710 +13384 67707 +49338 67694 +12828 67690 +30957 67687 +19454 67685 +18278 67654 +33513 67647 +14398 67646 +22594 67641 +11786 67639 +11876 67631 +2965 67624 +21427 67616 +20959 67612 +18376 67612 +26566 67609 +7201 67606 +27878 67604 +35933 67590 +13174 67587 +13839 67580 +4210 67579 +25112 67570 +46533 67567 +15967 67567 +9288 67558 +39154 67553 +39091 67544 +19917 67544 +20474 67543 +23774 67539 +38058 67537 +15299 67536 +19082 67534 +3795 67530 +26329 67512 +30455 67511 +41422 67509 +37016 67505 +32741 67502 +30062 67498 +24362 67497 +19206 67497 +36755 67486 +8457 67486 +26071 67485 +38495 67483 +21769 67476 +40983 67460 +41618 67451 +19325 67443 +14904 67441 +27088 67432 +32952 67427 +36507 67406 +29180 67403 +27947 67402 +30438 67398 +19930 67392 +20307 67391 +34187 67386 +45899 67384 +13033 67380 +26821 67373 +36933 67370 +7199 67368 +36592 67367 +34430 67365 +38080 67336 +3153 67334 +17688 67334 +32404 67333 +49369 67330 +20718 67327 +25327 67321 +43487 67314 +43229 67311 +10044 67310 +31277 67305 +29684 67302 +19903 67297 +22590 67291 +27183 67288 +15754 67287 +19222 67287 +22744 67282 +39046 67281 +38798 67277 +10613 67264 +39690 67260 +29711 67255 +21407 67245 +6207 67241 +19074 67237 +27029 67230 +28889 67226 +28267 67224 +12483 67216 +35335 67215 +49118 67214 +47097 67210 +16796 67207 +4233 67207 +16011 67191 +12458 67184 +22115 67183 +48831 67181 +19040 67180 +5570 67179 +4140 67176 +17983 67174 +35060 67167 +40688 67167 +26573 67160 +16365 67158 +24332 67149 +17756 67147 +35022 67146 +13042 67146 +2205 67137 +35127 67135 +40399 67133 +31147 67131 +24769 67131 +31931 67100 +27252 67100 +19909 67095 +19954 67094 +29630 67093 +141 67090 +19118 67089 +28332 67086 +45269 67082 +13824 67082 +36709 67080 +23594 67072 +27498 67061 +34922 67056 +44421 67043 +43765 67042 +27681 67037 +3314 67021 +31096 67016 +25250 67005 +17759 67001 +18571 66993 +9553 66992 +20179 66988 +29426 66977 +39944 66971 +19939 66965 +13624 66964 +9690 66958 +25772 66950 +5943 66948 +914 66937 +27208 66937 +38707 66931 +17010 66916 +8196 66915 +20868 66914 +42849 66913 +33021 66909 +23959 66906 +16354 66884 +26191 66878 +29595 66874 +9426 66872 +32800 66871 +25192 66866 +20100 66864 +21770 66862 +22050 66851 +24715 66851 +31422 66848 +46574 66843 +28105 66838 +14925 66831 +49858 66830 +15137 66829 +36131 66822 +11825 66818 +24530 66807 +13924 66806 +24716 66805 +26590 66799 +29451 66789 +16326 66788 +20836 66786 +30065 66785 +24796 66784 +7060 66782 +30060 66781 +30849 66779 +17585 66776 +24905 66776 +7680 66766 +6271 66765 +41037 66764 +46355 66764 +44272 66760 +29793 66742 +26500 66741 +8480 66739 +20917 66730 +23311 66727 +43861 66724 +20124 66724 +40563 66706 +41665 66705 +40524 66696 +21777 66695 +21536 66694 +23717 66689 +19609 66688 +46039 66687 +31879 66686 +24507 66676 +26888 66675 +34037 66674 +26810 66672 +40752 66671 +29290 66670 +25721 66667 +10363 66663 +40376 66660 +10419 66658 +34143 66645 +15274 66641 +21549 66633 +22427 66614 +37878 66586 +50236 66581 +42540 66581 +40645 66581 +22231 66577 +42847 66574 +45277 66567 +26432 66561 +24561 66549 +19485 66549 +34245 66545 +23834 66544 +40030 66540 +28459 66539 +7916 66538 +19507 66528 +20975 66527 +16552 66525 +36290 66524 +36254 66523 +45109 66517 +35696 66517 +20796 66516 +23771 66513 +26636 66502 +34639 66497 +4680 66491 +41913 66486 +28233 66484 +45327 66484 +37521 66479 +20048 66470 +25878 66469 +4804 66460 +5792 66455 +49398 66448 +21240 66426 +43757 66408 +27861 66400 +47515 66397 +15417 66396 +29105 66386 +37131 66383 +35459 66374 +4600 66368 +27415 66365 +22256 66364 +12012 66358 +28962 66357 +28225 66354 +15997 66351 +24386 66347 +21225 66346 +24012 66345 +26596 66344 +35449 66338 +47447 66338 +25858 66334 +41514 66330 +42542 66326 +29287 66319 +36398 66313 +32019 66307 +26206 66303 +26928 66297 +44294 66297 +21697 66296 +49767 66295 +33078 66293 +27718 66291 +30466 66290 +23134 66287 +39370 66268 +25433 66267 +25988 66261 +24506 66244 +30555 66234 +20578 66233 +37339 66227 +21718 66226 +24922 66224 +1937 66223 +17402 66221 +20615 66220 +21433 66216 +35032 66210 +23586 66204 +17824 66200 +33171 66198 +37556 66196 +30888 66190 +37499 66186 +21712 66183 +45177 66183 +31846 66165 +13381 66156 +25809 66154 +19088 66145 +19180 66144 +39288 66144 +33769 66140 +24578 66140 +26936 66139 +32826 66138 +41946 66127 +28123 66126 +49783 66126 +24049 66125 +42439 66123 +36524 66121 +37951 66112 +19645 66109 +36031 66104 +23275 66104 +17393 66104 +35466 66102 +18104 66083 +3527 66079 +20729 66079 +15155 66078 +21377 66075 +23570 66072 +33628 66058 +23938 66048 +47699 66043 +14990 66037 +34937 66035 +31453 66030 +19445 66030 +19348 66026 +45346 66016 +15079 66007 +35715 66005 +38548 66000 +38153 65991 +37517 65991 +30166 65984 +45603 65971 +9562 65967 +24777 65964 +25424 65959 +23667 65959 +25103 65957 +40935 65951 +35754 65946 +40701 65945 +27561 65943 +40274 65937 +44107 65934 +20675 65933 +38826 65927 +25318 65925 +26043 65925 +34268 65910 +48041 65908 +48117 65899 +28079 65896 +40633 65893 +31302 65891 +33280 65891 +36696 65891 +39745 65887 +35043 65887 +24291 65887 +36963 65885 +38262 65885 +40691 65883 +29050 65883 +34299 65881 +38111 65877 +23654 65875 +20834 65872 +18159 65871 +17400 65869 +27697 65862 +41611 65849 +46669 65845 +18558 65830 +35724 65825 +27851 65810 +29107 65807 +27262 65806 +13040 65802 +35845 65801 +23263 65798 +38459 65796 +30171 65791 +43774 65789 +20511 65786 +31320 65784 +41223 65776 +16996 65766 +22947 65764 +48309 65762 +40863 65761 +6785 65758 +38222 65757 +6804 65742 +33071 65740 +37276 65739 +33122 65739 +18629 65736 +23818 65735 +27475 65734 +32860 65727 +8622 65724 +23442 65717 +32358 65707 +27474 65707 +23824 65698 +36386 65698 +5577 65688 +10496 65681 +41943 65678 +11454 65675 +35966 65675 +19672 65672 +4807 65671 +13513 65669 +32969 65668 +28756 65657 +33568 65656 +35953 65653 +19244 65645 +29569 65643 +25425 65639 +31757 65634 +18273 65628 +6015 65625 +23009 65625 +22670 65621 +19972 65621 +15424 65619 +14192 65616 +35954 65615 +32445 65608 +29202 65608 +48164 65606 +16980 65604 +42475 65604 +32596 65603 +17118 65600 +44801 65600 +46406 65590 +32735 65589 +28011 65586 +27284 65582 +7583 65578 +43207 65573 +16931 65572 +11991 65561 +19233 65546 +29155 65542 +18298 65537 +33262 65533 +21154 65530 +43741 65528 +15456 65524 +39718 65508 +32569 65507 +15705 65504 +20350 65502 +24587 65498 +38164 65495 +29398 65495 +43393 65493 +24562 65489 +19114 65489 +20668 65477 +30538 65466 +34700 65463 +22049 65448 +34548 65447 +34014 65440 +27182 65432 +41702 65430 +21669 65427 +23372 65425 +35446 65425 +38178 65413 +21144 65412 +17749 65408 +34892 65405 +29232 65404 +43742 65401 +42151 65399 +25834 65394 +34315 65393 +14837 65392 +40685 65390 +16654 65389 +21607 65389 +35297 65378 +21816 65366 +19023 65364 +26852 65347 +27807 65347 +19736 65337 +21850 65336 +18574 65335 +21172 65322 +16833 65304 +31532 65302 +30973 65298 +29168 65274 +32762 65273 +9057 65269 +29450 65265 +8164 65261 +2760 65256 +35485 65254 +19865 65248 +46051 65245 +12984 65244 +43226 65242 +30172 65239 +18955 65239 +6084 65232 +31381 65230 +29614 65225 +31747 65202 +13369 65200 +38863 65198 +46653 65189 +37796 65188 +41140 65183 +12295 65181 +22547 65180 +39479 65177 +1506 65172 +25212 65167 +41902 65161 +39148 65157 +30793 65151 +29235 65150 +1361 65149 +6874 65148 +31001 65148 +2552 65138 +48484 65133 +13202 65132 +29242 65132 +37030 65130 +47154 65121 +21374 65119 +28526 65113 +44119 65112 +37180 65104 +28491 65102 +33368 65100 +44143 65096 +44381 65091 +17890 65088 +31956 65082 +18668 65067 +31781 65053 +18345 65041 +40992 65037 +9647 65032 +42125 65030 +35351 65029 +42498 65016 +9357 65005 +33522 64997 +30510 64996 +29065 64992 +30889 64992 +16966 64975 +26614 64971 +33936 64956 +40807 64948 +27819 64940 +27995 64930 +17301 64924 +31647 64923 +41310 64920 +30713 64919 +1491 64917 +11837 64914 +39360 64913 +24755 64910 +12552 64907 +25547 64904 +9822 64903 +34369 64900 +23981 64897 +34623 64895 +5493 64885 +49042 64883 +32914 64880 +26647 64878 +25241 64876 +19281 64872 +32753 64865 +26139 64864 +44386 64859 +27281 64851 +24739 64846 +44918 64838 +37566 64837 +22406 64833 +29844 64824 +42824 64805 +38063 64802 +14189 64796 +19664 64796 +5226 64782 +45463 64775 +47735 64766 +24780 64765 +22033 64742 +27048 64735 +28010 64734 +20083 64723 +27044 64718 +5458 64715 +37097 64709 +35971 64708 +3308 64701 +46811 64664 +34295 64662 +41896 64661 +27200 64652 +9062 64637 +26016 64631 +21890 64630 +26886 64628 +25980 64626 +33663 64621 +38719 64620 +3948 64618 +22780 64614 +22739 64601 +45556 64598 +44626 64582 +45125 64580 +40308 64579 +27899 64569 +24311 64568 +30375 64567 +32232 64565 +23591 64564 +49872 64553 +4323 64552 +6978 64549 +29382 64548 +19727 64540 +3231 64538 +30449 64533 +48906 64531 +21016 64531 +22248 64528 +16184 64525 +27276 64518 +21643 64514 +48769 64514 +39203 64511 +28530 64511 +23176 64508 +31332 64504 +31290 64502 +28654 64500 +7919 64500 +25397 64494 +28527 64492 +47595 64484 +33052 64481 +26215 64471 +18396 64466 +32607 64461 +21825 64459 +5360 64453 +16669 64450 +44179 64446 +28197 64443 +10286 64436 +19836 64433 +33806 64431 +25293 64419 +48361 64416 +43419 64408 +20108 64400 +38355 64398 +39358 64397 +15202 64381 +30256 64355 +36740 64355 +34998 64354 +36304 64343 +38320 64341 +18971 64336 +33081 64334 +34518 64329 +32083 64325 +30000 64322 +22701 64318 +19818 64306 +37313 64295 +40978 64294 +26730 64290 +24846 64271 +36121 64270 +27558 64261 +36009 64260 +18977 64259 +41926 64244 +26609 64238 +46779 64238 +28387 64238 +21826 64235 +40907 64224 +29671 64224 +24989 64214 +22656 64210 +25592 64206 +20743 64200 +25784 64193 +42392 64193 +24939 64189 +13398 64180 +45399 64171 +14635 64156 +26001 64155 +32029 64152 +48677 64145 +29927 64140 +42288 64135 +48302 64135 +4515 64131 +2101 64129 +38230 64125 +25655 64125 +14076 64121 +28779 64117 +5647 64115 +15217 64114 +36092 64113 +41967 64112 +16161 64107 +16487 64103 +22494 64102 +25063 64100 +33041 64096 +27759 64090 +24488 64083 +28973 64075 +46889 64075 +17515 64063 +31725 64060 +23769 64050 +35675 64047 +20161 64046 +26688 64040 +26112 64036 +28153 64031 +26470 64024 +44210 64015 +34571 64013 +42754 64007 +3921 64003 +21230 63999 +24356 63996 +46949 63996 +49860 63995 +43442 63995 +35128 63992 +2384 63991 +32033 63990 +9182 63984 +16274 63982 +15225 63979 +4070 63978 +31639 63975 +22465 63974 +41651 63973 +9448 63964 +35198 63962 +32504 63961 +45187 63958 +13103 63957 +12960 63950 +38360 63945 +13880 63937 +9804 63930 +11313 63921 +19772 63918 +47592 63915 +29333 63914 +20546 63914 +41073 63912 +37924 63907 +34821 63904 +24916 63903 +35444 63900 +44352 63891 +25076 63890 +4648 63889 +43985 63881 +6719 63881 +7803 63878 +35377 63878 +38523 63877 +38285 63866 +33899 63865 +6929 63864 +40352 63860 +46539 63858 +22303 63842 +36437 63837 +30061 63837 +35756 63836 +18041 63835 +41533 63833 +30987 63821 +22967 63820 +18722 63818 +34744 63812 +22745 63805 +25997 63798 +49577 63798 +17510 63796 +25519 63785 +41892 63784 +41965 63770 +19538 63757 +33702 63754 +25368 63752 +16096 63747 +45732 63743 +11362 63741 +38174 63739 +25776 63730 +36302 63728 +31101 63726 +24643 63725 +25699 63722 +46994 63713 +4636 63712 +47693 63712 +14838 63705 +10300 63703 +20646 63699 +28784 63696 +20634 63694 +22358 63685 +20144 63683 +28477 63682 +11184 63680 +24000 63673 +21261 63668 +32885 63667 +9819 63666 +16773 63664 +29020 63660 +37134 63652 +31477 63638 +46729 63633 +45555 63632 +11866 63631 +15903 63631 +14646 63631 +19130 63628 +18542 63623 +33753 63623 +25963 63620 +17385 63616 +29207 63608 +37461 63607 +19265 63594 +21508 63593 +22160 63584 +26536 63580 +42628 63580 +14254 63575 +43376 63574 +36762 63562 +34796 63561 +38315 63559 +31104 63552 +33861 63550 +18951 63549 +34351 63535 +25268 63532 +45064 63530 +29353 63529 +5659 63528 +35201 63526 +20853 63522 +19811 63520 +37043 63510 +12535 63506 +25976 63503 +30705 63500 +43868 63497 +11176 63484 +31352 63473 +39387 63471 +25493 63470 +45067 63470 +18768 63469 +30186 63465 +32249 63463 +31480 63457 +39011 63446 +22335 63443 +7861 63443 +24793 63441 +14602 63437 +17641 63429 +31498 63427 +25134 63426 +17297 63425 +29494 63420 +16486 63417 +21060 63409 +30663 63408 +35296 63401 +32967 63395 +33228 63392 +21729 63392 +17140 63374 +5317 63374 +30292 63371 +38234 63366 +37286 63360 +10141 63355 +19734 63350 +11804 63348 +19474 63340 +22247 63340 +12944 63322 +24018 63317 +38558 63316 +44936 63315 +33675 63309 +26224 63307 +26732 63300 +18058 63298 +27965 63296 +26684 63290 +27887 63277 +27221 63276 +1193 63271 +33453 63265 +21585 63258 +33578 63258 +40959 63243 +18720 63239 +24995 63234 +27004 63227 +27127 63224 +30647 63222 +20334 63219 +12997 63206 +24231 63200 +35554 63196 +25557 63194 +46989 63192 +48316 63191 +40074 63184 +38606 63184 +10994 63179 +47894 63176 +19442 63173 +15714 63170 +11284 63168 +46411 63167 +38129 63165 +24478 63158 +21255 63158 +49482 63158 +1407 63144 +21388 63141 +24069 63136 +34990 63135 +38646 63131 +25678 63125 +34596 63123 +45189 63122 +34040 63116 +34231 63112 +18905 63097 +11857 63093 +38358 63092 +36806 63091 +28776 63083 +37931 63081 +31892 63075 +29439 63072 +33290 63072 +47005 63070 +26424 63069 +25560 63058 +40430 63056 +48158 63054 +17464 63051 +22113 63047 +35065 63031 +50075 63023 +21667 63021 +20034 63019 +29943 63018 +28092 63017 +2108 63016 +41640 63013 +1132 63012 +20624 63009 +36876 63005 +27305 63002 +20381 63002 +39523 62996 +20478 62995 +49250 62994 +14797 62994 +13060 62992 +44310 62991 +23574 62990 +45476 62967 +26830 62965 +11296 62964 +9362 62963 +32507 62960 +31379 62955 +45033 62949 +23284 62947 +35938 62944 +22131 62944 +29135 62942 +25077 62938 +21578 62927 +25773 62927 +39372 62925 +3990 62921 +43341 62916 +23816 62916 +31091 62911 +33075 62909 +32442 62896 +18248 62881 +38014 62879 +26304 62872 +22352 62867 +8081 62866 +12976 62864 +50231 62857 +24921 62842 +18636 62837 +19846 62837 +41155 62821 +26574 62810 +42923 62806 +20844 62801 +31198 62794 +19650 62783 +17240 62778 +16856 62777 +11555 62777 +18049 62755 +25381 62751 +20043 62746 +36903 62744 +15624 62742 +23853 62741 +43678 62733 +6435 62733 +42952 62723 +14967 62723 +13049 62721 +19052 62720 +31351 62705 +23145 62704 +40596 62682 +27065 62678 +32805 62667 +33745 62661 +23417 62652 +27408 62651 +25071 62648 +35037 62647 +49067 62645 +38534 62633 +30712 62631 +42838 62626 +37629 62624 +25654 62620 +4142 62620 +44760 62618 +15845 62603 +3447 62602 +34809 62595 +7004 62593 +46137 62589 +45329 62587 +42862 62584 +26421 62583 +13696 62574 +15835 62573 +34529 62558 +25544 62533 +42345 62518 +23183 62516 +29987 62514 +25789 62501 +38162 62501 +39854 62496 +26397 62485 +20822 62484 +36839 62474 +31682 62473 +31443 62465 +32135 62459 +47955 62453 +37095 62453 +43097 62450 +34061 62442 +33633 62434 +28340 62429 +24786 62427 +17103 62419 +27327 62417 +40359 62413 +28234 62412 +36997 62408 +7254 62406 +8593 62406 +25580 62406 +31086 62406 +6652 62405 +21965 62398 +2445 62396 +28246 62395 +37075 62394 +31720 62392 +28767 62388 +11458 62385 +18534 62382 +36391 62380 +24648 62377 +31163 62375 +32385 62372 +22679 62367 +42328 62360 +24475 62355 +23741 62350 +36354 62339 +24672 62336 +29817 62332 +33782 62331 +19531 62331 +26218 62328 +17470 62306 +30050 62304 +43415 62300 +16985 62293 +38487 62277 +28364 62268 +47304 62257 +23421 62252 +10458 62249 +20260 62247 +25543 62245 +18869 62239 +36735 62235 +43256 62231 +28574 62228 +25562 62227 +42521 62222 +36768 62221 +230 62217 +30040 62215 +33880 62212 +35104 62207 +21489 62201 +13001 62195 +35256 62193 +24986 62193 +49018 62190 +9338 62183 +27855 62171 +31603 62153 +12721 62149 +34685 62149 +46977 62149 +41969 62142 +37732 62140 +20316 62132 +21888 62131 +43611 62127 +28671 62123 +23180 62123 +38137 62122 +12480 62120 +32628 62118 +35024 62117 +27495 62108 +49145 62102 +30250 62091 +24734 62064 +30913 62064 +34429 62054 +46327 62051 +11531 62033 +13154 62028 +19418 62024 +34475 62022 +38087 62021 +231 62021 +15396 62008 +34199 62003 +1715 61999 +48674 61998 +46409 61995 +30339 61994 +36897 61990 +42261 61989 +42621 61987 +29573 61985 +13377 61984 +31513 61978 +29042 61962 +15605 61953 +48734 61946 +24203 61944 +38725 61937 +10901 61934 +30868 61934 +29772 61921 +18175 61918 +31689 61915 +41804 61914 +25943 61906 +26519 61903 +29837 61901 +15810 61896 +15813 61893 +23117 61892 +17776 61889 +46227 61866 +35357 61862 +35685 61857 +43431 61855 +40129 61852 +40808 61849 +44014 61847 +13096 61846 +42554 61846 +39292 61842 +49002 61837 +22121 61814 +47891 61810 +32925 61807 +26394 61798 +40587 61798 +30721 61796 +15630 61788 +26951 61786 +23336 61766 +32302 61755 +22531 61745 +25781 61738 +23755 61736 +22236 61733 +12043 61731 +36743 61729 +18188 61723 +46072 61722 +39123 61718 +30755 61718 +7829 61705 +41718 61704 +43296 61704 +20010 61702 +34677 61701 +14046 61698 +16991 61697 +37553 61697 +3560 61695 +20676 61692 +42198 61685 +31934 61682 +26879 61678 +15785 61673 +24308 61668 +42356 61666 +22859 61660 +36858 61655 +45402 61650 +31810 61638 +47688 61619 +22703 61619 +17481 61615 +35517 61613 +44410 61596 +35228 61593 +12003 61583 +40349 61581 +29026 61576 +48273 61569 +28218 61562 +38920 61559 +25731 61552 +22655 61546 +32945 61545 +28161 61541 +46611 61534 +45022 61528 +13193 61514 +21056 61506 +47554 61503 +15357 61502 +32757 61495 +31238 61489 +30391 61486 +42518 61482 +24407 61479 +32046 61475 +39244 61451 +24246 61448 +10533 61447 +28082 61441 +35733 61435 +44252 61429 +21868 61424 +31517 61419 +17758 61418 +23236 61418 +12283 61412 +29301 61405 +24925 61402 +45996 61401 +8001 61395 +47727 61390 +36605 61372 +19177 61372 +39094 61368 +26386 61359 +10531 61355 +34166 61355 +32533 61342 +29259 61333 +33917 61333 +33652 61327 +44838 61322 +11977 61317 +41930 61315 +30349 61314 +28357 61311 +34081 61307 +17886 61305 +20266 61299 +6990 61299 +28344 61296 +34264 61290 +29036 61289 +34282 61284 +32202 61283 +14756 61281 +35003 61281 +36246 61280 +19915 61265 +37307 61264 +24312 61257 +32985 61253 +18289 61249 +12332 61245 +17766 61244 +34845 61239 +45369 61233 +30082 61222 +21652 61216 +23695 61211 +33585 61199 +15405 61190 +16081 61188 +15015 61185 +41476 61183 +19590 61181 +23859 61170 +18863 61167 +18490 61161 +11187 61159 +42786 61148 +44843 61148 +27346 61147 +35406 61139 +33867 61134 +44840 61129 +36773 61125 +36432 61123 +50070 61122 +32028 61117 +45562 61116 +12938 61113 +14831 61105 +20942 61104 +13335 61100 +48405 61090 +34226 61088 +8481 61085 +15478 61083 +49748 61083 +21207 61079 +25144 61077 +46286 61073 +31469 61072 +43130 61062 +39822 61061 +25907 61048 +36045 61046 +47561 61037 +22769 61036 +34838 61036 +5424 61034 +38633 61033 +11007 61028 +31083 61025 +20759 61025 +30224 61022 +44914 61021 +22483 61020 +16869 61019 +31788 61019 +19798 61014 +50198 61001 +7355 60988 +21580 60971 +23855 60969 +27135 60966 +48654 60963 +38931 60960 +29194 60955 +40390 60945 +30877 60939 +36405 60929 +42073 60929 +40542 60924 +39772 60914 +30084 60911 +18392 60902 +38822 60895 +46277 60890 +4298 60888 +14049 60881 +32972 60873 +27962 60865 +29716 60851 +11249 60842 +15537 60840 +45005 60835 +26124 60831 +42291 60828 +29461 60826 +26717 60826 +28607 60825 +39161 60824 +36964 60824 +34207 60824 +26457 60816 +5592 60812 +15194 60805 +9158 60805 +21581 60802 +26370 60800 +48937 60796 +18032 60795 +31604 60791 +26340 60789 +1405 60786 +37525 60786 +22320 60783 +38502 60765 +39952 60762 +25738 60756 +32064 60753 +33677 60751 +30873 60749 +22828 60744 +46744 60739 +24217 60738 +29370 60732 +41645 60726 +39656 60717 +48535 60715 +26194 60715 +44288 60714 +23098 60713 +45155 60711 +42226 60701 +46898 60673 +23625 60672 +29862 60671 +48857 60666 +25276 60654 +33681 60648 +28836 60647 +15266 60643 +41852 60638 +13966 60622 +25219 60620 +31600 60617 +31549 60616 +48639 60615 +23089 60611 +47319 60606 +25121 60605 +38554 60603 +29199 60601 +30544 60598 +25221 60595 +17436 60594 +19659 60593 +20022 60593 +28641 60590 +17115 60590 +43599 60589 +15584 60589 +27692 60585 +42183 60566 +15947 60565 +42196 60562 +32392 60560 +25704 60558 +22448 60555 +21932 60549 +24458 60547 +39330 60543 +29537 60537 +32528 60527 +29635 60520 +44997 60518 +45073 60514 +28837 60511 +47802 60499 +46766 60497 +45475 60497 +17020 60484 +14977 60483 +42619 60480 +25906 60478 +30771 60467 +2658 60460 +19814 60460 +47552 60455 +41886 60448 +36106 60444 +47956 60433 +15634 60428 +11577 60425 +20343 60420 +18680 60417 +49984 60413 +28142 60403 +4681 60401 +44790 60400 +40636 60397 +23895 60390 +32745 60388 +46151 60384 +9372 60384 +32874 60383 +32964 60374 +26844 60369 +46193 60365 +18823 60359 +26653 60359 +30942 60355 +37816 60352 +22705 60342 +45194 60327 +44111 60323 +23440 60322 +44358 60319 +27211 60315 +33024 60303 +40917 60302 +35771 60302 +39092 60300 +40769 60294 +19554 60291 +29738 60290 +47403 60284 +30909 60271 +46191 60271 +25546 60261 +32484 60261 +27452 60257 +38965 60250 +13704 60250 +31022 60249 +30063 60249 +46134 60246 +35855 60243 +39829 60238 +4668 60237 +18218 60237 +29542 60233 +37684 60224 +48403 60210 +37470 60207 +12645 60205 +21926 60196 +13764 60195 +16558 60195 +4348 60194 +31939 60194 +27492 60183 +34608 60180 +37159 60179 +48423 60162 +47485 60161 +29556 60161 +39097 60161 +34664 60152 +49200 60151 +37914 60142 +25500 60137 +18549 60126 +28443 60112 +20713 60101 +13894 60095 +25094 60095 +22740 60094 +42441 60086 +43898 60085 +28816 60073 +44949 60068 +45585 60067 +23556 60066 +16983 60058 +28415 60056 +19974 60049 +33511 60047 +44443 60044 +39480 60041 +24250 60038 +30874 60037 +21630 60035 +21831 60035 +18777 60032 +29787 60031 +30393 60023 +38426 60015 +40588 60009 +37240 60004 +2755 59997 +28188 59996 +25996 59996 +29255 59994 +20895 59989 +34146 59983 +41953 59981 +32625 59979 +17248 59979 +5104 59972 +48283 59969 +37391 59969 +37119 59967 +15429 59965 +35020 59965 +30954 59962 +31309 59954 +27782 59938 +6138 59937 +37897 59935 +35122 59932 +26781 59931 +21208 59929 +25639 59924 +39671 59923 +23133 59922 +15120 59920 +33760 59907 +31782 59900 +5080 59899 +44391 59896 +20757 59894 +47148 59891 +35142 59889 +46684 59885 +17792 59884 +32727 59878 +20802 59870 +19679 59863 +36800 59857 +33636 59854 +15716 59853 +2090 59853 +24758 59852 +17806 59850 +32715 59848 +27173 59847 +8490 59846 +27951 59844 +41407 59843 +47919 59843 +20785 59838 +31432 59834 +28214 59832 +30855 59825 +4278 59824 +35647 59823 +32609 59822 +35984 59820 +27312 59812 +11532 59807 +35734 59805 +3752 59802 +22909 59794 +33421 59791 +43923 59788 +22253 59785 +30503 59777 +44525 59772 +34164 59772 +19548 59770 +26258 59768 +16667 59767 +25062 59765 +41161 59755 +17007 59751 +43728 59748 +48269 59743 +7890 59742 +22186 59740 +15627 59733 +20867 59729 +44437 59723 +22017 59722 +31195 59722 +25457 59721 +33775 59719 +44192 59717 +22415 59711 +22631 59708 +42614 59705 +8146 59693 +6564 59688 +22366 59684 +25181 59682 +3692 59681 +11469 59681 +37216 59677 +20403 59676 +36788 59673 +13051 59672 +9777 59668 +39777 59660 +38979 59657 +35101 59653 +31829 59642 +30490 59641 +34311 59634 +33588 59631 +39042 59629 +15086 59629 +42351 59619 +22127 59613 +20446 59613 +32193 59611 +22487 59610 +24696 59607 +23691 59598 +2355 59598 +22470 59594 +30632 59593 +16066 59586 +22994 59578 +18611 59575 +16206 59575 +30720 59574 +19524 59571 +42997 59570 +13759 59568 +44871 59566 +23759 59565 +39909 59558 +21967 59549 +30814 59546 +28416 59542 +15409 59539 +25453 59537 +21444 59536 +41282 59530 +23662 59528 +30447 59521 +42386 59519 +36506 59517 +43084 59516 +31889 59513 +6310 59502 +30860 59487 +43771 59483 +45023 59478 +7019 59472 +44354 59465 +33983 59464 +41790 59457 +6263 59445 +25530 59441 +31321 59441 +32468 59441 +24233 59440 +48130 59438 +19685 59431 +5919 59426 +25904 59423 +29912 59421 +9214 59415 +26412 59413 +31392 59409 +15308 59408 +30030 59404 +24938 59402 +25177 59394 +30113 59390 +39421 59380 +29895 59379 +637 59379 +29731 59377 +25024 59376 +23760 59371 +4924 59370 +47312 59370 +19107 59368 +7978 59362 +26219 59359 +38351 59358 +10015 59354 +49737 59352 +2122 59351 +32814 59351 +9242 59350 +21958 59347 +11086 59347 +35831 59334 +8307 59333 +1425 59327 +36219 59316 +15046 59308 +23752 59307 +18640 59302 +16696 59300 +29491 59298 +21228 59290 +17533 59289 +15625 59288 +42692 59287 +48421 59280 +31831 59278 +23387 59264 +4284 59263 +7896 59262 +24707 59254 +16962 59245 +17963 59238 +34102 59236 +22370 59236 +34365 59235 +44677 59233 +34967 59231 +20476 59229 +21047 59228 +24221 59226 +37604 59224 +10110 59224 +35311 59223 +16331 59221 +24080 59216 +12140 59213 +18287 59204 +28087 59203 +36745 59201 +19933 59201 +46527 59199 +1579 59198 +24873 59187 +11327 59179 +8453 59171 +17280 59162 +38462 59156 +27430 59155 +26029 59149 +32480 59149 +26289 59148 +39720 59144 +25525 59137 +39337 59135 +43675 59128 +37484 59115 +44691 59114 +19943 59106 +32295 59101 +39733 59099 +24544 59095 +43855 59091 +50024 59086 +4288 59084 +4699 59083 +33198 59083 +42541 59070 +46488 59067 +30473 59066 +31922 59065 +28347 59052 +39566 59035 +14255 59033 +42167 59029 +26221 59028 +24314 59027 +34123 59026 +20057 59025 +36866 59020 +14915 59014 +33163 59014 +30519 59013 +13806 59001 +31365 59000 +27091 58999 +29417 58999 +43775 58997 +42290 58994 +27869 58990 +26648 58982 +22025 58979 +37438 58979 +42845 58975 +18756 58971 +46997 58969 +10611 58969 +12494 58967 +40425 58967 +50251 58964 +47686 58963 +20323 58958 +33976 58945 +49486 58945 +37899 58943 +24848 58943 +44660 58932 +15451 58930 +42207 58925 +46089 58925 +30829 58919 +30130 58917 +26855 58915 +33779 58912 +28476 58907 +33009 58904 +34161 58898 +23405 58896 +21185 58889 +41524 58885 +35774 58884 +50089 58880 +43945 58873 +23482 58867 +26241 58853 +24090 58852 +4055 58849 +25751 58830 +21617 58829 +36761 58820 +24889 58820 +42599 58819 +38028 58818 +9809 58816 +47220 58809 +35577 58807 +41556 58807 +27410 58805 +27870 58805 +36608 58803 +24501 58798 +28902 58792 +45083 58791 +38879 58785 +18792 58785 +22579 58780 +22618 58773 +19577 58770 +38223 58764 +41870 58746 +17731 58745 +34818 58744 +25616 58740 +42436 58738 +29237 58738 +35110 58734 +20918 58733 +39459 58725 +28738 58715 +37463 58713 +39812 58709 +46767 58707 +14306 58705 +28783 58701 +20733 58699 +9962 58697 +44236 58696 +23278 58694 +29898 58694 +35209 58689 +34854 58685 +36438 58681 +25804 58680 +46007 58680 +26698 58677 +26875 58672 +22503 58672 +3140 58670 +40894 58669 +16793 58662 +44970 58661 +28844 58652 +34988 58638 +40442 58638 +33269 58635 +15147 58634 +29327 58633 +21695 58622 +20368 58617 +30485 58613 +40934 58611 +40389 58609 +21935 58608 +32141 58606 +31155 58593 +46463 58590 +35374 58589 +41701 58588 +30588 58588 +7556 58582 +35651 58581 +48014 58574 +10951 58573 +22775 58569 +20839 58564 +42994 58559 +41643 58555 +21887 58552 +22217 58545 +39291 58543 +26031 58542 +2455 58537 +25966 58536 +33596 58532 +46123 58530 +45633 58527 +35719 58524 +36830 58521 +41480 58517 +20058 58513 +23303 58513 +31095 58511 +40950 58509 +28317 58508 +18050 58492 +44108 58487 +36642 58468 +19655 58465 +41457 58463 +27562 58462 +33085 58453 +39871 58444 +19296 58443 +30937 58443 +7162 58440 +26877 58436 +23182 58431 +44848 58423 +9228 58421 +21529 58418 +44493 58418 +24040 58414 +36457 58410 +40929 58405 +3639 58404 +26880 58403 +3490 58403 +12282 58393 +30726 58389 +36514 58387 +42293 58387 +42928 58378 +26658 58374 +32215 58370 +10199 58369 +31698 58368 +37824 58366 +25432 58364 +35717 58359 +24490 58347 +20644 58343 +29493 58341 +47328 58335 +26061 58334 +49457 58333 +3796 58323 +14164 58318 +30002 58318 +36586 58317 +26986 58314 +28606 58312 +22040 58310 +43989 58306 +14019 58297 +24421 58293 +7483 58288 +18850 58283 +17627 58279 +35014 58255 +32881 58255 +12137 58253 +39708 58249 +20632 58244 +27010 58234 +49318 58218 +9896 58209 +19924 58190 +33315 58190 +40665 58189 +49537 58186 +28305 58184 +6020 58183 +23694 58181 +50135 58179 +41213 58174 +30275 58170 +44559 58165 +36944 58162 +40264 58158 +30853 58143 +41034 58141 +30051 58128 +25108 58126 +21326 58123 +40424 58118 +47665 58118 +26100 58118 +26186 58117 +33824 58116 +21002 58116 +19567 58105 +19420 58099 +31818 58099 +47928 58097 +31403 58095 +34851 58094 +17259 58085 +23649 58080 +4612 58079 +49622 58074 +27306 58073 +23961 58067 +17213 58060 +23112 58048 +41326 58037 +26668 58033 +32872 58029 +38302 58027 +32333 58025 +27931 58021 +37915 58020 +47689 58020 +45239 58013 +21290 58011 +28429 58011 +16841 58007 +26582 58002 +46451 58002 +32483 58001 +24178 57999 +30607 57998 +38493 57996 +40915 57983 +38099 57982 +48453 57976 +43474 57970 +37051 57967 +24260 57964 +25798 57964 +32095 57962 +34056 57956 +29861 57953 +21554 57950 +23179 57940 +8448 57938 +23810 57932 +36162 57927 +10483 57927 +40977 57926 +44150 57920 +28502 57918 +45587 57910 +21422 57909 +30592 57908 +32613 57895 +13623 57890 +37599 57890 +42127 57890 +26969 57887 +18368 57885 +21265 57877 +27268 57871 +29345 57862 +20709 57862 +45775 57858 +46278 57852 +16969 57849 +16799 57847 +37718 57845 +1859 57835 +39257 57835 +32677 57812 +23258 57812 +36896 57812 +15710 57809 +34177 57809 +14721 57805 +41429 57804 +19591 57799 +30239 57797 +37976 57793 +33871 57788 +35654 57781 +33555 57780 +24725 57779 +20477 57769 +29652 57766 +2966 57764 +37562 57760 +15898 57751 +24228 57750 +13010 57750 +11159 57746 +9851 57740 +48591 57739 +40228 57735 +16475 57733 +34552 57731 +46691 57726 +35589 57719 +37538 57717 +3333 57717 +46905 57712 +24521 57712 +32453 57712 +29361 57708 +36292 57703 +23706 57697 +48500 57690 +24560 57689 +24082 57684 +37356 57680 +34010 57679 +5507 57677 +31449 57675 +36583 57673 +21239 57664 +25260 57662 +5656 57660 +43310 57659 +45031 57659 +39828 57655 +39619 57654 +26192 57654 +7540 57648 +27163 57642 +39722 57639 +8675 57634 +30176 57634 +28451 57630 +42417 57625 +20730 57612 +38175 57608 +31522 57604 +29405 57604 +31421 57592 +38242 57585 +49165 57573 +46507 57569 +30415 57568 +33611 57531 +9493 57527 +30300 57527 +41648 57526 +21606 57519 +30775 57514 +29254 57513 +33285 57511 +29955 57497 +47458 57491 +17701 57484 +28882 57476 +25913 57473 +28885 57472 +26798 57469 +10641 57466 +46368 57462 +49325 57455 +34356 57454 +39616 57454 +28539 57452 +14322 57451 +21169 57445 +25631 57441 +34029 57439 +20450 57438 +41045 57437 +30524 57435 +7956 57430 +43701 57424 +36644 57420 +31265 57420 +40718 57412 +5663 57408 +34034 57406 +17855 57404 +32521 57402 +29151 57392 +45108 57388 +36541 57386 +28506 57372 +37751 57368 +19200 57366 +6540 57363 +13102 57358 +23244 57352 +9956 57349 +20752 57346 +2796 57338 +21960 57327 +21593 57325 +27684 57319 +34954 57316 +28602 57313 +30255 57309 +9503 57289 +92 57268 +35622 57267 +34763 57264 +23603 57258 +39654 57255 +20151 57254 +33265 57249 +47879 57242 +10841 57235 +45496 57232 +24070 57230 +33658 57225 +34262 57220 +28521 57218 +35113 57216 +33835 57211 +32906 57205 +7213 57201 +11417 57197 +14270 57194 +31977 57187 +17950 57178 +38045 57170 +42368 57164 +4799 57161 +26042 57158 +40837 57146 +29265 57144 +14455 57141 +48714 57140 +21976 57126 +23601 57117 +25541 57116 +24494 57115 +15327 57113 +43999 57107 +7640 57103 +35437 57099 +36327 57091 +37352 57086 +10055 57082 +14269 57078 +44978 57077 +44333 57074 +44042 57065 +34735 57059 +31734 57056 +32679 57052 +32132 57050 +38814 57042 +5233 57039 +2131 57037 +23349 57036 +23713 57034 +38935 57024 +7549 57023 +39255 57022 +36205 57015 +23181 57007 +12392 57000 +4611 56997 +47697 56991 +10445 56985 +46884 56981 +30132 56975 +15412 56974 +16899 56965 +7757 56961 +30042 56960 +43349 56955 +35006 56948 +45966 56948 +25004 56945 +14274 56929 +29267 56926 +31027 56918 +23164 56918 +24182 56915 +25292 56902 +24928 56902 +29360 56900 +44343 56899 +42476 56895 +32701 56892 +36995 56889 +34428 56886 +25509 56886 +41430 56885 +2571 56885 +45774 56882 +18911 56874 +14603 56874 +43748 56867 +45247 56860 +29229 56854 +29466 56847 +30761 56839 +44128 56834 +35786 56825 +43379 56824 +24338 56822 +23666 56814 +28980 56814 +24760 56811 +45658 56806 +42627 56806 +20779 56803 +30777 56791 +37610 56790 +38244 56789 +21688 56787 +8467 56787 +47609 56787 +47518 56786 +17539 56785 +23292 56784 +25677 56778 +6331 56778 +12410 56776 +4182 56760 +22580 56753 +23153 56752 +46429 56749 +37439 56748 +2715 56743 +23848 56742 +25565 56740 +27605 56739 +39490 56731 +22156 56730 +18459 56725 +26497 56719 +41898 56715 +25325 56711 +41252 56710 +18967 56709 +44483 56703 +35658 56701 +44019 56697 +35362 56684 +38435 56682 +43479 56681 +47186 56681 +45644 56679 +27762 56677 +37884 56675 +30935 56659 +25309 56658 +6959 56658 +29456 56655 +28639 56646 +31731 56635 +49627 56635 +32774 56628 +10855 56627 +21450 56620 +25266 56612 +30508 56607 +37990 56606 +30179 56602 +14920 56602 +29653 56601 +26718 56596 +48988 56590 +26744 56589 +41778 56586 +37373 56586 +11944 56581 +21584 56570 +37761 56569 +13978 56568 +22395 56567 +21795 56564 +14565 56563 +9610 56556 +45795 56554 +16465 56554 +40421 56554 +38423 56551 +28625 56551 +22595 56550 +45209 56538 +28903 56537 +20938 56528 +6003 56526 +49197 56525 +20467 56516 +26980 56515 +44718 56514 +37318 56513 +26274 56511 +25398 56509 +18111 56505 +26771 56503 +44500 56502 +36791 56501 +15078 56500 +39845 56496 +8443 56486 +45400 56483 +16571 56481 +31395 56477 +36096 56475 +15157 56474 +36656 56459 +20694 56450 +25421 56435 +15170 56430 +24913 56430 +27479 56430 +27592 56429 +9444 56424 +39406 56421 +23097 56418 +11970 56414 +38608 56412 +33340 56409 +43425 56406 +15729 56403 +19197 56400 +15721 56391 +1783 56378 +27719 56373 +43995 56372 +37954 56368 +28109 56358 +35803 56358 +33482 56352 +35337 56350 +32882 56344 +4815 56338 +39034 56337 +47608 56334 +33181 56327 +20806 56323 +21295 56321 +28593 56319 +44067 56314 +16909 56311 +49037 56310 +22986 56305 +29452 56304 +24662 56297 +9682 56294 +47015 56286 +24831 56285 +7349 56285 +36074 56282 +33973 56279 +32736 56278 +22638 56278 +11645 56278 +26379 56277 +12663 56274 +18846 56271 +38398 56269 +42158 56265 +42395 56261 +41120 56258 +49842 56258 +48781 56253 +32254 56248 +47012 56247 +22196 56247 +5944 56245 +21755 56232 +30033 56219 +47164 56219 +20798 56212 +28633 56207 +19363 56204 +39451 56202 +40559 56195 +20807 56192 +20831 56189 +37198 56188 +21783 56184 +33736 56183 +32472 56179 +38583 56177 +28916 56177 +38820 56176 +40088 56171 +30668 56136 +20281 56136 +47586 56130 +18271 56125 +9685 56124 +25769 56120 +11190 56116 +33946 56112 +39630 56109 +32578 56106 +27079 56102 +12632 56092 +11402 56091 +29957 56084 +16199 56073 +29406 56069 +36448 56063 +45942 56060 +26271 56059 +19932 56054 +40870 56054 +27217 56052 +7015 56046 +33600 56044 +33255 56038 +11865 56033 +25280 56033 +26444 56031 +25696 56031 +2065 56019 +45897 56008 +17459 56005 +6008 56004 +11671 56003 +44366 56001 +24041 55998 +1488 55996 +32530 55992 +35923 55988 +11218 55985 +31599 55983 +30080 55976 +21693 55973 +43811 55970 +40745 55968 +33247 55956 +33907 55953 +30069 55944 +11194 55942 +14036 55942 +25466 55940 +36610 55939 +37121 55938 +3391 55932 +21074 55932 +28286 55931 +39965 55927 +26384 55926 +12752 55921 +16814 55920 +22613 55915 +15096 55898 +16670 55896 +25190 55889 +29871 55885 +18603 55880 +27134 55878 +15051 55871 +17878 55869 +43859 55861 +7315 55860 +35326 55852 +11522 55852 +5378 55849 +27364 55837 +27107 55833 +35210 55831 +19311 55830 +18879 55824 +5955 55818 +23523 55816 +26009 55814 +32848 55813 +5015 55811 +26509 55809 +23159 55806 +44250 55802 +3346 55802 +29802 55802 +24369 55802 +40855 55798 +28090 55795 +39205 55793 +26974 55791 +28359 55790 +13984 55790 +37714 55786 +29663 55782 +24993 55780 +20768 55775 +19334 55762 +49112 55756 +37084 55748 +42408 55740 +10570 55739 +27845 55732 +37329 55724 +16927 55723 +23729 55720 +4311 55715 +17622 55707 +42760 55700 +37325 55691 +7293 55688 +20981 55688 +16340 55686 +34632 55682 +36070 55679 +14500 55678 +50066 55675 +25148 55670 +14652 55668 +20436 55668 +25390 55658 +29385 55652 +39670 55647 +24718 55646 +23243 55641 +23208 55638 +26012 55637 +18993 55632 +37514 55622 +46049 55621 +29527 55618 +30038 55617 +21054 55616 +19770 55610 +24984 55608 +11572 55606 +32710 55597 +36512 55593 +23604 55588 +39343 55587 +39150 55583 +41069 55580 +28829 55578 +28034 55573 +28046 55565 +41950 55564 +41328 55563 +24420 55562 +32151 55558 +43653 55551 +25068 55547 +29093 55545 +32560 55544 +38808 55541 +45851 55541 +45982 55540 +43308 55535 +21579 55532 +25036 55520 +34110 55510 +47261 55508 +41764 55506 +10482 55505 +33627 55504 +20937 55497 +6720 55495 +48968 55483 +22228 55479 +20073 55476 +24651 55475 +41046 55474 +43858 55473 +2023 55471 +42221 55463 +19630 55459 +40776 55457 +23678 55455 +25411 55446 +28713 55443 +43042 55438 +22853 55436 +38400 55433 +33025 55431 +49108 55431 +18676 55426 +24588 55426 +35106 55418 +44923 55414 +24720 55411 +30276 55404 +43688 55403 +27675 55399 +12668 55398 +40801 55396 +37708 55386 +33839 55382 +19008 55378 +5547 55375 +25749 55373 +24706 55368 +10333 55361 +18239 55361 +26917 55359 +26452 55350 +33710 55348 +26494 55342 +38927 55341 +26521 55341 +36132 55339 +30423 55339 +30912 55336 +42573 55334 +36989 55319 +1159 55317 +26785 55317 +25179 55316 +6254 55315 +9886 55315 +37322 55315 +41360 55313 +32084 55306 +4215 55303 +21093 55295 +41817 55292 +42431 55288 +35924 55287 +36440 55283 +37254 55279 +36778 55270 +41683 55269 +28203 55268 +21836 55268 +28580 55267 +19569 55260 +33848 55259 +28563 55259 +36358 55240 +39350 55237 +38102 55229 +29981 55228 +34840 55222 +37728 55218 +49375 55217 +12582 55217 +28385 55203 +44385 55201 +9606 55200 +27903 55194 +42334 55191 +34395 55188 +13349 55188 +29089 55184 +24773 55181 +34629 55179 +30020 55178 +38651 55178 +3380 55177 +24051 55176 +35935 55175 +38191 55172 +27224 55154 +17866 55153 +46107 55151 +37206 55147 +37314 55142 +28295 55130 +27414 55129 +39206 55129 +46362 55127 +12613 55119 +24013 55119 +37465 55110 +46849 55105 +46132 55100 +43911 55096 +30204 55092 +44470 55088 +33388 55088 +14854 55084 +43881 55084 +15227 55080 +28450 55073 +44922 55068 +24815 55067 +21478 55067 +49696 55063 +19402 55060 +28273 55059 +24119 55057 +31375 55056 +26646 55054 +30013 55048 +29009 55044 +50106 55032 +22153 55030 +35130 55022 +47545 55019 +49006 55019 +39019 55013 +31878 54992 +37745 54990 +9731 54987 +2786 54984 +1237 54980 +15084 54978 +18908 54970 +37808 54965 +38283 54963 +32327 54961 +36066 54961 +37752 54960 +35860 54960 +40931 54957 +33517 54955 +37768 54955 +21748 54952 +22584 54951 +27362 54948 +49570 54941 +23507 54936 +21192 54934 +34012 54933 +3604 54928 +49407 54927 +39834 54919 +32111 54909 +46921 54896 +35546 54890 +38730 54887 +33291 54885 +14578 54872 +30623 54867 +34905 54864 +13402 54851 +25365 54845 +45747 54842 +41352 54842 +24053 54840 +43997 54836 +33466 54832 +36274 54831 +28423 54820 +27704 54818 +24071 54814 +28616 54814 +5554 54807 +1945 54798 +29914 54797 +41493 54795 +19851 54794 +32502 54786 +31087 54777 +24673 54771 +49247 54770 +27043 54766 +24039 54762 +19980 54751 +30902 54750 +40513 54747 +49195 54742 +44282 54741 +6962 54740 +33446 54738 +42397 54735 +49185 54735 +31501 54730 +35368 54728 +34573 54726 +41454 54719 +16417 54718 +16422 54702 +42239 54701 +38337 54695 +18681 54683 +1442 54682 +45813 54682 +6329 54675 +11965 54674 +41875 54669 +42803 54656 +47692 54656 +14906 54654 +18349 54651 +29579 54648 +7046 54648 +30696 54642 +30229 54641 +37113 54638 +28568 54637 +47470 54636 +29689 54632 +23917 54627 +38031 54626 +44117 54623 +17848 54622 +49873 54615 +15655 54615 +43052 54614 +158 54613 +7106 54607 +6395 54603 +22995 54602 +30522 54599 +30827 54589 +35233 54585 +35573 54579 +32281 54578 +31174 54577 +3787 54571 +34782 54569 +33201 54556 +44864 54556 +31413 54556 +6474 54555 +50219 54539 +22699 54535 +2210 54524 +4163 54521 +30357 54521 +27142 54520 +24646 54519 +28572 54516 +29673 54514 +23135 54512 +32387 54512 +16279 54498 +43251 54489 +6775 54488 +24571 54483 +33799 54482 +22963 54472 +16661 54472 +46116 54471 +38796 54465 +15851 54456 +37305 54454 +27101 54452 +33630 54448 +42857 54446 +43593 54445 +18148 54437 +29128 54422 +27721 54421 +17046 54415 +26210 54413 +20483 54411 +48994 54409 +31884 54408 +16945 54405 +23452 54403 +36232 54402 +19839 54401 +30413 54400 +11227 54394 +37551 54379 +13875 54376 +26637 54372 +13691 54362 +17458 54359 +21626 54349 +26808 54348 +19880 54341 +25806 54340 +10687 54336 +27695 54334 +34958 54334 +42971 54330 +42718 54327 +29850 54320 +20279 54312 +28682 54304 +38964 54284 +42358 54281 +31998 54278 +31531 54265 +29983 54262 +19179 54261 +40153 54257 +33124 54257 +3145 54254 +10845 54253 +25229 54249 +2418 54241 +25288 54238 +13649 54236 +29243 54236 +1684 54235 +27864 54231 +30386 54227 +41758 54226 +17008 54222 +46194 54218 +26819 54216 +17693 54216 +9633 54216 +40457 54213 +12830 54211 +49862 54210 +29704 54209 +13268 54209 +22405 54207 +40514 54205 +6780 54204 +25695 54200 +8600 54185 +42626 54183 +17380 54181 +25360 54178 +15581 54176 +35082 54174 +39585 54163 +30953 54160 +16371 54148 +28849 54147 +16653 54143 +10867 54143 +16491 54137 +38716 54134 +27888 54133 +49637 54129 +19780 54129 +13409 54124 +6718 54118 +8570 54118 +23048 54115 +25501 54100 +37408 54097 +31075 54093 +13436 54090 +39095 54089 +30064 54085 +33584 54078 +20931 54076 +16272 54070 +36188 54070 +42658 54069 +31003 54068 +39543 54067 +47404 54064 +16360 54062 +23375 54058 +25478 54054 +38886 54052 +6800 54049 +28729 54035 +7537 54035 +43968 54033 +45488 54033 +41262 54032 +22819 54032 +25006 54032 +49179 54032 +45159 54028 +42813 54028 +14951 54027 +15385 54026 +20609 54023 +24888 54021 +46346 54011 +25473 54010 +32595 54009 +35605 54006 +22792 54005 +46924 53997 +15003 53994 +49980 53992 +19004 53991 +9037 53986 +40727 53986 +18861 53980 +41314 53971 +22965 53970 +44953 53968 +32807 53966 +30740 53952 +47137 53950 +26903 53934 +27996 53933 +42157 53933 +40595 53933 +30690 53932 +22220 53931 +12496 53928 +5560 53928 +41440 53923 +36847 53922 +24046 53919 +19216 53915 +37264 53915 +38510 53911 +24810 53909 +24124 53906 +16283 53900 +37476 53899 +26143 53896 +45524 53883 +29889 53882 +35606 53878 +14599 53877 +22373 53874 +31465 53866 +31670 53855 +27395 53855 +44348 53848 +23011 53847 +14618 53847 +46081 53846 +34208 53839 +2916 53837 +24785 53828 +47358 53827 +45861 53821 +28285 53818 +40570 53809 +12907 53808 +37079 53803 +42569 53792 +18882 53788 +29729 53785 +18444 53782 +34047 53777 +28746 53774 +35159 53772 +23989 53771 +16702 53767 +34470 53762 +9767 53761 +21875 53758 +36909 53757 +42077 53751 +19641 53747 +40497 53744 +30767 53737 +15106 53732 +30220 53732 +37009 53730 +29154 53726 +13039 53723 +693 53718 +23069 53718 +29959 53714 +42211 53714 +26359 53711 +3655 53710 +30048 53694 +28790 53688 +28701 53687 +42427 53682 +33773 53681 +35196 53676 +42916 53673 +8544 53661 +37063 53660 +26893 53655 +44633 53654 +19156 53654 +35019 53645 +40355 53644 +23408 53643 +21378 53639 +29109 53638 +21252 53638 +16984 53632 +27078 53629 +39712 53628 +24896 53624 +17017 53619 +26788 53616 +38870 53615 +12072 53614 +35730 53613 +34216 53613 +43459 53609 +20247 53600 +19145 53600 +5738 53598 +28023 53598 +11505 53582 +30545 53572 +21949 53572 +22974 53571 +11802 53566 +26600 53561 +28932 53556 +23419 53553 +43022 53552 +26300 53550 +22711 53549 +1932 53546 +32650 53542 +36892 53536 +33846 53534 +34527 53531 +9453 53529 +24343 53528 +2133 53526 +26408 53521 +19482 53516 +40598 53513 +34858 53513 +29944 53509 +23499 53509 +40332 53509 +13653 53506 +30135 53503 +29724 53501 +19648 53498 +49383 53497 +45483 53495 +4344 53489 +26837 53478 +35413 53477 +30630 53469 +25025 53464 +40924 53457 +30686 53455 +34292 53453 +29932 53445 +32693 53445 +40447 53439 +44886 53438 +25607 53437 +47461 53435 +46233 53432 +45680 53429 +45777 53428 +20225 53424 +25805 53422 +42568 53421 +34491 53420 +19708 53419 +39028 53418 +33506 53415 +23304 53414 +27804 53410 +36747 53397 +29219 53395 +28678 53393 +26910 53391 +35583 53388 +38264 53387 +45517 53387 +27926 53385 +7181 53382 +45088 53382 +48854 53378 +28805 53378 +31390 53376 +41027 53375 +22890 53369 +28898 53368 +29576 53368 +44178 53364 +21654 53362 +24764 53356 +49019 53352 +24977 53350 +49134 53349 +14120 53348 +48886 53346 +45263 53346 +29117 53341 +24502 53339 +20107 53335 +37105 53333 +33565 53329 +24691 53326 +21870 53320 +47756 53316 +40377 53314 +37896 53313 +42645 53308 +22910 53306 +38934 53284 +26770 53282 +38039 53278 +12398 53275 +37406 53273 +29481 53269 +10244 53259 +35138 53258 +25596 53250 +22464 53242 +48209 53237 +14344 53233 +46269 53231 +45676 53229 +16149 53216 +15200 53214 +6689 53213 +27433 53210 +26074 53207 +47216 53205 +29762 53200 +15952 53198 +7249 53195 +26306 53194 +26827 53185 +28554 53185 +22495 53185 +2226 53182 +35023 53182 +31645 53181 +23199 53180 +27482 53178 +34599 53178 +20191 53177 +33067 53174 +44404 53171 +34666 53171 +30792 53170 +47974 53170 +16886 53170 +32150 53168 +50151 53164 +34070 53163 +39133 53148 +22577 53145 +43588 53145 +25503 53145 +24556 53143 +26270 53130 +37890 53124 +12514 53120 +17379 53106 +13200 53101 +25610 53099 +34671 53093 +140 53091 +45078 53087 +27654 53083 +21910 53078 +42404 53064 +18232 53057 +28669 53054 +46929 53051 +19329 53049 +25336 53042 +41746 53036 +34304 53034 +43092 53028 +22478 53022 +20465 53014 +32420 53012 +22042 53003 +12067 53001 +41535 53000 +27601 52999 +36364 52989 +39127 52985 +5569 52983 +28436 52975 +13450 52973 +35287 52972 +36601 52968 +28947 52967 +29028 52961 +47714 52961 +21467 52957 +244 52954 +29624 52945 +20845 52944 +34502 52941 +35980 52941 +30392 52936 +28005 52935 +26926 52930 +23536 52926 +48320 52918 +40471 52914 +29030 52908 +44870 52900 +45183 52897 +47499 52897 +2009 52895 +26133 52884 +29883 52883 +31445 52877 +25754 52873 +34579 52856 +43809 52853 +40583 52852 +41074 52852 +36444 52850 +31405 52844 +15087 52843 +43657 52840 +21905 52838 +31363 52837 +48498 52832 +42453 52829 +21129 52825 +39730 52824 +16843 52823 +32104 52822 +16072 52816 +44588 52815 +31541 52815 +12154 52809 +20219 52802 +34383 52800 +16021 52799 +5343 52799 +14355 52789 +39083 52788 +24109 52785 +39569 52775 +20029 52769 +46133 52765 +41731 52761 +12628 52748 +40753 52744 +10489 52743 +24235 52736 +34234 52736 +28518 52736 +22659 52725 +43738 52724 +27360 52720 +33667 52719 +35918 52717 +6826 52717 +30301 52717 +22817 52715 +426 52710 +34260 52700 +35782 52699 +43524 52693 +7289 52692 +26708 52692 +32325 52690 +39465 52687 +44610 52686 +1391 52686 +22624 52685 +32162 52667 +46034 52666 +40847 52662 +42805 52661 +18896 52661 +14400 52660 +37928 52659 +18673 52648 +23146 52640 +27429 52639 +22416 52632 +40212 52625 +33719 52625 +33305 52619 +21616 52619 +27544 52595 +21940 52589 +28251 52588 +9076 52588 +40789 52577 +28007 52574 +42058 52572 +33906 52563 +25138 52560 +33050 52558 +25879 52557 +47535 52556 +39983 52546 +22629 52543 +25304 52537 +40302 52527 +31609 52523 +21937 52520 +14849 52518 +18754 52518 +39435 52515 +48378 52513 +21874 52510 +21696 52510 +32673 52504 +45215 52500 +35510 52499 +45910 52497 +27834 52496 +33731 52489 +35118 52483 +28308 52479 +33757 52476 +30500 52475 +28610 52464 +41559 52463 +24680 52451 +44663 52449 +32932 52449 +17472 52447 +31177 52443 +43934 52431 +16442 52430 +48115 52429 +20348 52427 +40851 52424 +28131 52421 +33364 52420 +21103 52419 +43348 52417 +19184 52416 +27152 52410 +22672 52409 +45059 52406 +40151 52402 +4443 52397 +49270 52393 +36000 52391 +29541 52391 +23444 52390 +13147 52384 +46837 52383 +29622 52383 +23609 52379 +35239 52370 +36317 52362 +2255 52360 +2162 52359 +11275 52357 +49362 52356 +30791 52352 +27406 52346 +31825 52344 +42152 52339 +24982 52336 +22124 52336 +28739 52334 +19545 52332 +32783 52327 +41772 52322 +24813 52321 +31638 52319 +35146 52312 +10624 52311 +20032 52310 +50050 52305 +45867 52302 +48249 52302 +32631 52298 +8506 52297 +33158 52291 +32063 52290 +20370 52286 +19948 52285 +3172 52285 +19473 52284 +49661 52284 +23656 52278 +32262 52270 +23908 52268 +9943 52263 +33093 52257 +21311 52257 +32784 52257 +32554 52256 +37358 52253 +32911 52252 +41077 52248 +31969 52246 +26749 52244 +33348 52244 +37370 52235 +42724 52234 +23127 52234 +41546 52229 +9741 52228 +24869 52222 +29574 52220 +45594 52219 +47905 52217 +41594 52214 +22234 52206 +34180 52205 +24297 52203 +40476 52201 +23871 52200 +14885 52199 +15245 52190 +7086 52188 +24727 52183 +28581 52180 +6845 52180 +21379 52176 +24354 52173 +11042 52171 +30645 52170 +11712 52166 +14656 52159 +40879 52156 +22209 52156 +47318 52152 +46212 52149 +42433 52143 +21600 52139 +11883 52139 +21657 52137 +25928 52133 +36912 52131 +38252 52126 +35025 52123 +22450 52118 +36846 52117 +29617 52111 +42710 52110 +30993 52108 +26339 52103 +48334 52095 +20135 52089 +19058 52087 +47058 52085 +21671 52085 +21972 52077 +41085 52076 +43276 52074 +42236 52065 +36455 52063 +38479 52063 +34762 52062 +31967 52061 +26760 52058 +24653 52055 +24551 52052 +31420 52051 +32598 52048 +13034 52046 +20549 52045 +10743 52042 +16455 52038 +1905 52035 +8172 52034 +21717 52032 +42888 52026 +34172 52023 +13240 52016 +17164 52015 +27124 52015 +9083 52013 +32857 52002 +39732 52001 +25090 52000 +32837 51993 +17310 51987 +27089 51983 +28939 51981 +21095 51980 +35107 51967 +28139 51965 +35865 51963 +43423 51955 +23462 51949 +46584 51949 +26179 51944 +28030 51942 +40084 51942 +5240 51939 +20764 51937 +46595 51936 +31328 51936 +35849 51934 +45415 51932 +35031 51930 +37089 51917 +43867 51914 +25441 51904 +19918 51896 +13660 51895 +27394 51894 +42640 51893 +48576 51891 +24652 51885 +47647 51872 +34537 51870 +20123 51868 +19973 51868 +25049 51866 +38458 51864 +32931 51863 +33332 51860 +30821 51858 +38309 51853 +29853 51842 +44194 51831 +25237 51830 +21440 51828 +40676 51823 +44703 51823 +25209 51822 +19617 51821 +28504 51820 +22440 51819 +11858 51811 +38207 51809 +35690 51803 +22583 51802 +31665 51800 +23395 51795 +38695 51794 +36134 51789 +47983 51788 +30170 51788 +41675 51786 +17550 51784 +40500 51782 +9994 51782 +22521 51779 +34477 51778 +21963 51777 +47972 51777 +37237 51767 +25917 51765 +33473 51765 +22850 51763 +31933 51763 +19835 51763 +37789 51761 +31874 51757 +36780 51755 +1599 51750 +37468 51740 +29366 51740 +36289 51731 +34392 51731 +47290 51722 +33259 51719 +16658 51714 +32384 51713 +17941 51713 +14390 51703 +23904 51703 +19955 51700 +38736 51694 +42609 51694 +33865 51693 +40426 51673 +7005 51663 +11590 51655 +38597 51652 +20418 51644 +14020 51643 +33401 51643 +44956 51638 +32543 51626 +48722 51623 +36426 51621 +31888 51616 +4728 51615 +48946 51612 +19971 51611 +5809 51604 +34426 51599 +19619 51599 +43976 51593 +15288 51591 +31378 51588 +33841 51587 +27073 51583 +45552 51582 +28719 51582 +33275 51581 +23169 51580 +33134 51573 +27978 51572 +18942 51572 +49068 51570 +28178 51569 +33575 51569 +20389 51560 +8140 51560 +39246 51558 +26942 51555 +1991 51554 +12405 51541 +33002 51539 +34358 51531 +46795 51531 +4555 51518 +36283 51518 +31236 51518 +11600 51509 +42729 51506 +9171 51503 +25579 51502 +814 51501 +49408 51493 +13341 51489 +41575 51489 +41698 51484 +23469 51467 +47680 51463 +15612 51463 +27979 51458 +50054 51457 +44459 51457 +37557 51452 +18738 51450 +27795 51448 +8010 51447 +30580 51441 +21174 51434 +45978 51434 +30878 51425 +32283 51425 +44537 51420 +42179 51416 +30059 51408 +29567 51407 +25643 51404 +24976 51403 +26507 51402 +45316 51399 +36775 51395 +47706 51393 +27635 51393 +45744 51392 +17831 51391 +39561 51387 +17993 51384 +13358 51383 +49474 51382 +32379 51378 +40124 51370 +26466 51368 +21386 51368 +13563 51360 +49470 51355 +26666 51353 +31935 51352 +30594 51351 +10455 51345 +31663 51333 +46413 51331 +27001 51329 +45047 51329 +24826 51328 +36224 51313 +27062 51308 +47451 51304 +44882 51300 +36168 51297 +38290 51296 +46184 51291 +36129 51291 +22818 51284 +46892 51280 +18347 51273 +20106 51271 +3983 51270 +25326 51268 +38916 51261 +16724 51260 +46685 51260 +14059 51259 +31822 51256 +25337 51255 +13776 51255 +36464 51245 +45962 51245 +47474 51241 +16961 51238 +10293 51234 +25829 51227 +32959 51227 +44413 51224 +47437 51223 +16240 51217 +21786 51209 +9844 51209 +29754 51205 +45651 51197 +19970 51179 +22134 51177 +9339 51165 +34996 51164 +37319 51155 +24431 51155 +17489 51153 +47225 51146 +20809 51141 +28942 51141 +33733 51140 +30274 51139 +30346 51138 +36856 51134 +19365 51129 +26405 51124 +28053 51104 +29227 51103 +13247 51102 +24555 51100 +25080 51088 +18486 51085 +23983 51085 +24900 51081 +39386 51079 +21526 51079 +25761 51078 +35294 51075 +28621 51071 +30137 51062 +34557 51061 +34501 51061 +41715 51057 +15370 51052 +30436 51050 +31582 51049 +41088 51049 +39774 51047 +50214 51040 +25916 51040 +44322 51029 +38656 51025 +40794 51024 +37194 51021 +27627 51020 +16939 51015 +24931 51014 +29440 51010 +32306 51008 +37803 51007 +34246 51006 +41135 51002 +35949 50997 +29296 50997 +1066 50992 +2738 50991 +36250 50981 +38026 50975 +48133 50974 +40603 50972 +22308 50965 +36433 50954 +17328 50953 +9843 50953 +32090 50951 +5251 50944 +44851 50934 +23044 50932 +28750 50925 +34130 50924 +11141 50919 +28241 50917 +20706 50916 +3590 50913 +24953 50912 +24417 50908 +38000 50905 +44044 50905 +35864 50898 +7146 50894 +27826 50894 +8021 50891 +48694 50888 +48756 50887 +35595 50885 +36312 50870 +25291 50855 +15547 50852 +31459 50850 +44998 50845 +41191 50841 +9219 50826 +16437 50813 +32792 50812 +14878 50805 +7512 50797 +25037 50796 +30158 50787 +23534 50784 +46760 50782 +12215 50775 +40304 50772 +45926 50771 +14535 50770 +48470 50762 +33586 50761 +8715 50761 +34678 50761 +34531 50760 +39636 50758 +149 50753 +14587 50750 +24570 50749 +15701 50748 +48684 50747 +38494 50732 +34933 50731 +45221 50726 +28471 50726 +29097 50723 +41329 50723 +39554 50718 +48664 50712 +32766 50697 +28210 50694 +6519 50692 +7053 50688 +17966 50686 +28895 50684 +39517 50683 +33836 50671 +14753 50663 +35280 50660 +8422 50654 +18424 50653 +7489 50648 +21766 50645 +49554 50637 +34890 50631 +14677 50628 +41598 50626 +24550 50623 +40975 50623 +14250 50622 +22334 50622 +18407 50613 +7860 50598 +26298 50597 +38336 50596 +17139 50591 +34723 50591 +19260 50591 +32244 50586 +19950 50581 +32740 50577 +48506 50573 +37843 50572 +23568 50571 +15513 50571 +27351 50564 +30778 50556 +37671 50554 +21081 50550 +21973 50548 +8927 50545 +23016 50544 +26633 50537 +2519 50537 +2452 50533 +46985 50530 +29176 50521 +11217 50514 +24140 50505 +45683 50504 +26434 50502 +42908 50502 +22881 50500 +37005 50484 +49411 50476 +25762 50476 +27318 50473 +28866 50470 +38236 50468 +23053 50467 +7528 50466 +30862 50464 +45808 50462 +12832 50455 +43943 50448 +44833 50448 +47930 50444 +7983 50440 +46117 50435 +37069 50427 +27150 50422 +12935 50421 +31132 50418 +37315 50417 +29387 50414 +28548 50413 +33091 50413 +36816 50412 +36497 50409 +16142 50409 +28754 50402 +27014 50395 +36494 50392 +46234 50389 +21044 50387 +29480 50380 +43183 50377 +27338 50373 +18833 50371 +31679 50370 +31109 50367 +27168 50366 +22654 50362 +33474 50355 +25047 50354 +30865 50354 +34291 50353 +33343 50348 +26250 50345 +36994 50342 +41379 50341 +26929 50336 +24464 50328 +35515 50322 +26477 50320 +33955 50315 +14633 50314 +47908 50314 +24218 50313 +36628 50313 +37953 50310 +22924 50309 +41494 50309 +28655 50296 +26881 50289 +28135 50289 +43081 50275 +30810 50272 +30959 50261 +20566 50259 +21592 50255 +22008 50255 +27164 50249 +44127 50245 +49939 50241 +44216 50237 +40996 50230 +19291 50228 +22463 50225 +31295 50220 +47528 50218 +14007 50214 +46106 50210 +31354 50208 +26413 50206 +36569 50204 +28993 50199 +21072 50194 +36701 50191 +22969 50191 +32697 50186 +10915 50183 +28806 50178 +1796 50175 +22150 50168 +28560 50164 +16601 50150 +6975 50149 +6315 50149 +17651 50140 +43268 50139 +37930 50136 +20458 50134 +34154 50133 +27460 50129 +43029 50127 +6728 50123 +18217 50117 +28368 50114 +29835 50107 +44209 50105 +29642 50098 +40947 50092 +38625 50091 +48195 50079 +35259 50077 +29000 50077 +22524 50075 +20402 50071 +43070 50067 +10228 50059 +25474 50054 +22960 50054 +26934 50051 +39538 50048 +14298 50043 +23560 50040 +45773 50032 +29358 50031 +3678 50028 +22029 50028 +25505 50024 +36195 50015 +24757 50015 +28498 50006 +9492 50005 +20812 50005 +49523 49984 +39604 49984 +25985 49971 +14414 49966 +27367 49965 +4850 49956 +50224 49950 +29098 49944 +44038 49938 +46008 49937 +32909 49937 +11871 49933 +29997 49933 +7497 49929 +20647 49911 +15520 49911 +34252 49911 +30408 49909 +49653 49906 +34321 49904 +23313 49903 +37197 49902 +34929 49901 +29974 49898 +16704 49893 +27793 49884 +41777 49884 +9132 49873 +34342 49867 +20473 49864 +15951 49864 +18294 49863 +28009 49861 +19149 49861 +19833 49859 +36272 49857 +17003 49853 +17961 49851 +33307 49843 +2145 49840 +27114 49840 +12303 49838 +19730 49834 +35372 49831 +24641 49823 +24987 49820 +24298 49815 +33331 49815 +28002 49811 +19142 49808 +24650 49805 +43581 49805 +26783 49800 +28685 49797 +43543 49791 +30980 49791 +27499 49784 +46864 49783 +29076 49783 +25885 49779 +33941 49777 +22980 49777 +36665 49775 +21139 49763 +29380 49762 +27735 49761 +46562 49759 +30747 49759 +31555 49758 +45711 49758 +28545 49747 +21738 49746 +44654 49734 +36071 49732 +21632 49723 +39916 49723 +45132 49721 +18527 49720 +47479 49718 +39533 49714 +47246 49714 +18231 49708 +13560 49705 +10684 49703 +28389 49700 +5709 49699 +20916 49692 +43178 49690 +36714 49689 +11365 49683 +48064 49679 +44379 49675 +26332 49674 +27736 49672 +2037 49665 +30261 49665 +47725 49661 +30353 49656 +39567 49653 +33019 49650 +12418 49640 +27623 49639 +32984 49638 +23378 49635 +26279 49634 +20781 49629 +32724 49624 +49266 49623 +22603 49622 +25386 49617 +46988 49608 +41471 49598 +15140 49597 +31994 49596 +44579 49595 +45264 49589 +20728 49587 +47681 49584 +35223 49584 +28796 49577 +35454 49574 +36540 49572 +34069 49565 +31247 49565 +30367 49565 +21956 49560 +20596 49555 +25330 49548 +39620 49547 +33381 49545 +27625 49542 +24604 49522 +41987 49519 +7279 49518 +8170 49515 +31186 49508 +49769 49496 +28733 49492 +37648 49490 +43590 49484 +34355 49484 +28774 49483 +14435 49476 +4264 49476 +5786 49473 +27317 49473 +36240 49472 +42687 49470 +25783 49463 +42251 49460 +40973 49454 +44466 49448 +25110 49444 +29765 49442 +34613 49442 +30310 49441 +27997 49432 +36054 49431 +20909 49427 +43244 49427 +32520 49414 +30967 49413 +21212 49412 +38748 49408 +34597 49406 +38085 49403 +25856 49402 +46305 49400 +25439 49400 +10122 49386 +46834 49384 +29894 49383 +10341 49374 +13190 49373 +25743 49369 +45614 49367 +31331 49365 +46882 49365 +16994 49363 +48487 49361 +28887 49360 +23674 49356 +43039 49352 +31250 49350 +44135 49348 +36483 49342 +26467 49341 +12324 49341 +23385 49336 +5494 49330 +43974 49327 +24183 49324 +23059 49319 +22808 49316 +3385 49311 +39159 49311 +36197 49310 +24147 49304 +6924 49302 +8930 49301 +33045 49292 +19429 49288 +38591 49285 +25739 49285 +39476 49283 +29723 49269 +25069 49269 +36270 49268 +38376 49265 +42478 49265 +46855 49263 +42070 49262 +13208 49255 +30834 49251 +1784 49245 +33427 49243 +31719 49240 +14346 49240 +15885 49235 +46903 49229 +12428 49225 +35866 49223 +4872 49218 +13363 49213 +35135 49211 +12604 49209 +41391 49206 +34485 49198 +39735 49197 +24063 49190 +3097 49186 +36682 49185 +43144 49184 +19065 49181 +32344 49177 +25744 49174 +22557 49169 +33168 49166 +17482 49164 +32510 49162 +42815 49158 +25986 49154 +36686 49154 +43106 49152 +21045 49150 +36293 49150 +44188 49148 +17946 49128 +44636 49125 +28324 49121 +15783 49119 +32195 49119 +22716 49115 +18351 49113 +37726 49108 +29778 49103 +31070 49101 +16441 49100 +37731 49095 +45094 49089 +21152 49085 +28381 49076 +28068 49076 +41334 49076 +9044 49068 +16200 49064 +36110 49063 +5089 49060 +26006 49059 +24483 49044 +43400 49042 +24582 49038 +47755 49036 +28596 49032 +27843 49027 +48253 49023 +34205 49021 +50209 49013 +19742 49006 +42734 49005 +32600 49004 +39614 49000 +29373 48996 +29604 48993 +19694 48991 +47170 48989 +45759 48988 +28027 48981 +46930 48975 +23660 48974 +19796 48973 +30574 48968 +41716 48956 +42737 48956 +45588 48956 +26580 48950 +38408 48948 +47978 48944 +21201 48944 +36717 48942 +37632 48938 +14709 48933 +36294 48933 +28704 48933 +12212 48926 +29016 48916 +36212 48912 +46854 48910 +35495 48905 +25702 48904 +2867 48903 +36114 48901 +40343 48900 +34715 48895 +35183 48886 +31015 48884 +21123 48884 +16596 48880 +19112 48876 +30587 48871 +25859 48867 +27631 48852 +25791 48847 +29887 48846 +12920 48846 +16620 48844 +44043 48839 +24470 48838 +49132 48837 +38674 48837 +27441 48831 +23094 48828 +17721 48826 +10903 48825 +31274 48824 +22511 48824 +19620 48819 +21892 48813 +47817 48808 +18651 48808 +27045 48804 +19427 48799 +23334 48799 +31261 48798 +17336 48796 +48199 48791 +9787 48786 +37221 48772 +43579 48768 +36351 48764 +36242 48761 +43127 48760 +14253 48757 +35199 48750 +32772 48748 +25922 48745 +28854 48743 +30930 48743 +32219 48728 +28144 48724 +20198 48721 +19753 48720 +32793 48715 +48039 48715 +26664 48713 +25644 48711 +29708 48709 +26110 48708 +13972 48703 +39397 48694 +38914 48689 +19794 48689 +21680 48676 +37116 48673 +39005 48672 +22073 48657 +31608 48655 +29403 48647 +25242 48646 +33154 48641 +28202 48637 +26535 48633 +35094 48632 +37184 48631 +28952 48630 +35862 48624 +46516 48622 +29091 48620 +33764 48618 +33226 48617 +41220 48613 +4428 48613 +19542 48612 +33541 48610 +38034 48608 +31983 48606 +22057 48597 +22746 48593 +27241 48574 +25533 48572 +4300 48571 +40611 48569 +46395 48566 +18949 48559 +29332 48557 +27630 48555 +37813 48554 +31597 48550 +43274 48548 +25602 48546 +34541 48540 +28025 48538 +24611 48538 +12415 48533 +13262 48531 +25459 48527 +49553 48527 +34874 48527 +46057 48527 +49372 48523 +38866 48509 +24778 48508 +34224 48505 +22437 48495 +45468 48490 +17975 48490 +24256 48482 +29031 48476 +37882 48475 +39163 48473 +40255 48472 +26811 48466 +36450 48465 +1852 48465 +27121 48464 +19367 48463 +19961 48459 +44518 48455 +44655 48454 +40793 48454 +12464 48453 +34384 48452 +34902 48450 +36172 48441 +16339 48441 +29079 48437 +10760 48437 +33469 48435 +27587 48432 +36776 48426 +35208 48425 +19054 48425 +25516 48424 +34078 48422 +41973 48414 +11122 48412 +38173 48400 +4584 48399 +34882 48397 +26277 48391 +34434 48390 +29683 48382 +7752 48377 +45983 48376 +40910 48374 +31848 48373 +39769 48366 +12222 48364 +49752 48364 +23726 48363 +17019 48362 +26366 48359 +37091 48357 +36991 48356 +49851 48341 +22374 48341 +47572 48339 +21309 48338 +23337 48332 +28262 48329 +29246 48326 +49598 48325 +26130 48321 +18690 48315 +38594 48314 +23487 48314 +12573 48312 +31017 48310 +45736 48307 +21273 48305 +8648 48300 +29355 48295 +41089 48293 +7672 48292 +28714 48291 +20661 48276 +34912 48270 +45568 48263 +8671 48262 +44485 48260 +2690 48257 +27912 48256 +45494 48256 +21645 48246 +36559 48242 +46567 48239 +30622 48235 +26557 48229 +44523 48227 +36960 48227 +19867 48223 +10461 48221 +17723 48219 +29549 48218 +44862 48218 +24904 48218 +45142 48216 +23100 48213 +37279 48209 +9887 48209 +46470 48205 +18230 48202 +45511 48202 +48850 48202 +39037 48199 +33119 48197 +24814 48195 +42792 48194 +9042 48189 +39685 48187 +23253 48186 +34276 48182 +43373 48177 +43120 48174 +36193 48173 +5090 48171 +49902 48167 +24894 48161 +36833 48160 +43663 48159 +37248 48150 +28799 48150 +42898 48146 +32656 48145 +39339 48140 +47199 48135 +31068 48133 +35455 48131 +21725 48119 +47916 48119 +41336 48115 +33271 48114 +33977 48108 +35008 48107 +49432 48106 +30383 48106 +24975 48102 +47773 48099 +23506 48097 +33133 48091 +36612 48090 +5306 48090 +36998 48065 +37073 48064 +20206 48062 +11934 48060 +32728 48056 +45425 48054 +16341 48052 +36281 48044 +17363 48035 +35465 48033 +15880 48030 +16357 48025 +21527 48023 +22275 48023 +33502 48015 +13672 48014 +41679 48012 +31986 48012 +28867 48011 +33755 48009 +24776 48006 +8357 48004 +37292 47998 +34746 47994 +37027 47992 +40519 47990 +36279 47980 +6417 47979 +36083 47978 +28456 47977 +31114 47970 +38527 47969 +34567 47968 +39302 47965 +45504 47964 +25600 47960 +32377 47960 +27712 47958 +33424 47957 +10811 47948 +28771 47935 +35262 47924 +26539 47917 +29282 47917 +3643 47915 +39412 47915 +18255 47913 +14750 47910 +38088 47905 +6398 47904 +40470 47903 +45246 47901 +40662 47899 +24504 47899 +41229 47898 +49292 47898 +46160 47896 +35309 47896 +47329 47892 +40453 47890 +26352 47889 +22682 47885 +21262 47885 +39441 47884 +27238 47884 +28378 47879 +14132 47878 +21969 47877 +31533 47876 +33851 47875 +30582 47875 +22342 47874 +15075 47874 +40955 47871 +35332 47870 +27244 47864 +26181 47863 +22832 47858 +43956 47853 +29846 47852 +2697 47844 +20666 47844 +26281 47843 +18268 47843 +9928 47836 +2083 47836 +32350 47834 +37342 47831 +21315 47829 +27132 47825 +15809 47819 +38987 47819 +34375 47811 +47668 47809 +41836 47806 +25284 47803 +17273 47802 +39862 47800 +16311 47791 +1476 47789 +47853 47786 +39027 47783 +48213 47783 +8012 47782 +33063 47765 +27445 47759 +34849 47745 +41970 47743 +42170 47742 +42876 47742 +1161 47739 +36742 47733 +47151 47729 +13115 47725 +18301 47725 +35272 47723 +27219 47708 +3416 47707 +28177 47699 +38738 47699 +29770 47687 +26465 47686 +38383 47685 +22069 47680 +21456 47679 +14174 47674 +23539 47667 +46247 47665 +34459 47663 +22179 47659 +22449 47658 +42791 47654 +39603 47653 +50169 47652 +26667 47651 +44498 47645 +44868 47642 +28876 47640 +26701 47638 +5109 47637 +19327 47637 +31064 47634 +30573 47633 +35905 47625 +45232 47624 +43616 47621 +28217 47618 +48642 47617 +27086 47617 +42992 47611 +42327 47611 +23688 47609 +26282 47598 +36099 47596 +40852 47596 +43766 47594 +33297 47593 +49466 47591 +17209 47585 +43670 47582 +24748 47581 +32353 47571 +48342 47569 +38757 47561 +32905 47561 +32085 47556 +9250 47542 +35729 47542 +37663 47540 +16112 47538 +33694 47537 +31556 47535 +28277 47531 +25104 47526 +38429 47525 +39074 47524 +40404 47523 +46739 47521 +28940 47519 +27329 47518 +23753 47514 +29041 47510 +31267 47506 +35163 47506 +18150 47505 +33159 47503 +24811 47492 +22148 47488 +25342 47487 +38812 47487 +39283 47486 +25808 47483 +15486 47483 +12808 47482 +30103 47481 +19159 47481 +32572 47477 +30430 47475 +25881 47464 +37486 47459 +27506 47446 +28334 47432 +21561 47418 +18801 47417 +30180 47414 +39383 47411 +25587 47410 +28091 47408 +41626 47405 +47228 47403 +29419 47402 +1480 47400 +30363 47399 +33319 47394 +43558 47393 +27946 47391 +16623 47391 +37749 47390 +31271 47390 +18619 47388 +44003 47374 +21458 47373 +29181 47371 +38119 47362 +31807 47362 +10963 47356 +28060 47355 +46922 47354 +37331 47351 +27577 47345 +32884 47333 +22515 47327 +19853 47322 +19354 47319 +40431 47314 +16275 47311 +5915 47310 +44339 47303 +35489 47299 +43398 47294 +35096 47293 +18799 47293 +46577 47288 +29005 47287 +25534 47286 +27657 47281 +20298 47279 +42299 47272 +27613 47271 +29341 47270 +45723 47268 +38908 47263 +37164 47253 +38260 47252 +1251 47243 +47848 47234 +35817 47231 +28453 47230 +28878 47227 +5065 47223 +20214 47217 +13220 47216 +25649 47214 +38186 47210 +46822 47209 +34693 47207 +17827 47207 +19520 47204 +19756 47202 +24249 47201 +32514 47200 +2844 47198 +49416 47195 +36874 47186 +39962 47184 +37800 47183 +30669 47181 +43100 47180 +25671 47175 +15450 47174 +28888 47160 +34079 47156 +19484 47156 +42274 47154 +6278 47154 +37793 47153 +33816 47153 +31693 47149 +47218 47148 +33858 47146 +40689 47137 +21287 47137 +40105 47133 +30571 47133 +32402 47132 +38009 47131 +24294 47129 +22930 47120 +19707 47115 +34941 47114 +42425 47112 +19498 47111 +28609 47109 +34134 47104 +44056 47100 +19976 47094 +17678 47084 +47562 47083 +15310 47073 +16458 47068 +37228 47064 +23756 47063 +15590 47060 +19355 47056 +42906 47056 +45512 47056 +32529 47052 +11593 47052 +31528 47052 +21713 47051 +28880 47045 +20800 47043 +22291 47033 +40511 47032 +48569 47016 +48060 47011 +43799 47003 +20113 47001 +36579 46998 +36748 46996 +34263 46996 +49520 46994 +38686 46986 +35177 46971 +42967 46970 +14336 46967 +45184 46965 +47413 46965 +34789 46965 +45139 46964 +3065 46962 +23921 46961 +28687 46955 +32654 46950 +44738 46946 +35153 46942 +32761 46937 +17647 46935 +46150 46934 +38761 46931 +35355 46921 +12454 46913 +30876 46903 +3552 46902 +27458 46902 +23277 46901 +10465 46898 +35662 46897 +16643 46895 +41794 46895 +39524 46892 +42899 46892 +39164 46887 +36101 46879 +11370 46878 +12221 46873 +32293 46870 +29641 46869 +28424 46863 +28309 46861 +13775 46860 +39767 46856 +29343 46855 +34155 46853 +35932 46852 +46875 46850 +50173 46846 +35426 46843 +7615 46840 +42004 46840 +46447 46837 +28786 46834 +37048 46829 +32122 46829 +29280 46827 +1091 46827 +10709 46824 +42571 46820 +30468 46812 +35526 46807 +28807 46786 +6059 46785 +9313 46784 +43652 46779 +39640 46776 +48568 46764 +5957 46763 +25194 46763 +15432 46761 +50186 46760 +26135 46759 +36360 46755 +11019 46751 +30097 46749 +27952 46749 +32808 46748 +3935 46742 +25584 46738 +47377 46725 +26996 46725 +24134 46725 +24709 46720 +23585 46717 +36865 46715 +31910 46710 +16854 46708 +36093 46703 +48853 46698 +19900 46698 +43104 46694 +31349 46692 +41238 46686 +21521 46678 +49412 46678 +44026 46677 +45332 46677 +14406 46675 +25371 46664 +43257 46655 +33656 46655 +46861 46652 +27201 46649 +24944 46646 +29101 46642 +32343 46636 +17999 46634 +21306 46629 +32294 46629 +49104 46628 +23581 46624 +30875 46614 +48794 46612 +45645 46611 +45516 46610 +32396 46608 +20922 46608 +38406 46606 +11321 46605 +49612 46603 +35441 46600 +49190 46592 +9805 46592 +48701 46584 +30968 46582 +30066 46577 +46340 46575 +31377 46575 +35901 46567 +22208 46565 +21232 46562 +19108 46557 +45542 46557 +21194 46551 +35631 46546 +42580 46546 +29619 46542 +40943 46540 +44614 46537 +26957 46535 +32393 46532 +27377 46527 +47219 46524 +26975 46524 +24415 46523 +19680 46517 +40530 46513 +30811 46500 +30264 46491 +37054 46491 +18011 46489 +23250 46486 +21499 46482 +37818 46472 +22032 46469 +19831 46467 +34498 46463 +38833 46459 +12732 46458 +25710 46456 +29433 46452 +31207 46449 +47272 46447 +34569 46444 +33018 46438 +5530 46432 +18040 46428 +39891 46423 +28033 46423 +47353 46423 +33471 46415 +16678 46414 +35941 46414 +34191 46402 +18970 46402 +15183 46398 +17220 46397 +35180 46394 +13665 46383 +23126 46383 +32556 46382 +23087 46378 +47295 46377 +47196 46366 +31043 46364 +8110 46360 +13494 46358 +40698 46356 +40363 46355 +31168 46353 +35182 46351 +11143 46351 +34935 46346 +34122 46340 +27791 46338 +31172 46333 +40702 46330 +6172 46330 +16682 46322 +49597 46319 +23564 46319 +28590 46318 +31894 46316 +32542 46310 +24299 46303 +23214 46302 +6202 46299 +25598 46299 +13350 46299 +30548 46293 +49825 46288 +40062 46287 +34862 46283 +21163 46283 +20637 46283 +47560 46276 +30381 46274 +11599 46272 +27717 46270 +6120 46269 +21242 46269 +20025 46268 +33431 46263 +23447 46262 +36319 46260 +46980 46258 +32806 46258 +15008 46253 +29950 46251 +33570 46249 +14894 46239 +26761 46237 +19270 46234 +32921 46233 +42818 46228 +7671 46225 +37644 46222 +27254 46222 +47984 46221 +32729 46216 +2909 46212 +40657 46210 +29621 46209 +33713 46208 +14106 46206 +26034 46200 +22204 46196 +14265 46195 +11035 46193 +42437 46189 +44302 46189 +49967 46178 +44301 46177 +22633 46176 +40490 46165 +36930 46164 +8320 46159 +18093 46152 +11528 46151 +37026 46151 +35045 46149 +35370 46147 +30168 46146 +4772 46145 +43747 46137 +41130 46132 +34076 46130 +35088 46124 +16505 46123 +37428 46120 +20638 46115 +43007 46112 +43944 46109 +16258 46106 +22567 46098 +36214 46098 +23807 46093 +46972 46091 +9939 46091 +25384 46088 +29283 46087 +37293 46082 +7296 46079 +26707 46079 +36513 46072 +26173 46070 +32794 46069 +37891 46068 +42886 46058 +28328 46053 +44131 46048 +29397 46040 +42232 46039 +40741 46034 +26000 46034 +12205 46032 +37457 46032 +20964 46031 +19252 46030 +26753 46030 +32479 46029 +39078 46024 +25419 46023 +27642 46023 +45827 46017 +42097 46016 +32773 46015 +24227 46013 +35842 46009 +49300 46004 +32204 46002 +37554 46000 +23589 45991 +45046 45989 +48742 45987 +30355 45987 +34038 45986 +15307 45982 +39784 45980 +39987 45978 +39956 45976 +28388 45976 +22823 45976 +26473 45974 +22024 45969 +48437 45964 +38960 45958 +39716 45952 +31545 45952 +18458 45949 +45344 45949 +33561 45942 +34875 45941 +15861 45939 +32120 45939 +35639 45935 +32769 45930 +26990 45927 +43953 45925 +28954 45923 +38540 45916 +50172 45916 +40033 45912 +31448 45909 +39067 45905 +44769 45904 +29094 45900 +44768 45900 +18638 45900 +47244 45896 +29722 45892 +39960 45887 +46330 45887 +24372 45887 +16761 45865 +28107 45865 +8154 45864 +28031 45863 +36111 45861 +47950 45860 +32894 45859 +13908 45858 +47513 45857 +34151 45856 +27404 45855 +20453 45853 +14214 45848 +20751 45847 +43066 45844 +48901 45838 +20454 45834 +28271 45830 +35988 45827 +17134 45825 +11669 45820 +34243 45820 +47722 45818 +26471 45818 +39131 45806 +28448 45805 +46033 45802 +34316 45800 +20300 45799 +37676 45798 +34195 45795 +7550 45791 +26063 45787 +39420 45776 +30232 45771 +28183 45769 +26641 45765 +44818 45764 +40192 45764 +43176 45763 +36145 45762 +12942 45758 +26463 45756 +45878 45751 +26153 45749 +12192 45748 +37347 45747 +20979 45745 +44846 45740 +24843 45739 +42324 45735 +44220 45734 +28858 45732 +28351 45729 +27151 45728 +35011 45726 +26592 45726 +39235 45724 +27198 45724 +21721 45721 +37975 45715 +29442 45714 +17860 45699 +36383 45699 +23268 45698 +38754 45697 +37900 45696 +48709 45695 +37261 45695 +9926 45687 +45238 45687 +34152 45684 +2397 45683 +26333 45682 +37550 45681 +40966 45675 +31151 45673 +27769 45670 +8687 45664 +30185 45662 +27412 45661 +15787 45653 +44820 45649 +33536 45648 +33900 45647 +38705 45646 +25417 45642 +21767 45642 +22362 45640 +39678 45636 +50220 45628 +35073 45627 +41523 45625 +14065 45624 +24344 45624 +43153 45623 +31749 45622 +44652 45618 +36058 45615 +45413 45615 +39453 45611 +40641 45609 +27126 45601 +40136 45601 +36910 45601 +49628 45598 +32920 45597 +43492 45595 +49323 45589 +18005 45585 +29211 45582 +21762 45581 +49796 45577 +23831 45574 +36308 45571 +27758 45570 +40835 45569 +34486 45557 +37234 45553 +11720 45552 +34885 45548 +31629 45546 +46644 45537 +27253 45531 +26919 45525 +22136 45522 +29924 45521 +31489 45520 +49987 45520 +22564 45510 +29546 45510 +11537 45509 +10415 45508 +19259 45507 +24572 45499 +12304 45497 +40606 45494 +47748 45493 +42329 45492 +33790 45490 +15854 45490 +24170 45487 +45958 45477 +18076 45476 +41740 45473 +23550 45470 +38854 45468 +16183 45461 +41113 45458 +11055 45456 +35940 45455 +36394 45452 +21064 45451 +46544 45449 +9673 45449 +31059 45449 +9548 45448 +36769 45446 +28737 45442 +40307 45440 +23672 45439 +34663 45437 +23162 45434 +38689 45431 +42139 45426 +45200 45424 +40900 45424 +17620 45423 +38878 45422 +47657 45421 +39493 45401 +33182 45401 +19467 45398 +30795 45397 +16260 45391 +32367 45390 +35550 45386 +49341 45385 +40461 45380 +18219 45374 +8638 45371 +26484 45361 +21447 45360 +19941 45353 +10554 45352 +1823 45347 +29369 45345 +24702 45344 +13255 45341 +15098 45339 +29462 45334 +39373 45334 +41529 45333 +36941 45332 +23945 45328 +16010 45325 +50028 45324 +41928 45319 +21396 45318 +27162 45318 +49655 45310 +37222 45308 +26526 45303 +19852 45300 +27742 45296 +33105 45293 +4247 45291 +47410 45289 +37327 45285 +22717 45285 +46958 45282 +41053 45278 +20109 45272 +25514 45268 +6425 45263 +23272 45241 +21856 45239 +27036 45239 +24168 45230 +43939 45229 +15379 45223 +21352 45223 +25903 45223 +26949 45221 +30019 45218 +31130 45216 +41484 45215 +15125 45210 +32993 45207 +25685 45203 +38048 45200 +16911 45199 +45049 45195 +38932 45191 +9143 45188 +35794 45186 +37743 45183 +42186 45178 +33464 45174 +19172 45170 +17594 45164 +30551 45164 +36288 45164 +17133 45162 +27371 45162 +26093 45154 +48335 45154 +24309 45147 +28747 45147 +31991 45146 +38862 45145 +28978 45145 +29062 45144 +26887 45143 +25591 45143 +8145 45141 +29735 45139 +35906 45138 +23426 45132 +42850 45127 +36098 45122 +22035 45122 +35822 45120 +47573 45116 +26692 45113 +2361 45108 +15878 45107 +48687 45101 +24513 45096 +33270 45094 +22458 45092 +5145 45089 +26704 45087 +38378 45077 +26822 45071 +21515 45070 +43533 45069 +25830 45065 +42753 45063 +6940 45049 +28192 45048 +34683 45045 +32123 45042 +10035 45039 +44486 45035 +5042 45034 +29051 45032 +40859 45030 +7580 45025 +40687 45024 +30506 45013 +28959 45012 +42802 45012 +32816 45007 +22826 44996 +9474 44990 +31955 44989 +33187 44989 +49232 44988 +31202 44986 +9781 44978 +39076 44977 +39308 44974 +34965 44971 +24094 44964 +228 44960 +30549 44958 +23175 44953 +35613 44952 +17727 44938 +29985 44937 +11369 44930 +48604 44924 +22210 44923 +47877 44920 +10698 44909 +25797 44905 +20414 44904 +34030 44904 +27666 44903 +40904 44898 +39652 44897 +42828 44896 +27573 44894 +24288 44891 +49498 44886 +20485 44883 +24158 44882 +14558 44881 +17237 44879 +6293 44877 +22245 44876 +39679 44873 +13921 44870 +42490 44862 +38041 44860 +22911 44854 +46690 44854 +39128 44852 +39826 44851 +27109 44840 +23055 44839 +18875 44838 +19678 44838 +41520 44837 +5135 44836 +35584 44835 +30246 44828 +45222 44827 +24143 44827 +39411 44818 +49901 44817 +30559 44817 +48592 44815 +33963 44813 +23191 44808 +45892 44797 +30621 44795 +32154 44793 +25964 44793 +7949 44790 +34019 44787 +9243 44787 +1420 44786 +36703 44785 +42587 44782 +13131 44771 +46825 44767 +26820 44766 +9745 44766 +2498 44761 +42606 44758 +17817 44754 +18487 44752 +29524 44752 +32160 44750 +38189 44750 +39085 44745 +36587 44742 +10858 44739 +42378 44730 +31142 44728 +42495 44724 +19461 44719 +16806 44718 +15452 44716 +35873 44714 +39778 44711 +28801 44710 +26782 44709 +34480 44705 +33224 44703 +1269 44703 +32435 44702 +16364 44699 +36712 44699 +44913 44696 +11318 44691 +23537 44690 +11787 44689 +34903 44688 +21599 44683 +33896 44682 +27746 44679 +30568 44673 +30627 44670 +26576 44668 +43273 44658 +18107 44654 +29805 44652 +35429 44650 +49668 44648 +15411 44640 +47169 44633 +32581 44631 +37324 44631 +33344 44627 +25245 44625 +20084 44614 +33612 44606 +32167 44606 +19251 44605 +23684 44597 +7316 44597 +33732 44594 +23234 44593 +29086 44588 +12998 44588 +37378 44584 +31428 44584 +35212 44582 +47747 44581 +41630 44577 +26524 44575 +30086 44574 +16578 44571 +26978 44571 +47162 44567 +30844 44566 +4717 44565 +46662 44563 +14002 44556 +21259 44556 +9295 44554 +21441 44549 +33097 44546 +25498 44544 +30982 44543 +7921 44543 +27393 44541 +20914 44540 +27829 44539 +48073 44537 +22981 44534 +41909 44533 +232 44532 +34966 44528 +48028 44524 +31103 44522 +31450 44521 +22154 44521 +3924 44512 +17045 44511 +47830 44509 +43171 44509 +29697 44507 +46505 44502 +32049 44502 +21395 44500 +25307 44497 +45760 44496 +26774 44494 +22763 44486 +38094 44484 +34336 44481 +33852 44479 +46598 44478 +35278 44478 +36915 44475 +23454 44472 +6549 44471 +31648 44466 +22047 44464 +47533 44455 +48127 44454 +36582 44453 +38728 44453 +1184 44453 +23051 44443 +33189 44442 +38590 44435 +48965 44433 +32225 44432 +33418 44428 +41545 44427 +47399 44426 +25129 44425 +22194 44423 +9139 44423 +25929 44422 +32954 44421 +21062 44419 +48368 44416 +18206 44413 +15650 44407 +37829 44404 +27326 44404 +10441 44401 +26818 44398 +30416 44397 +1955 44397 +42917 44396 +37748 44394 +27688 44394 +38777 44394 +10487 44393 +41007 44390 +42610 44378 +35889 44373 +4607 44372 +15815 44371 +41218 44363 +28815 44362 +41980 44359 +38760 44357 +26243 44355 +20177 44355 +30004 44351 +24088 44348 +10121 44346 +45998 44341 +21282 44341 +48279 44340 +22651 44339 +45817 44335 +12754 44332 +23341 44328 +29262 44327 +26217 44324 +36619 44321 +18944 44321 +40459 44320 +50232 44317 +32235 44315 +28875 44314 +22790 44313 +46547 44313 +38176 44310 +8086 44307 +38962 44305 +40891 44299 +30796 44295 +25882 44295 +28826 44294 +35075 44292 +29183 44291 +33935 44290 +38868 44289 +16429 44288 +14418 44287 +43285 44280 +2249 44278 +24705 44274 +41985 44271 +25760 44270 +32630 44269 +14280 44268 +7390 44265 +41819 44261 +26422 44260 +22229 44256 +43150 44253 +28417 44252 +37437 44249 +46474 44245 +39835 44244 +42546 44238 +35471 44234 +10832 44232 +29643 44231 +25921 44227 +17042 44226 +48184 44221 +21497 44212 +25393 44210 +38883 44207 +27166 44207 +23109 44206 +14650 44198 +46673 44197 +40199 44184 +8866 44184 +22481 44183 +28994 44182 +20221 44181 +20305 44181 +27892 44179 +35173 44176 +23891 44174 +34290 44173 +28265 44166 +24487 44164 +32662 44164 +19822 44164 +43493 44160 +14922 44159 +18415 44158 +31137 44154 +29244 44153 +31706 44152 +28629 44152 +22068 44145 +35158 44145 +20766 44142 +36880 44140 +22413 44135 +23801 44135 +10539 44131 +13782 44130 +23671 44125 +23467 44124 +43842 44112 +35879 44109 +20969 44104 +40581 44101 +37692 44100 +31487 44096 +42575 44095 +35238 44095 +35245 44093 +36710 44091 +30530 44088 +41954 44085 +44795 44078 +13420 44076 +39167 44074 +36183 44073 +24857 44067 +9218 44066 +41719 44065 +12194 44063 +45830 44062 +38316 44057 +40772 44057 +26826 44049 +32481 44036 +37371 44030 +39857 44029 +33964 44028 +38794 44027 +46339 44023 +38951 44019 +37278 44017 +18938 44012 +40790 44012 +14902 44011 +33879 44009 +32114 44002 +19854 44002 +29962 44001 +41566 43997 +27405 43980 +33082 43978 +48695 43976 +19501 43972 +45024 43971 +19129 43971 +33260 43971 +43644 43970 +46218 43963 +29791 43962 +47041 43962 +32270 43961 +11285 43960 +44490 43956 +4276 43950 +39675 43948 +39438 43943 +39041 43943 +37029 43941 +26364 43936 +44752 43934 +29830 43928 +48918 43924 +21254 43920 +38291 43916 +46593 43910 +12009 43909 +30471 43901 +47600 43900 +18372 43897 +35424 43887 +37316 43881 +47096 43881 +38368 43877 +39917 43875 +25735 43874 +31283 43872 +28944 43871 +2338 43871 +25334 43867 +18761 43862 +22401 43856 +22507 43853 +32913 43850 +22172 43849 +46221 43847 +19700 43846 +27191 43844 +42458 43838 +28433 43838 +36263 43835 +24818 43828 +35501 43825 +19394 43824 +38775 43821 +35968 43820 +40568 43802 +38003 43798 +30247 43788 +24414 43787 +21533 43786 +29623 43783 +45129 43777 +37973 43774 +34624 43773 +38181 43771 +18016 43766 +31906 43758 +15546 43750 +31012 43748 +32240 43744 +49565 43741 +37660 43741 +46496 43736 +7944 43735 +36151 43734 +40045 43730 +49676 43728 +39824 43726 +42998 43725 +23163 43724 +16972 43722 +30365 43718 +16302 43716 +35534 43711 +36184 43711 +8102 43710 +33589 43709 +32776 43708 +48271 43698 +35473 43696 +16912 43694 +27064 43694 +32237 43692 +31210 43686 +27937 43684 +49906 43684 +38299 43677 +13583 43672 +15312 43671 +44180 43669 +41249 43662 +34099 43659 +37965 43652 +32222 43650 +48292 43647 +32813 43645 +28097 43641 +16185 43641 +28748 43636 +28541 43636 +25183 43632 +36324 43631 +36051 43631 +5280 43627 +31229 43626 +46982 43625 +36708 43624 +34221 43622 +33184 43621 +14868 43619 +20784 43609 +36334 43608 +45527 43607 +44200 43601 +15526 43601 +47766 43599 +24027 43598 +33593 43589 +34759 43587 +32018 43585 +19842 43584 +24605 43582 +36622 43577 +32309 43575 +13926 43563 +14421 43561 +12776 43559 +29563 43556 +31804 43553 +32469 43546 +16018 43542 +15559 43541 +36811 43540 +33729 43534 +48660 43533 +30502 43527 +25703 43522 +23457 43519 +5960 43519 +26956 43499 +36266 43496 +26578 43485 +29637 43484 +34740 43484 +24498 43483 +43457 43482 +35222 43480 +45354 43479 +28179 43476 +28765 43469 +40621 43464 +32067 43462 +18701 43462 +39770 43457 +20296 43457 +17841 43455 +18824 43455 +40677 43453 +16698 43452 +37416 43446 +42657 43444 +15092 43443 +41711 43441 +31856 43438 +25088 43436 +33875 43434 +42409 43433 +45789 43431 +16242 43428 +27936 43426 +45262 43425 +16706 43425 +24747 43423 +46301 43422 +48417 43418 +29899 43417 +43350 43416 +22619 43414 +27895 43410 +46427 43408 +17879 43406 +30477 43403 +29956 43397 +50146 43390 +47443 43389 +45235 43385 +34188 43384 +34127 43381 +42438 43381 +38553 43380 +7314 43380 +43715 43380 +30786 43377 +34884 43376 +49235 43376 +44792 43372 +33680 43370 +40856 43364 +41878 43363 +34170 43363 +31165 43361 +17323 43360 +18328 43358 +48257 43358 +27693 43351 +41893 43349 +34020 43343 +33884 43341 +28032 43339 +28403 43335 +29935 43334 +37317 43333 +32956 43329 +39552 43326 +28965 43322 +27773 43320 +34419 43317 +21333 43317 +22388 43311 +29217 43310 +8433 43307 +15597 43302 +18871 43296 +43833 43295 +38132 43288 +27472 43285 +44441 43285 +31182 43282 +44699 43282 +34837 43276 +35254 43274 +12789 43274 +20589 43271 +22141 43268 +26096 43267 +37247 43262 +26263 43259 +22922 43259 +29338 43255 +31669 43255 +44866 43252 +12305 43251 +37871 43250 +28006 43244 +42504 43242 +29482 43235 +48303 43232 +44972 43232 +47977 43222 +25527 43220 +6361 43218 +48614 43217 +18443 43212 +18228 43212 +26067 43212 +45391 43209 +27400 43206 +50058 43205 +26723 43204 +30184 43202 +33805 43197 +38096 43188 +41225 43184 +16890 43181 +31860 43173 +33516 43172 +36062 43170 +17011 43165 +34994 43163 +39673 43155 +39381 43152 +41756 43151 +32311 43150 +34454 43149 +35405 43148 +42841 43147 +45707 43144 +32995 43137 +22411 43133 +27257 43129 +44429 43129 +8333 43128 +31772 43127 +33292 43123 +42749 43121 +47143 43120 +41175 43119 +28003 43117 +38065 43116 +49462 43116 +45691 43112 +22555 43110 +44595 43110 +32721 43109 +45375 43107 +27780 43106 +26481 43105 +45715 43102 +31780 43099 +21684 43096 +43383 43096 +37432 43093 +34397 43093 +47129 43092 +25948 43087 +20791 43084 +48705 43082 +24322 43079 +33177 43076 +40586 43074 +24823 43067 +25674 43063 +7089 43061 +33763 43060 +43175 43053 +42048 43049 +31779 43046 +24339 43044 +17685 43044 +37451 43044 +40221 43043 +11429 43041 +47770 43039 +28680 43034 +18536 43032 +42116 43026 +7248 43023 +23005 43016 +33877 43012 +13789 43012 +38561 43010 +49756 43006 +40958 43006 +23777 43005 +25900 43004 +44268 43004 +28699 43001 +25135 43001 +27827 42996 +31505 42994 +4546 42994 +23875 42990 +50081 42989 +44941 42985 +27921 42970 +17937 42964 +35947 42961 +33619 42959 +44255 42956 +32423 42955 +31588 42954 +33655 42954 +43075 42953 +28168 42952 +37291 42950 +41269 42950 +21701 42949 +24698 42944 +25550 42944 +9401 42937 +37727 42928 +17119 42924 +37493 42919 +42689 42916 +25470 42915 +28384 42914 +46664 42912 +29368 42904 +44243 42900 +29464 42893 +44874 42892 +34087 42891 +26937 42890 +42410 42886 +33802 42883 +9660 42879 +27000 42870 +42694 42869 +13544 42864 +20636 42857 +22391 42854 +34147 42853 +23588 42852 +26731 42848 +40898 42848 +48256 42844 +18809 42842 +28846 42840 +42142 42835 +38091 42834 +26765 42828 +36641 42826 +29225 42825 +44316 42824 +29138 42819 +44717 42816 +36175 42816 +10155 42813 +26084 42813 +17072 42812 +24111 42810 +31866 42808 +40171 42806 +37625 42804 +16262 42804 +48784 42803 +15054 42801 +29877 42801 +50217 42796 +26460 42794 +49241 42794 +40502 42791 +39511 42791 +6583 42786 +24827 42784 +35456 42783 +29063 42783 +30445 42783 +38225 42779 +19248 42776 +31372 42769 +27798 42769 +38504 42768 +48666 42768 +28963 42766 +42020 42764 +24893 42754 +30079 42754 +27617 42753 +31534 42740 +22389 42738 +35581 42737 +35463 42737 +37367 42734 +36557 42731 +24409 42729 +33720 42727 +36589 42724 +21173 42721 +26832 42714 +47971 42712 +7115 42708 +23120 42706 +20977 42702 +29867 42701 +25713 42697 +40913 42690 +7878 42688 +43770 42687 +25669 42686 +21260 42680 +44630 42677 +37343 42676 +43259 42670 +48516 42669 +37040 42668 +34031 42666 +33068 42665 +23065 42665 +16756 42663 +48222 42657 +23355 42657 +23963 42656 +13441 42655 +49076 42653 +3566 42647 +28909 42644 +32024 42642 +23045 42629 +46699 42624 +42890 42623 +36004 42623 +36566 42622 +23297 42621 +22607 42621 +49852 42618 +29407 42618 +19213 42617 +26607 42616 +13169 42606 +47018 42602 +25993 42602 +44756 42599 +16676 42594 +32692 42591 +37245 42590 +27188 42589 +19071 42588 +37784 42586 +39408 42585 +44554 42585 +25156 42581 +29874 42578 +39536 42578 +44932 42570 +40936 42567 +42774 42565 +36630 42564 +40854 42561 +28055 42556 +31464 42554 +44147 42548 +30011 42548 +32415 42547 +10210 42541 +36355 42541 +44524 42539 +34259 42538 +3922 42538 +30253 42534 +1279 42534 +34940 42533 +27877 42532 +1926 42531 +8633 42531 +36808 42529 +22645 42528 +32234 42528 +38900 42525 +29739 42520 +27462 42509 +29718 42504 +42511 42503 +25402 42497 +36706 42493 +22269 42489 +14452 42483 +33833 42482 +29514 42479 +40982 42469 +10369 42469 +47364 42468 +44731 42466 +22732 42465 +37000 42461 +34202 42455 +28731 42452 +5759 42450 +39676 42448 +44945 42448 +3453 42445 +17193 42445 +24195 42441 +38698 42441 +26115 42436 +15956 42435 +8898 42432 +29789 42431 +18084 42430 +27420 42429 +37565 42429 +45510 42425 +48113 42424 +39631 42424 +29404 42421 +45807 42420 +47741 42418 +42852 42417 +18338 42416 +50073 42414 +28158 42411 +28848 42409 +37100 42409 +3548 42407 +20570 42407 +14423 42403 +35505 42402 +39473 42401 +26571 42398 +30481 42395 +37626 42391 +27591 42389 +33992 42388 +37753 42387 +29605 42385 +24278 42380 +28668 42376 +11440 42372 +29648 42371 +44299 42366 +25578 42360 +23064 42358 +40648 42357 +2803 42357 +39935 42351 +30149 42343 +39080 42343 +22337 42333 +44574 42328 +43074 42324 +26814 42317 +27041 42314 +13323 42314 +29615 42308 +22087 42302 +45928 42297 +48233 42296 +48750 42291 +1427 42285 +28874 42278 +33264 42276 +22886 42272 +36167 42261 +39795 42260 +31581 42246 +29376 42245 +31016 42244 +38682 42241 +16708 42237 +37560 42236 +31474 42234 +47507 42227 +23298 42222 +23188 42222 +30852 42221 +34483 42220 +33233 42214 +17708 42214 +46941 42212 +11984 42209 +30507 42207 +27756 42201 +32553 42198 +37589 42197 +23922 42195 +29608 42194 +33863 42191 +22587 42188 +46714 42184 +23262 42183 +27621 42182 +18940 42179 +48120 42178 +16694 42175 +36160 42173 +33599 42168 +49314 42168 +49157 42166 +26511 42160 +37301 42151 +48138 42149 +22852 42147 +10679 42144 +48066 42143 +32947 42139 +48111 42138 +43573 42133 +26118 42132 +28127 42131 +24137 42130 +7418 42123 +49030 42123 +16091 42120 +36189 42119 +33752 42116 +43542 42116 +34702 42114 +34225 42112 +38769 42110 +43014 42105 +41439 42099 +21405 42099 +25862 42091 +44615 42090 +15126 42090 +30850 42086 +37862 42079 +31025 42078 +26050 42078 +44312 42074 +24286 42070 +35765 42067 +47508 42067 +41151 42067 +26554 42066 +37935 42065 +14477 42065 +20848 42065 +37299 42064 +43086 42063 +26195 42063 +31296 42061 +41354 42054 +38999 42049 +35268 42048 +20356 42039 +1344 42033 +13847 42032 +26114 42031 +35969 42026 +50062 42024 +36684 42023 +40544 42022 +14220 42021 +24804 42016 +35927 42015 +22023 42012 +17875 42006 +12491 42002 +14382 42000 +22327 41999 +41957 41998 +28827 41988 +40795 41986 +26741 41982 +39725 41979 +29172 41978 +35576 41976 +33417 41974 +36415 41973 +47438 41973 +7167 41970 +8430 41967 +34731 41967 +28742 41967 +49377 41966 +41437 41961 +42843 41960 +13500 41957 +35250 41957 +37099 41956 +33970 41954 +31987 41953 +35154 41952 +42450 41948 +44341 41944 +44564 41944 +47122 41943 +41773 41936 +24761 41935 +41126 41935 +25460 41934 +46009 41933 +43190 41933 +28870 41916 +28964 41912 +30023 41911 +32433 41910 +33546 41906 +41318 41902 +48268 41899 +24549 41899 +46587 41893 +4658 41878 +28946 41877 +38068 41876 +46742 41876 +36939 41875 +30648 41873 +32726 41869 +28637 41867 +31085 41866 +49829 41861 +30124 41849 +4194 41845 +27118 41844 +43261 41842 +44158 41841 +21155 41839 +23362 41833 +29592 41827 +44775 41825 +643 41823 +46703 41822 +30161 41821 +49431 41819 +42085 41812 +35197 41807 +29503 41805 +29300 41804 +43965 41801 +36783 41797 +31692 41794 +38616 41791 +49774 41791 +47181 41788 +45450 41784 +42209 41781 +44992 41780 +16350 41778 +37876 41773 +49101 41773 +26795 41767 +33914 41764 +25803 41764 +34372 41762 +43520 41759 +42140 41752 +37620 41742 +40815 41737 +30990 41734 +28467 41730 +37872 41728 +22190 41728 +49589 41727 +18155 41725 +29933 41722 +46412 41720 +40181 41719 +21269 41714 +37993 41709 +22697 41707 +23237 41706 +37764 41706 +24368 41701 +37267 41700 +43381 41699 +24907 41699 +47547 41699 +44160 41698 +20443 41697 +28511 41696 +49595 41696 +49088 41693 +31430 41690 +27256 41689 +35981 41683 +47444 41682 +33662 41680 +31060 41678 +14204 41678 +35650 41677 +47095 41675 +30550 41673 +36550 41670 +49744 41667 +6736 41667 +10187 41666 +40448 41665 +5239 41661 +35547 41660 +38421 41659 +5234 41658 +46609 41649 +16163 41647 +29691 41646 +29038 41645 +22883 41645 +25096 41640 +44284 41637 +14579 41628 +24574 41625 +11993 41622 +47952 41619 +41189 41618 +40280 41617 +40690 41615 +23220 41606 +35701 41605 +35205 41604 +45058 41604 +14304 41602 +18683 41602 +31468 41601 +26247 41600 +49839 41596 +17353 41592 +38631 41591 +37429 41590 +42047 41590 +38801 41590 +26204 41589 +11268 41586 +28658 41584 +42123 41579 +16074 41575 +27487 41572 +46175 41572 +25445 41564 +36953 41562 +41273 41559 +43851 41558 +29633 41556 +15763 41554 +30517 41553 +5324 41552 +36034 41551 +12110 41544 +31113 41543 +27368 41542 +23685 41538 +48590 41530 +35698 41527 +1403 41526 +25010 41525 +28279 41521 +33687 41517 +35308 41516 +47500 41515 +49634 41515 +24181 41513 +47100 41508 +35672 41507 +31598 41504 +29145 41504 +21699 41491 +35616 41490 +46361 41489 +49297 41482 +14522 41481 +33878 41481 +20016 41478 +38005 41473 +29473 41473 +34228 41470 +29952 41470 +28778 41469 +29655 41467 +7569 41467 +45800 41462 +43499 41461 +27176 41460 +6816 41454 +37128 41454 +30041 41453 +17223 41449 +46157 41449 +40185 41446 +34824 41446 +32927 41445 +31999 41437 +43157 41437 +46515 41434 +15315 41431 +28703 41425 +31056 41421 +40372 41419 +45318 41414 +32747 41414 +45615 41414 +13290 41400 +25521 41397 +40066 41394 +30144 41393 +48501 41391 +50210 41390 +23619 41390 +18146 41386 +36256 41384 +32910 41382 +40252 41381 +26427 41381 +43902 41378 +28386 41377 +29906 41375 +34592 41371 +19176 41367 +41353 41365 +49739 41364 +26591 41362 +34781 41361 +18897 41349 +41565 41346 +50017 41345 +49994 41342 +32421 41342 +27596 41340 +29187 41337 +36015 41337 +46907 41336 +38019 41334 +32960 41334 +20597 41333 +32946 41318 +31235 41318 +28705 41316 +46073 41308 +41779 41307 +27263 41305 +46264 41303 +32576 41299 +35785 41299 +30674 41299 +24767 41292 +37462 41292 +35784 41287 +27761 41285 +48155 41283 +29517 41282 +47380 41282 +41738 41273 +47579 41271 +37984 41270 +31289 41268 +36213 41268 +27199 41266 +43018 41265 +40960 41262 +49468 41260 +33238 41252 +44763 41246 +13730 41245 +48015 41244 +23800 41239 +36049 41238 +39053 41236 +14945 41236 +43554 41236 +28408 41234 +23815 41220 +15739 41219 +22709 41217 +31685 41217 +27228 41209 +24909 41205 +41860 41197 +31843 41193 +41188 41192 +35525 41189 +15350 41187 +29915 41178 +46946 41176 +24399 41174 +49759 41170 +2992 41169 +14125 41168 +14437 41163 +15365 41162 +31641 41162 +37303 41159 +28069 41159 +37886 41159 +43255 41153 +46013 41152 +17253 41151 +18100 41150 +22749 41149 +28845 41148 +35683 41146 +34797 41137 +35341 41137 +28711 41131 +17561 41129 +29013 41124 +6732 41122 +22985 41112 +37812 41100 +20072 41098 +33826 41097 +7942 41096 +21911 41096 +30963 41094 +40323 41091 +41483 41088 +29206 41086 +41404 41078 +20595 41077 +20827 41075 +35261 41075 +5248 41073 +20509 41072 +5630 41067 +20797 41065 +21437 41060 +24403 41046 +49683 41046 +30649 41046 +42474 41043 +26337 41041 +35951 41041 +37114 41040 +28226 41036 +25675 41034 +34963 41032 +23628 41032 +45261 41025 +38560 41010 +49690 41008 +14988 41003 +37668 41003 +34819 41002 +15995 40999 +47031 40998 +28494 40998 +34173 40992 +34528 40991 +19404 40990 +37489 40987 +26863 40987 +48741 40985 +27205 40985 +44447 40979 +2424 40976 +27481 40973 +12272 40969 +31360 40968 +1594 40962 +34815 40960 +22572 40956 +42549 40955 +30252 40955 +22189 40954 +40018 40952 +24825 40952 +48119 40948 +7144 40943 +40551 40943 +46773 40942 +30125 40941 +25321 40937 +16941 40932 +38413 40932 +46131 40924 +47061 40923 +35097 40917 +25429 40915 +49059 40913 +20724 40912 +31875 40906 +43671 40898 +39271 40887 +23209 40882 +21338 40881 +49536 40878 +42040 40877 +31586 40876 +42618 40876 +39915 40874 +27230 40871 +16232 40868 +9409 40868 +39178 40866 +23700 40866 +39001 40860 +36090 40859 +34150 40858 +28933 40856 +30943 40848 +46896 40845 +24277 40842 +48258 40841 +33631 40835 +3456 40830 +23716 40825 +46450 40824 +31824 40821 +3606 40820 +34006 40819 +27710 40817 +18995 40808 +32752 40805 +40338 40804 +32696 40798 +41597 40785 +40827 40784 +18784 40782 +41837 40780 +29420 40779 +34551 40766 +47342 40762 +48171 40761 +24602 40749 +31193 40749 +40661 40747 +21731 40736 +32213 40734 +18568 40733 +33230 40732 +39047 40730 +18308 40724 +34614 40723 +4440 40723 +43414 40722 +37357 40710 +20133 40708 +43754 40706 +40224 40701 +39262 40700 +34863 40698 +30299 40695 +27969 40689 +10520 40689 +30412 40688 +44146 40684 +46426 40681 +20215 40679 +17477 40678 +38411 40673 +29979 40673 +33657 40671 +44516 40671 +22791 40668 +43188 40661 +17815 40659 +16647 40658 +19752 40656 +23673 40655 +27649 40651 +44473 40650 +28211 40648 +46919 40647 +4949 40647 +23315 40645 +34654 40644 +43726 40642 +28186 40638 +37172 40636 +35629 40634 +22523 40633 +32181 40631 +27017 40630 +33107 40629 +33822 40626 +20816 40625 +29252 40622 +45687 40622 +38193 40618 +33904 40617 +44096 40612 +22117 40611 +45566 40610 +27143 40603 +33595 40601 +36138 40600 +38645 40597 +41246 40593 +27416 40592 +15433 40592 +26350 40591 +31149 40590 +20199 40586 +8039 40584 +37880 40575 +32170 40570 +21286 40569 +19891 40568 +48967 40564 +28239 40556 +33598 40555 +40183 40555 +43933 40547 +37459 40545 +21426 40544 +26506 40544 +39240 40541 +14137 40540 +43288 40534 +29165 40532 +28304 40532 +24136 40527 +16432 40526 +47997 40522 +37001 40522 +24265 40515 +43972 40513 +24096 40506 +28982 40504 +23347 40500 +26897 40497 +18145 40494 +44076 40493 +35797 40490 +12919 40490 +44285 40488 +44620 40484 +29580 40482 +40771 40479 +25270 40479 +28552 40479 +35303 40477 +23142 40459 +43988 40456 +37546 40454 +36190 40452 +27644 40451 +39088 40450 +40773 40447 +35055 40446 +33294 40441 +12306 40439 +40144 40437 +36820 40431 +13989 40429 +9806 40429 +26754 40428 +27776 40428 +39864 40426 +41994 40425 +29130 40418 +17006 40414 +33229 40413 +40617 40411 +20068 40407 +18377 40403 +30745 40401 +24197 40394 +8954 40393 +21898 40391 +17340 40391 +36660 40386 +45443 40386 +35520 40381 +45678 40381 +37885 40381 +1822 40380 +41720 40379 +20924 40378 +16529 40371 +39281 40369 +40906 40345 +50180 40342 +33256 40339 +38308 40329 +12487 40328 +44715 40323 +40986 40323 +19079 40320 +47014 40317 +36691 40309 +39221 40308 +39160 40308 +43003 40307 +22829 40300 +19230 40298 +19564 40292 +35492 40288 +26208 40285 +36489 40283 +17579 40282 +28263 40279 +45378 40274 +48390 40274 +38639 40273 +46902 40273 +28446 40272 +39273 40268 +31564 40268 +31343 40268 +40000 40266 +25319 40265 +30407 40265 +35934 40258 +40048 40255 +42545 40254 +34297 40253 +1396 40249 +44456 40245 +31089 40245 +12234 40236 +13758 40229 +12747 40228 +27774 40228 +22932 40226 +33550 40225 +31975 40223 +40634 40214 +49882 40211 +49952 40208 +32647 40206 +32566 40194 +26995 40192 +31580 40191 +42837 40182 +40614 40181 +35275 40180 +15206 40172 +29972 40168 +48226 40167 +39002 40165 +25447 40165 +28598 40164 +1094 40157 +31031 40157 +16191 40155 +48434 40151 +32094 40150 +13376 40144 +26411 40140 +48759 40139 +29303 40137 +29909 40137 +36672 40136 +25728 40136 +46629 40132 +37341 40125 +23516 40122 +31278 40122 +34725 40120 +41415 40119 +44771 40116 +13260 40112 +23721 40110 +28551 40108 +34644 40108 +37271 40105 +39796 40101 +27192 40099 +20851 40097 +45030 40097 +28715 40097 +27778 40096 +42613 40096 +31454 40091 +40909 40090 +42892 40089 +47149 40089 +31434 40088 +39663 40082 +35939 40080 +30748 40076 +42119 40074 +41903 40074 +13550 40071 +32497 40070 +9545 40066 +21390 40065 +35856 40063 +15786 40060 +34839 40060 +28930 40058 +29776 40056 +17465 40056 +32699 40055 +19706 40054 +22243 40050 +46190 40050 +17288 40042 +32798 40040 +12149 40040 +33950 40033 +46295 40031 +37140 40026 +47624 40026 +41522 40025 +25960 40024 +48367 40022 +36529 40021 +37986 40020 +26977 40020 +35878 40019 +29947 40013 +28103 40013 +45548 40010 +47492 40003 +42858 40002 +41448 40002 +47042 40001 +47140 39987 +49870 39984 +37607 39979 +31792 39970 +28700 39963 +48218 39958 +33039 39958 +29119 39956 +13564 39956 +43200 39953 +21992 39948 +43314 39941 +19638 39941 +29559 39931 +17823 39928 +49290 39927 +12795 39925 +50045 39923 +49866 39916 +24492 39909 +32933 39906 +42371 39904 +13164 39903 +29268 39899 +6746 39896 +32476 39893 +19936 39893 +48456 39893 +38537 39889 +33237 39888 +40678 39885 +25035 39883 +27948 39877 +8890 39877 +36158 39876 +25059 39864 +19815 39864 +48153 39863 +47004 39863 +30238 39861 +45099 39856 +41378 39841 +30820 39839 +36849 39836 +26884 39834 +21037 39832 +21749 39829 +35090 39814 +45419 39812 +29414 39809 +39564 39808 +36326 39801 +42826 39801 +25356 39800 +46379 39796 +14822 39795 +48296 39795 +41532 39791 +43896 39791 +32492 39791 +27753 39790 +48128 39789 +17613 39787 +48904 39786 +49420 39782 +15208 39781 +19231 39779 +17580 39775 +19720 39775 +22865 39773 +29939 39772 +42514 39766 +37215 39765 +40320 39762 +31219 39761 +35582 39761 +35016 39760 +35769 39759 +43424 39758 +9239 39754 +38787 39752 +31869 39746 +25788 39746 +31323 39745 +26533 39740 +39369 39736 +24616 39733 +37987 39731 +45036 39731 +31819 39730 +6382 39727 +37524 39727 +48580 39725 +30495 39725 +36661 39724 +46851 39721 +37464 39717 +34244 39717 +23906 39713 +29767 39711 +21991 39707 +17128 39706 +34025 39705 +20292 39696 +31146 39690 +44221 39679 +28900 39673 +44628 39673 +24676 39672 +14465 39667 +48881 39665 +37738 39664 +20423 39663 +48424 39661 +43278 39661 +34857 39660 +30052 39658 +36906 39657 +33512 39651 +7350 39646 +24274 39645 +37722 39645 +41534 39644 +25944 39644 +42880 39641 +27790 39641 +30304 39640 +31650 39637 +29184 39637 +34417 39636 +13849 39634 +25920 39634 +43113 39625 +36573 39617 +45513 39616 +3736 39616 +26923 39613 +39138 39613 +45763 39610 +36161 39604 +21304 39604 +32110 39602 +37606 39597 +34970 39597 +37683 39585 +3476 39583 +38228 39582 +26019 39581 +30095 39580 +18117 39577 +46041 39577 +12156 39574 +46004 39574 +39510 39573 +35382 39572 +31651 39571 +45121 39568 +32592 39566 +6813 39566 +48022 39565 +35057 39561 +19462 39560 +7153 39559 +36511 39559 +47788 39558 +23021 39555 +36756 39547 +45220 39546 +41824 39545 +13916 39544 +31543 39542 +28567 39541 +24509 39539 +47331 39535 +36637 39534 +22457 39531 +15999 39530 +44803 39530 +33530 39529 +14202 39528 +43359 39528 +41696 39528 +26478 39524 +6892 39524 +26322 39515 +46738 39513 +37623 39512 +28369 39509 +30614 39502 +32683 39495 +30354 39494 +48914 39487 +39798 39485 +13986 39485 +47769 39481 +16097 39479 +38496 39479 +49286 39476 +31794 39471 +44618 39471 +45829 39466 +39593 39461 +15675 39459 +36264 39458 +41891 39449 +8328 39446 +21758 39446 +24936 39437 +42625 39430 +16673 39421 +41410 39418 +27799 39418 +30280 39417 +37952 39416 +34453 39413 +39502 39412 +33919 39409 +41865 39409 +27998 39408 +42402 39406 +25032 39399 +30219 39397 +20104 39395 +15977 39394 +30147 39393 +37149 39393 +10751 39388 +31356 39387 +36012 39380 +3627 39377 +16144 39374 +23392 39364 +33183 39360 +36050 39357 +27172 39351 +28257 39349 +38550 39345 +29253 39338 +28138 39320 +43170 39318 +36068 39318 +42966 39318 +35445 39317 +37810 39316 +22348 39315 +45459 39315 +46035 39312 +26792 39303 +24118 39300 +14473 39299 +45602 39298 +49807 39295 +29132 39292 +20968 39290 +25256 39286 +18060 39283 +30940 39281 +38443 39276 +30948 39268 +32133 39267 +25770 39263 +22950 39256 +30032 39256 +27673 39254 +47405 39250 +40684 39249 +35216 39245 +28447 39240 +12425 39240 +24906 39237 +31209 39234 +35091 39233 +44226 39232 +46637 39219 +27112 39216 +41871 39213 +25061 39213 +46414 39210 +27588 39209 +27218 39207 +30272 39205 +27715 39202 +27289 39202 +41040 39201 +29687 39200 +29449 39200 +43264 39198 +37246 39198 +22800 39197 +24254 39197 +7240 39196 +48026 39193 +43949 39193 +2878 39191 +27171 39188 +20245 39186 +33411 39183 +30501 39183 +26260 39182 +21917 39180 +32117 39172 +47099 39171 +24876 39170 +44708 39169 +25345 39169 +35711 39168 +36519 39165 +27970 39161 +35077 39159 +34328 39158 +47745 39156 +34451 39155 +39832 39152 +24940 39151 +25959 39150 +38648 39141 +17763 39141 +17525 39133 +36332 39131 +11485 39129 +16874 39128 +45546 39126 +27777 39119 +13974 39112 +29965 39111 +18047 39111 +25667 39111 +36339 39109 +31270 39109 +42578 39108 +46878 39108 +48343 39107 +28396 39104 +39865 39104 +37019 39102 +30904 39100 +2336 39100 +39399 39090 +25191 39089 +31111 39080 +26090 39076 +33334 39075 +44058 39073 +44940 39073 +8388 39070 +31427 39063 +29558 39061 +37294 39060 +37949 39057 +21009 39053 +42021 39050 +16409 39045 +40468 39043 +37721 39041 +24870 39039 +36280 39037 +28130 39034 +34741 39033 +40693 39027 +32926 39026 +50191 39025 +44880 39023 +35995 39022 +47750 39021 +32618 39017 +17764 39014 +13278 39009 +44175 39000 +21077 38999 +42201 38998 +34691 38996 +32789 38989 +41209 38985 +36146 38975 +14916 38973 +6311 38970 +50008 38970 +39263 38969 +21492 38968 +42340 38967 +24591 38965 +28520 38962 +28597 38951 +40440 38951 +27345 38949 +23326 38943 +11587 38943 +35184 38933 +28174 38926 +19631 38924 +44697 38924 +45447 38923 +31930 38921 +31557 38918 +34326 38914 +33866 38914 +20044 38912 +27680 38912 +43112 38908 +48049 38907 +25286 38897 +23451 38895 +40273 38893 +22278 38891 +39694 38890 +47959 38890 +49335 38889 +24772 38887 +41884 38886 +44091 38881 +23350 38878 +15720 38876 +33770 38876 +41264 38875 +16414 38873 +24726 38871 +8479 38865 +34611 38862 +40123 38862 +27203 38861 +25707 38859 +46337 38856 +43064 38856 +28936 38855 +36534 38852 +40817 38852 +39110 38851 +21895 38839 +45396 38839 +41342 38837 +44512 38829 +29512 38827 +27917 38827 +21858 38825 +49590 38822 +39477 38821 +20803 38816 +29213 38814 +43942 38807 +17771 38807 +20861 38805 +40905 38799 +11961 38798 +38379 38790 +27564 38790 +48563 38788 +24693 38787 +21153 38784 +28420 38783 +34894 38773 +23092 38771 +49524 38769 +34738 38767 +16099 38767 +37285 38766 +17469 38764 +24279 38760 +48622 38756 +22429 38755 +27009 38754 +47347 38749 +45699 38746 +30110 38740 +17250 38737 +46702 38733 +41370 38730 +48572 38727 +44770 38724 +43929 38714 +35038 38714 +17278 38711 +48274 38710 +6860 38704 +41414 38703 +28825 38699 +42375 38699 +14407 38698 +45429 38693 +41149 38689 +49847 38686 +23036 38685 +10067 38683 +22466 38683 +25785 38674 +34737 38671 +41063 38670 +33849 38667 +41883 38659 +36453 38657 +43032 38653 +35919 38652 +6773 38647 +2466 38644 +15366 38642 +20777 38640 +40268 38636 +40291 38635 +41516 38633 +19044 38633 +35524 38629 +32144 38628 +31805 38627 +38086 38626 +40270 38621 +36336 38621 +33387 38620 +17818 38619 +35800 38617 +34360 38615 +20772 38614 +9821 38611 +46288 38609 +22691 38600 +48422 38597 +25272 38591 +29645 38590 +41018 38589 +20711 38588 +24962 38587 +49510 38587 +47828 38586 +33520 38585 +14368 38577 +32261 38575 +20211 38575 +39198 38573 +46895 38569 +38889 38566 +32391 38562 +44666 38561 +47257 38554 +35710 38554 +47098 38553 +39013 38552 +37274 38544 +23483 38536 +28585 38532 +32109 38530 +46791 38526 +22813 38524 +35870 38516 +25782 38513 +46357 38512 +9271 38512 +45211 38510 +27033 38510 +28976 38498 +45248 38498 +33047 38498 +36380 38494 +47713 38493 +37086 38490 +30624 38490 +38745 38488 +30514 38487 +26611 38487 +2886 38486 +10873 38485 +43351 38484 +41562 38483 +42191 38481 +34841 38481 +45994 38474 +45712 38474 +22921 38473 +41567 38467 +28925 38464 +21577 38461 +21084 38459 +12331 38458 +4187 38456 +29133 38455 +26529 38455 +41013 38451 +48998 38450 +31475 38448 +22657 38445 +29551 38444 +47313 38442 +45769 38424 +38192 38424 +34688 38420 +42240 38414 +18483 38412 +34135 38410 +33488 38403 +31759 38398 +23571 38396 +36946 38395 +41929 38393 +40003 38393 +40878 38387 +34898 38380 +13155 38379 +18160 38378 +2711 38376 +27640 38374 +26485 38370 +41737 38366 +21759 38355 +27486 38353 +38612 38348 +11350 38348 +33591 38347 +28159 38346 +48486 38344 +25823 38344 +46657 38340 +21186 38338 +46312 38332 +29951 38331 +33065 38329 +43504 38328 +22302 38327 +11726 38323 +22622 38322 +35325 38320 +28204 38319 +32680 38318 +23460 38317 +35112 38316 +14964 38314 +34176 38313 +11878 38313 +48406 38312 +40037 38311 +12794 38311 +33048 38310 +30601 38310 +49844 38305 +40964 38304 +31081 38297 +16979 38294 +23402 38292 +17108 38291 +26284 38289 +46852 38285 +39153 38284 +12574 38284 +39180 38282 +27960 38281 +1753 38281 +28627 38280 +40798 38279 +37256 38278 +47982 38274 +45074 38274 +20643 38274 +28588 38272 +2529 38269 +24920 38267 +37624 38267 +9198 38265 +7391 38263 +49187 38262 +23463 38259 +44488 38253 +21797 38246 +26950 38245 +35174 38243 +12940 38229 +34920 38224 +28802 38217 +34641 38216 +44234 38214 +34464 38206 +21985 38199 +24845 38188 +37681 38185 +44767 38185 +38182 38184 +48106 38182 +16913 38182 +17610 38180 +24745 38174 +21610 38172 +30533 38169 +26423 38168 +9289 38166 +34425 38161 +29214 38160 +45342 38154 +14215 38152 +20876 38146 +38425 38145 +31342 38143 +32221 38140 +48566 38137 +32674 38136 +9249 38132 +46730 38132 +32357 38131 +3897 38129 +47022 38124 +20658 38124 +34332 38120 +35427 38120 +26660 38119 +13733 38118 +39580 38113 +32744 38109 +39574 38109 +49400 38108 +37995 38104 +26392 38101 +36919 38101 +37104 38097 +43955 38091 +31572 38090 +14974 38085 +20957 38085 +37630 38085 +2289 38083 +39146 38079 +28140 38078 +39589 38076 +33162 38065 +12409 38065 +29057 38064 +31773 38062 +34869 38059 +10179 38054 +21847 38048 +36539 38044 +24528 38043 +34089 38041 +40496 38041 +47130 38038 +47993 38036 +36817 38036 +38585 38035 +41119 38032 +30376 38031 +46942 38031 +41031 38028 +31359 38025 +48531 38025 +32877 38023 +28306 38023 +3029 38020 +29033 38017 +34948 38016 +44376 38015 +33592 38013 +23580 38012 +38520 38011 +35894 38006 +8014 38001 +39853 37999 +48843 37998 +28302 37993 +16718 37987 +32052 37986 +20911 37986 +47028 37978 +48731 37970 +21382 37967 +44495 37962 +17974 37958 +34298 37958 +6601 37958 +32201 37957 +39015 37956 +29649 37954 +13501 37951 +23437 37948 +32611 37938 +25840 37935 +39348 37935 +29068 37935 +25499 37933 +41861 37932 +27734 37931 +42761 37930 +8637 37929 +30989 37927 +37868 37926 +32547 37921 +18703 37920 +25778 37917 +37333 37911 +37646 37910 +36824 37907 +4435 37901 +39525 37900 +33155 37899 +44151 37890 +45156 37889 +34774 37885 +18069 37882 +35529 37877 +45916 37870 +32337 37866 +36459 37864 +27560 37862 +29538 37862 +30453 37860 +15511 37857 +20413 37857 +21241 37855 +21953 37854 +42320 37853 +26697 37852 +48102 37850 +46783 37847 +49252 37846 +12987 37840 +26983 37839 +16398 37839 +21537 37839 +36016 37837 +17680 37837 +13983 37836 +34441 37836 +37537 37835 +25171 37835 +44390 37829 +37643 37829 +41714 37826 +25870 37822 +24783 37819 +29526 37818 +14191 37818 +34987 37817 +24184 37817 +41320 37813 +18890 37808 +32900 37805 +39556 37800 +43793 37798 +28116 37795 +33508 37787 +32754 37787 +14487 37785 +38209 37779 +45223 37779 +45162 37778 +31018 37777 +20605 37775 +24245 37773 +20497 37773 +24612 37771 +44888 37767 +43503 37766 +49255 37765 +41680 37762 +25404 37762 +42110 37756 +47796 37754 +9652 37754 +22497 37740 +6743 37737 +33178 37735 +44904 37733 +29082 37732 +38853 37730 +23740 37724 +34505 37724 +24180 37715 +20856 37715 +22312 37711 +31227 37707 +20506 37704 +25322 37700 +28727 37699 +40585 37697 +35548 37696 +37611 37694 +45498 37692 +43307 37690 +40714 37688 +35557 37687 +14539 37686 +40249 37684 +17828 37684 +41664 37684 +49874 37684 +37980 37679 +14184 37678 +42101 37673 +26308 37671 +13603 37668 +34872 37666 +45846 37666 +48928 37665 +36575 37664 +28481 37662 +42743 37662 +38272 37662 +22119 37659 +50060 37658 +15803 37651 +18019 37650 +43783 37649 +42241 37647 +21728 37646 +39082 37626 +36234 37622 +40755 37614 +23653 37610 +44766 37607 +43744 37604 +8578 37601 +45054 37601 +31510 37600 +32338 37599 +834 37595 +30194 37591 +40593 37584 +24213 37583 +25149 37579 +48681 37575 +46016 37572 +27140 37567 +33509 37567 +44635 37562 +47825 37562 +38359 37561 +13689 37551 +32298 37549 +45237 37543 +24468 37539 +43709 37534 +9451 37532 +26872 37526 +22913 37525 +18601 37524 +19570 37521 +22163 37517 +34771 37517 +49727 37511 +40919 37510 +38306 37505 +22226 37503 +44507 37500 +33004 37499 +41236 37491 +47416 37490 +35168 37489 +15884 37488 +32156 37485 +21780 37484 +32161 37482 +15552 37479 +32493 37479 +39501 37479 +46313 37477 +13437 37473 +36689 37468 +32209 37464 +32299 37460 +29024 37458 +39114 37454 +26168 37454 +20264 37453 +36237 37450 +35598 37448 +40547 37444 +12905 37430 +38060 37421 +20943 37420 +47739 37419 +5020 37415 +24991 37414 +26987 37412 +34039 37411 +25083 37408 +29270 37408 +22648 37399 +39484 37399 +22991 37397 +36238 37392 +40868 37391 +16284 37386 +5841 37383 +40327 37382 +38127 37380 +35688 37374 +35996 37373 +15042 37371 +22324 37369 +24222 37366 +41268 37365 +24275 37365 +33036 37362 +30692 37362 +32012 37362 +48306 37361 +44827 37356 +26705 37354 +44331 37353 +36322 37351 +29555 37350 +16598 37342 +47055 37339 +33735 37339 +18937 37337 +35761 37335 +26045 37333 +13732 37328 +25648 37325 +38876 37325 +36097 37322 +15445 37321 +47062 37318 +27277 37318 +30788 37316 +11413 37316 +27522 37314 +14577 37310 +34594 37309 +42346 37304 +36478 37303 +42264 37302 +29157 37302 +31300 37302 +36340 37299 +35723 37296 +49789 37293 +7434 37292 +20855 37288 +40410 37287 +44368 37286 +45075 37283 +44049 37282 +28740 37279 +31287 37278 +17529 37277 +17138 37271 +40170 37271 +26448 37263 +30685 37263 +22419 37258 +41846 37255 +33966 37253 +42951 37251 +46675 37251 +26979 37249 +50071 37249 +24839 37248 +34378 37244 +25399 37238 +37088 37238 +27255 37237 +47584 37236 +30499 37235 +10572 37235 +18173 37231 +28647 37231 +3289 37225 +4110 37224 +39813 37220 +16802 37217 +26912 37216 +36349 37214 +43227 37212 +47039 37210 +37306 37209 +10877 37205 +29318 37205 +37901 37204 +5111 37204 +16087 37202 +40953 37199 +29807 37198 +45143 37198 +49417 37193 +36412 37190 +43369 37190 +40152 37176 +21839 37167 +41917 37166 +17618 37151 +19385 37151 +44388 37148 +34401 37143 +49558 37140 +18209 37140 +47704 37139 +32593 37136 +35148 37132 +38439 37126 +43386 37115 +38884 37115 +25095 37111 +28164 37104 +22059 37103 +3276 37102 +36035 37102 +20482 37101 +46043 37100 +47203 37089 +8316 37089 +32817 37087 +35826 37081 +26787 37078 +48629 37077 +27656 37075 +46466 37075 +15876 37064 +38975 37064 +39973 37064 +33492 37060 +43222 37058 +37220 37058 +47566 37049 +42418 37046 +42515 37043 +20599 37042 +34289 37042 +37526 37041 +43497 37041 +36179 37039 +47248 37034 +47767 37032 +19944 37027 +33438 37026 +33840 37025 +12726 37025 +39931 37024 +21633 37020 +40991 37016 +28522 37014 +40796 37012 +25796 37008 +35737 37003 +34494 37001 +22728 36999 +29992 36997 +47494 36996 +34387 36992 +45782 36990 +31897 36990 +31485 36981 +13284 36981 +39207 36980 +12143 36980 +44232 36977 +37873 36975 +46105 36973 +25161 36970 +23211 36968 +39851 36967 +18747 36964 +41315 36964 +32455 36960 +21013 36959 +30638 36957 +13857 36955 +29587 36954 +35052 36953 +43751 36953 +28017 36952 +24678 36948 +18426 36944 +45927 36942 +20053 36940 +21303 36938 +24624 36937 +31243 36936 +15746 36935 +34063 36935 +45161 36932 +14055 36932 +28200 36929 +37064 36926 +25369 36924 +42851 36921 +45043 36915 +46696 36908 +26108 36908 +24603 36907 +8214 36907 +27149 36904 +42252 36900 +19737 36893 +24974 36890 +41039 36886 +42664 36885 +17943 36885 +30332 36885 +5925 36880 +27207 36880 +17024 36878 +26972 36871 +6730 36869 +32008 36867 +20961 36862 +21217 36860 +47631 36849 +43186 36848 +42017 36846 +15826 36843 +5723 36842 +31169 36838 +27057 36838 +37192 36829 +29968 36829 +20427 36826 +18910 36823 +33811 36822 +48179 36822 +27402 36822 +36599 36817 +43203 36811 +32231 36808 +32532 36804 +29349 36804 +16725 36803 +47496 36802 +36860 36795 +38669 36791 +45497 36785 +4234 36784 +23464 36776 +41400 36770 +28071 36761 +16286 36760 +47460 36759 +26086 36753 +36838 36750 +38221 36748 +44257 36741 +16198 36730 +10632 36727 +29598 36725 +32107 36725 +31766 36720 +38051 36719 +39454 36717 +45995 36715 +19689 36713 +45948 36711 +28788 36710 +36844 36708 +2757 36708 +21981 36706 +49230 36704 +22988 36703 +18457 36701 +37409 36700 +22279 36698 +41521 36691 +38722 36691 +36973 36682 +35867 36678 +32626 36677 +44584 36674 +32651 36674 +20459 36673 +47888 36672 +35458 36671 +34716 36669 +19146 36665 +45609 36664 +29763 36659 +43206 36655 +44725 36655 +31503 36650 +40381 36649 +16234 36647 +42926 36646 +41693 36645 +19803 36640 +33862 36639 +21902 36630 +33603 36629 +35356 36625 +49606 36620 +48373 36619 +10410 36617 +49890 36612 +39635 36611 +9710 36610 +38323 36610 +33109 36610 +25285 36609 +11407 36606 +38386 36599 +48261 36598 +47387 36594 +33928 36591 +25622 36591 +17822 36591 +21281 36589 +46525 36588 +25946 36587 +49581 36580 +34834 36579 +10180 36579 +33560 36576 +43562 36576 +20141 36569 +45386 36569 +44275 36564 +44098 36563 +16485 36560 +36901 36559 +7783 36557 +25418 36556 +33266 36551 +35139 36546 +24452 36542 +37087 36538 +24875 36538 +27800 36537 +34711 36537 +28857 36534 +26407 36533 +23020 36533 +29286 36531 +5113 36529 +30734 36525 +24770 36523 +41554 36520 +10879 36518 +28728 36517 +16928 36514 +21202 36503 +32172 36503 +34538 36501 +38961 36499 +36344 36496 +41539 36494 +49414 36493 +27982 36491 +34266 36483 +35218 36480 +32708 36474 +30425 36468 +34503 36467 +30463 36464 +45535 36457 +31029 36450 +37543 36449 +22620 36449 +24970 36448 +37454 36447 +46407 36444 +34109 36442 +27372 36441 +7182 36439 +35353 36438 +39287 36436 +28868 36435 +39338 36433 +29134 36428 +18941 36425 +44094 36418 +25653 36415 +33781 36413 +42639 36412 +11269 36410 +46711 36403 +27563 36401 +9394 36398 +37062 36397 +33138 36395 +29561 36386 +31925 36377 +47339 36364 +25278 36362 +31521 36360 +16770 36356 +38265 36356 +35137 36352 +35714 36346 +22530 36342 +31858 36341 +13862 36341 +47108 36340 +10775 36334 +24943 36333 +41579 36330 +42910 36328 +946 36325 +40007 36319 +45764 36316 +35161 36316 +40140 36308 +40737 36289 +39190 36287 +43115 36287 +13712 36286 +45503 36280 +26829 36280 +48786 36269 +47223 36267 +30991 36264 +9407 36260 +13928 36258 +28907 36255 +37355 36254 +27785 36251 +32698 36248 +32037 36243 +47640 36241 +36958 36238 +19967 36234 +38078 36231 +33015 36231 +16071 36226 +27653 36222 +23565 36222 +46467 36222 +42362 36221 +29429 36220 +47260 36217 +33408 36216 +29801 36215 +23318 36214 +29728 36213 +31738 36212 +22396 36208 +35015 36203 +8003 36198 +47805 36195 +16266 36193 +35899 36184 +46619 36182 +27967 36177 +19795 36173 +45062 36165 +28979 36163 +46548 36163 +16751 36161 +24149 36159 +35028 36155 +27896 36154 +41710 36150 +40857 36147 +38036 36146 +32950 36142 +34581 36135 +28817 36130 +48215 36128 +44967 36127 +41247 36127 +36082 36123 +42447 36116 +46208 36113 +13080 36113 +49777 36112 +11656 36112 +28301 36109 +9534 36108 +35863 36104 +43627 36102 +25382 36102 +37814 36095 +49547 36094 +31188 36089 +31753 36089 +20702 36087 +26915 36082 +46062 36080 +18776 36079 +26492 36077 +31093 36075 +42046 36069 +36666 36067 +38046 36066 +47453 36065 +35519 36064 +40757 36064 +44001 36063 +19583 36062 +41754 36061 +15047 36061 +37115 36061 +6744 36058 +46769 36055 +25998 36053 +48803 36049 +39239 36047 +14078 36047 +16942 36045 +28974 36044 +49162 36042 +40071 36039 +46656 36039 +1608 36037 +28896 36036 +44875 36035 +15640 36035 +24911 36031 +42028 36031 +48776 36031 +38200 36029 +42941 36022 +18055 36021 +42882 36014 +37185 36014 +20930 36012 +33027 36010 +8745 36003 +29008 36002 +47478 36002 +48667 36002 +44611 36000 +34993 35998 +27889 35998 +44823 35994 +42391 35988 +32855 35987 +16233 35985 +43840 35983 +35484 35982 +27760 35976 +33289 35970 +9590 35970 +38304 35969 +19963 35962 +50142 35961 +48357 35955 +40697 35955 +48450 35952 +28953 35946 +48574 35944 +42231 35942 +49879 35942 +38185 35931 +25022 35930 +15127 35929 +22713 35923 +49445 35922 +21360 35921 +29099 35917 +36527 35910 +43960 35909 +49363 35905 +20205 35904 +37223 35903 +32755 35900 +8413 35899 +49772 35896 +45484 35895 +37125 35895 +13313 35894 +47268 35893 +42292 35892 +46292 35887 +35531 35880 +42870 35880 +17230 35878 +27081 35878 +26682 35876 +42972 35876 +2699 35875 +27533 35875 +33499 35873 +24856 35867 +24305 35866 +21227 35865 +41214 35864 +15681 35860 +34232 35858 +42900 35856 +30005 35850 +34080 35849 +42594 35847 +28592 35843 +39226 35843 +44571 35842 +41528 35833 +26993 35830 +35431 35828 +23835 35828 +26598 35820 +31412 35805 +41840 35803 +46658 35802 +14231 35801 +33982 35794 +23881 35794 +15048 35791 +8926 35791 +27040 35789 +31170 35786 +44581 35779 +45457 35779 +39674 35779 +32227 35778 +25153 35774 +2880 35769 +7704 35767 +23468 35762 +23971 35758 +47083 35754 +42517 35751 +17303 35748 +36102 35744 +30741 35743 +47401 35734 +43185 35725 +30727 35722 +32568 35720 +44907 35720 +9982 35713 +40761 35712 +28096 35711 +29976 35704 +39129 35700 +10886 35696 +34923 35692 +42462 35690 +32274 35688 +19085 35678 +19926 35674 +28430 35673 +17287 35666 +49684 35665 +35021 35663 +28297 35658 +43388 35658 +27659 35654 +37571 35653 +26968 35653 +40120 35653 +43131 35653 +49909 35652 +16990 35649 +15484 35647 +49334 35646 +41108 35642 +37413 35642 +41685 35640 +38598 35635 +30615 35635 +27249 35635 +31730 35634 +15204 35633 +12646 35631 +40126 35629 +42927 35628 +46635 35627 +41254 35626 +30528 35620 +39539 35620 +27019 35616 +11794 35615 +41226 35610 +41125 35610 +31814 35609 +34756 35609 +18567 35607 +49009 35603 +47664 35603 +31211 35600 +41854 35597 +28843 35595 +3693 35594 +31716 35591 +21288 35591 +24877 35586 +46242 35583 +44989 35583 +41123 35582 +50176 35577 +44355 35575 +43342 35575 +35545 35574 +23836 35571 +39863 35570 +13911 35570 +18943 35558 +34689 35558 +26268 35552 +45240 35547 +38991 35547 +5148 35544 +46814 35535 +3396 35534 +42370 35530 +28676 35528 +29954 35523 +22240 35522 +30483 35522 +48525 35509 +39081 35508 +29818 35499 +46829 35499 +48202 35493 +42088 35491 +32112 35488 +36079 35487 +48383 35487 +37111 35486 +10612 35485 +9614 35485 +36668 35481 +32549 35481 +37210 35480 +14016 35477 +42423 35477 +22860 35475 +48986 35473 +7094 35473 +27158 35471 +28922 35468 +34253 35460 +38791 35458 +33689 35457 +47868 35448 +40054 35446 +30835 35446 +35960 35441 +31010 35438 +11869 35437 +16017 35432 +46715 35430 +23833 35430 +21495 35430 +22606 35428 +40608 35427 +25086 35427 +25341 35425 +9213 35424 +38227 35424 +40450 35418 +28640 35418 +26273 35416 +20735 35411 +29500 35409 +46631 35407 +24575 35399 +40070 35397 +39384 35396 +31192 35395 +42106 35393 +25540 35390 +16451 35387 +49935 35380 +10722 35379 +46626 35371 +7717 35370 +13383 35370 +33148 35364 +47476 35363 +21170 35360 +38912 35356 +42508 35356 +46909 35350 +38190 35349 +40300 35348 +35704 35345 +43978 35343 +20405 35335 +25437 35334 +24724 35332 +26246 35330 +11488 35329 +18893 35328 +27392 35326 +14312 35325 +46534 35325 +20315 35325 +45735 35322 +24242 35319 +45405 35317 +33083 35316 +28850 35315 +48748 35314 +6704 35312 +47163 35312 +33016 35310 +18795 35307 +39741 35305 +33960 35302 +48221 35300 +32779 35299 +35005 35295 +27189 35294 +11371 35292 +34035 35284 +20987 35280 +43511 35277 +14647 35269 +34690 35268 +48358 35267 +35643 35266 +26313 35264 +34036 35264 +45350 35263 +48420 35263 +46348 35261 +34785 35258 +43958 35257 +30248 35252 +39885 35252 +25089 35251 +15496 35245 +27797 35240 +26276 35239 +6145 35233 +38848 35230 +31102 35227 +32171 35225 +40078 35223 +42825 35219 +49654 35218 +48656 35218 +15148 35216 +29087 35211 +49679 35210 +6812 35205 +40831 35205 +36972 35201 +42741 35189 +40515 35184 +31649 35174 +6760 35169 +40395 35168 +24820 35166 +25511 35165 +10355 35164 +48636 35163 +13295 35163 +34670 35161 +40589 35155 +23042 35154 +30267 35154 +7154 35154 +36446 35144 +40446 35139 +44565 35137 +21019 35128 +33211 35128 +49530 35126 +27339 35122 +32786 35117 +38238 35113 +28048 35113 +42002 35109 +36738 35109 +23144 35107 +38267 35105 +42339 35105 +33416 35105 +32575 35104 +44219 35104 +8928 35102 +25269 35097 +6433 35096 +49575 35089 +40326 35089 +36014 35087 +2762 35080 +29069 35075 +48347 35074 +37733 35074 +25898 35071 +34341 35070 +32653 35065 +43837 35062 +39155 35059 +23912 35058 +17018 35051 +11385 35051 +29204 35045 +29174 35043 +44981 35040 +39496 35037 +42836 35032 +31009 35026 +31566 35026 +32142 35020 +32996 35020 +34658 35017 +20203 35016 +45098 35010 +12315 35005 +12611 35003 +21018 34996 +4733 34995 +36057 34994 +3168 34984 +42607 34983 +25991 34979 +42422 34979 +31947 34976 +46652 34975 +44590 34974 +39797 34972 +40871 34969 +44810 34969 +19203 34962 +26551 34961 +41795 34952 +30802 34949 +44306 34948 +41586 34939 +29096 34936 +36564 34935 +33341 34931 +38006 34931 +41918 34931 +43716 34931 +45315 34930 +38135 34929 +42988 34929 +37041 34928 +29322 34914 +38373 34914 +20810 34913 +40115 34913 +32886 34910 +41699 34908 +45652 34907 +34859 34905 +39389 34901 +21881 34898 +14385 34898 +32143 34897 +45085 34897 +13265 34896 +45017 34895 +39233 34893 +45172 34892 +38966 34888 +27910 34888 +29122 34887 +39740 34880 +30505 34877 +37212 34877 +32185 34869 +16520 34868 +41231 34865 +33121 34865 +34721 34860 +17190 34857 +34730 34856 +41747 34856 +28684 34850 +40294 34847 +26525 34838 +32129 34834 +47421 34829 +38842 34825 +4655 34821 +45407 34816 +24100 34815 +30971 34811 +31166 34808 +46964 34803 +39439 34801 +33475 34799 +42022 34799 +43853 34797 +44806 34796 +26895 34794 +17309 34794 +21928 34793 +28872 34793 +30707 34790 +34278 34788 +47450 34785 +32621 34780 +31476 34778 +29298 34770 +46881 34770 +45502 34767 +38008 34766 +45525 34763 +12792 34761 +46622 34759 +44643 34757 +23123 34753 +26381 34752 +36209 34744 +24750 34742 +23644 34740 +30373 34739 +41855 34738 +6471 34737 +31397 34737 +41478 34736 +26146 34736 +24736 34733 +43189 34733 +21524 34731 +38621 34728 +36932 34727 +29448 34726 +12679 34717 +47193 34711 +49278 34711 +20641 34710 +17171 34709 +46891 34701 +23219 34700 +31135 34696 +39229 34694 +31607 34694 +47027 34690 +19559 34686 +24917 34683 +33607 34679 +35556 34679 +29552 34676 +47495 34674 +46017 34673 +24043 34668 +27676 34664 +31778 34656 +6604 34655 +5459 34649 +43507 34645 +49791 34645 +30805 34643 +29903 34641 +33301 34640 +6406 34638 +35131 34630 +42793 34628 +41186 34627 +38731 34627 +23924 34626 +41157 34625 +49304 34622 +27531 34622 +40760 34620 +8232 34617 +50136 34615 +18072 34615 +39793 34608 +22356 34606 +32051 34603 +9668 34602 +23998 34602 +40716 34601 +36177 34597 +40137 34595 +25536 34594 +44182 34592 +37661 34591 +41431 34590 +25400 34588 +46230 34584 +30199 34583 +39322 34583 +30584 34582 +36135 34580 +49827 34580 +31786 34580 +27159 34579 +12893 34579 +28439 34573 +41654 34564 +49600 34559 +26675 34558 +32016 34558 +32686 34557 +48598 34556 +22409 34556 +38206 34551 +23552 34550 +12621 34549 +29275 34547 +24155 34545 +25822 34541 +30361 34540 +47428 34538 +47610 34538 +46709 34535 +44845 34534 +42767 34533 +45334 34533 +32545 34530 +39460 34529 +42727 34528 +32375 34528 +34484 34522 +40131 34521 +43051 34519 +27920 34516 +35569 34513 +34382 34513 +30780 34506 +26303 34504 +28881 34499 +32378 34493 +36033 34486 +24895 34484 +32759 34479 +21090 34473 +30155 34472 +46549 34472 +45226 34469 +36043 34468 +49817 34468 +37782 34466 +35483 34466 +22754 34465 +42205 34464 +19543 34460 +20341 34457 +48537 34457 +29129 34456 +28782 34455 +5376 34452 +13425 34451 +17684 34448 +13907 34447 +49119 34445 +47434 34442 +50128 34441 +31514 34440 +28362 34428 +47229 34422 +34532 34419 +23496 34419 +48596 34418 +28692 34416 +45918 34414 +30435 34413 +39326 34409 +36006 34402 +14940 34402 +36105 34400 +18667 34398 +33040 34395 +32386 34395 +23338 34392 +40487 34390 +30148 34389 +30616 34385 +34057 34381 +41250 34380 +30338 34373 +36367 34371 +29923 34370 +18007 34369 +21901 34368 +19336 34366 +41321 34366 +23079 34364 +16819 34362 +21897 34362 +39981 34360 +8731 34357 +40309 34352 +24732 34352 +13484 34350 +48985 34350 +21132 34348 +33149 34348 +44474 34347 +34452 34347 +42879 34346 +32591 34345 +21466 34343 +45698 34338 +41393 34335 +29528 34332 +23222 34331 +24665 34329 +24792 34324 +34418 34321 +27988 34317 +17516 34305 +39191 34289 +46848 34288 +39838 34287 +48180 34286 +23573 34286 +34568 34280 +29215 34276 +18554 34275 +33620 34274 +47620 34273 +30749 34271 +30804 34270 +47662 34268 +42566 34261 +41549 34253 +16488 34252 +30121 34251 +32561 34247 +25663 34246 +43527 34243 +36819 34242 +43067 34237 +21328 34233 +33451 34231 +26309 34230 +44995 34227 +27359 34226 +31924 34225 +31667 34223 +30806 34220 +39152 34220 +32781 34219 +48586 34217 +12021 34216 +30560 34214 +9043 34211 +21557 34203 +32820 34198 +45972 34193 +31082 34192 +19453 34188 +27860 34185 +8619 34185 +49243 34184 +49840 34181 +12745 34180 +27641 34180 +15132 34173 +43079 34172 +26864 34170 +32861 34170 +48118 34167 +14127 34166 +45436 34164 +44674 34159 +46913 34159 +36968 34157 +30458 34152 +8134 34149 +35162 34147 +19366 34144 +23874 34139 +33443 34136 +24525 34136 +40830 34124 +26854 34124 +34864 34124 +20146 34122 +17744 34115 +45138 34114 +49708 34113 +28736 34104 +45655 34101 +25312 34099 +25344 34097 +40565 34092 +45241 34086 +32191 34086 +5590 34083 +28065 34079 +40834 34073 +49248 34072 +22544 34071 +15856 34066 +45275 34064 +43802 34061 +30639 34056 +48833 34053 +44244 34050 +40930 34048 +33456 34044 +38347 34041 +33282 34041 +11785 34041 +49792 34034 +45997 34030 +33776 34029 +21482 34029 +48521 34026 +46517 34026 +37407 34020 +34887 34017 +11361 34017 +32292 34016 +49911 34012 +33688 34010 +32192 34010 +38923 34007 +34826 34006 +16884 33999 +25378 33998 +44645 33991 +26235 33991 +19588 33983 +37052 33979 +35408 33978 +47146 33976 +16768 33975 +5131 33974 +47367 33968 +47165 33965 +34011 33964 +40473 33963 +33535 33962 +34194 33959 +43865 33958 +34493 33957 +46196 33952 +38130 33952 +24552 33950 +34449 33949 +41237 33949 +49868 33942 +15479 33940 +10851 33939 +15004 33939 +32118 33936 +19452 33930 +39680 33926 +49699 33922 +36021 33922 +29167 33914 +10661 33914 +49000 33912 +5784 33912 +36531 33902 +37375 33900 +14681 33900 +25899 33899 +46317 33899 +45168 33898 +6501 33898 +14755 33897 +25239 33890 +28484 33889 +48696 33888 +31279 33888 +40121 33887 +45179 33883 +45954 33880 +13867 33879 +24367 33879 +24202 33877 +41160 33873 +15717 33868 +49099 33867 +4212 33865 +47742 33861 +35820 33858 +7774 33856 +27012 33854 +47761 33854 +44505 33853 +39565 33850 +27030 33848 +44580 33848 +40813 33848 +44283 33847 +46390 33844 +33930 33842 +18417 33842 +41475 33835 +45061 33834 +27018 33833 +35176 33832 +42959 33824 +37252 33824 +37970 33822 +30673 33822 +22088 33821 +44240 33821 +34230 33821 +40210 33817 +40494 33812 +49831 33809 +39388 33804 +46524 33795 +47550 33795 +18674 33790 +45365 33786 +8555 33783 +26160 33780 +44428 33780 +41016 33779 +30403 33778 +36449 33772 +21111 33770 +26569 33767 +40949 33765 +37183 33759 +29626 33758 +43332 33758 +42893 33757 +4882 33755 +22996 33752 +41956 33751 +46706 33751 +10034 33748 +39579 33748 +46307 33745 +27696 33744 +47887 33741 +48878 33738 +34192 33731 +30936 33717 +29197 33716 +45939 33712 +20504 33710 +42874 33710 +39993 33709 +43818 33706 +34423 33705 +43553 33703 +38156 33702 +31760 33701 +47866 33689 +32258 33685 +30558 33682 +39642 33680 +32838 33679 +36202 33679 +44005 33677 +37594 33673 +5431 33673 +34396 33664 +31834 33662 +19535 33660 +36535 33654 +30222 33647 +37933 33646 +41350 33645 +19512 33643 +37397 33640 +47810 33639 +9483 33638 +31677 33631 +25839 33628 +27729 33626 +42807 33620 +40237 33620 +36585 33615 +36923 33615 +46646 33605 +16037 33592 +25700 33590 +22239 33583 +21034 33580 +30318 33577 +35086 33577 +38341 33571 +25383 33570 +38050 33570 +27757 33567 +42265 33566 +40722 33565 +38584 33564 +42099 33561 +45306 33560 +4468 33552 +29794 33550 +32551 33549 +39425 33548 +36466 33543 +35640 33540 +41259 33537 +28681 33535 +25353 33533 +30368 33531 +34125 33526 +34925 33525 +45642 33525 +49032 33524 +48977 33518 +19067 33516 +33397 33514 +34846 33514 +34322 33512 +45313 33511 +27503 33509 +42688 33502 +21863 33502 +33046 33498 +40150 33484 +34511 33479 +28460 33476 +33057 33475 +24765 33473 +31914 33473 +37080 33472 +38869 33472 +21944 33470 +34065 33468 +24733 33466 +32709 33465 +23291 33464 +37881 33463 +36223 33462 +34412 33459 +11484 33459 +42444 33458 +33671 33457 +49384 33454 +26028 33453 +41421 33448 +34918 33447 +10772 33446 +32565 33443 +15877 33442 +16973 33442 +42129 33441 +36218 33440 +49142 33438 +28543 33437 +17911 33437 +41751 33436 +47105 33435 +43210 33433 +10646 33432 +45832 33414 +46737 33408 +37353 33405 +36845 33405 +17717 33405 +41001 33405 +33287 33403 +40174 33399 +38090 33398 +9895 33397 +44371 33395 +35883 33391 +38397 33390 +16539 33381 +23379 33372 +45950 33372 +48595 33371 +28115 33369 +36794 33368 +36736 33364 +27385 33364 +35516 33361 +29477 33360 +28977 33358 +32329 33357 +38327 33357 +14596 33356 +40102 33353 +44364 33346 +34514 33345 +33055 33343 +28440 33341 +48519 33338 +24674 33335 +27450 33332 +22262 33330 +48240 33325 +7693 33325 +38123 33324 +27665 33323 +30070 33322 +35872 33322 +28149 33321 +42262 33319 +46961 33315 +47433 33314 +23035 33311 +10881 33308 +43880 33307 +47591 33307 +19393 33307 +15672 33301 +14152 33298 +12327 33297 +46174 33296 +33327 33296 +43165 33292 +33537 33292 +40229 33290 +44735 33287 +41109 33280 +40534 33277 +49364 33277 +42913 33274 +47002 33273 +18254 33270 +10440 33263 +43672 33260 +29606 33260 +19807 33258 +33166 33256 +14970 33253 +49859 33251 +25590 33251 +20218 33250 +42968 33250 +7033 33249 +39976 33248 +37024 33248 +24285 33242 +35225 33241 +35527 33241 +46225 33240 +40508 33237 +49291 33237 +39486 33235 +26677 33233 +30798 33229 +41623 33228 +36825 33225 +29201 33223 +34318 33223 +41117 33219 +43344 33219 +23863 33217 +33304 33216 +28208 33214 +38751 33212 +36893 33209 +44277 33207 +34157 33207 +18934 33204 +42361 33204 +21209 33201 +44619 33195 +29077 33193 +45324 33193 +24133 33191 +20548 33189 +40625 33187 +43852 33186 +32057 33186 +47729 33178 +38677 33177 +16000 33176 +37754 33176 +49838 33175 +31553 33170 +22735 33165 +13438 33163 +19428 33162 +32841 33161 +46382 33161 +23340 33158 +21449 33156 +33626 33155 +35358 33152 +39300 33151 +46401 33149 +13828 33147 +41921 33147 +26097 33146 +40896 33144 +18430 33141 +21484 33141 +37502 33139 +20451 33139 +26531 33138 +28290 33137 +47691 33135 +46226 33131 +13935 33126 +27303 33124 +26230 33124 +32297 33121 +35491 33118 +49910 33115 +29001 33112 +34277 33110 +50100 33109 +41907 33108 +3997 33107 +42255 33104 +8820 33102 +41975 33101 +20087 33096 +26069 33092 +25873 33088 +36032 33087 +47917 33086 +45010 33082 +19596 33082 +33774 33080 +22794 33077 +49178 33077 +24863 33075 +42598 33072 +7843 33070 +41558 33064 +39989 33064 +49349 33063 +39966 33061 +30100 33059 +38348 33054 +43044 33049 +35461 33044 +30799 33043 +41802 33041 +27626 33039 +41879 33038 +26552 33038 +26200 33036 +47645 33036 +27961 33036 +28072 33034 +30290 33032 +29607 33023 +32563 33022 +27706 33021 +30703 33019 +30742 33010 +42647 33006 +14094 33006 +26845 33001 +38486 33000 +39351 32999 +43360 32998 +46243 32998 +44716 32993 +47222 32991 +33679 32989 +44937 32986 +35725 32982 +36638 32982 +2707 32979 +44462 32977 +40995 32977 +31675 32976 +26970 32967 +22340 32967 +40883 32966 +42491 32964 +21222 32960 +30689 32952 +43494 32950 +31722 32945 +45550 32944 +37844 32942 +45779 32939 +38749 32938 +43088 32936 +20028 32934 +32349 32933 +7775 32931 +34115 32929 +43635 32926 +44304 32923 +4192 32923 +42617 32923 +24613 32919 +42146 32919 +30181 32919 +17605 32915 +44712 32915 +31511 32908 +48145 32906 +34394 32906 +34942 32904 +37255 32902 +33754 32897 +33106 32895 +44720 32895 +38042 32895 +29061 32894 +17037 32890 +44861 32887 +28734 32885 +6984 32878 +44377 32872 +45451 32865 +43567 32865 +17204 32865 +43133 32864 +19220 32864 +13896 32862 +38573 32861 +44656 32853 +29584 32853 +33690 32852 +33712 32852 +27913 32852 +43598 32849 +28287 32848 +30896 32845 +42659 32844 +22011 32844 +28562 32843 +36115 32843 +49573 32838 +13451 32833 +18484 32833 +49559 32825 +37792 32824 +28877 32823 +37807 32821 +44142 32820 +47782 32817 +25009 32812 +8017 32810 +42633 32809 +31921 32808 +49331 32807 +17980 32805 +23830 32799 +34868 32796 +19998 32793 +25042 32792 +35858 32792 +37107 32791 +27783 32789 +24595 32784 +40968 32782 +45360 32780 +46392 32776 +43279 32773 +28468 32772 +40416 32770 +15214 32769 +31900 32765 +47958 32764 +17954 32763 +49237 32758 +42200 32751 +24192 32751 +22357 32750 +32409 32750 +36222 32749 +20904 32745 +5893 32745 +31899 32744 +6913 32742 +33390 32740 +41038 32739 +43560 32737 +41122 32732 +35333 32730 +20429 32730 +45041 32729 +9838 32729 +46806 32726 +22284 32724 +37398 32723 +31707 32723 +37202 32722 +41210 32717 +14334 32716 +48743 32715 +42755 32714 +27714 32713 +35108 32712 +31640 32712 +31550 32711 +48829 32711 +5993 32703 +46843 32701 +43107 32698 +13710 32698 +39959 32696 +35634 32692 +26150 32690 +42377 32690 +21560 32689 +39395 32686 +34833 32684 +40597 32683 +32525 32681 +11595 32680 +40017 32679 +6146 32677 +27424 32677 +30429 32677 +26948 32675 +32764 32674 +28327 32670 +36001 32669 +41674 32668 +49725 32666 +8380 32665 +29251 32661 +30766 32660 +11750 32656 +8521 32652 +48329 32647 +42128 32647 +34598 32643 +33504 32642 +32303 32640 +10196 32640 +34465 32636 +28557 32635 +18528 32634 +3293 32630 +45558 32630 +36765 32629 +36187 32629 +4080 32627 +32220 32625 +27781 32620 +36465 32613 +40191 32612 +28221 32602 +46188 32598 +41154 32598 +49245 32597 +29441 32597 +2467 32596 +1696 32595 +46419 32587 +3171 32585 +42859 32579 +35290 32577 +27652 32573 +39430 32573 +33169 32569 +37785 32565 +31084 32565 +36396 32562 +47019 32559 +24700 32558 +10847 32555 +23196 32551 +40501 32548 +39541 32545 +34563 32542 +48715 32541 +36318 32540 +33298 32539 +47320 32539 +34409 32539 +30127 32538 +30995 32537 +38691 32535 +48704 32533 +802 32530 +41418 32529 +16177 32526 +25855 32526 +26231 32525 +35857 32522 +19457 32522 +30123 32521 +35599 32521 +45802 32521 +35084 32519 +30434 32518 +43136 32515 +47948 32514 +47115 32513 +26342 32510 +18746 32509 +50102 32504 +39474 32502 +44193 32497 +27222 32497 +9311 32497 +8899 32494 +2027 32494 +38985 32491 +23364 32486 +8317 32483 +47119 32482 +4416 32477 +37312 32475 +3243 32475 +21181 32468 +42513 32468 +27342 32468 +36155 32468 +36731 32467 +40527 32467 +47820 32464 +46624 32460 +37433 32459 +46959 32451 +32389 32450 +13636 32449 +35669 32446 +15156 32445 +12773 32444 +36841 32444 +35563 32441 +39577 32433 +28062 32427 +49038 32426 +29854 32418 +47008 32417 +34751 32413 +48621 32412 +40525 32411 +27212 32409 +47054 32406 +45120 32403 +21551 32403 +49198 32398 +47619 32397 +14781 32397 +43082 32396 +50185 32395 +26265 32395 +33440 32391 +34275 32385 +47386 32379 +41428 32378 +35959 32377 +45889 32375 +34323 32372 +39599 32370 +41750 32368 +46991 32365 +40536 32363 +41527 32361 +39937 32361 +32282 32356 +35931 32353 +8569 32352 +11714 32346 +39804 32337 +5463 32336 +31496 32333 +33389 32333 +34757 32330 +21184 32330 +32637 32325 +40836 32323 +6863 32321 +46400 32320 +35322 32315 +43091 32309 +27287 32302 +42844 32297 +38515 32293 +24885 32293 +30832 32291 +40535 32290 +35352 32289 +38010 32288 +31883 32285 +32446 32283 +30824 32275 +36947 32275 +48363 32273 +38441 32273 +25818 32259 +36331 32253 +27728 32250 +41961 32248 +30869 32246 +40180 32244 +36855 32239 +37609 32238 +40529 32237 +30733 32237 +29930 32235 +42278 32235 +42414 32235 +24438 32233 +40409 32227 +34357 32224 +18731 32224 +41996 32224 +46345 32221 +40028 32219 +37110 32217 +33604 32217 +38945 32214 +11906 32214 +44262 32213 +42798 32212 +17801 32211 +32850 32211 +48910 32206 +28230 32205 +35880 32205 +35389 32203 +50099 32200 +35054 32196 +47749 32193 +47968 32193 +46945 32191 +28981 32190 +33267 32187 +34791 32176 +34071 32169 +33785 32167 +27453 32167 +11036 32166 +38424 32164 +22696 32164 +38436 32164 +47543 32163 +40699 32158 +46908 32158 +33064 32156 +12307 32156 +8988 32154 +29564 32152 +37700 32146 +36711 32145 +29855 32133 +13330 32132 +30291 32132 +12211 32132 +29813 32130 +20910 32121 +35897 32120 +19825 32119 +18998 32116 +29509 32113 +9381 32113 +45665 32108 +26286 32105 +30819 32104 +34114 32103 +14428 32102 +48455 32099 +12288 32098 +19540 32095 +39713 32092 +37417 32091 +43638 32090 +44553 32089 +46420 32089 +37109 32088 +37679 32087 +23775 32080 +34977 32079 +13625 32077 +19728 32075 +26583 32074 +29236 32073 +32522 32069 +18211 32065 +32638 32063 +36651 32062 +31717 32062 +43508 32058 +32045 32049 +36078 32048 +14070 32047 +42454 32043 +49127 32040 +35314 32038 +41951 32031 +43893 32031 +46967 32031 +36784 32029 +12826 32024 +47029 32022 +38049 32020 +13418 32019 +33326 32017 +36053 32013 +34616 32011 +37479 32010 +17130 32010 +43043 32009 +20535 32009 +36943 32008 +32066 32007 +12048 32006 +48837 31988 +47239 31984 +18627 31978 +42304 31974 +47267 31974 +9615 31967 +44794 31966 +30336 31965 +21175 31960 +13934 31957 +41106 31953 +14832 31953 +31980 31952 +47893 31950 +35164 31948 +44414 31947 +34587 31946 +35026 31943 +32732 31939 +2812 31939 +50160 31938 +42932 31938 +35371 31934 +43684 31933 +16338 31933 +45931 31930 +40296 31927 +21267 31926 +27049 31923 +26226 31921 +31570 31920 +34052 31918 +27977 31911 +20267 31909 +36182 31907 +44446 31906 +50034 31903 +29907 31900 +38444 31898 +6739 31897 +32544 31889 +41894 31888 +31180 31882 +27128 31882 +29695 31881 +34646 31880 +41244 31878 +31850 31878 +29620 31878 +32718 31876 +49285 31871 +19914 31870 +37861 31868 +47653 31867 +40087 31866 +36005 31858 +35033 31857 +22146 31853 +13634 31853 +49458 31850 +16664 31849 +44273 31846 +46173 31844 +50195 31840 +40313 31839 +25838 31838 +40330 31837 +30981 31836 +46461 31836 +32652 31836 +43545 31833 +13760 31832 +27679 31828 +42949 31828 +33435 31827 +48093 31827 +29348 31820 +18061 31817 +28660 31817 +27832 31811 +49264 31807 +22602 31805 +18770 31803 +17953 31802 +22431 31797 +32702 31794 +39248 31792 +44849 31792 +13057 31791 +15574 31788 +50013 31787 +35365 31787 +30440 31784 +27842 31784 +32159 31780 +14315 31779 +30387 31775 +29920 31769 +38074 31768 +44476 31766 +18113 31766 +31190 31764 +36329 31760 +39638 31757 +41809 31756 +38811 31756 +29856 31753 +48475 31748 +34444 31748 +47926 31746 +40068 31745 +46108 31742 +40774 31742 +8746 31729 +18216 31729 +35081 31728 +47107 31728 +23808 31727 +33191 31726 +25765 31724 +38331 31721 +34082 31721 +25553 31720 +34446 31720 +40112 31717 +24642 31717 +2730 31717 +29548 31716 +20540 31710 +28466 31703 +1668 31703 +38781 31703 +18906 31698 +50215 31698 +43068 31697 +23583 31695 +40980 31694 +44773 31693 +41286 31692 +45848 31686 +26742 31685 +11295 31679 +13842 31678 +22787 31678 +21168 31677 +30723 31676 +44776 31676 +40334 31674 +37552 31673 +36369 31673 +23892 31671 +27137 31669 +17249 31669 +31690 31667 +35808 31664 +23061 31661 +12413 31661 +41856 31657 +47259 31653 +47841 31652 +7010 31651 +40032 31644 +1409 31644 +43712 31644 +30092 31643 +38724 31640 +41177 31636 +50225 31632 +48509 31629 +26239 31621 +29075 31619 +30114 31610 +5902 31608 +48723 31599 +39809 31598 +41413 31596 +13658 31595 +49753 31595 +31911 31594 +14541 31593 +33144 31591 +22858 31588 +23876 31587 +35661 31581 +47865 31579 +29474 31578 +47254 31573 +24645 31573 +36544 31572 +26883 31572 +15541 31562 +37719 31562 +41655 31561 +47316 31560 +29516 31560 +26429 31555 +25994 31552 +28483 31550 +42204 31548 +28469 31541 +40869 31539 +28937 31538 +36402 31538 +11509 31537 +15622 31532 +30577 31530 +36959 31529 +30362 31528 +15839 31525 +44889 31523 +30057 31523 +45112 31522 +20587 31518 +25489 31513 +39254 31510 +47734 31507 +47204 31506 +45445 31506 +46070 31504 +35436 31503 +34750 31501 +23410 31499 +42248 31497 +36404 31493 +13595 31490 +35361 31490 +38240 31488 +43195 31486 +36654 31484 +44181 31481 +42193 31480 +33115 31479 +38850 31477 +29540 31475 +26065 31475 +24838 31474 +41631 31470 +27867 31466 +10289 31466 +6734 31465 +31713 31463 +27990 31459 +15829 31457 +43693 31455 +31240 31452 +44346 31447 +24988 31446 +49013 31446 +48973 31445 +23409 31445 +38024 31443 +39844 31442 +29288 31433 +37820 31431 +25480 31428 +34810 31426 +27854 31424 +33544 31422 +38475 31421 +23849 31420 +15560 31419 +40094 31417 +41517 31407 +43609 31402 +27578 31389 +36362 31387 +34144 31387 +30911 31386 +43556 31382 +26290 31375 +38531 31374 +42403 31371 +39955 31371 +37989 31370 +41399 31368 +18856 31366 +13470 31366 +45309 31360 +44205 31360 +35587 31359 +34665 31357 +39514 31354 +42596 31345 +47158 31341 +31345 31340 +44577 31340 +24932 31340 +22788 31338 +37582 31335 +48354 31333 +41062 31333 +48214 31332 +31602 31329 +23491 31328 +32847 31326 +36186 31326 +44722 31325 +36920 31325 +42461 31323 +46377 31322 +42497 31319 +39532 31317 +8004 31316 +19777 31316 +40873 31315 +35592 31314 +44834 31312 +40579 31311 +25405 31307 +32339 31304 +32749 31301 +38361 31299 +6967 31297 +27720 31295 +44678 31293 +27597 31289 +22067 31288 +35853 31288 +41118 31279 +13506 31277 +16454 31268 +22674 31267 +31601 31261 +38890 31261 +40818 31257 +17251 31256 +37137 31255 +48491 31254 +29191 31250 +45852 31248 +43371 31246 +46113 31242 +6231 31241 +48917 31234 +30182 31233 +17111 31233 +2591 31232 +6558 31229 +18758 31229 +11655 31228 +15093 31228 +27689 31226 +43578 31225 +28516 31223 +42153 31222 +39362 31220 +29750 31220 +38940 31214 +32424 31213 +32974 31213 +6820 31213 +10372 31208 +34190 31201 +40064 31193 +483 31186 +19191 31185 +44534 31184 +29498 31172 +35376 31168 +31672 31164 +48172 31156 +7985 31154 +30195 31153 +15407 31153 +48824 31152 +22664 31147 +34209 31146 +40562 31145 +1383 31144 +24774 31143 +38468 31141 +31632 31140 +23930 31135 +47961 31132 +44811 31131 +32624 31130 +40142 31124 +25989 31123 +40939 31122 +9093 31122 +16875 31120 +35552 31117 +12559 31116 +38405 31115 +35462 31112 +24686 31111 +5505 31109 +41536 31109 +35963 31106 +37382 31103 +41116 31102 +30847 31100 +46128 31099 +40750 31098 +50043 31096 +38545 31095 +42394 31095 +33026 31094 +29190 31093 +49309 31080 +40623 31079 +10631 31078 +18126 31074 +39975 31073 +21460 31073 +11569 31072 +37208 31072 +48308 31071 +32035 31068 +44527 31068 +17632 31066 +29571 31066 +26823 31058 +22556 31057 +32471 31056 +10987 31055 +29833 31054 +43325 31054 +31816 31053 +35132 31049 +47587 31048 +11525 31047 +28292 31045 +14585 31043 +5255 31040 +36454 31039 +42285 31037 +45583 31037 +33850 31036 +32075 31035 +31575 31024 +24453 31021 +47667 31020 +33145 31016 +23418 31016 +36441 31015 +47779 31013 +22589 31013 +46124 31011 +46638 31004 +47395 31001 +31306 31001 +42045 30999 +42958 30995 +5568 30994 +27881 30994 +30842 30992 +29171 30989 +46398 30983 +18171 30981 +34952 30976 +44085 30975 +46494 30974 +25594 30973 +27763 30971 +37658 30971 +49982 30968 +38410 30964 +38098 30960 +26375 30959 +23819 30952 +48692 30947 +34726 30944 +47448 30940 +38007 30939 +49556 30938 +45466 30937 +47751 30927 +49344 30926 +31756 30925 +10507 30922 +48044 30920 +38246 30917 +19019 30916 +35665 30914 +33037 30912 +33649 30912 +15419 30905 +36730 30904 +31055 30899 +39877 30898 +37687 30897 +39130 30895 +37959 30889 +21224 30883 +30714 30880 +23553 30876 +42308 30876 +47414 30873 +7562 30873 +37840 30872 +44300 30867 +36869 30866 +47519 30860 +21589 30860 +49591 30858 +41199 30856 +25790 30854 +46627 30853 +48097 30851 +48939 30849 +44245 30847 +32395 30844 +34499 30841 +35522 30841 +8527 30840 +48805 30839 +37967 30839 +43829 30837 +32827 30834 +24280 30833 +8725 30829 +37783 30828 +34107 30827 +43613 30825 +35152 30822 +37302 30822 +25150 30821 +11551 30816 +48669 30814 +33901 30812 +22070 30812 +7694 30810 +20314 30801 +22822 30799 +31916 30798 +21211 30797 +5388 30794 +34800 30789 +46531 30789 +49083 30789 +48454 30785 +46265 30781 +18546 30778 +42768 30777 +39939 30776 +31297 30769 +48020 30767 +45230 30765 +32859 30765 +23680 30765 +46564 30763 +34672 30762 +14744 30760 +37533 30760 +5266 30755 +41653 30754 +43035 30752 +23235 30748 +33058 30748 +50076 30741 +29048 30739 +34406 30739 +17403 30730 +6664 30726 +46394 30726 +22009 30724 +38668 30722 +16955 30721 +38939 30721 +46168 30712 +17786 30706 +44313 30704 +31380 30701 +20283 30701 +44809 30701 +23322 30698 +33686 30691 +40712 30691 +20189 30690 +30797 30688 +45455 30688 +23914 30686 +21988 30679 +45540 30677 +38609 30674 +37501 30673 +45794 30670 +25487 30669 +50153 30663 +26561 30659 +47136 30658 +27944 30657 +40826 30654 +46867 30652 +16775 30650 +35859 30649 +40145 30648 +37421 30647 +36798 30645 +29159 30644 +22796 30640 +29925 30640 +29455 30635 +25323 30634 +38647 30629 +47607 30626 +23210 30621 +21753 30616 +42835 30616 +48241 30613 +29879 30611 +25777 30608 +42732 30599 +40590 30597 +49529 30596 +41184 30594 +26518 30591 +1246 30591 +17891 30590 +14278 30590 +45092 30588 +41442 30587 +39132 30584 +41843 30583 +31504 30582 +42337 30580 +11367 30578 +35776 30578 +41110 30577 +15948 30575 +37482 30575 +29958 30572 +31237 30570 +13761 30569 +36417 30567 +10167 30565 +35601 30562 +13507 30559 +42025 30558 +41332 30557 +30949 30555 +44353 30551 +37300 30550 +44432 30542 +22074 30542 +22477 30541 +45260 30541 +37850 30540 +32989 30539 +16587 30539 +38198 30536 +42833 30535 +9810 30534 +27102 30530 +31591 30530 +48001 30530 +38708 30530 +35175 30526 +11457 30525 +25857 30524 +10535 30524 +44693 30517 +37530 30515 +29331 30514 +27047 30513 +41997 30513 +36140 30505 +45593 30502 +48446 30497 +18462 30496 +37795 30492 +15912 30480 +12489 30479 +41574 30476 +26156 30469 +30111 30462 +45581 30462 +3107 30460 +43983 30460 +35607 30459 +42708 30449 +45370 30448 +31284 30446 +31344 30444 +46981 30442 +39875 30440 +49567 30438 +18119 30435 +37179 30433 +31414 30433 +27233 30432 +11830 30431 +39775 30427 +46125 30424 +39977 30423 +46206 30423 +45292 30422 +45432 30422 +5687 30418 +37144 30416 +30098 30416 +42016 30410 +42160 30401 +30119 30400 +33890 30398 +26613 30397 +29083 30394 +23970 30394 +46418 30387 +18837 30385 +22271 30385 +47131 30383 +8908 30383 +44607 30379 +32642 30379 +48336 30378 +39867 30376 +28683 30376 +42041 30376 +32973 30375 +41395 30374 +40101 30372 +16923 30371 +4791 30369 +43409 30368 +25310 30367 +6935 30364 +18242 30356 +42912 30355 +32039 30354 +30864 30353 +44657 30352 +44714 30351 +50020 30350 +44472 30349 +2170 30349 +40168 30347 +11245 30346 +32077 30341 +38942 30337 +27661 30336 +25253 30335 +32690 30334 +46937 30332 +48644 30329 +42806 30327 +29053 30324 +16591 30324 +41091 30320 +43374 30319 +41968 30316 +44497 30312 +35788 30306 +33393 30306 +13136 30303 +36877 30300 +24099 30300 +50086 30295 +22734 30295 +49023 30290 +39498 30288 +13192 30285 +48575 30281 +45919 30273 +39311 30272 +1470 30270 +44297 30270 +27285 30267 +45204 30266 +30983 30263 +20965 30263 +21236 30261 +20598 30260 +23739 30256 +45374 30248 +38097 30247 +40594 30245 +27431 30239 +17121 30238 +26992 30230 +17478 30230 +18291 30229 +46944 30225 +20985 30225 +10875 30222 +14308 30220 +39659 30219 +43746 30217 +31700 30217 +16390 30217 +38101 30217 +42018 30205 +46694 30202 +46853 30200 +34566 30199 +39293 30192 +27332 30192 +11890 30188 +30955 30188 +47207 30187 +24123 30184 +30285 30183 +48778 30177 +37854 30175 +41187 30172 +49066 30170 +19691 30169 +44016 30168 +39331 30164 +38319 30163 +47791 30158 +39145 30157 +21520 30157 +9421 30157 +42779 30156 +41197 30154 +39469 30153 +37211 30151 +8995 30150 +23232 30147 +34466 30143 +36159 30142 +42715 30138 +6267 30135 +48830 30135 +9006 30133 +37263 30129 +21342 30117 +18343 30113 +26343 30112 +16992 30107 +48045 30106 +41834 30105 +46818 30103 +18597 30102 +26514 30102 +47188 30101 +42551 30101 +8334 30099 +37583 30096 +26024 30095 +46215 30094 +45479 30092 +48870 30091 +40810 30086 +20216 30085 +46454 30082 +36522 30081 +36154 30081 +45709 30080 +13056 30080 +46590 30078 +3120 30076 +49181 30074 +25085 30073 +27085 30071 +38895 30070 +41728 30070 +49259 30070 +49804 30069 +45397 30069 +38268 30066 +34313 30060 +30962 30058 +38596 30057 +34120 30054 +17925 30052 +41056 30051 +47052 30047 +31408 30035 +31338 30033 +40428 30031 +6160 30028 +43452 30025 +40800 30020 +50233 30016 +45404 30011 +28051 30005 +33034 30003 +46801 30002 +43080 29999 +43338 29992 +50156 29991 +35844 29989 +30450 29987 +27210 29987 +39980 29986 +48966 29986 +30737 29986 +41869 29981 +44670 29979 +28821 29976 +19840 29972 +35340 29971 +21747 29970 +30099 29969 +14918 29968 +15352 29965 +28171 29964 +45389 29956 +39052 29954 +48920 29946 +31887 29944 +41881 29944 +33551 29938 +49522 29936 +43662 29932 +25491 29930 +15979 29930 +38771 29929 +26674 29928 +28659 29926 +12018 29921 +49180 29917 +25372 29916 +42242 29914 +37617 29912 +30523 29909 +29284 29906 +48483 29903 +40723 29901 +33394 29898 +38578 29897 +44852 29895 +46336 29894 +45612 29893 +8751 29892 +46776 29892 +49389 29891 +42134 29890 +33706 29890 +33001 29887 +20217 29882 +45598 29876 +41402 29875 +41690 29875 +24983 29870 +33927 29869 +37759 29867 +48733 29866 +30665 29866 +19759 29866 +44222 29859 +45308 29857 +49207 29857 +49350 29857 +35674 29856 +41067 29856 +36890 29851 +38384 29849 +46790 29848 +10023 29848 +43123 29846 +44740 29843 +47973 29840 +35682 29839 +38273 29829 +32081 29828 +16202 29827 +36036 29824 +47091 29807 +33303 29806 +28479 29806 +44522 29804 +19106 29803 +10666 29799 +43466 29799 +38303 29799 +47334 29792 +41044 29783 +47523 29783 +10434 29783 +44924 29775 +33273 29772 +40999 29770 +5987 29769 +43779 29766 +14949 29763 +27660 29762 +42671 29756 +45164 29755 +33868 29755 +27375 29753 +35195 29750 +22741 29748 +13724 29747 +42275 29739 +21627 29737 +45151 29736 +38849 29732 +44109 29731 +42738 29730 +17443 29730 +47980 29729 +28342 29728 +41266 29728 +16801 29727 +36741 29726 +14994 29725 +48105 29710 +35186 29709 +20373 29708 +45967 29708 +44345 29704 +28042 29702 +38723 29694 +33370 29691 +15942 29690 +33820 29688 +29315 29688 +39071 29687 +45045 29686 +44173 29685 +9035 29685 +14464 29679 +34106 29676 +34692 29663 +26631 29661 +39475 29658 +31046 29656 +44637 29655 +43981 29651 +28315 29650 +34404 29650 +41432 29649 +34547 29648 +37205 29647 +19892 29645 +44141 29644 +42023 29642 +33618 29642 +27066 29638 +32871 29637 +30442 29634 +33300 29634 +25164 29629 +48266 29629 +31940 29628 +41070 29625 +47643 29624 +40315 29617 +40077 29614 +38255 29613 +5037 29608 +28540 29607 +36313 29606 +21772 29605 +39189 29598 +50103 29597 +31282 29597 +26842 29597 +39537 29593 +47001 29592 +15388 29592 +36296 29589 +36065 29587 +30457 29581 +44247 29581 +48492 29579 +43309 29579 +44039 29578 +41608 29578 +43247 29576 +23212 29574 +34956 29573 +49471 29573 +28166 29570 +45171 29565 +39894 29565 +46784 29564 +23154 29563 +32272 29561 +36262 29556 +33562 29552 +27468 29551 +33350 29551 +36089 29549 +41621 29549 +37627 29549 +33013 29546 +40293 29543 +23073 29536 +24010 29533 +4254 29529 +20490 29528 +20489 29528 +14222 29527 +10985 29515 +44064 29511 +30808 29510 +25412 29508 +23947 29503 +9945 29501 +43700 29498 +45894 29495 +26847 29494 +41933 29489 +41098 29489 +41004 29489 +18425 29489 +46833 29486 +34709 29484 +45153 29483 +28035 29482 +27984 29480 +14830 29475 +38756 29475 +22456 29473 +41288 29473 +19513 29472 +39423 29472 +31785 29470 +48061 29469 +30335 29468 +31224 29464 +25414 29462 +28311 29461 +15703 29461 +21844 29456 +34986 29456 +48325 29456 +46066 29455 +25871 29453 +42656 29452 +28219 29447 +49996 29447 +8503 29445 +48663 29445 +31125 29440 +39612 29434 +19640 29433 +35000 29433 +42256 29431 +34576 29428 +40469 29426 +34091 29425 +13132 29423 +15527 29421 +34772 29420 +45274 29418 +42176 29416 +26161 29412 +25214 29412 +2820 29409 +33531 29408 +49402 29407 +33043 29407 +48116 29406 +12321 29406 +42706 29405 +40096 29404 +49549 29403 +14550 29402 +9788 29402 +19491 29400 +47135 29399 +3659 29397 +24742 29396 +32148 29390 +36673 29390 +49186 29386 +49305 29384 +17408 29382 +47778 29379 +46166 29378 +16713 29377 +35972 29365 +32858 29364 +36100 29364 +26455 29359 +36724 29358 +28828 29355 +48963 29354 +38126 29353 +24284 29349 +37999 29346 +45255 29340 +36377 29339 +49663 29336 +39846 29336 +36993 29333 +39521 29329 +23657 29328 +33974 29326 +40557 29324 +5817 29322 +45888 29321 +36593 29320 +45622 29318 +22919 29317 +1857 29317 +49419 29314 +1554 29308 +47895 29307 +35752 29306 +44115 29304 +46504 29303 +31444 29302 +29392 29298 +42079 29290 +39858 29289 +41602 29284 +36221 29284 +35518 29281 +22061 29277 +13243 29273 +33407 29272 +41128 29272 +48411 29271 +30923 29270 +49758 29269 +38930 29265 +3762 29262 +22704 29261 +33432 29259 +18428 29253 +42677 29252 +45906 29252 +32770 29251 +20481 29250 +27260 29243 +45981 29243 +27494 29242 +17666 29241 +11395 29240 +39267 29239 +47844 29234 +25127 29233 +35435 29231 +33576 29229 +40297 29228 +32823 29228 +23868 29224 +20407 29224 +44475 29223 +35588 29218 +14172 29218 +37035 29214 +47737 29214 +26354 29213 +26433 29212 +36215 29204 +35229 29204 +37018 29201 +29175 29199 +16067 29198 +35227 29197 +36060 29197 +36153 29193 +20564 29191 +42665 29189 +26726 29188 +47874 29187 +47628 29182 +37590 29182 +42772 29180 +45039 29180 +47327 29178 +37170 29175 +38765 29173 +2028 29172 +28810 29169 +46058 29169 +37290 29168 +30826 29168 +43272 29166 +27108 29163 +31763 29161 +26868 29157 +47565 29154 +22502 29154 +47325 29154 +36042 29147 +44499 29141 +18467 29138 +25532 29138 +9798 29136 +31552 29129 +41307 29128 +42933 29125 +44460 29123 +38614 29120 +47501 29119 +27824 29118 +35821 29118 +33038 29117 +18208 29111 +46563 29109 +41768 29103 +43713 29098 +26079 29098 +38133 29096 +13812 29094 +41550 29090 +42560 29088 +23927 29087 +45105 29084 +41978 29083 +45843 29083 +37207 29082 +47541 29081 +18986 29075 +43365 29074 +35778 29074 +34522 29070 +35354 29069 +42674 29064 +20978 29063 +39471 29063 +25897 29059 +15653 29058 +45174 29057 +38492 29057 +35367 29056 +30460 29052 +33949 29050 +34975 29048 +34843 29047 +31697 29046 +26946 29045 +38349 29044 +24615 29044 +37904 29042 +38888 29040 +48388 29038 +21519 29037 +45087 29035 +45586 29035 +35451 29029 +40582 29028 +31203 29026 +26567 29025 +43033 29020 +39879 29018 +25933 29016 +30557 29016 +44407 29013 +43906 29013 +23067 29008 +45690 29006 +17152 29002 +30704 29001 +40867 29000 +36435 29000 +15695 28998 +36617 28996 +34770 28995 +48095 28994 +31963 28991 +48217 28988 +36022 28988 +45190 28980 +44434 28978 +49452 28977 +30015 28976 +8658 28973 +21734 28973 +41606 28973 +28529 28973 +25681 28968 +1827 28967 +28129 28963 +41301 28961 +8175 28959 +30480 28958 +37281 28957 +37106 28957 +46167 28955 +18191 28955 +39024 28955 +30760 28954 +37585 28954 +8374 28950 +30131 28949 +22277 28949 +44533 28948 +26748 28947 +17667 28945 +45040 28945 +10952 28938 +26793 28936 +31423 28927 +39055 28924 +38763 28924 +44073 28924 +49726 28915 +30083 28910 +20125 28910 +43139 28907 +34267 28904 +2609 28901 +18698 28899 +45312 28899 +44105 28898 +35214 28895 +43798 28894 +33543 28893 +45985 28891 +40284 28890 +36789 28889 +22390 28886 +24283 28886 +47166 28886 +34274 28880 +25848 28879 +35257 28879 +13372 28876 +9854 28871 +37011 28871 +43317 28866 +40227 28864 +14881 28864 +32513 28864 +47289 28863 +30271 28860 +49040 28856 +25350 28852 +29896 28847 +28043 28846 +44885 28844 +26882 28842 +3676 28842 +42464 28841 +36669 28838 +20984 28836 +28184 28836 +42957 28831 +44594 28829 +10336 28829 +34279 28826 +37830 28825 +43286 28823 +27598 28823 +38258 28823 +47422 28821 +11612 28816 +48397 28814 +9010 28810 +33957 28809 +41217 28806 +47940 28805 +25019 28805 +39951 28805 +37402 28802 +39033 28801 +47301 28797 +17426 28796 +38530 28794 +28720 28794 +36667 28793 +48259 28792 +31985 28789 +44624 28788 +37385 28787 +23886 28787 +48707 28784 +39705 28784 +47034 28782 +25002 28781 +41739 28780 +28884 28779 +40738 28779 +32328 28778 +36311 28775 +29973 28764 +43646 28761 +29436 28761 +37061 28760 +38369 28759 +10100 28759 +28912 28755 +35768 28753 +11836 28753 +45501 28752 +31710 28749 +30372 28747 +43141 28747 +25298 28746 +44964 28744 +46283 28743 +29308 28741 +27766 28740 +40658 28739 +23345 28738 +36403 28736 +16114 28735 +34324 28734 +41234 28733 +37209 28731 +23742 28727 +40675 28727 +17540 28724 +43124 28720 +26189 28719 +33818 28718 +30803 28718 +28256 28717 +33472 28715 +15166 28715 +9600 28713 +32990 28705 +48134 28705 +18398 28705 +16438 28703 +13394 28699 +42078 28698 +41042 28689 +8387 28688 +48384 28685 +28371 28684 +15730 28684 +36795 28681 +48993 28681 +31398 28678 +18509 28678 +33645 28675 +33316 28673 +43295 28673 +32266 28671 +49734 28671 +11090 28670 +45805 28670 +36286 28668 +15791 28668 +47183 28668 +45004 28665 +34433 28665 +33903 28663 +33053 28662 +22018 28660 +44732 28658 +21066 28657 +48464 28654 +14696 28653 +29726 28652 +34385 28645 +39269 28642 +35219 28639 +34628 28637 +15059 28636 +22215 28636 +46142 28634 +29416 28633 +30807 28631 +45456 28630 +3986 28622 +49775 28622 +36871 28621 +33639 28620 +35288 28618 +40200 28617 +37657 28615 +19928 28615 +31416 28610 +25388 28607 +42222 28600 +29088 28600 +34561 28598 +29814 28598 +11479 28597 +34831 28596 +49779 28596 +49301 28595 +43181 28589 +7495 28589 +45084 28589 +40075 28587 +37136 28585 +36273 28579 +36720 28578 +12292 28570 +32531 28569 +33355 28569 +49236 28569 +48698 28569 +10272 28568 +39947 28567 +37760 28566 +27971 28562 +43028 28556 +47094 28554 +40479 28552 +16244 28552 +49320 28552 +12455 28548 +22789 28546 +18296 28542 +37243 28541 +29305 28540 +28426 28537 +27610 28531 +7966 28529 +26976 28521 +49465 28521 +27145 28519 +50162 28517 +38901 28516 +49978 28510 +49212 28508 +39519 28508 +29766 28507 +40316 28506 +26075 28506 +35641 28506 +32459 28505 +45858 28504 +34391 28504 +29892 28501 +45055 28501 +50134 28500 +34512 28500 +27310 28499 +39418 28498 +25517 28497 +14054 28496 +29893 28492 +1354 28492 +34415 28491 +42374 28491 +47093 28490 +33741 28490 +29381 28488 +40838 28486 +32658 28482 +26920 28481 +28950 28479 +39185 28478 +39063 28477 +15978 28475 +32199 28475 +45703 28470 +33748 28467 +21544 28466 +42161 28464 +29390 28461 +38121 28459 +25812 28457 +39440 28455 +30343 28454 +31128 28453 +27841 28451 +35095 28445 +45096 28444 +47606 28444 +34515 28443 +2772 28442 +50072 28439 +14972 28438 +35447 28435 +18214 28433 +40802 28432 +30591 28426 +31828 28424 +24394 28423 +39144 28423 +45283 28420 +39843 28415 +47568 28414 +37283 28414 +39924 28408 +44034 28404 +30743 28400 +49435 28396 +28441 28392 +42388 28389 +39202 28388 +49227 28388 +48415 28386 +32187 28384 +49507 28382 +48377 28382 +36821 28371 +30419 28371 +40329 28367 +39238 28367 +46060 28365 +40463 28362 +38702 28362 +34722 28361 +38840 28361 +1121 28361 +45231 28359 +48969 28357 +44264 28356 +93 28353 +49401 28353 +16351 28347 +21979 28345 +43347 28342 +40023 28342 +20045 28341 +42699 28338 +35046 28338 +28927 28334 +11041 28333 +45663 28328 +28674 28326 +35868 28326 +33479 28324 +49871 28323 +25730 28322 +38867 28318 +49164 28316 +29937 28315 +42816 28313 +45414 28313 +4350 28312 +45393 28308 +20544 28302 +28709 28302 +47629 28302 +50042 28301 +38968 28297 +50190 28292 +38081 28291 +39982 28288 +37360 28283 +47266 28282 +45730 28282 +27966 28280 +15545 28276 +49242 28275 +45146 28274 +31729 28271 +39988 28270 +42102 28268 +42212 28267 +31524 28265 +25220 28264 +30465 28263 +35537 28261 +48355 28255 +36371 28253 +18909 28247 +37473 28246 +48352 28235 +22048 28231 +40271 28229 +2970 28219 +14427 28216 +42331 28214 +28270 28214 +34296 28212 +41704 28212 +48929 28206 +20235 28206 +23664 28204 +49957 28204 +31194 28198 +32042 28191 +38570 28190 +37750 28189 +39856 28184 +13021 28184 +49206 28183 +46240 28180 +38158 28179 +41487 28177 +28413 28177 +12619 28177 +49729 28176 +40721 28176 +13406 28172 +49786 28172 +35757 28168 +24633 28163 +33821 28162 +42831 28157 +46559 28156 +36013 28155 +45600 28150 +38404 28150 +34212 28146 +22840 28141 +27451 28140 +27713 28140 +30045 28139 +21741 28137 +44541 28137 +27087 28135 +36123 28134 +33253 28128 +34458 28124 +27934 28123 +49078 28121 +21716 28120 +30120 28119 +49698 28118 +33830 28118 +47890 28118 +16977 28116 +27417 28115 +14986 28114 +26997 28113 +23820 28108 +22907 28108 +43764 28107 +11747 28105 +30191 28102 +39048 28101 +45694 28100 +26165 28100 +24830 28099 +43622 28092 +28531 28091 +26120 28087 +25139 28084 +38773 28084 +49080 28077 +27507 28076 +35117 28072 +39247 28072 +47341 28072 +49308 28071 +37507 28065 +45296 28065 +18779 28065 +46845 28061 +30586 28060 +38589 28059 +33212 28056 +43472 28049 +12738 28046 +37964 28043 +5190 28035 +31048 28035 +48982 28034 +23723 28033 +14785 28031 +48927 28031 +40038 28030 +48706 28024 +20095 28022 +46553 28022 +39645 28019 +23578 28019 +43882 28019 +37419 28011 +24752 28009 +49448 28007 +39349 28006 +34307 28003 +17712 28002 +40272 28002 +42341 28000 +22479 28000 +49399 27997 +20539 27997 +43905 27996 +50065 27996 +14402 27994 +14118 27992 +32030 27992 +41537 27990 +44335 27989 +37734 27989 +18982 27988 +48100 27986 +22768 27984 +26651 27983 +47943 27981 +38622 27979 +32182 27979 +34432 27974 +50077 27971 +48074 27970 +46873 27969 +19027 27968 +40504 27966 +39594 27964 +24472 27964 +40788 27963 +34476 27962 +32695 27958 +18262 27958 +29999 27957 +22795 27956 +47992 27954 +48281 27952 +38476 27951 +35891 27949 +31014 27944 +42723 27943 +39171 27940 +20663 27938 +14534 27937 +26780 27936 +46473 27934 +42935 27934 +34066 27934 +36957 27930 +25021 27929 +41194 27925 +38792 27920 +18712 27919 +24997 27919 +34526 27915 +19549 27913 +47695 27913 +9428 27913 +22532 27909 +48101 27905 +41366 27900 +47884 27898 +31655 27897 +2379 27897 +15182 27896 +25264 27895 +34182 27894 +39606 27894 +29471 27891 +40093 27891 +29998 27890 +49587 27889 +10950 27885 +24961 27883 +22109 27877 +49354 27877 +42775 27876 +50011 27875 +25515 27872 +43845 27871 +48293 27866 +34353 27864 +31903 27864 +36908 27861 +41669 27858 +24965 27858 +45671 27856 +29769 27855 +40444 27851 +37381 27848 +23170 27848 +33378 27842 +25868 27837 +28056 27836 +46927 27836 +47332 27836 +49295 27835 +43534 27834 +36257 27828 +49620 27828 +21264 27827 +28240 27827 +31228 27825 +41851 27823 +37879 27822 +40369 27820 +48238 27819 +23561 27817 +9224 27808 +47832 27805 +22491 27803 +20094 27802 +1916 27800 +48051 27797 +46163 27791 +22476 27787 +48360 27780 +31907 27776 +21634 27775 +21602 27774 +47338 27774 +32407 27774 +17049 27773 +28945 27765 +13495 27764 +26833 27761 +22715 27759 +45743 27758 +18164 27754 +44772 27754 +45381 27753 +29409 27746 +23542 27745 +27349 27742 +32242 27741 +32935 27737 +47382 27734 +19966 27734 +39842 27734 +42218 27734 +8961 27731 +36165 27728 +31862 27725 +32100 27725 +32880 27725 +46647 27725 +27272 27718 +40845 27717 +49887 27713 +27178 27708 +32312 27705 +36495 27702 +42154 27702 +8019 27699 +49428 27697 +43843 27694 +9720 27693 +46778 27692 +36220 27691 +44129 27691 +34642 27689 +27552 27687 +40610 27681 +44477 27680 +31548 27679 +39336 27679 +27242 27678 +19456 27677 +30141 27673 +32417 27671 +42811 27669 +30536 27661 +37927 27659 +39704 27653 +40941 27651 +43914 27651 +15614 27649 +41605 27648 +34669 27646 +31578 27645 +40903 27643 +45437 27642 +21005 27642 +45273 27642 +45291 27638 +46213 27636 +29228 27630 +41717 27630 +49120 27628 +42510 27626 +41368 27626 +31702 27625 +40049 27625 +28532 27624 +38120 27624 +15316 27623 +44071 27623 +36185 27621 +40067 27619 +48797 27619 +43916 27619 +45756 27618 +27527 27616 +42773 27615 +34413 27615 +40933 27607 +33660 27605 +46605 27604 +44653 27604 +46054 27602 +44583 27602 +36277 27596 +20651 27594 +24992 27589 +36063 27588 +31877 27588 +20148 27586 +43071 27581 +38937 27580 +48815 27575 +41889 27575 +24639 27574 +33277 27564 +45541 27563 +27461 27555 +35190 27551 +41991 27546 +38877 27545 +39379 27544 +22212 27542 +46351 27542 +43417 27541 +15073 27540 +47632 27539 +42907 27538 +42260 27536 +41927 27534 +41278 27534 +39613 27533 +21642 27529 +32957 27528 +17896 27527 +26764 27524 +36491 27520 +41337 27517 +37034 27512 +25251 27512 +28781 27511 +47351 27511 +39669 27508 +38690 27508 +23743 27501 +36086 27498 +26213 27492 +45828 27489 +16203 27478 +29095 27478 +23551 27476 +48788 27475 +43552 27475 +39805 27472 +37147 27471 +46229 27470 +28063 27468 +9054 27466 +30406 27466 +39214 27465 +36999 27463 +49976 27463 +37962 27462 +45733 27461 +43110 27461 +8963 27460 +7513 27459 +8376 27458 +11449 27458 +31691 27457 +41015 27456 +48611 27450 +35830 27450 +28478 27449 +33834 27447 +24038 27447 +49824 27439 +47077 27437 +20419 27436 +25366 27436 +27099 27435 +37494 27435 +43886 27429 +40643 27428 +32523 27424 +39390 27424 +46030 27421 +36378 27415 +30695 27412 +49340 27412 +32388 27408 +31733 27407 +41663 27406 +33396 27405 +33771 27405 +34523 27405 +23535 27404 +30730 27399 +48637 27395 +44668 27389 +37532 27388 +28462 27386 +48029 27384 +48690 27383 +47602 27379 +46236 27378 +31841 27377 +48016 27376 +30702 27367 +34059 27365 +44463 27361 +47366 27361 +24481 27354 +40724 27352 +33032 27351 +42878 27347 +47256 27346 +12475 27343 +25741 27342 +38905 27342 +25942 27342 +38056 27342 +26030 27341 +25481 27340 +31196 27339 +6470 27338 +45955 27336 +47483 27335 +26512 27334 +21397 27333 +30919 27326 +32804 27324 +32799 27321 +50139 27315 +28067 27305 +39868 27304 +47024 27301 +30122 27297 +38332 27295 +27516 27293 +46718 27292 +36157 27291 +19226 27291 +44999 27289 +43919 27289 +39783 27287 +36688 27287 +40604 27285 +50202 27284 +6221 27279 +47270 27279 +41760 27278 +38305 27275 +34247 27275 +44011 27273 +29100 27271 +47092 27267 +30225 27267 +1372 27265 +44854 27263 +34044 27262 +20626 27261 +43489 27260 +35660 27260 +44246 27259 +39602 27258 +45831 27239 +44350 27237 +41241 27235 +42701 27235 +15724 27226 +42653 27220 +39699 27217 +30932 27215 +45716 27212 +40027 27211 +8647 27211 +19041 27210 +36653 27204 +46753 27203 +48186 27202 +11082 27199 +48094 27196 +13729 27193 +43356 27191 +23024 27190 +32700 27188 +43299 27188 +37846 27187 +33372 27184 +40706 27184 +35402 27182 +32585 27182 +11486 27178 +44517 27175 +26543 27168 +40134 27167 +21739 27162 +14853 27161 +43501 27159 +42777 27158 +37621 27157 +40187 27157 +48058 27155 +1369 27154 +15823 27153 +22482 27152 +49642 27150 +28638 27149 +38747 27147 +30944 27147 +37238 27146 +33507 27146 +39135 27145 +9820 27144 +32508 27140 +39059 27137 +7592 27136 +38170 27135 +37701 27132 +29032 27130 +33069 27129 +23747 27127 +33844 27124 +41138 27122 +47441 27118 +43621 27117 +31912 27113 +40844 27113 +22213 27111 +34374 27109 +25836 27106 +46688 27103 +42420 27099 +36116 27096 +42162 27096 +13579 27096 +42984 27093 +37847 27090 +38634 27087 +39840 27087 +34249 27085 +40306 27082 +17225 27081 +12570 27078 +46280 27078 +34489 27075 +37268 27072 +42087 27072 +42642 27067 +14965 27066 +36615 27065 +32040 27061 +34482 27061 +11373 27057 +35882 27055 +33882 27055 +47816 27051 +25431 27049 +41820 27045 +47867 27045 +34377 27041 +48673 27039 +40462 27035 +40303 27033 +4997 27033 +36936 27032 +29198 27031 +40922 27028 +16767 27025 +26461 27021 +11619 27015 +37645 27014 +40888 27014 +42467 27013 +47721 27012 +30146 27012 +22689 27010 +18090 27008 +29858 27006 +41479 27006 +34873 27006 +44389 27005 +47856 27004 +49880 27002 +17882 27001 +44658 26999 +42564 26995 +25114 26995 +16986 26995 +33400 26992 +38125 26991 +33008 26976 +21646 26969 +18760 26967 +35502 26965 +39204 26963 +38893 26962 +31662 26960 +21548 26956 +48166 26955 +31499 26949 +39899 26947 +35328 26941 +43650 26941 +27853 26940 +46630 26939 +44536 26939 +10628 26936 +33638 26934 +36517 26933 +44138 26933 +29371 26930 +43587 26929 +49202 26927 +21459 26924 +45560 26922 +46901 26922 +47509 26920 +45768 26920 +4008 26920 +11588 26919 +45613 26919 +36887 26918 +32998 26917 +42238 26913 +44102 26912 +42347 26911 +49924 26909 +47564 26906 +24914 26905 +25627 26903 +25140 26903 +32050 26901 +10694 26899 +38340 26899 +32460 26898 +19150 26898 +24295 26894 +37127 26893 +29326 26887 +20288 26883 +41385 26883 +35722 26880 +39903 26879 +45834 26877 +37673 26877 +43056 26874 +34471 26871 +22824 26871 +43483 26864 +16088 26863 +39277 26860 +36626 26859 +39392 26859 +36072 26855 +6396 26852 +47277 26852 +31339 26851 +42673 26844 +30768 26843 +39750 26843 +8844 26843 +38810 26838 +29744 26834 +23615 26833 +33117 26830 +47655 26830 +39402 26828 +43236 26828 +32373 26822 +45879 26815 +35499 26813 +44545 26813 +29627 26811 +15684 26810 +36243 26810 +19164 26808 +7936 26807 +50166 26805 +44053 26804 +37563 26803 +44068 26800 +6887 26799 +24398 26796 +32152 26794 +48140 26794 +42270 26791 +27916 26789 +35375 26789 +32399 26789 +20457 26784 +47187 26784 +48350 26783 +49680 26782 +29153 26781 +27838 26780 +43548 26776 +35677 26775 +32731 26775 +46090 26773 +48234 26773 +37241 26770 +44947 26768 +27115 26767 +9797 26766 +31020 26765 +7347 26764 +5829 26761 +37547 26761 +31837 26760 +15603 26760 +44426 26758 +45363 26757 +41284 26753 +33558 26752 +7398 26751 +49382 26745 +39954 26733 +21672 26733 +41112 26728 +23515 26727 +34314 26725 +33916 26725 +25726 26724 +61 26723 +24055 26720 +38823 26720 +32712 26717 +37320 26717 +40429 26716 +24229 26714 +46721 26713 +11538 26713 +42894 26709 +9642 26708 +34436 26705 +36375 26703 +49571 26700 +48730 26700 +31160 26698 +41306 26698 +11027 26697 +45922 26694 +40024 26693 +39279 26692 +37004 26685 +15509 26674 +37694 26673 +30504 26673 +34256 26671 +34159 26671 +24855 26671 +31654 26671 +45253 26669 +43232 26669 +48511 26668 +38013 26668 +28152 26668 +36625 26667 +24026 26665 +46139 26665 +38753 26663 +14050 26662 +24798 26660 +50131 26659 +17367 26656 +40215 26653 +38635 26653 +43333 26648 +43606 26648 +46119 26646 +22166 26646 +34424 26638 +47157 26636 +46252 26631 +34801 26631 +35059 26630 +9990 26629 +45233 26628 +43396 26626 +42335 26625 +30992 26623 +46027 26617 +34835 26617 +41243 26616 +38364 26613 +48211 26612 +25113 26612 +43674 26609 +42143 26605 +7892 26603 +15297 26601 +22785 26600 +47510 26595 +10352 26592 +44480 26592 +23722 26588 +35916 26588 +43435 26587 +29081 26587 +34951 26586 +46266 26585 +35124 26584 +39764 26581 +34606 26581 +6680 26580 +29120 26580 +31855 26579 +33552 26579 +12230 26579 +32811 26574 +33988 26573 +29819 26570 +38750 26570 +44782 26567 +36217 26563 +34184 26561 +26756 26558 +36879 26557 +49950 26557 +45643 26557 +22842 26556 +40432 26555 +50206 26551 +31280 26549 +18849 26548 +23113 26543 +49672 26538 +36649 26538 +33812 26536 +38726 26535 +44672 26534 +43891 26534 +46813 26534 +34825 26533 +50212 26532 +45968 26530 +35273 26521 +43973 26521 +35243 26515 +35617 26515 +16705 26513 +43470 26508 +27927 26507 +49296 26507 +36565 26507 +43439 26506 +42934 26504 +20854 26502 +49263 26494 +33542 26492 +22854 26491 +41129 26488 +24209 26487 +34743 26487 +32097 26478 +45625 26469 +32180 26468 +46272 26468 +32073 26468 +19956 26464 +15161 26462 +47224 26461 +44118 26461 +16421 26461 +26885 26458 +18855 26457 +45481 26450 +46705 26449 +48012 26445 +35824 26444 +36328 26444 +22298 26444 +47590 26441 +35246 26441 +48626 26439 +49410 26431 +35871 26427 +35360 26420 +15391 26418 +39600 26415 +49981 26411 +44062 26411 +36039 26402 +35998 26402 +42434 26401 +39035 26398 +36141 26398 +49353 26395 +8775 26393 +36210 26387 +44017 26386 +35419 26385 +24129 26384 +42961 26384 +39978 26380 +44976 26379 +24669 26378 +5950 26373 +30787 26369 +24897 26367 +27155 26366 +35076 26361 +42922 26361 +44853 26359 +26007 26358 +32643 26358 +27258 26356 +45060 26351 +35989 26346 +49376 26343 +31714 26339 +28556 26339 +21327 26337 +40295 26337 +25940 26337 +26988 26337 +42132 26336 +41207 26336 +32128 26335 +133 26334 +27411 26333 +38887 26332 +44832 26329 +46557 26328 +41024 26327 +38568 26323 +49889 26319 +12401 26319 +34450 26317 +33538 26316 +35068 26314 +8818 26313 +26869 26313 +33569 26313 +24446 26312 +7751 26310 +46101 26309 +36905 26308 +48122 26307 +48242 26306 +32746 26305 +41084 26304 +25905 26302 +21781 26301 +49861 26301 +7826 26301 +47696 26295 +20591 26292 +49588 26291 +35804 26291 +45553 26290 +49159 26289 +38151 26288 +44092 26282 +45188 26281 +28915 26280 +15905 26279 +47442 26276 +37151 26274 +43607 26273 +28841 26273 +31026 26272 +44983 26271 +3238 26268 +13635 26268 +22005 26265 +17457 26264 +34339 26264 +23339 26264 +44680 26256 +38714 26255 +45812 26255 +46750 26253 +37580 26252 +41672 26251 +22508 26244 +30390 26238 +46275 26236 +47263 26235 +40715 26233 +12722 26231 +18251 26230 +26550 26227 +42352 26223 +26769 26223 +37513 26223 +26556 26222 +44314 26221 +36373 26219 +45418 26218 +42970 26214 +15151 26211 +13882 26205 +21931 26203 +49631 26202 +41885 26200 +29203 26197 +38784 26196 +31076 26192 +5820 26191 +15838 26190 +15457 26187 +45905 26186 +47436 26186 +47391 26185 +29108 26179 +34140 26178 +34229 26175 +36149 26174 +20415 26166 +31216 26165 +22968 26164 +25396 26162 +24327 26161 +42149 26157 +39340 26154 +39548 26153 +40080 26147 +37939 26146 +35404 26146 +1782 26145 +45499 26144 +33961 26141 +46782 26136 +43936 26135 +11755 26133 +8730 26129 +42679 26127 +46952 26126 +35264 26124 +39761 26118 +15594 26112 +16296 26110 +35126 26109 +40571 26109 +32285 26107 +22246 26107 +30657 26106 +48755 26105 +28399 26103 +40427 26098 +44986 26096 +11178 26096 +40670 26095 +40622 26093 +43330 26092 +49913 26092 +37193 26091 +48152 26089 +11713 26089 +42683 26087 +20375 26087 +31952 26086 +32494 26085 +1580 26085 +47589 26084 +48299 26082 +30410 26078 +20017 26074 +27111 26068 +22138 26067 +45014 26066 +33985 26066 +40016 26066 +40194 26063 +29386 26063 +24125 26062 +8497 26061 +37171 26061 +25328 26060 +42049 26059 +38333 26057 +40086 26057 +1038 26050 +45102 26050 +41021 26048 +43500 26047 +23398 26046 +48916 26044 +42465 26044 +15258 26044 +27814 26043 +49208 26039 +14824 26035 +32361 26032 +46698 26031 +28173 26031 +30162 26030 +48247 26029 +35679 26027 +23768 26023 +48517 26021 +39535 26020 +17969 26019 +33137 26014 +48151 26013 +35586 26012 +28573 26007 +31491 26007 +25055 25998 +40467 25997 +14140 25997 +38603 25994 +42713 25994 +28341 25992 +3637 25991 +19527 25991 +38210 25988 +41193 25988 +41212 25987 +27749 25986 +13940 25984 +46423 25982 +46938 25981 +40567 25980 +49192 25977 +28555 25977 +36460 25977 +37853 25977 +42067 25972 +49596 25963 +18844 25960 +21417 25959 +47964 25959 +39527 25959 +29703 25959 +49455 25959 +49272 25957 +29143 25954 +31471 25952 +19242 25951 +29675 25949 +48077 25949 +19652 25946 +39625 25945 +28155 25945 +45584 25944 +37639 25943 +32997 25940 +27646 25938 +19663 25934 +27898 25933 +9402 25932 +42914 25932 +35074 25930 +34179 25924 +25801 25923 +32056 25923 +49085 25919 +46232 25919 +41283 25916 +25324 25915 +39928 25913 +46069 25909 +35536 25908 +47300 25908 +22743 25907 +30960 25903 +17611 25903 +15431 25903 +27796 25893 +35667 25892 +49673 25891 +48356 25891 +36618 25888 +22224 25886 +32622 25882 +9515 25880 +17071 25878 +20366 25877 +26620 25875 +18540 25874 +40580 25873 +32093 25872 +9165 25870 +22122 25869 +20944 25863 +25132 25863 +44757 25851 +44186 25851 +42801 25850 +43395 25848 +46308 25848 +45471 25846 +41139 25845 +38248 25843 +24958 25841 +22801 25840 +40085 25839 +40335 25834 +38683 25834 +24670 25833 +15659 25831 +40492 25829 +44511 25827 +27446 25823 +9187 25820 +40398 25819 +36410 25809 +11158 25807 +49393 25805 +34626 25803 +1519 25803 +36392 25802 +15361 25800 +45785 25800 +50243 25799 +49761 25796 +34347 25792 +42415 25792 +17995 25791 +43426 25789 +27202 25787 +33933 25783 +38766 25779 +3890 25779 +34830 25779 +21907 25775 +25603 25774 +25816 25774 +23864 25773 +35904 25770 +44100 25765 +22588 25761 +25548 25754 +34752 25752 +49014 25750 +31334 25749 +11724 25739 +30129 25733 +44006 25733 +36597 25726 +39212 25726 +41498 25723 +34565 25722 +39506 25722 +15159 25722 +34938 25718 +29936 25718 +16204 25718 +9304 25716 +22891 25715 +15946 25714 +11120 25711 +26710 25709 +42810 25708 +39624 25707 +49311 25706 +35150 25705 +39901 25703 +27770 25703 +49385 25702 +45957 25702 +36966 25699 +46322 25691 +35987 25689 +21283 25689 +38652 25683 +41687 25680 +27190 25679 +34201 25677 +44938 25676 +39195 25672 +28283 25670 +31011 25669 +20871 25666 +24062 25666 +26733 25662 +34510 25658 +49609 25654 +41465 25653 +42224 25652 +38077 25652 +40615 25649 +30790 25648 +25883 25647 +38944 25647 +48341 25647 +40694 25646 +38393 25646 +40090 25646 +47284 25644 +29757 25641 +43888 25638 +34896 25638 +28702 25636 +38324 25633 +31222 25632 +39646 25619 +36766 25618 +45469 25614 +42716 25613 +43478 25611 +33156 25608 +47406 25605 +10853 25603 +44813 25597 +35412 25595 +26634 25590 +37196 25584 +20521 25578 +40932 25575 +49636 25572 +40358 25570 +37887 25568 +19781 25568 +37982 25565 +47388 25565 +43574 25560 +31406 25559 +44570 25553 +30200 25551 +30433 25549 +23104 25538 +43871 25537 +49310 25535 +7658 25534 +40336 25532 +41041 25532 +40509 25531 +39054 25527 +35042 25526 +32457 25526 +41617 25523 +48961 25519 +37809 25519 +14684 25518 +33540 25518 +43340 25516 +6566 25516 +47059 25514 +42969 25513 +33965 25513 +39272 25512 +45934 25510 +4012 25505 +41168 25502 +30389 25499 +32842 25498 +35342 25497 +49639 25497 +30841 25496 +29478 25496 +39038 25496 +37710 25490 +41799 25487 +42354 25485 +39376 25483 +50112 25482 +38409 25480 +20303 25479 +22616 25477 +27863 25477 +21999 25476 +41613 25474 +4087 25473 +38239 25473 +48071 25471 +48581 25468 +45048 25467 +42325 25466 +28626 25465 +44548 25463 +8194 25463 +50150 25463 +46439 25462 +37161 25461 +11703 25459 +46591 25458 +19715 25458 +35551 25457 +38311 25453 +47455 25452 +22975 25452 +42457 25450 +49175 25448 +30682 25447 +43555 25445 +34784 25433 +24075 25430 +13327 25429 +8803 25428 +46969 25426 +25864 25421 +12966 25417 +28374 25413 +31527 25411 +17060 25404 +29192 25403 +26659 25401 +18003 25400 +44730 25399 +42083 25395 +32102 25394 +35120 25392 +48962 25392 +42865 25391 +37330 25391 +42387 25389 +41083 25388 +38366 25388 +31298 25387 +46465 25384 +33437 25377 +27739 25376 +48243 25374 +47043 25373 +30999 25373 +49265 25369 +22509 25369 +43924 25365 +14221 25364 +47774 25362 +24597 25358 +31054 25358 +46365 25357 +37230 25355 +47872 25353 +44662 25353 +46586 25352 +36023 25351 +36244 25347 +47912 25345 +32912 25345 +42733 25339 +47902 25337 +45480 25335 +47075 25333 +32334 25330 +49441 25326 +14375 25325 +28971 25324 +33022 25318 +40829 25315 +3030 25314 +24316 25314 +24812 25314 +49723 25312 +36087 25306 +45042 25304 +38450 25303 +41014 25300 +37593 25299 +6321 25297 +15071 25296 +27266 25296 +11808 25295 +43737 25295 +41724 25293 +41639 25291 +31260 25290 +24289 25289 +49830 25289 +41593 25283 +39930 25282 +25197 25279 +18402 25279 +37287 25278 +34126 25274 +41087 25273 +32218 25273 +7904 25271 +37572 25270 +40472 25267 +43528 25265 +41131 25265 +13009 25264 +40362 25260 +30563 25260 +48579 25252 +42982 25247 +43928 25246 +34637 25245 +49025 25244 +38328 25243 +7470 25240 +37309 25240 +38229 25232 +28998 25232 +50087 25230 +44993 25229 +25886 25225 +46859 25223 +46603 25218 +38922 25218 +41028 25212 +44777 25211 +43694 25211 +29334 25210 +15014 25209 +20682 25209 +7321 25209 +35965 25206 +40923 25203 +11834 25202 +11591 25200 +23066 25199 +8981 25196 +22876 25195 +36681 25194 +26961 25194 +20049 25192 +46797 25192 +21302 25190 +45837 25188 +48280 25187 +39264 25179 +39710 25175 +31953 25174 +48550 25173 +35310 25171 +22083 25171 +36676 25171 +46172 25170 +43773 25170 +3959 25168 +42203 25167 +49069 25166 +33459 25166 +40169 25165 +39591 25161 +32173 25161 +30566 25156 +47869 25156 +21855 25153 +36509 25152 +47842 25150 +36793 25148 +45714 25147 +46469 25140 +23308 25139 +38541 25138 +38322 25135 +12522 25129 +44737 25124 +36094 25124 +36207 25123 +23466 25121 +36705 25118 +42426 25118 +39006 25115 +49391 25114 +22178 25114 +32000 25113 +28380 25112 +25364 25110 +21720 25108 +38611 25107 +46122 25105 +35305 25103 +43114 25097 +12992 25096 +37635 25096 +8143 25089 +45921 25083 +38915 25083 +23351 25083 +35155 25083 +38150 25079 +41786 25078 +30283 25072 +31037 25071 +43354 25069 +42156 25069 +11121 25063 +47855 25062 +47309 25060 +35907 25057 +30752 25056 +42076 25056 +10072 25055 +25100 25054 +24016 25053 +35267 25053 +48418 25052 +46951 25051 +13814 25050 +46487 25048 +12502 25048 +46040 25048 +25975 25047 +42421 25046 +38446 25045 +8327 25044 +26595 25043 +25045 25040 +24074 25040 +19731 25037 +48606 25031 +38163 25029 +31418 25028 +37145 25027 +36536 25027 +10394 25023 +20679 25023 +24580 25021 +25252 25017 +33206 25017 +32079 25017 +42765 25016 +34980 25016 +39056 25015 +8323 25014 +49897 25012 +44103 25009 +48315 25006 +48149 25005 +49564 25003 +38259 25000 +49805 24999 +47716 24998 +41638 24998 +46029 24994 +33573 24994 +32140 24991 +42925 24990 +33646 24989 +36108 24989 +19128 24988 +18089 24988 +38212 24986 +29991 24984 +14767 24984 +38997 24980 +40540 24979 +38712 24976 +49483 24975 +33580 24974 +49593 24974 +33123 24968 +24151 24966 +47142 24966 +41864 24966 +35709 24963 +47362 24962 +11683 24961 +33342 24959 +48981 24957 +38076 24955 +13095 24953 +33548 24952 +39677 24951 +19608 24949 +38329 24949 +37008 24949 +26222 24948 +47487 24945 +29647 24944 +18096 24940 +23549 24939 +28690 24939 +36080 24938 +17094 24937 +33958 24937 +27186 24932 +41290 24929 +45186 24928 +27106 24925 +46943 24924 +25774 24919 +48990 24918 +33525 24917 +45110 24914 +32436 24913 +21038 24911 +33869 24911 +46594 24910 +40314 24910 +38506 24908 +17593 24908 +14519 24908 +39497 24906 +27034 24906 +33827 24904 +15501 24904 +23129 24903 +20966 24903 +28191 24902 +45741 24901 +49629 24901 +47481 24901 +21912 24900 +41121 24899 +38171 24898 +18123 24893 +36898 24893 +44491 24892 +46592 24891 +27037 24891 +45245 24888 +44496 24887 +35993 24885 +32908 24885 +14374 24884 +43576 24878 +28693 24876 +49619 24875 +47291 24874 +34620 24870 +37098 24867 +43281 24866 +42294 24864 +16549 24861 +46734 24855 +49154 24854 +18318 24854 +45420 24854 +40488 24850 +25755 24846 +38859 24845 +46087 24832 +23567 24829 +39318 24829 +4363 24828 +40862 24826 +39595 24825 +49380 24816 +48921 24812 +38089 24809 +32158 24808 +31384 24804 +41890 24801 +45780 24801 +47007 24794 +22433 24793 +13046 24793 +38524 24792 +38054 24792 +10259 24789 +10149 24784 +39416 24783 +45320 24783 +17570 24777 +28281 24775 +18829 24774 +46291 24773 +34403 24771 +36366 24768 +20012 24767 +43250 24766 +14829 24765 +21370 24762 +31115 24759 +5470 24758 +36864 24755 +38483 24753 +36306 24751 +27314 24750 +33788 24749 +42662 24749 +48348 24748 +47843 24745 +15158 24743 +44689 24743 +10534 24734 +48766 24734 +26959 24732 +41285 24732 +20563 24732 +34559 24728 +47851 24722 +47221 24722 +43161 24720 +18186 24713 +44201 24711 +29438 24701 +2273 24701 +5790 24699 +30443 24696 +49440 24695 +35324 24690 +39468 24689 +49105 24689 +30021 24687 +44951 24685 +10818 24684 +15067 24682 +48651 24682 +42526 24680 +31407 24680 +28045 24679 +48135 24678 +38721 24677 +12276 24677 +47462 24676 +37833 24672 +35247 24666 +42505 24666 +43370 24663 +44841 24660 +35708 24660 +48025 24659 +42446 24656 +24508 24656 +46529 24655 +11235 24655 +39064 24649 +37229 24648 +44089 24648 +15480 24643 +36627 24642 +45731 24641 +24148 24641 +24044 24636 +30671 24634 +38657 24632 +45141 24630 +45629 24628 +39744 24626 +34817 24623 +43603 24623 +41180 24621 +45423 24620 +48430 24619 +30398 24619 +47243 24617 +27808 24616 +19151 24614 +42108 24610 +42137 24604 +43963 24603 +6274 24601 +46273 24600 +33215 24599 +29774 24597 +44088 24597 +30964 24597 +45002 24595 +34625 24595 +29977 24591 +36480 24589 +47390 24588 +35699 24585 +39837 24583 +24467 24583 +47780 24581 +38084 24577 +43486 24576 +14311 24574 +47337 24573 +47352 24572 +29969 24571 +41544 24567 +50147 24554 +37102 24551 +27093 24549 +32321 24549 +37570 24548 +41755 24548 +20719 24546 +35740 24545 +4085 24542 +49599 24541 +20639 24537 +24204 24535 +9205 24529 +34490 24525 +11928 24520 +32226 24517 +42707 24513 +33669 24512 +45578 24509 +27569 24507 +40383 24506 +29810 24506 +42315 24505 +31150 24503 +46042 24503 +37251 24500 +46674 24494 +43658 24494 +41061 24491 +35649 24490 +43151 24490 +24432 24488 +34370 24487 +40791 24486 +39686 24485 +23802 24482 +11964 24482 +29603 24481 +36144 24481 +28770 24478 +39009 24474 +32627 24472 +44514 24469 +47153 24469 +24629 24468 +33088 24468 +33192 24466 +44526 24463 +19879 24462 +29185 24461 +49246 24457 +32062 24452 +30539 24448 +36657 24447 +39158 24445 +40286 24444 +47833 24444 +40812 24444 +33938 24442 +43166 24441 +12626 24441 +48024 24439 +29408 24438 +28151 24430 +28124 24427 +49848 24426 +47208 24421 +7654 24421 +48957 24419 +46282 24419 +33317 24416 +7499 24416 +9291 24410 +37614 24408 +41472 24408 +26714 24408 +1968 24407 +29839 24407 +8921 24406 +46462 24405 +49932 24404 +26797 24403 +33227 24402 +29346 24402 +37466 24402 +27750 24397 +36777 24390 +38172 24388 +36551 24385 +27825 24382 +46580 24379 +45874 24379 +40464 24377 +2602 24376 +38675 24373 +32427 24373 +45185 24371 +30214 24369 +16935 24368 +21321 24365 +49447 24363 +43122 24359 +44022 24359 +49975 24358 +22155 24357 +44958 24353 +36629 24352 +16877 24351 +28390 24350 +45956 24348 +38875 24348 +44631 24340 +43935 24339 +26151 24338 +37542 24335 +41258 24333 +29501 24333 +49919 24328 +36397 24324 +10186 24323 +42744 24318 +22812 24314 +19677 24313 +24631 24313 +19466 24312 +48474 24311 +1284 24310 +31882 24310 +37076 24309 +44966 24306 +29400 24302 +49968 24301 +37068 24301 +38974 24299 +44695 24298 +28343 24294 +35615 24287 +164 24283 +27380 24283 +20507 24282 +49956 24281 +46098 24279 +45607 24279 +34310 24278 +34287 24277 +45343 24277 +34535 24276 +32277 24273 +49895 24271 +43224 24268 +30643 24267 +39682 24266 +40206 24263 +6525 24261 +29506 24260 +40787 24258 +40305 24256 +44979 24255 +38232 24253 +45000 24250 +21597 24249 +30152 24242 +46899 24242 +48244 24241 +30028 24240 +33971 24240 +35970 24239 +48225 24239 +16733 24235 +42651 24235 +32217 24235 +44789 24230 +23640 24229 +29173 24228 +36680 24228 +28722 24218 +18900 24215 +38746 24215 +45478 24215 +27784 24214 +40609 24213 +44681 24205 +43677 24201 +36979 24194 +37992 24194 +39100 24193 +29616 24193 +5880 24189 +44065 24187 +42364 24186 +35403 24184 +27514 24183 +41959 24181 +43138 24177 +6463 24174 +27871 24174 +24899 24174 +45976 24172 +43885 24167 +46074 24167 +47709 24165 +36393 24160 +37169 24156 +31964 24155 +35846 24155 +32562 24153 +21436 24148 +41515 24147 +3593 24147 +31304 24146 +36749 24140 +22845 24138 +33698 24136 +26586 24134 +38909 24134 +49191 24132 +37002 24132 +9531 24128 +45969 24121 +34407 24119 +30546 24118 +47378 24117 +46164 24115 +41132 24109 +45355 24108 +39721 24108 +23779 24104 +42227 24104 +36247 24102 +33220 24101 +39380 24100 +43957 24098 +50127 24097 +19946 24096 +16804 24096 +24809 24096 +8053 24096 +30578 24095 +21179 24095 +37528 24095 +48412 24091 +45016 24089 +22702 24087 +15580 24086 +45477 24086 +25666 24084 +38595 24083 +47840 24080 +48996 24075 +41797 24073 +12757 24070 +48083 24066 +23279 24057 +36760 24056 +4616 24056 +38873 24054 +33981 24050 +40076 24049 +46628 24047 +44822 24047 +46641 24043 +35602 24042 +20980 24041 +42363 24038 +38380 24037 +5561 24037 +39706 24035 +17839 24031 +48787 24029 +5918 24029 +23317 24027 +23495 24027 +1289 24026 +37469 24026 +34583 24023 +23449 24020 +47753 24016 +36570 24013 +36323 24012 +43329 24012 +33102 24009 +44530 24009 +41526 24007 +46323 24007 +22293 24006 +30484 24002 +22214 24002 +34467 24000 +32139 23999 +36434 23999 +31035 23998 +46794 23995 +25492 23994 +22764 23993 +46436 23988 +48036 23982 +27550 23977 +43220 23970 +29457 23965 +12337 23961 +35638 23960 +18068 23958 +29601 23957 +26750 23957 +41367 23955 +37396 23955 +35010 23955 +40739 23954 +48793 23954 +47923 23954 +19400 23946 +23582 23946 +40569 23945 +47953 23945 +37282 23945 +42937 23943 +36298 23942 +49048 23942 +33832 23940 +11604 23939 +45460 23938 +24667 23933 +41656 23931 +45660 23929 +49745 23929 +48196 23928 +49163 23928 +28605 23923 +25189 23919 +21924 23918 +18124 23917 +38339 23917 +43836 23913 +28029 23913 +44093 23913 +30377 23912 +19913 23911 +14527 23905 +13581 23903 +49741 23903 +48643 23902 +38431 23902 +43495 23899 +26679 23897 +49854 23893 +49941 23893 +38989 23892 +47906 23888 +42570 23886 +37682 23885 +25265 23883 +33302 23880 +48055 23878 +5244 23876 +31072 23872 +38976 23871 +42319 23870 +4118 23869 +16795 23867 +47740 23867 +18736 23864 +49904 23862 +21727 23862 +26468 23855 +38547 23854 +47673 23853 +34822 23852 +163 23851 +45176 23849 +39882 23848 +44664 23848 +26255 23847 +18305 23840 +37730 23836 +38525 23832 +48710 23831 +31162 23827 +37998 23821 +31835 23819 +48255 23817 +47944 23817 +26141 23811 +42648 23811 +37431 23810 +35286 23807 +33872 23807 +40125 23807 +45564 23806 +33921 23803 +26650 23802 +46601 23801 +40103 23801 +33665 23800 +43759 23799 +16469 23797 +36907 23796 +41029 23793 +29458 23791 +38107 23791 +21659 23791 +21238 23786 +47873 23785 +40758 23777 +46138 23775 +45326 23775 +31497 23774 +44206 23773 +46396 23771 +49378 23769 +39472 23766 +12124 23764 +8349 23761 +33079 23761 +50117 23759 +26713 23756 +15189 23755 +43788 23752 +41174 23751 +22177 23749 +35978 23748 +29422 23745 +39215 23745 +25962 23744 +29258 23744 +49566 23743 +13866 23740 +23450 23738 +45543 23731 +42305 23730 +44865 23729 +16735 23729 +41564 23728 +38737 23727 +45679 23726 +43469 23721 +42584 23719 +33610 23717 +26404 23712 +40437 23711 +22820 23711 +36462 23710 +48459 23709 +32953 23709 +15576 23705 +16140 23704 +39256 23701 +49225 23699 +33819 23698 +49716 23695 +14163 23694 +5546 23694 +40600 23693 +46324 23686 +29922 23685 +50229 23684 +42795 23682 +38147 23681 +49992 23678 +40887 23673 +4972 23666 +7568 23664 +39058 23663 +41882 23662 +49077 23661 +35092 23661 +31466 23660 +38871 23658 +32583 23658 +23899 23658 +20649 23656 +32672 23655 +49695 23653 +45727 23652 +22090 23650 +43743 23649 +39305 23649 +32703 23646 +40022 23644 +39549 23643 +48675 23643 +39609 23642 +33969 23640 +43149 23638 +39623 23638 +44263 23637 +10292 23633 +14117 23632 +44002 23629 +34085 23624 +44290 23622 +34728 23620 +22802 23616 +27603 23615 +43538 23614 +42206 23610 +42051 23610 +46731 23606 +42537 23605 +45289 23604 +20207 23604 +40198 23599 +39139 23595 +43831 23593 +48112 23591 +40732 23590 +44027 23590 +12448 23589 +38539 23589 +24113 23587 +11097 23583 +50082 23581 +26175 23580 +26295 23579 +36330 23575 +23925 23573 +47132 23570 +47674 23570 +48898 23570 +42155 23570 +3882 23569 +43147 23569 +35066 23568 +49148 23568 +36052 23567 +40660 23567 +10956 23566 +48862 23564 +28688 23564 +37923 23564 +49124 23563 +43711 23562 +16554 23561 +36357 23557 +20530 23551 +47687 23551 +35716 23545 +43964 23545 +39773 23538 +42489 23534 +35910 23533 +42582 23532 +46983 23531 +43980 23530 +34645 23529 +38249 23529 +40354 23524 +32904 23520 +37132 23519 +41081 23517 +20470 23515 +37622 23514 +37071 23510 +44471 23509 +26417 23508 +19080 23508 +47398 23507 +47836 23505 +40321 23495 +48275 23493 +38440 23489 +9186 23489 +44915 23489 +44682 23483 +37225 23476 +33042 23474 +31191 23474 +28081 23472 +41859 23466 +30746 23465 +12866 23465 +32535 23465 +13128 23464 +30106 23462 +19881 23458 +39003 23457 +34131 23448 +27500 23447 +38706 23447 +16595 23444 +35848 23442 +32663 23439 +13283 23437 +43984 23436 +43951 23434 +12610 23433 +48903 23430 +41983 23428 +16837 23423 +20896 23423 +20926 23420 +6830 23419 +38275 23415 +40882 23413 +28435 23413 +30764 23413 +34376 23412 +44551 23412 +39119 23412 +36024 23411 +43011 23410 +43218 23400 +17821 23400 +12814 23397 +43918 23394 +9184 23392 +39252 23389 +38224 23385 +30193 23384 +8038 23383 +46789 23377 +50035 23376 +35085 23374 +39382 23373 +46099 23372 +32477 23365 +15022 23362 +18923 23361 +31823 23360 +38159 23360 +45158 23356 +41634 23356 +28708 23353 +34717 23351 +36891 23350 +24102 23350 +49951 23349 +35366 23344 +43482 23342 +22938 23341 +3928 23337 +42590 23337 +24518 23337 +36914 23334 +10567 23333 +41412 23333 +37308 23332 +11360 23331 +17320 23331 +42608 23330 +43334 23326 +23986 23320 +12975 23320 +49623 23319 +31847 23318 +43234 23314 +39108 23313 +42272 23313 +15540 23312 +26087 23306 +34917 23306 +40893 23301 +41303 23297 +38205 23293 +37364 23287 +30488 23287 +49392 23286 +5058 23282 +38473 23280 +2572 23280 +40455 23280 +24383 23279 +19056 23273 +17015 23272 +37474 23272 +28812 23271 +37249 23269 +45428 23269 +32320 23268 +26789 23265 +8247 23264 +45514 23263 +41447 23262 +46443 23260 +44465 23260 +11953 23259 +47230 23255 +41632 23255 +28260 23253 +41619 23252 +47829 23249 +48389 23249 +37703 23247 +47446 23246 +45244 23246 +47234 23245 +37020 23243 +29629 23243 +26121 23238 +12207 23230 +42297 23228 +26499 23226 +24206 23224 +46844 23224 +36488 23222 +19492 23222 +40740 23220 +45806 23219 +47669 23219 +39997 23213 +29022 23211 +42762 23210 +17116 23206 +11929 23204 +49433 23201 +35697 23199 +18960 23198 +47452 23196 +37021 23193 +42661 23193 +26288 23193 +37515 23185 +38733 23185 +48827 23185 +37825 23182 +10208 23181 +17802 23178 +24393 23178 +48943 23174 +38346 23164 +35990 23163 +45095 23153 +6328 23153 +45317 23152 +34420 23152 +37603 23149 +42936 23147 +38569 23145 +44816 23143 +38575 23142 +42611 23141 +28989 23141 +5130 23141 +13382 23136 +45822 23135 +38402 23134 +45799 23133 +21705 23133 +27595 23130 +27959 23128 +40047 23121 +49317 23118 +47626 23117 +36931 23116 +43921 23115 +33241 23115 +41677 23115 +39409 23114 +42013 23114 +47938 23113 +37153 23113 +46416 23112 +42989 23110 +27280 23109 +38741 23106 +33759 23100 +37823 23100 +39328 23098 +11051 23096 +24637 23091 +36809 23090 +36913 23089 +34578 23089 +46318 23087 +39352 23084 +29688 23080 +47134 23076 +22037 23074 +22914 23073 +26523 23072 +43096 23072 +41775 23072 +26955 23065 +28391 23064 +36859 23064 +40821 23063 +46755 23062 +46565 23060 +43407 23059 +27850 23053 +48520 23051 +162 23050 +32863 23047 +30273 23046 +32126 23044 +5418 23043 +19005 23042 +38022 23041 +41167 23040 +48197 23034 +49846 23033 +30677 23029 +47049 23029 +30816 23029 +36996 23028 +31234 23025 +47910 23022 +41322 23022 +43187 23022 +43132 23018 +35394 23017 +45207 23014 +25456 23013 +13781 23011 +36974 23008 +28675 23008 +27740 23008 +6815 23007 +49473 23006 +22628 23005 +36505 23002 +35549 22999 +48154 22998 +30085 22997 +34776 22996 +47803 22995 +38313 22985 +35467 22983 +47627 22977 +24095 22973 +45657 22971 +40965 22969 +19873 22965 +46408 22962 +37869 22960 +48934 22957 +23433 22957 +45604 22956 +36951 22949 +27413 22948 +29693 22948 +46338 22945 +43366 22944 +23441 22935 +47057 22933 +45631 22931 +39667 22930 +38254 22929 +38472 22927 +35623 22926 +32962 22925 +48577 22922 +26699 22919 +24547 22919 +45191 22917 +39466 22916 +48461 22916 +47047 22916 +45621 22913 +38533 22910 +25066 22910 +47104 22909 +42279 22907 +34427 22906 +29978 22903 +27226 22902 +34251 22894 +11456 22893 +43072 22887 +32725 22882 +35962 22879 +39320 22879 +9282 22875 +8186 22873 +49867 22871 +45534 22868 +35391 22868 +37895 22866 +3209 22864 +46493 22861 +39448 22861 +18745 22858 +41785 22856 +25528 22854 +31705 22853 +33616 22852 +39518 22852 +43156 22849 +43705 22847 +32488 22847 +42449 22841 +38959 22839 +27294 22838 +31218 22837 +47045 22835 +34765 22825 +48542 22824 +43539 22822 +22941 22821 +47112 22819 +40881 22816 +44457 22816 +31452 22815 +31516 22812 +33962 22812 +49820 22811 +34343 22810 +41923 22806 +38831 22806 +36678 22805 +40328 22805 +34416 22805 +45001 22805 +10563 22804 +34574 22804 +39799 22803 +41338 22803 +39545 22802 +37060 22801 +40681 22801 +39495 22801 +25262 22800 +15620 22799 +45974 22799 +22398 22796 +49721 22795 +21464 22789 +46671 22789 +40141 22788 +24717 22781 +48913 22781 +38666 22780 +41649 22776 +48883 22773 +12334 22773 +36729 22773 +33887 22772 +48426 22767 +33697 22767 +48811 22767 +28407 22765 +25257 22764 +44455 22762 +35876 22761 +25184 22761 +46831 22759 +47194 22755 +23569 22753 +40408 22751 +49914 22746 +37737 22740 +29152 22739 +37618 22739 +33808 22737 +30296 22735 +20021 22735 +29910 22733 +30610 22733 +36734 22733 +31751 22731 +45331 22731 +4779 22722 +37711 22720 +41569 22719 +42872 22716 +32082 22709 +41589 22708 +50007 22705 +41093 22702 +37864 22701 +32322 22697 +48817 22697 +6346 22692 +46735 22690 +36723 22684 +41612 22684 +50193 22684 +33104 22684 +15358 22683 +37831 22679 +39563 22679 +46642 22678 +35171 22677 +44361 22670 +33791 22666 +31024 22665 +49326 22665 +39610 22664 +50078 22661 +38967 22653 +40554 22652 +30631 22644 +35369 22640 +48868 22633 +45650 22633 +35885 22632 +39101 22631 +47883 22630 +39709 22629 +34153 22627 +46186 22625 +20120 22623 +30731 22622 +50208 22621 +43485 22621 +41491 22621 +44582 22620 +27427 22617 +37857 22615 +34439 22613 +32060 22610 +40819 22606 +12339 22601 +36699 22599 +33700 22599 +46376 22599 +43570 22598 +37576 22594 +8586 22594 +36785 22577 +2981 22577 +24061 22576 +37893 22571 +42725 22571 +27543 22568 +50105 22568 +49453 22567 +42794 22566 +42050 22561 +19697 22561 +46241 22560 +39985 22558 +11716 22558 +47449 22553 +46199 22552 +46094 22552 +43565 22552 +43337 22551 +22806 22551 +42081 22550 +35291 22549 +40231 22547 +37972 22545 +16344 22543 +24675 22541 +40652 22541 +39512 22538 +45068 22535 +47102 22534 +40149 22534 +27964 22534 +35562 22533 +49177 22527 +41190 22526 +32263 22526 +46446 22525 +30342 22524 +50023 22522 +48345 22522 +49079 22522 +35700 22518 +48559 22518 +33924 22517 +39925 22517 +34185 22516 +37856 22516 +47372 22515 +48324 22514 +48513 22512 +17077 22512 +13654 22511 +47314 22510 +47480 22509 +48872 22507 +31242 22507 +46350 22506 +36260 22502 +36088 22501 +40861 22499 +36561 22496 +41453 22496 +48950 22492 +36118 22485 +33246 22484 +36048 22483 +48683 22483 +35475 22481 +21263 22479 +43101 22477 +21760 22474 +23286 22473 +24244 22471 +39093 22470 +45070 22469 +35544 22467 +25261 22466 +46309 22464 +38422 22462 +16541 22457 +39779 22457 +21595 22455 +45019 22449 +34961 22447 +46001 22445 +49920 22443 +47302 22441 +42069 22434 +37028 22434 +15428 22434 +47356 22433 +18946 22432 +15390 22429 +2976 22426 +50046 22426 +44784 22425 +47684 22423 +48079 22421 +41172 22419 +46141 22419 +8885 22417 +16023 22416 +59 22415 +48419 22413 +48792 22412 +42122 22410 +46682 22409 +16515 22408 +21480 22407 +18332 22405 +9542 22404 +28764 22404 +46389 22395 +42516 22395 +36368 22395 +4413 22392 +5712 22390 +33932 22388 +14673 22384 +46082 22384 +44550 22383 +42398 22380 +22443 22379 +30374 22378 +44641 22377 +38298 22373 +39174 22371 +23607 22368 +35739 22367 +40956 22365 +22539 22364 +30908 22356 +47599 22355 +48370 22354 +42846 22354 +42043 22351 +27489 22348 +32146 22348 +47355 22348 +42163 22347 +48359 22347 +43355 22346 +18588 22345 +26625 22344 +47159 22340 +25954 22339 +32617 22339 +42666 22338 +7143 22338 +13900 22334 +46804 22334 +50021 22333 +34346 22331 +23637 22330 +47067 22330 +30117 22327 +16367 22326 +31144 22321 +40619 22320 +49624 22318 +7917 22316 +32676 22315 +42330 22313 +20498 22308 +36801 22305 +21582 22305 +29680 22300 +44074 22300 +44228 22300 +30206 22297 +16297 22296 +30077 22296 +21706 22292 +40749 22291 +20686 22288 +2914 22286 +41425 22284 +35012 22282 +20737 22281 +41734 22279 +5138 22278 +28860 22265 +44382 22261 +44665 22261 +31992 22259 +50197 22259 +40985 22258 +22104 22256 +47679 22255 +15095 22253 +42061 22252 +26644 22249 +45029 22249 +19797 22247 +49799 22240 +47475 22240 +49801 22239 +39299 22233 +34812 22232 +32867 22230 +38986 22225 +30243 22224 +46836 22224 +42487 22220 +32538 22219 +42111 22218 +47459 22213 +39018 22210 +37334 22209 +10525 22208 +22438 22203 +22006 22202 +30314 22200 +20041 22199 +47324 22198 +48427 22198 +29162 22192 +34907 22189 +3045 22187 +20099 22186 +38371 22183 +40190 22177 +29783 22177 +42695 22174 +41144 22173 +39483 22172 +24358 22171 +47354 22169 +42399 22166 +49647 22160 +32005 22156 +49456 22154 +34053 22153 +33076 22150 +40438 22143 +31206 22140 +14957 22140 +22610 22138 +38549 22134 +48005 22133 +36203 22132 +47557 22131 +27816 22130 +34463 22128 +32751 22127 +49093 22127 +45279 22124 +38374 22123 +33602 22122 +42714 22121 +40499 22119 +50012 22114 +21914 22114 +39949 22114 +47363 22113 +39743 22113 +40163 22112 +40333 22107 +37272 22106 +40762 22104 +34330 22102 +33310 22101 +38356 22100 +45984 22100 +27483 22097 +12756 22095 +35282 22093 +30277 22092 +18238 22092 +13185 22090 +34158 22090 +29581 22090 +1609 22088 +30924 22087 +18793 22086 +49754 22085 +28645 22083 +22899 22080 +37615 22077 +44230 22075 +37636 22073 +31754 22072 +9063 22070 +24867 22067 +22939 22061 +29865 22061 +30846 22057 +46255 22057 +48624 22053 +19011 22052 +48284 22051 +48480 22046 +36530 22044 +48248 22043 +41165 22040 +30694 22039 +42366 22037 +45382 22034 +22404 22032 +11690 22031 +45787 22030 +27021 22021 +45737 22017 +32516 22017 +24326 22016 +42144 22012 +24132 22007 +25174 22007 +35642 22004 +29059 22003 +42947 22003 +13518 22003 +39243 22002 +25392 22000 +49172 21998 +34950 21998 +44106 21997 +42107 21997 +45757 21997 +30863 21997 +36956 21995 +41289 21995 +26178 21994 +47850 21991 +12715 21991 +35973 21987 +15879 21985 +37794 21984 +31800 21980 +3040 21979 +41211 21979 +25545 21979 +45173 21979 +25835 21978 +28911 21977 +24449 21977 +48201 21972 +45314 21970 +6781 21969 +32165 21968 +38903 21961 +48332 21959 +37567 21958 +33794 21957 +17377 21954 +37414 21954 +49990 21950 +45792 21945 +34096 21942 +48632 21941 +29266 21939 +47241 21936 +37735 21934 +31982 21933 +49306 21933 +16557 21932 +37338 21930 +39544 21928 +49329 21926 +19154 21925 +2510 21924 +31891 21922 +32648 21922 +29249 21921 +45971 21918 +49508 21917 +39653 21917 +49514 21916 +21401 21916 +29054 21916 +9556 21915 +47929 21912 +48721 21901 +48512 21899 +19117 21897 +30464 21895 +45572 21892 +35071 21891 +37776 21888 +35735 21887 +41075 21887 +43704 21887 +45012 21886 +40841 21881 +34530 21877 +16138 21875 +43231 21871 +39122 21871 +36620 21871 +38310 21870 +38629 21862 +42380 21860 +36249 21856 +43434 21852 +38352 21852 +41304 21850 +29299 21850 +42712 21849 +49231 21849 +6925 21848 +44569 21847 +17833 21843 +35681 21843 +45213 21842 +17417 21841 +49586 21840 +32430 21836 +47124 21836 +21619 21833 +49706 21833 +47814 21833 +34959 21832 +42534 21829 +46431 21827 +45259 21821 +37747 21820 +26846 21819 +39553 21816 +39004 21816 +33374 21814 +48004 21811 +34546 21811 +2765 21810 +28718 21809 +16353 21808 +29792 21808 +27279 21807 +27379 21802 +43315 21801 +47128 21801 +21280 21800 +5604 21798 +34828 21797 +23206 21795 +40876 21790 +16501 21788 +47909 21786 +38092 21786 +43184 21785 +21243 21785 +35133 21784 +18166 21781 +38703 21780 +49986 21780 +45654 21779 +26302 21777 +47885 21777 +37434 21773 +39918 21773 +47023 21772 +41308 21771 +10878 21768 +44539 21768 +35678 21765 +27629 21765 +34137 21764 +41423 21761 +23203 21760 +39558 21759 +27672 21758 +14933 21757 +48008 21755 +40885 21752 +36314 21752 +39586 21751 +40081 21751 +41580 21748 +49888 21748 +37613 21745 +33398 21744 +7399 21743 +26776 21740 +48949 21739 +27836 21736 +33345 21736 +27398 21735 +35815 21733 +24198 21733 +42905 21733 +17069 21731 +8917 21731 +42704 21731 +38995 21729 +37094 21729 +43134 21729 +44815 21728 +50064 21727 +38456 21726 +44860 21725 +36299 21723 +10198 21722 +41789 21722 +39689 21722 +19895 21722 +18790 21719 +34302 21717 +40275 21717 +36201 21716 +41813 21715 +48070 21714 +4994 21713 +30732 21711 +17746 21706 +43862 21704 +36496 21701 +44903 21700 +29700 21698 +46472 21697 +31383 21694 +35664 21692 +44461 21688 +41287 21687 +35809 21686 +44430 21685 +37777 21682 +6753 21678 +32869 21678 +12909 21678 +28037 21676 +41291 21674 +24047 21673 +36271 21671 +46492 21670 +17148 21669 +48783 21667 +33608 21663 +40234 21662 +45661 21659 +35721 21654 +44593 21653 +40184 21651 +47076 21651 +37804 21651 +27291 21650 +49714 21649 +49010 21643 +36643 21638 +28694 21638 +25155 21637 +41548 21636 +1707 21636 +45409 21634 +24156 21630 +49983 21621 +44251 21621 +35890 21614 +45722 21614 +41660 21613 +24225 21612 +40877 21608 +43509 21605 +47411 21604 +27904 21604 +10139 21598 +19539 21595 +45090 21592 +42780 21589 +41275 21589 +49386 21586 +48091 21583 +16153 21583 +38466 21581 +39771 21580 +32567 21577 +41822 21577 +29052 21576 +26008 21575 +34820 21574 +13036 21573 +46962 21571 +47326 21570 +46960 21563 +40618 21562 +32970 21562 +43536 21559 +16907 21557 +48311 21550 +38783 21550 +39526 21548 +46526 21547 +45551 21546 +49719 21544 +35472 21541 +16005 21540 +47336 21537 +25118 21536 +26330 21536 +46197 21533 +13047 21533 +42529 21532 +26738 21532 +22742 21529 +43666 21523 +20523 21522 +26305 21520 +40823 21513 +32439 21511 +27537 21511 +44149 21509 +39117 21504 +39942 21500 +40405 21498 +43289 21497 +41477 21497 +27891 21497 +41542 21496 +39880 21494 +28957 21493 +46663 21491 +40385 21489 +13874 21486 +30521 21485 +22667 21480 +4442 21479 +40058 21475 +46290 21475 +46762 21473 +18835 21472 +17354 21469 +46724 21465 +46606 21464 +44439 21464 +41577 21462 +48924 21461 +10954 21455 +7041 21454 +37505 21449 +40495 21448 +46661 21444 +47205 21442 +24318 21442 +49555 21442 +32058 21436 +41810 21432 +45649 21431 +33987 21430 +27743 21423 +29840 21423 +37055 21421 +14863 21420 +46298 21418 +39530 21417 +40276 21416 +44857 21415 +7494 21413 +26092 21413 +39801 21406 +33979 21401 +36664 21396 +42194 21395 +43624 21391 +42309 21390 +28073 21388 +40860 21388 +49747 21382 +48649 21381 +38856 21381 +38605 21377 +43780 21376 +29796 21374 +32824 21374 +2281 21373 +43860 21373 +49688 21371 +28402 21368 +46044 21362 +46885 21356 +40128 21355 +16820 21355 +20847 21354 +48176 21352 +26922 21351 +48250 21351 +45025 21350 +37483 21348 +49809 21346 +29733 21345 +45521 21338 +42921 21335 +42055 21334 +18635 21332 +21389 21329 +44994 21328 +20378 21327 +10138 21324 +49184 21322 +46916 21322 +48425 21321 +36376 21319 +40671 21317 +41557 21315 +16692 21312 +48661 21310 +42634 21306 +28041 21301 +46509 21301 +30986 21300 +49276 21299 +44048 21299 +33462 21296 +26261 21293 +49640 21293 +49102 21292 +6618 21288 +45193 21287 +40954 21287 +44137 21286 +43125 21285 +47101 21282 +5771 21281 +33484 21277 +14889 21276 +38576 21275 +41101 21271 +42533 21269 +46293 21266 +31938 21266 +37475 21266 +49692 21264 +18038 21260 +48200 21260 +29067 21259 +40785 21258 +44987 21258 +40889 21256 +39357 21251 +43523 21248 +45993 21247 +19006 21245 +30931 21241 +1116 21237 +30606 21237 +43803 21227 +45505 21225 +13967 21223 +43168 21223 +46254 21223 +36208 21222 +12065 21221 +28677 21221 +49478 21220 +46585 21219 +41292 21216 +39578 21216 +39344 21216 +43293 21212 +42280 21207 +21301 21206 +33621 21206 +41666 21201 +30662 21201 +35875 21201 +31199 21201 +30650 21201 +31120 21200 +17483 21196 +32243 21191 +35418 21188 +43403 21182 +39201 21176 +42960 21173 +22805 21171 +39736 21171 +37304 21171 +32200 21168 +42472 21168 +44281 21168 +13903 21167 +39312 21167 +40051 21166 +4184 21164 +10683 21163 +39251 21157 +32371 21154 +39166 21154 +33883 21153 +32968 21152 +49973 21151 +21101 21148 +41066 21146 +26155 21146 +23709 21141 +41590 21140 +39111 21138 +31495 21137 +41460 21137 +48987 21127 +13821 21127 +38073 21123 +46005 21123 +23155 21122 +47988 21116 +2868 21115 +38906 21115 +48564 21112 +40245 21112 +40173 21103 +31880 21103 +44162 21100 +27403 21100 +134 21099 +34865 21094 +11156 21093 +36981 21089 +43016 21089 +21871 21086 +27559 21086 +45682 21085 +31350 21083 +43874 21082 +39023 21078 +46847 21076 +38505 21074 +46067 21071 +48182 21066 +21687 21066 +28730 21057 +50189 21056 +42875 21054 +34335 21052 +11229 21050 +10922 21050 +37348 21049 +36927 21046 +42726 21046 +46550 21045 +26588 21044 +13065 21040 +13090 21037 +22834 21036 +13082 21035 +42268 21030 +45630 21029 +8451 21029 +47792 21024 +28757 21022 +34312 21022 +49461 21022 +49233 21019 +38407 21018 +16742 21017 +39186 21016 +35140 21014 +20949 21013 +45458 21008 +49979 21004 +37323 21001 +43626 21001 +30420 21001 +32915 20999 +43253 20992 +45136 20991 +40700 20990 +1889 20989 +26123 20988 +33553 20987 +38546 20983 +46271 20981 +5853 20980 +42015 20974 +25283 20974 +39216 20973 +30565 20972 +43820 20971 +44957 20971 +35145 20971 +37472 20971 +43610 20970 +36445 20969 +49381 20968 +47283 20968 +14975 20966 +34178 20966 +32704 20965 +35720 20963 +36902 20961 +34647 20959 +35030 20959 +46612 20958 +5877 20957 +35378 20956 +32975 20954 +35755 20947 +14238 20947 +41166 20946 +46378 20944 +27556 20942 +38846 20941 +18521 20938 +35388 20935 +40116 20935 +36549 20933 +48399 20928 +37531 20927 +39449 20923 +47303 20919 +12999 20917 +25235 20916 +27701 20916 +42005 20914 +46146 20913 +33483 20912 +37117 20907 +39327 20907 +30234 20907 +43633 20902 +18122 20898 +45493 20895 +6933 20893 +7845 20884 +33723 20877 +24418 20874 +39904 20872 +48736 20870 +8660 20868 +26307 20867 +49122 20866 +33449 20863 +30270 20862 +30893 20861 +48726 20861 +45072 20860 +38636 20860 +49282 20859 +42010 20859 +49418 20858 +34359 20856 +37716 20855 +24543 20853 +33284 20852 +40157 20851 +17156 20851 +48809 20847 +49697 20843 +40957 20842 +34742 20840 +42747 20839 +41858 20836 +37769 20835 +22058 20835 +43363 20833 +48680 20831 +5927 20830 +13228 20826 +33856 20826 +37780 20825 +48230 20824 +40375 20819 +48212 20819 +3712 20814 +45719 20809 +26267 20809 +23072 20809 +41043 20808 +33647 20804 +49940 20803 +47287 20798 +3138 20795 +27129 20794 +32599 20788 +22012 20787 +30018 20780 +49024 20775 +39118 20775 +30317 20775 +36337 20773 +28004 20773 +47551 20773 +47904 20770 +43690 20770 +39513 20765 +18448 20763 +9790 20762 +28022 20761 +45325 20759 +33056 20758 +31409 20757 +39940 20757 +25983 20755 +40979 20754 +45485 20753 +45704 20752 +36059 20752 +4552 20746 +34288 20745 +40285 20744 +29878 20744 +35156 20742 +37902 20741 +48263 20741 +29593 20740 +43510 20740 +41195 20739 +49542 20738 +40523 20738 +43221 20737 +2400 20736 +39895 20734 +40182 20732 +49625 20732 +8778 20731 +50032 20729 +32276 20725 +40564 20723 +24441 20722 +28352 20717 +6261 20716 +33797 20715 +34618 20715 +49707 20715 +45929 20713 +4878 20713 +43116 20713 +47009 20712 +28833 20712 +38066 20708 +16946 20707 +16575 20707 +45693 20706 +48052 20704 +34435 20703 +46460 20703 +26817 20700 +22266 20699 +39738 20697 +27068 20694 +13561 20693 +35997 20692 +37806 20691 +45128 20690 +39831 20689 +24661 20687 +47861 20686 +20098 20686 +44004 20680 +44261 20678 +39242 20675 +41148 20674 +11586 20671 +45118 20671 +4832 20671 +40043 20667 +44184 20659 +22797 20659 +25033 20657 +43322 20654 +22454 20653 +33258 20653 +34793 20649 +38220 20648 +40248 20648 +36722 20646 +43996 20645 +48189 20643 +10079 20642 +43540 20641 +47913 20635 +16709 20626 +43193 20622 +37765 20614 +39768 20612 +45028 20607 +47484 20606 +12743 20606 +45632 20606 +13962 20604 +21250 20597 +38082 20597 +28667 20596 +42740 20595 +49561 20587 +15535 20587 +43367 20585 +41364 20585 +43410 20584 +40987 20584 +10672 20583 +49109 20581 +38055 20580 +40179 20578 +42583 20574 +25642 20573 +24753 20570 +34124 20569 +49944 20568 +49351 20563 +49288 20562 +31041 20560 +34456 20556 +41019 20556 +32918 20553 +50000 20552 +37845 20550 +33239 20549 +41344 20546 +19738 20545 +35926 20544 +43391 20542 +36265 20542 +47799 20541 +37349 20539 +45900 20534 +47408 20528 +35474 20526 +42105 20524 +49853 20523 +42885 20521 +41473 20519 +40005 20515 +39179 20514 +46984 20512 +2201 20512 +36424 20512 +12843 20509 +29508 20507 +23909 20504 +39378 20503 +46415 20502 +30691 20502 +47276 20501 +45668 20495 +42175 20493 +41017 20493 +17982 20491 +50213 20491 +48062 20490 +37860 20490 +40175 20489 +23661 20489 +1508 20487 +37224 20484 +14761 20479 +48159 20478 +10294 20477 +44817 20476 +47426 20473 +37258 20473 +41847 20473 +34046 20471 +18009 20470 +38586 20470 +29715 20470 +47702 20469 +43169 20466 +33993 20462 +44218 20460 +41745 20459 +34400 20459 +44925 20458 +32059 20458 +5965 20458 +20456 20448 +39731 20446 +45758 20446 +48866 20445 +39188 20445 +43912 20444 +11062 20440 +34813 20437 +42117 20437 +45421 20436 +33758 20435 +46827 20433 +34739 20431 +34524 20429 +15578 20427 +6019 20426 +40638 20426 +25228 20426 +24008 20424 +37289 20423 +11415 20422 +42778 20422 +47666 20421 +4456 20417 +37680 20416 +29984 20416 +35736 20416 +38053 20415 +45538 20413 +45940 20413 +47897 20412 +37827 20412 +5084 20411 +17665 20411 +24461 20410 +34300 20409 +41495 20409 +23346 20408 +42918 20407 +28613 20406 +31674 20405 +43420 20404 +45590 20403 +46387 20402 +27288 20401 +44927 20400 +47969 20400 +45840 20399 +42215 20396 +21470 20396 +41136 20392 +16366 20388 +43685 20387 +39403 20386 +41302 20384 +46759 20383 +45850 20380 +40147 20379 +24882 20376 +35396 20376 +41232 20373 +30139 20372 +33272 20369 +48429 20369 +43707 20367 +45366 20365 +35200 20365 +37802 20363 +18688 20363 +41143 20362 +19633 20361 +24019 20356 +40486 20356 +47818 20354 +21197 20353 +9832 20347 +35908 20344 +42650 20342 +45089 20342 +28664 20341 +33854 20341 +22274 20340 +38907 20338 +35810 20337 +45720 20336 +26939 20333 +25271 20332 +24334 20332 +19317 20331 +34169 20330 +33367 20329 +48229 20326 +4977 20324 +42819 20321 +31589 20317 +46352 20316 +44576 20312 +50181 20307 +43631 20301 +41486 20301 +48466 20298 +36603 20298 +41295 20298 +40820 20297 +29988 20295 +49125 20294 +46023 20294 +41706 20293 +35964 20293 +45280 20292 +46435 20287 +32238 20286 +37453 20284 +32069 20276 +42199 20275 +45842 20274 +46169 20268 +48294 20266 +31080 20265 +40940 20264 +32739 20262 +37424 20257 +32923 20256 +35828 20256 +20655 20255 +26794 20254 +39315 20254 +36928 20254 +40433 20252 +33967 20246 +5217 20240 +45131 20238 +42213 20237 +21319 20237 +33402 20235 +24531 20235 +45339 20232 +39368 20222 +30813 20221 +21618 20219 +40797 20219 +38885 20215 +16634 20204 +33186 20201 +31634 20199 +50140 20197 +29734 20196 +38628 20195 +48544 20192 +20397 20187 +33112 20184 +13749 20183 +3962 20183 +39096 20181 +45009 20180 +37852 20180 +46147 20178 +46341 20176 +32475 20171 +4575 20170 +42353 20170 +36076 20170 +33549 20167 +24981 20167 +46235 20166 +47106 20165 +49898 20165 +30641 20163 +48909 20161 +17488 20159 +25746 20155 +47623 20154 +36600 20154 +48136 20151 +30192 20145 +18541 20142 +31148 20142 +31633 20136 +29884 20131 +48782 20129 +30518 20125 +15289 20115 +32795 20114 +32738 20113 +38312 20109 +44793 20108 +47957 20103 +23012 20098 +33463 20089 +49089 20089 +49921 20088 +41065 20086 +48380 20083 +49674 20079 +47985 20078 +16714 20077 +42871 20077 +30189 20077 +49644 20075 +43875 20074 +14141 20073 +25990 20073 +39697 20073 +25099 20071 +30969 20069 +26266 20064 +18765 20063 +28358 20063 +49934 20062 +45385 20060 +18156 20060 +37988 20059 +17992 20059 +40035 20051 +44736 20050 +44260 20047 +13515 20045 +40423 20044 +45797 20043 +36870 20042 +49012 20035 +44400 20032 +48386 20028 +33939 20026 +25506 20025 +23928 20024 +42411 20022 +13651 20022 +39120 20021 +45427 20021 +41198 20018 +22528 20017 +48463 20014 +47642 20012 +8416 20009 +21918 20007 +40311 20005 +22809 20002 +25979 20002 +39664 19999 +30087 19998 +36961 19997 +37826 19996 +43697 19995 +3229 19992 +39210 19990 +43057 19984 +15609 19981 +35490 19980 +48295 19977 +27787 19976 +38697 19972 +21860 19971 +40393 19970 +46180 19968 +49244 19964 +36226 19964 +45106 19961 +30396 19959 +49995 19957 +47675 19953 +32362 19947 +18314 19946 +25679 19945 +37601 19944 +47578 19944 +32318 19943 +13628 19943 +42276 19941 +48291 19940 +45401 19938 +13887 19933 +36725 19932 +41600 19930 +44136 19929 +41948 19928 +47069 19926 +33366 19923 +45252 19920 +46636 19911 +37015 19908 +13073 19906 +42428 19904 +38918 19903 +43901 19901 +30322 19900 +39878 19899 +47330 19892 +33199 19890 +46617 19889 +27968 19889 +41938 19887 +41752 19886 +42091 19885 +44123 19885 +39628 19882 +40465 19881 +48057 19881 +44639 19880 +18380 19879 +1060 19878 +45472 19877 +11988 19875 +4971 19874 +17910 19872 +22690 19867 +42939 19864 +47468 19863 +23651 19858 +20112 19858 +42430 19856 +34487 19851 +46201 19845 +34437 19840 +41257 19837 +37775 19833 +50085 19831 +38438 19831 +42074 19827 +38237 19826 +45107 19822 +46314 19820 +49503 19820 +29401 19820 +22157 19819 +25117 19815 +42413 19815 +34792 19812 +20448 19811 +46758 19806 +43600 19805 +33101 19805 +32852 19803 +44908 19800 +37203 19799 +45111 19798 +12658 19791 +25661 19788 +32941 19784 +40573 19783 +39107 19777 +44705 19772 +42250 19772 +49488 19770 +9622 19769 +45721 19762 +40256 19761 +34964 19761 +42601 19761 +43876 19761 +32001 19760 +46792 19757 +38802 19755 +48376 19752 +41722 19750 +27894 19748 +48177 19747 +45113 19746 +40848 19745 +11877 19744 +12235 19744 +36276 19743 +18048 19742 +42790 19741 +48408 19736 +34631 19736 +47601 19734 +15514 19733 +36104 19727 +42769 19726 +44881 19726 +39872 19725 +44617 19724 +36693 19721 +48745 19720 +32316 19720 +38654 19717 +9872 19716 +23172 19712 +43808 19710 +48882 19709 +43580 19707 +44578 19705 +39696 19701 +42962 19698 +30235 19698 +27992 19698 +44928 19697 +43673 19697 +49650 19696 +40654 19695 +45531 19695 +17751 19694 +19007 19692 +41647 19691 +41261 19690 +3121 19689 +16998 19689 +38970 19688 +45288 19686 +32616 19685 +42789 19684 +38070 19683 +46356 19683 +48584 19681 +46438 19679 +23812 19678 +48108 19678 +10803 19678 +47824 19675 +30620 19674 +12359 19674 +41604 19669 +41895 19668 +31184 19667 +27880 19665 +45124 19664 +47070 19661 +46883 19660 +46302 19656 +44667 19652 +23093 19650 +16684 19649 +43432 19646 +44969 19644 +16530 19643 +49097 19643 +6854 19643 +17573 19642 +7992 19637 +31000 19634 +44977 19633 +40682 19631 +28631 19629 +38457 19628 +15507 19626 +49196 19625 +35438 19624 +47576 19619 +26694 19617 +46020 19615 +47308 19615 +41458 19612 +43571 19610 +28761 19608 +40572 19606 +41914 19605 +48739 19605 +20280 19603 +47491 19602 +32928 19601 +25346 19599 +46258 19598 +43451 19592 +47493 19590 +46028 19584 +32454 19583 +10600 19583 +43813 19580 +47482 19574 +31094 19573 +37167 19571 +42690 19569 +42343 19568 +42463 19564 +44363 19556 +28869 19555 +42776 19547 +40417 19545 +33505 19545 +44893 19544 +43026 19544 +49216 19542 +20335 19541 +28254 19539 +40747 19538 +30409 19536 +31551 19536 +6694 19534 +38445 19534 +42115 19533 +30903 19532 +24292 19531 +48030 19530 +42214 19526 +50168 19524 +48638 19523 +20193 19521 +20502 19520 +48110 19519 +15911 19517 +48194 19515 +40659 19515 +43284 19514 +26729 19513 +40620 19511 +35795 19510 +49446 19508 +38843 19508 +18442 19507 +23396 19504 +34694 19504 +26766 19500 +38142 19497 +38828 19495 +33070 19495 +21968 19493 +46061 19493 +44549 19492 +50080 19491 +28106 19487 +13344 19486 +18099 19484 +41381 19483 +26025 19482 +33751 19479 +44399 19479 +22165 19479 +48392 19474 +38793 19474 +42321 19471 +21889 19465 +38117 19459 +43778 19457 +46894 19455 +34431 19454 +46572 19452 +37586 19449 +46632 19449 +29891 19447 +40279 19447 +46417 19447 +48264 19446 +26233 19445 +36580 19444 +38015 19444 +49837 19443 +37522 19443 +28455 19442 +18885 19441 +46333 19437 +43908 19436 +10201 19435 +27300 19433 +31286 19429 +48488 19428 +30765 19426 +49115 19425 +31563 19422 +38314 19420 +35609 19415 +43948 19414 +22135 19412 +31613 19407 +40639 19405 +31972 19404 +47521 19401 +34615 19394 +40972 19394 +38139 19392 +30482 19390 +39912 19389 +37832 19389 +39332 19388 +18584 19388 +42548 19386 +47899 19386 +47498 19385 +38071 19384 +44879 19381 +32340 19380 +1261 19379 +45960 19378 +39147 19373 +22504 19372 +45864 19371 +49123 19371 +46442 19370 +39089 19370 +50161 19369 +38350 19369 +38202 19362 +41599 19361 +38600 19361 +23190 19359 +22550 19359 +38954 19358 +23952 19358 +46871 19356 +47990 19356 +39112 19355 +21722 19355 +42956 19354 +44649 19354 +40756 19352 +34808 19349 +30424 19349 +42032 19348 +11021 19348 +40207 19347 +49064 19346 +35143 19345 +49953 19344 +36275 19344 +38852 19344 +49147 19343 +34026 19341 +39800 19341 +25512 19341 +43006 19341 +4989 19336 +24480 19335 +41456 19335 +21463 19335 +9411 19330 +23645 19328 +23435 19326 +46239 19325 +7462 19324 +47760 19323 +35854 19323 +9378 19322 +18713 19321 +35167 19320 +43719 19319 +45006 19319 +42130 19315 +48856 19313 +48634 19313 +46886 19311 +38177 19310 +47369 19309 +42759 19308 +24945 19307 +31577 19306 +44215 19304 +28864 19303 +41642 19302 +43182 19298 +46925 19296 +38509 19295 +44468 19295 +49319 19292 +48764 19292 +37839 19291 +42336 19291 +31050 19287 +44338 19283 +28114 19281 +46096 19278 +16606 19277 +45734 19276 +25052 19274 +34023 19272 +44031 19269 +37050 19268 +41833 19267 +38116 19266 +36555 19264 +29289 19263 +48092 19261 +18363 19260 +41541 19258 +23522 19258 +47931 19257 +45989 19254 +47683 19253 +49701 19250 +46052 19249 +34621 19246 +33203 19245 +40711 19243 +47900 19243 +47233 19242 +2874 19237 +46501 19232 +49111 19231 +18386 19230 +38062 19230 +46056 19227 +15676 19227 +38375 19226 +45838 19225 +47200 19223 +48724 19223 +39102 19222 +49439 19218 +42075 19216 +18144 19214 +45881 19211 +46872 19211 +29344 19210 +47949 19209 +39573 19207 +41925 19206 +45349 19206 +44896 19203 +38685 19202 +44397 19198 +41169 19197 +21177 19196 +8460 19196 +35123 19194 +48573 19184 +18095 19182 +16956 19180 +1914 19179 +24990 19179 +16786 19176 +23562 19176 +10042 19175 +25029 19175 +34619 19175 +37652 19174 +36813 19172 +36948 19170 +20773 19161 +24412 19159 +20763 19158 +29304 19157 +27319 19156 +38412 19156 +22918 19155 +46109 19154 +40441 19147 +37798 19144 +39405 19140 +48150 19139 +39785 19137 +29114 19137 +38604 19131 +43462 19129 +44408 19124 +45122 19121 +17434 19121 +34200 19120 +46749 19118 +27130 19114 +35509 19113 +49509 19113 +50010 19112 +42407 19112 +45452 19112 +44960 19110 +46868 19108 +45696 19107 +48096 19106 +46270 19105 +27270 19098 +22902 19089 +34393 19089 +10519 19088 +33074 19085 +43864 19082 +42174 19080 +25633 19080 +44560 19078 +48630 19075 +11850 19073 +48524 19071 +36305 19069 +22869 19066 +18719 19064 +43048 19060 +44891 19058 +28871 19058 +47670 19057 +47880 19057 +44513 19053 +14688 19052 +31643 19047 +48260 19047 +33651 19047 +39413 19045 +46263 19044 +33571 19039 +17191 19036 +43009 19036 +44176 19033 +41792 19032 +34779 19031 +28886 19030 +46158 19027 +46993 19026 +48825 19023 +45944 19022 +19741 19019 +41583 19018 +21936 19016 +49845 19013 +48398 19012 +25058 19011 +5467 19008 +36282 19007 +41492 19006 +24512 19005 +48187 19004 +42763 19002 +29274 19001 +16982 19001 +47237 19001 +33325 19001 +50027 18997 +42435 18992 +35230 18989 +46589 18989 +40631 18985 +18749 18983 +26368 18982 +36228 18982 +21663 18979 +4027 18979 +13807 18978 +36499 18978 +36109 18977 +46176 18972 +26491 18968 +39443 18965 +49974 18963 +31457 18963 +46103 18954 +47785 18952 +47016 18951 +27175 18951 +43986 18950 +41072 18949 +34408 18947 +48813 18946 +7290 18945 +47396 18945 +41280 18945 +48645 18944 +30187 18936 +50167 18933 +43339 18933 +38650 18932 +29163 18931 +12650 18931 +40736 18930 +47915 18929 +17181 18928 +39584 18927 +20619 18925 +47305 18924 +44063 18924 +30142 18919 +44214 18918 +10027 18917 +26967 18914 +41279 18912 +48170 18912 +45225 18910 +50222 18905 +22192 18898 +46285 18894 +39324 18893 +37446 18892 +20297 18891 +43119 18888 +13163 18888 +43465 18885 +13297 18883 +41627 18882 +49397 18881 +43445 18881 +37458 18881 +47941 18881 +48246 18878 +8289 18877 +37889 18876 +42562 18875 +33995 18875 +49964 18873 +49052 18862 +49857 18861 +32606 18860 +41874 18856 +44598 18854 +29277 18853 +46623 18852 +41765 18852 +42011 18850 +38169 18849 +27991 18847 +34349 18846 +5312 18842 +46250 18839 +21704 18836 +40251 18831 +26901 18830 +23770 18828 +47350 18821 +27214 18820 +48930 18819 +19735 18817 +35387 18813 +39509 18810 +29410 18809 +47807 18803 +49489 18800 +17040 18794 +48462 18794 +34850 18790 +34411 18788 +34610 18787 +47694 18786 +2203 18785 +40401 18782 +43427 18779 +13264 18779 +25775 18778 +43819 18776 +27636 18773 +46476 18770 +37941 18768 +33086 18768 +41955 18764 +45705 18762 +45441 18758 +43300 18751 +8535 18747 +43854 18745 +44265 18740 +38318 18739 +35087 18736 +18991 18735 +8522 18729 +45857 18727 +37691 18725 +44685 18725 +49499 18724 +27215 18723 +34718 18721 +35950 18719 +43564 18718 +23168 18717 +24085 18717 +19489 18716 +16448 18715 +43822 18715 +48665 18711 +49550 18711 +42145 18708 +10639 18702 +26836 18702 +48697 18696 +39568 18695 +27844 18695 +48772 18694 +47397 18693 +48037 18692 +25367 18692 +39615 18686 +44504 18685 +48442 18683 +43385 18683 +5666 18682 +42675 18680 +43883 18680 +24647 18678 +42271 18676 +41403 18674 +5914 18674 +26892 18674 +48864 18672 +48647 18667 +17566 18666 +14391 18664 +42283 18664 +32031 18662 +21113 18661 +24212 18661 +48381 18658 +37400 18656 +46649 18649 +24360 18645 +34919 18645 +47613 18644 +39550 18641 +33934 18639 +41998 18636 +16424 18628 +32942 18626 +24112 18626 +36241 18625 +26098 18622 +40414 18621 +36818 18620 +48648 18618 +47812 18613 +49130 18607 +9333 18606 +21930 18604 +41050 18602 +26991 18599 +47901 18598 +42443 18596 +30380 18595 +42938 18592 +45791 18592 +34829 18591 +43722 18590 +44372 18589 +36255 18584 +28919 18584 +17271 18581 +23862 18578 +25955 18577 +24121 18576 +42482 18574 +36056 18572 +44675 18570 +40804 18569 +24355 18569 +49816 18569 +42702 18568 +37905 18568 +46743 18567 +36949 18567 +10430 18566 +41060 18561 +8456 18559 +25612 18558 +31728 18557 +44269 18555 +29919 18555 +42405 18554 +16579 18546 +24084 18544 +20150 18544 +34888 18543 +46692 18542 +30297 18541 +31737 18540 +18497 18538 +22062 18535 +48489 18534 +44622 18532 +49720 18532 +23736 18528 +47085 18527 +34712 18524 +35443 18524 +44087 18524 +32501 18516 +19537 18515 +26341 18513 +28547 18510 +32527 18508 +35712 18505 +45639 18502 +41330 18501 +47838 18500 +48719 18500 +25593 18499 +38752 18498 +44955 18496 +27547 18490 +39866 18490 +6900 18490 +22285 18490 +17350 18490 +43930 18489 +32179 18487 +44586 18486 +48808 18482 +9802 18478 +35064 18476 +19424 18473 +50205 18471 +29982 18470 +48861 18470 +49330 18470 +42307 18469 +47037 18465 +44055 18465 +33096 18465 +42412 18464 +35035 18463 +48006 18463 +43389 18462 +34333 18462 +39334 18459 +15026 18458 +46752 18457 +10859 18456 +44753 18455 +43950 18454 +41183 18453 +8032 18452 +24145 18449 +47646 18445 +47275 18444 +13031 18444 +44327 18442 +41815 18441 +46678 18441 +44573 18439 +38226 18434 +42604 18433 +26994 18431 +46296 18430 +29929 18429 +39921 18429 +13607 18421 +42493 18421 +39168 18419 +38902 18416 +46026 18412 +44396 18410 +38187 18410 +37452 18410 +19465 18406 +49150 18402 +43557 18402 +49424 18399 +48578 18393 +33838 18392 +41531 18390 +41962 18390 +16845 18385 +35775 18380 +48305 18380 +46900 18379 +11605 18378 +31439 18378 +27949 18377 +48337 18369 +23789 18368 +46681 18368 +21086 18368 +43191 18367 +31257 18366 +23733 18366 +39492 18364 +43680 18361 +45839 18358 +47963 18356 +34652 18353 +8206 18352 +42739 18347 +41610 18346 +43559 18346 +44295 18341 +41228 18336 +23156 18336 +43446 18336 +26857 18335 +40908 18335 +45943 18335 +6969 18330 +44719 18329 +43521 18329 +38565 18327 +49666 18324 +46093 18323 +45702 18321 +44659 18315 +40350 18313 +45628 18312 +17418 18309 +46860 18307 +30352 18307 +47512 18307 +48618 18306 +43277 18303 +17881 18301 +38588 18300 +34897 18298 +35350 18293 +31989 18291 +36823 18290 +33131 18289 +39776 18289 +50074 18289 +48602 18286 +40348 18281 +45486 18277 +48414 18276 +47488 18275 +21118 18274 +35513 18271 +45214 18270 +16536 18267 +43468 18267 +27789 18267 +43847 18264 +17425 18262 +41850 18261 +12357 18261 +36954 18258 +50029 18255 +40052 18254 +34773 18253 +14124 18252 +43283 18248 +46478 18248 +48603 18244 +45319 18244 +31255 18241 +48740 18240 +43059 18238 +31671 18236 +38451 18236 +11931 18232 +32216 18224 +47389 18218 +44172 18216 +49033 18214 +32007 18214 +22055 18214 +22592 18212 +44847 18211 +40451 18211 +48892 18209 +41051 18207 +28524 18205 +24511 18205 +30613 18203 +40505 18201 +33142 18200 +45515 18198 +48653 18198 +44375 18197 +13337 18196 +44612 18192 +32203 18188 +17407 18187 +38642 18187 +45256 18185 +48767 18185 +4667 18183 +43665 18182 +48156 18180 +30026 18180 +13161 18180 +37699 18179 +42001 18179 +50052 18177 +28089 18176 +20627 18176 +38508 18171 +9680 18171 +29080 18170 +42635 18169 +42955 18169 +35635 18164 +45157 18162 +33923 18160 +38219 18160 +40287 18158 +42996 18155 +36523 18155 +35251 18152 +8417 18152 +19872 18151 +24704 18150 +23108 18145 +43312 18144 +41963 18142 +23428 18135 +46079 18130 +31793 18124 +45891 18123 +31664 18119 +25723 18118 +17791 18117 +44494 18117 +44305 18116 +29670 18116 +31904 18115 +22851 18114 +38161 18108 +19945 18106 +31305 18105 +31715 18103 +45007 18101 +19743 18100 +17725 18095 +49051 18091 +47582 18090 +43172 18088 +34390 18085 +29550 18084 +48075 18084 +44542 18083 +15493 18083 +47570 18081 +44826 18078 +29746 18077 +32106 18074 +19241 18074 +47274 18070 +37363 18070 +38692 18067 +6012 18067 +47859 18061 +29149 18060 +34097 18059 +27935 18054 +24437 18054 +39961 18049 +48441 18044 +23611 18037 +43243 18037 +41263 18034 +38627 18032 +13072 18031 +43563 18028 +48945 18028 +32315 18026 +26486 18023 +28575 18019 +43239 18018 +35914 18016 +48165 18016 +28095 18011 +42187 18007 +16306 18006 +21015 18005 +41596 18004 +42622 18003 +42312 18003 +43668 17999 +44307 17998 +36804 17997 +47240 17993 +49348 17993 +26388 17987 +5307 17986 +38471 17985 +34167 17985 +34706 17982 +48465 17981 +44694 17977 +30867 17976 +36659 17975 +32208 17975 +32512 17973 +43887 17967 +41235 17966 +44114 17966 +45728 17963 +22467 17962 +49641 17960 +20308 17957 +42237 17956 +39134 17954 +39633 17952 +44743 17949 +38179 17947 +41841 17945 +47896 17934 +49900 17929 +49366 17926 +45523 17919 +39601 17918 +12360 17917 +49803 17910 +32449 17909 +19357 17908 +48054 17902 +31007 17899 +37045 17899 +9097 17898 +47612 17893 +14445 17893 +43098 17891 +40143 17891 +39450 17890 +33429 17888 +41601 17886 +35051 17881 +32649 17881 +49915 17880 +40663 17875 +44887 17874 +47111 17871 +31554 17869 +49875 17868 +19319 17868 +17391 17866 +37994 17865 +37836 17863 +20846 17860 +16870 17859 +27524 17858 +4742 17855 +22469 17855 +47726 17851 +41095 17849 +41732 17849 +46618 17848 +49313 17847 +46914 17846 +24802 17846 +27664 17844 +44543 17843 +39417 17837 +38213 17836 +48556 17831 +22272 17831 +34575 17827 +31761 17827 +21176 17827 +49579 17827 +37152 17825 +30541 17822 +24510 17821 +19816 17821 +22993 17820 +37074 17818 +49234 17818 +43656 17817 +41990 17817 +46440 17815 +49254 17815 +48623 17812 +44253 17811 +48090 17810 +48086 17810 +15311 17810 +37057 17809 +30675 17808 +11827 17808 +44203 17802 +22512 17799 +46888 17798 +44943 17789 +44552 17787 +26446 17786 +47965 17783 +13221 17780 +45938 17780 +11535 17780 +10702 17777 +29519 17777 +35298 17773 +17850 17772 +28906 17768 +44661 17766 +37637 17764 +29917 17764 +1764 17764 +34337 17762 +24290 17761 +22407 17760 +49437 17759 +18243 17757 +28905 17756 +45462 17756 +44532 17754 +49436 17753 +20164 17753 +29857 17748 +18819 17747 +10260 17747 +47152 17738 +46046 17738 +48444 17734 +17177 17733 +37828 17730 +24609 17724 +8760 17719 +39647 17715 +41351 17714 +37288 17714 +48947 17714 +20683 17709 +26055 17708 +45114 17707 +11049 17705 +42676 17704 +43739 17702 +31960 17702 +30233 17702 +42188 17698 +50223 17698 +35838 17697 +44897 17695 +23792 17695 +47758 17692 +12875 17691 +44259 17690 +25683 17688 +43805 17686 +37394 17686 +43046 17681 +49782 17681 +44069 17680 +48047 17675 +43490 17672 +24559 17671 +44020 17670 +23977 17670 +27454 17664 +45616 17663 +28863 17661 +41369 17661 +2463 17659 +29126 17658 +45216 17653 +12443 17647 +20630 17643 +29247 17642 +47731 17641 +43732 17638 +41705 17636 +45825 17635 +47044 17635 +40104 17635 +44169 17634 +25652 17634 +23521 17634 +41793 17633 +35379 17633 +32661 17631 +33108 17629 +44060 17628 +8645 17628 +42322 17623 +39121 17621 +33481 17619 +46371 17616 +43336 17615 +47191 17615 +39090 17610 +34128 17603 +47056 17595 +42084 17595 +40392 17594 +41141 17594 +30226 17592 +27369 17588 +43209 17587 +30329 17585 +2216 17584 +31950 17583 +40839 17583 +32870 17580 +37346 17579 +14911 17577 +35731 17576 +11030 17576 +42782 17576 +11018 17576 +40324 17575 +31336 17575 +37779 17575 +43946 17573 +49856 17572 +45738 17571 +43787 17570 +40260 17567 +34684 17566 +44934 17566 +20971 17565 +50183 17563 +46159 17563 +44673 17562 +24035 17561 +28329 17561 +38744 17560 +40201 17558 +45465 17557 +46267 17556 +39887 17553 +26331 17551 +48236 17546 +42258 17546 +10328 17546 +43605 17545 +39572 17541 +44965 17539 +45670 17534 +43027 17534 +46841 17533 +35909 17532 +30074 17531 +50238 17530 +36252 17522 +49075 17517 +45636 17515 +31310 17514 +46665 17514 +44000 17514 +46519 17510 +39570 17507 +38497 17506 +39437 17505 +38896 17504 +35911 17502 +18538 17502 +39816 17499 +46248 17497 +27020 17497 +47184 17490 +20238 17490 +17614 17490 +40301 17490 +35744 17489 +11981 17488 +43589 17485 +19899 17484 +41094 17483 +46887 17483 +35961 17481 +43625 17479 +46432 17476 +48884 17475 +27480 17474 +44121 17474 +6030 17473 +28321 17469 +43529 17469 +35211 17468 +29123 17468 +2862 17463 +26940 17461 +48555 17460 +43063 17460 +38624 17458 +42488 17457 +42964 17452 +33913 17451 +13303 17450 +18999 17447 +2847 17442 +44973 17441 +29882 17440 +25020 17436 +42131 17434 +45080 17434 +38385 17433 +37592 17423 +43240 17423 +44442 17421 +14913 17419 +37696 17416 +29511 17416 +6671 17416 +49602 17416 +26458 17415 +49703 17414 +21594 17414 +45182 17409 +48818 17406 +14606 17403 +37296 17403 +39491 17397 +41761 17396 +45964 17396 +15082 17393 +33441 17391 +45935 17391 +41376 17388 +42581 17388 +46468 17386 +27374 17386 +40647 17385 +41409 17385 +31098 17383 +41047 17382 +36479 17381 +45648 17380 +841 17379 +20745 17379 +44078 17378 +42965 17378 +17005 17376 +13462 17375 +3591 17374 +42980 17373 +49257 17370 +29078 17370 +23037 17369 +13249 17368 +32895 17367 +50165 17366 +45761 17363 +37174 17360 +25213 17357 +49303 17354 +24866 17353 +36227 17352 +39183 17351 +44153 17345 +39923 17344 +4148 17343 +48926 17343 +18431 17343 +46815 17343 +29150 17341 +43292 17341 +47370 17341 +19295 17335 +19211 17334 +46717 17331 +49211 17330 +48327 17329 +5772 17327 +43994 17322 +19285 17321 +34997 17317 +44764 17317 +24064 17316 +49356 17316 +18471 17316 +33554 17316 +43223 17314 +28780 17314 +19676 17314 +49394 17313 +36191 17312 +4566 17311 +43683 17310 +35881 17310 +42494 17306 +26908 17306 +48552 17305 +43146 17300 +41159 17291 +41700 17290 +48228 17290 +42643 17288 +16750 17287 +13287 17286 +11134 17284 +44329 17284 +41561 17283 +45949 17282 +36352 17281 +38513 17277 +24189 17277 +19768 17271 +37078 17271 +42301 17265 +34667 17256 +35945 17254 +33975 17251 +35157 17251 +47776 17251 +49114 17249 +44077 17245 +42141 17242 +9108 17242 +27187 17240 +14993 17240 +46781 17239 +35625 17238 +48959 17237 +48188 17236 +15522 17235 +24729 17232 +48964 17230 +40733 17229 +33635 17226 +47946 17224 +28425 17218 +31042 17216 +22658 17215 +49525 17213 +32178 17211 +36728 17204 +46807 17203 +28599 17201 +20020 17201 +38679 17198 +2570 17197 +40545 17194 +41455 17194 +47127 17192 +40384 17191 +42703 17191 +42563 17187 +30583 17187 +9244 17187 +26245 17184 +48952 17183 +31686 17181 +34211 17174 +44900 17174 +14817 17166 +32186 17166 +37201 17164 +45659 17161 +25435 17159 +39196 17158 +44980 17157 +38208 17155 +44417 17152 +31993 17147 +47544 17145 +48894 17144 +12220 17144 +37496 17141 +38864 17139 +44489 17133 +30010 17132 +26755 17131 +40202 17130 +27919 17130 +47555 17125 +49730 17120 +46091 17120 +20306 17116 +42978 17116 +47871 17116 +39422 17115 +24196 17109 +2110 17108 +37007 17107 +43514 17105 +44249 17104 +43228 17104 +10221 17104 +26632 17102 +38418 17102 +31401 17102 +46095 17102 +43515 17101 +44528 17101 +46953 17101 +32717 17101 +43687 17097 +50006 17096 +23509 17095 +33528 17092 +45765 17089 +11962 17088 +29544 17087 +30009 17085 +3475 17079 +18547 17077 +30516 17077 +49289 17076 +34090 17073 +31951 17072 +45559 17072 +41644 17071 +44877 17069 +31126 17069 +38972 17069 +50055 17061 +35149 17061 +37883 17060 +37460 17060 +17147 17058 +49189 17057 +44729 17055 +44171 17054 +16593 17053 +49053 17046 +31021 17044 +25551 17044 +37354 17043 +33691 17042 +43619 17041 +46112 17039 +30344 17037 +44183 17037 +33980 17033 +36170 17031 +47603 17028 +43413 17026 +46798 17025 +39061 17021 +41827 17020 +7909 17019 +47672 17015 +40108 17011 +49253 17010 +26439 17004 +48528 17003 +49645 16995 +26419 16995 +49532 16993 +44589 16990 +35745 16989 +44950 16988 +13321 16985 +21237 16981 +40262 16980 +26358 16978 +41867 16977 +42520 16972 +36038 16970 +46114 16970 +50019 16969 +47210 16968 +47036 16967 +46571 16967 +16327 16966 +46542 16965 +3471 16963 +21993 16961 +35743 16958 +48840 16956 +32803 16956 +33704 16956 +31741 16956 +36390 16952 +46877 16947 +24079 16945 +34472 16941 +29273 16940 +15252 16940 +46809 16938 +26560 16938 +5744 16937 +45886 16934 +28322 16933 +41267 16931 +20650 16928 +24428 16927 +22545 16923 +33336 16919 +32381 16917 +39106 16914 +42385 16912 +42820 16907 +44701 16906 +31740 16906 +30910 16906 +30173 16905 +38484 16904 +49459 16904 +22123 16903 +24978 16902 +32428 16902 +21628 16901 +17227 16900 +34600 16896 +20704 16895 +14474 16895 +10774 16892 +44509 16891 +45100 16888 +20204 16886 +42856 16886 +42054 16885 +47641 16884 +40277 16884 +44116 16883 +25606 16883 +38947 16878 +49700 16877 +47539 16877 +48431 16876 +41300 16875 +33202 16874 +14038 16873 +40849 16872 +48262 16871 +46803 16870 +45557 16870 +24443 16869 +47156 16868 +50119 16868 +35330 16862 +37649 16862 +48518 16859 +45467 16851 +33430 16851 +38467 16850 +43807 16849 +39528 16846 +45053 16841 +42946 16841 +47898 16838 +49735 16836 +20181 16836 +15128 16833 +27579 16833 +49768 16833 +32707 16831 +45952 16827 +45666 16824 +4797 16819 +16948 16817 +47360 16816 +40193 16815 +44383 16814 +32264 16805 +22251 16800 +50226 16799 +11076 16795 +34478 16795 +35915 16793 +40602 16791 +41784 16790 +33701 16790 +30817 16790 +46503 16789 +40926 16788 +26584 16786 +40673 16785 +45206 16784 +31644 16776 +43723 16775 +46928 16774 +46303 16773 +40575 16773 +40984 16768 +23251 16767 +45862 16766 +37165 16765 +31830 16762 +44384 16758 +46704 16757 +15356 16757 +36594 16753 +43824 16753 +42253 16751 +27365 16750 +45987 16747 +47907 16745 +38353 16742 +21275 16739 +49669 16738 +49787 16736 +20526 16736 +50242 16733 +25574 16732 +20960 16730 +39765 16728 +17117 16725 +48747 16725 +17932 16724 +47424 16723 +46862 16723 +23750 16723 +40366 16722 +46012 16720 +2212 16720 +43140 16720 +42483 16717 +19423 16717 +43952 16714 +31438 16709 +29828 16708 +41134 16708 +47994 16705 +44647 16704 +29248 16703 +50174 16697 +42600 16696 +37512 16693 +43275 16691 +42672 16689 +49500 16688 +39557 16687 +49087 16686 +31615 16682 +17041 16681 +44030 16677 +46405 16676 +37336 16676 +41563 16676 +36924 16673 +17854 16669 +42313 16668 +49822 16663 +45601 16662 +41872 16660 +18769 16659 +49541 16659 +45902 16657 +49685 16656 +46325 16653 +37797 16651 +43021 16650 +40380 16650 +44207 16647 +37689 16645 +31595 16644 +26984 16641 +32196 16637 +23545 16636 +33135 16631 +40435 16631 +23197 16627 +41103 16626 +34823 16625 +49750 16621 +41723 16620 +45372 16619 +39217 16618 +16466 16613 +42024 16607 +38567 16606 +25932 16605 +41934 16605 +44395 16602 +44741 16599 +10264 16599 +47886 16597 +21785 16597 +49481 16589 +39702 16589 +48820 16588 +39984 16586 +33730 16585 +43937 16580 +25660 16579 +47921 16574 +49784 16574 +49512 16569 +29106 16567 +15960 16565 +25595 16560 +47787 16560 +49733 16560 +45126 16558 +20921 16556 +34008 16551 +28566 16549 +27080 16549 +42053 16546 +41922 16541 +45770 16541 +43615 16539 +11603 16536 +22752 16534 +39852 16533 +45310 16533 +31185 16532 +49718 16531 +44931 16527 +2625 16525 +22288 16524 +45692 16524 +48979 16522 +45508 16522 +31624 16518 +48428 16516 +17617 16514 +30378 16509 +48109 16507 +1876 16507 +46410 16506 +20277 16505 +11180 16502 +33947 16498 +34921 16496 +38734 16494 +46761 16494 +43835 16491 +47846 16487 +975 16486 +24474 16485 +27747 16483 +36452 16482 +48409 16482 +38138 16479 +44780 16475 +44869 16474 +49682 16474 +28958 16469 +49878 16465 +30602 16464 +2819 16462 +47616 16457 +46256 16457 +48365 16455 +32478 16453 +48612 16452 +38140 16451 +44134 16448 +39933 16448 +47849 16446 +28601 16443 +43645 16442 +17883 16436 +34129 16434 +45290 16434 +42557 16432 +11337 16432 +34570 16427 +49294 16424 +950 16420 +49592 16418 +49922 16417 +27053 16415 +42691 16410 +50088 16410 +45766 16409 +48679 16409 +26026 16404 +47548 16403 +44671 16402 +19368 16398 +38827 16394 +42480 16391 +23331 16389 +31836 16388 +38257 16388 +23878 16386 +14194 16385 +46918 16382 +40503 16382 +32044 16381 +24695 16378 +46957 16375 +48098 16375 +42853 16369 +6316 16368 +41511 16367 +49287 16367 +20223 16366 +16780 16365 +37841 16364 +34007 16361 +48839 16360 +5904 16360 +44856 16358 +35648 16358 +49949 16356 +42486 16354 +24117 16354 +34675 16351 +47597 16351 +45500 16350 +37688 16350 +38556 16348 +48530 16348 +27786 16348 +40282 16344 +8382 16343 +40218 16342 +20580 16340 +47503 16338 +24366 16336 +19673 16333 +47580 16330 +29319 16327 +41466 16324 +44152 16322 +44921 16322 +9903 16321 +30617 16319 +32796 16318 +27858 16316 +31387 16316 +49936 16316 +14609 16306 +49055 16305 +33204 16305 +26537 16298 +26382 16296 +47728 16296 +27727 16293 +6137 16291 +49971 16288 +43608 16288 +43686 16287 +33320 16286 +17295 16286 +44422 16285 +16650 16284 +48173 16280 +35758 16279 +47247 16275 +45605 16271 +17976 16266 +45571 16262 +46736 16261 +49943 16260 +50123 16251 +44761 16251 +29487 16250 +42459 16249 +47346 16248 +22380 16246 +8703 16245 +48338 16243 +38806 16242 +15130 16240 +49638 16240 +38195 16237 +45925 16231 +5487 16231 +40962 16226 +19381 16225 +42787 16223 +46511 16221 +29788 16220 +46660 16219 +42416 16219 +35504 16218 +46500 16216 +44778 16214 +46192 16214 +49617 16212 +28505 16210 +49327 16209 +31752 16206 +25122 16206 +50234 16205 +8907 16205 +45506 16201 +36150 16200 +37739 16199 +50014 16198 +45877 16197 +41450 16195 +17307 16193 +22082 16192 +27383 16188 +37491 16187 +29193 16187 +12764 16186 +49307 16186 +44698 16185 +36925 16185 +21312 16184 +28751 16183 +11145 16181 +11956 16179 +41984 16178 +6629 16174 +32657 16174 +33908 16172 +35772 16172 +42060 16170 +24526 16167 +44750 16166 +47155 16165 +42572 16165 +48771 16162 +2294 16161 +38017 16157 +31371 16156 +23858 16156 +46839 16147 +35258 16144 +39077 16143 +30636 16138 +49193 16138 +27722 16138 +6534 16137 +43977 16127 +31404 16126 +37166 16126 +3410 16123 +45439 16122 +19335 16121 +32335 16118 +47192 16116 +29665 16115 +37849 16113 +49614 16108 +37577 16108 +927 16104 +30402 16100 +34661 16100 +26398 16099 +42080 16098 +45362 16097 +8236 16096 +21369 16096 +21145 16095 +36885 16092 +48960 16090 +46393 16090 +1274 16089 +44324 16087 +45701 16086 +45224 16086 +30472 16085 +25282 16084 +49717 16083 +47385 16083 +7871 16081 +45050 16080 +44229 16078 +15117 16076 +33894 16076 +11627 16073 +40019 16073 +2542 16072 +38710 16071 +49218 16071 +37059 16068 +9438 16067 +43252 16067 +46556 16067 +48372 16067 +50068 16065 +40299 16065 +48628 16062 +37540 16062 +41650 16059 +15748 16057 +17889 16055 +19806 16054 +37678 16050 +40353 16050 +6377 16043 +43795 16040 +44132 16040 +41299 16040 +41496 16034 +14203 16034 +32587 16032 +48167 16031 +48655 16030 +40521 16028 +45304 16026 +15044 16021 +35869 16015 +45127 16015 +47467 16015 +40225 16014 +41201 16012 +43199 16011 +33422 16009 +45653 16008 +43128 16008 +28361 16006 +31276 16004 +10446 16003 +21300 16000 +45116 15998 +45909 15997 +41936 15995 +45618 15995 +13461 15995 +27456 15994 +49959 15992 +46342 15991 +44125 15988 +49533 15987 +41932 15984 +32038 15983 +48223 15979 +36568 15973 +46756 15971 +49517 15971 +37925 15969 +29873 15967 +40538 15965 +28795 15963 +33175 15960 +37163 15960 +39701 15957 +20579 15954 +31852 15952 +21498 15952 +9134 15945 +46992 15941 +35684 15941 +44427 15940 +44609 15936 +47860 15935 +47374 15929 +17752 15927 +12474 15923 +35976 15921 +26252 15919 +30143 15918 +2161 15916 +43324 15914 +26314 15912 +22149 15909 +31062 15908 +43438 15908 +40382 15907 +32271 15906 +10412 15906 +47202 15895 +48548 15895 +20817 15892 +50192 15891 +6132 15888 +6791 15886 +34893 15885 +49760 15883 +48717 15882 +46018 15882 +29142 15882 +37774 15881 +25673 15877 +49199 15877 +39285 15876 +31124 15874 +26254 15871 +37640 15870 +20183 15869 +20941 15868 +38819 15868 +43758 15866 +37204 15863 +41513 15859 +17538 15856 +36303 15856 +9736 15854 +44796 15851 +48640 15849 +47567 15849 +14808 15849 +26035 15848 +28951 15836 +7302 15836 +43335 15834 +38659 15834 +27323 15830 +34414 15829 +28475 15828 +43568 15827 +49985 15827 +49659 15825 +45570 15820 +39766 15815 +8750 15815 +30576 15810 +43230 15809 +50187 15806 +25842 15806 +12596 15804 +32351 15804 +26676 15801 +44561 15798 +31152 15798 +31821 15797 +32876 15796 +40970 15796 +34320 15793 +42612 15792 +45804 15787 +46484 15785 +44254 15782 +43776 15780 +13538 15774 +31153 15772 +39182 15771 +44040 15770 +38069 15770 +32610 15769 +45573 15769 +39306 15768 +48896 15766 +36148 15764 +41416 15764 +47634 15763 +32434 15759 +26505 15758 +48691 15756 +42834 15753 +38025 15751 +38498 15750 +35217 15750 +15965 15748 +9147 15747 +32412 15747 +22685 15747 +43421 15746 +14405 15744 +47976 15741 +23192 15740 +42911 15739 +10189 15739 +45453 15739 +23857 15738 +24548 15736 +16362 15736 +40340 15735 +23847 15733 +10506 15733 +48081 15727 +30262 15727 +38770 15726 +30397 15726 +38699 15725 +37838 15725 +48554 15724 +28421 15720 +43643 15720 +15562 15720 +35191 15720 +35486 15719 +2554 15718 +37677 15718 +24476 15718 +42796 15718 +44367 15714 +11539 15713 +48129 15712 +39649 15712 +48980 15712 +16482 15710 +44084 15705 +33715 15702 +20967 15700 +46064 15699 +49373 15699 +37667 15699 +48089 15699 +37182 15698 +41327 15698 +23260 15694 +23937 15693 +29522 15690 +43550 15690 +26171 15685 +27381 15683 +43846 15676 +43585 15675 +31492 15675 +20076 15672 +37926 15672 +39848 15671 +42864 15670 +21637 15665 +9954 15663 +24979 15662 +43502 15662 +37488 15661 +24330 15661 +34572 15659 +43148 15658 +29179 15658 +37744 15657 +43498 15656 +44650 15656 +40298 15655 +31973 15655 +48540 15654 +14623 15654 +50182 15652 +41178 15651 +23029 15650 +43959 15648 +31635 15648 +30324 15646 +44095 15646 +49766 15645 +22825 15642 +49961 15639 +21565 15638 +41009 15637 +8831 15636 +35334 15632 +34083 15628 +47876 15628 +13912 15625 +37564 15623 +42027 15622 +32177 15616 +39515 15612 +41003 15609 +32163 15604 +36739 15604 +49156 15600 +34871 15600 +39346 15595 +6917 15594 +40146 15592 +49141 15591 +32614 15590 +34799 15588 +43390 15587 +48891 15587 +19326 15585 +28028 15584 +31583 15581 +49039 15579 +44774 15578 +39208 15578 +37058 15575 +8596 15570 +13274 15569 +5383 15568 +43155 15566 +38795 15566 +30454 15565 +48161 15564 +33523 15562 +40531 15560 +34736 15559 +42991 15558 +4723 15556 +48587 15555 +45801 15553 +39790 15551 +22071 15546 +40784 15543 +44235 15543 +34117 15542 +45217 15541 +28419 15541 +12358 15538 +32304 15537 +40584 15534 +45305 15532 +49423 15529 +47420 15525 +44124 15519 +41607 15518 +43639 15514 +35814 15512 +43863 15510 +13023 15510 +43884 15508 +49469 15505 +47211 15504 +43938 15504 +11943 15503 +26558 15501 +28897 15501 +36645 15495 +45752 15494 +32257 15492 +50069 15486 +4338 15486 +28000 15481 +45257 15476 +50132 15476 +40100 15476 +39817 15472 +43584 15470 +28777 15468 +20488 15468 +41636 15468 +44370 15466 +45097 15465 +2789 15465 +50095 15463 +38194 15463 +26873 15461 +28418 15460 +34931 15457 +32812 15457 +32633 15456 +19718 15456 +46486 15455 +34160 15445 +28261 15444 +49262 15440 +22856 15432 +26703 15428 +47273 15426 +27435 15426 +16063 15416 +3647 15414 +42401 15414 +47038 15413 +46856 15413 +9884 15412 +13286 15409 +48712 15407 +15667 15406 +38956 15402 +48718 15401 +48290 15400 +38279 15398 +44905 15395 +25853 15395 +37966 15389 +46874 15387 +43045 15386 +46693 15386 +46088 15385 +6114 15383 +23883 15383 +43060 15382 +45295 15382 +41949 15381 +50141 15379 +29748 15376 +28249 15372 +48019 15372 +46926 15368 +43927 15364 +18187 15363 +45219 15363 +35958 15361 +27590 15360 +35948 15359 +40400 15356 +38782 15355 +15019 15348 +45575 15345 +44629 15340 +19983 15339 +34972 15329 +15908 15329 +48970 15323 +6054 15323 +21908 15320 +42528 15318 +39807 15316 +43526 15311 +47359 15311 +22480 15309 +39444 15306 +41829 15304 +39957 15303 +41712 15299 +24045 15299 +33412 15298 +31058 15296 +37600 15294 +41572 15292 +12664 15291 +27532 15290 +32934 15287 +35578 15280 +42987 15279 +46209 15274 +45930 15273 +43637 15271 +41830 15270 +39896 15266 +46970 15263 +49594 15262 +45860 15261 +42891 15260 +45781 15259 +35241 15259 +46048 15257 +23412 15253 +26799 15251 +30068 15248 +43547 15246 +41171 15242 +38743 15241 +24422 15241 +33415 15240 +138 15238 +40373 15237 +46725 15235 +47553 15233 +46319 15233 +39748 15230 +46152 15230 +28985 15230 +30058 15226 +43926 15225 +38973 15224 +16778 15218 +28299 15217 +49823 15215 +29469 15215 +19279 15215 +25475 15210 +8311 15210 +33695 15205 +38711 15205 +38503 15202 +40743 15202 +41426 15201 +32487 15201 +42854 15199 +21341 15199 +29421 15198 +38845 15197 +42318 15194 +16688 15191 +43142 15190 +48800 15188 +6092 15186 +31927 15185 +44884 15185 +21556 15183 +48935 15182 +46162 15181 +36757 15180 +44688 15180 +32347 15178 +48307 15177 +19703 15169 +35614 15169 +44638 15162 +46940 15160 +13711 15159 +49713 15159 +49557 15156 +49757 15156 +21394 15154 +44739 15153 +34210 15152 +39950 15149 +21774 15142 +42121 15141 +18773 15141 +43872 15139 +11629 15138 +47922 15136 +10513 15128 +47789 15124 +23136 15123 +40118 15119 +30133 15118 +22599 15118 +49841 15117 +48627 15115 +47141 15109 +19532 15107 +44328 15104 +46560 15101 +27110 15097 +44890 15096 +45438 15096 +8657 15095 +50239 15094 +27751 15089 +42333 15088 +4617 15087 +44045 15086 +39505 15086 +18234 15086 +45134 15084 +30118 15081 +31776 15081 +42863 15077 +31097 15076 +42456 15075 +48597 15072 +48326 15071 +30988 15070 +48143 15070 +49764 15067 +47927 15065 +32775 15064 +46359 15063 +28248 15062 +5177 15060 +38786 15059 +35718 15059 +47920 15057 +24926 15056 +18958 15054 +39341 15052 +47529 15051 +42007 15051 +43030 15050 +24439 15048 +38607 15046 +43760 15043 +43290 15041 +9748 15039 +23737 15036 +49552 15033 +50049 15032 +23587 15032 +30653 15031 +46838 15029 +41443 15028 +44821 15028 +27363 15024 +33897 15022 +44721 15020 +29664 15020 +40616 15016 +35115 15011 +50254 15011 +21061 15011 +18375 15010 +35187 15010 +42632 15009 +49170 15009 +33594 15008 +22500 15008 +42220 15006 +49005 15002 +41587 15000 +7607 14995 +43440 14988 +43592 14986 +47752 14985 +47588 14983 +8412 14982 +41676 14980 +33033 14980 +15011 14980 +18037 14978 +21812 14975 +26589 14975 +49352 14975 +28920 14969 +37801 14967 +42003 14960 +26236 14957 +45338 14953 +33498 14952 +43817 14948 +39289 14943 +27100 14939 +43454 14939 +46607 14933 +41553 14931 +38419 14930 +42589 14928 +49954 14927 +46498 14926 +39870 14925 +24837 14924 +37449 14924 +49544 14921 +25969 14920 +46528 14919 +39780 14914 +39974 14914 +26209 14911 +27976 14910 +45210 14905 +43826 14905 +45580 14903 +17562 14894 +37977 14893 +36640 14888 +45345 14886 +42092 14885 +18826 14885 +43725 14878 +33944 14877 +30348 14877 +42349 14875 +22666 14874 +34218 14874 +5346 14873 +44237 14871 +40046 14868 +48439 14864 +22003 14861 +44802 14860 +37943 14860 +41919 14860 +45924 14859 +15002 14859 +47464 14858 +33842 14855 +38342 14853 +36235 14851 +35284 14851 +22442 14851 +45820 14851 +33136 14849 +33335 14848 +45718 14847 +38216 14843 +40726 14841 +49988 14840 +47736 14840 +49113 14839 +27 14839 +44734 14839 +25641 14833 +10669 14833 +45489 14830 +42068 14828 +47379 14826 +37162 14818 +44929 14818 +45071 14813 +40475 14806 +16683 14805 +16993 14802 +49336 14801 +29112 14796 +25222 14791 +18891 14789 +48732 14788 +47439 14783 +44037 14783 +44521 14782 +34294 14779 +15888 14778 +15091 14777 +19805 14777 +40646 14773 +37218 14772 +48328 14772 +48635 14771 +34984 14768 +49131 14766 +49800 14766 +41812 14765 +42012 14764 +34900 14764 +41255 14759 +22002 14758 +17212 14745 +25558 14743 +14158 14742 +16214 14739 +4937 14737 +32965 14736 +26323 14735 +46300 14734 +21191 14734 +26859 14727 +39162 14724 +43040 14724 +45079 14722 +11828 14719 +48232 14717 +32588 14713 +5024 14710 +45424 14708 +13947 14707 +18361 14705 +12378 14701 +40655 14698 +28205 14698 +27248 14697 +25443 14697 +16363 14697 +15642 14696 +27039 14695 +40008 14695 +36984 14693 +40576 14692 +38196 14686 +42544 14682 +29161 14680 +30709 14675 +41294 14675 +34909 14673 +48192 14664 +49021 14663 +36921 14662 +46857 14661 +48449 14660 +40411 14658 +50148 14656 +40938 14654 +39276 14653 +41543 14653 +50067 14652 +38661 14651 +34733 14647 +45596 14647 +37702 14640 +26752 14639 +34213 14638 +50228 14637 +46910 14637 +19030 14637 +46770 14636 +37284 14635 +45165 14635 +32198 14635 +19042 14631 +50091 14627 +44469 14619 +35423 14617 +47236 14612 +24465 14611 +46821 14611 +25608 14611 +18874 14608 +7701 14606 +43411 14599 +49463 14595 +42812 14590 +49649 14589 +42909 14585 +48768 14585 +47003 14585 +35384 14583 +29677 14580 +48252 14579 +43617 14579 +30128 14575 +48322 14574 +29644 14570 +17740 14568 +35626 14567 +47733 14567 +38168 14566 +10514 14564 +45574 14561 +16016 14559 +47440 14558 +40884 14555 +48775 14552 +30907 14552 +30619 14550 +4651 14549 +33532 14548 +44690 14546 +2942 14544 +31642 14542 +49092 14535 +41058 14534 +43069 14528 +16926 14527 +41347 14526 +49442 14525 +35400 14524 +49395 14523 +14810 14522 +46480 14519 +32286 14519 +27682 14518 +48810 14514 +49047 14511 +20906 14506 +17126 14505 +47176 14504 +31308 14501 +47121 14499 +42770 14497 +45576 14497 +24744 14497 +47053 14491 +48777 14488 +41508 14488 +49656 14487 +46633 14484 +48888 14483 +29823 14479 +30945 14478 +49049 14476 +47517 14474 +26745 14473 +13738 14470 +30056 14470 +32380 14469 +45426 14465 +27650 14464 +50031 14463 +38112 14461 +11221 14460 +16423 14458 +39662 14456 +33809 14454 +8008 14454 +17956 14453 +33322 14452 +44959 14449 +37898 14447 +33060 14445 +49299 14444 +48615 14441 +48751 14440 +21653 14437 +49497 14436 +8542 14430 +41811 14430 +27651 14428 +34329 14428 +44440 14426 +31746 14426 +39259 14426 +39968 14418 +36624 14417 +49421 14416 +48925 14415 +20317 14415 +46385 14414 +43001 14411 +20142 14409 +15439 14404 +25231 14402 +46876 14399 +46869 14399 +47744 14397 +41823 14393 +30975 14393 +32929 14390 +47719 14389 +41981 14389 +42406 14388 +37888 14387 +49477 14385 +49491 14385 +31849 14381 +3973 14380 +29709 14379 +26927 14379 +37535 14378 +33622 14377 +48132 14377 +48000 14375 +47032 14374 +46489 14373 +26904 14370 +45448 14366 +43448 14362 +31385 14362 +39562 14356 +19194 14355 +42298 14355 +26981 14354 +45431 14353 +44344 14353 +37912 14352 +38933 14349 +9778 14348 +46816 14346 +38532 14345 +33281 14344 +37670 14343 +48168 14340 +48072 14339 +9420 14338 +49540 14334 +27831 14331 +36614 14330 +31885 14329 +42681 14329 +45272 14327 +43496 14326 +41670 14326 +49917 14325 +40312 14322 +41467 14322 +20808 14321 +19199 14319 +24460 14317 +43781 14314 +32431 14312 +45620 14307 +46823 14305 +45473 14302 +48479 14301 +34165 14301 +40728 14300 +29562 14297 +38658 14296 +24886 14293 +32041 14286 +41339 14286 +32279 14283 +44438 14282 +34319 14282 +48942 14280 +10962 14277 +50005 14272 +9488 14271 +46457 14269 +48571 14267 +49815 14265 +39029 14264 +48546 14263 +47323 14263 +36152 14262 +24236 14261 +46558 14259 +27125 14256 +22114 14255 +35666 14253 +31870 14252 +22313 14249 +21964 14248 +31709 14248 +28339 14247 +37044 14245 +45895 14241 +21745 14239 +49359 14239 +19757 14237 +41507 14228 +12166 14226 +27271 14226 +43201 14225 +41988 14224 +28363 14223 +30781 14222 +2464 14222 +25820 14220 +30598 14218 +37056 14215 +13680 14215 +46712 14215 +26335 14214 +30561 14214 +21475 14214 +35009 14214 +30637 14205 +47381 14204 +44762 14203 +40967 14199 +37067 14199 +30708 14197 +46751 14196 +45619 14195 +44640 14194 +38512 14191 +32780 14191 +49072 14188 +41276 14184 +35728 14183 +48254 14179 +36774 14177 +21446 14175 +24331 14175 +21308 14174 +40029 14173 +46645 14173 +18789 14170 +40566 14169 +38199 14167 +42259 14165 +35166 14162 +34084 14159 +39941 14154 +45610 14150 +165 14149 +36934 14148 +29725 14146 +43679 14146 +8427 14140 +50221 14133 +38709 14131 +43464 14130 +41844 14127 +39363 14127 +12210 14126 +36439 14123 +42096 14121 +26616 14119 +11324 14118 +49274 14117 +39066 14116 +44676 14116 +12367 14114 +49538 14106 +49298 14105 +46370 14104 +20939 14099 +17957 14099 +45020 14097 +36422 14094 +24220 14093 +17429 14091 +14689 14090 +16607 14088 +6657 14080 +49605 14078 +47525 14076 +27846 14072 +38538 14071 +38917 14070 +4672 14066 +33676 14066 +45491 14065 +45937 14061 +45688 14060 +37569 14056 +27056 14054 +49892 14052 +46456 14052 +8279 14045 +34636 14045 +9560 14044 +32571 14044 +15249 14037 +39687 14037 +42190 14035 +33804 14031 +39963 14027 +39367 14026 +19986 14025 +45388 14023 +43745 14021 +26032 14019 +38033 14015 +18274 14015 +38414 14014 +46223 14014 +46670 14013 +20258 14008 +24996 14006 +44996 14004 +49937 14000 +11748 13999 +47033 13998 +46588 13997 +25832 13996 +47583 13994 +40189 13994 +25468 13993 +28934 13992 +43058 13989 +31119 13986 +39270 13984 +44540 13982 +49810 13981 +50124 13978 +42977 13977 +48887 13976 +44787 13976 +47743 13976 +38489 13972 +16827 13971 +41181 13966 +45412 13965 +32372 13964 +38474 13962 +11547 13961 +37917 13960 +36532 13959 +8933 13955 +34622 13954 +48785 13953 +25202 13953 +45579 13952 +41725 13952 +41011 13948 +23032 13947 +48270 13943 +47577 13941 +49938 13939 +31322 13937 +39141 13936 +31978 13935 +33210 13924 +13088 13919 +16624 13917 +32269 13912 +33640 13908 +36474 13906 +44007 13905 +5551 13905 +37158 13899 +37280 13898 +45323 13898 +48276 13898 +39546 13896 +25968 13896 +30076 13892 +48905 13890 +50107 13890 +45917 13890 +48208 13889 +31268 13880 +35452 13876 +43447 13876 +48013 13873 +9946 13873 +40216 13869 +43163 13869 +26052 13866 +46200 13866 +17055 13865 +45882 13854 +43954 13852 +36091 13851 +47593 13850 +46380 13849 +35192 13849 +30575 13849 +40443 13849 +28988 13843 +31200 13843 +35099 13841 +29971 13831 +50063 13830 +49923 13830 +29182 13829 +10331 13827 +31319 13827 +17244 13819 +22617 13818 +24923 13814 +49271 13814 +39629 13814 +19721 13809 +39319 13809 +10634 13808 +23459 13803 +36829 13802 +34981 13800 +45920 13800 +49916 13798 +34062 13794 +8409 13787 +24860 13787 +37160 13786 +47011 13785 +28296 13780 +10037 13778 +49903 13775 +46863 13772 +32844 13768 +27685 13767 +35920 13766 +35511 13765 +145 13758 +11797 13757 +26628 13754 +46974 13754 +8817 13753 +42817 13752 +37944 13750 +43402 13746 +24608 13743 +37894 13742 +36007 13741 +29160 13733 +43019 13728 +46224 13728 +23901 13726 +10406 13725 +42056 13723 +45328 13722 +40042 13720 +44279 13716 +26435 13715 +34045 13715 +43591 13715 +9711 13714 +32006 13714 +23633 13713 +3408 13712 +32401 13712 +38610 13709 +45945 13709 +23697 13708 +15235 13707 +16779 13706 +50004 13705 +19937 13704 +50115 13704 +11297 13701 +33361 13699 +33414 13695 +40628 13694 +45086 13692 +47197 13687 +42029 13687 +41568 13686 +16872 13686 +15404 13684 +26184 13682 +33150 13681 +38713 13674 +46003 13672 +31399 13672 +47343 13670 +35957 13666 +40036 13664 +31621 13662 +45884 13657 +45821 13657 +48351 13655 +19046 13655 +39079 13652 +32346 13651 +25188 13650 +48802 13645 +38655 13644 +23504 13643 +9026 13640 +41417 13640 +36848 13639 +26515 13635 +44984 13635 +25844 13634 +49223 13632 +43062 13632 +48340 13632 +23215 13629 +45287 13627 +19599 13624 +50018 13623 +33740 13621 +18185 13617 +35789 13616 +27378 13613 +46181 13611 +49090 13610 +8897 13610 +6585 13602 +17928 13602 +43873 13600 +25467 13600 +22748 13599 +45487 13598 +27576 13596 +18817 13596 +28075 13595 +37675 13593 +47079 13592 +37295 13591 +6526 13591 +45152 13590 +43202 13588 +26612 13584 +39529 13584 +8690 13584 +34352 13583 +8219 13581 +48440 13580 +39366 13578 +42369 13573 +41221 13572 +37369 13571 +43832 13568 +4841 13565 +22747 13562 +34844 13561 +43179 13561 +22031 13560 +45359 13560 +39754 13555 +43364 13555 +49240 13554 +49480 13552 +46716 13548 +30922 13548 +24697 13544 +49931 13543 +4408 13542 +39385 13541 +42295 13541 +22143 13540 +46299 13540 +45669 13540 +47549 13539 +2719 13538 +25716 13537 +16864 13536 +32383 13536 +43785 13534 +23636 13531 +11381 13530 +46597 13529 +41547 13527 +16881 13519 +34680 13519 +19243 13516 +26099 13515 +41452 13512 +33696 13512 +48846 13507 +41277 13501 +42093 13498 +32287 13498 +33216 13492 +39481 13491 +36797 13489 +34469 13489 +18968 13488 +44097 13482 +37441 13477 +48078 13476 +46810 13475 +8973 13475 +48207 13474 +21598 13472 +45361 13470 +38535 13470 +14171 13469 +19153 13469 +40754 13467 +36842 13463 +39494 13460 +27237 13457 +48210 13456 +23988 13456 +49818 13455 +41435 13452 +44394 13451 +27054 13450 +46963 13448 +48435 13448 +26064 13447 +22290 13446 +36537 13443 +40379 13443 +30556 13437 +28570 13437 +42244 13437 +41684 13435 +4208 13434 +4195 13431 +38824 13431 +16665 13427 +40040 13427 +28724 13425 +31106 13416 +34533 13415 +30526 13414 +16701 13413 +34088 13413 +26205 13411 +6002 13408 +45180 13408 +45855 13408 +41887 13405 +23579 13402 +20272 13400 +38729 13396 +46171 13385 +47373 13381 +46328 13377 +49928 13375 +49613 13373 +48317 13373 +28272 13371 +40226 13370 +31584 13369 +48374 13360 +49711 13358 +28093 13356 +25057 13356 +10502 13351 +6627 13351 +49020 13351 +47575 13350 +46713 13349 +25742 13349 +19965 13349 +9227 13348 +45330 13341 +36572 13341 +23784 13339 +43378 13334 +30590 13332 +20783 13329 +40601 13329 +23293 13327 +21063 13325 +18706 13324 +39819 13323 +1003 13321 +46038 13318 +49965 13312 +49203 13308 +43473 13307 +47821 13304 +43541 13303 +43129 13301 +48010 13300 +48770 13297 +27705 13295 +18649 13293 +47269 13291 +47138 13290 +42709 13287 +48445 13286 +31961 13285 +30491 13282 +44855 13281 +22282 13278 +47996 13278 +50098 13277 +31479 13275 +16719 13274 +23132 13270 +28665 13268 +48339 13266 +47811 13266 +5310 13264 +42561 13263 +48042 13262 +49345 13262 +147 13262 +30729 13259 +42460 13251 +17493 13245 +48735 13245 +47637 13240 +10004 13239 +33066 13238 +38389 13238 +45440 13236 +37934 13236 +33235 13234 +36635 13232 +43005 13231 +38064 13230 +33146 13229 +36751 13227 +25934 13226 +46826 13224 +18691 13221 +29391 13218 +12441 13215 +25628 13213 +24191 13212 +19767 13209 +7629 13208 +38144 13198 +36802 13198 +11200 13191 +43121 13190 +11497 13188 +31902 13188 +20344 13188 +45991 13185 +16228 13184 +161 13183 +12837 13182 +24951 13179 +42269 13178 +37654 13177 +40050 13177 +41265 13177 +48282 13175 +19298 13174 +40642 13170 +12638 13168 +47870 13167 +26371 13166 +36484 13163 +39684 13160 +25563 13159 +32937 13155 +35655 13154 +8986 13152 +33942 13148 +46053 13147 +15141 13147 +23898 13146 +34973 13145 +45589 13143 +41419 13141 +15553 13136 +2480 13134 +22916 13132 +48050 13132 +4076 13128 +33703 13125 +32374 13122 +32245 13120 +28835 13119 +44835 13119 +41625 13115 +45149 13114 +41464 13113 +48532 13111 +38861 13107 +43699 13103 +37328 13103 +34215 13102 +8944 13101 +23951 13101 +40177 13101 +17705 13100 +22184 13099 +48104 13096 +38437 13096 +48849 13095 +47161 13092 +47777 13092 +31544 13089 +40989 13088 +46689 13088 +33351 13087 +12541 13087 +38416 13085 +39260 13083 +46522 13080 +14360 13077 +43311 13075 +45322 13074 +29730 13067 +45387 13064 +35428 13063 +15303 13061 +24258 13060 +49998 13057 +36335 13056 +50036 13051 +29678 13051 +3631 13048 +30034 13047 +37638 13042 +48646 13042 +38233 13041 +31562 13040 +45470 13039 +48331 13036 +49569 13036 +41944 13035 +23283 13030 +24529 13028 +38280 13028 +38301 13027 +49881 13026 +49989 13024 +46772 13021 +15168 13021 +50137 13020 +36578 13020 +37471 13019 +35072 13018 +41730 13017 +45915 13017 +40779 13017 +19286 13013 +41616 13009 +47757 13007 +47238 13005 +45923 13005 +30395 13005 +42332 13004 +31315 13004 +17178 12995 +19511 12995 +28710 12993 +42512 12993 +32463 12990 +48502 12989 +10065 12986 +43211 12985 +39972 12981 +15160 12980 +23763 12973 +32822 12969 +50096 12969 +50154 12965 +2109 12963 +42531 12963 +17934 12960 +28216 12955 +44621 12954 +36456 12953 +47264 12952 +33382 12948 +19669 12944 +24223 12944 +9504 12942 +42136 12941 +47321 12941 +16680 12935 +26393 12934 +37368 12931 +24819 12928 +34193 12926 +40522 12925 +49635 12925 +44883 12922 +31249 12921 +38788 12921 +42072 12917 +43756 12917 +40039 12916 +36492 12916 +47622 12914 +34605 12909 +17804 12902 +47231 12899 +49966 12899 +38994 12898 +49812 12897 +38996 12897 +46316 12896 +35542 12894 +24159 12892 +7553 12892 +44445 12891 +31872 12888 +13615 12885 +36420 12882 +9571 12880 +41362 12879 +15384 12878 +38559 12871 +40763 12869 +38395 12867 +32845 12865 +48065 12865 +37559 12857 +29310 12856 +30754 12849 +35111 12849 +36206 12846 +40683 12842 +45946 12842 +36315 12840 +23343 12839 +36416 12839 +39371 12837 +5204 12836 +35596 12834 +31567 12830 +25867 12826 +46802 12825 +49776 12825 +44733 12822 +41022 12817 +48689 12815 +45883 12815 +13739 12808 +43306 12806 +35936 12805 +13410 12799 +45013 12792 +47839 12792 +23725 12789 +17569 12788 +43362 12779 +18771 12775 +32307 12774 +49671 12773 +32768 12769 +32397 12768 +41935 12765 +20907 12764 +35285 12764 +11064 12761 +9816 12761 +49479 12759 +20575 12757 +35487 12754 +36413 12753 +18602 12753 +49814 12752 +45203 12752 +16850 12751 +45885 12748 +34883 12743 +37971 12742 +41689 12741 +47407 12741 +33029 12725 +40951 12722 +47048 12722 +39913 12719 +31461 12717 +39485 12711 +14784 12708 +45815 12705 +37138 12704 +19246 12700 +36010 12698 +48385 12692 +27076 12691 +40533 12687 +43970 12686 +23850 12686 +45849 12683 +42538 12682 +45816 12678 +32505 12678 +49681 12677 +46482 12676 +35832 12676 +46475 12670 +11692 12669 +24477 12669 +47536 12668 +5974 12668 +42585 12660 +20654 12658 +22857 12658 +15776 12658 +33200 12658 +2410 12658 +48021 12656 +12708 12652 +13702 12652 +46817 12650 +50114 12649 +30779 12645 +32999 12644 +40731 12641 +48940 12637 +21065 12636 +23606 12633 +38394 12630 +33737 12624 +28787 12623 +22510 12620 +42147 12620 +48076 12619 +45333 12607 +35619 12606 +48619 12602 +50129 12600 +48752 12597 +6993 12596 +36967 12595 +13490 12594 +44241 12591 +23778 12588 +12102 12587 +13918 12583 +47081 12582 +28353 12581 +19758 12573 +28512 12572 +34236 12569 +42904 12568 +38950 12567 +33222 12564 +34054 12561 +35644 12559 +37122 12559 +44605 12555 +27131 12555 +34197 12550 +39049 12548 +34848 12546 +33244 12542 +25297 12541 +34719 12541 +33090 12541 +16995 12540 +19093 12538 +21886 12538 +46596 12537 +11124 12537 +36543 12536 +49268 12534 +44072 12528 +39571 12527 +15521 12526 +16838 12524 +38829 12522 +23216 12521 +34513 12520 +41270 12518 +46774 12516 +49275 12512 +20860 12511 +15690 12509 +41912 12506 +41509 12505 +8052 12505 +50194 12503 +44814 12502 +32898 12501 +13744 12501 +50056 12499 +38047 12496 +48978 12488 +37715 12487 +16722 12487 +32940 12482 +46740 12481 +20519 12481 +25206 12480 +9746 12476 +41173 12475 +40114 12474 +11213 12463 +37997 12461 +43969 12457 +41659 12454 +21488 12454 +49260 12453 +48033 12452 +47914 12452 +22872 12451 +30670 12449 +25484 12446 +38557 12444 +49343 12443 +32278 12443 +33403 12442 +17737 12442 +5397 12441 +31294 12439 +33268 12437 +43629 12434 +48508 12434 +34507 12433 +26999 12431 +23939 12430 +39758 12429 +50230 12427 +40127 12424 +48353 12423 +43794 12421 +39060 12421 +41500 12418 +29110 12418 +28050 12418 +39442 12415 +46723 12412 +31435 12411 +41030 12404 +13138 12403 +26033 12403 +45859 12400 +13838 12398 +30330 12394 +40371 12393 +19904 12392 +32828 12390 +34517 12390 +32765 12388 +37805 12382 +49360 12382 +25630 12377 +40346 12374 +47125 12368 +49732 12367 +22472 12366 +36836 12365 +21535 12365 +29257 12363 +42390 12362 +9023 12361 +41371 12359 +48919 12359 +27594 12358 +37892 12356 +47858 12354 +43669 12354 +28377 12346 +39719 12342 +49169 12341 +12667 12340 +43696 12338 +47425 12337 +44266 12337 +26462 12336 +49886 12336 +49367 12336 +11310 12334 +48059 12334 +9081 12333 +22614 12333 +42800 12325 +38011 12323 +40002 12323 +30688 12323 +39020 12322 +26144 12322 +41857 12321 +27897 12320 +35448 12317 +39889 12313 +46765 12310 +45063 12310 +12447 12310 +32500 12309 +13846 12306 +22762 12303 +12687 12301 +45383 12300 +40357 12298 +48178 12296 +31959 12296 +15969 12294 +37983 12291 +26125 12288 +44895 12287 +41571 12286 +48533 12279 +39176 12279 +44961 12276 +47087 12275 +21604 12275 +48842 12273 +24606 12268 +48401 12266 +33165 12263 +26709 12263 +35777 12263 +45394 12261 +18226 12257 +28743 12255 +36407 12252 +18299 12251 +43849 12248 +39237 12245 +36372 12244 +44029 12240 +11098 12239 +5835 12238 +38345 12236 +45880 12235 +29465 12235 +49449 12234 +41240 12233 +34024 12232 +42616 12232 +45076 12227 +45947 12227 +49153 12227 +36163 12220 +45482 12220 +17599 12218 +25458 12218 +39969 12211 +4253 12210 +49460 12208 +43205 12208 +39250 12206 +43196 12205 +43659 12204 +33100 12204 +43294 12204 +3569 12202 +41469 12202 +31373 12201 +47315 12199 +41105 12199 +38797 12193 +30547 12192 +28913 12191 +49755 12189 +44804 12188 +46433 12187 +28768 12187 +43519 12186 +6394 12186 +34653 12185 +43752 12183 +33857 12180 +29035 12180 +38857 12179 +40165 12179 +49106 12178 +48859 12177 +19968 12176 +47759 12174 +50218 12172 +48287 12169 +32004 12167 +17001 12167 +47392 12165 +45903 12153 +48224 12153 +39794 12149 +49016 12148 +28904 12147 +42019 12144 +44433 12143 +25568 12143 +38980 12141 +49152 12139 +45959 12135 +22936 12132 +44080 12124 +10020 12124 +48819 12122 +46262 12121 +5971 12113 +39294 12112 +20074 12110 +10739 12110 +4549 12107 +50053 12105 +25973 12105 +38832 12102 +28496 12102 +45904 12101 +45965 12097 +40553 12095 +31313 12095 +49526 12095 +48387 12093 +20556 12092 +47051 12092 +46986 12090 +33559 12086 +34055 12083 +32668 12073 +26018 12072 +8538 12072 +39436 12069 +43992 12068 +33793 12067 +42931 12066 +36261 12064 +24890 12063 +25097 12061 +4880 12061 +39356 12057 +48085 12057 +30431 12053 +35364 12051 +27299 12050 +32558 12050 +4567 12048 +32366 12048 +47617 12047 +32893 12046 +17360 12044 +37265 12042 +42217 12040 +48482 12035 +46771 12034 +31616 12032 +49387 12032 +49724 12031 +47110 12027 +48478 12025 +23705 12022 +33674 12021 +34049 12020 +23047 12017 +13379 12016 +28366 12014 +37516 12011 +27882 12010 +25504 12009 +49977 12008 +42500 12005 +17497 12004 +46971 12001 +26806 12000 +27815 12000 +40337 11998 +42756 11996 +31520 11995 +35751 11995 +49662 11994 +41958 11991 +35100 11991 +50057 11987 +49907 11987 +8240 11987 +47556 11985 +31789 11979 +43634 11977 +43345 11976 +48871 11975 +46920 11972 +47311 11970 +22897 11970 +7828 11969 +39715 11967 +30946 11967 +41048 11967 +15267 11964 +48481 11958 +41691 11956 +14670 11955 +34223 11955 +48251 11953 +45970 11953 +49972 11948 +48048 11948 +46987 11945 +45713 11943 +49664 11940 +25526 11938 +43249 11936 +36493 11934 +30887 11934 +43516 11927 +28169 11926 +34627 11926 +41451 11925 +31040 11924 +18438 11911 +37819 11908 +12084 11907 +30735 11903 +37310 11899 +3419 11898 +34591 11897 +48139 11896 +46024 11895 +3317 11895 +21485 11894 +35928 11894 +42277 11892 +26418 11892 +39746 11891 +27331 11890 +49054 11889 +33641 11889 +49667 11886 +50241 11882 +46205 11882 +40122 11880 +49100 11877 +48779 11876 +31943 11873 +16188 11873 +29124 11866 +44962 11862 +48391 11859 +42945 11853 +27535 11852 +44212 11851 +48758 11849 +39626 11849 +49098 11842 +41389 11836 +47771 11836 +41906 11827 +49826 11824 +44224 11821 +43947 11819 +40824 11817 +7132 11815 +41787 11814 +27738 11813 +23471 11813 +26493 11810 +39991 11808 +5206 11806 +48693 11800 +40866 11799 +43920 11797 +49321 11795 +42839 11794 +47615 11793 +48300 11792 +20568 11791 +47878 11790 +35713 11789 +21353 11788 +17597 11788 +28349 11785 +45656 11784 +33279 11783 +28409 11781 +49238 11781 +38217 11780 +43838 11780 +38673 11778 +37963 11777 +34399 11774 +17331 11774 +49740 11767 +33739 11766 +41697 11763 +6904 11758 +48203 11756 +20253 11755 +22201 11750 +44411 11749 +27490 11747 +38664 11744 +48068 11741 +32068 11740 +48668 11739 +33308 11736 +48616 11735 +47465 11732 +41842 11730 +46768 11729 +21244 11725 +41316 11724 +36137 11724 +47854 11722 +44479 11721 +46686 11721 +46575 11719 +47335 11719 +31312 11718 +46246 11714 +31030 11712 +44899 11712 +41796 11709 +42832 11709 +36687 11706 +15517 11703 +23765 11702 +24919 11700 +47998 11700 +48040 11697 +47088 11694 +48989 11690 +20274 11687 +28185 11687 +42919 11682 +41733 11682 +30203 11676 +21559 11675 +29158 11675 +47080 11675 +11250 11672 +6522 11671 +28400 11671 +22615 11668 +34656 11667 +46276 11656 +28117 11653 +11242 11646 +30025 11645 +47801 11644 +41397 11644 +35392 11644 +50122 11643 +50164 11643 +50244 11642 +35044 11642 +42686 11641 +49947 11640 +46793 11631 +27554 11630 +32580 11629 +36136 11628 +40656 11625 +45599 11624 +39806 11623 +12710 11620 +45380 11619 +11355 11619 +42393 11617 +46569 11614 +14228 11612 +44139 11604 +41502 11602 +40578 11601 +30655 11601 +20310 11595 +42552 11593 +35884 11592 +13877 11591 +49835 11590 +36807 11590 +40512 11589 +37549 11589 +46978 11587 +25034 11585 +40843 11584 +47532 11581 +11116 11577 +47103 11570 +29003 11566 +45065 11564 +44325 11564 +23187 11560 +47278 11560 +37598 11559 +45845 11558 +36631 11557 +41497 11557 +36230 11557 +42230 11556 +47520 11555 +27423 11554 +48507 11553 +47073 11553 +43429 11553 +29666 11552 +45873 11552 +27471 11549 +49205 11549 +41801 11548 +33926 11544 +12286 11544 +43821 11542 +3135 11538 +26377 11537 +37038 11535 +36716 11534 +48851 11533 +42827 11532 +45167 11531 +43750 11531 +17574 11525 +25373 11524 +27708 11524 +50211 11524 +37103 11520 +49210 11517 +44700 11517 +171 11513 +42112 11511 +31217 11511 +46404 11510 +44351 11510 +48394 11510 +25952 11510 +16211 11509 +40945 11507 +24127 11506 +32873 11506 +34398 11504 +35237 11499 +46032 11494 +26559 11490 +48032 11489 +47456 11482 +49502 11480 +25038 11480 +27923 11479 +35763 11479 +33428 11478 +17808 11477 +25896 11476 +34028 11473 +31299 11472 +25166 11471 +35204 11466 +46573 11465 +36409 11463 +49633 11460 +40805 11460 +24491 11459 +32589 11459 +36325 11451 +45793 11448 +26528 11448 +14630 11447 +45611 11443 +37947 11441 +42118 11440 +855 11436 +49405 11434 +36113 11432 +43800 11429 +47514 11425 +49103 11425 +37705 11425 +31948 11425 +18196 11424 +40976 11422 +44558 11421 +26894 11417 +40244 11416 +20789 11414 +35027 11411 +42267 11410 +49027 11408 +25877 11407 +49062 11406 +32866 11404 +32723 11401 +40969 11400 +39359 11398 +35417 11397 +44602 11396 +44933 11396 +45847 11395 +50038 11390 +27683 11389 +23525 11389 +46006 11388 +19729 11386 +24370 11386 +22460 11381 +42323 11379 +46244 11374 +16377 11373 +31013 11369 +40510 11363 +8641 11363 +29590 11360 +25908 11355 +40539 11350 +44342 11347 +25093 11346 +14244 11346 +43797 11346 +30891 11345 +46523 11342 +7345 11339 +25981 11337 +33780 11337 +31034 11337 +49962 11336 +30687 11335 +46483 11334 +32604 11329 +47754 11329 +36501 11328 +48932 11325 +17874 11324 +27754 11322 +17829 11319 +44454 11319 +41821 11317 +34388 11317 +33103 11317 +49534 11316 +18092 11315 +28406 11315 +45710 11311 +50120 11311 +8294 11309 +44982 11306 +49689 11305 +45569 11300 +43506 11298 +19331 11297 +20361 11297 +23900 11294 +45069 11292 +47700 11288 +29245 11284 +29625 11282 +25956 11280 +25196 11279 +10313 11275 +34788 11275 +48907 11271 +42039 11268 +47942 11267 +49773 11266 +47933 11265 +43143 11264 +33257 11259 +23517 11258 +42848 11256 +45686 11254 +47710 11252 +42469 11252 +26921 11250 +45057 11250 +44059 11249 +43917 11247 +44990 11247 +46063 11243 +25552 11240 +32236 11239 +47621 11238 +37405 11237 +50253 11233 +45637 11233 +13170 11233 +15988 11232 +31374 11231 +32503 11230 +38263 11228 +27865 11228 +39504 11224 +42044 11223 +31560 11221 +9129 11216 +34163 11214 +41311 11213 +35539 11213 +38295 11213 +41036 11211 +17903 11208 +50138 11202 +37039 11197 +39461 11196 +14011 11191 +46747 11190 +49585 11189 +22422 11187 +10738 11186 +36216 11185 +35543 11183 +17257 11182 +36287 11182 +21069 11181 +41142 11181 +39396 11177 +46375 11176 +24542 11173 +40204 11173 +35694 11173 +30308 11171 +41603 11170 +37837 11166 +11199 11165 +34978 11165 +45623 11165 +45641 11164 +49371 11164 +18827 11160 +8385 11159 +18001 11159 +44315 11158 +43399 11157 +47954 11157 +41877 11154 +38100 11143 +6410 11140 +45536 11139 +27530 11137 +34368 11131 +19739 11128 +33801 11122 +43535 11121 +36639 11118 +11958 11117 +40605 11110 +40916 11108 +49332 11107 +31317 11100 +31366 11092 +12836 11089 +23872 11088 +39522 11086 +12504 11085 +32157 11084 +49166 11080 +46667 11077 +8184 11072 +14467 11066 +8938 11063 +28054 11063 +16375 11061 +18312 11060 +39010 11060 +30411 11053 +39936 11052 +38470 11052 +34714 11051 +40865 11050 +46830 11048 +34878 11047 +49948 11047 +21359 11047 +30366 11044 +31808 11044 +32854 11044 +32124 11043 +29177 11043 +34971 11042 +42523 11032 +37232 11025 +41899 11024 +29425 11023 +46403 11016 +28015 11014 +47235 11013 +32296 11010 +11309 11010 +33999 11007 +35263 11004 +41163 11003 +42400 11002 +35836 10999 +19714 10998 +30196 10996 +38592 10994 +44248 10994 +2268 10990 +30417 10990 +2359 10988 +45052 10986 +47114 10985 +34729 10982 +46540 10982 +24879 10980 +46576 10979 +20364 10979 +16910 10975 +33288 10966 +27122 10966 +2325 10965 +46015 10963 +20475 10960 +39084 10959 +49365 10958 +41692 10948 +40734 10947 +4748 10946 +35753 10942 +36528 10941 +27744 10941 +13618 10939 +38778 10934 +26932 10934 +47123 10933 +46354 10931 +42751 10929 +33219 10928 +29231 10926 +44023 10925 +46161 10918 +29427 10917 +45117 10917 +13554 10912 +36350 10910 +48889 10910 +36225 10909 +5782 10909 +50121 10907 +31079 10906 +31587 10904 +45307 10903 +29490 10901 +37275 10899 +10237 10892 +42944 10889 +7242 10887 +43595 10886 +41102 10886 +48804 10885 +45771 10885 +31657 10884 +36164 10882 +33225 10882 +23177 10875 +42641 10874 +47371 10873 +36075 10870 +28427 10867 +45104 10863 +46687 10862 +19377 10860 +14229 10860 +23635 10860 +32422 10858 +38104 10858 +34768 10857 +16401 10857 +27301 10856 +46259 10855 +43460 10854 +31529 10853 +15270 10850 +23295 10849 +46931 10847 +29084 10844 +46399 10842 +44401 10840 +25559 10837 +21398 10836 +43649 10835 +48017 10833 +13845 10832 +42126 10830 +41646 10824 +36020 10824 +36055 10822 +4154 10819 +25958 10814 +49017 10813 +29375 10812 +24849 10811 +12117 10811 +27204 10810 +25609 10810 +45021 10809 +44101 10803 +42963 10800 +17035 10799 +32499 10798 +32087 10798 +34189 10797 +135 10793 +41384 10793 +15057 10789 +39426 10789 +22164 10787 +33738 10784 +46490 10778 +41920 10776 +30075 10775 +47368 10775 +47126 10774 +45591 10766 +31458 10763 +18464 10763 +28189 10761 +44713 10761 +36873 10759 +46425 10758 +45745 10755 +38432 10754 +23861 10752 +25233 10750 +39428 10749 +39932 10749 +47831 10748 +22774 10743 +26438 10742 +46153 10741 +27741 10736 +14758 10733 +6677 10731 +16474 10728 +43041 10728 +38517 10727 +43632 10726 +23296 10724 +37297 10724 +47707 10719 +9259 10715 +36835 10714 +17959 10708 +7377 10708 +27295 10707 +25825 10707 +46666 10704 +29812 10702 +48239 10698 +46145 10697 +47293 10695 +35422 10695 +18569 10689 +4644 10688 +20380 10687 +12563 10684 +28463 10683 +32896 10680 +41408 10679 +39892 10679 +34745 10678 +41825 10676 +50002 10672 +44412 10668 +31500 10666 +32053 10665 +24188 10665 +41766 10664 +46453 10663 +37032 10661 +46045 10660 +32977 10660 +28434 10658 +38023 10655 +43076 10652 +23882 10650 +41245 10650 +34145 10649 +42536 10642 +23479 10641 +20761 10641 +24266 10640 +17868 10640 +38993 10639 +27354 10638 +48432 10636 +41713 10636 +13687 10635 +36342 10634 +35317 10626 +34421 10625 +16710 10624 +41331 10623 +21469 10619 +34021 10619 +44484 10617 +45416 10616 +43137 10615 +33205 10613 +48594 10612 +49143 10610 +20362 10610 +49610 10602 +30207 10601 +41355 10599 +18962 10598 +36423 10596 +33095 10596 +26176 10593 +38363 10592 +34811 10591 +33455 10584 +1250 10583 +49958 10580 +45357 10579 +28535 10574 +38969 10572 +39353 10569 +29990 10568 +30163 10565 +46835 10564 +25258 10563 +36381 10562 +19148 10558 +32936 10557 +42104 10546 +48821 10544 +13492 10543 +49107 10541 +40292 10537 +15769 10536 +43327 10535 +46720 10532 +41361 10531 +43522 10530 +42303 10528 +42559 10527 +21136 10526 +49519 10523 +42306 10517 +33727 10515 +40178 10513 +46648 10506 +39948 10506 +11631 10506 +42808 10505 +38910 10502 +40961 10502 +30116 10501 +33581 10497 +40686 10497 +28834 10496 +49993 10495 +38678 10493 +48235 10488 +25617 10486 +30823 10477 +27932 10477 +42233 10470 +13725 10469 +42829 10468 +47504 10466 +13168 10459 +29434 10458 +29824 10457 +31519 10457 +34239 10453 +46812 10449 +45776 10441 +5632 10440 +18331 10439 +16025 10436 +47394 10436 +41831 10435 +43382 10435 +47784 10432 +39499 10431 +16317 10427 +11507 10417 +34957 10416 +28586 10412 +47659 10411 +22684 10404 +38670 10404 +49821 10394 +40937 10392 +48992 10391 +20620 10389 +25561 10382 +6636 10380 +27071 10377 +21399 10377 +32026 10375 +13811 10374 +25599 10374 +11008 10373 +11209 10373 +32574 10369 +47281 10368 +40872 10366 +41986 10363 +17602 10360 +34853 10358 +35653 10357 +47310 10356 +34734 10344 +18980 10343 +31226 10340 +43215 10339 +44944 10339 +41133 10337 +12355 10336 +42902 10336 +23920 10334 +35796 10329 +37311 10325 +44876 10325 +31771 10323 +18766 10320 +17398 10320 +4613 10320 +30815 10319 +26480 10317 +3302 10313 +44356 10312 +9120 10312 +25008 10312 +47800 10311 +46865 10310 +35478 10306 +48169 10298 +25198 10293 +8813 10291 +24541 10286 +25015 10286 +30228 10285 +37908 10284 +48601 10284 +15858 10282 +20452 10281 +7984 10281 +14793 10280 +33767 10277 +41910 10276 +41635 10276 +47179 10274 +16672 10271 +34542 10270 +20166 10266 +38449 10265 +46437 10262 +29836 10260 +29127 10258 +29011 10257 +50094 10255 +41319 10254 +44403 10253 +18888 10250 +36827 10245 +27438 10243 +35811 10240 +20692 10240 +32577 10239 +43358 10235 +38357 10234 +29967 10234 +30532 10233 +48915 10232 +29279 10232 +12099 10231 +38490 10230 +47790 10230 +23650 10228 +40148 10227 +32506 10225 +9125 10219 +43372 10215 +25361 10214 +45268 10210 +32011 10207 +26568 10207 +21297 10203 +42697 10201 +23896 10201 +23813 10198 +39284 10196 +29389 10196 +2729 10196 +43173 10195 +44988 10193 +34651 10188 +9140 10186 +43823 10185 +37867 10182 +43031 10181 +35762 10180 +38785 10177 +42302 10177 +41735 10176 +21708 10173 +19653 10169 +30078 10168 +46543 10167 +46364 10164 +26059 10163 +31262 10163 +44420 10163 +31653 10161 +26489 10159 +48676 10158 +43401 10154 +5578 10151 +37555 10150 +32750 10146 +20696 10142 +37442 10139 +16663 10139 +39884 10139 +40630 10139 +17183 10139 +40119 10138 +28673 10137 +32864 10135 +46445 10134 +49765 10131 +48436 10130 +18557 10129 +22903 10129 +42577 10128 +22708 10127 +33450 10126 +2335 10124 +46334 10122 +36120 10118 +44710 10112 +42219 10112 +39040 10108 +47635 10107 +40783 10103 +35048 10101 +47167 10099 +38955 10099 +19448 10098 +44894 10092 +27488 10091 +33243 10090 +42981 10089 +16848 10088 +39648 10087 +43012 10086 +47262 10086 +21130 10086 +46204 10085 +2639 10084 +45725 10082 +38037 10081 +23712 10080 +34555 10080 +49004 10077 +34175 10075 +23406 10074 +26453 10072 +13567 10072 +49710 10071 +16321 10071 +46934 10067 +36400 10065 +14938 10057 +37786 10056 +33747 10056 +35541 10054 +41510 10052 +47375 10051 +26164 10051 +26259 10049 +27160 10047 +49678 10047 +35203 10045 +45638 10045 +168 10045 +40880 10044 +32986 10034 +32498 10029 +32273 10026 +24052 10025 +50130 10022 +36081 10022 +41880 10020 +48002 10017 +34946 10016 +21970 10012 +8626 10011 +31795 10010 +44187 10010 +48816 10008 +18374 10008 +23828 10007 +48174 10007 +32305 10000 +46178 9992 +49738 9991 +22294 9986 +34362 9978 +31494 9968 +45278 9968 +43530 9966 +136 9966 +38166 9965 +45778 9963 +22227 9962 +50022 9958 +38284 9955 +36652 9953 +39698 9952 +50039 9948 +17904 9947 +46654 9946 +19309 9946 +26952 9939 +38803 9938 +49229 9935 +46148 9934 +8951 9934 +29964 9931 +45018 9929 +35114 9929 +45348 9928 +33526 9925 +44280 9924 +48146 9923 +27467 9915 +44362 9915 +26449 9913 +30579 9913 +24937 9906 +37023 9905 +49450 9900 +46372 9899 +31694 9898 +19081 9895 +40006 9891 +36268 9888 +40082 9885 +42109 9881 +18503 9880 +47636 9878 +48220 9877 +34657 9876 +18491 9874 +18354 9871 +44696 9869 +18324 9868 +31628 9867 +50152 9865 +44347 9865 +38266 9864 +39169 9863 +42985 9861 +22920 9861 +48672 9857 +20778 9853 +28995 9853 +11253 9847 +11208 9846 +48708 9838 +41312 9835 +9522 9833 +24614 9830 +38247 9829 +45740 9829 +31281 9826 +40994 9823 +11893 9823 +38924 9822 +45444 9821 +22399 9821 +43061 9821 +16419 9821 +48510 9820 +25586 9817 +38420 9816 +35416 9816 +46805 9815 +41192 9814 +17447 9808 +48832 9807 +14989 9802 +19628 9802 +32447 9801 +40555 9797 +48605 9795 +14876 9794 +41474 9793 +34588 9791 +49429 9791 +49091 9790 +23124 9790 +4557 9789 +23478 9788 +23274 9785 +28008 9784 +44616 9782 +30358 9779 +49675 9776 +48038 9775 +31646 9775 +33369 9773 +5484 9770 +44287 9768 +3467 9765 +35812 9764 +41952 9761 +38516 9756 +40491 9753 +30763 9751 +25843 9751 +49677 9748 +50044 9746 +42698 9745 +4115 9745 +43601 9743 +32836 9739 +42282 9739 +39213 9738 +34960 9737 +26132 9735 +19598 9734 +30470 9732 +31482 9730 +151 9730 +41897 9727 +42593 9722 +35787 9716 +48551 9715 +50111 9708 +35312 9708 +16496 9705 +35226 9705 +48035 9703 +27092 9702 +28026 9701 +49060 9700 +24108 9700 +26488 9694 +32336 9693 +37741 9690 +37534 9687 +42860 9683 +48899 9682 +39729 9680 +7436 9676 +24874 9675 +49576 9675 +24794 9670 +44519 9670 +49204 9669 +11838 9669 +25105 9669 +42595 9665 +38514 9665 +20134 9664 +47644 9661 +27236 9660 +27016 9659 +33113 9659 +49415 9656 +49705 9654 +25689 9654 +23954 9652 +43467 9650 +47698 9647 +19124 9647 +31197 9645 +24762 9645 +27104 9637 +43263 9635 +44906 9635 +49284 9634 +16174 9628 +34348 9628 +46973 9627 +31324 9625 +20628 9622 +24251 9621 +47376 9619 +10132 9619 +35850 9611 +3531 9609 +32976 9606 +24593 9606 +49283 9604 +41866 9604 +32301 9603 +34100 9603 +37450 9596 +49548 9592 +33392 9591 +22738 9590 +26813 9590 +41682 9588 +38004 9587 +49136 9585 +34067 9585 +19352 9584 +18224 9584 +21180 9583 +31303 9583 +4211 9581 +28798 9580 +45066 9580 +37608 9578 +33460 9578 +32582 9577 +43698 9577 +41485 9572 +1651 9569 +11388 9569 +42448 9559 +39432 9557 +13731 9557 +44378 9555 +35827 9555 +20035 9546 +50061 9542 +5553 9541 +33823 9536 +27954 9533 +45973 9533 +34544 9532 +49616 9532 +46449 9531 +5862 9528 +24445 9526 +31337 9524 +42742 9524 +33959 9520 +46906 9519 +45697 9518 +39876 9515 +43352 9515 +650 9513 +24735 9513 +48662 9510 +43304 9507 +31361 9507 +42138 9505 +46149 9504 +21483 9504 +24858 9499 +43620 9494 +22439 9493 +42579 9492 +19389 9491 +43830 9490 +30073 9489 +43909 9489 +44242 9481 +8238 9481 +27812 9480 +34073 9465 +33049 9460 +46170 9458 +41942 9457 +32983 9456 +24427 9454 +28987 9453 +3295 9452 +48141 9451 +10258 9450 +20878 9448 +16154 9438 +31673 9431 +43767 9430 +42667 9429 +24371 9429 +27691 9426 +49584 9425 +14929 9419 +43111 9417 +30515 9417 +47526 9415 +32248 9411 +7687 9407 +8256 9407 +32778 9406 +47986 9395 +15764 9391 +13976 9391 +37447 9383 +32071 9378 +44393 9376 +18880 9371 +19626 9370 +38072 9364 +13955 9361 +47017 9359 +29845 9358 +2653 9355 +49328 9353 +8446 9351 +36969 9349 +41576 9346 +39639 9345 +18762 9343 +49651 9339 +33614 9336 +9789 9327 +42034 9326 +27981 9326 +40454 9321 +26185 9321 +39433 9321 +28600 9321 +44902 9321 +48703 9319 +49269 9316 +18000 9316 +43505 9316 +18378 9308 +43254 9307 +48011 9307 +30177 9306 +48490 9301 +49215 9301 +42591 9299 +42830 9297 +37541 9296 +35732 9295 +9270 9295 +49022 9292 +22344 9289 +34870 9286 +48175 9285 +35567 9284 +31923 9284 +2015 9283 +36650 9282 +49043 9281 +45708 9279 +17294 9278 +12775 9278 +39587 9278 +24781 9277 +36482 9276 +38501 9276 +44858 9274 +38382 9272 +27387 9267 +33312 9267 +50059 9265 +25007 9265 +37974 9264 +50184 9261 +41176 9260 +48795 9258 +34713 9258 +44746 9256 +45836 9255 +50104 9246 +39581 9245 +11579 9243 +46031 9240 +49793 9240 +30611 9232 +36992 9230 +47060 9225 +25031 9224 +34508 9222 +45595 9218 +48237 9216 +33003 9215 +49160 9213 +45896 9211 +48137 9209 +50037 9208 +43476 9207 +35624 9206 +37497 9204 +39297 9202 +9540 9201 +33873 9198 +49171 9197 +39424 9196 +41156 9191 +20520 9190 +23484 9187 +47502 9187 +29071 9181 +41501 9180 +14079 9179 +45742 9177 +39022 9177 +4926 9172 +31699 9171 +10842 9170 +42821 9169 +40803 9169 +46998 9167 +8669 9167 +27493 9166 +38720 9165 +44079 9164 +16745 9161 +47650 9156 +37781 9153 +27302 9150 +46130 9149 +15167 9143 +49955 9141 +43755 9140 +47605 9138 +47786 9138 +28104 9138 +34257 9136 +37037 9134 +16128 9134 +48780 9133 +48865 9130 +38136 9124 +32988 9120 +45461 9110 +44839 9108 +43594 9107 +48936 9105 +43681 9102 +41937 9096 +47845 9094 +48565 9092 +26513 9086 +9163 9084 +46363 9083 +38694 9081 +47415 9080 +29859 9080 +28432 9080 +14626 9077 +31539 9076 +49361 9072 +37219 9071 +48382 9069 +48713 9067 +24620 9065 +22811 9061 +42930 9061 +43135 9060 +38755 9060 +44975 9058 +44357 9057 +45597 9057 +19302 9057 +39104 9055 +47808 9053 +44788 9050 +29196 9049 +41054 9043 +39445 9042 +37950 9039 +32255 9038 +33954 9036 +44164 9034 +30961 9031 +49333 9031 +45266 9031 +44597 9030 +42532 9027 +23150 9026 +48407 9026 +26805 9026 +32809 9023 +1260 9023 +43301 9022 +48671 9020 +34043 9019 +40998 9017 +30104 9015 +28489 9011 +48163 9005 +49084 9003 +24864 8996 +47250 8993 +40918 8992 +40213 8991 +40342 8989 +31962 8988 +50246 8988 +49539 8986 +33510 8984 +28832 8983 +37591 8983 +50201 8982 +26483 8975 +30951 8972 +31571 8972 +34879 8967 +2020 8966 +10002 8964 +35299 8963 +23031 8960 +10124 8957 +12982 8952 +40061 8951 +32021 8949 +40220 8948 +46261 8948 +44909 8946 +49863 8943 +36702 8942 +32233 8940 +14632 8939 +40921 8939 +33395 8934 +7339 8929 +49545 8927 +39627 8926 +29233 8924 +40729 8923 +31131 8919 +30294 8918 +40436 8917 +28741 8916 +38804 8915 +24567 8913 +30978 8907 +35477 8903 +14694 8902 +30369 8900 +45390 8897 +32515 8895 +21975 8892 +27755 8892 +44687 8891 +41170 8887 +41757 8884 +33278 8883 +37130 8881 +27084 8880 +42567 8877 +50255 8875 +18414 8874 +37875 8874 +38825 8873 +35748 8871 +47035 8868 +26080 8862 +14571 8856 +39031 8848 +30053 8847 +48515 8846 +50171 8845 +37561 8841 +46870 8841 +45267 8841 +37326 8839 +24915 8836 +19929 8832 +21410 8831 +50003 8825 +39050 8825 +31223 8824 +44702 8824 +41374 8822 +37587 8821 +22317 8820 +35411 8817 +24565 8816 +21587 8815 +36965 8813 +49034 8811 +42678 8807 +36609 8804 +48826 8802 +35194 8802 +42033 8801 +33563 8796 +23230 8795 +27090 8794 +47999 8793 +47724 8792 +31484 8791 +49161 8787 +44623 8786 +48599 8783 +30309 8782 +33889 8780 +27077 8778 +43343 8775 +30459 8775 +39583 8773 +4109 8772 +35349 8767 +45410 8766 +29530 8765 +29706 8762 +29568 8760 +44751 8759 +43604 8759 +49065 8759 +30333 8758 +17772 8757 +39657 8756 +35224 8754 +31426 8752 +35450 8752 +22731 8750 +23919 8746 +42597 8744 +38693 8742 +48725 8740 +45893 8737 +32971 8735 +25780 8733 +38949 8732 +46071 8727 +29970 8725 +6418 8719 +7887 8719 +48190 8719 +17312 8717 +10806 8715 +49891 8715 +48951 8715 +17994 8714 +7566 8712 +30178 8712 +49693 8710 +39596 8710 +20652 8707 +47344 8701 +1019 8701 +35636 8699 +15860 8698 +25145 8694 +10555 8692 +2924 8692 +19864 8690 +33491 8690 +48879 8690 +26662 8685 +41152 8683 +16335 8681 +48410 8681 +14373 8679 +48126 8675 +43093 8673 +50110 8672 +33493 8666 +49797 8665 +43940 8665 +8651 8657 +48476 8655 +29238 8652 +28348 8649 +44556 8645 +7682 8639 +40254 8638 +38844 8634 +29609 8634 +37092 8629 +39665 8625 +42781 8624 +26743 8621 +31744 8620 +49715 8619 +29363 8618 +46655 8612 +19120 8612 +6262 8611 +45634 8611 +18974 8611 +16763 8611 +32348 8610 +34171 8608 +16130 8607 +48400 8605 +39401 8603 +28809 8601 +49658 8599 +13525 8596 +10376 8593 +26385 8592 +39075 8592 +41686 8591 +47185 8590 +50252 8589 +40963 8587 +40725 8585 +34381 8584 +46068 8584 +17088 8582 +49133 8582 +13366 8580 +35749 8579 +38582 8578 +40552 8578 +46211 8577 +28128 8577 +21700 8576 +21448 8575 +30866 8574 +40217 8573 +40607 8571 +21746 8568 +17394 8563 +35316 8559 +44529 8557 +6022 8553 +6963 8553 +26169 8553 +32943 8551 +38480 8549 +41652 8548 +37746 8547 +41876 8547 +49543 8543 +41023 8542 +40205 8540 +28495 8537 +35116 8537 +30513 8536 +47806 8536 +35898 8534 +12257 8533 +48877 8532 +43877 8529 +36663 8528 +47497 8527 +37366 8526 +33360 8524 +25215 8520 +37877 8520 +30635 8517 +35986 8511 +29790 8510 +38984 8508 +33346 8506 +31803 8505 +39307 8505 +30039 8502 +17971 8501 +16447 8500 +34363 8499 +30634 8498 +14210 8496 +40166 8494 +20543 8492 +37049 8490 +22222 8489 +1331 8483 +36414 8483 +47511 8483 +48897 8481 +37724 8477 +26177 8476 +47862 8475 +20500 8474 +26627 8472 +47072 8467 +44157 8467 +46059 8466 +50097 8464 +4218 8463 +40746 8463 +29321 8463 +27161 8461 +25387 8458 +47178 8457 +46650 8454 +49785 8448 +43180 8447 +41814 8444 +39274 8443 +39464 8440 +36790 8439 +40172 8438 +38093 8438 +46707 8437 +46555 8435 +29908 8430 +44901 8427 +34500 8423 +44939 8421 +14875 8419 +17452 8413 +35994 8412 +19991 8409 +49221 8409 +49261 8409 +23248 8408 +49876 8407 +42246 8405 +48495 8398 +24146 8397 +28636 8396 +50245 8395 +42372 8393 +31092 8393 +28929 8391 +38958 8386 +28501 8386 +48333 8386 +48923 8385 +41762 8384 +12561 8380 +39051 8380 +34094 8379 +39234 8378 +25676 8376 +44863 8376 +39605 8373 +35566 8372 +18800 8370 +21373 8370 +44873 8370 +29694 8366 +46304 8366 +28642 8363 +38108 8362 +35179 8360 +38401 8359 +39782 8357 +32352 8356 +35886 8354 +18924 8354 +32491 8353 +45979 8351 +43164 8349 +43430 8348 +46915 8346 +39014 8342 +32194 8337 +45258 8333 +35125 8331 +38983 8331 +33194 8330 +37372 8329 +13511 8328 +23377 8325 +31330 8324 +29034 8323 +19793 8321 +34836 8321 +44482 8319 +39218 8310 +38334 8305 +40341 8304 +40239 8300 +27005 8298 +32061 8297 +35313 8296 +27442 8293 +38800 8291 +21564 8291 +35610 8290 +47345 8290 +35252 8287 +47245 8283 +23227 8279 +45464 8278 +34075 8275 +34306 8273 +9237 8272 +40828 8268 +41518 8267 +12190 8261 +33141 8259 +30414 8259 +43219 8259 +46537 8256 +41771 8253 +34777 8249 +39400 8247 +39335 8247 +44669 8246 +45561 8242 +31652 8239 +35900 8238 +40482 8234 +46214 8229 +37181 8227 +34648 8227 +41743 8226 +3358 8225 +33240 8223 +37231 8222 +40952 8221 +17050 8221 +44308 8220 +33722 8217 +13250 8216 +42731 8216 +37253 8215 +34220 8213 +25442 8212 +39228 8208 +3485 8202 +35580 8202 +26469 8202 +19261 8201 +35215 8201 +45236 8200 +41694 8198 +42663 8193 +28196 8190 +19668 8187 +27726 8186 +33313 8184 +13743 8183 +40484 8183 +25817 8179 +42344 8170 +26751 8167 +46104 8163 +21893 8163 +28316 8162 +10621 8158 +46513 8156 +16553 8156 +28859 8156 +44258 8155 +45796 8150 +34367 8149 +14706 8145 +7581 8140 +34913 8140 +46207 8139 +30725 8139 +47217 8136 +32821 8135 +6111 8134 +23119 8133 +639 8131 +39914 8128 +23893 8127 +31979 8123 +2076 8121 +39057 8118 +44837 8116 +47685 8111 +42861 8111 +46135 8109 +40407 8107 +36156 8102 +49031 8099 +49044 8095 +4824 8092 +49316 8091 +43710 8088 +38897 8087 +42383 8085 +2459 8082 +16271 8082 +32125 8082 +22838 8078 +33891 8076 +39227 8075 +21188 8075 +42229 8075 +37659 8075 +41420 8074 +42181 8072 +41873 8070 +19749 8070 +44783 8068 +33497 8068 +30476 8068 +3766 8068 +45270 8067 +42950 8065 +48504 8061 +20722 8060 +41798 8059 +25078 8058 +49648 8058 +26841 8051 +45192 8051 +44606 8051 +40708 8051 +17327 8049 +6722 8048 +47477 8045 +44449 8044 +48087 8042 +29631 8041 +29701 8040 +43546 8039 +37596 8035 +42842 8035 +48121 8035 +28398 8034 +28473 8034 +22519 8033 +35498 8032 +39296 8031 +25811 8029 +40612 8027 +43982 8027 +35479 8027 +29938 8026 +35565 8024 +24021 8022 +44380 8021 +15400 8011 +12977 8010 +23946 8007 +45875 8007 +38544 8003 +35943 8003 +39241 8002 +23620 7999 +38447 7998 +11494 7998 +33615 7996 +36950 7990 +22882 7987 +25601 7986 +34983 7986 +30313 7984 +26362 7983 +19053 7982 +20988 7974 +47723 7974 +47306 7973 +25120 7970 +25668 7969 +46284 7968 +16207 7963 +36211 7959 +27866 7957 +18267 7950 +40550 7947 +35612 7946 +24966 7941 +46084 7939 +35888 7938 +36018 7936 +43436 7935 +16411 7932 +8614 7928 +41620 7925 +38676 7923 +13352 7919 +40374 7915 +19119 7915 +33711 7911 +49413 7910 +25938 7910 +35013 7909 +32594 7909 +42338 7905 +24708 7904 +30108 7902 +31141 7893 +44601 7891 +35937 7886 +48589 7880 +18036 7877 +40079 7877 +46014 7875 +48620 7874 +43689 7874 +46680 7867 +43618 7866 +40483 7866 +31762 7864 +35070 7855 +46203 7850 +44800 7846 +18445 7844 +16124 7841 +47298 7840 +34389 7839 +41032 7838 +20147 7836 +39170 7830 +26437 7828 +48699 7828 +36837 7824 +25513 7823 +38882 7821 +20287 7816 +28480 7814 +15512 7813 +45772 7811 +49864 7811 +47409 7805 +43730 7804 +49578 7802 +47889 7792 +38587 7792 +14763 7790 +49158 7789 +32640 7788 +48890 7773 +38978 7769 +37410 7766 +1606 7765 +38638 7763 +8934 7763 +45532 7762 +32901 7760 +49833 7760 +32981 7759 +24598 7755 +33567 7753 +43655 7745 +39301 7743 +45154 7740 +14575 7736 +26654 7736 +34867 7735 +45081 7734 +44791 7734 +29566 7734 +49467 7732 +29261 7727 +29531 7725 +44567 7725 +34447 7725 +29271 7724 +27732 7723 +46842 7720 +35991 7718 +26072 7715 +32155 7713 +21833 7710 +30027 7709 +29821 7702 +38453 7702 +25537 7701 +38293 7700 +17633 7699 +44196 7698 +43397 7698 +90 7697 +42452 7694 +18718 7694 +476 7692 +50250 7690 +32458 7690 +27245 7689 +49302 7685 +36990 7675 +33377 7672 +35929 7670 +49493 7667 +48844 7664 +38454 7661 +33765 7661 +21121 7659 +33952 7659 +48288 7654 +32888 7654 +44061 7653 +42942 7644 +48762 7640 +44503 7640 +41490 7640 +27261 7638 +24200 7636 +37025 7629 +30608 7627 +46616 7625 +31851 7622 +36899 7622 +25211 7621 +35319 7621 +24640 7619 +27273 7616 +37767 7616 +26201 7613 +36077 7613 +43642 7610 +45681 7608 +29888 7604 +40767 7603 +35574 7602 +34695 7601 +15263 7601 +49930 7600 +17373 7599 +41127 7594 +32875 7594 +37957 7593 +48027 7589 +33574 7585 +41628 7583 +19959 7582 +19256 7580 +25039 7576 +11730 7576 +167 7576 +48494 7574 +30394 7567 +2532 7564 +33249 7562 +37548 7561 +37189 7558 +36868 7554 +38288 7553 +34866 7551 +48836 7547 +37362 7542 +50047 7542 +25831 7540 +49632 7540 +47663 7537 +25354 7537 +15023 7535 +34270 7535 +49279 7534 +38767 7533 +48570 7527 +39103 7526 +29676 7523 +42940 7522 +45746 7519 +41274 7517 +49454 7513 +49942 7509 +46075 7509 +41025 7508 +27419 7507 +24328 7507 +32602 7506 +48549 7505 +11157 7501 +14566 7497 +16813 7496 +37384 7492 +26425 7491 +41681 7490 +38002 7487 +39087 7485 +36008 7484 +8767 7479 +10268 7479 +6879 7477 +50207 7476 +39427 7473 +48561 7472 +44644 7470 +26652 7468 +27421 7467 +20998 7466 +12851 7465 +38391 7464 +49121 7464 +37047 7459 +35244 7454 +36840 7450 +38231 7449 +4857 7443 +25977 7441 +31129 7436 +42455 7436 +40548 7434 +39225 7433 +30328 7432 +12951 7431 +21004 7429 +50126 7425 +30659 7424 +49580 7424 +38415 7423 +36295 7422 +31508 7421 +29668 7414 +11935 7407 +15205 7407 +37065 7406 +26003 7404 +39230 7403 +30281 7401 +41356 7400 +38837 7396 +36500 7394 +16229 7389 +34204 7389 +41206 7388 +38141 7386 +10479 7385 +44920 7382 +34554 7381 +46126 7375 +49970 7373 +30306 7373 +42670 7373 +40842 7370 +26696 7370 +2595 7366 +41341 7365 +44742 7362 +47213 7359 +38143 7359 +45509 7354 +13773 7354 +18806 7352 +6976 7352 +41570 7349 +35255 7348 +32441 7347 +30240 7344 +44374 7340 +39197 7334 +46604 7333 +33161 7330 +12631 7327 +35590 7321 +45282 7319 +34586 7313 +21620 7313 +18408 7311 +21357 7310 +34953 7306 +18161 7306 +49686 7300 +14624 7297 +33383 7296 +15330 7295 +38482 7291 +31272 7290 +34926 7287 +35570 7287 +23958 7286 +44563 7284 +26953 7280 +37365 7279 +49869 7277 +31493 7275 +49492 7273 +24042 7273 +48799 7273 +43892 7267 +48451 7265 +37834 7261 +42296 7260 +28732 7259 +17563 7259 +32771 7256 +7641 7254 +48272 7254 +24259 7253 +7426 7252 +15582 7250 +28923 7247 +37404 7246 +29294 7245 +47133 7244 +40506 7242 +32949 7236 +45898 7234 +41079 7233 +20086 7230 +47516 7229 +36284 7228 +45627 7225 +34048 7224 +13498 7223 +40368 7219 +5881 7218 +37504 7216 +44724 7216 +29006 7215 +46728 7213 +42006 7213 +49572 7209 +49712 7206 +23843 7202 +49008 7201 +26747 7199 +42771 7198 +21215 7195 +41931 7194 +19573 7193 +32105 7192 +49379 7189 +20468 7188 +44679 7182 +37821 7176 +40097 7176 +34947 7176 +28039 7175 +44292 7173 +11861 7172 +25294 7171 +7790 7168 +20740 7167 +25106 7161 +35119 7159 +39156 7156 +39478 7155 +38020 7154 +44812 7152 +42466 7148 +37697 7147 +36648 7145 +46251 7145 +24396 7145 +39861 7142 +27447 7141 +18065 7136 +1054 7134 +25306 7131 +44360 7130 +9833 7126 +48593 7125 +31400 7124 +47937 7123 +48582 7122 +25919 7121 +37139 7120 +37740 7119 +20438 7118 +3680 7114 +41208 7112 +33433 7105 +43894 7104 +28721 7102 +30542 7101 +44387 7094 +16950 7091 +46933 7089 +47585 7088 +43404 7087 +36772 7086 +27477 7084 +33010 7083 +32645 7079 +8243 7078 +41488 7077 +24583 7074 +8700 7067 +47294 7066 +42317 7064 +43827 7063 +30769 7063 +50025 7059 +36822 7058 +48631 7054 +17664 7051 +39881 7050 +49144 7049 +48485 7040 +15742 7040 +19746 7039 +29205 7039 +36883 7037 +45853 7034 +17679 7032 +31755 7030 +36881 7027 +32797 7025 +43878 7022 +42550 7022 +37945 7021 +27958 7021 +26629 7018 +41499 7015 +45166 7003 +45336 7001 +39839 6998 +9770 6994 +44054 6993 +41298 6992 +23493 6991 +31968 6990 +44227 6987 +19312 6987 +8191 6983 +43320 6982 +34379 6981 +49516 6981 +36803 6976 +32550 6976 +44759 6975 +27536 6973 +40139 6971 +36526 6970 +31919 6969 +23228 6968 +49095 6968 +40988 6967 +33357 6966 +22621 6966 +43387 6964 +29492 6964 +18994 6963 +46546 6963 +17733 6963 +43441 6960 +49828 6955 +10773 6955 +2315 6955 +46975 6953 +11477 6951 +46683 6950 +42979 6950 +34455 6948 +36834 6946 +7559 6944 +43456 6943 +49138 6942 +24834 6941 +31590 6941 +19615 6939 +31122 6939 +39902 6938 +46102 6937 +44276 6937 +17043 6937 +26964 6935 +36754 6932 +40331 6931 +39200 6930 +46002 6930 +26062 6926 +41744 6924 +14472 6918 +15380 6916 +41633 6916 +37269 6916 +44991 6916 +18202 6912 +44159 6911 +20026 6910 +31419 6910 +49370 6910 +48124 6909 +29307 6908 +24342 6901 +29223 6893 +29221 6891 +33786 6883 +39458 6882 +50188 6882 +38241 6880 +22585 6876 +3836 6870 +29017 6868 +49562 6865 +28018 6862 +41748 6859 +33349 6859 +49802 6859 +33028 6853 +5970 6851 +40853 6847 +48791 6844 +49709 6842 +14629 6836 +13078 6836 +36607 6834 +38027 6832 +50178 6831 +49176 6830 +43437 6830 +42696 6829 +37674 6827 +31053 6814 +42171 6814 +34242 6813 +49670 6812 +30603 6812 +40478 6812 +36843 6811 +22927 6810 +49422 6806 +43740 6805 +12962 6805 +16776 6802 +18453 6795 +40107 6791 +40397 6791 +22955 6788 +47172 6788 +28745 6781 +38035 6781 +13576 6781 +41862 6779 +12629 6779 +46120 6776 +33684 6775 +10387 6773 +19593 6769 +31815 6763 +46025 6760 +33077 6759 +18615 6759 +17505 6756 +46022 6753 +48567 6753 +14950 6753 +36122 6748 +17922 6745 +18743 6744 +28298 6743 +40091 6743 +32916 6740 +43828 6736 +20684 6732 +16444 6731 +26257 6731 +28100 6730 +8077 6728 +48895 6728 +10166 6727 +13263 6726 +41008 6725 +25947 6725 +35608 6724 +46477 6723 +32184 6722 +36894 6719 +46421 6719 +44591 6717 +25385 6715 +40981 6714 +31442 6712 +48084 6712 +43517 6709 +38899 6709 +43572 6709 +24361 6708 +14126 6708 +30642 6708 +31798 6706 +43614 6705 +21206 6705 +26447 6703 +16951 6700 +47781 6700 +24835 6696 +47117 6693 +40360 6691 +43612 6691 +49790 6689 +20613 6683 +50016 6681 +35645 6676 +41901 6675 +32665 6675 +35750 6670 +32851 6669 +46369 6669 +33485 6669 +38998 6666 +40901 6666 +37436 6663 +38688 6663 +23935 6660 +46077 6658 +19576 6654 +33650 6652 +12904 6652 +15732 6646 +2333 6644 +42310 6644 +45038 6643 +28019 6642 +26152 6638 +35802 6636 +44501 6635 +19209 6633 +16764 6632 +40044 6631 +27619 6630 +41242 6629 +19578 6629 +45685 6627 +15145 6622 +23767 6617 +10623 6615 +37870 6614 +10677 6613 +12095 6612 +31116 6611 +39462 6610 +36733 6608 +43777 6608 +10699 6607 +44819 6602 +4812 6600 +13556 6600 +28101 6595 +42503 6595 +139 6594 +37150 6591 +40920 6590 +35835 6585 +47537 6584 +46610 6583 +31945 6582 +29942 6579 +26599 6578 +23806 6578 +16594 6576 +35630 6574 +45271 6573 +49884 6573 +2358 6572 +18400 6571 +44798 6568 +42481 6559 +42685 6557 +50015 6552 +7113 6550 +31857 6550 +32138 6548 +29347 6548 +48765 6547 +46076 6544 +32429 6542 +49728 6540 +33080 6540 +45869 6534 +17649 6533 +42948 6532 +26663 6526 +48529 6526 +35220 6525 +24388 6525 +16873 6521 +47206 6518 +15973 6517 +47918 6517 +25200 6516 +42135 6516 +46613 6516 +10118 6513 +20602 6509 +10126 6508 +49811 6508 +15341 6508 +25423 6507 +41202 6507 +38095 6500 +28465 6498 +48103 6497 +25163 6496 +46374 6496 +42485 6493 +31358 6492 +45119 6492 +29918 6491 +27856 6489 +40322 6489 +13942 6488 +40914 6488 +25636 6486 +19314 6486 +20689 6485 +13194 6482 +33373 6480 +27024 6480 +35742 6478 +7419 6477 +48852 6477 +49918 6473 +23520 6471 +31436 6468 +6668 6467 +27195 6464 +31809 6462 +16943 6454 +5515 6450 +39841 6448 +47050 6448 +36533 6446 +40059 6446 +37835 6443 +33314 6440 +48954 6439 +41251 6438 +47705 6437 +45337 6437 +14302 6436 +7107 6434 +28820 6430 +40481 6427 +2979 6425 +20863 6424 +44478 6423 +26272 6423 +43975 6421 +10786 6421 +49028 6420 +43331 6419 +44910 6418 +16947 6418 +32113 6417 +24469 6417 +17302 6413 +39788 6411 +47524 6410 +8419 6406 +15583 6402 +48413 6401 +44935 6400 +14279 6399 +46311 6398 +40864 6397 +27956 6396 +41767 6396 +46228 6394 +14823 6393 +25941 6390 +30520 6390 +47116 6389 +44213 6384 +35726 6382 +33333 6381 +31627 6369 +25486 6369 +46850 6354 +37340 6353 +23903 6350 +41591 6349 +40387 6347 +42440 6347 +12453 6343 +36604 6342 +15256 6342 +8892 6339 +40230 6339 +49110 6338 +38835 6337 +8371 6334 +4927 6331 +15103 6329 +49096 6327 +46780 6326 +27634 6324 +49126 6322 +3480 6319 +18853 6318 +46315 6318 +20168 6316 +48493 6316 +38452 6310 +7798 6309 +34000 6309 +35746 6309 +38936 6309 +43020 6308 +33283 6308 +36515 6306 +34674 6305 +49770 6304 +45234 6299 +32485 6298 +6432 6296 +47150 6296 +29027 6295 +28663 6293 +40705 6293 +17203 6292 +48379 6289 +15864 6281 +33410 6279 +47065 6274 +42887 6273 +46253 6271 +29880 6270 +6077 6269 +36133 6259 +41614 6254 +26129 6253 +42797 6249 +40339 6249 +48545 6248 +14061 6247 +30014 6246 +33716 6245 +27745 6244 +42243 6239 +42501 6236 +17739 6232 +13968 6232 +33353 6229 +35283 6229 +27608 6228 +42721 6225 +21053 6213 +32223 6211 +48790 6210 +31915 6208 +45565 6206 +46634 6204 +28698 6204 +21117 6201 +7727 6200 +29356 6199 +13499 6197 +34617 6194 +20557 6192 +45783 6188 +48613 6187 +21496 6186 +28066 6185 +44758 6183 +43749 6182 +39431 6181 +45582 6180 +47852 6177 +33276 6175 +33480 6172 +47242 6165 +29291 6161 +35790 6160 +22961 6160 +8273 6155 +46521 6153 +39707 6151 +43099 6145 +21878 6143 +17224 6141 +31688 6139 +43714 6139 +32555 6131 +47285 6129 +28410 6128 +45977 6128 +25302 6127 +34556 6125 +21106 6123 +1457 6123 +15324 6122 +43641 6122 +8269 6119 +17258 6118 +44336 6117 +45841 6114 +14399 6112 +49045 6108 +40825 6106 +7551 6106 +21330 6102 +24881 6101 +45353 6100 +36046 6096 +34104 6094 +29610 6093 +48468 6092 +22079 6089 +26687 6089 +20301 6086 +16136 6084 +31932 6083 +15118 6079 +48659 6078 +31614 6075 +28497 6075 +49999 6072 +17663 6069 +49563 6068 +48814 6067 +45352 6066 +49780 6065 +28735 6065 +45677 6065 +35381 6065 +34539 6062 +46127 6060 +35460 6054 +50237 6048 +31801 6048 +28813 6047 +28873 6047 +20257 6044 +43036 6041 +26504 6040 +32394 6037 +26522 6033 +49621 6033 +41470 6033 +6624 6031 +23966 6030 +17606 6030 +33365 6029 +45751 6029 +16681 6027 +45833 6026 +13253 6026 +3547 6022 +22360 6015 +46428 6015 +30676 6014 +38727 6014 +26563 6013 +46700 6012 +42784 6011 +25961 6010 +13007 6009 +16906 6008 +33170 6005 +6828 6003 +10310 6002 +37235 6001 +41076 6001 +33299 6000 +32072 5999 +42788 5998 +47625 5993 +32103 5993 +48749 5992 +35955 5991 +36787 5990 +43904 5986 +46840 5981 +16604 5979 +49515 5978 +146 5978 +45529 5976 +6384 5973 +42814 5970 +40640 5969 +32323 5968 +48433 5967 +36346 5966 +46786 5965 +47190 5965 +45901 5963 +35503 5960 +30710 5957 +34280 5952 +46231 5952 +39787 5949 +37656 5946 +38344 5945 +31559 5942 +41394 5941 +25199 5941 +12614 5941 +27520 5937 +16547 5936 +43844 5935 +35302 5932 +43108 5932 +43392 5932 +49277 5931 +31695 5931 +50175 5930 +39792 5926 +38618 5921 +28452 5921 +29800 5921 +46080 5919 +47658 5919 +35843 5919 +39456 5917 +46785 5913 +43368 5913 +46297 5911 +30337 5911 +30894 5908 +47427 5908 +28851 5907 +40890 5903 +15437 5903 +47288 5901 +38103 5900 +43990 5893 +30956 5892 +8864 5892 +18716 5891 +15333 5885 +39286 5884 +14182 5883 +50051 5879 +39919 5878 +42881 5874 +46294 5871 +50144 5870 +37393 5864 +38114 5860 +46144 5857 +33494 5854 +40692 5852 +33476 5849 +44278 5846 +37736 5844 +37573 5835 +23028 5835 +24546 5834 +45430 5834 +33380 5833 +27038 5832 +34350 5825 +41581 5823 +48867 5822 +5477 5818 +14490 5815 +45592 5815 +31502 5814 +44968 5811 +38183 5808 +32706 5808 +42254 5807 +49946 5806 +36732 5806 +28482 5805 +23054 5803 +20331 5799 +26517 5795 +45854 5793 +36343 5792 +20659 5791 +29146 5788 +42133 5781 +39429 5778 +30486 5776 +47255 5775 +44199 5773 +46538 5773 +39938 5772 +26801 5771 +49485 5768 +43198 5766 +48789 5765 +37725 5763 +18114 5762 +24954 5762 +39888 5760 +27097 5756 +26604 5753 +8911 5748 +48034 5745 +14369 5744 +13173 5744 +31859 5743 +47173 5739 +30543 5737 +24321 5735 +24840 5735 +23869 5735 +8552 5733 +33245 5732 +10541 5730 +46599 5729 +47471 5729 +36727 5726 +35628 5723 +5218 5721 +28644 5720 +45608 5714 +150 5711 +46219 5710 +47522 5708 +40632 5705 +30854 5700 +46383 5698 +41358 5696 +29515 5693 +48941 5693 +27553 5692 +27072 5684 +42098 5683 +30003 5682 +27915 5682 +12730 5675 +41057 5675 +32345 5672 +26495 5672 +42419 5669 +44566 5669 +48902 5665 +37672 5658 +37942 5658 +24243 5657 +48536 5654 +33853 5652 +27148 5649 +34227 5647 +32978 5645 +17844 5643 +40874 5642 +37865 5642 +37960 5637 +23252 5636 +12895 5636 +22027 5635 +31391 5631 +44270 5628 +18709 5628 +36341 5624 +49687 5624 +39908 5623 +49691 5619 +44596 5615 +9933 5614 +21665 5613 +41667 5613 +37770 5612 +29121 5608 +40653 5605 +20946 5602 +24912 5601 +42014 5596 +18439 5596 +21789 5592 +49494 5588 +49406 5588 +35977 5585 +36064 5580 +8572 5578 +45912 5578 +36147 5575 +30474 5573 +43532 5573 +49081 5572 +45198 5572 +41915 5568 +35320 5566 +4026 5566 +26848 5565 +22915 5564 +46748 5562 +47834 5560 +37418 5560 +42975 5559 +34198 5558 +40925 5558 +47730 5555 +48838 5555 +31205 5555 +9898 5552 +44319 5551 +26621 5550 +31396 5550 +28561 5544 +35738 5539 +29513 5538 +32833 5537 +41317 5536 +47454 5530 +18203 5530 +26416 5529 +13236 5527 +27389 5527 +47772 5527 +25753 5523 +19380 5521 +13074 5520 +39575 5519 +36027 5517 +33815 5516 +26109 5515 +36198 5513 +46710 5511 +16743 5510 +31626 5508 +50177 5508 +25554 5506 +24524 5502 +43925 5500 +46727 5499 +30698 5492 +31971 5491 +13305 5490 +1824 5490 +28861 5489 +42182 5483 +15239 5483 +36347 5483 +17576 5482 +44709 5479 +13930 5478 +34042 5475 +28914 5475 +10789 5474 +47419 5471 +42809 5470 +31441 5469 +34634 5461 +4945 5454 +50248 5452 +8360 5450 +38113 5446 +24333 5446 +47357 5445 +50092 5443 +23010 5440 +47402 5436 +27694 5433 +27193 5433 +29320 5432 +30892 5424 +32635 5424 +33582 5421 +27313 5420 +40164 5418 +47960 5415 +2515 5412 +46639 5411 +46510 5406 +23599 5406 +26287 5405 +35676 5403 +26858 5403 +25901 5403 +46183 5402 +41622 5396 +21149 5391 +13890 5388 +38270 5386 +10894 5385 +35575 5382 +46566 5379 +28956 5379 +3204 5377 +42031 5374 +27031 5374 +20180 5371 +46085 5368 +35393 5368 +8293 5368 +49357 5367 +49239 5367 +43406 5361 +38615 5361 +13149 5359 +40710 5358 +45201 5355 +46452 5353 +30934 5353 +43085 5352 +23416 5351 +30493 5347 +33634 5345 +42036 5343 +22468 5341 +31253 5340 +36204 5339 +14387 5339 +24426 5328 +45530 5326 +44831 5324 +42114 5324 +44225 5322 +48562 5320 +30156 5320 +40759 5319 +49057 5318 +42895 5316 +11921 5308 +29672 5307 +48652 5306 +44531 5306 +34924 5306 +9602 5304 +44604 5303 +9924 5301 +16478 5301 +9350 5300 +40546 5298 +45417 5293 +33092 5292 +50125 5292 +46360 5289 +4358 5288 +25624 5288 +49574 5285 +30230 5282 +28614 5280 +40627 5277 +28122 5274 +28268 5272 +7131 5270 +46897 5270 +38032 5269 +29302 5268 +44515 5267 +23391 5265 +35780 5263 +47783 5263 +44156 5262 +14168 5262 +43651 5260 +36428 5260 +2578 5254 +29945 5251 +27830 5246 +24378 5244 +33855 5244 +25454 5237 +39007 5235 +42547 5232 +45133 5230 +41995 5230 +10615 5229 +45647 5227 +23969 5223 +13233 5221 +46455 5221 +15036 5220 +26357 5219 +34769 5215 +44859 5214 +39407 5210 +36632 5210 +47911 5209 +30345 5209 +32465 5207 +26870 5203 +26828 5198 +35895 5196 +31711 5189 +37855 5187 +47892 5187 +46177 5186 +34442 5182 +43866 5179 +33898 5173 +45300 5166 +50157 5165 +20837 5165 +38948 5164 +3239 5161 +29934 5159 +17495 5158 +48657 5151 +40232 5150 +21656 5150 +23793 5148 +24015 5148 +18925 5145 +30581 5144 +45941 5139 +45285 5137 +33762 5136 +31898 5135 +34877 5134 +49742 5134 +44727 5131 +35781 5131 +24516 5130 +38518 5127 +46911 5120 +24364 5117 +42507 5115 +44296 5114 +29660 5112 +11924 5111 +12485 5111 +11788 5107 +49702 5105 +44562 5104 +45140 5102 +27925 5098 +31659 5097 +7203 5095 +30218 5094 +19016 5093 +39934 5090 +2500 5088 +19763 5086 +17056 5084 +42588 5081 +16107 5081 +40577 5080 +5320 5080 +22077 5078 +33172 5073 +21413 5071 +34371 5071 +44683 5064 +48473 5061 +20231 5059 +9881 5059 +46367 5058 +23314 5057 +23758 5057 +25444 5055 +36487 5054 +39668 5053 +49140 5051 +43054 5049 +15924 5048 +14978 5048 +46384 5048 +12859 5046 +46037 5045 +35913 5044 +49451 5039 +40009 5037 +42042 5035 +36125 5033 +15782 5032 +45547 5031 +27586 5030 +44451 5029 +28653 5024 +46329 5022 +48278 5022 +44086 5022 +27448 5020 +9315 5020 +26802 5019 +18559 5018 +27282 5017 +1880 5016 +39219 5012 +6349 5012 +32091 5011 +44330 5011 +38799 5007 +19160 5006 +15966 5006 +16372 5005 +8029 5002 +16358 5002 +21351 5000 +24515 5000 +46808 4995 +18097 4995 +42901 4993 +31127 4993 +16671 4992 +28839 4990 +26365 4989 +17964 4980 +21800 4977 +49444 4977 +41908 4974 +26737 4973 +43869 4973 +47763 4972 +6560 4971 +15848 4971 +40809 4967 +35346 4966 +19722 4963 +20701 4963 +43260 4962 +19563 4961 +48267 4957 +33825 4956 +45368 4952 +23838 4947 +44032 4945 +37124 4945 +13371 4943 +41296 4941 +39551 4941 +32096 4940 +43661 4939 +26294 4937 +49912 4936 +40480 4935 +26163 4935 +14605 4934 +38715 4933 +14749 4932 +38197 4930 +33054 4929 +48477 4928 +40083 4926 +21906 4926 +31611 4925 +47639 4924 +22880 4923 +22369 4920 +41848 4919 +47020 4919 +33129 4918 +11239 4916 +28935 4914 +25757 4914 +38566 4909 +17595 4908 +48738 4908 +33132 4906 +48948 4904 +43810 4899 +39455 4896 +30589 4895 +10791 4891 +41387 4891 +28258 4888 +28508 4885 +16580 4884 +20891 4884 +22646 4883 +22400 4882 +33734 4881 +35170 4880 +14910 4878 +30016 4876 +17097 4874 +17861 4871 +47466 4869 +29352 4869 +11995 4868 +37120 4866 +3784 4861 +21465 4860 +43418 4857 +46514 4855 +24455 4849 +45101 4845 +23686 4845 +25391 4844 +45403 4843 +21231 4842 +39650 4840 +43762 4835 +39559 4834 +45175 4834 +45243 4832 +31811 4832 +48191 4832 +38821 4827 +6222 4824 +47333 4817 +19109 4816 +46047 4814 +46615 4810 +40281 4810 +13442 4805 +27772 4801 +43660 4798 +30150 4796 +31382 4792 +33248 4790 +41411 4789 +32966 4772 +47025 4769 +17491 4769 +36982 4766 +44231 4764 +35236 4764 +46111 4757 +27566 4749 +15865 4747 +20369 4746 +33744 4743 +29639 4741 +31425 4739 +30001 4736 +38521 4736 +25101 4734 +42995 4733 +46602 4733 +49094 4730 +25169 4730 +33196 4728 +43167 4726 +44120 4725 +42384 4723 +21962 4717 +17143 4715 +8285 4712 +41424 4712 +44239 4711 +41233 4710 +10267 4710 +41446 4706 +24201 4703 +40133 4701 +16040 4699 +14631 4699 +42553 4698 +5192 4698 +1189 4696 +32608 4696 +6009 4695 +4368 4694 +45299 4694 +39555 4694 +18932 4693 +36476 4690 +40347 4690 +19031 4690 +33234 4689 +42223 4687 +41097 4682 +43118 4679 +40850 4679 +9169 4676 +40651 4676 +33518 4672 +46995 4665 +39622 4665 +49475 4665 +30071 4664 +48301 4663 +43377 4661 +43628 4659 +13361 4659 +40258 4652 +43258 4650 +6354 4648 +37712 4643 +39833 4643 +4183 4642 +28228 4637 +36504 4625 +19801 4622 +43471 4621 +20885 4619 +46659 4617 +42711 4616 +43194 4615 +48056 4613 +49438 4612 +45729 4612 +32101 4609 +33766 4608 +22552 4600 +26572 4600 +40402 4599 +20593 4596 +45914 4596 +37214 4596 +34749 4596 +24908 4594 +40799 4591 +23646 4587 +10353 4587 +20574 4587 +35779 4584 +23814 4582 +16254 4577 +36471 4576 +49604 4576 +41205 4576 +22015 4569 +44190 4568 +23821 4562 +32356 4561 +21109 4559 +20898 4559 +48538 4557 +42477 4556 +46155 4554 +29868 4552 +35442 4551 +48131 4549 +34562 4549 +41359 4548 +49788 4545 +33152 4545 +47109 4545 +16497 4544 +50200 4541 +10895 4538 +34149 4537 +19778 4535 +38365 4533 +48860 4532 +41966 4527 +24179 4525 +29507 4523 +33789 4522 +30736 4518 +33951 4515 +26865 4514 +25249 4512 +34293 4511 +19212 4511 +46021 4511 +31718 4510 +20688 4508 +27286 4506 +10103 4504 +40668 4500 +21575 4496 +44425 4496 +30388 4495 +15944 4494 +46824 4493 +33772 4488 +48682 4488 +19871 4486 +48885 4485 +38941 4483 +14116 4480 +22866 4478 +15414 4474 +9523 4473 +36942 4473 +45035 4467 +38110 4466 +35952 4462 +14224 4460 +46722 4459 +44850 4457 +3479 4456 +37273 4454 +19724 4453 +27955 4452 +35414 4450 +37155 4448 +43815 4447 +25461 4442 +46955 4442 +19359 4441 +33406 4439 +39347 4438 +32023 4436 +44538 4432 +7453 4430 +24517 4429 +31871 4427 +33661 4427 +34577 4425 +6038 4424 +48142 4424 +49646 4420 +30535 4418 +7098 4416 +34496 4414 +11306 4414 +46343 4413 +36571 4413 +36320 4410 +49511 4405 +3462 4404 +28266 4402 +29428 4401 +35632 4399 +17440 4399 +18847 4398 +42822 4397 +44642 4394 +33717 4392 +20360 4391 +43008 4391 +24214 4390 +36084 4386 +9625 4383 +50118 4383 +43597 4375 +49217 4375 +29323 4373 +38644 4373 +37773 4373 +47598 4368 +37012 4366 +44191 4366 +38148 4365 +36387 4363 +3749 4362 +17474 4358 +41185 4356 +47473 4347 +35741 4345 +36510 4334 +44166 4331 +17236 4329 +44409 4328 +15038 4327 +23795 4324 +38215 4320 +166 4320 +48003 4312 +36613 4307 +28961 4305 +16368 4304 +40247 4303 +38399 4300 +28712 4296 +50204 4290 +31680 4288 +37188 4287 +26545 4287 +33274 4285 +42432 4285 +45675 4282 +33678 4281 +32086 4278 +20015 4278 +23002 4278 +46217 4273 +39945 4272 +38564 4271 +13994 4270 +35813 4268 +37815 4268 +14459 4267 +38274 4267 +20129 4266 +44613 4262 +28376 4261 +11093 4255 +29720 4253 +48875 4252 +42359 4250 +28074 4242 +49347 4241 +37500 4240 +50109 4233 +18919 4232 +47935 4230 +30540 4229 +8193 4226 +24586 4223 +37766 4218 +23735 4218 +42903 4217 +46775 4217 +36117 4216 +29736 4214 +46733 4213 +39582 4209 +38043 4209 +7969 4208 +33968 4204 +18074 4203 +31895 4202 +32603 4199 +46320 4199 +14621 4196 +33062 4195 +14844 4194 +36911 4191 +3821 4190 +7678 4187 +27358 4187 +27050 4184 +3880 4183 +34795 4181 +38891 4180 +49751 4180 +38499 4172 +5435 4165 +4770 4165 +20826 4164 +8809 4163 +43316 4158 +49749 4155 +49531 4153 +9462 4149 +36781 4149 +32419 4147 +45077 4143 +43790 4135 +10168 4132 +34303 4132 +37351 4130 +28119 4129 +37148 4129 +11709 4128 +9152 4127 +42059 4126 +4256 4126 +43721 4126 +43270 4125 +38038 4124 +26639 4122 +32065 4121 +32495 4121 +49222 4120 +5754 4120 +30221 4120 +37920 4117 +28252 4117 +28474 4117 +45988 4115 +33321 4108 +41020 4108 +28333 4101 +25538 4097 +29453 4092 +44808 4089 +48608 4083 +42197 4083 +22773 4079 +10558 4077 +43654 4076 +29064 4075 +39223 4067 +41982 4067 +48082 4063 +34457 4061 +43094 4059 +38641 4059 +37344 4059 +30998 4058 +31069 4054 +9547 4052 +39275 4051 +25267 4049 +43987 4048 +48625 4045 +21945 4042 +26111 4042 +9783 4039 +10777 4037 +36259 4031 +28147 4031 +40203 4026 +41940 4026 +8594 4023 +50149 4023 +30564 4020 +45367 4020 +30950 4020 +22885 4018 +37655 4017 +38790 4017 +49041 4011 +28769 4011 +46672 4010 +44707 4010 +43525 4003 +44898 4001 +25576 4000 +36558 3996 +1832 3995 +47252 3991 +31775 3988 +49836 3987 +15005 3984 +24036 3984 +43384 3983 +3658 3983 +28428 3981 +15575 3978 +34635 3974 +45242 3974 +131 3972 +41781 3971 +36291 3969 +43233 3968 +33606 3968 +32032 3962 +11183 3962 +47924 3961 +28373 3959 +34703 3956 +40637 3956 +38815 3955 +37250 3953 +32252 3953 +42166 3952 +26317 3951 +19083 3950 +33236 3947 +49061 3946 +48558 3946 +32782 3945 +38640 3944 +19076 3943 +47823 3942 +34764 3938 +36598 3936 +24564 3934 +19275 3934 +22719 3934 +4412 3932 +32634 3931 +36770 3930 +46430 3929 +23772 3927 +34168 3927 +15179 3925 +49899 3924 +26396 3922 +46697 3922 +28789 3919 +1253 3918 +35767 3913 +45205 3912 +35967 3911 +41560 3910 +11452 3910 +44592 3909 +39266 3908 +24941 3907 +22522 3905 +16844 3905 +49128 3904 +48447 3902 +33501 3899 +30952 3899 +43895 3892 +26214 3890 +29230 3888 +7304 3887 +30326 3881 +12103 3881 +45027 3878 +37022 3877 +41671 3877 +39069 3875 +25255 3869 +46100 3868 +38807 3866 +30672 3863 +42592 3858 +47113 3856 +10263 3855 +22683 3855 +42009 3854 +49501 3854 +18401 3853 +41609 3850 +47638 3849 +26180 3847 +6353 3846 +39126 3846 +47365 3842 +44946 3839 +23803 3837 +8816 3836 +38491 3834 +41741 3824 +38855 3824 +18498 3824 +44028 3823 +27933 3819 +15322 3818 +40211 3816 +36616 3816 +45835 3815 +40034 3815 +32147 3814 +33800 3813 +28766 3808 +32169 3808 +25637 3808 +34148 3808 +2783 3804 +47798 3803 +42539 3801 +28338 3801 +31909 3799 +41482 3798 +19622 3798 +2827 3797 +18270 3793 +38874 3793 +45254 3792 +3273 3789 +41026 3789 +28404 3788 +37713 3787 +28611 3787 +38417 3783 +19776 3780 +10024 3778 +36463 3777 +5619 3776 +48754 3775 +32846 3774 +33486 3769 +26344 3769 +8912 3768 +33176 3767 +36850 3766 +45700 3762 +46220 3762 +48330 3758 +41989 3758 +44458 3757 +22170 3755 +41219 3754 +49926 3754 +29486 3749 +49151 3746 +17518 3746 +31929 3746 +44223 3744 +45684 3742 +30871 3741 +49908 3741 +29870 3740 +33796 3739 +26367 3736 +12208 3731 +36258 3730 +47279 3726 +19449 3720 +49933 3718 +23241 3713 +28622 3712 +10367 3711 +35395 3711 +35178 3708 +46799 3708 +10929 3708 +38637 3705 +31696 3702 +21849 3701 +30609 3698 +47071 3697 +48181 3696 +42473 3696 +41104 3694 +17256 3692 +18610 3691 +13757 3690 +49256 3687 +41552 3684 +15466 3678 +12351 3676 +46182 3675 +49560 3674 +37403 3673 +22929 3670 +43087 3664 +35791 3664 +37146 3661 +8935 3660 +34999 3660 +16301 3659 +43782 3657 +39967 3657 +18167 3656 +30265 3655 +18172 3655 +46676 3655 +31954 3651 +39345 3650 +37647 3650 +38663 3646 +32685 3642 +41162 3640 +46358 3633 +41000 3632 +23877 3631 +36674 3631 +42195 3624 +44747 3622 +48547 3622 +49849 3620 +39825 3619 +26037 3618 +26440 3616 +22810 3615 +37510 3614 +28883 3613 +43582 3611 +19754 3611 +23114 3607 +38601 3607 +45037 3602 +37017 3602 +33547 3600 +44435 3597 +47718 3593 +29046 3591 +47815 3591 +45798 3591 +32830 3590 +12145 3589 +28222 3589 +31410 3589 +39849 3587 +38296 3587 +37477 3587 +13751 3586 +40517 3586 +36977 3585 +10046 3583 +26060 3578 +48310 3574 +47348 3574 +42180 3574 +26198 3568 +27614 3567 +44318 3566 +33439 3564 +42840 3563 +41392 3562 +33539 3561 +33714 3560 +19488 3560 +50145 3558 +18717 3557 +24335 3556 +43907 3554 +17386 3554 +35279 3553 +40106 3546 +24741 3545 +36124 3545 +45890 3545 +35620 3544 +23826 3544 +44706 3541 +29085 3541 +47531 3538 +49388 3537 +34445 3530 +19618 3527 +28312 3525 +38481 3523 +42652 3522 +5649 3522 +2660 3518 +41512 3514 +36623 3510 +31417 3510 +17125 3510 +36436 3505 +26656 3504 +25205 3500 +33087 3498 +41055 3498 +46701 3497 +19557 3497 +14815 3495 +32070 3495 +27908 3490 +18337 3490 +4707 3490 +20336 3487 +26638 3487 +44464 3485 +36542 3482 +7555 3476 +43271 3474 +15109 3473 +48319 3470 +37996 3469 +39815 3468 +40025 3467 +39590 3466 +46326 3461 +39531 3459 +23513 3459 +37187 3458 +49139 3457 +39470 3456 +11430 3455 +3390 3454 +49794 3453 +45809 3452 +49026 3452 +5196 3451 +33637 3450 +48686 3449 +40832 3449 +10493 3448 +21677 3448 +26193 3448 +33931 3447 +43319 3447 +35770 3442 +46912 3442 +30331 3437 +20275 3437 +36851 3436 +35347 3436 +47026 3435 +46893 3435 +44267 3433 +29435 3432 +34248 3430 +46143 3430 +37077 3427 +33293 3425 +47322 3422 +47361 3419 +43158 3416 +43160 3413 +37481 3410 +36037 3409 +41090 3409 +41506 3408 +43477 3407 +40709 3406 +38841 3404 +35974 3403 +22492 3403 +18249 3400 +30448 3400 +10587 3398 +47209 3397 +7333 3396 +36028 3396 +26238 3396 +27002 3393 +49155 3393 +49601 3393 +31743 3392 +15340 3391 +5480 3388 +18589 3387 +33784 3384 +42165 3384 +35792 3383 +43212 3383 +32815 3381 +23380 3379 +37010 3377 +44942 3377 +24844 3376 +19364 3373 +11151 3370 +22630 3369 +47227 3368 +33707 3366 +26693 3366 +35202 3366 +10545 3365 +1518 3363 +44754 3361 +26158 3359 +34364 3357 +23480 3357 +19293 3355 +14257 3354 +38321 3347 +21251 3344 +44807 3341 +25064 3340 +50093 3339 +41707 3336 +48289 3329 +30818 3329 +7990 3323 +4228 3322 +27849 3320 +21744 3319 +43806 3316 +11127 3315 +22252 3313 +32341 3312 +42082 3309 +14012 3308 +44954 3304 +34258 3301 +18872 3299 +43216 3296 +44154 3295 +40778 3295 +26501 3294 +28012 3294 +44334 3294 +49945 3293 +44238 3293 +38460 3293 +21075 3291 +20621 3289 +44836 3288 +33409 3285 +49036 3284 +34805 3282 +39658 3281 +48522 3281 +35760 3275 +18604 3274 +33420 3274 +35265 3273 +45755 3267 +33447 3263 +35102 3258 +38040 3253 +33792 3252 +27983 3251 +31073 3250 +46257 3247 +18245 3245 +15759 3243 +40806 3237 +35500 3235 +3322 3234 +31254 3229 +20480 3229 +40781 3228 +41808 3227 +20255 3224 +35420 3223 +35147 3223 +8370 3220 +49893 3212 +41709 3209 +17643 3208 +39070 3207 +48402 3207 +33120 3207 +39232 3207 +20184 3206 +22873 3206 +13600 3203 +40063 3201 +45872 3200 +47118 3200 +48933 3196 +23111 3194 +35058 3190 +35488 3184 +28938 3184 +42976 3183 +19237 3182 +36356 3178 +49608 3177 +26104 3175 +46932 3174 +43303 3174 +49850 3174 +18865 3173 +43979 3173 +44090 3171 +32368 3167 +22935 3167 +49506 3166 +44217 3164 +28525 3162 +44177 3162 +35538 3160 +40310 3157 +30444 3156 +44450 3153 +35433 3152 +18441 3151 +32229 3147 +45160 3144 +47120 3143 +23652 3141 +29485 3140 +30467 3138 +6552 3135 +31008 3134 +46331 3131 +45866 3131 +6876 3130 +41436 3130 +22314 3125 +38542 3125 +32247 3121 +15485 3121 +16891 3119 +48641 3118 +38764 3117 +7047 3113 +35172 3110 +42289 3109 +42184 3108 +23321 3107 +34032 3103 +32963 3102 +36196 3102 +37046 3101 +3062 3100 +49496 3095 +37653 3093 +48650 3091 +13577 3080 +9681 3079 +40135 3078 +16818 3065 +28326 3064 +49116 3064 +48346 3060 +29565 3060 +36567 3058 +3179 3056 +44825 3056 +25850 3055 +31467 3050 +43214 3047 +24400 3047 +44632 3047 +10697 3047 +24852 3044 +19012 3043 +11639 3039 +24728 3038 +37778 3037 +12063 3034 +27914 3034 +29335 3032 +20329 3030 +4921 3030 +36248 3028 +46512 3025 +38128 3024 +45528 3022 +2225 3022 +41080 3022 +39184 3021 +32818 3020 +48975 3013 +12343 3012 +5697 3011 +45169 3010 +7449 3010 +37359 3010 +38012 3010 +42646 3009 +22428 3008 +33209 3007 +37395 3006 +13003 3000 +20859 2999 +35277 2996 +28811 2996 +13668 2995 +46010 2993 +42660 2993 +47445 2991 +41780 2991 +35001 2990 +17527 2986 +35528 2984 +26715 2983 +22305 2981 +16151 2980 +46679 2978 +39986 2978 +36767 2975 +2103 2974 +22417 2974 +36857 2973 +5144 2969 +41911 2968 +35533 2967 +33643 2967 +32275 2964 +41382 2964 +9981 2963 +13086 2962 +19306 2959 +46115 2958 +45311 2952 +23839 2948 +17174 2947 +38052 2942 +28865 2940 +44195 2940 +40792 2934 +47046 2933 +27950 2933 +38106 2932 +31579 2930 +25102 2921 +169 2921 +22678 2920 +39688 2920 +37799 2919 +26941 2918 +36970 2915 +19351 2912 +19857 2908 +30715 2907 +39355 2904 +35494 2902 +23360 2902 +45195 2899 +32467 2898 +41357 2895 +32197 2895 +11549 2894 +37693 2894 +14501 2891 +41111 2890 +22506 2887 +47064 2886 +43834 2884 +31619 2882 +27575 2881 +12467 2879 +39789 2876 +45147 2875 +14095 2875 +5319 2874 +20660 2872 +44779 2872 +48845 2872 +35041 2871 +10864 2871 +25490 2870 +22453 2869 +32418 2866 +38277 2862 +38577 2858 +46332 2857 +16166 2856 +23414 2849 +26327 2845 +40927 2843 +23289 2842 +32570 2841 +49883 2841 +38865 2839 +37597 2838 +11573 2837 +46757 2836 +33746 2834 +38816 2833 +45490 2833 +49885 2833 +29863 2832 +21828 2831 +44012 2829 +36112 2827 +49806 2821 +37422 2819 +48728 2817 +42120 2816 +39354 2815 +27945 2814 +47384 2810 +24503 2810 +19282 2807 +26232 2807 +49046 2805 +16589 2804 +38392 2803 +38354 2801 +3330 2800 +48700 2799 +45522 2798 +30205 2792 +37262 2790 +48107 2788 +9762 2784 +34582 2782 +19580 2780 +24087 2778 +30661 2776 +33870 2776 +38292 2774 +39516 2773 +47558 2772 +34889 2764 +38067 2759 +26623 2758 +43753 2758 +25925 2753 +27169 2749 +25714 2746 +50235 2745 +43217 2744 +37978 2742 +28457 2742 +27292 2742 +43695 2741 +29719 2737 +44608 2730 +37157 2729 +21384 2727 +35029 2727 +35558 2726 +48938 2726 +22367 2725 +40010 2725 +23025 2722 +42173 2719 +41068 2715 +23745 2715 +47145 2712 +44952 2711 +46954 2708 +23269 2708 +38281 2705 +32689 2705 +30487 2705 +14980 2703 +16141 2698 +38904 2694 +31966 2693 +37031 2693 +41203 2690 +49615 2690 +27510 2686 +30753 2684 +18566 2683 +19324 2677 +35470 2677 +1792 2677 +33114 2676 +42522 2674 +39323 2668 +47084 2666 +49763 2665 +17414 2664 +38448 2664 +49513 2664 +20869 2663 +4181 2663 +4209 2662 +47006 2662 +8709 2661 +37177 2655 +17022 2654 +21974 2651 +26142 2650 +21340 2647 +22203 2644 +34111 2640 +40099 2638 +34787 2636 +31077 2635 +28167 2633 +47030 2629 +37612 2628 +47174 2628 +15817 2626 +18699 2626 +48443 2626 +40257 2625 +35841 2625 +37156 2623 +6367 2622 +34727 2622 +40613 2621 +42216 2621 +49427 2617 +30293 2616 +44110 2614 +38269 2614 +16897 2613 +45537 2613 +37200 2612 +38235 2611 +40895 2610 +5541 2609 +49071 2608 +35693 2608 +45824 2606 +45135 2606 +39759 2604 +41807 2603 +11610 2603 +48162 2601 +8723 2595 +33910 2594 +4052 2584 +22887 2584 +49832 2584 +44711 2580 +34095 2577 +31301 2576 +16273 2575 +48147 2570 +37511 2568 +980 2568 +41776 2566 +35476 2563 +21471 2561 +46554 2560 +47469 2555 +19526 2555 +37411 2553 +19875 2549 +41972 2548 +32713 2547 +41256 2547 +1591 2545 +18748 2544 +25180 2543 +43706 2542 +30249 2541 +34876 2540 +23081 2540 +44163 2539 +47506 2539 +46471 2538 +9146 2537 +18796 2532 +16516 2529 +49337 2528 +22755 2528 +43583 2524 +30585 2519 +26759 2514 +4840 2510 +42974 2509 +42064 2508 +4766 2507 +44453 2506 +48658 2505 +31266 2505 +31447 2504 +39994 2501 +50030 2499 +28620 2499 +36379 2495 +38571 2493 +18022 2493 +43691 2492 +33986 2490 +44189 2487 +5525 2485 +34181 2480 +19411 2479 +36321 2474 +49505 2474 +758 2471 +36361 2470 +27528 2468 +32785 2467 +38925 2467 +49228 2467 +19382 2464 +44930 2463 +26011 2462 +31181 2462 +41375 2460 +49342 2458 +45492 2457 +30728 2455 +41033 2454 +42451 2453 +22180 2453 +18109 2452 +46495 2452 +31542 2445 +30599 2444 +47951 2441 +36670 2437 +41993 2437 +36621 2434 +34142 2433 +23157 2430 +36017 2425 +40117 2425 +35887 2424 +50163 2424 +47651 2424 +30216 2421 +48744 2419 +31512 2418 +35040 2416 +35847 2416 +23118 2414 +44046 2409 +48469 2408 +47764 2406 +21317 2405 +7168 2404 +41729 2404 +29510 2402 +49137 2400 +21055 2399 +30553 2396 +39194 2393 +22501 2390 +11792 2390 +38946 2385 +33768 2384 +29904 2383 +42558 2380 +25472 2376 +30312 2376 +48900 2375 +25380 2374 +39175 2373 +49795 2372 +34495 2371 +45454 2369 +20004 2369 +49174 2369 +17410 2368 +45672 2366 +39814 2364 +8707 2363 +30782 2363 +11884 2362 +21689 2362 +23870 2359 +21920 2356 +34804 2352 +22093 2347 +5208 2346 +9065 2345 +43145 2344 +14671 2342 +19515 2340 +13783 2337 +14961 2335 +45689 2335 +31631 2334 +46156 2333 +31838 2332 +29905 2331 +45724 2328 +41372 2325 +34720 2320 +47472 2317 +34932 2316 +36690 2316 +40452 2316 +33583 2314 +45519 2314 +24871 2310 +48505 2309 +43786 2309 +47708 2306 +46479 2306 +46121 2303 +40574 2302 +47010 2299 +37199 2298 +22022 2298 +24319 2296 +40246 2294 +44974 2290 +30347 2288 +45803 2286 +47086 2283 +39724 2283 +28618 2280 +39065 2278 +5641 2276 +30604 2275 +10142 2272 +21476 2272 +23257 2270 +40155 2269 +40259 2268 +28968 2265 +24771 2263 +46136 2261 +41947 2260 +36658 2257 +13557 2252 +27985 2250 +34354 2250 +48499 2249 +27600 2246 +27167 2246 +44448 2245 +8964 2243 +17750 2242 +16495 2239 +22542 2239 +47701 2239 +46947 2237 +49358 2237 +42624 2235 +49226 2234 +28198 2231 +48321 2230 +40485 2230 +41100 2230 +10778 2229 +23979 2228 +34196 2227 +21864 2226 +35705 2226 +43702 2225 +31844 2223 +39890 2221 +33725 2221 +40971 2220 +19858 2218 +36370 2217 +33912 2214 +10967 2210 +159 2209 +25178 2208 +8682 2204 +12731 2204 +31179 2204 +48760 2201 +47720 2200 +45876 2196 +42855 2195 +36127 2195 +34340 2193 +39910 2190 +28365 2187 +22330 2183 +6527 2182 +846 2179 +50249 2178 +17635 2177 +38155 2173 +32212 2170 +37859 2169 +28656 2165 +37633 2163 +45662 2163 +39971 2161 +40541 2161 +28762 2160 +48956 2160 +30856 2159 +49896 2157 +48806 2157 +21412 2155 +48869 2155 +49630 2154 +27703 2151 +32230 2150 +42986 2148 +49214 2145 +49665 2142 +43000 2141 +24861 2141 +31660 2141 +44405 2140 +42719 2140 +48369 2139 +39021 2136 +7879 2135 +49905 2132 +44208 2132 +37195 2131 +3679 2129 +47068 2128 +27275 2127 +42758 2124 +37665 2123 +47835 2120 +20290 2116 +42169 2115 +43049 2111 +45286 2111 +41052 2109 +11774 2107 +48523 2100 +32667 2100 +46237 2100 +45351 2098 +49504 2097 +17269 2092 +48720 2091 +36253 2089 +21017 2089 +42867 2089 +23004 2088 +26047 2088 +32734 2081 +46890 2080 +28624 2079 +36408 2078 +40413 2074 +42530 2074 +33044 2069 +32573 2068 +29250 2066 +45008 2065 +49374 2064 +15931 2062 +45908 2062 +47649 2061 +37910 2058 +19890 2057 +44971 2052 +49050 2050 +48362 2047 +10882 2047 +47063 2045 +49925 2036 +12126 2035 +5785 2034 +41444 2034 +46165 2034 +46582 2029 +40591 2028 +11771 2024 +40053 2022 +47732 2022 +9454 2020 +38667 2019 +36044 2019 +48285 2019 +34460 2019 +33920 2018 +46787 2017 +13715 2016 +25529 2015 +4322 2014 +25224 2013 +9201 2012 +39500 2012 +11139 2008 +14188 2005 +33111 2005 +26575 2000 +41753 1998 +19469 1998 +9959 1996 +48452 1995 +30640 1995 +39377 1994 +23595 1993 +24376 1992 +14092 1991 +43717 1990 +28337 1989 +48183 1987 +30167 1986 +25348 1985 +11405 1984 +41099 1983 +48375 1982 +16791 1982 +22046 1980 +23320 1978 +44010 1977 +28794 1974 +36916 1974 +22759 1974 +21876 1974 +28758 1973 +48046 1972 +16883 1967 +20560 1965 +30626 1965 +16878 1964 +45356 1960 +15798 1960 +31364 1959 +15566 1959 +30475 1958 +19182 1956 +20748 1953 +12376 1952 +13291 1952 +22793 1951 +18883 1950 +40356 1949 +50247 1947 +35523 1945 +46621 1944 +18237 1943 +29881 1941 +37430 1940 +41519 1938 +49434 1936 +39044 1936 +47656 1934 +31327 1933 +21583 1933 +28134 1933 +26947 1933 +39000 1931 +38860 1929 +44015 1923 +45507 1923 +26945 1922 +44797 1921 +28707 1920 +13726 1916 +9418 1914 +29829 1913 +40158 1912 +137 1911 +33000 1910 +4907 1909 +38649 1909 +42381 1908 +49834 1907 +35039 1904 +21058 1902 +32014 1901 +45364 1899 +45163 1899 +36918 1896 +36406 1895 +48617 1894 +5512 1894 +23907 1893 +43037 1890 +19667 1890 +32892 1889 +2559 1886 +38696 1882 +32080 1880 +22833 1877 +21756 1876 +11805 1875 +32259 1872 +35975 1872 +30201 1871 +45249 1867 +13328 1867 +22402 1864 +13292 1864 +36545 1863 +49322 1862 +1147 1857 +34013 1857 +44140 1856 +27809 1856 +35307 1855 +26410 1852 +43857 1849 +42973 1848 +34098 1848 +44867 1847 +19990 1845 +11967 1843 +46386 1843 +23381 1836 +31264 1832 +47292 1829 +42866 1827 +33030 1826 +35540 1819 +20608 1818 +39149 1818 +44625 1817 +46238 1814 +48763 1812 +26292 1811 +35453 1810 +43034 1807 +29886 1806 +24957 1806 +47768 1804 +10063 1804 +35611 1802 +46846 1800 +22130 1799 +37093 1797 +28899 1796 +43789 1795 +12676 1795 +38528 1790 +47160 1788 +33566 1787 +37498 1785 +40558 1785 +45554 1776 +1421 1775 +5294 1775 +17358 1774 +18253 1773 +27444 1772 +44728 1769 +19028 1768 +48344 1763 +12567 1762 +27670 1758 +2235 1757 +42915 1753 +35861 1752 +43475 1751 +25895 1750 +7479 1750 +22973 1742 +43267 1739 +38776 1738 +45170 1737 +50227 1736 +46651 1736 +42929 1734 +16892 1733 +49346 1731 +37133 1729 +37350 1729 +21943 1728 +22766 1726 +4011 1725 +49865 1725 +39621 1725 +35656 1721 +7450 1720 +49213 1719 +49877 1715 +25748 1715 +41115 1713 +29743 1711 +23193 1707 +43422 1705 +31347 1705 +33810 1704 +25275 1703 +27674 1702 +15090 1701 +39157 1697 +33648 1695 +42636 1694 +42063 1694 +20746 1694 +31913 1693 +24807 1691 +14859 1690 +44402 1689 +14908 1689 +40261 1688 +13736 1685 +49168 1683 +40267 1683 +11907 1683 +36655 1681 +39643 1681 +16410 1679 +39316 1677 +49819 1674 +19463 1673 +40507 1673 +31246 1673 +46448 1673 +42316 1671 +31687 1671 +9466 1671 +49603 1670 +7726 1667 +34156 1665 +25471 1663 +15020 1663 +34301 1661 +1947 1661 +36181 1660 +31326 1657 +30266 1656 +43458 1656 +41964 1650 +38469 1650 +44293 1649 +33692 1647 +43380 1643 +26678 1643 +43433 1642 +31758 1641 +47400 1639 +2477 1638 +36307 1638 +48991 1637 +10549 1636 +32332 1634 +16268 1633 +28966 1632 +30656 1632 +47226 1629 +46979 1629 +19373 1628 +35036 1626 +38929 1624 +6312 1624 +42615 1622 +45819 1621 +47863 1620 +47195 1619 +26933 1617 +46695 1615 +27711 1614 +18686 1613 +28394 1613 +33386 1612 +42524 1612 +48205 1608 +32360 1605 +9318 1604 +45626 1603 +48526 1601 +29710 1601 +25874 1601 +14610 1599 +3472 1597 +44309 1594 +39922 1594 +19510 1590 +38381 1590 +34121 1588 +28253 1587 +49660 1586 +29773 1585 +32048 1582 +34219 1582 +45664 1581 +46464 1579 +5271 1578 +48911 1578 +22110 1573 +42720 1572 +34018 1569 +38335 1569 +44704 1569 +15149 1568 +45137 1567 +42172 1566 +35339 1563 +43910 1562 +45250 1560 +13392 1560 +23022 1559 +33012 1559 +48277 1557 +16333 1553 +23764 1553 +19202 1553 +19060 1551 +30049 1550 +43993 1546 +32432 1541 +41657 1541 +12374 1540 +31905 1536 +45032 1536 +43408 1534 +39224 1531 +42717 1531 +44423 1528 +19876 1528 +12054 1524 +13412 1523 +34816 1522 +28383 1518 +30295 1516 +39298 1514 +21142 1513 +30327 1513 +19953 1512 +39850 1511 +21563 1505 +17653 1504 +5099 1502 +48123 1499 +30268 1498 +39181 1497 +45297 1495 +36854 1494 +21947 1494 +32639 1494 +21271 1493 +18200 1493 +45748 1491 +43961 1488 +7847 1488 +48534 1487 +43073 1486 +29826 1485 +49991 1483 +42492 1482 +3603 1481 +40704 1480 +43024 1476 +45739 1475 +35585 1471 +41222 1470 +3728 1468 +30360 1465 +40489 1465 +37455 1464 +29712 1463 +30136 1462 +28112 1462 +21052 1459 +33699 1459 +31676 1457 +44036 1454 +43729 1454 +39310 1453 +43078 1453 +33250 1451 +15926 1446 +44506 1442 +29869 1440 +28156 1440 +44520 1438 +47690 1435 +142 1433 +33223 1432 +48761 1432 +34775 1431 +11900 1431 +39187 1430 +47746 1429 +29316 1428 +42468 1428 +36365 1427 +27823 1424 +30562 1424 +31263 1423 +40031 1422 +3429 1421 +30159 1420 +44415 1418 +11039 1418 +47066 1415 +47090 1415 +47212 1412 +20491 1409 +34402 1408 +5008 1408 +28924 1406 +37706 1404 +26227 1404 +45384 1401 +47991 1401 +11411 1399 +20972 1399 +41049 1399 +19662 1399 +38211 1399 +46858 1397 +21314 1391 +13780 1391 +42360 1391 +33705 1387 +49736 1386 +36563 1386 +44013 1384 +37337 1383 +46923 1379 +49312 1379 +23242 1377 +46763 1375 +31490 1375 +49929 1373 +35944 1370 +11482 1370 +38300 1369 +50155 1366 +32590 1363 +43825 1362 +47881 1360 +46349 1360 +48585 1360 +40156 1359 +47797 1359 +37141 1358 +32326 1355 +41305 1355 +12630 1351 +45767 1350 +34945 1350 +29698 1349 +19438 1348 +25182 1344 +24457 1341 +43444 1340 +23473 1339 +24935 1339 +27422 1338 +49035 1338 +40403 1336 +41818 1334 +35600 1334 +6284 1331 +7665 1331 +36171 1328 +3343 1326 +45887 1325 +40092 1324 +23511 1321 +47215 1320 +23626 1319 +11833 1317 +16080 1317 +49267 1316 +46581 1313 +33756 1308 +45251 1308 +25787 1307 +16041 1304 +27901 1302 +36382 1300 +33885 1300 +37906 1296 +19529 1295 +45907 1294 +37239 1293 +3282 1290 +38834 1290 +29613 1290 +18170 1289 +45533 1288 +31221 1285 +6852 1285 +36698 1285 +19395 1285 +40406 1285 +42519 1285 +23106 1283 +47947 1282 +33180 1281 +46570 1281 +37508 1278 +42300 1277 +27813 1275 +8485 1274 +39391 1274 +45750 1272 +27730 1271 +42680 1270 +45999 1270 +23305 1269 +36695 1267 +47546 1266 +11641 1263 +28401 1263 +50084 1263 +26825 1258 +45302 1254 +14524 1254 +33295 1253 +12100 1253 +38953 1253 +34697 1252 +39222 1252 +38021 1252 +36560 1250 +36180 1249 +3422 1248 +40456 1246 +16100 1245 +38543 1242 +37390 1240 +11037 1238 +25216 1235 +42479 1234 +26866 1234 +48835 1233 +34708 1232 +47393 1231 +43727 1229 +39827 1228 +29557 1227 +40069 1227 +26796 1226 +19455 1226 +44197 1225 +34640 1225 +10748 1224 +2155 1224 +27765 1216 +29241 1215 +43117 1215 +47795 1212 +47738 1211 +49694 1211 +20046 1211 +47864 1209 +46344 1208 +33623 1206 +43763 1204 +14060 1200 +40161 1199 +24539 1196 +16648 1195 +29281 1193 +17912 1193 +22065 1188 +43971 1188 +37863 1186 +35061 1186 +25618 1184 +12927 1183 +18125 1181 +37523 1179 +37101 1179 +41333 1177 +160 1176 +12241 1175 +14804 1175 +17720 1174 +27370 1173 +41248 1170 +17816 1169 +46746 1167 +39520 1165 +38271 1163 +34255 1163 +11727 1162 +33232 1162 +43879 1162 +21950 1162 +39098 1161 +22944 1161 +36750 1161 +10779 1160 +34519 1160 +25918 1158 +45281 1157 +36461 1154 +7503 1153 +36297 1152 +29572 1152 +37650 1150 +7180 1149 +30351 1149 +36764 1147 +23046 1145 +32756 1144 +48472 1143 +29472 1143 +5227 1142 +14865 1140 +12813 1140 +28243 1137 +8943 1137 +37848 1136 +19969 1136 +47527 1135 +34261 1132 +21903 1129 +35260 1127 +14560 1126 +42030 1126 +41592 1124 +47171 1124 +12426 1122 +46956 1122 +12071 1121 +20662 1119 +21711 1117 +36713 1117 +48227 1116 +22781 1115 +30660 1115 +43551 1113 +43328 1111 +41504 1109 +18163 1108 +31431 1106 +48873 1103 +31051 1102 +38184 1102 +40816 1097 +37345 1096 +35103 1095 +34704 1094 +27764 1092 +41096 1088 +47429 1088 +35561 1087 +41468 1086 +49551 1085 +43089 1083 +27570 1082 +47945 1078 +18477 1077 +49146 1077 +6847 1076 +16208 1073 +16934 1072 +16281 1067 +37420 1067 +24239 1065 +50143 1065 +2698 1064 +39786 1063 +37168 1061 +47082 1060 +45340 1059 +38124 1058 +16922 1056 +46310 1054 +11568 1053 +42527 1053 +32131 1053 +29460 1053 +44082 1052 +32849 1052 +18083 1052 +46796 1049 +49056 1049 +13486 1045 +30719 1041 +46966 1039 +28255 1036 +23138 1035 +22273 1035 +12486 1030 +45145 1028 +49070 1027 +37380 1027 +39695 1025 +41661 1024 +47611 1019 +48144 1019 +22345 1018 +44204 1017 +45640 1017 +45115 1017 +42752 1016 +31944 1016 +40674 1015 +41343 1014 +8762 1014 +45148 1013 +44575 1011 +33843 1011 +30667 1010 +43266 1008 +44948 1006 +33252 1005 +26911 1005 +35930 1004 +42654 1003 +42245 1002 +21424 999 +33151 998 +45865 997 +44052 996 +40822 996 +23294 993 +31386 993 +12640 989 +9705 989 +20786 986 +16471 985 +29495 984 +49535 984 +12953 982 +25915 982 +9020 978 +11647 976 +1209 976 +13211 972 +47139 966 +43966 966 +22174 961 +41002 960 +46976 959 +39258 959 +46086 958 +39611 958 +43922 956 +48080 956 +39231 955 +35407 955 +19820 951 +21324 950 +19421 950 +47826 949 +45442 949 +33031 946 +48997 943 +24714 941 +33221 940 +23001 940 +6043 939 +41585 939 +26076 938 +27156 935 +47715 934 +40561 931 +29325 931 +48313 931 +48364 929 +47340 928 +12045 927 +37605 925 +36853 922 +30751 920 +12675 919 +47671 915 +40041 914 +38165 911 +9497 911 +36752 911 +10790 908 +7948 906 +15697 905 +44165 904 +33213 903 +28544 903 +35707 902 +44917 901 +29848 900 +12736 898 +26475 896 +31965 892 +37096 892 +25195 892 +48958 891 +45992 889 +38963 888 +38218 887 +27632 887 +42655 880 +36119 879 +32403 878 +25001 873 +37771 873 +17845 872 +18670 872 +29312 869 +41726 862 +15139 862 +25597 862 +43488 859 +21481 858 +25081 858 +40223 857 +45474 856 +22718 853 +27972 853 +46497 850 +32003 849 +21737 847 +24029 847 +47676 846 +31023 846 +37695 843 +21477 843 +37955 842 +42669 840 +32092 840 +43549 838 +39666 836 +39482 833 +19682 833 +46764 832 +7535 832 +6927 832 +14367 831 +28165 830 +25745 830 +34222 828 +31724 827 +143 824 +42000 823 +48670 823 +17158 821 +11839 820 +33524 817 +39105 816 +25902 815 +29554 815 +41791 815 +22316 814 +44646 813 +32291 809 +49426 805 +37456 803 +41309 803 +43015 803 +38551 800 +42565 799 +2549 797 +44829 796 +9263 796 +38461 795 +43889 795 +35269 794 +29994 794 +36003 794 +31917 793 +22270 791 +12424 791 +46274 791 +33470 790 +33778 789 +46092 788 +46036 785 +41335 784 +31456 784 +21775 783 +15211 783 +15081 782 +3786 781 +29841 781 +16116 780 +43297 777 +43903 776 +34976 772 +19587 772 +4431 772 +40391 771 +49201 771 +21763 769 +21091 768 +45298 767 +45990 766 +48265 763 +37374 760 +29847 760 +42924 759 +39763 756 +7961 755 +41150 753 +37014 753 +30298 751 +30838 750 +144 748 +175 748 +40664 746 +9202 745 +15362 745 +38626 745 +39415 744 +36875 741 +15886 741 +5392 741 +15306 741 +23497 739 +25410 738 +45398 736 +14030 735 +38430 734 +28133 734 +22758 733 +18189 733 +34284 730 +30138 729 +46741 728 +23991 728 +29940 727 +36584 727 +37423 727 +49568 726 +20379 725 +47903 722 +8795 721 +6141 719 +31813 719 +48678 717 +22341 716 +46202 715 +27802 714 +41293 713 +27352 711 +44148 709 +37961 708 +29785 706 +40477 705 +29412 705 +21948 704 +43900 701 +40367 699 +22446 699 +43718 697 +42684 692 +37772 690 +45911 688 +46866 686 +30831 685 +8728 685 +36685 685 +10108 684 +33918 684 +39618 682 +48746 681 +39317 681 +34585 680 +3556 679 +21955 678 +15790 674 +6336 673 +9603 673 +47505 672 +46189 668 +25226 668 +23719 667 +49518 665 +37083 664 +39691 661 +43238 658 +25295 656 +6480 653 +23884 653 +48688 652 +20645 650 +36310 650 +48972 650 +31424 647 +48702 646 +34534 642 +41216 642 +30916 640 +33084 639 +37588 639 +45358 638 +40265 637 +49007 636 +30929 636 +8815 635 +44274 634 +8955 633 +38145 630 +43357 628 +37581 627 +29705 626 +43962 626 +33116 626 +22934 624 +46506 623 +42376 622 +36142 622 +11966 622 +32284 620 +36278 619 +9841 617 +29966 616 +38105 616 +17433 615 +19779 614 +170 613 +25060 613 +7280 612 +36737 612 +29415 612 +24679 611 +22784 611 +47762 610 +982 608 +48416 606 +9468 604 +46083 603 +42348 602 +43023 599 +29047 599 +46532 598 +29582 598 +34237 597 +24442 597 +22263 596 +41146 595 +12712 594 +45617 593 +27643 593 +47078 593 +39893 591 +49704 589 +49339 589 +32288 588 +49969 587 +37176 586 +43913 586 +24032 585 +15474 583 +44557 583 +2468 581 +43055 580 +34580 580 +37190 576 +8773 576 +19841 575 +46118 573 +35266 573 +43291 571 +43491 569 +22906 568 +13700 567 +37266 567 +28657 566 +10523 566 +38593 566 +6710 565 +31175 565 +13562 562 +40717 561 +23854 555 +38574 555 +48553 554 +34758 554 +29740 553 +28235 552 +9882 552 +50101 549 +39747 547 +37527 547 +29953 545 +45228 545 +43095 543 +44627 542 +40748 540 +40948 539 +14077 535 +41092 535 +42314 535 +43636 531 +37763 530 +29212 529 +42757 527 +46777 526 +42249 524 +28350 524 +23224 523 +27032 522 +41215 521 +15083 519 +14726 519 +28360 518 +44051 518 +16148 517 +16792 515 +13298 515 +36338 514 +48393 513 +16150 512 +31854 512 +15661 511 +43648 511 +31817 511 +36470 510 +21316 507 +47581 506 +35430 506 +38463 505 +48774 505 +49135 504 +48801 503 +27246 502 +32015 502 +21364 501 +19785 498 +45520 498 +24122 497 +35067 497 +5622 497 +15961 496 +48557 493 +23287 492 +10144 491 +10097 491 +26867 487 +41888 487 +49409 484 +24336 484 +39488 483 +25248 482 +20398 481 +38892 479 +20322 479 +16626 477 +35508 476 +26791 476 +40290 475 +49546 475 +16253 474 +43768 473 +43197 473 +14099 470 +47418 469 +9287 468 +14692 468 +25053 467 +45227 466 +42164 466 +14512 466 +28632 465 +45986 464 +30134 462 +38519 462 +28955 460 +29636 458 +33489 457 +44057 457 +20036 457 +23376 456 +15272 455 +36166 452 +29577 450 +32541 449 +24373 448 +24689 448 +39737 447 +5571 447 +10160 446 +49582 446 +49015 446 +44755 444 +23711 442 +42804 441 +28013 441 +40672 440 +37143 440 +35439 439 +8346 439 +35050 437 +15116 436 +36447 436 +44926 435 +16323 434 +31768 434 +44174 431 +14777 430 +15853 430 +21451 428 +41340 428 +47232 428 +46999 427 +35799 425 +35555 425 +25131 424 +19927 423 +41349 423 +8983 421 +34991 417 +20213 416 +34949 415 +33682 415 +44332 414 +29589 413 +49771 413 +49129 412 +24186 412 +33803 411 +48893 410 +25645 408 +19415 407 +44547 407 +35235 405 +39321 404 +40786 403 +37389 401 +34650 401 +45717 399 +20590 397 +8994 396 +26054 393 +48031 393 +13697 391 +2955 390 +40266 386 +1977 385 +17405 384 +2115 382 +3207 382 +45082 381 +3156 381 +32517 381 +43102 381 +40537 380 +43375 378 +47932 378 +30109 377 +48737 376 +16165 373 +19629 371 +30072 370 +37485 368 +3286 368 +48995 367 +24493 367 +25465 366 +43848 366 +21253 365 +24440 364 +49607 359 +37227 359 +48541 358 +41573 358 +40386 353 +7601 352 +29851 352 +22133 352 +29659 352 +49011 351 +45218 351 +18004 350 +40720 349 +37467 346 +42783 345 +32398 343 +49960 342 +35306 342 +34345 341 +27007 341 +12404 340 +42543 339 +42638 338 +49167 335 +31208 334 +45784 333 +33296 333 +20512 332 +32742 332 +18356 331 +44587 331 +30165 329 +41433 327 +40364 327 +9049 326 +2918 326 +17683 325 +17681 325 +24077 324 +25611 324 +6166 323 +39681 323 +49855 323 +44256 322 +39333 322 +49073 320 +31937 320 +37509 317 +43077 316 +15842 316 +33384 315 +25497 315 +30965 315 +48497 315 +16193 315 +47987 313 +22725 313 +46140 313 +2804 312 +32524 311 +20532 310 +47939 310 +28542 310 +39192 309 +10125 308 +17773 305 +47423 305 +46268 301 +24363 301 +20513 300 +26498 300 +50179 299 +36469 298 +34543 297 +20115 297 +28696 297 +36786 296 +26712 294 +31090 293 +42062 290 +23131 287 +38326 287 +34373 287 +26095 286 +25626 286 +46424 285 +15506 285 +9000 285 +31936 283 +30800 282 +49058 281 +27551 280 +44648 276 +12240 276 +18803 275 +7589 274 +41641 274 +40109 274 +1896 273 +47000 273 +32518 272 +45379 272 +45635 272 +33529 272 +13765 270 +40420 270 +12845 269 +16201 268 +33465 268 +24001 267 +49249 267 +41853 267 +4010 266 +8418 265 +49813 265 +25581 264 +43481 263 +46788 262 +41629 262 +24710 261 +35144 261 +39907 260 +47021 260 +20801 260 +25128 259 +30812 259 +48304 258 +18115 257 +48404 256 +33829 256 +23846 255 +6112 254 +5997 254 +6987 253 +7677 253 +26503 253 +44161 253 +46491 252 +48231 251 +11910 251 +37669 250 +35702 249 +49464 248 +41323 248 +37233 247 +26998 246 +36828 246 +24806 244 +32207 243 +30762 242 +42637 242 +16103 242 +172 242 +49472 242 +25084 241 +7105 239 +19791 238 +41974 237 +45335 237 +31052 236 +37426 236 +13177 235 +38894 235 +37495 234 +25549 234 +25795 234 +43801 234 +43282 233 +13296 231 +38149 231 +24466 230 +28264 230 +34525 230 +48448 228 +36581 227 +36521 227 +49082 225 +48318 225 +8755 225 +36792 223 +43353 223 +19039 223 +43302 222 +20677 220 +47307 220 +32310 220 +31732 220 +19021 219 +17401 219 +36922 219 +12717 218 +33426 217 +46904 217 +33309 216 +33795 215 +29993 214 +4242 214 +32546 212 +35818 208 +47175 208 +11896 208 +34934 208 +48908 208 +42367 208 +43769 207 +50159 207 +35514 205 +45495 204 +43213 203 +2590 203 +4895 203 +27542 202 +39996 201 +35063 201 +47569 200 +40283 198 +37817 198 +1543 197 +37757 197 +14592 196 +27733 195 +39394 195 +13979 194 +12662 193 +2887 193 +42287 191 +22241 191 +32891 189 +11548 189 +26642 188 +11480 187 +15755 186 +10060 186 +15913 186 +45144 185 +42026 185 +19227 184 +33761 184 +25207 182 +38122 182 +44916 181 +29113 181 +22831 179 +13945 179 +6681 178 +34901 178 +22322 178 +50108 177 +29164 176 +23544 174 +46545 174 +15351 173 +22496 173 +39742 173 +11606 173 +36796 173 +26229 171 +39115 171 +35572 170 +16959 169 +30716 168 +37642 167 +44651 167 +27852 167 +40073 167 +37913 167 +45975 166 +46968 166 +38362 165 +29446 165 +7359 164 +17473 163 +40012 163 +47258 163 +46677 162 +49390 162 +45150 162 +24022 161 +24898 160 +32239 160 +45434 159 +27509 159 +39703 159 +3587 157 +11974 157 +36726 156 +15685 155 +38653 154 +24618 154 +17532 153 +24973 151 +41386 151 +48944 151 +41678 151 +7804 149 +33991 148 +21959 146 +14531 145 +41441 144 +37412 144 +41658 143 +6399 142 +2941 142 +33893 141 +41849 140 +11919 140 +44912 139 +46541 138 +32088 137 +35604 137 +48610 137 +50033 137 +19476 137 +25926 136 +23330 136 +49843 136 +50001 135 +48999 134 +48458 134 +36301 133 +16646 132 +47271 132 +40111 132 +23329 132 +41313 131 +23728 130 +33207 128 +47542 127 +37082 127 +10269 126 +39434 126 +48204 125 +43667 124 +6533 122 +49273 122 +40361 122 +49324 121 +25589 121 +8438 120 +2432 120 +17761 120 +10052 120 +37435 119 +47981 118 +12869 116 +34607 115 +41006 113 +47596 113 +45123 112 +43246 110 +40719 110 +26700 110 +41945 109 +152 109 +16098 109 +6598 109 +47253 107 +46249 107 +29021 106 +45563 106 +47147 105 +45539 105 +35343 105 +43241 105 +50216 105 +43734 105 +35922 104 +29226 104 +44431 104 +221 104 +25970 104 +27293 103 +1841 103 +14468 103 +42311 103 +47559 102 +49643 102 +35069 101 +8115 101 +44233 100 +16303 100 +47282 99 +40629 99 +39008 99 +32509 98 +32917 98 +42396 98 +176 97 +36169 96 +48753 96 +10253 94 +40516 94 +3523 93 +48185 93 +49086 93 +44104 92 +31204 92 +43839 91 +27924 90 +17933 90 +49476 89 +36596 89 +46745 89 +44167 87 +30531 87 +47486 87 +40549 86 +35992 85 +34832 85 +33468 85 +46996 84 +40493 84 +43242 84 +7134 83 +11585 80 +22640 80 +37811 80 +42869 80 +39860 79 +25719 78 +23984 77 +11273 76 +41481 76 +20554 75 +13198 75 +28725 74 +42210 74 +155 73 +48727 73 +35318 72 +41365 71 +32511 70 +40345 70 +31708 69 +44872 69 +41832 69 +9968 69 +28670 69 +30478 67 +10298 67 +11885 66 +11737 66 +17787 66 +41939 65 +41868 65 +39467 65 +41538 64 +44546 63 +20503 62 +47682 61 +4060 61 +16068 60 +48457 59 +36473 59 +45449 59 +50113 58 +15040 58 +46110 58 +47614 58 +50116 57 +43313 57 +18945 57 +29795 57 +37858 56 +26534 55 +18433 55 +39886 55 +33490 54 +40415 54 +36704 53 +44785 53 +43899 51 +32865 51 +47530 51 +39364 50 +42535 49 +25887 49 +37981 49 +49527 49 +33937 48 +36940 48 +22315 47 +48874 47 +4204 47 +23596 47 +31478 47 +42035 46 +9286 46 +38016 46 +11689 46 +42785 45 +46222 45 +33994 45 +47794 45 +43897 45 +42877 45 +48953 44 +44686 44 +37545 44 +45435 43 +34386 42 +7260 42 +13171 42 +23926 41 +24307 40 +31820 40 +44392 39 +48600 38 +41230 38 +38390 38 +13150 37 +154 37 +38243 36 +29447 36 +44326 36 +29646 35 +19073 35 +15041 35 +45433 34 +42382 32 +38377 32 +47540 32 +47936 32 +47432 31 +49149 31 +30432 31 +39280 30 +48366 30 +12943 29 +37922 29 +14695 29 +13426 29 +16782 28 +49997 28 +36926 28 +23613 28 +22675 28 +35098 28 +6408 28 +33153 28 +31739 28 +43518 27 +7782 27 +29752 27 +23363 26 +45823 26 +43053 26 +40236 24 +37787 24 +49029 24 +33023 24 +20804 24 +34103 24 +27013 23 +37991 23 +42943 23 +44033 23 +41380 22 +25698 22 +42750 22 +25362 22 +44555 22 +22039 20 +42983 20 +45762 20 +27006 20 +12677 19 +48219 19 +43394 19 +37662 19 +17553 18 +45422 18 +30439 18 +14341 18 +11592 17 +33929 17 +17629 17 +42889 17 +153 17 +14223 17 +43010 17 +32843 16 +48193 16 +35793 16 +36475 16 +31161 15 +16822 15 +47182 15 +44112 14 +34604 14 +31881 14 +46402 13 +36937 12 +28500 12 +24731 12 +27584 12 +43796 11 +36481 11 +40703 11 +19049 10 +44444 10 +4690 10 +42470 9 +41977 9 +36917 9 +46948 9 +10658 9 +38250 9 +43298 8 +45953 8 +36929 8 +42234 7 +38160 7 +47490 7 +40235 7 +5808 7 +45786 7 +36490 6 +5367 6 +27534 6 +21807 5 +36886 5 +39693 5 +30684 5 +37226 5 +15243 5 +34633 5 +22997 5 +25658 4 +45321 4 +8980 4 +47648 4 +34206 4 +43569 4 +36862 3 +49778 3 +45392 3 +42066 3 +36130 3 +46939 3 +6438 3 +34842 2 +48527 2 +38370 2 +34473 2 +40278 2 +20174 2 +5815 1 +9364 1 +39142 1 +47703 1 +49074 1 +31536 1 +14827 1 +23090 1 +43735 1 +24847 1 +40219 1 +32437 1 +31727 1 +124 0 +125 0 +173 0 +174 0 +177 0 +178 0 +179 0 +180 0 +181 0 +182 0 +183 0 +184 0 +185 0 +186 0 +187 0 +188 0 +189 0 +190 0 +191 0 +192 0 +193 0 +194 0 +195 0 +196 0 +197 0 +198 0 +199 0 +200 0 +201 0 +202 0 +203 0 +204 0 +205 0 +206 0 +207 0 +208 0 +209 0 +210 0 +211 0 +212 0 +213 0 +214 0 +215 0 +216 0 +217 0 +218 0 +219 0 +628 0 +1849 0 +4603 0 +5624 0 +8828 0 +11504 0 +12781 0 +17811 0 +17900 0 +18472 0 +22686 0 +22757 0 +23282 0 +23614 0 +23785 0 +24293 0 +24934 0 +25193 0 +25502 0 +25992 0 +28666 0 +29342 0 +29372 0 +30202 0 +30208 0 +30209 0 +30210 0 +30211 0 +30212 0 +30213 0 +30897 0 +30898 0 +30899 0 +30905 0 +30906 0 +31032 0 +31538 0 +31573 0 +31576 0 +31666 0 +31765 0 +31783 0 +31886 0 +31957 0 +32047 0 +32406 0 +33434 0 +33454 0 +33477 0 +33813 0 +34027 0 +34448 0 +34504 0 +34516 0 +35207 0 +35496 0 +35579 0 +36173 0 +36174 0 +36935 0 +36938 0 +37444 0 +37574 0 +37579 0 +37631 0 +37842 0 +38214 0 +39165 0 +39172 0 +39177 0 +39253 0 +39374 0 +39446 0 +39655 0 +39714 0 +39749 0 +39752 0 +39753 0 +39755 0 +39756 0 +39757 0 +39803 0 +39811 0 +39820 0 +39821 0 +39906 0 +40240 0 +40241 0 +40242 0 +41297 0 +41383 0 +41551 0 +42089 0 +42090 0 +42202 0 +42424 0 +42496 0 +42586 0 +42728 0 +43038 0 +43065 0 +43177 0 +43361 0 +43453 0 +44320 0 +45003 0 +45199 0 +45544 0 +45545 0 +45706 0 +46600 0 +47198 0 +47571 0 +47654 0 +47934 0 +48069 0 +48396 0 +49731 0 +49781 0 +50009 0 +50256 0 +madeupword0000 0 +madeupword0001 0 +madeupword0002 0 \ No newline at end of file diff --git a/dotnet/samples/KernelSyntaxExamples/Resources/EnglishRoberta/encoder.json b/dotnet/samples/KernelSyntaxExamples/Resources/EnglishRoberta/encoder.json new file mode 100644 index 000000000000..396f9b4d98c7 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Resources/EnglishRoberta/encoder.json @@ -0,0 +1,50259 @@ +{ + "!": 0, + "\"": 1, + "#": 2, + "$": 3, + "%": 4, + "&": 5, + "'": 6, + "(": 7, + ")": 8, + "*": 9, + "+": 10, + ",": 11, + "-": 12, + ".": 13, + "/": 14, + "0": 15, + "1": 16, + "2": 17, + "3": 18, + "4": 19, + "5": 20, + "6": 21, + "7": 22, + "8": 23, + "9": 24, + ":": 25, + ";": 26, + "<": 27, + "=": 28, + ">": 29, + "?": 30, + "@": 31, + "A": 32, + "B": 33, + "C": 34, + "D": 35, + "E": 36, + "F": 37, + "G": 38, + "H": 39, + "I": 40, + "J": 41, + "K": 42, + "L": 43, + "M": 44, + "N": 45, + "O": 46, + "P": 47, + "Q": 48, + "R": 49, + "S": 50, + "T": 51, + "U": 52, + "V": 53, + "W": 54, + "X": 55, + "Y": 56, + "Z": 57, + "[": 58, + "\\": 59, + "]": 60, + "^": 61, + "_": 62, + "`": 63, + "a": 64, + "b": 65, + "c": 66, + "d": 67, + "e": 68, + "f": 69, + "g": 70, + "h": 71, + "i": 72, + "j": 73, + "k": 74, + "l": 75, + "m": 76, + "n": 77, + "o": 78, + "p": 79, + "q": 80, + "r": 81, + "s": 82, + "t": 83, + "u": 84, + "v": 85, + "w": 86, + "x": 87, + "y": 88, + "z": 89, + "{": 90, + "|": 91, + "}": 92, + "~": 93, + "\u00a1": 94, + "\u00a2": 95, + "\u00a3": 96, + "\u00a4": 97, + "\u00a5": 98, + "\u00a6": 99, + "\u00a7": 100, + "\u00a8": 101, + "\u00a9": 102, + "\u00aa": 103, + "\u00ab": 104, + "\u00ac": 105, + "\u00ae": 106, + "\u00af": 107, + "\u00b0": 108, + "\u00b1": 109, + "\u00b2": 110, + "\u00b3": 111, + "\u00b4": 112, + "\u00b5": 113, + "\u00b6": 114, + "\u00b7": 115, + "\u00b8": 116, + "\u00b9": 117, + "\u00ba": 118, + "\u00bb": 119, + "\u00bc": 120, + "\u00bd": 121, + "\u00be": 122, + "\u00bf": 123, + "\u00c0": 124, + "\u00c1": 125, + "\u00c2": 126, + "\u00c3": 127, + "\u00c4": 128, + "\u00c5": 129, + "\u00c6": 130, + "\u00c7": 131, + "\u00c8": 132, + "\u00c9": 133, + "\u00ca": 134, + "\u00cb": 135, + "\u00cc": 136, + "\u00cd": 137, + "\u00ce": 138, + "\u00cf": 139, + "\u00d0": 140, + "\u00d1": 141, + "\u00d2": 142, + "\u00d3": 143, + "\u00d4": 144, + "\u00d5": 145, + "\u00d6": 146, + "\u00d7": 147, + "\u00d8": 148, + "\u00d9": 149, + "\u00da": 150, + "\u00db": 151, + "\u00dc": 152, + "\u00dd": 153, + "\u00de": 154, + "\u00df": 155, + "\u00e0": 156, + "\u00e1": 157, + "\u00e2": 158, + "\u00e3": 159, + "\u00e4": 160, + "\u00e5": 161, + "\u00e6": 162, + "\u00e7": 163, + "\u00e8": 164, + "\u00e9": 165, + "\u00ea": 166, + "\u00eb": 167, + "\u00ec": 168, + "\u00ed": 169, + "\u00ee": 170, + "\u00ef": 171, + "\u00f0": 172, + "\u00f1": 173, + "\u00f2": 174, + "\u00f3": 175, + "\u00f4": 176, + "\u00f5": 177, + "\u00f6": 178, + "\u00f7": 179, + "\u00f8": 180, + "\u00f9": 181, + "\u00fa": 182, + "\u00fb": 183, + "\u00fc": 184, + "\u00fd": 185, + "\u00fe": 186, + "\u00ff": 187, + "\u0100": 188, + "\u0101": 189, + "\u0102": 190, + "\u0103": 191, + "\u0104": 192, + "\u0105": 193, + "\u0106": 194, + "\u0107": 195, + "\u0108": 196, + "\u0109": 197, + "\u010a": 198, + "\u010b": 199, + "\u010c": 200, + "\u010d": 201, + "\u010e": 202, + "\u010f": 203, + "\u0110": 204, + "\u0111": 205, + "\u0112": 206, + "\u0113": 207, + "\u0114": 208, + "\u0115": 209, + "\u0116": 210, + "\u0117": 211, + "\u0118": 212, + "\u0119": 213, + "\u011a": 214, + "\u011b": 215, + "\u011c": 216, + "\u011d": 217, + "\u011e": 218, + "\u011f": 219, + "\u0120": 220, + "\u0121": 221, + "\u0122": 222, + "\u0123": 223, + "\u0124": 224, + "\u0125": 225, + "\u0126": 226, + "\u0127": 227, + "\u0128": 228, + "\u0129": 229, + "\u012a": 230, + "\u012b": 231, + "\u012c": 232, + "\u012d": 233, + "\u012e": 234, + "\u012f": 235, + "\u0130": 236, + "\u0131": 237, + "\u0132": 238, + "\u0133": 239, + "\u0134": 240, + "\u0135": 241, + "\u0136": 242, + "\u0137": 243, + "\u0138": 244, + "\u0139": 245, + "\u013a": 246, + "\u013b": 247, + "\u013c": 248, + "\u013d": 249, + "\u013e": 250, + "\u013f": 251, + "\u0140": 252, + "\u0141": 253, + "\u0142": 254, + "\u0143": 255, + "\u0120t": 256, + "\u0120a": 257, + "he": 258, + "in": 259, + "re": 260, + "on": 261, + "\u0120the": 262, + "er": 263, + "\u0120s": 264, + "at": 265, + "\u0120w": 266, + "\u0120o": 267, + "en": 268, + "\u0120c": 269, + "it": 270, + "is": 271, + "an": 272, + "or": 273, + "es": 274, + "\u0120b": 275, + "ed": 276, + "\u0120f": 277, + "ing": 278, + "\u0120p": 279, + "ou": 280, + "\u0120an": 281, + "al": 282, + "ar": 283, + "\u0120to": 284, + "\u0120m": 285, + "\u0120of": 286, + "\u0120in": 287, + "\u0120d": 288, + "\u0120h": 289, + "\u0120and": 290, + "ic": 291, + "as": 292, + "le": 293, + "\u0120th": 294, + "ion": 295, + "om": 296, + "ll": 297, + "ent": 298, + "\u0120n": 299, + "\u0120l": 300, + "st": 301, + "\u0120re": 302, + "ve": 303, + "\u0120e": 304, + "ro": 305, + "ly": 306, + "\u0120be": 307, + "\u0120g": 308, + "\u0120T": 309, + "ct": 310, + "\u0120S": 311, + "id": 312, + "ot": 313, + "\u0120I": 314, + "ut": 315, + "et": 316, + "\u0120A": 317, + "\u0120is": 318, + "\u0120on": 319, + "im": 320, + "am": 321, + "ow": 322, + "ay": 323, + "ad": 324, + "se": 325, + "\u0120that": 326, + "\u0120C": 327, + "ig": 328, + "\u0120for": 329, + "ac": 330, + "\u0120y": 331, + "ver": 332, + "ur": 333, + "\u0120u": 334, + "ld": 335, + "\u0120st": 336, + "\u0120M": 337, + "'s": 338, + "\u0120he": 339, + "\u0120it": 340, + "ation": 341, + "ith": 342, + "ir": 343, + "ce": 344, + "\u0120you": 345, + "il": 346, + "\u0120B": 347, + "\u0120wh": 348, + "ol": 349, + "\u0120P": 350, + "\u0120with": 351, + "\u01201": 352, + "ter": 353, + "ch": 354, + "\u0120as": 355, + "\u0120we": 356, + "\u0120(": 357, + "nd": 358, + "ill": 359, + "\u0120D": 360, + "if": 361, + "\u01202": 362, + "ag": 363, + "ers": 364, + "ke": 365, + "\u0120\"": 366, + "\u0120H": 367, + "em": 368, + "\u0120con": 369, + "\u0120W": 370, + "\u0120R": 371, + "her": 372, + "\u0120was": 373, + "\u0120r": 374, + "od": 375, + "\u0120F": 376, + "ul": 377, + "ate": 378, + "\u0120at": 379, + "ri": 380, + "pp": 381, + "ore": 382, + "\u0120The": 383, + "\u0120se": 384, + "us": 385, + "\u0120pro": 386, + "\u0120ha": 387, + "um": 388, + "\u0120are": 389, + "\u0120de": 390, + "ain": 391, + "and": 392, + "\u0120or": 393, + "igh": 394, + "est": 395, + "ist": 396, + "ab": 397, + "rom": 398, + "\u0120N": 399, + "th": 400, + "\u0120com": 401, + "\u0120G": 402, + "un": 403, + "op": 404, + "00": 405, + "\u0120L": 406, + "\u0120not": 407, + "ess": 408, + "\u0120ex": 409, + "\u0120v": 410, + "res": 411, + "\u0120E": 412, + "ew": 413, + "ity": 414, + "ant": 415, + "\u0120by": 416, + "el": 417, + "os": 418, + "ort": 419, + "oc": 420, + "qu": 421, + "\u0120from": 422, + "\u0120have": 423, + "\u0120su": 424, + "ive": 425, + "ould": 426, + "\u0120sh": 427, + "\u0120this": 428, + "nt": 429, + "ra": 430, + "pe": 431, + "ight": 432, + "art": 433, + "ment": 434, + "\u0120al": 435, + "ust": 436, + "end": 437, + "--": 438, + "all": 439, + "\u0120O": 440, + "ack": 441, + "\u0120ch": 442, + "\u0120le": 443, + "ies": 444, + "red": 445, + "ard": 446, + "\u00e2\u0122": 447, + "out": 448, + "\u0120J": 449, + "\u0120ab": 450, + "ear": 451, + "iv": 452, + "ally": 453, + "our": 454, + "ost": 455, + "gh": 456, + "pt": 457, + "\u0120pl": 458, + "ast": 459, + "\u0120can": 460, + "ak": 461, + "ome": 462, + "ud": 463, + "The": 464, + "\u0120his": 465, + "\u0120do": 466, + "\u0120go": 467, + "\u0120has": 468, + "ge": 469, + "'t": 470, + "\u0120U": 471, + "rou": 472, + "\u0120sa": 473, + "\u0120j": 474, + "\u0120but": 475, + "\u0120wor": 476, + "\u0120all": 477, + "ect": 478, + "\u0120k": 479, + "ame": 480, + "\u0120will": 481, + "ok": 482, + "\u0120whe": 483, + "\u0120they": 484, + "ide": 485, + "01": 486, + "ff": 487, + "ich": 488, + "pl": 489, + "ther": 490, + "\u0120tr": 491, + "..": 492, + "\u0120int": 493, + "ie": 494, + "ure": 495, + "age": 496, + "\u0120ne": 497, + "ial": 498, + "ap": 499, + "ine": 500, + "ice": 501, + "\u0120me": 502, + "\u0120out": 503, + "ans": 504, + "one": 505, + "ong": 506, + "ions": 507, + "\u0120who": 508, + "\u0120K": 509, + "\u0120up": 510, + "\u0120their": 511, + "\u0120ad": 512, + "\u01203": 513, + "\u0120us": 514, + "ated": 515, + "ous": 516, + "\u0120more": 517, + "ue": 518, + "og": 519, + "\u0120St": 520, + "ind": 521, + "ike": 522, + "\u0120so": 523, + "ime": 524, + "per": 525, + ".\"": 526, + "ber": 527, + "iz": 528, + "act": 529, + "\u0120one": 530, + "\u0120said": 531, + "\u0120-": 532, + "are": 533, + "\u0120your": 534, + "cc": 535, + "\u0120Th": 536, + "\u0120cl": 537, + "ep": 538, + "ake": 539, + "able": 540, + "ip": 541, + "\u0120cont": 542, + "\u0120which": 543, + "ia": 544, + "\u0120im": 545, + "\u0120about": 546, + "\u0120were": 547, + "very": 548, + "ub": 549, + "\u0120had": 550, + "\u0120en": 551, + "\u0120comp": 552, + ",\"": 553, + "\u0120In": 554, + "\u0120un": 555, + "\u0120ag": 556, + "ire": 557, + "ace": 558, + "au": 559, + "ary": 560, + "\u0120would": 561, + "ass": 562, + "ry": 563, + "\u0120\u00e2\u0122": 564, + "cl": 565, + "ook": 566, + "ere": 567, + "so": 568, + "\u0120V": 569, + "ign": 570, + "ib": 571, + "\u0120off": 572, + "\u0120te": 573, + "ven": 574, + "\u0120Y": 575, + "ile": 576, + "ose": 577, + "ite": 578, + "orm": 579, + "\u0120201": 580, + "\u0120res": 581, + "\u0120man": 582, + "\u0120per": 583, + "\u0120other": 584, + "ord": 585, + "ult": 586, + "\u0120been": 587, + "\u0120like": 588, + "ase": 589, + "ance": 590, + "ks": 591, + "ays": 592, + "own": 593, + "ence": 594, + "\u0120dis": 595, + "ction": 596, + "\u0120any": 597, + "\u0120app": 598, + "\u0120sp": 599, + "int": 600, + "ress": 601, + "ations": 602, + "ail": 603, + "\u01204": 604, + "ical": 605, + "\u0120them": 606, + "\u0120her": 607, + "ount": 608, + "\u0120Ch": 609, + "\u0120ar": 610, + "\u0120if": 611, + "\u0120there": 612, + "\u0120pe": 613, + "\u0120year": 614, + "av": 615, + "\u0120my": 616, + "\u0120some": 617, + "\u0120when": 618, + "ough": 619, + "ach": 620, + "\u0120than": 621, + "ru": 622, + "ond": 623, + "ick": 624, + "\u0120over": 625, + "vel": 626, + "\u0120qu": 627, + "\u010a\u010a": 628, + "\u0120sc": 629, + "reat": 630, + "ree": 631, + "\u0120It": 632, + "ound": 633, + "port": 634, + "\u0120also": 635, + "\u0120part": 636, + "fter": 637, + "\u0120kn": 638, + "\u0120bec": 639, + "\u0120time": 640, + "ens": 641, + "\u01205": 642, + "ople": 643, + "\u0120what": 644, + "\u0120no": 645, + "du": 646, + "mer": 647, + "ang": 648, + "\u0120new": 649, + "----": 650, + "\u0120get": 651, + "ory": 652, + "ition": 653, + "ings": 654, + "\u0120just": 655, + "\u0120into": 656, + "\u01200": 657, + "ents": 658, + "ove": 659, + "te": 660, + "\u0120people": 661, + "\u0120pre": 662, + "\u0120its": 663, + "\u0120rec": 664, + "\u0120tw": 665, + "ian": 666, + "irst": 667, + "ark": 668, + "ors": 669, + "\u0120work": 670, + "ade": 671, + "ob": 672, + "\u0120she": 673, + "\u0120our": 674, + "wn": 675, + "ink": 676, + "lic": 677, + "\u012019": 678, + "\u0120He": 679, + "ish": 680, + "nder": 681, + "ause": 682, + "\u0120him": 683, + "ons": 684, + "\u0120[": 685, + "\u0120ro": 686, + "form": 687, + "ild": 688, + "ates": 689, + "vers": 690, + "\u0120only": 691, + "oll": 692, + "\u0120spe": 693, + "ck": 694, + "ell": 695, + "amp": 696, + "\u0120acc": 697, + "\u0120bl": 698, + "ious": 699, + "urn": 700, + "ft": 701, + "ood": 702, + "\u0120how": 703, + "hed": 704, + "\u0120'": 705, + "\u0120after": 706, + "aw": 707, + "\u0120att": 708, + "ov": 709, + "ne": 710, + "\u0120play": 711, + "erv": 712, + "ict": 713, + "\u0120could": 714, + "itt": 715, + "\u0120am": 716, + "\u0120first": 717, + "\u01206": 718, + "\u0120act": 719, + "\u0120$": 720, + "ec": 721, + "hing": 722, + "ual": 723, + "ull": 724, + "\u0120comm": 725, + "oy": 726, + "old": 727, + "ces": 728, + "ater": 729, + "\u0120fe": 730, + "\u0120bet": 731, + "we": 732, + "iff": 733, + "\u0120two": 734, + "ock": 735, + "\u0120back": 736, + ").": 737, + "ident": 738, + "\u0120under": 739, + "rough": 740, + "sel": 741, + "xt": 742, + "\u0120may": 743, + "round": 744, + "\u0120po": 745, + "ph": 746, + "iss": 747, + "\u0120des": 748, + "\u0120most": 749, + "\u0120did": 750, + "\u0120add": 751, + "ject": 752, + "\u0120inc": 753, + "fore": 754, + "\u0120pol": 755, + "ont": 756, + "\u0120again": 757, + "clud": 758, + "tern": 759, + "\u0120know": 760, + "\u0120need": 761, + "\u0120cons": 762, + "\u0120co": 763, + "\u0120.": 764, + "\u0120want": 765, + "\u0120see": 766, + "\u01207": 767, + "ning": 768, + "iew": 769, + "\u0120This": 770, + "ced": 771, + "\u0120even": 772, + "\u0120ind": 773, + "ty": 774, + "\u0120We": 775, + "ath": 776, + "\u0120these": 777, + "\u0120pr": 778, + "\u0120use": 779, + "\u0120because": 780, + "\u0120fl": 781, + "ng": 782, + "\u0120now": 783, + "\u0120\u00e2\u0122\u0135": 784, + "com": 785, + "ise": 786, + "\u0120make": 787, + "\u0120then": 788, + "ower": 789, + "\u0120every": 790, + "\u0120Un": 791, + "\u0120sec": 792, + "oss": 793, + "uch": 794, + "\u0120em": 795, + "\u0120=": 796, + "\u0120Re": 797, + "ied": 798, + "rit": 799, + "\u0120inv": 800, + "lect": 801, + "\u0120supp": 802, + "ating": 803, + "\u0120look": 804, + "man": 805, + "pect": 806, + "\u01208": 807, + "row": 808, + "\u0120bu": 809, + "\u0120where": 810, + "ific": 811, + "\u0120years": 812, + "ily": 813, + "\u0120diff": 814, + "\u0120should": 815, + "\u0120rem": 816, + "Th": 817, + "In": 818, + "\u0120ev": 819, + "day": 820, + "'re": 821, + "rib": 822, + "\u0120rel": 823, + "ss": 824, + "\u0120def": 825, + "\u0120right": 826, + "\u0120sy": 827, + "),": 828, + "les": 829, + "000": 830, + "hen": 831, + "\u0120through": 832, + "\u0120Tr": 833, + "__": 834, + "\u0120way": 835, + "\u0120don": 836, + "\u0120,": 837, + "\u012010": 838, + "ased": 839, + "\u0120ass": 840, + "ublic": 841, + "\u0120reg": 842, + "\u0120And": 843, + "ix": 844, + "\u0120very": 845, + "\u0120includ": 846, + "other": 847, + "\u0120imp": 848, + "oth": 849, + "\u0120sub": 850, + "\u0120\u00e2\u0122\u0136": 851, + "\u0120being": 852, + "arg": 853, + "\u0120Wh": 854, + "==": 855, + "ible": 856, + "\u0120does": 857, + "ange": 858, + "ram": 859, + "\u01209": 860, + "ert": 861, + "ps": 862, + "ited": 863, + "ational": 864, + "\u0120br": 865, + "\u0120down": 866, + "\u0120many": 867, + "aking": 868, + "\u0120call": 869, + "uring": 870, + "ities": 871, + "\u0120ph": 872, + "ics": 873, + "als": 874, + "\u0120dec": 875, + "ative": 876, + "ener": 877, + "\u0120before": 878, + "ility": 879, + "\u0120well": 880, + "\u0120much": 881, + "erson": 882, + "\u0120those": 883, + "\u0120such": 884, + "\u0120ke": 885, + "\u0120end": 886, + "\u0120But": 887, + "ason": 888, + "ting": 889, + "\u0120long": 890, + "ef": 891, + "\u0120think": 892, + "ys": 893, + "\u0120bel": 894, + "\u0120sm": 895, + "its": 896, + "ax": 897, + "\u0120own": 898, + "\u0120prov": 899, + "\u0120set": 900, + "ife": 901, + "ments": 902, + "ble": 903, + "ward": 904, + "\u0120show": 905, + "\u0120pres": 906, + "ms": 907, + "omet": 908, + "\u0120ob": 909, + "\u0120say": 910, + "\u0120Sh": 911, + "ts": 912, + "ful": 913, + "\u0120eff": 914, + "\u0120gu": 915, + "\u0120inst": 916, + "und": 917, + "ren": 918, + "cess": 919, + "\u0120ent": 920, + "\u0120You": 921, + "\u0120good": 922, + "\u0120start": 923, + "ince": 924, + "\u0120made": 925, + "tt": 926, + "stem": 927, + "olog": 928, + "up": 929, + "\u0120|": 930, + "ump": 931, + "\u0120hel": 932, + "vern": 933, + "ular": 934, + "ually": 935, + "\u0120ac": 936, + "\u0120mon": 937, + "\u0120last": 938, + "\u0120200": 939, + "10": 940, + "\u0120stud": 941, + "ures": 942, + "\u0120Ar": 943, + "self": 944, + "ars": 945, + "meric": 946, + "ues": 947, + "cy": 948, + "\u0120min": 949, + "ollow": 950, + "\u0120col": 951, + "io": 952, + "\u0120mod": 953, + "\u0120count": 954, + "\u0120Com": 955, + "hes": 956, + "\u0120fin": 957, + "air": 958, + "ier": 959, + "\u00e2\u0122\u0136": 960, + "read": 961, + "ank": 962, + "atch": 963, + "ever": 964, + "\u0120str": 965, + "\u0120point": 966, + "ork": 967, + "\u0120New": 968, + "\u0120sur": 969, + "ool": 970, + "alk": 971, + "ement": 972, + "\u0120used": 973, + "ract": 974, + "ween": 975, + "\u0120same": 976, + "oun": 977, + "\u0120Al": 978, + "ci": 979, + "\u0120differe": 980, + "\u0120while": 981, + "--------": 982, + "\u0120game": 983, + "cept": 984, + "\u0120sim": 985, + "...": 986, + "\u0120inter": 987, + "ek": 988, + "\u0120report": 989, + "\u0120produ": 990, + "\u0120still": 991, + "led": 992, + "ah": 993, + "\u0120here": 994, + "\u0120world": 995, + "\u0120though": 996, + "\u0120num": 997, + "arch": 998, + "imes": 999, + "ale": 1000, + "\u0120Se": 1001, + "\u0120If": 1002, + "//": 1003, + "\u0120Le": 1004, + "\u0120ret": 1005, + "\u0120ref": 1006, + "\u0120trans": 1007, + "ner": 1008, + "ution": 1009, + "ters": 1010, + "\u0120take": 1011, + "\u0120Cl": 1012, + "\u0120conf": 1013, + "way": 1014, + "ave": 1015, + "\u0120going": 1016, + "\u0120sl": 1017, + "ug": 1018, + "\u0120Americ": 1019, + "\u0120spec": 1020, + "\u0120hand": 1021, + "\u0120between": 1022, + "ists": 1023, + "\u0120De": 1024, + "oot": 1025, + "It": 1026, + "\u0120ear": 1027, + "\u0120against": 1028, + "\u0120high": 1029, + "gan": 1030, + "az": 1031, + "ather": 1032, + "\u0120exp": 1033, + "\u0120op": 1034, + "\u0120ins": 1035, + "\u0120gr": 1036, + "\u0120help": 1037, + "\u0120requ": 1038, + "ets": 1039, + "ins": 1040, + "\u0120Pro": 1041, + "ism": 1042, + "\u0120found": 1043, + "land": 1044, + "ata": 1045, + "uss": 1046, + "ames": 1047, + "\u0120person": 1048, + "\u0120great": 1049, + "pr": 1050, + "\u0120sign": 1051, + "\u0120An": 1052, + "'ve": 1053, + "\u0120somet": 1054, + "\u0120ser": 1055, + "hip": 1056, + "\u0120run": 1057, + "\u0120:": 1058, + "\u0120ter": 1059, + "irect": 1060, + "\u0120follow": 1061, + "\u0120det": 1062, + "ices": 1063, + "\u0120find": 1064, + "12": 1065, + "\u0120mem": 1066, + "\u0120cr": 1067, + "ered": 1068, + "ex": 1069, + "\u0120ext": 1070, + "uth": 1071, + "ense": 1072, + "co": 1073, + "\u0120team": 1074, + "ving": 1075, + "ouse": 1076, + "ash": 1077, + "att": 1078, + "ved": 1079, + "\u0120system": 1080, + "\u0120As": 1081, + "der": 1082, + "ives": 1083, + "min": 1084, + "\u0120lead": 1085, + "\u0120Bl": 1086, + "cent": 1087, + "\u0120around": 1088, + "\u0120govern": 1089, + "\u0120cur": 1090, + "velop": 1091, + "any": 1092, + "\u0120cour": 1093, + "alth": 1094, + "ages": 1095, + "ize": 1096, + "\u0120car": 1097, + "ode": 1098, + "\u0120law": 1099, + "\u0120read": 1100, + "'m": 1101, + "con": 1102, + "\u0120real": 1103, + "\u0120support": 1104, + "\u012012": 1105, + "....": 1106, + "\u0120really": 1107, + "ness": 1108, + "\u0120fact": 1109, + "\u0120day": 1110, + "\u0120both": 1111, + "ying": 1112, + "\u0120serv": 1113, + "\u0120For": 1114, + "\u0120three": 1115, + "\u0120wom": 1116, + "\u0120med": 1117, + "ody": 1118, + "\u0120They": 1119, + "50": 1120, + "\u0120exper": 1121, + "ton": 1122, + "\u0120each": 1123, + "akes": 1124, + "\u0120che": 1125, + "\u0120cre": 1126, + "ines": 1127, + "\u0120rep": 1128, + "19": 1129, + "gg": 1130, + "illion": 1131, + "\u0120grou": 1132, + "ute": 1133, + "ik": 1134, + "We": 1135, + "get": 1136, + "ER": 1137, + "\u0120met": 1138, + "\u0120says": 1139, + "ox": 1140, + "\u0120during": 1141, + "ern": 1142, + "ized": 1143, + "ared": 1144, + "\u0120fam": 1145, + "ically": 1146, + "\u0120happ": 1147, + "\u0120Is": 1148, + "\u0120char": 1149, + "med": 1150, + "vent": 1151, + "\u0120gener": 1152, + "ient": 1153, + "ple": 1154, + "iet": 1155, + "rent": 1156, + "11": 1157, + "ves": 1158, + "ption": 1159, + "\u012020": 1160, + "formation": 1161, + "\u0120cor": 1162, + "\u0120offic": 1163, + "ield": 1164, + "\u0120too": 1165, + "ision": 1166, + "\u0120inf": 1167, + "\u0120Z": 1168, + "the": 1169, + "oad": 1170, + "\u0120public": 1171, + "\u0120prog": 1172, + "ric": 1173, + "**": 1174, + "\u0120war": 1175, + "\u0120power": 1176, + "view": 1177, + "\u0120few": 1178, + "\u0120loc": 1179, + "\u0120different": 1180, + "\u0120state": 1181, + "\u0120head": 1182, + "'ll": 1183, + "\u0120poss": 1184, + "\u0120stat": 1185, + "ret": 1186, + "ants": 1187, + "\u0120val": 1188, + "\u0120iss": 1189, + "\u0120cle": 1190, + "ivers": 1191, + "anc": 1192, + "\u0120expl": 1193, + "\u0120another": 1194, + "\u0120Q": 1195, + "\u0120av": 1196, + "thing": 1197, + "nce": 1198, + "Wh": 1199, + "\u0120child": 1200, + "\u0120since": 1201, + "ired": 1202, + "less": 1203, + "\u0120life": 1204, + "\u0120develop": 1205, + "ittle": 1206, + "\u0120dep": 1207, + "\u0120pass": 1208, + "\u00e3\u0125": 1209, + "\u0120turn": 1210, + "orn": 1211, + "This": 1212, + "bers": 1213, + "ross": 1214, + "\u0120Ad": 1215, + "\u0120fr": 1216, + "\u0120resp": 1217, + "\u0120second": 1218, + "oh": 1219, + "\u0120/": 1220, + "\u0120disc": 1221, + "\u0120&": 1222, + "\u0120something": 1223, + "\u0120comple": 1224, + "\u0120ed": 1225, + "\u0120fil": 1226, + "\u0120month": 1227, + "aj": 1228, + "uc": 1229, + "\u0120government": 1230, + "\u0120without": 1231, + "\u0120leg": 1232, + "\u0120dist": 1233, + "\u0120put": 1234, + "\u0120quest": 1235, + "ann": 1236, + "\u0120prot": 1237, + "20": 1238, + "\u0120never": 1239, + "ience": 1240, + "\u0120level": 1241, + "\u0120art": 1242, + "\u0120things": 1243, + "\u0120might": 1244, + "\u0120effect": 1245, + "\u0120contro": 1246, + "\u0120cent": 1247, + "\u012018": 1248, + "\u0120allow": 1249, + "\u0120belie": 1250, + "chool": 1251, + "ott": 1252, + "\u0120incre": 1253, + "\u0120feel": 1254, + "\u0120result": 1255, + "\u0120lot": 1256, + "\u0120fun": 1257, + "ote": 1258, + "\u0120ty": 1259, + "erest": 1260, + "\u0120contin": 1261, + "\u0120using": 1262, + "\u0120big": 1263, + "201": 1264, + "\u0120ask": 1265, + "\u0120best": 1266, + "\u0120)": 1267, + "IN": 1268, + "\u0120opp": 1269, + "30": 1270, + "\u0120number": 1271, + "iness": 1272, + "St": 1273, + "lease": 1274, + "\u0120ca": 1275, + "\u0120must": 1276, + "\u0120direct": 1277, + "\u0120gl": 1278, + "\u0120<": 1279, + "\u0120open": 1280, + "\u0120post": 1281, + "\u0120come": 1282, + "\u0120seem": 1283, + "ording": 1284, + "\u0120week": 1285, + "ately": 1286, + "ital": 1287, + "\u0120el": 1288, + "riend": 1289, + "\u0120far": 1290, + "\u0120tra": 1291, + "inal": 1292, + "\u0120pri": 1293, + "\u0120US": 1294, + "\u0120place": 1295, + "\u0120form": 1296, + "\u0120told": 1297, + "\":": 1298, + "ains": 1299, + "ature": 1300, + "\u0120Trump": 1301, + "\u0120stand": 1302, + "\u0120#": 1303, + "ider": 1304, + "\u0120Fr": 1305, + "\u0120next": 1306, + "\u0120soc": 1307, + "\u0120pur": 1308, + "\u0120let": 1309, + "\u0120little": 1310, + "\u0120hum": 1311, + "\u0120i": 1312, + "ron": 1313, + "15": 1314, + "\u012015": 1315, + "\u0120commun": 1316, + "\u0120mark": 1317, + "\u0120There": 1318, + "\u0120wr": 1319, + "\u0120That": 1320, + "\u0120information": 1321, + "ways": 1322, + "\u0120bus": 1323, + "app": 1324, + "\u0120invest": 1325, + "me": 1326, + "\u0120hard": 1327, + "ained": 1328, + "ead": 1329, + "\u0120import": 1330, + "\u0120appro": 1331, + "\u0120test": 1332, + "\u0120tri": 1333, + "\u0120rest": 1334, + "osed": 1335, + "\u0120full": 1336, + "\u0120care": 1337, + "\u0120Sp": 1338, + "\u0120case": 1339, + "ON": 1340, + "\u0120sk": 1341, + "\u0120less": 1342, + "\u0120+": 1343, + "\u0120partic": 1344, + "\u0120Pl": 1345, + "ably": 1346, + "uck": 1347, + "ished": 1348, + "chn": 1349, + "be": 1350, + "\u0120list": 1351, + "ator": 1352, + "\u0120top": 1353, + "\u0120adv": 1354, + "\u0120Be": 1355, + "ruct": 1356, + "\u0120dem": 1357, + "ration": 1358, + "ling": 1359, + "gy": 1360, + "reen": 1361, + "ger": 1362, + "\u0120home": 1363, + "\u0120left": 1364, + "\u0120better": 1365, + "\u0120data": 1366, + "\u012011": 1367, + "\u0120attack": 1368, + "\u0120proble": 1369, + "line": 1370, + "ards": 1371, + "\u0120beh": 1372, + "ral": 1373, + "\u0120How": 1374, + "\u0120She": 1375, + "arge": 1376, + "\u0120--": 1377, + "://": 1378, + "\u0120bro": 1379, + "\u0120Ph": 1380, + "ats": 1381, + "\u0120build": 1382, + "ww": 1383, + "ided": 1384, + "aim": 1385, + "ases": 1386, + "ency": 1387, + "\u0120main": 1388, + "ined": 1389, + "\u0120including": 1390, + "\u0120{": 1391, + "\u0120got": 1392, + "\u0120interest": 1393, + "\u0120keep": 1394, + "\u0120X": 1395, + "\u0120eas": 1396, + "aining": 1397, + "\u0120class": 1398, + "\u00e2\u0122\u00a6": 1399, + "\u0120No": 1400, + "\u0120var": 1401, + "\u0120small": 1402, + "ample": 1403, + "AT": 1404, + "\u0120ide": 1405, + "\u0120So": 1406, + "\u0120rece": 1407, + "\u0120polit": 1408, + "\u0120mov": 1409, + "\u0120plan": 1410, + "\u0120percent": 1411, + "iving": 1412, + "\u0120camp": 1413, + "\u0120pay": 1414, + "14": 1415, + "sc": 1416, + "ised": 1417, + "\u0120unt": 1418, + "oney": 1419, + "ploy": 1420, + "====": 1421, + "\u0120didn": 1422, + "\u0120Ind": 1423, + "els": 1424, + "ertain": 1425, + "\u0120pos": 1426, + "____": 1427, + "iver": 1428, + "\u0120process": 1429, + "\u0120program": 1430, + "ified": 1431, + "\u0120Rep": 1432, + "16": 1433, + "uro": 1434, + "ology": 1435, + "atter": 1436, + "ina": 1437, + "\u0120name": 1438, + "\u0120All": 1439, + "\u0120four": 1440, + "\u0120return": 1441, + "vious": 1442, + "bs": 1443, + "\u0120called": 1444, + "\u0120move": 1445, + "\u0120Sc": 1446, + "ird": 1447, + "\u0120group": 1448, + "\u0120bre": 1449, + "\u0120men": 1450, + "\u0120cap": 1451, + "ten": 1452, + "ee": 1453, + "\u0120dri": 1454, + "leg": 1455, + "here": 1456, + "uthor": 1457, + "\u0120pat": 1458, + "\u0120current": 1459, + "ides": 1460, + "\u0120pop": 1461, + "to": 1462, + "ention": 1463, + "\u0120always": 1464, + "\u0120mil": 1465, + "\u0120women": 1466, + "\u012016": 1467, + "\u0120old": 1468, + "iven": 1469, + "raph": 1470, + "\u0120Or": 1471, + "ror": 1472, + "ently": 1473, + "\u0120near": 1474, + "\u0120Ex": 1475, + "ream": 1476, + "sh": 1477, + "\u012014": 1478, + "\u0120free": 1479, + "ission": 1480, + "stand": 1481, + "\u0120Con": 1482, + "ality": 1483, + "used": 1484, + "13": 1485, + "\u0120design": 1486, + "\u0120change": 1487, + "\u0120chang": 1488, + "\u0120bo": 1489, + "\u0120vis": 1490, + "ember": 1491, + "\u0120book": 1492, + "ready": 1493, + "\u0120kill": 1494, + "25": 1495, + "pped": 1496, + "\u0120away": 1497, + "\u0120able": 1498, + "\u0120country": 1499, + "\u0120const": 1500, + "arn": 1501, + "\u0120order": 1502, + "AR": 1503, + "ior": 1504, + "ium": 1505, + "orth": 1506, + "18": 1507, + "ailable": 1508, + "\u0120sw": 1509, + "\u0120million": 1510, + "\u012013": 1511, + "atic": 1512, + "ted": 1513, + "\u0120Go": 1514, + "\u0120oper": 1515, + "eng": 1516, + "\u0120thing": 1517, + "ajor": 1518, + "conom": 1519, + "\u0120Comm": 1520, + "\u0120why": 1521, + "ured": 1522, + "ural": 1523, + "\u0120school": 1524, + "by": 1525, + "\u0120Mar": 1526, + "\u0120aff": 1527, + "\u0120days": 1528, + "\u0120ann": 1529, + "ush": 1530, + "ane": 1531, + "If": 1532, + "eg": 1533, + "\u0120prof": 1534, + "\u0120health": 1535, + "outh": 1536, + "But": 1537, + "ional": 1538, + ".,": 1539, + "\u0120sol": 1540, + "\u0120already": 1541, + "\u012030": 1542, + "\u0120charact": 1543, + "He": 1544, + "\u0120friend": 1545, + "ES": 1546, + "ians": 1547, + "icle": 1548, + "'d": 1549, + "\u0120On": 1550, + "\u0120least": 1551, + "\u0120prom": 1552, + "\u0120dr": 1553, + "\u0120hist": 1554, + "ither": 1555, + "\u0120est": 1556, + "iqu": 1557, + "17": 1558, + "son": 1559, + "\u0120tell": 1560, + "\u0120talk": 1561, + "ohn": 1562, + "oint": 1563, + "lection": 1564, + "AN": 1565, + "\u0120until": 1566, + "augh": 1567, + "\u0120later": 1568, + "\u0120ve": 1569, + "\u0120view": 1570, + "ending": 1571, + "ived": 1572, + "\u0120word": 1573, + "ware": 1574, + "\u0120cost": 1575, + "\u0120enough": 1576, + "\u0120give": 1577, + "\u0120United": 1578, + "\u0120techn": 1579, + "arent": 1580, + "OR": 1581, + "\u0120par": 1582, + "\u0120Dr": 1583, + "\u01202016": 1584, + "rist": 1585, + "ering": 1586, + "\u0120\u00c2": 1587, + "\u0120large": 1588, + "side": 1589, + "acy": 1590, + "ccess": 1591, + "\u0120win": 1592, + "\u0120important": 1593, + "\u0120199": 1594, + "\u0120doesn": 1595, + "\u012017": 1596, + "\u0120business": 1597, + "\u0120clear": 1598, + "\u0120rese": 1599, + "\",": 1600, + "ury": 1601, + "\u0120equ": 1602, + "aster": 1603, + "alf": 1604, + "\u0120American": 1605, + "nect": 1606, + "\u0120expect": 1607, + "iversity": 1608, + "\u0120occ": 1609, + "\u0120Fl": 1610, + "\u0120kind": 1611, + "\u0120mean": 1612, + "\u0120past": 1613, + "\u0120dev": 1614, + "\u0120bas": 1615, + "let": 1616, + "raft": 1617, + "\u0120organ": 1618, + "\u0120del": 1619, + "\u0120perform": 1620, + "\u0120story": 1621, + "\u0120season": 1622, + "\u0120Col": 1623, + "\u0120claim": 1624, + "\u0120came": 1625, + "\u0120within": 1626, + "\u0120line": 1627, + "\u0120project": 1628, + "\u0120At": 1629, + "\u0120control": 1630, + "ended": 1631, + "\u0120Sy": 1632, + "\u0120air": 1633, + "ization": 1634, + "\u0120*": 1635, + "ley": 1636, + "\u0120money": 1637, + "idd": 1638, + "You": 1639, + "for": 1640, + "\u0120family": 1641, + "\u0120making": 1642, + "\u0120bit": 1643, + "\u0120police": 1644, + "\u0120happen": 1645, + "\u0120vers": 1646, + "ony": 1647, + "uff": 1648, + "\u0120When": 1649, + "\u0120sit": 1650, + "ideo": 1651, + "lf": 1652, + "ison": 1653, + "\u0120sure": 1654, + "gin": 1655, + "\u0120appear": 1656, + "\u0120light": 1657, + "\u0120es": 1658, + "of": 1659, + "\u0120water": 1660, + "\u0120times": 1661, + "not": 1662, + "\u0120grow": 1663, + "\u0120company": 1664, + "\u0120Te": 1665, + "ows": 1666, + "\u0120mar": 1667, + "ource": 1668, + "iol": 1669, + "arm": 1670, + "br": 1671, + "\u0120example": 1672, + "\u0120conc": 1673, + "\u0120fore": 1674, + "\u0120To": 1675, + "pro": 1676, + "EN": 1677, + "ries": 1678, + "\u012025": 1679, + "\u0120Can": 1680, + "ney": 1681, + "\u0120actually": 1682, + "\u0120ever": 1683, + "urity": 1684, + "aken": 1685, + "aps": 1686, + "\u0120tax": 1687, + "\u0120major": 1688, + "ama": 1689, + "\u0120often": 1690, + "eral": 1691, + "\u0120human": 1692, + "\u0120job": 1693, + "ister": 1694, + "\u0120available": 1695, + "ocr": 1696, + "enn": 1697, + "aid": 1698, + "ivid": 1699, + "\u0120record": 1700, + "?\"": 1701, + "\u0120sing": 1702, + "\u0120Am": 1703, + "idence": 1704, + "\u0120news": 1705, + "ster": 1706, + "\u0120econom": 1707, + "\u0120following": 1708, + "\u0120Br": 1709, + "ising": 1710, + "\u0120hour": 1711, + "most": 1712, + "ument": 1713, + "\u0120sex": 1714, + "\u0120desc": 1715, + "\u0120become": 1716, + "\u0120Ed": 1717, + "\u0120took": 1718, + "\u0120having": 1719, + "\u0120product": 1720, + "ault": 1721, + "As": 1722, + "aring": 1723, + "\u0120means": 1724, + "\u0120hop": 1725, + "une": 1726, + "\u0120cho": 1727, + "\u0120certain": 1728, + "\u0120non": 1729, + "\u0120deal": 1730, + "24": 1731, + "lement": 1732, + "oci": 1733, + "ene": 1734, + "\u0120side": 1735, + "\u0120Pr": 1736, + "\u0120May": 1737, + "\u0120reason": 1738, + "ued": 1739, + "ched": 1740, + "ulation": 1741, + "\u0120elect": 1742, + "\u0120official": 1743, + "\u0120possible": 1744, + "\u0120hold": 1745, + "ands": 1746, + "ots": 1747, + "\u0120city": 1748, + "ories": 1749, + "\u0120sever": 1750, + "\u0120children": 1751, + "\u0120once": 1752, + "\u0120activ": 1753, + "ler": 1754, + "\u0120night": 1755, + "itions": 1756, + "\u0120John": 1757, + "ape": 1758, + "play": 1759, + "\u0120done": 1760, + "\u0120lim": 1761, + "\u0120working": 1762, + "\u0120Pres": 1763, + "orld": 1764, + "eb": 1765, + "\u0120Co": 1766, + "\u0120body": 1767, + "ails": 1768, + "utes": 1769, + "\u0120Mr": 1770, + "\u0120whether": 1771, + "\u0120author": 1772, + "rop": 1773, + "\u0120proper": 1774, + "\u0120seen": 1775, + ");": 1776, + "\u0120fac": 1777, + "\u0120Su": 1778, + "\u0120cond": 1779, + "iting": 1780, + "\u0120course": 1781, + "\u0120}": 1782, + "----------------": 1783, + "aign": 1784, + "\u0120event": 1785, + "\u0120eng": 1786, + "\u0120pot": 1787, + "\u0120intern": 1788, + "iam": 1789, + "\u0120short": 1790, + "empt": 1791, + "\u00e3\u0124": 1792, + "\u0120God": 1793, + "ilar": 1794, + "80": 1795, + "\u0120orig": 1796, + "IS": 1797, + "ourn": 1798, + "ability": 1799, + "itive": 1800, + "\u0120dam": 1801, + "\u0120100": 1802, + "\u0120press": 1803, + "\u0120doing": 1804, + "\u0120protect": 1805, + "ring": 1806, + "\u0120thought": 1807, + "\u0120question": 1808, + "rew": 1809, + "\u0120War": 1810, + "\u0120several": 1811, + "\u0120State": 1812, + "\u0120given": 1813, + "\u0120fund": 1814, + "\u0120Tw": 1815, + "\u0120went": 1816, + "ances": 1817, + "work": 1818, + "por": 1819, + "my": 1820, + "40": 1821, + "\u0120arg": 1822, + "artment": 1823, + "ustom": 1824, + "\u0120polic": 1825, + "\u0120meet": 1826, + "\u0120creat": 1827, + "22": 1828, + "\u0120States": 1829, + "\u0120games": 1830, + "raw": 1831, + "uture": 1832, + "\u0120understand": 1833, + "urs": 1834, + "\u0120Ob": 1835, + "lish": 1836, + "sy": 1837, + "\u0120makes": 1838, + "\u0120won": 1839, + "agon": 1840, + "\u0120htt": 1841, + "\u0120love": 1842, + "ential": 1843, + "\u0120complete": 1844, + "par": 1845, + "\u0120Im": 1846, + "AL": 1847, + "\u0120account": 1848, + "\u00c2\u0142": 1849, + "ored": 1850, + "vert": 1851, + "\u0120ident": 1852, + "\u01202015": 1853, + "\u0120others": 1854, + "\u0120Min": 1855, + "iber": 1856, + "verage": 1857, + "There": 1858, + "itional": 1859, + "dd": 1860, + "\u0120prob": 1861, + "\u0120young": 1862, + "\u0120along": 1863, + "\u0120according": 1864, + "\u0120yet": 1865, + "\u0120members": 1866, + "\u0120What": 1867, + "oid": 1868, + "\u0120Man": 1869, + "And": 1870, + "\u0120among": 1871, + "ai": 1872, + "\u0120employ": 1873, + "\u0120Res": 1874, + "\u0120>": 1875, + "\u0120invol": 1876, + "\u0120low": 1877, + "af": 1878, + "\u0120Car": 1879, + "\u0120hig": 1880, + "\u0120One": 1881, + "\u0120Sec": 1882, + "ination": 1883, + "\u0120likely": 1884, + "\u0120ant": 1885, + "aged": 1886, + "\u0120Russ": 1887, + "\u0120ben": 1888, + "\u0120rele": 1889, + "For": 1890, + "back": 1891, + "\u0120Not": 1892, + "\u0120president": 1893, + "ball": 1894, + "\u0120access": 1895, + "ividual": 1896, + "\u0120Dem": 1897, + "\u0120Euro": 1898, + "60": 1899, + "\u0120known": 1900, + "irl": 1901, + "\u0120Gr": 1902, + "\u0120early": 1903, + "use": 1904, + "iety": 1905, + "\u00e2\u0122\u0135": 1906, + "\u0120fight": 1907, + "\u0120sent": 1908, + "\u0120today": 1909, + "\u0120market": 1910, + "\".": 1911, + "\u0120based": 1912, + "\u0120strong": 1913, + "urther": 1914, + "\u0120deb": 1915, + "mber": 1916, + "\u0120problem": 1917, + "\u0120death": 1918, + "\u0120social": 1919, + "imate": 1920, + "AS": 1921, + "ortun": 1922, + "\u0120campaign": 1923, + "ery": 1924, + "Ch": 1925, + "\u0120ey": 1926, + "ially": 1927, + "\u0120mus": 1928, + "wh": 1929, + "pos": 1930, + "\u0120er": 1931, + "\u0120saf": 1932, + "\u0120months": 1933, + "iron": 1934, + "\u0120viol": 1935, + "\u0120five": 1936, + "\u0120stre": 1937, + "\u0120players": 1938, + "inc": 1939, + "ald": 1940, + "year": 1941, + "aun": 1942, + "\u0120success": 1943, + "\u0120present": 1944, + "erence": 1945, + "\u01202014": 1946, + "\u0120sugg": 1947, + "\u0120particular": 1948, + "\u0120try": 1949, + "\u0120suggest": 1950, + "\u0120Christ": 1951, + "ones": 1952, + "\u0120priv": 1953, + "23": 1954, + "\u0120crit": 1955, + "\u0120land": 1956, + "\u0120local": 1957, + "ify": 1958, + "29": 1959, + "\u0120aut": 1960, + "ED": 1961, + "\u0120Gu": 1962, + "\u0120mult": 1963, + "\u0120political": 1964, + "\u0120asked": 1965, + "\u0120former": 1966, + "itter": 1967, + "ript": 1968, + "\u0120close": 1969, + "\u0120pract": 1970, + "\u0120York": 1971, + "\u0120getting": 1972, + "\u0120across": 1973, + "\u0120comb": 1974, + "\u0120believe": 1975, + "\u0120z": 1976, + "\u0120toget": 1977, + "\u0120together": 1978, + "\u0120Cent": 1979, + "irc": 1980, + "\u0120individual": 1981, + "\u0120Mc": 1982, + "27": 1983, + "isk": 1984, + "\u0120Eng": 1985, + "\u0120face": 1986, + "\u012024": 1987, + "\u0120value": 1988, + "\u0120area": 1989, + "ev": 1990, + "\u0120writ": 1991, + "\u0120President": 1992, + "\u0120vot": 1993, + "\u0120key": 1994, + "\u0120mom": 1995, + "put": 1996, + "\u0120anything": 1997, + "\u0120experience": 1998, + "attle": 1999, + "\u0120mind": 2000, + "aff": 2001, + "omm": 2002, + "\u0120future": 2003, + "ged": 2004, + "\u0120cut": 2005, + "\u0120tot": 2006, + "itch": 2007, + "\u0120video": 2008, + "\u0120investig": 2009, + "\u0120net": 2010, + "\u0120My": 2011, + "rict": 2012, + "ien": 2013, + ".)": 2014, + "\u0120impro": 2015, + "though": 2016, + "wards": 2017, + "\u0120connect": 2018, + "\u0120Med": 2019, + "selves": 2020, + "ensive": 2021, + "mb": 2022, + "ober": 2023, + "ators": 2024, + "An": 2025, + "\u012050": 2026, + "\u0120redu": 2027, + "resent": 2028, + "\u0120above": 2029, + "\u0120fre": 2030, + "\u0120Europe": 2031, + "sw": 2032, + "\u0120amount": 2033, + "\u0120App": 2034, + "\u0120either": 2035, + "\u0120milit": 2036, + "\u0120anal": 2037, + "\u0120fail": 2038, + "\u0120En": 2039, + "ales": 2040, + "\u0120special": 2041, + "\u0120black": 2042, + "IT": 2043, + "cher": 2044, + "\u0120looking": 2045, + "\u0120fire": 2046, + "yn": 2047, + "\u0120almost": 2048, + "oon": 2049, + "\u0120study": 2050, + "\u0120miss": 2051, + "ches": 2052, + "rown": 2053, + "\u0120tre": 2054, + "\u0120community": 2055, + "\u0120media": 2056, + "\u0120food": 2057, + "\u0120comes": 2058, + "\u0120University": 2059, + "\u0120single": 2060, + "What": 2061, + "uly": 2062, + "\u0120half": 2063, + "ague": 2064, + "hod": 2065, + "\u0120Republic": 2066, + "\u0120started": 2067, + "\u0120quick": 2068, + "oto": 2069, + "book": 2070, + "\u0120issue": 2071, + "itor": 2072, + "\u0120else": 2073, + "\u0120consider": 2074, + "26": 2075, + "rodu": 2076, + "\u0120taken": 2077, + "28": 2078, + "99": 2079, + "\u0120With": 2080, + "\u0120true": 2081, + "\u0120wa": 2082, + "\u0120trad": 2083, + "\u0120ago": 2084, + "\u0120mess": 2085, + "ief": 2086, + "\u0120added": 2087, + "oke": 2088, + "\u0120bad": 2089, + "\u0120fav": 2090, + "33": 2091, + "\u0120similar": 2092, + "ask": 2093, + "\u0120Don": 2094, + "\u0120character": 2095, + "orts": 2096, + "\u0120House": 2097, + "\u0120reported": 2098, + "\u0120type": 2099, + "val": 2100, + "iod": 2101, + "\u0120However": 2102, + "\u0120targ": 2103, + "\u0120entire": 2104, + "pping": 2105, + "\u0120history": 2106, + "\u0120live": 2107, + "ffic": 2108, + "........": 2109, + "ederal": 2110, + "\u0120trying": 2111, + "\u0120discuss": 2112, + "\u0120Har": 2113, + "aces": 2114, + "lished": 2115, + "\u0120self": 2116, + "osp": 2117, + "rest": 2118, + "\u0120room": 2119, + "elt": 2120, + "\u0120fall": 2121, + "olution": 2122, + "\u0120et": 2123, + "\u0120x": 2124, + "\u0120isn": 2125, + "\u0120idea": 2126, + "bo": 2127, + "\u0120sound": 2128, + "\u0120Dep": 2129, + "\u0120someone": 2130, + "cially": 2131, + "ully": 2132, + "\u0120foc": 2133, + "\u0120object": 2134, + "ift": 2135, + "aper": 2136, + "\u0120player": 2137, + "\u0120rather": 2138, + "\u0120service": 2139, + "ashing": 2140, + "\u0120Do": 2141, + "\u0120Part": 2142, + "rug": 2143, + "mon": 2144, + "ply": 2145, + "\u0120mor": 2146, + "\u0120nothing": 2147, + "\u0120provide": 2148, + "IC": 2149, + "ung": 2150, + "\u0120party": 2151, + "\u0120exist": 2152, + "\u0120mag": 2153, + "70": 2154, + "\u0120rul": 2155, + "\u0120house": 2156, + "\u0120behind": 2157, + "\u0120however": 2158, + "\u0120World": 2159, + "\u0120sum": 2160, + "\u0120applic": 2161, + "\u0120;": 2162, + "\u0120function": 2163, + "gr": 2164, + "\u0120Pol": 2165, + "\u0120front": 2166, + "200": 2167, + "\u0120series": 2168, + "\u0120tem": 2169, + "\u0120typ": 2170, + "ills": 2171, + "\u0120opt": 2172, + "\u0120points": 2173, + "\u0120below": 2174, + "itted": 2175, + "\u0120specific": 2176, + "\u01202017": 2177, + "umb": 2178, + "\u0120ra": 2179, + "\u0120previous": 2180, + "\u0120pret": 2181, + "reme": 2182, + "\u0120custom": 2183, + "\u0120court": 2184, + "\u0120Me": 2185, + "\u0120repl": 2186, + "\u0120whole": 2187, + "go": 2188, + "cer": 2189, + "\u0120treat": 2190, + "\u0120Act": 2191, + "\u0120probably": 2192, + "\u0120learn": 2193, + "ender": 2194, + "\u0120Ass": 2195, + "\u0120version": 2196, + "now": 2197, + "\u0120check": 2198, + "\u0120Cal": 2199, + "RE": 2200, + "minist": 2201, + "On": 2202, + "ources": 2203, + "\u0120benef": 2204, + "\u0120doc": 2205, + "\u0120deter": 2206, + "\u0120enc": 2207, + "\u0120super": 2208, + "\u0120address": 2209, + "\u0120vict": 2210, + "\u01202013": 2211, + "\u0120meas": 2212, + "tr": 2213, + "\u0120field": 2214, + "When": 2215, + "\u0120signific": 2216, + "uge": 2217, + "\u0120feat": 2218, + "\u0120common": 2219, + "load": 2220, + "\u0120begin": 2221, + "\u0120bring": 2222, + "\u0120action": 2223, + "erman": 2224, + "\u0120describ": 2225, + "\u0120indust": 2226, + "\u0120wanted": 2227, + "ried": 2228, + "ming": 2229, + "\u0120attempt": 2230, + "45": 2231, + "fer": 2232, + "\u0120due": 2233, + "ression": 2234, + "##": 2235, + "\u0120shall": 2236, + "\u0120six": 2237, + "oo": 2238, + "\u0120step": 2239, + "\u0120pub": 2240, + "\u0120himself": 2241, + "\u012023": 2242, + "\u0120cop": 2243, + "\u0120dest": 2244, + "\u0120stop": 2245, + "AC": 2246, + "ibility": 2247, + "\u0120lab": 2248, + "icult": 2249, + "\u0120hours": 2250, + "\u0120create": 2251, + "\u0120further": 2252, + "\u0120America": 2253, + "\u0120City": 2254, + "\u0120dou": 2255, + "head": 2256, + "ST": 2257, + "\u0120North": 2258, + "cing": 2259, + "\u0120national": 2260, + "ule": 2261, + "\u0120Inst": 2262, + "\u0120taking": 2263, + "\u0120Qu": 2264, + "irt": 2265, + "\u0120red": 2266, + "\u0120research": 2267, + "viron": 2268, + "\u0120Ge": 2269, + "\u0120break": 2270, + "ana": 2271, + "\u0120space": 2272, + "aterial": 2273, + "\u0120recent": 2274, + "\u0120Ab": 2275, + "\u0120general": 2276, + "\u0120hit": 2277, + "\u0120period": 2278, + "\u0120everything": 2279, + "ively": 2280, + "\u0120phys": 2281, + "\u0120saying": 2282, + "anks": 2283, + "\u0120cou": 2284, + "\u0120cult": 2285, + "aced": 2286, + "eal": 2287, + "uation": 2288, + "\u0120coun": 2289, + "lu": 2290, + "\u0120include": 2291, + "\u0120position": 2292, + "\u0120After": 2293, + "\u0120Canad": 2294, + "\u0120Em": 2295, + "\u0120imm": 2296, + "\u0120Red": 2297, + "\u0120pick": 2298, + "\u0120compl": 2299, + "\u0120matter": 2300, + "reg": 2301, + "ext": 2302, + "angu": 2303, + "isc": 2304, + "ole": 2305, + "aut": 2306, + "\u0120compet": 2307, + "eed": 2308, + "fect": 2309, + "\u012021": 2310, + "\u0120Sen": 2311, + "\u0120These": 2312, + "asing": 2313, + "\u0120cannot": 2314, + "\u0120init": 2315, + "\u0120relations": 2316, + "ached": 2317, + "\u0120bar": 2318, + "\u012040": 2319, + "\u0120TH": 2320, + "\u01202012": 2321, + "\u0120vol": 2322, + "\u0120ground": 2323, + "\u0120security": 2324, + "\u0120upd": 2325, + "ilt": 2326, + "35": 2327, + "\u0120concern": 2328, + "\u0120Just": 2329, + "\u0120white": 2330, + "\u0120seems": 2331, + "\u0120Her": 2332, + "pecially": 2333, + "ients": 2334, + "\u0120announ": 2335, + "\u0120fig": 2336, + "ights": 2337, + "\u0120stri": 2338, + "like": 2339, + "ids": 2340, + "\u0120sus": 2341, + "\u0120watch": 2342, + "\u0120\u00e2": 2343, + "\u0120wind": 2344, + "\u0120Cont": 2345, + "\u0120itself": 2346, + "\u0120mass": 2347, + "Al": 2348, + "yle": 2349, + "ique": 2350, + "\u0120National": 2351, + "\u0120abs": 2352, + "\u0120pack": 2353, + "\u0120outside": 2354, + "\u0120anim": 2355, + "\u0120pain": 2356, + "eter": 2357, + "\u0120manag": 2358, + "duct": 2359, + "ogn": 2360, + "\u0120]": 2361, + "\u0120Sept": 2362, + "sec": 2363, + "off": 2364, + "\u0120Jan": 2365, + "\u0120foot": 2366, + "ades": 2367, + "\u0120third": 2368, + "\u0120mot": 2369, + "\u0120evidence": 2370, + "inton": 2371, + "\u0120threat": 2372, + "apt": 2373, + "ples": 2374, + "cle": 2375, + "\u0120lo": 2376, + "\u0120decl": 2377, + "\u0120item": 2378, + "medi": 2379, + "\u0120represent": 2380, + "omb": 2381, + "amer": 2382, + "\u0120significant": 2383, + "ograph": 2384, + "su": 2385, + "\u0120cal": 2386, + "ires": 2387, + "0000": 2388, + "ID": 2389, + "AM": 2390, + "\u0120simply": 2391, + "\u0120longer": 2392, + "\u0120file": 2393, + "OT": 2394, + "che": 2395, + "So": 2396, + "ateg": 2397, + "org": 2398, + "\u0120His": 2399, + "\u0120ener": 2400, + "\u0120dom": 2401, + "\u0120upon": 2402, + "ili": 2403, + "\":\"": 2404, + "\u0120themselves": 2405, + "\u0120coming": 2406, + "\u0120quite": 2407, + "\u0120difficult": 2408, + "\u0120Bar": 2409, + "ilities": 2410, + "rel": 2411, + "ends": 2412, + "cial": 2413, + "64": 2414, + "\u0120woman": 2415, + "rap": 2416, + "yr": 2417, + "\u0120necess": 2418, + "ips": 2419, + "\u0120text": 2420, + "\u0120require": 2421, + "\u0120military": 2422, + "\u0120review": 2423, + "\u0120respons": 2424, + "75": 2425, + "\u0120subject": 2426, + "\u0120instead": 2427, + "\u0120issues": 2428, + "\u0120gen": 2429, + "\",\"": 2430, + "\u0120minutes": 2431, + "\u0120weap": 2432, + "ray": 2433, + "amed": 2434, + "time": 2435, + "bl": 2436, + "How": 2437, + "\u0120code": 2438, + "\u0120Sm": 2439, + "\u0120higher": 2440, + "\u0120Ste": 2441, + "ris": 2442, + "\u0120page": 2443, + "\u0120students": 2444, + "\u0120Intern": 2445, + "\u0120method": 2446, + "\u0120Aug": 2447, + "\u0120Per": 2448, + "\u0120Ag": 2449, + "\u0120policy": 2450, + "\u0120Sw": 2451, + "\u0120exec": 2452, + "\u0120accept": 2453, + "ume": 2454, + "ribut": 2455, + "\u0120words": 2456, + "\u0120final": 2457, + "\u0120changes": 2458, + "\u0120Democr": 2459, + "\u0120friends": 2460, + "\u0120respect": 2461, + "\u0120ep": 2462, + "\u0120compan": 2463, + "ivil": 2464, + "\u0120damage": 2465, + "****": 2466, + "ogle": 2467, + "vironment": 2468, + "\u0120neg": 2469, + "ental": 2470, + "\u0120ap": 2471, + "\u0120total": 2472, + "ival": 2473, + "!\"": 2474, + "lim": 2475, + "\u0120needs": 2476, + "\u0120agre": 2477, + "\u0120development": 2478, + "\u0120age": 2479, + "iple": 2480, + "21": 2481, + "\u0120results": 2482, + "\u0120Af": 2483, + "Sh": 2484, + "\u0120gun": 2485, + "\u0120Obama": 2486, + "roll": 2487, + "\u0120@": 2488, + "\u0120rights": 2489, + "\u0120Brit": 2490, + "\u0120running": 2491, + "\u0120wasn": 2492, + "\u0120port": 2493, + "\u0120rate": 2494, + "\u0120pretty": 2495, + "\u0120target": 2496, + "\u0120saw": 2497, + "\u0120circ": 2498, + "\u0120works": 2499, + "icro": 2500, + "alt": 2501, + "over": 2502, + "www": 2503, + "That": 2504, + "lier": 2505, + "\u0120everyone": 2506, + "ude": 2507, + "\u0120pie": 2508, + "iddle": 2509, + "rael": 2510, + "\u0120rad": 2511, + "\u0120block": 2512, + "\u0120walk": 2513, + "To": 2514, + "\u00e3\u0123": 2515, + "nes": 2516, + "\u0120Aust": 2517, + "aul": 2518, + "rote": 2519, + "\u0120South": 2520, + "ession": 2521, + "oph": 2522, + "\u0120shows": 2523, + "\u0120site": 2524, + "\u0120jo": 2525, + "\u0120risk": 2526, + "clus": 2527, + "lt": 2528, + "\u0120inj": 2529, + "iding": 2530, + "\u0120Spe": 2531, + "\u0120chall": 2532, + "irm": 2533, + "\u012022": 2534, + "itting": 2535, + "str": 2536, + "\u0120hy": 2537, + "LE": 2538, + "key": 2539, + "\u0120began": 2540, + "atur": 2541, + "ashington": 2542, + "lam": 2543, + "\u0120Dav": 2544, + "bit": 2545, + "\u0120size": 2546, + "\u0120Par": 2547, + "38": 2548, + "ournal": 2549, + "face": 2550, + "\u0120decision": 2551, + "\u0120larg": 2552, + "\u0120jud": 2553, + "rect": 2554, + "\u0120continue": 2555, + "\u0120Oct": 2556, + "overed": 2557, + "\u0120Int": 2558, + "========": 2559, + "\u0120parent": 2560, + "\u0120Will": 2561, + "\u0120easy": 2562, + "\u0120drug": 2563, + "anger": 2564, + "\u0120sense": 2565, + "\u0120di": 2566, + "iday": 2567, + "\u0120energy": 2568, + "istic": 2569, + "\u0120associ": 2570, + "arter": 2571, + "obal": 2572, + "eks": 2573, + "\u0120El": 2574, + "urch": 2575, + "\u0120girl": 2576, + "oe": 2577, + "itle": 2578, + "\u012028": 2579, + "\u0120Che": 2580, + "\u0120request": 2581, + "\u0120soon": 2582, + "\u0120host": 2583, + "ky": 2584, + "\u0120states": 2585, + "omes": 2586, + "\u0120material": 2587, + "lex": 2588, + "\u0120moment": 2589, + "\u0120answ": 2590, + "onse": 2591, + "\u0120especially": 2592, + "\u0120norm": 2593, + "\u0120services": 2594, + "pite": 2595, + "ran": 2596, + "\u0120role": 2597, + "44": 2598, + "):": 2599, + "\u0120cred": 2600, + "Cl": 2601, + "________": 2602, + "\u0120mat": 2603, + "\u0120log": 2604, + "\u0120Clinton": 2605, + "OU": 2606, + "\u0120office": 2607, + "\u012026": 2608, + "\u0120charg": 2609, + "\u0120track": 2610, + "ma": 2611, + "\u0120heart": 2612, + "\u0120ball": 2613, + "\u0120personal": 2614, + "\u0120building": 2615, + "na": 2616, + "set": 2617, + "body": 2618, + "\u0120Black": 2619, + "\u0120increase": 2620, + "itten": 2621, + "\u0120needed": 2622, + "36": 2623, + "32": 2624, + "=\"": 2625, + "\u0120lost": 2626, + "\u0120became": 2627, + "\u0120groups": 2628, + "\u0120Mus": 2629, + "\u0120wrote": 2630, + "\u0120Pe": 2631, + "\u0120prop": 2632, + "joy": 2633, + "\u00c3\u00a9": 2634, + "\u0120White": 2635, + "\u0120dead": 2636, + ".'": 2637, + "\u0120http": 2638, + "\u0120webs": 2639, + "OS": 2640, + "\u0120inside": 2641, + "\u0120wrong": 2642, + "\u0120statement": 2643, + "\u0120...": 2644, + "yl": 2645, + "\u0120film": 2646, + "\u0120music": 2647, + "\u0120share": 2648, + "ification": 2649, + "\u0120release": 2650, + "\u0120forward": 2651, + "\u0120stay": 2652, + "\u0120comput": 2653, + "itte": 2654, + "ser": 2655, + "\u0120original": 2656, + "\u0120card": 2657, + "\u0120cand": 2658, + "\u0120div": 2659, + "atural": 2660, + "\u0120favor": 2661, + "OM": 2662, + "\u0120cases": 2663, + "uses": 2664, + "\u0120section": 2665, + "\u0120leave": 2666, + "ging": 2667, + "oved": 2668, + "\u0120Washington": 2669, + "39": 2670, + "\u0120Gl": 2671, + "\u0120required": 2672, + "action": 2673, + "apan": 2674, + "oor": 2675, + "iter": 2676, + "\u0120King": 2677, + "\u0120countries": 2678, + "\u0120German": 2679, + "lling": 2680, + "\u012027": 2681, + "34": 2682, + "\u0120questions": 2683, + "\u0120prim": 2684, + "\u0120cell": 2685, + "\u0120shoot": 2686, + "\u0120anyone": 2687, + "\u0120West": 2688, + "\u0120affect": 2689, + "epend": 2690, + "\u0120online": 2691, + "\u0120Israel": 2692, + "\u0120September": 2693, + "\u0120ability": 2694, + "\u0120content": 2695, + "ises": 2696, + "\u0120reve": 2697, + "\u0120laun": 2698, + "\u0120indic": 2699, + "\u0120force": 2700, + "cast": 2701, + "\u0120sold": 2702, + "aving": 2703, + "fl": 2704, + "\u0120soft": 2705, + "\u0120companies": 2706, + "ceed": 2707, + "\u0120article": 2708, + "\u0120aud": 2709, + "\u0120rev": 2710, + "\u0120educ": 2711, + "\u0120playing": 2712, + "05": 2713, + "\u0120held": 2714, + "ctor": 2715, + "\u0120released": 2716, + "\u0120federal": 2717, + "37": 2718, + "\u0120administ": 2719, + "\u0120interview": 2720, + "\u0120install": 2721, + "\u0120received": 2722, + "\u0120source": 2723, + "uk": 2724, + "Ph": 2725, + "\u0120serious": 2726, + "\u0120created": 2727, + "\u0120cause": 2728, + "\u0120immedi": 2729, + "\u0120defin": 2730, + "uel": 2731, + "\u0120Department": 2732, + "ctions": 2733, + "\u0120Cour": 2734, + "\u0120Now": 2735, + "ze": 2736, + "ites": 2737, + "itution": 2738, + "\u0120late": 2739, + "\u0120speak": 2740, + "ners": 2741, + "\u0120legal": 2742, + "ari": 2743, + "\u0120Cor": 2744, + "\u0120weeks": 2745, + "\u0120model": 2746, + "\u0120pred": 2747, + "\u0120exact": 2748, + "BC": 2749, + "\u0120By": 2750, + "ING": 2751, + "osing": 2752, + "\u0120takes": 2753, + "\u0120regard": 2754, + "\u0120opportun": 2755, + "\u0120price": 2756, + "\u0120198": 2757, + "\u0120Apr": 2758, + "fully": 2759, + "\u0120ord": 2760, + "\u0120problems": 2761, + "ruction": 2762, + "ham": 2763, + "\u0120Count": 2764, + "lege": 2765, + "\u0120leaders": 2766, + "ET": 2767, + "lev": 2768, + "\u0120deep": 2769, + "ological": 2770, + "ese": 2771, + "haps": 2772, + "\u0120Some": 2773, + "\u0120pers": 2774, + "\u0120contract": 2775, + "\u0120relationship": 2776, + "sp": 2777, + "oud": 2778, + "\u0120base": 2779, + "48": 2780, + "mit": 2781, + "Ad": 2782, + "ancial": 2783, + "\u0120consum": 2784, + "\u0120potential": 2785, + "\u0120langu": 2786, + "rem": 2787, + "eth": 2788, + "\u0120relig": 2789, + "ressed": 2790, + "66": 2791, + "\u0120link": 2792, + "\u0120lower": 2793, + "ayer": 2794, + "\u0120June": 2795, + "\u0120fem": 2796, + "unt": 2797, + "erc": 2798, + "urd": 2799, + "\u0120contact": 2800, + "\u0120ill": 2801, + "\u0120mother": 2802, + "\u0120estab": 2803, + "htt": 2804, + "\u0120March": 2805, + "\u0120Bro": 2806, + "\u0120China": 2807, + "\u012029": 2808, + "\u0120squ": 2809, + "\u0120provided": 2810, + "\u0120average": 2811, + "asons": 2812, + "\u01202011": 2813, + "\u0120exam": 2814, + "lin": 2815, + "55": 2816, + "ned": 2817, + "\u0120perfect": 2818, + "\u0120tou": 2819, + "alse": 2820, + "ux": 2821, + "\u0120buy": 2822, + "\u0120shot": 2823, + "\u0120collect": 2824, + "\u0120phot": 2825, + "\u0120played": 2826, + "\u0120surpr": 2827, + "\u0120officials": 2828, + "\u0120simple": 2829, + "avy": 2830, + "\u0120industry": 2831, + "\u0120hands": 2832, + "ground": 2833, + "\u0120pull": 2834, + "\u0120round": 2835, + "\u0120user": 2836, + "\u0120range": 2837, + "uary": 2838, + "\u0120private": 2839, + "ops": 2840, + "ees": 2841, + "\u0120ways": 2842, + "\u0120Mich": 2843, + "\u0120veh": 2844, + "\u0120except": 2845, + "\u0120terms": 2846, + "imum": 2847, + "pper": 2848, + "ION": 2849, + "ores": 2850, + "\u0120Dragon": 2851, + "oul": 2852, + "\u0120den": 2853, + "\u0120performance": 2854, + "\u0120bill": 2855, + "cil": 2856, + "47": 2857, + "\u0120environment": 2858, + "\u0120exc": 2859, + "add": 2860, + "\u0120worth": 2861, + "\u0120pict": 2862, + "\u0120chance": 2863, + "\u01202018": 2864, + "bor": 2865, + "\u0120speed": 2866, + "iction": 2867, + "\u0120alleg": 2868, + "\u0120Japan": 2869, + "atory": 2870, + "reet": 2871, + "\u0120match": 2872, + "\u0120II": 2873, + "\u0120stru": 2874, + "order": 2875, + "\u0120ste": 2876, + "\u0120living": 2877, + "\u0120struct": 2878, + "ino": 2879, + "\u0120separ": 2880, + "hern": 2881, + "\u0120response": 2882, + "\u0120enjoy": 2883, + "\u0120via": 2884, + "AD": 2885, + "uments": 2886, + "acebook": 2887, + "\u0120member": 2888, + "ibr": 2889, + "izing": 2890, + "\u0120tool": 2891, + "\u0120Mon": 2892, + "\u0120While": 2893, + "hood": 2894, + "\u0120Ang": 2895, + "\u0120Def": 2896, + "\u0120offer": 2897, + "Tr": 2898, + "aur": 2899, + "\u0120turned": 2900, + "\u0120July": 2901, + "down": 2902, + "anced": 2903, + "\u0120recently": 2904, + "\u0120Ear": 2905, + "\u0120ce": 2906, + "\u0120Star": 2907, + "\u0120Cong": 2908, + "rought": 2909, + "\u0120blood": 2910, + "\u0120hope": 2911, + "\u0120comment": 2912, + "aint": 2913, + "\u0120arri": 2914, + "iles": 2915, + "\u0120particip": 2916, + "ought": 2917, + "ription": 2918, + "08": 2919, + "49": 2920, + "\u0120gave": 2921, + "\u0120select": 2922, + "\u0120killed": 2923, + "sych": 2924, + "\u0120goes": 2925, + "ij": 2926, + "\u0120coll": 2927, + "\u0120impact": 2928, + "atives": 2929, + "\u0120Ser": 2930, + "09": 2931, + "\u0120August": 2932, + "\u0120boy": 2933, + "de": 2934, + "\u0120Des": 2935, + "\u0120felt": 2936, + "US": 2937, + "\u0120expected": 2938, + "\u0120image": 2939, + "\u0120Mark": 2940, + "ccording": 2941, + "oice": 2942, + "EC": 2943, + "\u0120Mag": 2944, + "ened": 2945, + "hold": 2946, + "\u0120Post": 2947, + "\u0120prevent": 2948, + "No": 2949, + "\u0120involved": 2950, + "\u0120eyes": 2951, + "\u0120quickly": 2952, + "At": 2953, + "unk": 2954, + "\u0120behav": 2955, + "\u0120ur": 2956, + "\u0120led": 2957, + "come": 2958, + "ey": 2959, + "\u0120candid": 2960, + "\u0120earlier": 2961, + "\u0120focus": 2962, + "ety": 2963, + "Pro": 2964, + "ledge": 2965, + "ixed": 2966, + "illed": 2967, + "\u0120popular": 2968, + "AP": 2969, + "\u0120sett": 2970, + "light": 2971, + "\u0120various": 2972, + "inks": 2973, + "\u0120levels": 2974, + "\u0120road": 2975, + "ellig": 2976, + "ables": 2977, + "hel": 2978, + "ittee": 2979, + "\u0120Gener": 2980, + "ype": 2981, + "\u0120heard": 2982, + "icles": 2983, + "\u0120mis": 2984, + "\u0120users": 2985, + "\u0120San": 2986, + "\u0120improve": 2987, + "\u0120father": 2988, + "\u0120search": 2989, + "They": 2990, + "vil": 2991, + "\u0120profess": 2992, + "\u0120knew": 2993, + "\u0120loss": 2994, + "\u0120events": 2995, + "65": 2996, + "\u0120billion": 2997, + "07": 2998, + "02": 2999, + "\u0120News": 3000, + "\u0120AM": 3001, + "\u0120cover": 3002, + "where": 3003, + "ension": 3004, + "\u0120bott": 3005, + "\u0120areas": 3006, + "ences": 3007, + "ope": 3008, + "\u0120Twitter": 3009, + "ael": 3010, + "\u0120gets": 3011, + "\u0120Google": 3012, + "\u0120sn": 3013, + "iant": 3014, + "\u0120vote": 3015, + "\u0120nearly": 3016, + "\u0120included": 3017, + "\u0120recogn": 3018, + "zz": 3019, + "mm": 3020, + "aled": 3021, + "\u0120happened": 3022, + "04": 3023, + "\u0120hot": 3024, + "\u0120whose": 3025, + "\u0120civil": 3026, + "\u0120suff": 3027, + "oes": 3028, + "itiz": 3029, + "\u0120Syri": 3030, + "\u0120respond": 3031, + "\u0120hon": 3032, + "\u0120features": 3033, + "\u0120economic": 3034, + "\u0120April": 3035, + "rim": 3036, + "\u0120technology": 3037, + "\u0120option": 3038, + "aging": 3039, + "\u0120purch": 3040, + "Re": 3041, + "\u0120lat": 3042, + "chie": 3043, + "isl": 3044, + "\u0120recomm": 3045, + "uf": 3046, + "\u0120training": 3047, + "\u0120effects": 3048, + "\u0120fast": 3049, + "\u01202010": 3050, + "\u0120occur": 3051, + "\u0120website": 3052, + "\u0120email": 3053, + "\u0120sens": 3054, + "ech": 3055, + "\u0120oil": 3056, + "\u0120influ": 3057, + "\u0120currently": 3058, + "\u0120Sch": 3059, + "\u0120Add": 3060, + "\u0120goal": 3061, + "\u0120scient": 3062, + "\u0120conv": 3063, + "100": 3064, + "emy": 3065, + "\u0120decided": 3066, + "\u0120travel": 3067, + "\u0120mention": 3068, + "LL": 3069, + "03": 3070, + "\u0120election": 3071, + "\u0120phone": 3072, + "\u0120looks": 3073, + "\u0120situation": 3074, + "\u0120cy": 3075, + "\u0120hor": 3076, + "bed": 3077, + "\u0120Court": 3078, + "aily": 3079, + "aves": 3080, + "\u0120quality": 3081, + "\u0120Comp": 3082, + "wise": 3083, + "\u0120table": 3084, + "\u0120staff": 3085, + "\u0120Wind": 3086, + "ett": 3087, + "\u0120tried": 3088, + "idered": 3089, + "\u0120addition": 3090, + "\u0120box": 3091, + "\u0120lack": 3092, + "arily": 3093, + "\u0120wide": 3094, + "\u0120mid": 3095, + "\u0120board": 3096, + "ysis": 3097, + "\u0120anti": 3098, + "ha": 3099, + "\u0120dig": 3100, + "ening": 3101, + "\u0120dro": 3102, + "Con": 3103, + "68": 3104, + "\u0120slow": 3105, + "based": 3106, + "sequ": 3107, + "\u0120path": 3108, + "Ex": 3109, + "aker": 3110, + "\u0120worked": 3111, + "\u0120pen": 3112, + "\u0120engine": 3113, + "\u0120looked": 3114, + "\u0120Super": 3115, + "\u0120Serv": 3116, + "\u0120victim": 3117, + "Un": 3118, + "\u0120property": 3119, + "\u0120introdu": 3120, + "\u0120execut": 3121, + "\u0120PM": 3122, + "Le": 3123, + "\u0120color": 3124, + "\u0120More": 3125, + "\u012060": 3126, + "\u0120network": 3127, + "\u0120date": 3128, + "cul": 3129, + "idge": 3130, + "\u0120extra": 3131, + "31": 3132, + "\u0120sle": 3133, + "67": 3134, + "\u0120wond": 3135, + "\u0120reports": 3136, + "just": 3137, + "\u0120Austral": 3138, + "\u0120capital": 3139, + "\u0120ens": 3140, + "\u0120command": 3141, + "\u0120allowed": 3142, + "\u0120prep": 3143, + "\u0120capt": 3144, + "hib": 3145, + "\u0120numbers": 3146, + "chan": 3147, + "\u0120fair": 3148, + "mp": 3149, + "oms": 3150, + "\u0120reach": 3151, + "With": 3152, + "tain": 3153, + "\u0120broad": 3154, + "\u0120couple": 3155, + "ecause": 3156, + "lying": 3157, + "\u0120Feb": 3158, + "\u0120screen": 3159, + "\u0120lives": 3160, + "\u0120prior": 3161, + "\u0120Congress": 3162, + "Ar": 3163, + "\u0120approach": 3164, + "\u0120emer": 3165, + "aries": 3166, + "\u0120Dis": 3167, + "serv": 3168, + "\u0120Ne": 3169, + "\u0120built": 3170, + "cies": 3171, + "\u0120repe": 3172, + "\u0120rules": 3173, + "force": 3174, + "\u0120Pal": 3175, + "\u0120financial": 3176, + "\u0120considered": 3177, + "\u0120Char": 3178, + "nces": 3179, + "\u0120IS": 3180, + "\u0120brought": 3181, + "\u0120bi": 3182, + "iers": 3183, + "\u0120Sim": 3184, + "OP": 3185, + "\u0120products": 3186, + "\u0120visit": 3187, + "\u0120document": 3188, + "\u0120conduct": 3189, + "\u0120completely": 3190, + "ining": 3191, + "\u0120Calif": 3192, + "ibly": 3193, + "\u0120written": 3194, + "\u0120TV": 3195, + "ements": 3196, + "\u0120draw": 3197, + "One": 3198, + "\u0120published": 3199, + "\u0120secret": 3200, + "rain": 3201, + "het": 3202, + "\u0120Facebook": 3203, + "onday": 3204, + "\u0120Up": 3205, + "\u0120sexual": 3206, + "\u0120thous": 3207, + "\u0120Pat": 3208, + "\u0120ess": 3209, + "\u0120standard": 3210, + "\u0120arm": 3211, + "ges": 3212, + "ection": 3213, + "\u0120fell": 3214, + "\u0120foreign": 3215, + "ani": 3216, + "\u0120Friday": 3217, + "\u0120regular": 3218, + "inary": 3219, + "\u0120increased": 3220, + "\u0120usually": 3221, + "\u0120demon": 3222, + "\u0120dark": 3223, + "\u0120additional": 3224, + "rol": 3225, + "\u0120Of": 3226, + "\u0120production": 3227, + "!!": 3228, + "undred": 3229, + "\u0120international": 3230, + "idents": 3231, + "\u0120Free": 3232, + "roup": 3233, + "\u0120race": 3234, + "\u0120mach": 3235, + "\u0120huge": 3236, + "All": 3237, + "lear": 3238, + "ovember": 3239, + "\u0120town": 3240, + "\u0120attention": 3241, + "\u0120Off": 3242, + "yond": 3243, + "\u0120Then": 3244, + "field": 3245, + "\u0120terror": 3246, + "raz": 3247, + "\u0120Bo": 3248, + "\u0120meeting": 3249, + "\u0120Park": 3250, + "\u0120arrest": 3251, + "\u0120fear": 3252, + "\u0120aw": 3253, + "\u0120Val": 3254, + "oring": 3255, + "',": 3256, + "\u0120extreme": 3257, + "arr": 3258, + "\u0120workers": 3259, + "After": 3260, + "\u012031": 3261, + "net": 3262, + "ament": 3263, + "\u0120directly": 3264, + "\u0120population": 3265, + "ube": 3266, + "\u0120October": 3267, + "\u0120IN": 3268, + "\u0120January": 3269, + "59": 3270, + "\u0120David": 3271, + "\u0120cross": 3272, + "cember": 3273, + "\u0120First": 3274, + "\u0120message": 3275, + "irit": 3276, + "\u0120nation": 3277, + "\u0120poll": 3278, + "isions": 3279, + "\u0120answer": 3280, + "ny": 3281, + "isode": 3282, + "\u0120carry": 3283, + "\u0120Russia": 3284, + "\u0120hear": 3285, + "ength": 3286, + "roy": 3287, + "\u0120natural": 3288, + "inally": 3289, + "\u0120dog": 3290, + "mitted": 3291, + "\u0120trade": 3292, + "\u0120subst": 3293, + "\u0120multiple": 3294, + "\u0120Afric": 3295, + "\u0120fans": 3296, + "\u0120sort": 3297, + "\u0120global": 3298, + "ication": 3299, + "\u0120Wed": 3300, + "ara": 3301, + "\u0120achie": 3302, + "\u0120language": 3303, + "vey": 3304, + "\u0120tal": 3305, + "\u0120necessary": 3306, + "\u0120details": 3307, + "\u0120sen": 3308, + "\u0120Sund": 3309, + "\u0120Reg": 3310, + "\u0120Rec": 3311, + "06": 3312, + "\u0120sil": 3313, + "ressive": 3314, + "\u0120medical": 3315, + "unch": 3316, + "ornia": 3317, + "\u0120und": 3318, + "fort": 3319, + "ocks": 3320, + "\u0120Monday": 3321, + "uesday": 3322, + "craft": 3323, + "77": 3324, + "urt": 3325, + "\u0120ver": 3326, + "\u0120Hill": 3327, + "\u0120receive": 3328, + "\u0120morning": 3329, + "estern": 3330, + "\u0120bank": 3331, + "\u0120sat": 3332, + "irth": 3333, + "\u0120High": 3334, + "\u0120device": 3335, + "\u0120THE": 3336, + "\u0120Center": 3337, + "\u0120safe": 3338, + "\u0120ple": 3339, + "\u0120Canada": 3340, + "\u0120systems": 3341, + "\u0120assist": 3342, + "\u0120surv": 3343, + "\u0120battle": 3344, + "\u0120Soc": 3345, + "vertis": 3346, + "She": 3347, + "\u0120paper": 3348, + "\u0120growth": 3349, + "\u0120cast": 3350, + "Sc": 3351, + "\u0120plans": 3352, + "lled": 3353, + "\u0120parts": 3354, + "\u0120wall": 3355, + "\u0120movement": 3356, + "\u0120practice": 3357, + "imately": 3358, + "\u0120display": 3359, + "\u0120sometimes": 3360, + "omp": 3361, + "\u0120Paul": 3362, + "\u0120Yes": 3363, + "king": 3364, + "58": 3365, + "oly": 3366, + "\u0120son": 3367, + "\u0120avoid": 3368, + "okes": 3369, + "\u0120Jew": 3370, + "\u0120towards": 3371, + "asc": 3372, + "\u0120//": 3373, + "\u0120Kore": 3374, + "\u0120talking": 3375, + "\u0120correct": 3376, + "\u0120spent": 3377, + "icks": 3378, + "iable": 3379, + "eared": 3380, + "\u0120term": 3381, + "\u0120wants": 3382, + "oming": 3383, + "\u0120ut": 3384, + "\u0120doub": 3385, + "\u0120forces": 3386, + "\u0120please": 3387, + "69": 3388, + "\u0120November": 3389, + "atform": 3390, + "ondon": 3391, + "\u0120ones": 3392, + "\u0120immediately": 3393, + "\u0120Russian": 3394, + "\u0120Met": 3395, + "\u0120deg": 3396, + "\u0120parents": 3397, + "CH": 3398, + "\u0120Americans": 3399, + "aly": 3400, + "\u0120Mod": 3401, + "\u0120shown": 3402, + "\u0120conditions": 3403, + "\u0120stuff": 3404, + "\u0120reb": 3405, + "\u0120Your": 3406, + "\u0120includes": 3407, + "nown": 3408, + "\u0120Sam": 3409, + "\u0120experien": 3410, + "mission": 3411, + "\u0120Even": 3412, + "aught": 3413, + "\u0120announced": 3414, + "\u0120Republican": 3415, + "\u0120determin": 3416, + "\u0120described": 3417, + "\u0120County": 3418, + "()": 3419, + "\u0120door": 3420, + "\u0120changed": 3421, + "\u0120neigh": 3422, + "\u0120Here": 3423, + "\u0120clean": 3424, + "\u0120pan": 3425, + "\u0120December": 3426, + "\u0120European": 3427, + "iring": 3428, + "apter": 3429, + "\u0120club": 3430, + "\u0120Tuesday": 3431, + "\u0120paid": 3432, + "\u0120Net": 3433, + "\u0120attacks": 3434, + "\u0120characters": 3435, + "\u0120alone": 3436, + "\u0120director": 3437, + "dom": 3438, + "\u012035": 3439, + "\u0120load": 3440, + "\u0120rout": 3441, + "\u0120California": 3442, + "\u0120finally": 3443, + "\u0120rac": 3444, + "\u0120contr": 3445, + "\u0120exactly": 3446, + "resh": 3447, + "pri": 3448, + "\u0120Islam": 3449, + "\u0120nature": 3450, + "\u0120career": 3451, + "\u0120latest": 3452, + "\u0120convers": 3453, + "\u0120Sl": 3454, + "pose": 3455, + "cient": 3456, + "\u0120Inc": 3457, + "ivity": 3458, + "88": 3459, + "\u0120Att": 3460, + "\u0120Mor": 3461, + "nesday": 3462, + "\u0120weight": 3463, + "ken": 3464, + "\u0120note": 3465, + "\u0120teams": 3466, + "\u0120\\": 3467, + "airs": 3468, + "\u0120Green": 3469, + "\u0120hundred": 3470, + "onent": 3471, + "\u0120streng": 3472, + "\u0120consist": 3473, + "icated": 3474, + "\u0120regul": 3475, + "\u0120lic": 3476, + "astic": 3477, + "\u0120ten": 3478, + "ursday": 3479, + "elligence": 3480, + "ously": 3481, + "\u0120UK": 3482, + "BI": 3483, + "\u0120costs": 3484, + "\u0120independ": 3485, + "\u0120AP": 3486, + "\u0120normal": 3487, + "\u0120hom": 3488, + "\u0120obvious": 3489, + "\u0120swe": 3490, + "\u0120star": 3491, + "\u0120ready": 3492, + "acher": 3493, + "\u0120implement": 3494, + "gest": 3495, + "\u0120song": 3496, + "\u0120Get": 3497, + "\u0120Lab": 3498, + "\u0120interesting": 3499, + "using": 3500, + "\u0120giving": 3501, + "\u0120Sunday": 3502, + "\u0120etc": 3503, + "\u0120middle": 3504, + "\u0120remember": 3505, + "right": 3506, + "osition": 3507, + "utions": 3508, + "\u0120max": 3509, + "46": 3510, + "\u0120yourself": 3511, + "\u0120demand": 3512, + "\u0120treatment": 3513, + "\u0120danger": 3514, + "\u0120Cons": 3515, + "\u0120guy": 3516, + "\u0120British": 3517, + "\u0120physical": 3518, + "\u0120related": 3519, + "\u0120remain": 3520, + "\u0120couldn": 3521, + "\u0120refer": 3522, + "\u0120citiz": 3523, + "box": 3524, + "ENT": 3525, + "board": 3526, + "\u0120inn": 3527, + "IG": 3528, + "ero": 3529, + "\u0120Street": 3530, + "ospital": 3531, + "rench": 3532, + "chers": 3533, + "\u0120stra": 3534, + "OL": 3535, + "ager": 3536, + "\u0120AN": 3537, + "\u0120easily": 3538, + "IA": 3539, + "enge": 3540, + "iny": 3541, + "\u0120clos": 3542, + "ocked": 3543, + "\u0120uses": 3544, + "\u0120Coun": 3545, + "Im": 3546, + "uild": 3547, + "??": 3548, + "more": 3549, + "\u0120ang": 3550, + "\u0120write": 3551, + "olute": 3552, + "57": 3553, + "\u0120leader": 3554, + "\u0120reading": 3555, + "": 3784, + "\u0120figure": 3785, + "\u0120disapp": 3786, + "enty": 3787, + "\u0120software": 3788, + "\u0120ult": 3789, + "\u0120officers": 3790, + "New": 3791, + "Is": 3792, + "\u0120remains": 3793, + "\u0120India": 3794, + "\u0120psych": 3795, + "rief": 3796, + "\u0120cat": 3797, + "esc": 3798, + "\u0120observ": 3799, + "\u0120stage": 3800, + "\u0120Dark": 3801, + "\u0120enter": 3802, + "change": 3803, + "\u0120passed": 3804, + "\u0120despite": 3805, + "\u0120Out": 3806, + "\u0120movie": 3807, + "rs": 3808, + "\u0120voice": 3809, + "mine": 3810, + "\u0120Play": 3811, + "\u0120toward": 3812, + "\u0120Ter": 3813, + "\u0120region": 3814, + "\u0120values": 3815, + "orters": 3816, + "\u0120mount": 3817, + "\u0120officer": 3818, + "\u0120Other": 3819, + "ban": 3820, + "\u0120hous": 3821, + "wood": 3822, + "room": 3823, + "IV": 3824, + "\u0120Sun": 3825, + "see": 3826, + "\u0120Over": 3827, + "rog": 3828, + "90": 3829, + "\u0120lay": 3830, + "\u0120Tur": 3831, + "awn": 3832, + "\u0120pressure": 3833, + "\u0120Sub": 3834, + "\u0120books": 3835, + "edom": 3836, + "\u0120Sand": 3837, + "AA": 3838, + "ago": 3839, + "\u0120reasons": 3840, + "ford": 3841, + "\u0120activity": 3842, + "UT": 3843, + "Now": 3844, + "\u0120Senate": 3845, + "cell": 3846, + "night": 3847, + "\u0120calls": 3848, + "inter": 3849, + "\u0120letter": 3850, + "\u0120Rob": 3851, + "\u0120Je": 3852, + "\u0120choose": 3853, + "\u0120Law": 3854, + "Get": 3855, + "Be": 3856, + "\u0120rob": 3857, + "\u0120types": 3858, + "\u0120platform": 3859, + "\u0120quarter": 3860, + "RA": 3861, + "\u0120Time": 3862, + "\u0120maybe": 3863, + "\u0120Cr": 3864, + "95": 3865, + "pre": 3866, + "\u0120moving": 3867, + "\u0120lif": 3868, + "\u0120gold": 3869, + "\u0120som": 3870, + "\u0120patients": 3871, + "\u0120truth": 3872, + "\u0120Ke": 3873, + "urance": 3874, + "antly": 3875, + "mar": 3876, + "\u0120charge": 3877, + "\u0120Great": 3878, + "\u0120cele": 3879, + "--------------------------------": 3880, + "\u0120rock": 3881, + "roid": 3882, + "ancy": 3883, + "\u0120credit": 3884, + "aud": 3885, + "By": 3886, + "\u0120Every": 3887, + "\u0120moved": 3888, + "inger": 3889, + "ribution": 3890, + "\u0120names": 3891, + "\u0120straight": 3892, + "\u0120Health": 3893, + "\u0120Well": 3894, + "\u0120feature": 3895, + "\u0120rule": 3896, + "\u0120sche": 3897, + "inated": 3898, + "\u0120Michael": 3899, + "berg": 3900, + "41": 3901, + "iled": 3902, + "band": 3903, + "\u0120click": 3904, + "\u0120Angel": 3905, + "onents": 3906, + "\u00c2\u0143": 3907, + "\u0120Iraq": 3908, + "\u0120Saturday": 3909, + "\u0120aware": 3910, + "part": 3911, + "\u0120pattern": 3912, + "OW": 3913, + "\u0120Let": 3914, + "\u0120grad": 3915, + "igned": 3916, + "\u0120associated": 3917, + "\u0120style": 3918, + "no": 3919, + "iation": 3920, + "aith": 3921, + "ilies": 3922, + "\u0120stories": 3923, + "uration": 3924, + "\u0120individuals": 3925, + "\u0120\u00e2\u0122\u00a6": 3926, + "miss": 3927, + "\u0120Associ": 3928, + "ishing": 3929, + "aby": 3930, + "\u0120summer": 3931, + "\u0120Ben": 3932, + "\u012032": 3933, + "\u0120arch": 3934, + "uty": 3935, + "\u0120Texas": 3936, + "hol": 3937, + "\u0120fully": 3938, + "\u0120mill": 3939, + "\u0120followed": 3940, + "\u0120Bill": 3941, + "\u0120Indian": 3942, + "\u0120Secret": 3943, + "\u0120Bel": 3944, + "\u0120February": 3945, + "\u0120jobs": 3946, + "\u0120seemed": 3947, + "\u0120Govern": 3948, + "ipped": 3949, + "\u0120reality": 3950, + "\u0120lines": 3951, + "\u0120park": 3952, + "\u0120measure": 3953, + "\u0120Our": 3954, + "IM": 3955, + "\u0120brother": 3956, + "\u0120growing": 3957, + "\u0120ban": 3958, + "\u0120estim": 3959, + "\u0120cry": 3960, + "\u0120School": 3961, + "\u0120mechan": 3962, + "\u0120OF": 3963, + "\u0120Windows": 3964, + "\u0120rates": 3965, + "\u0120Oh": 3966, + "\u0120positive": 3967, + "\u0120culture": 3968, + "istics": 3969, + "ica": 3970, + "\u0120har": 3971, + "ya": 3972, + "itely": 3973, + "ipp": 3974, + "\u0120map": 3975, + "encies": 3976, + "\u0120William": 3977, + "II": 3978, + "akers": 3979, + "56": 3980, + "\u0120Mart": 3981, + "\u0120Rem": 3982, + "\u0120altern": 3983, + "itude": 3984, + "\u0120coach": 3985, + "rowd": 3986, + "Don": 3987, + "\u0120kids": 3988, + "\u0120journal": 3989, + "\u0120corpor": 3990, + "\u0120false": 3991, + "\u0120web": 3992, + "\u0120sleep": 3993, + "\u0120contain": 3994, + "\u0120sto": 3995, + "\u0120bed": 3996, + "iverse": 3997, + "\u0120Rich": 3998, + "\u0120Chinese": 3999, + "\u0120pun": 4000, + "\u0120meant": 4001, + "known": 4002, + "\u0120notice": 4003, + "\u0120favorite": 4004, + "aven": 4005, + "\u0120condition": 4006, + "\u0120purpose": 4007, + "))": 4008, + "\u0120organization": 4009, + "\u0120challeng": 4010, + "\u0120manufact": 4011, + "\u0120susp": 4012, + "\u0120Ac": 4013, + "\u0120critic": 4014, + "unes": 4015, + "uclear": 4016, + "\u0120mer": 4017, + "vention": 4018, + "\u012080": 4019, + "\u0120mist": 4020, + "\u0120Us": 4021, + "\u0120Tor": 4022, + "http": 4023, + "olf": 4024, + "\u0120larger": 4025, + "\u0120advant": 4026, + "\u0120resear": 4027, + "\u0120actions": 4028, + "ml": 4029, + "\u0120kept": 4030, + "\u0120aim": 4031, + ",'": 4032, + "col": 4033, + "\u0120benefits": 4034, + "ifying": 4035, + "\u0120actual": 4036, + "\u0120International": 4037, + "\u0120vehicle": 4038, + "\u0120chief": 4039, + "\u0120efforts": 4040, + "\u0120League": 4041, + "\u0120Most": 4042, + "\u0120wait": 4043, + "\u0120adult": 4044, + "\u0120overall": 4045, + "\u0120speech": 4046, + "\u0120highly": 4047, + "\u0120female": 4048, + "\u0120error": 4049, + "\u0120effective": 4050, + "54": 4051, + "\u0120encour": 4052, + "well": 4053, + "\u0120failed": 4054, + "\u0120conserv": 4055, + "\u0120programs": 4056, + "\u0120trou": 4057, + "\u0120ahead": 4058, + "500": 4059, + "vertisement": 4060, + "IP": 4061, + "\u0120Found": 4062, + "pir": 4063, + "\u0120%": 4064, + "\u0120crime": 4065, + "ander": 4066, + "\u0120location": 4067, + "\u0120Iran": 4068, + "\u0120behavior": 4069, + "azing": 4070, + "\u0120rare": 4071, + "\u0120emb": 4072, + "\u0120caused": 4073, + "\u0120ship": 4074, + "\u0120active": 4075, + "\u0120contribut": 4076, + "\u0120green": 4077, + "\u0120acqu": 4078, + "\u0120reflect": 4079, + "venue": 4080, + "\u0120firm": 4081, + "\u0120birth": 4082, + "].": 4083, + "\u0120clearly": 4084, + "\u0120emot": 4085, + "\u0120agency": 4086, + "riage": 4087, + "\u0120memory": 4088, + "98": 4089, + "SA": 4090, + "\u0120See": 4091, + "acing": 4092, + "CC": 4093, + "\u0120biggest": 4094, + "\u0120rap": 4095, + "\u0120basic": 4096, + "\u0120band": 4097, + "eat": 4098, + "\u0120suspect": 4099, + "\u0120Mac": 4100, + "\u012090": 4101, + "mark": 4102, + "istan": 4103, + "\u0120spread": 4104, + "ams": 4105, + "ki": 4106, + "asy": 4107, + "rav": 4108, + "\u0120Rober": 4109, + "\u0120demonstr": 4110, + "rated": 4111, + "\u0120absolute": 4112, + "\u0120places": 4113, + "\u0120impl": 4114, + "ibrary": 4115, + "\u0120cards": 4116, + "\u0120destroy": 4117, + "\u0120virt": 4118, + "vere": 4119, + "\u0120appeared": 4120, + "yan": 4121, + "point": 4122, + "\u0120beg": 4123, + "\u0120temper": 4124, + "spe": 4125, + "anted": 4126, + "ears": 4127, + "\u0120Direct": 4128, + "\u0120length": 4129, + "\u0120blog": 4130, + "amb": 4131, + "\u0120integ": 4132, + "\u0120resources": 4133, + "acc": 4134, + "iful": 4135, + "\u0120spot": 4136, + "\u0120forced": 4137, + "\u0120thousands": 4138, + "\u0120Minister": 4139, + "\u0120qual": 4140, + "\u0120French": 4141, + "atically": 4142, + "\u0120generally": 4143, + "\u0120drink": 4144, + "\u0120thus": 4145, + "IL": 4146, + "odes": 4147, + "\u0120appropri": 4148, + "\u0120Read": 4149, + "\u0120whom": 4150, + "\u0120eye": 4151, + "\u0120college": 4152, + "\u012045": 4153, + "irection": 4154, + "\u0120ensure": 4155, + "\u0120apparent": 4156, + "iders": 4157, + "\u0120religious": 4158, + "\u0120minor": 4159, + "olic": 4160, + "\u0120tro": 4161, + "\u0120Why": 4162, + "ribute": 4163, + "met": 4164, + "\u0120primary": 4165, + "\u0120developed": 4166, + "\u0120peace": 4167, + "\u0120skin": 4168, + "ste": 4169, + "ava": 4170, + "\u0120blue": 4171, + "\u0120families": 4172, + "\u0120ir": 4173, + "\u0120apply": 4174, + "\u0120inform": 4175, + "\u0120Smith": 4176, + "CT": 4177, + "ii": 4178, + "\u0120limit": 4179, + "\u0120resist": 4180, + "................": 4181, + "umn": 4182, + "\u0120conflic": 4183, + "\u0120twe": 4184, + "udd": 4185, + "\u0120Tom": 4186, + "\u0120liter": 4187, + "que": 4188, + "bon": 4189, + "\u0120hair": 4190, + "\u0120eventually": 4191, + "\u0120pus": 4192, + "\u0120helped": 4193, + "\u0120agg": 4194, + "orney": 4195, + "\u0120Apple": 4196, + "\u0120fit": 4197, + "\u0120Sur": 4198, + "\u0120prem": 4199, + "\u0120sales": 4200, + "\u0120seconds": 4201, + "\u0120strength": 4202, + "\u0120feeling": 4203, + "\u00bf\u00bd": 4204, + "\u0120tour": 4205, + "\u0120knows": 4206, + "oom": 4207, + "\u0120exerc": 4208, + "\u0120somew": 4209, + "\u00ef\u00bf\u00bd": 4210, + ">>": 4211, + "\u0120spokes": 4212, + "\u0120ideas": 4213, + "\u0120regist": 4214, + "soft": 4215, + "\u0120Del": 4216, + "\u0120PC": 4217, + "\u0120propos": 4218, + "\u0120launch": 4219, + "\u0120bottom": 4220, + "TH": 4221, + "\u0120Please": 4222, + "vest": 4223, + "itz": 4224, + "\u0120Inter": 4225, + "\u0120script": 4226, + "\u0120rat": 4227, + "arning": 4228, + "\u0120il": 4229, + "\u0120Jer": 4230, + "\u0120Are": 4231, + "\u0120whatever": 4232, + "oken": 4233, + "cience": 4234, + "\u0120mode": 4235, + "\u0120agree": 4236, + "\u0120sources": 4237, + "\u0120initial": 4238, + "\u0120restrict": 4239, + "\u0120wonder": 4240, + "usion": 4241, + "####": 4242, + "\u0120Sil": 4243, + "ville": 4244, + "\u0120burn": 4245, + "tw": 4246, + "asion": 4247, + "\u0120\u00c2\u00a3": 4248, + "\u0120nor": 4249, + "uing": 4250, + "\u0120reached": 4251, + "\u0120sun": 4252, + "\u0120categ": 4253, + "igration": 4254, + "\u0120cook": 4255, + "\u0120promot": 4256, + "\u0120male": 4257, + "\u0120climate": 4258, + "\u0120fix": 4259, + "\u0120alleged": 4260, + "UR": 4261, + "alled": 4262, + "\u0120images": 4263, + "Cont": 4264, + "ota": 4265, + "\u0120schools": 4266, + "ios": 4267, + "\u0120drop": 4268, + "\u0120stream": 4269, + "\u0120Mo": 4270, + "\u0120previously": 4271, + "aling": 4272, + "\u0120pet": 4273, + "\u0120double": 4274, + "\u0120(@": 4275, + "annel": 4276, + "\u0120default": 4277, + "ties": 4278, + "\u0120rank": 4279, + "\u0120Dec": 4280, + "\u0120Council": 4281, + "\u0120weapon": 4282, + "\u0120stock": 4283, + "\u0120analy": 4284, + "\u0120Str": 4285, + "\u0120picture": 4286, + "\u0120Police": 4287, + "ference": 4288, + "\u0120century": 4289, + "\u0120citizens": 4290, + "\u0120onto": 4291, + "\u0120expand": 4292, + "\u0120hero": 4293, + "\u0120Sol": 4294, + "\u0120wild": 4295, + "\u0120update": 4296, + "\u0120customers": 4297, + "ront": 4298, + "def": 4299, + "\u0120lik": 4300, + "\u0120criminal": 4301, + "\u0120Christian": 4302, + "SP": 4303, + "76": 4304, + "\u0120leaving": 4305, + "\u0120otherwise": 4306, + "\u0120Dist": 4307, + "\u0120basis": 4308, + "52": 4309, + "53": 4310, + "icip": 4311, + "\u0120Ber": 4312, + "\u0120recommend": 4313, + "\u0120floor": 4314, + "\u0120crowd": 4315, + "oles": 4316, + "\u012070": 4317, + "\u0120central": 4318, + "\u0120Ev": 4319, + "\u0120dream": 4320, + "\u0120download": 4321, + "\u0120confir": 4322, + "\u0120Thom": 4323, + "\u0120window": 4324, + "\u0120happens": 4325, + "\u0120unit": 4326, + "\u0120tend": 4327, + "\u0120spl": 4328, + "\u0120becomes": 4329, + "\u0120fighting": 4330, + "\u0120predict": 4331, + "\u0120Press": 4332, + "\u0120Power": 4333, + "\u0120heavy": 4334, + "aked": 4335, + "\u0120fan": 4336, + "orter": 4337, + "ategy": 4338, + "BA": 4339, + "izes": 4340, + "\u0120spend": 4341, + "Here": 4342, + "\u01202007": 4343, + "\u0120adop": 4344, + "\u0120Ham": 4345, + "\u0120football": 4346, + "\u0120Port": 4347, + "oday": 4348, + "51": 4349, + "ampions": 4350, + "\u0120transfer": 4351, + "ht": 4352, + "\u012038": 4353, + "term": 4354, + "acity": 4355, + "\u0120bur": 4356, + "],": 4357, + "ternal": 4358, + "rig": 4359, + "but": 4360, + "\u0120therefore": 4361, + "\u0120Because": 4362, + "resp": 4363, + "rey": 4364, + "\u0120mission": 4365, + "Some": 4366, + "\u0120noted": 4367, + "\u0120assum": 4368, + "\u0120disease": 4369, + "\u0120edit": 4370, + "\u0120progress": 4371, + "rd": 4372, + "\u0120Brown": 4373, + "ocal": 4374, + "\u0120adding": 4375, + "\u0120raised": 4376, + "\u0120Any": 4377, + "\u0120tick": 4378, + "\u0120seeing": 4379, + "\u0120People": 4380, + "\u0120agreement": 4381, + "\u0120server": 4382, + "\u0120wat": 4383, + "\u0120debate": 4384, + "\u0120supposed": 4385, + "iling": 4386, + "\u0120largest": 4387, + "\u0120successful": 4388, + "\u0120Pri": 4389, + "\u0120Democratic": 4390, + "\u0120jump": 4391, + "\u0120Syria": 4392, + "\u0120owners": 4393, + "\u0120offers": 4394, + "\u0120shooting": 4395, + "\u0120effic": 4396, + "sey": 4397, + "\u0120haven": 4398, + "verse": 4399, + "tered": 4400, + "\u0120Light": 4401, + "imal": 4402, + "\u0120Big": 4403, + "\u0120defend": 4404, + "\u0120beat": 4405, + "\u0120records": 4406, + "%)": 4407, + "\u0120scen": 4408, + "\u0120employees": 4409, + "\u0120devices": 4410, + "hem": 4411, + "\u0120commer": 4412, + "\u0120Mex": 4413, + "\u0120benefit": 4414, + "\u0120Prof": 4415, + "\u0120illeg": 4416, + "\u0120surface": 4417, + "\u0120Also": 4418, + "\u0120harm": 4419, + "ingly": 4420, + "wide": 4421, + "\u0120Alex": 4422, + "\u0120shut": 4423, + "\u0120Cur": 4424, + "\u0120lose": 4425, + "pm": 4426, + "\u0120challenge": 4427, + "semb": 4428, + "\u0120station": 4429, + "\u0120intelligence": 4430, + "\u0120accur": 4431, + "\u0120Flor": 4432, + "\u0120requires": 4433, + "\u0120Mal": 4434, + "bum": 4435, + "\u0120hospital": 4436, + "\u0120spirit": 4437, + "\u0120offered": 4438, + "\u0120produce": 4439, + "\u0120Commun": 4440, + "\u0120creating": 4441, + "\u0120cris": 4442, + "spect": 4443, + "\u0120ended": 4444, + "\u0120daily": 4445, + "\u0120voters": 4446, + "lands": 4447, + "ias": 4448, + "ih": 4449, + "ona": 4450, + "\u0120smart": 4451, + "\u0120Office": 4452, + "\u0120Lord": 4453, + "rial": 4454, + "\u0120Internet": 4455, + "\u0120circum": 4456, + "\u0120extremely": 4457, + "'.": 4458, + "\u0120opinion": 4459, + "\u0120Mil": 4460, + "\u0120gain": 4461, + "BS": 4462, + "\u0120Fin": 4463, + "yp": 4464, + "\u0120useful": 4465, + "\u0120budget": 4466, + "\u0120comfort": 4467, + "isf": 4468, + "\u0120background": 4469, + "eline": 4470, + "\u0120episode": 4471, + "\u0120enemy": 4472, + "\u0120trial": 4473, + "\u0120establish": 4474, + "date": 4475, + "\u0120Cap": 4476, + "\u0120continues": 4477, + "\u0120showing": 4478, + "\u0120Union": 4479, + "with": 4480, + "\u0120posted": 4481, + "\u0120System": 4482, + "\u0120eat": 4483, + "rian": 4484, + "\u0120rise": 4485, + "\u0120Germany": 4486, + "ils": 4487, + "\u0120signed": 4488, + "\u0120vill": 4489, + "\u0120grand": 4490, + "mor": 4491, + "\u0120England": 4492, + "\u0120projects": 4493, + "umber": 4494, + "\u0120conference": 4495, + "za": 4496, + "\u0120responsible": 4497, + "\u0120Arab": 4498, + "\u0120learned": 4499, + "\u00e2\u0122\u0136\u00e2\u0122\u0136": 4500, + "ipping": 4501, + "\u0120George": 4502, + "OC": 4503, + "\u0120returned": 4504, + "\u0120Australia": 4505, + "\u0120brief": 4506, + "Qu": 4507, + "\u0120brand": 4508, + "illing": 4509, + "abled": 4510, + "\u0120highest": 4511, + "\u0120train": 4512, + "\u0120Commission": 4513, + "while": 4514, + "\u0120nom": 4515, + "ception": 4516, + "\u0120mut": 4517, + "\u0120Blue": 4518, + "\u0120incident": 4519, + "vant": 4520, + "86": 4521, + "\u0120ID": 4522, + "\u0120nuclear": 4523, + "74": 4524, + "\u0120Like": 4525, + "\u0120RE": 4526, + "\u0120Micro": 4527, + "li": 4528, + "mail": 4529, + "\u0120charges": 4530, + "89": 4531, + "\u0120adjust": 4532, + "ado": 4533, + "\u0120earth": 4534, + "NA": 4535, + "\u0120prices": 4536, + "PA": 4537, + "\u0120draft": 4538, + "\u0120runs": 4539, + "\u0120candidate": 4540, + "enses": 4541, + "\u0120management": 4542, + "\u0120Phil": 4543, + "\u0120Miss": 4544, + "\u0120teach": 4545, + "gram": 4546, + "\u0120understanding": 4547, + "ait": 4548, + "icago": 4549, + "Add": 4550, + "\u0120Ep": 4551, + "secut": 4552, + "\u0120separate": 4553, + "\u0120instance": 4554, + "\u0120eth": 4555, + "\u0120unless": 4556, + "********": 4557, + "\u0120Fore": 4558, + "inate": 4559, + "\u0120operations": 4560, + "Sp": 4561, + "\u0120faith": 4562, + "gar": 4563, + "\u0120Church": 4564, + "ronic": 4565, + "\u0120config": 4566, + "osure": 4567, + "\u0120activities": 4568, + "\u0120traditional": 4569, + "\u012036": 4570, + "\u0120direction": 4571, + "\u0120machine": 4572, + "\u0120surround": 4573, + "\u0120push": 4574, + "unction": 4575, + "\u0120EU": 4576, + "\u0120easier": 4577, + "\u0120argument": 4578, + "GB": 4579, + "\u0120micro": 4580, + "\u0120spending": 4581, + "izations": 4582, + "\u0120theory": 4583, + "adow": 4584, + "\u0120calling": 4585, + "\u0120Last": 4586, + "\u0120der": 4587, + "\u0120influence": 4588, + "\u0120commit": 4589, + "\u0120photo": 4590, + "\u0120unc": 4591, + "istry": 4592, + "gn": 4593, + "aste": 4594, + "acks": 4595, + "\u0120disp": 4596, + "ady": 4597, + "do": 4598, + "\u0120Good": 4599, + "\u0120`": 4600, + "\u0120wish": 4601, + "\u0120revealed": 4602, + "\u00c2\u0142\u00c2\u0142": 4603, + "lig": 4604, + "\u0120enforce": 4605, + "\u0120Committee": 4606, + "\u0120chem": 4607, + "\u0120miles": 4608, + "\u0120interested": 4609, + "\u0120solution": 4610, + "icy": 4611, + "inct": 4612, + "\u0120->": 4613, + "\u0120Det": 4614, + "\u0120removed": 4615, + "\u0120compar": 4616, + "eah": 4617, + "\u0120plant": 4618, + "\u0120Since": 4619, + "\u0120achieve": 4620, + "\u0120advantage": 4621, + "\u0120slightly": 4622, + "bing": 4623, + "\u0120placed": 4624, + "under": 4625, + "2015": 4626, + "\u0120Mad": 4627, + "\u0120tim": 4628, + "oses": 4629, + "\u0120cru": 4630, + "\u0120Rock": 4631, + "\u0120mostly": 4632, + "\u0120negative": 4633, + "\u0120setting": 4634, + "\u0120produced": 4635, + "\u0120mur": 4636, + "\u0120connection": 4637, + "\u0120Mer": 4638, + "\u0120driver": 4639, + "\u0120executive": 4640, + "\u0120assault": 4641, + "\u0120born": 4642, + "\u0120Ver": 4643, + "tained": 4644, + "\u0120structure": 4645, + "\u0120reduce": 4646, + "\u0120decades": 4647, + "\u0120ded": 4648, + "uke": 4649, + "\u0120Many": 4650, + "idden": 4651, + "\u0120league": 4652, + "Se": 4653, + "\u0120join": 4654, + "\u0120disco": 4655, + "\u0120die": 4656, + "cks": 4657, + "actions": 4658, + "\u0120assess": 4659, + "agn": 4660, + "\u0120goals": 4661, + "ours": 4662, + "IR": 4663, + "\u0120senior": 4664, + "iller": 4665, + "mod": 4666, + "ipment": 4667, + "ocol": 4668, + "uy": 4669, + "\u0120Que": 4670, + "\u0120parties": 4671, + "irgin": 4672, + "\u0120learning": 4673, + "itable": 4674, + "\u0120street": 4675, + "\u0120camera": 4676, + "App": 4677, + "\u0120skills": 4678, + "bre": 4679, + "cious": 4680, + "\u0120celebr": 4681, + "\u0120Franc": 4682, + "\u0120existing": 4683, + "\u0120willing": 4684, + "lor": 4685, + "\u0120id": 4686, + "\u0120Space": 4687, + "\u0120critical": 4688, + "\u0120La": 4689, + "ortunately": 4690, + "\u0120serve": 4691, + "\u0120cold": 4692, + "\u0120species": 4693, + "TS": 4694, + "\u0120animals": 4695, + "\u0120Bay": 4696, + "\u0120older": 4697, + "\u0120Under": 4698, + "estic": 4699, + "\u0120Tre": 4700, + "\u0120teacher": 4701, + "\u0120prefer": 4702, + "vis": 4703, + "\u0120thread": 4704, + "\u0120Matt": 4705, + "\u0120manager": 4706, + "\u00e3\u0125\u00bb": 4707, + "\u0120professional": 4708, + "\u0120Vol": 4709, + "\u0120notes": 4710, + "These": 4711, + "ula": 4712, + "\u0120fresh": 4713, + "ented": 4714, + "uzz": 4715, + "edy": 4716, + "clusion": 4717, + "\u0120Rel": 4718, + "\u0120doubt": 4719, + "EO": 4720, + "\u0120opened": 4721, + "\u0120Bit": 4722, + "Advertisement": 4723, + "\u0120guess": 4724, + "\u0120UN": 4725, + "\u0120sequ": 4726, + "\u0120explain": 4727, + "otten": 4728, + "\u0120attract": 4729, + "aks": 4730, + "\u0120string": 4731, + "\u0120context": 4732, + "ossible": 4733, + "\u0120Republicans": 4734, + "\u0120solid": 4735, + "\u0120cities": 4736, + "\u0120asking": 4737, + "\u0120random": 4738, + "ups": 4739, + "uries": 4740, + "arant": 4741, + "dden": 4742, + "gl": 4743, + "\u0120Florida": 4744, + "\u0120depend": 4745, + "\u0120Scott": 4746, + "\u012033": 4747, + "\u0120iT": 4748, + "icon": 4749, + "\u0120mentioned": 4750, + "\u01202000": 4751, + "\u0120claimed": 4752, + "\u0120definitely": 4753, + "ulf": 4754, + "\u0120core": 4755, + "\u0120opening": 4756, + "\u0120Const": 4757, + "which": 4758, + "\u0120Tra": 4759, + "AG": 4760, + "72": 4761, + "\u0120believed": 4762, + "ada": 4763, + "\u012048": 4764, + "\u0120Security": 4765, + "yright": 4766, + "\u0120Pet": 4767, + "\u0120Lou": 4768, + "\u0120holding": 4769, + "================": 4770, + "\u0120ice": 4771, + "\u0120brow": 4772, + "\u0120authorities": 4773, + "host": 4774, + "word": 4775, + "\u0120score": 4776, + "\u0120Div": 4777, + "\u0120cells": 4778, + "\u0120transl": 4779, + "\u0120neighbor": 4780, + "\u0120remove": 4781, + "uct": 4782, + "\u0120district": 4783, + "\u0120According": 4784, + "\u0120worse": 4785, + "\u0120concerns": 4786, + "\u0120presidential": 4787, + "\u0120policies": 4788, + "\u0120Hall": 4789, + "73": 4790, + "\u0120hus": 4791, + "AY": 4792, + "\u01202006": 4793, + "\u0120Jud": 4794, + "\u0120independent": 4795, + "\u0120Justice": 4796, + "iliar": 4797, + "print": 4798, + "ighter": 4799, + "\u0120protection": 4800, + "zen": 4801, + "\u0120sudden": 4802, + "house": 4803, + "\u0120Jes": 4804, + "PR": 4805, + "\u0120Inf": 4806, + "\u0120bul": 4807, + "\u0120_": 4808, + "\u0120Service": 4809, + "\u0120PR": 4810, + "\u0120strategy": 4811, + "ffect": 4812, + "\u0120girls": 4813, + "\u0120missing": 4814, + "oyal": 4815, + "\u0120Team": 4816, + "ulated": 4817, + "\u0120dat": 4818, + "\u0120politics": 4819, + "abor": 4820, + "According": 4821, + "\u0120spell": 4822, + "\u0120graph": 4823, + "orthern": 4824, + "TC": 4825, + "Ab": 4826, + "\u0120labor": 4827, + "isher": 4828, + "\u0120kick": 4829, + "\u0120iTunes": 4830, + "\u0120steps": 4831, + "poses": 4832, + "\u0120smaller": 4833, + "En": 4834, + "bert": 4835, + "\u0120roll": 4836, + "\u0120researchers": 4837, + "\u0120closed": 4838, + "\u0120transport": 4839, + "\u0120lawy": 4840, + "________________": 4841, + "\u0120Chicago": 4842, + "\u0120aspect": 4843, + "\u0120none": 4844, + "\u0120marriage": 4845, + "96": 4846, + "\u0120elements": 4847, + "\u0120Fre": 4848, + "\u0120Sal": 4849, + "\u0120dram": 4850, + "FC": 4851, + "top": 4852, + "equ": 4853, + "\u0120hearing": 4854, + "\u0120supported": 4855, + "\u0120testing": 4856, + "cohol": 4857, + "\u0120massive": 4858, + "\u0120stick": 4859, + "\u0120guard": 4860, + "isco": 4861, + "phone": 4862, + "From": 4863, + "However": 4864, + "\u0120border": 4865, + "\u0120copy": 4866, + "ography": 4867, + "list": 4868, + "71": 4869, + "\u0120owner": 4870, + "class": 4871, + "ruit": 4872, + "rate": 4873, + "\u0120Once": 4874, + "\u0120digital": 4875, + "\u0120task": 4876, + "ERS": 4877, + "\u0120incred": 4878, + "tes": 4879, + "++": 4880, + "\u0120France": 4881, + "\u0120breat": 4882, + "owl": 4883, + "\u0120issued": 4884, + "\u0120Western": 4885, + "\u0120detect": 4886, + "\u0120partners": 4887, + "\u0120shared": 4888, + "\u0120Call": 4889, + "\u0120cancer": 4890, + "ache": 4891, + "ribe": 4892, + "\u0120explained": 4893, + "\u0120heat": 4894, + "{\"": 4895, + "\u0120investment": 4896, + "\u0120Book": 4897, + "\u0120wood": 4898, + "\u0120tools": 4899, + "\u0120Although": 4900, + "\u0120belief": 4901, + "\u0120crisis": 4902, + "\u0120ge": 4903, + "\u0120MP": 4904, + "\u0120operation": 4905, + "type": 4906, + "~~": 4907, + "ga": 4908, + "\u0120contains": 4909, + "anta": 4910, + "\u0120express": 4911, + "\u0120Group": 4912, + "\u0120Journal": 4913, + "ka": 4914, + "\u0120amb": 4915, + "\u0120USA": 4916, + "\u0120finding": 4917, + "\u0120funding": 4918, + "how": 4919, + "\u0120established": 4920, + "ideos": 4921, + "\u0120degree": 4922, + "\u0120dangerous": 4923, + "anging": 4924, + "\u0120freedom": 4925, + "pport": 4926, + "outhern": 4927, + "\u0120church": 4928, + "\u0120catch": 4929, + "\u0120Two": 4930, + "\u0120presence": 4931, + "\u0120Guard": 4932, + "Up": 4933, + "\u0120authority": 4934, + "\u0120Project": 4935, + "\u0120button": 4936, + "\u0120consequ": 4937, + "\u0120valid": 4938, + "\u0120weak": 4939, + "\u0120starts": 4940, + "\u0120reference": 4941, + "\u0120Mem": 4942, + "\")": 4943, + "UN": 4944, + "orage": 4945, + "\u0120Open": 4946, + "\u0120collection": 4947, + "ym": 4948, + "gency": 4949, + "\u0120beautiful": 4950, + "ros": 4951, + "\u0120tells": 4952, + "\u0120waiting": 4953, + "nel": 4954, + "\u0120providing": 4955, + "\u0120Democrats": 4956, + "\u0120daughter": 4957, + "\u0120master": 4958, + "\u0120purposes": 4959, + "\u0120Japanese": 4960, + "\u0120equal": 4961, + "\u0120turns": 4962, + "\u0120documents": 4963, + "\u0120watching": 4964, + "Res": 4965, + "\u0120ran": 4966, + "2014": 4967, + "\u0120reject": 4968, + "\u0120Korea": 4969, + "\u0120victims": 4970, + "Level": 4971, + "erences": 4972, + "\u0120witness": 4973, + "\u012034": 4974, + "\u0120reform": 4975, + "coming": 4976, + "\u0120occup": 4977, + "\u0120caught": 4978, + "\u0120traffic": 4979, + "ading": 4980, + "\u0120models": 4981, + "ario": 4982, + "\u0120served": 4983, + "\u0120batter": 4984, + "uate": 4985, + "\u0120Secretary": 4986, + "\u0120agreed": 4987, + "\u0120truly": 4988, + "ynam": 4989, + "\u0120Ret": 4990, + "\u0120units": 4991, + "\u0120Research": 4992, + "hand": 4993, + "azine": 4994, + "\u0120Mike": 4995, + "\u0120variety": 4996, + "otal": 4997, + "\u0120amazing": 4998, + "\u0120confirmed": 4999, + "\u0120entirely": 5000, + "\u0120purchase": 5001, + "\u0120element": 5002, + "\u0120cash": 5003, + "\u0120determine": 5004, + "De": 5005, + "\u0120cars": 5006, + "\u0120Wall": 5007, + "\u00e2\u0138": 5008, + "\u0120views": 5009, + "\u0120drugs": 5010, + "\u0120department": 5011, + "\u0120Step": 5012, + "uit": 5013, + "\u012039": 5014, + "asure": 5015, + "\u0120Class": 5016, + "\u0120covered": 5017, + "\u0120Bank": 5018, + "\u0120mere": 5019, + "uana": 5020, + "\u0120multi": 5021, + "\u0120mix": 5022, + "\u0120unlike": 5023, + "levision": 5024, + "\u0120stopped": 5025, + "\u0120sem": 5026, + "\u0120Gal": 5027, + "ules": 5028, + "\u0120wel": 5029, + "\u0120Johnson": 5030, + "la": 5031, + "\u0120skill": 5032, + "\u0120becoming": 5033, + "rie": 5034, + "\u0120appropriate": 5035, + "fe": 5036, + "ellow": 5037, + "\u0120Prot": 5038, + "ulate": 5039, + "ocation": 5040, + "\u0120weekend": 5041, + "odies": 5042, + "\u0120sites": 5043, + "\u0120animal": 5044, + "\u0120Tim": 5045, + "\u0120scale": 5046, + "\u0120charged": 5047, + "\u0120instruct": 5048, + "illa": 5049, + "\u0120methods": 5050, + "\u0120cert": 5051, + "\u0120judge": 5052, + "\u0120Hel": 5053, + "\u0120dollars": 5054, + "\u0120standing": 5055, + "\u0120Squ": 5056, + "\u0120debt": 5057, + "liam": 5058, + "\u0120driving": 5059, + "\u0120Sum": 5060, + "\u0120Edition": 5061, + "\u0120album": 5062, + "andon": 5063, + "IF": 5064, + "\u0120Uk": 5065, + "63": 5066, + "ader": 5067, + "\u0120commercial": 5068, + "esh": 5069, + "\u0120Government": 5070, + "\u0120discovered": 5071, + "\u0120output": 5072, + "\u0120Hillary": 5073, + "\u0120Carol": 5074, + "\u01202005": 5075, + "\u0120abuse": 5076, + "ancing": 5077, + "\u0120switch": 5078, + "\u0120annual": 5079, + "Tw": 5080, + "\u0120stated": 5081, + "agement": 5082, + "inner": 5083, + "\u0120democr": 5084, + "\u0120residents": 5085, + "\u0120allowing": 5086, + "\u0120factors": 5087, + "odd": 5088, + "\u0120fuck": 5089, + "emies": 5090, + "\u0120occurred": 5091, + "oti": 5092, + "\u0120north": 5093, + "\u0120Public": 5094, + "\u0120injury": 5095, + "\u0120insurance": 5096, + "CL": 5097, + "olly": 5098, + "\u00e3\u0122": 5099, + "\u0120repeated": 5100, + "\u0120arms": 5101, + "anged": 5102, + "\u0120construction": 5103, + "\u0120fle": 5104, + "PU": 5105, + "icians": 5106, + "\u0120forms": 5107, + "\u0120McC": 5108, + "antic": 5109, + "\u0120mental": 5110, + "pire": 5111, + "\u0120equipment": 5112, + "\u0120fant": 5113, + "\u0120discussion": 5114, + "\u0120regarding": 5115, + "kin": 5116, + "arp": 5117, + "\u0120chair": 5118, + "ogue": 5119, + "\u0120proceed": 5120, + "\u0120Id": 5121, + "Our": 5122, + "\u0120murder": 5123, + "Man": 5124, + "\u012049": 5125, + "asp": 5126, + "\u0120supply": 5127, + "\u0120input": 5128, + "\u0120wealth": 5129, + "liament": 5130, + "\u0120proced": 5131, + "orial": 5132, + "\u0120Stat": 5133, + "\u0120NFL": 5134, + "hens": 5135, + "\u0120Institute": 5136, + "\u0120putting": 5137, + "ournament": 5138, + "etic": 5139, + "\u0120located": 5140, + "\u0120kid": 5141, + "eria": 5142, + "run": 5143, + "\u0120princ": 5144, + "\u0120!": 5145, + "going": 5146, + "\u0120Bet": 5147, + "\u0120clot": 5148, + "\u0120telling": 5149, + "\u0120proposed": 5150, + "iot": 5151, + "orry": 5152, + "\u0120funds": 5153, + "gment": 5154, + "\u0120Life": 5155, + "\u0120baby": 5156, + "\u0120Back": 5157, + "\u0120spoke": 5158, + "Image": 5159, + "\u0120earn": 5160, + "\u0120AT": 5161, + "gu": 5162, + "\u0120exchange": 5163, + "\u0120Lin": 5164, + "oving": 5165, + "\u0120pair": 5166, + "More": 5167, + "azon": 5168, + "\u0120arrested": 5169, + "\u0120killing": 5170, + "can": 5171, + "\u0120Card": 5172, + "yd": 5173, + "\u0120identified": 5174, + "\u0120mobile": 5175, + "\u0120thanks": 5176, + "onym": 5177, + "\u0120Form": 5178, + "\u0120hundreds": 5179, + "\u0120Chris": 5180, + "\u0120Cat": 5181, + "\u0120trend": 5182, + "hat": 5183, + "\u0120Av": 5184, + "oman": 5185, + "\u0120electric": 5186, + "\u0120Wil": 5187, + "SE": 5188, + "Of": 5189, + "\u0120restaur": 5190, + "oted": 5191, + "\u0120trig": 5192, + "\u0120nine": 5193, + "\u0120bomb": 5194, + "Why": 5195, + "\u00c2\u00af": 5196, + "\u0120coverage": 5197, + "\u0120appeal": 5198, + "\u0120Robert": 5199, + "\u0120Sup": 5200, + "\u0120finished": 5201, + "\u0120flow": 5202, + "\u0120deliver": 5203, + "\u0120calcul": 5204, + "\u0120photos": 5205, + "\u0120phil": 5206, + "\u0120pieces": 5207, + "\u0120appre": 5208, + "kes": 5209, + "\u0120rough": 5210, + "Do": 5211, + "\u0120partner": 5212, + "\u0120concerned": 5213, + "\u012037": 5214, + "\u0120Gen": 5215, + "Col": 5216, + "ctors": 5217, + "\u0120=>": 5218, + "state": 5219, + "\u0120suggested": 5220, + "\u0120Force": 5221, + "CE": 5222, + "\u0120herself": 5223, + "\u0120Plan": 5224, + "works": 5225, + "ooth": 5226, + "rency": 5227, + "\u0120corner": 5228, + "\u0120husband": 5229, + "\u0120internet": 5230, + "\u0120Aut": 5231, + "ems": 5232, + "osen": 5233, + "\u0120Atl": 5234, + "gen": 5235, + "\u0120balance": 5236, + "62": 5237, + "\u0120sounds": 5238, + "text": 5239, + "\u0120arr": 5240, + "oves": 5241, + "\u0120millions": 5242, + "\u0120radio": 5243, + "\u0120satisf": 5244, + "\u0120Dam": 5245, + "Mr": 5246, + "Go": 5247, + "Spe": 5248, + "\u0120combat": 5249, + "rant": 5250, + "\u0120Gree": 5251, + "\u0120fuel": 5252, + "\u0120distance": 5253, + "\u0120tests": 5254, + "\u0120decre": 5255, + "\u0120Er": 5256, + "\u0120managed": 5257, + "DS": 5258, + "\u0120tit": 5259, + "\u0120measures": 5260, + "\u0120Liber": 5261, + "\u0120attend": 5262, + "ashed": 5263, + "\u0120Jose": 5264, + "\u0120Night": 5265, + "dit": 5266, + "\u0120Nov": 5267, + "\u0120End": 5268, + "outs": 5269, + "\u0120generation": 5270, + "\u0120advoc": 5271, + "yth": 5272, + "\u0120conversation": 5273, + "\u0120Sky": 5274, + "active": 5275, + "cel": 5276, + "rier": 5277, + "\u0120Frank": 5278, + "\u0120gender": 5279, + "\u0120concent": 5280, + "\u0120carried": 5281, + "anda": 5282, + "\u0120Virgin": 5283, + "\u0120arrived": 5284, + "icide": 5285, + "aded": 5286, + "\u0120failure": 5287, + "\u0120minimum": 5288, + "lets": 5289, + "\u0120worst": 5290, + "\u0120keeping": 5291, + "\u0120intended": 5292, + "\u0120illegal": 5293, + "\u0120subsc": 5294, + "\u0120determined": 5295, + "\u0120trip": 5296, + "Yes": 5297, + "\u0120raise": 5298, + "\u0120~": 5299, + "\u0120feels": 5300, + "\u0120package": 5301, + "\u0120Jo": 5302, + "hi": 5303, + "2016": 5304, + "real": 5305, + "\u0120fra": 5306, + "\u0120symb": 5307, + "Me": 5308, + "ucky": 5309, + "pret": 5310, + "\u0120Kh": 5311, + "\u0120Edit": 5312, + "\u0120Web": 5313, + "emic": 5314, + "\u0120Color": 5315, + "\u0120justice": 5316, + "Int": 5317, + "\u0120farm": 5318, + "cknow": 5319, + "\">": 5320, + "eless": 5321, + "\u0120reduced": 5322, + "\u0120500": 5323, + "xx": 5324, + "\u0120Rad": 5325, + "\u0120Wood": 5326, + "\u0120clin": 5327, + "\u0120hyp": 5328, + "iler": 5329, + "ura": 5330, + "kins": 5331, + "85": 5332, + "61": 5333, + "\u0120Their": 5334, + "\u0120Mary": 5335, + "\u0120san": 5336, + "\u0120novel": 5337, + "\u0120Who": 5338, + "\u0120capacity": 5339, + "\u0120impossible": 5340, + "\u0120plays": 5341, + "\u0120minister": 5342, + "ijuana": 5343, + "icate": 5344, + "\u0120Set": 5345, + "\u0120fram": 5346, + "\u0120ing": 5347, + "\u0120communities": 5348, + "\u0120FBI": 5349, + "ita": 5350, + "\u0120bon": 5351, + "\u0120strateg": 5352, + "\u0120interests": 5353, + "lock": 5354, + "gers": 5355, + "mas": 5356, + "\u0120AND": 5357, + "\u0120conflict": 5358, + "\u0120requirements": 5359, + "\u0120sac": 5360, + "\u0120operating": 5361, + "ini": 5362, + "related": 5363, + "\u0120committed": 5364, + "\u0120relatively": 5365, + "\u0120south": 5366, + "\u00c2\u00af\u00c2\u00af": 5367, + "\u0120afford": 5368, + "\u0120identity": 5369, + "\u0120decisions": 5370, + "\u0120accused": 5371, + "place": 5372, + "\u0120victory": 5373, + "och": 5374, + "iat": 5375, + "Name": 5376, + "Com": 5377, + "tion": 5378, + "eds": 5379, + "\u0120seek": 5380, + "\u0120tight": 5381, + "\u0120Images": 5382, + "\u0120initi": 5383, + "\u0120humans": 5384, + "\u0120familiar": 5385, + "\u0120audience": 5386, + "\u0120internal": 5387, + "venture": 5388, + "\u0120sides": 5389, + "\u0120TO": 5390, + "\u0120dim": 5391, + "\u0120conclud": 5392, + "\u0120appoint": 5393, + "\u0120enforcement": 5394, + "\u0120Jim": 5395, + "\u0120Association": 5396, + "\u0120circumst": 5397, + "\u0120Canadian": 5398, + "\u0120joined": 5399, + "\u0120differences": 5400, + "\u0120Los": 5401, + "\u0120protest": 5402, + "\u0120twice": 5403, + "win": 5404, + "\u0120glass": 5405, + "arsh": 5406, + "\u0120Army": 5407, + "\u0120expression": 5408, + "\u0120decide": 5409, + "\u0120planning": 5410, + "ania": 5411, + "\u0120handle": 5412, + "\u0120Microsoft": 5413, + "\u0120Nor": 5414, + "\u0120maximum": 5415, + "\u0120Rev": 5416, + "\u0120sea": 5417, + "\u0120eval": 5418, + "\u0120helps": 5419, + "ref": 5420, + "\u0120bound": 5421, + "\u0120mouth": 5422, + "\u0120standards": 5423, + "\u0120clim": 5424, + "\u0120Camp": 5425, + "\u0120Fox": 5426, + "cles": 5427, + "\u0120army": 5428, + "\u0120Techn": 5429, + "acking": 5430, + "xy": 5431, + "SS": 5432, + "\u012042": 5433, + "\u0120bug": 5434, + "\u0120Ukrain": 5435, + "\u0120Max": 5436, + "\u0120Jones": 5437, + "\u0120Show": 5438, + "lo": 5439, + "\u0120planet": 5440, + "\u012075": 5441, + "\u0120winning": 5442, + "\u0120faster": 5443, + "\u0120spect": 5444, + "\u0120broken": 5445, + "TR": 5446, + "\u0120defined": 5447, + "\u0120healthy": 5448, + "\u0120competition": 5449, + "https": 5450, + "\u0120Island": 5451, + "\u0120Fe": 5452, + "\u0120announce": 5453, + "\u0120Cup": 5454, + "\u0120Instead": 5455, + "\u0120client": 5456, + "\u0120possibly": 5457, + "section": 5458, + "ocket": 5459, + "look": 5460, + "\u0120finish": 5461, + "\u0120crew": 5462, + "\u0120reserv": 5463, + "\u0120editor": 5464, + "\u0120hate": 5465, + "\u0120sale": 5466, + "\u0120controvers": 5467, + "\u0120pages": 5468, + "wing": 5469, + "\u0120numer": 5470, + "\u0120opposition": 5471, + "\u01202004": 5472, + "\u0120refuge": 5473, + "\u0120flight": 5474, + "\u0120apart": 5475, + "\u0120Lat": 5476, + "Americ": 5477, + "\u0120Africa": 5478, + "\u0120applications": 5479, + "\u0120Palest": 5480, + "\u0120Bur": 5481, + "\u0120gar": 5482, + "\u0120Social": 5483, + "\u0120upgr": 5484, + "\u0120shape": 5485, + "\u0120speaking": 5486, + "ansion": 5487, + "ao": 5488, + "\u0120Sn": 5489, + "\u0120worry": 5490, + "\u0120Britain": 5491, + "Please": 5492, + "roud": 5493, + "\u0120hun": 5494, + "\u0120introduced": 5495, + "\u0120diet": 5496, + "Ind": 5497, + "\u0120Second": 5498, + "\u0120functions": 5499, + "uts": 5500, + "\u0120Each": 5501, + "\u0120Jeff": 5502, + "\u0120stress": 5503, + "\u0120accounts": 5504, + "\u0120guarant": 5505, + "\u0120Ann": 5506, + "edia": 5507, + "\u0120honest": 5508, + "\u0120tree": 5509, + "\u0120African": 5510, + "\u0120Bush": 5511, + "},": 5512, + "\u0120sch": 5513, + "\u0120Only": 5514, + "\u0120fif": 5515, + "igan": 5516, + "\u0120exercise": 5517, + "\u0120Exp": 5518, + "\u0120scientists": 5519, + "\u0120legislation": 5520, + "\u0120Work": 5521, + "\u0120Spr": 5522, + "\u00c3\u0124": 5523, + "\u0120Human": 5524, + "\u0120\u00e8": 5525, + "\u0120survey": 5526, + "\u0120rich": 5527, + "rip": 5528, + "\u0120maintain": 5529, + "\u0120flo": 5530, + "\u0120leadership": 5531, + "stream": 5532, + "\u0120Islamic": 5533, + "\u012001": 5534, + "\u0120College": 5535, + "\u0120magic": 5536, + "\u0120Prime": 5537, + "\u0120figures": 5538, + "2017": 5539, + "inder": 5540, + "xual": 5541, + "\u0120Dead": 5542, + "\u0120absolutely": 5543, + "\u0120fourth": 5544, + "\u0120presented": 5545, + "respond": 5546, + "rible": 5547, + "\u0120alcohol": 5548, + "ato": 5549, + "\u0120DE": 5550, + "porary": 5551, + "\u0120grab": 5552, + "\u0120vari": 5553, + "\u0120quant": 5554, + "\u0120Photo": 5555, + "\u0120plus": 5556, + "rick": 5557, + "arks": 5558, + "\u0120alternative": 5559, + "\u0120pil": 5560, + "\u0120approx": 5561, + "that": 5562, + "\u0120objects": 5563, + "\u0120Ro": 5564, + "\u0120Android": 5565, + "\u0120significantly": 5566, + "\u0120Road": 5567, + "kay": 5568, + "Read": 5569, + "avor": 5570, + "\u0120acknow": 5571, + "\u0120HD": 5572, + "\u0120Sing": 5573, + "Or": 5574, + "\u0120Mont": 5575, + "\u0120uns": 5576, + "prof": 5577, + "\u0120negoti": 5578, + "\u0120Arch": 5579, + "iki": 5580, + "\u0120television": 5581, + "\u0120Jewish": 5582, + "\u0120committee": 5583, + "\u0120motor": 5584, + "\u0120appearance": 5585, + "\u0120sitting": 5586, + "\u0120strike": 5587, + "\u0120Down": 5588, + "comp": 5589, + "\u0120Hist": 5590, + "\u0120fold": 5591, + "acement": 5592, + "\u0120Louis": 5593, + "\u0120belong": 5594, + "\u0120\u00e2\u0122\u00a2": 5595, + "\u0120mort": 5596, + "\u0120prepared": 5597, + "\u012064": 5598, + "\u0120Master": 5599, + "\u0120indeed": 5600, + "\u0120Den": 5601, + "\u0120rent": 5602, + "TA": 5603, + "ourney": 5604, + "arc": 5605, + "Su": 5606, + "97": 5607, + "\u0120advice": 5608, + "\u0120changing": 5609, + "\u0120listed": 5610, + "\u0120launched": 5611, + "isation": 5612, + "\u0120Peter": 5613, + "ishes": 5614, + "\u0120lived": 5615, + "\u0120Mel": 5616, + "\u0120Supreme": 5617, + "\u0120Federal": 5618, + "\u0120);": 5619, + "ructure": 5620, + "\u0120sets": 5621, + "\u0120philos": 5622, + "uous": 5623, + "\u0120\u00c2\u0142": 5624, + "\u0120applied": 5625, + "\u0120NOT": 5626, + "\u0120housing": 5627, + "\u0120Mount": 5628, + "\u0120odd": 5629, + "\u0120sust": 5630, + "DA": 5631, + "fficient": 5632, + "\u0120?": 5633, + "olved": 5634, + "\u0120powers": 5635, + "\u0120thr": 5636, + "\u0120remaining": 5637, + "\u0120Water": 5638, + "LC": 5639, + "\u0120causes": 5640, + "\u00e3\u0123\u00ae": 5641, + "\u0120manner": 5642, + "ads": 5643, + "\u0120suggests": 5644, + "\u0120ends": 5645, + "standing": 5646, + "fig": 5647, + "\u0120Dun": 5648, + "idth": 5649, + "\u0120gay": 5650, + "\u0120termin": 5651, + "\u0120Angeles": 5652, + "MS": 5653, + "\u0120scientific": 5654, + "\u0120coal": 5655, + "apers": 5656, + "bar": 5657, + "\u0120Thomas": 5658, + "\u0120sym": 5659, + "\u0120Run": 5660, + "this": 5661, + "PC": 5662, + "igrants": 5663, + "\u0120minute": 5664, + "\u0120District": 5665, + "cellent": 5666, + "\u0120leaves": 5667, + "\u0120completed": 5668, + "amin": 5669, + "\u0120focused": 5670, + "\u0120monitor": 5671, + "\u0120vehicles": 5672, + "MA": 5673, + "\u0120Mass": 5674, + "\u0120Grand": 5675, + "\u0120affected": 5676, + "itutional": 5677, + "\u0120construct": 5678, + "\u0120follows": 5679, + "\u0120ton": 5680, + "reens": 5681, + "\u0120homes": 5682, + "\u0120Ext": 5683, + "\u0120Level": 5684, + "rast": 5685, + "\u0120Ir": 5686, + "\u0120elim": 5687, + "\u0120largely": 5688, + "\u0120Joe": 5689, + "\u0120votes": 5690, + "alls": 5691, + "\u0120businesses": 5692, + "\u0120Foundation": 5693, + "\u0120Central": 5694, + "\u0120yards": 5695, + "\u0120materials": 5696, + "ulner": 5697, + "\u0120guide": 5698, + "\u0120closer": 5699, + "ums": 5700, + "\u0120sports": 5701, + "eder": 5702, + "Just": 5703, + "\u0120taxes": 5704, + "84": 5705, + "\u0120Old": 5706, + "\u0120decade": 5707, + "ola": 5708, + "\u0120vir": 5709, + "\u0120dropped": 5710, + "\u0120delay": 5711, + "itect": 5712, + "\u0120secure": 5713, + "stein": 5714, + "level": 5715, + "\u0120treated": 5716, + "\u0120filed": 5717, + "aine": 5718, + "\u0120van": 5719, + "\u0120mir": 5720, + "\u0120column": 5721, + "icted": 5722, + "eper": 5723, + "\u0120rot": 5724, + "\u0120consult": 5725, + "\u0120entry": 5726, + "\u0120marijuana": 5727, + "\u0120Dou": 5728, + "\u0120apparently": 5729, + "oking": 5730, + "clusive": 5731, + "\u0120increases": 5732, + "ano": 5733, + "\u0120specifically": 5734, + "\u0120tele": 5735, + "ensions": 5736, + "\u0120religion": 5737, + "abilities": 5738, + "\u0120frame": 5739, + "\u0120Note": 5740, + "\u0120Lee": 5741, + "\u0120helping": 5742, + "\u0120edge": 5743, + "oston": 5744, + "\u0120organizations": 5745, + "\u00c3\u0125": 5746, + "\u0120Both": 5747, + "hips": 5748, + "\u0120bigger": 5749, + "\u0120boost": 5750, + "\u0120Stand": 5751, + "\u0120row": 5752, + "uls": 5753, + "abase": 5754, + "\u0120rid": 5755, + "Let": 5756, + "aren": 5757, + "rave": 5758, + "\u0120stret": 5759, + "PD": 5760, + "\u0120vision": 5761, + "\u0120wearing": 5762, + "\u0120appreci": 5763, + "\u0120award": 5764, + "\u0120Use": 5765, + "\u0120factor": 5766, + "war": 5767, + "ulations": 5768, + ")(": 5769, + "\u0120god": 5770, + "\u0120territ": 5771, + "\u0120param": 5772, + "asts": 5773, + "87": 5774, + "\u0120enemies": 5775, + "\u0120Games": 5776, + "FF": 5777, + "\u0120accident": 5778, + "Well": 5779, + "\u0120Martin": 5780, + "TER": 5781, + "\u0120ath": 5782, + "\u0120Hell": 5783, + "\u0120forg": 5784, + "\u0120veter": 5785, + "\u0120Medic": 5786, + "free": 5787, + "\u0120stars": 5788, + "\u0120expensive": 5789, + "\u0120acad": 5790, + "rawn": 5791, + "\u0120Whe": 5792, + "\u0120lock": 5793, + "\u0120format": 5794, + "\u0120soldiers": 5795, + "sm": 5796, + "\u0120agent": 5797, + "\u0120responsibility": 5798, + "ora": 5799, + "\u0120Science": 5800, + "\u0120rapid": 5801, + "\u0120tough": 5802, + "\u0120Jesus": 5803, + "\u0120believes": 5804, + "ML": 5805, + "\u0120wear": 5806, + "lete": 5807, + "\u00c3\u0125\u00c3\u0124": 5808, + "\u0120Dri": 5809, + "\u0120commission": 5810, + "\u0120Bob": 5811, + "Oh": 5812, + "aped": 5813, + "\u0120warm": 5814, + "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 5815, + "\u01202003": 5816, + "ortion": 5817, + "\u0120hasn": 5818, + "uster": 5819, + "\u0120univers": 5820, + "\u0120Ill": 5821, + "\u0120king": 5822, + "ologies": 5823, + "94": 5824, + "\u0120Tem": 5825, + "\u0120Mos": 5826, + "\u0120patient": 5827, + "\u0120Mexico": 5828, + "cean": 5829, + "\u0120Death": 5830, + "\u0120Sanders": 5831, + "you": 5832, + "\u0120Cast": 5833, + "\u0120Company": 5834, + "pty": 5835, + "\u0120happening": 5836, + "FP": 5837, + "\u0120Battle": 5838, + "\u0120bought": 5839, + "Am": 5840, + "Mod": 5841, + "Us": 5842, + "uters": 5843, + "\u0120Cre": 5844, + "\u0120Those": 5845, + "\u012044": 5846, + "iser": 5847, + "\u0120soul": 5848, + "\u0120Top": 5849, + "\u0120Harry": 5850, + "\u0120Aw": 5851, + "\u0120seat": 5852, + "ffee": 5853, + "\u0120revolution": 5854, + "\u0120(\"": 5855, + "\u0120During": 5856, + "ette": 5857, + "\u0120ring": 5858, + "\u0120offensive": 5859, + "\u0120returns": 5860, + "\u0120videos": 5861, + "\u0120discl": 5862, + "\u0120famous": 5863, + "enced": 5864, + "\u0120Sign": 5865, + "\u0120River": 5866, + "\u0120300": 5867, + "PM": 5868, + "\u0120Bus": 5869, + "\u0120CH": 5870, + "\u0120candidates": 5871, + "arden": 5872, + "\u0120percentage": 5873, + "\u0120visual": 5874, + "\u0120thank": 5875, + "\u0120trouble": 5876, + "nergy": 5877, + "\u01202001": 5878, + "\u0120prove": 5879, + "ashion": 5880, + "\u0120enh": 5881, + "\u0120Long": 5882, + "UM": 5883, + "\u0120connected": 5884, + "\u0120possibility": 5885, + "Over": 5886, + "\u0120expert": 5887, + "\u0120library": 5888, + "arts": 5889, + "\u0120Director": 5890, + "\u0120fellow": 5891, + "92": 5892, + "irty": 5893, + "\u0120dry": 5894, + "\u0120signs": 5895, + "\u0120Love": 5896, + "\u0120quiet": 5897, + "foot": 5898, + "\u0120pure": 5899, + "\u0120Hun": 5900, + "\u0120filled": 5901, + "phas": 5902, + "\u0120Elect": 5903, + "endment": 5904, + "\u0120Expl": 5905, + "\u0120unable": 5906, + "ns": 5907, + "mo": 5908, + "\u0120vast": 5909, + "obe": 5910, + "\u0120identify": 5911, + "apping": 5912, + "\u0120Carolina": 5913, + "gress": 5914, + "\u0120prote": 5915, + "\u0120fish": 5916, + "\u0120circumstances": 5917, + "razy": 5918, + "\u0120Phot": 5919, + "\u0120bodies": 5920, + "\u0120Mur": 5921, + "\u0120developing": 5922, + "\u0120AR": 5923, + "\u0120experienced": 5924, + "\u0120substant": 5925, + "\u0120Board": 5926, + "esome": 5927, + "\u0120domestic": 5928, + "\u0120combined": 5929, + "\u0120Put": 5930, + "\u0120chemical": 5931, + "\u0120Child": 5932, + "\u0120pool": 5933, + "\u0120Cy": 5934, + "\u0120egg": 5935, + "cons": 5936, + "sters": 5937, + "\u0120hurt": 5938, + "\u0120markets": 5939, + "\u0120conservative": 5940, + "\u0120supporters": 5941, + "\u0120agencies": 5942, + "idel": 5943, + "Ob": 5944, + "urb": 5945, + "\u012043": 5946, + "\u0120Defense": 5947, + "ye": 5948, + "\u0120Ap": 5949, + "dule": 5950, + "\u0120temperature": 5951, + "\u0120conducted": 5952, + "\u0120Chief": 5953, + "\u0120pulled": 5954, + "\u0120fol": 5955, + "Last": 5956, + "onto": 5957, + "osis": 5958, + "VER": 5959, + "Des": 5960, + "\u0120Pan": 5961, + "First": 5962, + "\u0120advance": 5963, + "\u0120license": 5964, + "rors": 5965, + "\u0120Jon": 5966, + "\u0120imagine": 5967, + "\u0120hell": 5968, + "\u0120fixed": 5969, + "\u0120incor": 5970, + "osite": 5971, + "\u0120Log": 5972, + "icken": 5973, + "]:": 5974, + "\u0120surprise": 5975, + "hab": 5976, + "\u0120craft": 5977, + "olt": 5978, + "\u0120Jul": 5979, + "\u0120dial": 5980, + "\u0120relevant": 5981, + "\u0120entered": 5982, + "\u0120leads": 5983, + "\u0120AD": 5984, + "\u0120Clean": 5985, + "\u0120pictures": 5986, + "essor": 5987, + "\u0120alt": 5988, + "\u0120paying": 5989, + "Per": 5990, + "\u0120Market": 5991, + "\u0120updates": 5992, + "amily": 5993, + "\u0120Type": 5994, + "\u0120Home": 5995, + "\u012055": 5996, + "sembly": 5997, + "rome": 5998, + "83": 5999, + "\u0120greatest": 6000, + "\u0120height": 6001, + "\u0120heav": 6002, + "aints": 6003, + "\u0120listen": 6004, + "aser": 6005, + "\u0120SH": 6006, + "\u0120capable": 6007, + "acle": 6008, + "\u0120perspect": 6009, + "inating": 6010, + "\u0120offering": 6011, + "rypt": 6012, + "\u0120Develop": 6013, + "abin": 6014, + "rc": 6015, + "\u0120bright": 6016, + "alty": 6017, + "arrow": 6018, + "\u0120suppl": 6019, + "inding": 6020, + "acked": 6021, + "gypt": 6022, + "\u0120Another": 6023, + "pg": 6024, + "\u0120Virginia": 6025, + "\u0120Lu": 6026, + "\u0120planned": 6027, + "\u0120pit": 6028, + "\u0120sweet": 6029, + "Type": 6030, + "\u0120Di": 6031, + "\u0120typically": 6032, + "\u0120Francisco": 6033, + "\u0120prospect": 6034, + "\u0120Dan": 6035, + "\u0120teen": 6036, + "rees": 6037, + "\u0120sched": 6038, + "\u0120hol": 6039, + "\u0120scr": 6040, + "\u0120lots": 6041, + "life": 6042, + "\u0120newsp": 6043, + "\u0120forget": 6044, + "\u0120None": 6045, + "\u0120Middle": 6046, + "\u0120Ryan": 6047, + "edd": 6048, + "\u0120severe": 6049, + "\u0120suit": 6050, + "ller": 6051, + "93": 6052, + "\u0120correspond": 6053, + "\u0120explos": 6054, + "uations": 6055, + "\u0120flag": 6056, + "game": 6057, + "rid": 6058, + "\u0120prin": 6059, + "\u0120Data": 6060, + "\u0120deploy": 6061, + "\u0120Enter": 6062, + "suit": 6063, + "ghan": 6064, + "\u0120Men": 6065, + "\u0120thoughts": 6066, + "\u0120matters": 6067, + "\u0120adapt": 6068, + "\u0120Ari": 6069, + "\u0120fill": 6070, + "\u0120forth": 6071, + "\u0120sam": 6072, + "\u012041": 6073, + "\u0120payment": 6074, + "\u0120Hor": 6075, + "\u0120spring": 6076, + "duc": 6077, + "\u0120losing": 6078, + "\u0120bringing": 6079, + "FO": 6080, + "ala": 6081, + "\u0120distribution": 6082, + "hered": 6083, + "bour": 6084, + "\u0120Israeli": 6085, + "oma": 6086, + "\u0120combination": 6087, + "\u0120plenty": 6088, + "VE": 6089, + "Can": 6090, + "\u0120Haw": 6091, + "\u0120perman": 6092, + "\u0120Special": 6093, + "\u0120tow": 6094, + "\u0120seeking": 6095, + "\u0120examples": 6096, + "\u0120classes": 6097, + "cr": 6098, + "\u0120beer": 6099, + "\u0120moves": 6100, + "\u0120IP": 6101, + "\u0120Kn": 6102, + "\u0120panel": 6103, + "Even": 6104, + "\u0120properly": 6105, + "\u0120ris": 6106, + "\u0120plug": 6107, + "\u0120estimated": 6108, + "Every": 6109, + "\u0120defensive": 6110, + "agraph": 6111, + "\u0120pregn": 6112, + "\u0120instit": 6113, + "\u0120Vict": 6114, + "\u0120volume": 6115, + "\u0120positions": 6116, + "\u0120links": 6117, + "\u0120Program": 6118, + "\u0120Week": 6119, + "agues": 6120, + "\u0120transform": 6121, + "ker": 6122, + "\u0120CEO": 6123, + "\u0120cas": 6124, + "\u0120opponent": 6125, + "\u0120tweet": 6126, + "\u0120Code": 6127, + "\u0120shop": 6128, + "\u0120fly": 6129, + "\u0120talks": 6130, + "\u0120bag": 6131, + "Phone": 6132, + "\u0120aid": 6133, + "\u0120plants": 6134, + "\u012065": 6135, + "\u0120attorney": 6136, + "arters": 6137, + "quest": 6138, + "\u0120Magic": 6139, + "\u0120begins": 6140, + "\u0120myster": 6141, + "\u0120environmental": 6142, + "\u0120storage": 6143, + "NN": 6144, + "\u0120marg": 6145, + "\u0120ske": 6146, + "\u0120metal": 6147, + "elly": 6148, + "\u0120ordered": 6149, + "\u0120remained": 6150, + "\u0120loved": 6151, + "\u0120prompt": 6152, + "\u0120updated": 6153, + "\u0120experts": 6154, + "\u0120walking": 6155, + "\u0120ancient": 6156, + "\u0120performed": 6157, + "ATE": 6158, + "\u0120neither": 6159, + "iency": 6160, + "\u0120manufacture": 6161, + "\u0120Pak": 6162, + "\u0120selected": 6163, + "\u0120mine": 6164, + "\u0120ultimately": 6165, + "\u0120explan": 6166, + "\u0120label": 6167, + "\u0120Services": 6168, + "ributed": 6169, + "Trump": 6170, + "\u0120syn": 6171, + "\u0120Ult": 6172, + "SC": 6173, + "\u0120meat": 6174, + "\u0120giant": 6175, + "\u0120Wars": 6176, + "\u0120ON": 6177, + "\u0120adm": 6178, + "\u0120interpret": 6179, + "\u0120evening": 6180, + "\u0120evil": 6181, + "\u0120Boston": 6182, + "\u0120Wild": 6183, + "\u0120\u00c3": 6184, + "\u0120Bitcoin": 6185, + "\u0120Amazon": 6186, + "Dr": 6187, + "\u0120Information": 6188, + "\u0120obviously": 6189, + "\u0120advanced": 6190, + "Photo": 6191, + "olar": 6192, + "\u0120weather": 6193, + "\u0120symbol": 6194, + "\u0120sole": 6195, + "\u0120potentially": 6196, + "oster": 6197, + "\u0120originally": 6198, + "mun": 6199, + "300": 6200, + "aze": 6201, + "essions": 6202, + "\u0120deck": 6203, + "\u0120stood": 6204, + "\u0120youth": 6205, + "\u0120Bern": 6206, + "Rep": 6207, + "\u0120Test": 6208, + "\u0120basically": 6209, + "otic": 6210, + "\u0120involve": 6211, + "olit": 6212, + "lyn": 6213, + "See": 6214, + "\u0120aircraft": 6215, + "\u0120confirm": 6216, + "EW": 6217, + "\u0120messages": 6218, + "\u0120Richard": 6219, + "\u0120kit": 6220, + "\u0120prohib": 6221, + "\u0120vulner": 6222, + "isters": 6223, + "\u0120existence": 6224, + "\u0120turning": 6225, + "\u0120SP": 6226, + "\u0120desire": 6227, + "\u0120flat": 6228, + "\u0120ment": 6229, + "season": 6230, + "anges": 6231, + "\u0120neighborhood": 6232, + "\u0120Lake": 6233, + "ATION": 6234, + "\u0120pointed": 6235, + "bur": 6236, + "\u0120innov": 6237, + "ucks": 6238, + "UL": 6239, + "\u0120professor": 6240, + "\u0120expressed": 6241, + "AB": 6242, + "icious": 6243, + "\u01202002": 6244, + "\u0120Dev": 6245, + "\u0120session": 6246, + "\u0120bare": 6247, + "sen": 6248, + "\u0120diss": 6249, + "\u0120Cath": 6250, + "\u0120Pass": 6251, + "\u0120Point": 6252, + "\u0120doctor": 6253, + "orrow": 6254, + "ailed": 6255, + "\u0120Rub": 6256, + "\u0120DC": 6257, + "\u0120Charl": 6258, + "person": 6259, + "\u0120writer": 6260, + "ighters": 6261, + "ureau": 6262, + "\u0120oblig": 6263, + "\u0120recorded": 6264, + "\u0120broke": 6265, + "\u0120orders": 6266, + "ilty": 6267, + "\u0120motion": 6268, + "inity": 6269, + "law": 6270, + "adium": 6271, + "\u0120immigration": 6272, + "\u0120contrast": 6273, + "\u0120batt": 6274, + "\u0120excellent": 6275, + "\u0120technical": 6276, + "ami": 6277, + "\u0120tun": 6278, + "\u0120cloud": 6279, + "\u0120Year": 6280, + "geon": 6281, + "\u0120creation": 6282, + "\u0120strange": 6283, + "\u0120auth": 6284, + "\u0120fort": 6285, + "born": 6286, + "\u0120extent": 6287, + "\u0120Today": 6288, + "\u0120Club": 6289, + "\u0120rain": 6290, + "\u0120sample": 6291, + "\u0120accepted": 6292, + "\u0120tact": 6293, + "\u0120fired": 6294, + "\u0120Son": 6295, + "\u0120stands": 6296, + "\u0120boot": 6297, + "\u012047": 6298, + "\u0120statements": 6299, + "\u0120versions": 6300, + "\u0120selling": 6301, + "ounded": 6302, + "\u01201990": 6303, + "\u0120weren": 6304, + "\u0120Watch": 6305, + "\u0120experiment": 6306, + "Post": 6307, + "\u0120retail": 6308, + "uled": 6309, + "Inst": 6310, + "unte": 6311, + "\u00e3\u0125\u00bc": 6312, + "\u0120depart": 6313, + "\u0120bond": 6314, + "ivery": 6315, + "ompl": 6316, + "\u0120reaction": 6317, + "\u0120Syrian": 6318, + "\u0120Pac": 6319, + "apped": 6320, + "aniel": 6321, + "DP": 6322, + "\u0120resolution": 6323, + "\u0120react": 6324, + "\u0120approved": 6325, + "onom": 6326, + "mond": 6327, + "\u0120Offic": 6328, + "---": 6329, + "\u0120replace": 6330, + "\u0120tack": 6331, + "\u0120sport": 6332, + "\u0120chain": 6333, + "\u0120emergency": 6334, + "rad": 6335, + "\u0120Palestin": 6336, + "\u012046": 6337, + "\u0120automatically": 6338, + "\u0120route": 6339, + "\u0120pal": 6340, + "\u0120banks": 6341, + "\u0120Paris": 6342, + "\u0120Media": 6343, + "road": 6344, + "icing": 6345, + "ixt": 6346, + "isted": 6347, + "\u0120grew": 6348, + "\u0120coord": 6349, + "\u0120Where": 6350, + "omin": 6351, + "\u0120subs": 6352, + "\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 6353, + "\u0120\u00c2\u00b1": 6354, + "\u0120corporate": 6355, + "\u0120selection": 6356, + "noon": 6357, + "\u0120Report": 6358, + "cs": 6359, + "cluding": 6360, + "orders": 6361, + "anche": 6362, + "\u0120Its": 6363, + "\u0120slowly": 6364, + "\u0120Egypt": 6365, + "\u0120Acc": 6366, + "\u0120colle": 6367, + "iques": 6368, + "EX": 6369, + "\u0120attempts": 6370, + "url": 6371, + "\u0120Cross": 6372, + "\u0120findings": 6373, + "\u0120SC": 6374, + "\u0120OR": 6375, + "\u0120index": 6376, + "ensity": 6377, + "\u0120Way": 6378, + "\u0120Land": 6379, + "\u0120shock": 6380, + "dis": 6381, + "\u0120dynam": 6382, + "\u0120cart": 6383, + "mosp": 6384, + "Since": 6385, + "iest": 6386, + "\u0120Boy": 6387, + "\u0120storm": 6388, + "\u0120Contin": 6389, + "2013": 6390, + "hew": 6391, + "ilit": 6392, + "\u0120essential": 6393, + "iquid": 6394, + "Other": 6395, + "ivered": 6396, + "\u0120reasonable": 6397, + "Act": 6398, + "\u0120subsequ": 6399, + "\u0120Pack": 6400, + "\u0120Fort": 6401, + "\u0120considering": 6402, + "\u0120university": 6403, + "log": 6404, + "\u0120married": 6405, + "\u0120illust": 6406, + "\u0120True": 6407, + "\u00a3\u0131": 6408, + "\u0120numerous": 6409, + "rastructure": 6410, + "\u0120seriously": 6411, + "\u0120referred": 6412, + "ua": 6413, + "\u0120consistent": 6414, + "onna": 6415, + "\u0120Real": 6416, + "ruption": 6417, + "ciples": 6418, + "\u0120facts": 6419, + "91": 6420, + "otes": 6421, + "erg": 6422, + "Then": 6423, + "\u0120accompl": 6424, + "Note": 6425, + "\u0120revenue": 6426, + "\u0120passing": 6427, + "\u0120mal": 6428, + "een": 6429, + "\u0120Yet": 6430, + "\u0120gather": 6431, + "terday": 6432, + "ework": 6433, + "\u0120Author": 6434, + "Pe": 6435, + "\u0120optim": 6436, + "\u0120rub": 6437, + "\u0120\u00e8\u00a3\u0131": 6438, + "\u0120unknown": 6439, + "stone": 6440, + "\u0120union": 6441, + "olve": 6442, + "\u0120opportunities": 6443, + "\u0120browser": 6444, + "\u0120Wal": 6445, + "\u0120Cost": 6446, + "\u0120reporting": 6447, + "sts": 6448, + "pet": 6449, + "\u0120sand": 6450, + "\u0120suddenly": 6451, + "\u0120surprising": 6452, + "\u0120VR": 6453, + "\u0120somewhat": 6454, + "\u0120Bas": 6455, + "ulture": 6456, + "izz": 6457, + "\u0120CD": 6458, + "\u0120challenges": 6459, + "\u0120settings": 6460, + "\u0120experiences": 6461, + "\u0120Full": 6462, + "\u0120cann": 6463, + "\u0120receiving": 6464, + "EST": 6465, + "\u0120joint": 6466, + "\u0120cultural": 6467, + "\u0120ast": 6468, + "82": 6469, + "astern": 6470, + "ceived": 6471, + "\u0120Cru": 6472, + "\u0120bull": 6473, + "pired": 6474, + "amm": 6475, + "\u0120facing": 6476, + "power": 6477, + "\u0120boss": 6478, + "\u0120Hol": 6479, + "\u0120instr": 6480, + "\u0120increasingly": 6481, + "\u0120shift": 6482, + "\u0120streets": 6483, + "\u0120Williams": 6484, + "abb": 6485, + "\u0120lie": 6486, + "\u0120laugh": 6487, + "\u0120Ca": 6488, + "PL": 6489, + "\u0120adults": 6490, + "\u0120customer": 6491, + "\u0120obtained": 6492, + "\u0120supporting": 6493, + "html": 6494, + "fire": 6495, + "\u0120detailed": 6496, + "\u0120picked": 6497, + "\u0120Right": 6498, + "lder": 6499, + "EE": 6500, + "stood": 6501, + "\u0120Kim": 6502, + "\u0120wire": 6503, + "\u0120sight": 6504, + "\u0120developers": 6505, + "\u0120persons": 6506, + "\u0120sad": 6507, + "\u0120cup": 6508, + "\u0120warning": 6509, + "\u0120boys": 6510, + "long": 6511, + "\u0120bird": 6512, + "fo": 6513, + "\u0120wal": 6514, + "\u0120observed": 6515, + "\u0120zone": 6516, + "iveness": 6517, + "\u0120channel": 6518, + "cript": 6519, + "\u0120refused": 6520, + "\u0120Again": 6521, + "\u0120suc": 6522, + "\u0120spokesman": 6523, + "\u0120Ref": 6524, + "rite": 6525, + "ouston": 6526, + "\u00e3\u0125\u00b3": 6527, + "\u0120Sher": 6528, + "\u0120acts": 6529, + "\u0120Name": 6530, + "\u0120struggle": 6531, + "arry": 6532, + "ometimes": 6533, + "\u0120discrim": 6534, + "HT": 6535, + "\u0120category": 6536, + "\u0120realize": 6537, + "\u0120employee": 6538, + "\u0120Afghan": 6539, + "enger": 6540, + "\u0120guns": 6541, + "\u0120Steve": 6542, + "\u0120Mot": 6543, + "\u0120Ol": 6544, + "oked": 6545, + "\u0120thick": 6546, + "\u0120fairly": 6547, + "illy": 6548, + "\u0120surve": 6549, + "\u0120Mat": 6550, + "weight": 6551, + "\u00e2\u0136": 6552, + "\u0120troops": 6553, + "\u0120agents": 6554, + "\u0120battery": 6555, + "\u0120motiv": 6556, + "\u00c3\u00a1": 6557, + "Sec": 6558, + "den": 6559, + "overy": 6560, + "LS": 6561, + "\u0120flu": 6562, + "\u0120confident": 6563, + "\u0120Oper": 6564, + "\u0120empty": 6565, + "\u0120phen": 6566, + "\u0120sector": 6567, + "\u0120excited": 6568, + "\u0120remote": 6569, + "aph": 6570, + "oen": 6571, + "\u0120destroyed": 6572, + "\u0120moral": 6573, + "\u0120HP": 6574, + "\u0120Ron": 6575, + "\u0120dress": 6576, + "\u0120Bat": 6577, + "\u0120lit": 6578, + "\u0120MS": 6579, + "\u0120af": 6580, + "HL": 6581, + "rum": 6582, + "isms": 6583, + "\u0120shouldn": 6584, + "\u0120sympt": 6585, + "\u0120Toronto": 6586, + "hetic": 6587, + "\u0120carbon": 6588, + "\u0120installed": 6589, + "\u0120violent": 6590, + "\u0120solar": 6591, + "ja": 6592, + "\u0120practices": 6593, + "\u0120ride": 6594, + "\u0120Penn": 6595, + "\u0120improved": 6596, + "\u0120audio": 6597, + "\u0120behavi": 6598, + "\u0120PS": 6599, + "\u0120eating": 6600, + "Data": 6601, + "\u0120Review": 6602, + "pass": 6603, + "claim": 6604, + "uated": 6605, + "angers": 6606, + "chen": 6607, + "\u0120properties": 6608, + "\u0120anywhere": 6609, + "Another": 6610, + "\u0120blow": 6611, + "\u0120Jackson": 6612, + "\u0120proud": 6613, + "\u0120plane": 6614, + "lines": 6615, + "\u0120square": 6616, + "\u0120proof": 6617, + "ansas": 6618, + "\u0120talked": 6619, + "makers": 6620, + "\u0120sister": 6621, + "\u0120holds": 6622, + "\u0120resident": 6623, + "\u0120==": 6624, + "\u0120resistance": 6625, + "\u0120split": 6626, + "\u0120prosecut": 6627, + "\u0120confidence": 6628, + "resents": 6629, + "\u0120cuts": 6630, + "\u0120exception": 6631, + "\u0120zero": 6632, + "Getty": 6633, + "\u0120copyright": 6634, + "\u0120totally": 6635, + "ormal": 6636, + "ifications": 6637, + "\u0120Australian": 6638, + "\u0120sick": 6639, + "\u0120150": 6640, + "\u0120household": 6641, + "\u0120fees": 6642, + "\u0120drivers": 6643, + "ogen": 6644, + "\u0120NY": 6645, + "\u0120necessarily": 6646, + "\u0120regulations": 6647, + "earing": 6648, + "sl": 6649, + "\u0120perspective": 6650, + "care": 6651, + "icial": 6652, + "His": 6653, + "\u0120escape": 6654, + "\u0120surprised": 6655, + "\u0120Van": 6656, + "urrent": 6657, + "\u0120vac": 6658, + "81": 6659, + "\u0120Thus": 6660, + "\u0120emphas": 6661, + "\u0120Champions": 6662, + "\u0120Ice": 6663, + "\u0120narr": 6664, + "\u0120heads": 6665, + "\u0120causing": 6666, + "bel": 6667, + "fortunately": 6668, + "\u0120Ma": 6669, + "\u0120targets": 6670, + "cipl": 6671, + "\u0120afternoon": 6672, + "\u0120adds": 6673, + "\u0120Maybe": 6674, + "\u0120Four": 6675, + "essed": 6676, + "plete": 6677, + "\u0120usual": 6678, + "cho": 6679, + "ingu": 6680, + "\u0120withd": 6681, + "\u0120Energy": 6682, + "\u0120Econom": 6683, + "OO": 6684, + "\u0120articles": 6685, + "\u0120injured": 6686, + "\u0120manage": 6687, + "\u0120explains": 6688, + "\u0120diagn": 6689, + "Rec": 6690, + "atures": 6691, + "\u0120linked": 6692, + "\u0120discussed": 6693, + "\u0120explo": 6694, + "\u0120occasion": 6695, + "athan": 6696, + "\u0120opposite": 6697, + "\u0120faces": 6698, + "\u0120denied": 6699, + "\u0120Knight": 6700, + "\u0120nut": 6701, + "\u0120approximately": 6702, + "\u0120disappoint": 6703, + "onymous": 6704, + "\u0120Best": 6705, + "\u0120Lo": 6706, + "\u0120Hy": 6707, + "\u0120Aff": 6708, + "\u0120voting": 6709, + "anwhile": 6710, + "\u0120III": 6711, + "\u0120institutions": 6712, + "agram": 6713, + "\u0120Daily": 6714, + "\u0120drag": 6715, + "\u0120nearby": 6716, + "\u0120guilty": 6717, + "\u0120conver": 6718, + "Pre": 6719, + "ship": 6720, + "\u0120reward": 6721, + "\u0120philosoph": 6722, + "\u0120SS": 6723, + "ugh": 6724, + "\u0120apps": 6725, + "friend": 6726, + "\u0120upper": 6727, + "\u0120advert": 6728, + "\u0120snow": 6729, + "\u0120frust": 6730, + "\u0120ourselves": 6731, + "Fr": 6732, + "\u0120Die": 6733, + "ampion": 6734, + "\u0120dismiss": 6735, + "\u0120cere": 6736, + "\u0120signal": 6737, + "from": 6738, + "\u0120).": 6739, + "\u012052": 6740, + "\u0120crimes": 6741, + "itors": 6742, + "estival": 6743, + "useum": 6744, + "\u0120council": 6745, + "\u0120Saud": 6746, + "May": 6747, + "\u0120Gun": 6748, + "ician": 6749, + "ether": 6750, + "\u0120sufficient": 6751, + "\u0120Hen": 6752, + "sole": 6753, + "\u0120historical": 6754, + "\u0120Far": 6755, + "\u0120Turn": 6756, + "\u0120pin": 6757, + "\u0120succeed": 6758, + "mat": 6759, + "lymp": 6760, + "\u0120tradition": 6761, + "\u0120Ok": 6762, + "\u0120cro": 6763, + "\u0120description": 6764, + "alle": 6765, + "\u0120sky": 6766, + "Te": 6767, + "\u0120widely": 6768, + "\u0120wave": 6769, + "\u0120definition": 6770, + "\u0120Jews": 6771, + "\u0120cycle": 6772, + "\u0120refere": 6773, + "\u0120brings": 6774, + "usal": 6775, + "\u0120alive": 6776, + "\u0120frequently": 6777, + "\u0120intention": 6778, + "\u0120Control": 6779, + "lv": 6780, + "ystem": 6781, + "\u0120privacy": 6782, + "gent": 6783, + "rence": 6784, + "\u0120Quest": 6785, + "\u0120Christmas": 6786, + "\u0120rail": 6787, + "\u0120cooper": 6788, + "\u0120tested": 6789, + "\u0120Capt": 6790, + "asks": 6791, + "\u0120comfortable": 6792, + "\u0120delivered": 6793, + "scape": 6794, + "\u0120depth": 6795, + "\u0120GOP": 6796, + "\u0120writes": 6797, + "\u0120assets": 6798, + "\u0120sav": 6799, + "iments": 6800, + "\u0120transition": 6801, + "\u0120artist": 6802, + "\u0120Look": 6803, + "\u0120lob": 6804, + "\u0120components": 6805, + "arity": 6806, + "\u0120walked": 6807, + "\u0120root": 6808, + "\u0120participants": 6809, + "\u0120noticed": 6810, + "\u0120resc": 6811, + "\u0120nav": 6812, + "\u0120Administ": 6813, + "da": 6814, + "utral": 6815, + "plate": 6816, + "\u0120importance": 6817, + "\u0120assert": 6818, + "iously": 6819, + "cription": 6820, + "\u0120injuries": 6821, + "\u0120Check": 6822, + "\u0120registered": 6823, + "\u0120intent": 6824, + "\u0120missed": 6825, + "ographic": 6826, + "\u0120sentence": 6827, + "ounter": 6828, + "\u0120assistance": 6829, + "evin": 6830, + "\u0120database": 6831, + "\u0120buildings": 6832, + "\u0120classic": 6833, + "\u0120thinks": 6834, + "\u0120Ohio": 6835, + "Pr": 6836, + "ugg": 6837, + "\u0120fee": 6838, + "pan": 6839, + "\u0120effectively": 6840, + "\u0120facility": 6841, + "\u0120bear": 6842, + "\u0120chapter": 6843, + "\u0120dogs": 6844, + "\u0120Columb": 6845, + "\u0120latter": 6846, + "itial": 6847, + "\u0120admitted": 6848, + "TV": 6849, + "\u0120Georg": 6850, + "\u0120posts": 6851, + "\\\\": 6852, + "\u0120lawyer": 6853, + "\u0120equival": 6854, + "\u0120mand": 6855, + "\u0120controlled": 6856, + "\u0120Walk": 6857, + "\u0120Andrew": 6858, + "\u0120menu": 6859, + "amental": 6860, + "\u0120protected": 6861, + "va": 6862, + "\u0120administr": 6863, + "oral": 6864, + "\u0120rein": 6865, + "\u0120Sar": 6866, + "\u0120amounts": 6867, + "\u0120native": 6868, + "\u0120Moon": 6869, + "\u0120represents": 6870, + "\u0120abandon": 6871, + "\u0120carrying": 6872, + "\u0120tank": 6873, + "mary": 6874, + "\u0120declared": 6875, + "Tube": 6876, + "\u0120hat": 6877, + "\u0120punish": 6878, + "ellect": 6879, + "mes": 6880, + "\u0120universe": 6881, + "\u0120Rod": 6882, + "phy": 6883, + "\u0120infrastructure": 6884, + "\u012051": 6885, + "\u0120opposed": 6886, + "ownt": 6887, + "ca": 6888, + "\u0120Make": 6889, + "\u0120hardware": 6890, + "\u0120coffee": 6891, + "Rel": 6892, + "bal": 6893, + "world": 6894, + "\u0120Saf": 6895, + "\u0120Sea": 6896, + "inals": 6897, + "\u0120owned": 6898, + "\u0120hall": 6899, + "ersion": 6900, + "\u0120describe": 6901, + "\u0120Pot": 6902, + "\u0120portion": 6903, + "\u0120atmosp": 6904, + "\u0120governments": 6905, + "\u0120depending": 6906, + "\u0120offense": 6907, + "\u0120trick": 6908, + "awa": 6909, + "\u0120Line": 6910, + "\u0120Vis": 6911, + "\u0120Hard": 6912, + "\u0120Orig": 6913, + "\u0120Click": 6914, + "\u0120desk": 6915, + "\u0120Valley": 6916, + "\u0120Sov": 6917, + "\u0120movies": 6918, + "\u0120remark": 6919, + "\u0120mail": 6920, + "\u0120conscious": 6921, + "\u0120ruling": 6922, + "\u0120Rights": 6923, + "\u0120medic": 6924, + "hent": 6925, + "\u0120Women": 6926, + "><": 6927, + "\u0120replaced": 6928, + "\u0120Prem": 6929, + "\u0120Thanks": 6930, + "\u0120renew": 6931, + "\u0120Ball": 6932, + "iform": 6933, + "\u0120shots": 6934, + "Comm": 6935, + "\u0120armed": 6936, + "\u0120constant": 6937, + "\u0120taste": 6938, + "\u0120realized": 6939, + "\u0120buff": 6940, + "\u0120mo": 6941, + "\u0120efficient": 6942, + "Most": 6943, + "oration": 6944, + "ifies": 6945, + "\u0120communication": 6946, + "\u0120flood": 6947, + "\u0120consequences": 6948, + "\u0120anyway": 6949, + "igg": 6950, + "\u0120GM": 6951, + "\u0120Thank": 6952, + "\u0120iron": 6953, + "\u0120evolution": 6954, + "\u0120Cop": 6955, + "twitter": 6956, + "\u012095": 6957, + "\u0120relationships": 6958, + "adel": 6959, + "\u0120Young": 6960, + "\u0120proposal": 6961, + "ayers": 6962, + "uilding": 6963, + "\u0120Hot": 6964, + "ORE": 6965, + "cos": 6966, + "\u0120collabor": 6967, + "PG": 6968, + "axy": 6969, + "\u0120knowing": 6970, + "\u0120supports": 6971, + "owed": 6972, + "\u0120controls": 6973, + "\u0120merely": 6974, + "umer": 6975, + "\u0120athlet": 6976, + "\u0120fashion": 6977, + "path": 6978, + "\u0120gift": 6979, + "\u0120era": 6980, + "AND": 6981, + "\u0120kinds": 6982, + "\u0120Korean": 6983, + "\u0120legit": 6984, + "ulous": 6985, + "\u0120essentially": 6986, + "\u0120therap": 6987, + "nic": 6988, + "\u0120suffered": 6989, + "\u0120hur": 6990, + "\u0120promise": 6991, + "\u0120excess": 6992, + "\u0120overw": 6993, + "\u0120prime": 6994, + "\u0120Houston": 6995, + "erry": 6996, + "\u0120Ms": 6997, + "RS": 6998, + "2012": 6999, + "\u0120stores": 7000, + "\u0120Olymp": 7001, + "\u0120journey": 7002, + "Although": 7003, + "Sub": 7004, + "\u0120Educ": 7005, + "\u0120Chapter": 7006, + "\u0120requests": 7007, + "\u0120consumers": 7008, + "\u0120tiny": 7009, + "\u0120isol": 7010, + "\u0120Fair": 7011, + "ba": 7012, + "\u0120YOU": 7013, + "\u0120crash": 7014, + "celer": 7015, + "\u0120emotional": 7016, + "\u0120goods": 7017, + "\u0120elected": 7018, + "\u0120moder": 7019, + "\u0120Linux": 7020, + "\u0120blocks": 7021, + "\u0120island": 7022, + "\u0120Society": 7023, + "\u0120elections": 7024, + "\u0120broadcast": 7025, + "\u0120cheap": 7026, + "\u0120nations": 7027, + "\u0120seasons": 7028, + "400": 7029, + "\u0120waste": 7030, + "\u0120Sat": 7031, + "\u0120fields": 7032, + "employ": 7033, + "\u0120profile": 7034, + "\u0120authors": 7035, + "ALL": 7036, + "\u0120Gra": 7037, + "west": 7038, + "\u0120Ty": 7039, + "\u0120deaths": 7040, + "\u0120vacc": 7041, + "\u0120formed": 7042, + "\u0120du": 7043, + "\u0120ongoing": 7044, + "\u0120Muslims": 7045, + "elf": 7046, + "igure": 7047, + "\u0120assume": 7048, + "\u0120Ukraine": 7049, + "water": 7050, + "\u0120coast": 7051, + "\u0120voted": 7052, + "gor": 7053, + "\u0120AS": 7054, + "\u0120Michigan": 7055, + "aza": 7056, + "\u0120Arm": 7057, + "iro": 7058, + "\u0120flex": 7059, + "asters": 7060, + "''": 7061, + "\u0120welcome": 7062, + "arl": 7063, + "\u0120locations": 7064, + "igation": 7065, + "\u0120Fil": 7066, + "\u0120buying": 7067, + "\u0120architect": 7068, + "\u0120harder": 7069, + "\u0120Cub": 7070, + "\u0120interface": 7071, + "\u0120restaurant": 7072, + "\u0120discover": 7073, + "\u0120exceed": 7074, + "\u0120favour": 7075, + "gery": 7076, + "\u0120duty": 7077, + "\u0120pitch": 7078, + "ador": 7079, + "\u0120Mach": 7080, + "boy": 7081, + "\u0120responded": 7082, + "\u0120extended": 7083, + "hers": 7084, + "Many": 7085, + "raid": 7086, + "ifer": 7087, + "\u0120Ins": 7088, + "Ser": 7089, + "\u0120medium": 7090, + "she": 7091, + "\u0120Sports": 7092, + "\u0120magazine": 7093, + "utation": 7094, + "\u0120limits": 7095, + "\u0120Gall": 7096, + "\u0120external": 7097, + "razil": 7098, + "\u0120younger": 7099, + "tle": 7100, + "\u0120remind": 7101, + "\u0120CON": 7102, + "\u0120immediate": 7103, + "\u0120hidden": 7104, + "\u0120volunte": 7105, + "\u0120simpl": 7106, + "odcast": 7107, + "\u0120phase": 7108, + "dr": 7109, + "\u0120plot": 7110, + "\u0120exposure": 7111, + "RI": 7112, + "ograp": 7113, + "vin": 7114, + "anish": 7115, + "\u0120Acad": 7116, + "\u0120Engine": 7117, + "\u0120expansion": 7118, + "\u0120Pay": 7119, + "Your": 7120, + "\u0120pushed": 7121, + "\u0120Ell": 7122, + "\u0120Head": 7123, + "\u0120marketing": 7124, + "\u0120AC": 7125, + "ket": 7126, + "\u0120hits": 7127, + "\u0120gro": 7128, + "\u0120Age": 7129, + "\u0120Scot": 7130, + "][": 7131, + "\u0120stim": 7132, + "\u0120iPhone": 7133, + "\u012a\u0134": 7134, + "\u0120narrow": 7135, + "\u0120Getty": 7136, + "\u0120Turkey": 7137, + "\u0120perfectly": 7138, + "\u0120enable": 7139, + "utch": 7140, + "\u0120precise": 7141, + "\u0120regime": 7142, + "\u0120shif": 7143, + "\u0120compens": 7144, + "gun": 7145, + "div": 7146, + "\u0120chosen": 7147, + "\u0120Ken": 7148, + "Any": 7149, + "\u0120trees": 7150, + "\u0120recommended": 7151, + "\u0120Ren": 7152, + "uable": 7153, + "\u0120HT": 7154, + "Follow": 7155, + "EG": 7156, + "\u0120Hand": 7157, + "\u0120Kenn": 7158, + "\u0120arguments": 7159, + "\u0120exists": 7160, + "\u0120bike": 7161, + "\u0120Conserv": 7162, + "\u0120breaking": 7163, + "\u0120Gar": 7164, + "\u0120crazy": 7165, + "\u0120virtual": 7166, + "aylor": 7167, + "ixel": 7168, + "\u01201980": 7169, + "\u0120permission": 7170, + "\u0120Series": 7171, + "\u0120consumer": 7172, + "\u0120closely": 7173, + "called": 7174, + "\u012054": 7175, + "\u0120hopes": 7176, + "\u0120array": 7177, + "\u0120Win": 7178, + "\u0120Labour": 7179, + "\u0120spons": 7180, + "\u0120Ire": 7181, + "\u0120pow": 7182, + "\u0120readers": 7183, + "\u0120employment": 7184, + "\u0120creature": 7185, + "\u0120resulting": 7186, + "\u0120accurate": 7187, + "\u0120moments": 7188, + "\u0120argued": 7189, + "\u0120ped": 7190, + "During": 7191, + "\u012053": 7192, + "\u0120Tal": 7193, + "\u0120sought": 7194, + "\u0120suffering": 7195, + "\u0120icon": 7196, + "lee": 7197, + "\u0120($": 7198, + "alian": 7199, + "\u00c2\u00b0": 7200, + "\u0120pra": 7201, + "\u0120bonus": 7202, + "(\"": 7203, + "ko": 7204, + "\u0120acting": 7205, + "DE": 7206, + "fall": 7207, + "\u0120comparison": 7208, + "\u0120smooth": 7209, + "\u0120NAS": 7210, + "upp": 7211, + "\u0120Joseph": 7212, + "eping": 7213, + "\u0120Take": 7214, + "\u0120Mid": 7215, + "\u0120sending": 7216, + "fast": 7217, + "\u0120Fall": 7218, + "\u0120dealing": 7219, + "user": 7220, + "\u0120Organ": 7221, + "Co": 7222, + "\u0120attached": 7223, + "\u0120sees": 7224, + "%.": 7225, + "\u0120typical": 7226, + "ART": 7227, + "\u0120finds": 7228, + "\u0120Asia": 7229, + "umin": 7230, + "\u0120Core": 7231, + "\u0120Ent": 7232, + "inent": 7233, + "uce": 7234, + "\u0120Blood": 7235, + "\u0120Never": 7236, + "\u0120emails": 7237, + "\u0120highlight": 7238, + "\u0120confront": 7239, + "atus": 7240, + "uted": 7241, + "\u0120unus": 7242, + "\u0120topic": 7243, + "\u0120Adam": 7244, + "\u0120ble": 7245, + "ati": 7246, + "\u0120understood": 7247, + "Set": 7248, + "struct": 7249, + "TP": 7250, + "\u0120mob": 7251, + "aa": 7252, + "\u0120Start": 7253, + "pected": 7254, + "sell": 7255, + "\u0120dedicated": 7256, + "\u0120CA": 7257, + "uan": 7258, + "\u0120songs": 7259, + "escription": 7260, + "\u0120tech": 7261, + "\u0120rape": 7262, + "\u0120aside": 7263, + "\u0120grant": 7264, + "\u012056": 7265, + "sub": 7266, + "\u0120argue": 7267, + "\u0120containing": 7268, + "\u0120schedule": 7269, + "\u0120liberal": 7270, + "\u0120publicly": 7271, + "\u0120heavily": 7272, + "\u0120Ut": 7273, + "iner": 7274, + "\u0120Section": 7275, + "\u0120Care": 7276, + "weet": 7277, + "ls": 7278, + "Dis": 7279, + "\u00e2\u0136\u0122": 7280, + "\u0120Follow": 7281, + "Back": 7282, + "\u0120IT": 7283, + "\u0120bes": 7284, + "ji": 7285, + "\u0120Hit": 7286, + "ested": 7287, + "\u0120everybody": 7288, + "\u0120Swed": 7289, + "\u0120femin": 7290, + "\u0120facilities": 7291, + "\u0120conven": 7292, + "Comp": 7293, + "\u0120OS": 7294, + "core": 7295, + "\u0120anx": 7296, + "\u0120division": 7297, + "\u0120Cam": 7298, + "\u0120Stan": 7299, + "mates": 7300, + "\u0120explore": 7301, + "plom": 7302, + "\u0120shares": 7303, + "pload": 7304, + "anes": 7305, + "\u0120ideal": 7306, + "eters": 7307, + "\u0120Base": 7308, + "\u0120plastic": 7309, + "\u0120distinct": 7310, + "\u0120Network": 7311, + "\u0120Seattle": 7312, + "\u0120trading": 7313, + "ensus": 7314, + "intend": 7315, + "\u0120exhib": 7316, + "\u0120initially": 7317, + "\u0120Food": 7318, + "\u0120thousand": 7319, + "\u0120Business": 7320, + "acter": 7321, + "\u0120paragraph": 7322, + "\u0120roughly": 7323, + "\u0120www": 7324, + "\u0120creative": 7325, + "\u0120Conf": 7326, + "\u0120consumption": 7327, + "\u0120films": 7328, + "agan": 7329, + "\u0120obtain": 7330, + "\u0120tall": 7331, + "\u0120tor": 7332, + "\u0120acknowled": 7333, + "\u0120grown": 7334, + "alo": 7335, + "KE": 7336, + "\u0120400": 7337, + "enders": 7338, + "taining": 7339, + "UG": 7340, + "\u0120suicide": 7341, + "\u0120watched": 7342, + "\u0120List": 7343, + "ali": 7344, + "rehens": 7345, + "\u0120surrounding": 7346, + "\u0120pip": 7347, + "\u0120flying": 7348, + "\u0120Java": 7349, + "ordan": 7350, + "\u0120serving": 7351, + "inations": 7352, + "post": 7353, + "\u0120sho": 7354, + "Av": 7355, + "\u0120jail": 7356, + "zy": 7357, + "\u01201999": 7358, + "\u0120>": 9609, + "orous": 9610, + "\u0120firms": 9611, + "screen": 9612, + "una": 9613, + "\u0120embarrass": 9614, + "ulse": 9615, + "\u0120letting": 9616, + "\u0120threw": 9617, + "iley": 9618, + "\u0120channels": 9619, + "lan": 9620, + "\u0120Vegas": 9621, + "\u0120sear": 9622, + "\u0120fantastic": 9623, + "arre": 9624, + "uzzle": 9625, + "\u0120Der": 9626, + "Those": 9627, + "\u0120swing": 9628, + "\u0120sheet": 9629, + "index": 9630, + "cover": 9631, + "ogan": 9632, + "\u0120variables": 9633, + "\u0120Tech": 9634, + "\u0120spoken": 9635, + "achel": 9636, + "\u0120Da": 9637, + "\u0120Mountain": 9638, + "\u0120loaded": 9639, + "\u0120footage": 9640, + "version": 9641, + "\u0120unl": 9642, + "\u0120Phoenix": 9643, + "\u0120throwing": 9644, + "\u0120firing": 9645, + "\u0120tracking": 9646, + "\u0120width": 9647, + "\u0120struggling": 9648, + "rooms": 9649, + "otion": 9650, + "\u0120monthly": 9651, + "\u0120Server": 9652, + "\u0120eggs": 9653, + "open": 9654, + "MC": 9655, + "\u01201993": 9656, + "\u0120hired": 9657, + "\u0120stayed": 9658, + "\u0120Allen": 9659, + "\u0120stro": 9660, + "\u012098": 9661, + "step": 9662, + "\u0120Turkish": 9663, + "\u0120fabric": 9664, + "isting": 9665, + "\u0120Dom": 9666, + "\u0120dates": 9667, + "\u0120pron": 9668, + "\u0120basketball": 9669, + "\u0120lucky": 9670, + "\u0120Arabia": 9671, + "\u0120assumed": 9672, + "esty": 9673, + "\u0120affairs": 9674, + "\u0120glad": 9675, + "\u0120Indeed": 9676, + "\u0120FA": 9677, + "\u0120Word": 9678, + "\u0120joining": 9679, + "ifice": 9680, + "pread": 9681, + "irts": 9682, + "\u0120Select": 9683, + "\u0120populations": 9684, + "aware": 9685, + "\u0120nose": 9686, + "\u0120complaints": 9687, + "start": 9688, + "\u0120scoring": 9689, + "Thanks": 9690, + "\u0120mining": 9691, + "\u0120visitors": 9692, + "SH": 9693, + "\u0120damaged": 9694, + "\u0120characteristics": 9695, + "\u0120Pent": 9696, + "DC": 9697, + "\u012083": 9698, + "\u0120Six": 9699, + "rates": 9700, + "\u0120flags": 9701, + "\u0120Brew": 9702, + "dog": 9703, + "Mark": 9704, + "////": 9705, + "\u0120execution": 9706, + "\u0120joke": 9707, + "phones": 9708, + "\u0120testimony": 9709, + "\u0120obst": 9710, + "QL": 9711, + "\u0120Cut": 9712, + "\u0120studied": 9713, + "\u0120Nintendo": 9714, + "icket": 9715, + "\u0120NBC": 9716, + "\u0120lad": 9717, + "\u0120Bra": 9718, + "\u0120Moh": 9719, + "\u0120kernel": 9720, + "\u0120overwhelming": 9721, + "\u0120aged": 9722, + "\u0120applicable": 9723, + "\u0120Cond": 9724, + "\u0120roads": 9725, + "\u0120Block": 9726, + "made": 9727, + "odge": 9728, + "\u0120commands": 9729, + "\u0120offices": 9730, + "veland": 9731, + "\u0120tut": 9732, + "\u0120receiver": 9733, + "\u0120Fro": 9734, + "\u0120shopping": 9735, + "\u0120iP": 9736, + "\u0120Stre": 9737, + "\u0120ABC": 9738, + "\u0120entertainment": 9739, + "\u0120Bow": 9740, + "orted": 9741, + "Mc": 9742, + "\u0120reads": 9743, + "grad": 9744, + "\u0120Collect": 9745, + "\u0120\u00e2\u012a\u0134": 9746, + "\u0120Capital": 9747, + "ederation": 9748, + "\u0120employer": 9749, + "\u0120involvement": 9750, + "\u0120anxiety": 9751, + "alia": 9752, + "\u0120roof": 9753, + "\u0120Among": 9754, + "\u0120Democrat": 9755, + "\u0120stats": 9756, + "\u0120Vill": 9757, + "\u0120constitutional": 9758, + "\u0120referring": 9759, + "itty": 9760, + "\u0120tackle": 9761, + "outube": 9762, + "\u0120backed": 9763, + "\u0120Hong": 9764, + "\u0120Broad": 9765, + "\u0120ele": 9766, + "\u0120Ott": 9767, + "\u01201992": 9768, + "hour": 9769, + "achusetts": 9770, + "Cal": 9771, + "\u0120defeated": 9772, + "\u012081": 9773, + "esp": 9774, + "\u0120seemingly": 9775, + "was": 9776, + "\u0120Jenn": 9777, + "\u0120Kurd": 9778, + "\u0120gene": 9779, + "\u0120discount": 9780, + "Ret": 9781, + "ECT": 9782, + "();": 9783, + "\u0120clubs": 9784, + "\u0120sid": 9785, + "\u0120Marsh": 9786, + "Check": 9787, + "\u0120pp": 9788, + "\u0120Eag": 9789, + "idespread": 9790, + "\u0120beings": 9791, + "FT": 9792, + "\u0120introduction": 9793, + "\u0120Change": 9794, + "ARD": 9795, + "\u0120110": 9796, + "adows": 9797, + "ierce": 9798, + "\u0120meal": 9799, + "author": 9800, + "\u0120Bang": 9801, + "lahoma": 9802, + "\u0120ranks": 9803, + "2011": 9804, + "????": 9805, + "max": 9806, + "\u0120collapse": 9807, + "\u0120opens": 9808, + "\u0120echo": 9809, + "\u0120soph": 9810, + "\u0120racist": 9811, + "\u0120enormous": 9812, + "\u0120waves": 9813, + "\u0120tap": 9814, + "\u0120comprehensive": 9815, + ".--": 9816, + "\u0120Roy": 9817, + "\u0120farmers": 9818, + "Related": 9819, + "aired": 9820, + "rones": 9821, + "\u0120Crim": 9822, + "\u0120proportion": 9823, + "\u0120designs": 9824, + "\u0120negotiations": 9825, + "\u0120virtually": 9826, + "\u0120Batman": 9827, + "\u0120warn": 9828, + "\u0120legitimate": 9829, + "mate": 9830, + "\u0120convention": 9831, + ",,": 9832, + "netic": 9833, + "\u0120SD": 9834, + "\u0120consistently": 9835, + "\u0120compensation": 9836, + "\u0120punishment": 9837, + "\u0120ye": 9838, + "\u0120tie": 9839, + "\u0120Bureau": 9840, + "irlf": 9841, + "\u0120Bu": 9842, + "\u0120Aren": 9843, + "\u0120Philipp": 9844, + "\u0120knife": 9845, + "\u0120memories": 9846, + "\u0120Ross": 9847, + "\u0120angle": 9848, + "\u012086": 9849, + "\u0120Thunder": 9850, + "\u0120rend": 9851, + "\u0120Tour": 9852, + "\u0120counts": 9853, + "sung": 9854, + "\u0120Imp": 9855, + "\u0120educational": 9856, + "\u0120accessible": 9857, + "COM": 9858, + "\u0120drew": 9859, + "yer": 9860, + "Gl": 9861, + "amine": 9862, + "ORT": 9863, + "OB": 9864, + "IB": 9865, + "master": 9866, + "\u0120trials": 9867, + "ogy": 9868, + "har": 9869, + "\u0120Trust": 9870, + "\u0120preferred": 9871, + "irlfriend": 9872, + "\u0120Nev": 9873, + "\u0120bin": 9874, + "\u0120cow": 9875, + "Page": 9876, + "\u0120signature": 9877, + "\u0120BL": 9878, + "700": 9879, + "\u0120retired": 9880, + "\u0120bytes": 9881, + "\u0120neighb": 9882, + "\u0120Legend": 9883, + "\u0120devast": 9884, + "\u0120suspected": 9885, + "isons": 9886, + "\u0120Pok\u00c3\u00a9mon": 9887, + "scale": 9888, + "\u0120capabilities": 9889, + "\u0120revel": 9890, + "\u0120cheese": 9891, + "dy": 9892, + "igrant": 9893, + "\u0120failing": 9894, + "bits": 9895, + "\u0120Heroes": 9896, + "\u0120Ghost": 9897, + "\u0120Scient": 9898, + "\u0120appointed": 9899, + "uri": 9900, + "\u0120institution": 9901, + "\u0120expanded": 9902, + "greg": 9903, + "\u0120monitoring": 9904, + "\u0120podcast": 9905, + "\u0120coalition": 9906, + "\u012096": 9907, + "Jo": 9908, + "\u0120stolen": 9909, + "\u0120Sab": 9910, + "\u0120stops": 9911, + "\u0120holiday": 9912, + "\u0120intr": 9913, + "Car": 9914, + "Black": 9915, + "\u0120LGBT": 9916, + "\u0120warming": 9917, + "\u0120Anderson": 9918, + "\u012089": 9919, + "\u0120producer": 9920, + "Med": 9921, + "\u0120accuracy": 9922, + "\u0120Marvel": 9923, + "izabeth": 9924, + "\u0120Patrick": 9925, + "mony": 9926, + "\u0120mini": 9927, + "acles": 9928, + "\u0120overt": 9929, + "they": 9930, + "\u0120membership": 9931, + "\u0120Ven": 9932, + "\u0120exch": 9933, + "\u0120removal": 9934, + "\u0120Dave": 9935, + "TY": 9936, + "mad": 9937, + "\u0120Find": 9938, + "\u0120adequ": 9939, + "\u0120ec": 9940, + "\u0120teeth": 9941, + "\u0120emotion": 9942, + "\u0120perm": 9943, + "\u0120solely": 9944, + "db": 9945, + "\u0120extraord": 9946, + "IGHT": 9947, + "cal": 9948, + "\u0120guidelines": 9949, + "\u0120dying": 9950, + "\u0120suspended": 9951, + "\u0120Premier": 9952, + "\u0120Anthony": 9953, + "elve": 9954, + "\u0120dad": 9955, + "\u0120Eth": 9956, + "\u0120Football": 9957, + "\u0120abandoned": 9958, + "\u0120<<": 9959, + "\u0120march": 9960, + "\u0120horror": 9961, + "\u00e2\u0122\u00a6\"": 9962, + "\u0120childhood": 9963, + "\u0120campaigns": 9964, + "\u0120lunch": 9965, + "\u0120Albert": 9966, + "block": 9967, + "\u00e2\u0138\u012a\u00e2\u0138\u012a": 9968, + "ounding": 9969, + "\u0120bone": 9970, + "organ": 9971, + "aders": 9972, + "\u0120Flash": 9973, + "\u0120Drive": 9974, + "\u0120tonight": 9975, + "\u0120wars": 9976, + "\u0120FL": 9977, + "\u0120formation": 9978, + "const": 9979, + "News": 9980, + "\u0120compe": 9981, + "orious": 9982, + "\u0120Staff": 9983, + "\u0120discussions": 9984, + "\u0120Protection": 9985, + "\u0120Jam": 9986, + "\u0120criteria": 9987, + "\u0120installation": 9988, + "\u0120accomplish": 9989, + "izza": 9990, + "\u0120publisher": 9991, + "\u0120rescue": 9992, + "\u0120Try": 9993, + "ULL": 9994, + "\u0120Som": 9995, + "\u0120Hop": 9996, + "oret": 9997, + "ths": 9998, + "ordon": 9999, + "\u0120pocket": 10000, + "\u0120Inv": 10001, + "Download": 10002, + "\u0120Crime": 10003, + "\u0120bene": 10004, + "\u0120Guide": 10005, + "\u0120Assembly": 10006, + "\u0120parameters": 10007, + "IE": 10008, + "\u0120Alexander": 10009, + "\u0120concert": 10010, + "\u0120Sche": 10011, + "\u0120shoes": 10012, + "\u0120visiting": 10013, + "\u0120recall": 10014, + "\u0120bub": 10015, + "\u0120rural": 10016, + "\u0120concrete": 10017, + "\u0120Ros": 10018, + "Next": 10019, + "Russ": 10020, + "\u0120loans": 10021, + "\u0120Shield": 10022, + "\u0120trem": 10023, + "hemat": 10024, + "kg": 10025, + "\u0120Harris": 10026, + "isition": 10027, + "\u0120Move": 10028, + "\u0120FC": 10029, + "\u0120fate": 10030, + "\u0120Cho": 10031, + "\u0120tired": 10032, + "\u0120principal": 10033, + "hist": 10034, + "iences": 10035, + "athy": 10036, + "\u0120sevent": 10037, + "\u0120mood": 10038, + "\u0120strategic": 10039, + "\u0120diseases": 10040, + "\u0120forum": 10041, + "\u0120tempor": 10042, + "\u0120headquarters": 10043, + "Par": 10044, + "ige": 10045, + "flix": 10046, + "\u0120guitar": 10047, + "\u012094": 10048, + "Only": 10049, + "\u0120releases": 10050, + "roph": 10051, + "================================": 10052, + "\u0120600": 10053, + "\u0120Continue": 10054, + "igate": 10055, + "\u0120Crit": 10056, + "system": 10057, + "\u0120disabled": 10058, + "\u0120unexpected": 10059, + "ithub": 10060, + "\u0120unclear": 10061, + "\u0120Est": 10062, + "\u0120contrad": 10063, + "\u0120strategies": 10064, + "ventures": 10065, + "\u0120passage": 10066, + "AME": 10067, + "\u0120improving": 10068, + "\u0120reveals": 10069, + "\u0120decrease": 10070, + "ova": 10071, + "\u0120annoy": 10072, + "\u0120Short": 10073, + "\u0120Library": 10074, + "\u0120cyber": 10075, + "nell": 10076, + "\u0120Hur": 10077, + "\u0120CB": 10078, + "\u0120photograp": 10079, + "UI": 10080, + "\u0120sed": 10081, + "Ge": 10082, + "\u012087": 10083, + "\u0120diverse": 10084, + "\u0120encouraged": 10085, + "\u0120conspiracy": 10086, + "\u0120birds": 10087, + "\u0120operator": 10088, + "\u0120handful": 10089, + "\u0120classified": 10090, + "?)": 10091, + "\u0120dramatic": 10092, + "\u0120investigators": 10093, + "ito": 10094, + "\u0120widespread": 10095, + "\u0120Room": 10096, + "----------------------------------------------------------------": 10097, + "\u0120collective": 10098, + "\u0120journalist": 10099, + "String": 10100, + "\u0120temperatures": 10101, + "ila": 10102, + "\u0120guid": 10103, + "\u0120inspect": 10104, + "\u0120missile": 10105, + "\u0120Mayor": 10106, + "\u0120manual": 10107, + "\u0120simultane": 10108, + "\u0120ratings": 10109, + "\u0120suck": 10110, + "\u012097": 10111, + "\u0120universal": 10112, + "\u0120pharm": 10113, + "\u0120disrupt": 10114, + "iano": 10115, + "AV": 10116, + "\u0120ft": 10117, + "\u0120statist": 10118, + "olds": 10119, + "\u0120Walker": 10120, + "php": 10121, + "\u0120undert": 10122, + "\u0120Las": 10123, + "ishop": 10124, + "ntil": 10125, + "reshold": 10126, + "\u0120Whether": 10127, + "Ms": 10128, + "\u0120deny": 10129, + "\u0120Cloud": 10130, + "\u0120provider": 10131, + "\u0120surviv": 10132, + "\u0120Update": 10133, + "has": 10134, + "\u0120mistakes": 10135, + "charge": 10136, + "pled": 10137, + "rity": 10138, + "\u0120node": 10139, + "\u0120Massachusetts": 10140, + "ools": 10141, + "lication": 10142, + "\u0120fails": 10143, + "emale": 10144, + "ori": 10145, + "backs": 10146, + "\u0120shirt": 10147, + "\u0120''": 10148, + "\u0120NAT": 10149, + "\u0120waters": 10150, + "elson": 10151, + "\u0120ease": 10152, + "\u0120scar": 10153, + "\u0120contents": 10154, + "mind": 10155, + "\u0120contribution": 10156, + "\u0120shr": 10157, + "\u0120handed": 10158, + "\u0120stability": 10159, + "\u0120trave": 10160, + "Em": 10161, + "\u0120mirror": 10162, + "123": 10163, + "\u0120weigh": 10164, + "\u0120fiction": 10165, + "ouver": 10166, + "istant": 10167, + "rition": 10168, + "\u0120Fed": 10169, + "\u0120physically": 10170, + "\u0120stake": 10171, + "\u0120Article": 10172, + "\u0120Arc": 10173, + "\u0120Lewis": 10174, + "\u0120Mind": 10175, + "\u0120demonstrate": 10176, + "\u0120profits": 10177, + "vision": 10178, + "omic": 10179, + "olid": 10180, + "\u0120battles": 10181, + "\u0120drives": 10182, + "\u0120eastern": 10183, + "\u0120Sony": 10184, + "!!!": 10185, + "aration": 10186, + "vard": 10187, + "\u0120GL": 10188, + "portation": 10189, + "\u012092": 10190, + "\u0120lawmakers": 10191, + "\u0120protecting": 10192, + "\u0120EPA": 10193, + "\u0120yeah": 10194, + "\u0120shame": 10195, + "olph": 10196, + "even": 10197, + "xit": 10198, + "\u0120attach": 10199, + "\u0120representing": 10200, + "\u0120obs": 10201, + "\u0120Utah": 10202, + "iffs": 10203, + "\u0120Freedom": 10204, + "\u00c3\u00b3": 10205, + "AK": 10206, + "\u0120incidents": 10207, + "itage": 10208, + "\u0120viewers": 10209, + "cd": 10210, + "\u0120mouse": 10211, + "\u0120clar": 10212, + "\u0120accordance": 10213, + "\u0120bot": 10214, + "cor": 10215, + "\u0120Summer": 10216, + "held": 10217, + "\u0120innocent": 10218, + "\u0120initiative": 10219, + "ols": 10220, + "________________________________": 10221, + "\u0120spots": 10222, + "pace": 10223, + "\u0120conventional": 10224, + "\u0120corporations": 10225, + "\u0120blocked": 10226, + "HD": 10227, + "attered": 10228, + "\u0120refers": 10229, + "\u0120buck": 10230, + "\u0120Digital": 10231, + "120": 10232, + "\u0120topics": 10233, + "TF": 10234, + "\u00c4\u0123": 10235, + "brid": 10236, + "reement": 10237, + "\u0120underlying": 10238, + "\u0120Member": 10239, + "\u0120investigating": 10240, + "\u0120pregnancy": 10241, + "\u0120touchdown": 10242, + "\u0120Band": 10243, + "\u0120Caller": 10244, + "\u0120instances": 10245, + "PP": 10246, + "wa": 10247, + "Good": 10248, + "\u01201991": 10249, + "\u0120Cold": 10250, + "\u0120fears": 10251, + "\u0120remarks": 10252, + "\u0128\u0134": 10253, + "atal": 10254, + "\u0120mit": 10255, + "\u0120experiments": 10256, + "ipt": 10257, + "Color": 10258, + "indu": 10259, + "Update": 10260, + "\u012093": 10261, + "Ag": 10262, + "\u0120\u00e5": 10263, + "ancouver": 10264, + "Both": 10265, + "\u0120judges": 10266, + "Object": 10267, + "\u0120stere": 10268, + "umbn": 10269, + "\u0120participation": 10270, + "\u0120Stars": 10271, + "\u0120Jere": 10272, + "\u0120weekly": 10273, + "\u0120Ban": 10274, + "\u0120conversations": 10275, + "\u0120Pitt": 10276, + "uz": 10277, + "\u0120Indiana": 10278, + "\u0120Kick": 10279, + "\u0120infection": 10280, + "\u0120heroes": 10281, + "\u0120settled": 10282, + "\u0120strip": 10283, + "\u0120hal": 10284, + "\u0120dump": 10285, + "\u0120Sci": 10286, + "\u0120les": 10287, + "\u0120references": 10288, + "\u0120URL": 10289, + "\u0120Bridge": 10290, + "\u0120wanting": 10291, + "Force": 10292, + "\u0120exclus": 10293, + "Meanwhile": 10294, + "mn": 10295, + "\u0120gentle": 10296, + "maker": 10297, + "senal": 10298, + "\u0120Gro": 10299, + "ouri": 10300, + "\u0120Rain": 10301, + "\u0120Alliance": 10302, + "\u0120lift": 10303, + "ela": 10304, + "SD": 10305, + "\u0120Cleveland": 10306, + "\u0120ranked": 10307, + "\u0120stadium": 10308, + "\u0120deadly": 10309, + "\u00e4\u00b8": 10310, + "\u0120riding": 10311, + "aria": 10312, + "\u0120Armor": 10313, + "\u0120documentation": 10314, + "\u0120Greece": 10315, + "reek": 10316, + "\u0120lens": 10317, + "\u0120Sa": 10318, + "\u0120gross": 10319, + "\u0120Emer": 10320, + "agers": 10321, + "\u0120Dub": 10322, + "\u0120Rh": 10323, + "\u0120AMD": 10324, + "\u0120arrival": 10325, + "\u0120desert": 10326, + "\u0120supplement": 10327, + "\u0120Resp": 10328, + "\u0120knee": 10329, + "\u0120margin": 10330, + "font": 10331, + "ogg": 10332, + "2010": 10333, + "\u0120Pir": 10334, + "\u0120Prom": 10335, + "ivals": 10336, + "\u0120intake": 10337, + "\u0120differently": 10338, + "ugs": 10339, + "\u0120bits": 10340, + "cluded": 10341, + "\u0120searching": 10342, + "\u0120Du": 10343, + "umble": 10344, + "\u0120functional": 10345, + "\u0120Baltimore": 10346, + "\u0120Could": 10347, + "\u0120desired": 10348, + "\u0120circuit": 10349, + "\u0120Lyn": 10350, + "\u0120GO": 10351, + "\u0120False": 10352, + "repre": 10353, + "':": 10354, + "alties": 10355, + "\u0120minim": 10356, + "\u0120drove": 10357, + "\u0120Should": 10358, + "\u0120hip": 10359, + "\u0120pros": 10360, + "\u0120utility": 10361, + "\u0120Nature": 10362, + "\u0120Mode": 10363, + "President": 10364, + "opp": 10365, + "rat": 10366, + "formance": 10367, + "\u0120concentration": 10368, + "\u0120font": 10369, + "\u0120Bud": 10370, + "\u0120amid": 10371, + "\u0120revers": 10372, + "\u0120ML": 10373, + "Bar": 10374, + "\u0120interaction": 10375, + "\u0120jurisd": 10376, + "\u0120spells": 10377, + "dep": 10378, + "fil": 10379, + "\u0120civilians": 10380, + "utter": 10381, + "\u0120Cooper": 10382, + "\u0120Below": 10383, + "\u0120entrance": 10384, + "\u0120convert": 10385, + "\u0120controversy": 10386, + "owered": 10387, + "\u0120contrary": 10388, + "\u0120arc": 10389, + "\u0120Executive": 10390, + "\u0120Officer": 10391, + "\u0120packages": 10392, + "\u0120progressive": 10393, + "width": 10394, + "\u0120reserved": 10395, + "vol": 10396, + "\u0120Samsung": 10397, + "\u0120printed": 10398, + "\u0120centers": 10399, + "\u0120introduce": 10400, + "\u0120Kennedy": 10401, + "\u0120odds": 10402, + "\u0120surely": 10403, + "\u0120independence": 10404, + "\u0120passengers": 10405, + "reprene": 10406, + "\u0120Beh": 10407, + "\u0120loves": 10408, + "\u0120ESPN": 10409, + "\u0120facilit": 10410, + "\u0120identical": 10411, + "\u0120doct": 10412, + "\u0120partnership": 10413, + "conf": 10414, + "\u0120Hide": 10415, + "\u0120confused": 10416, + "\u0120Cow": 10417, + "Men": 10418, + "\u0120wrest": 10419, + "\u0120Iraqi": 10420, + "\u0120holes": 10421, + "\u0120Studies": 10422, + "\u0120pregnant": 10423, + "hard": 10424, + "\u0120signals": 10425, + "IX": 10426, + "\u0120pulling": 10427, + "\u0120graduate": 10428, + "\u0120nominee": 10429, + "Date": 10430, + "\u0120permitted": 10431, + "\u0120\u00e2\u0124\u00ac": 10432, + "\u0120Oklahoma": 10433, + "Start": 10434, + "\u0120authorized": 10435, + "\u0120alarm": 10436, + "\u0120Cos": 10437, + "van": 10438, + "\u0120generations": 10439, + "cular": 10440, + "\u0120dragon": 10441, + "\u0120Software": 10442, + "\u0120Edward": 10443, + "\u0120controller": 10444, + "Sen": 10445, + "gered": 10446, + "\u0120Vik": 10447, + "\u0120approached": 10448, + "Thank": 10449, + "\u0120cance": 10450, + "\u0120formula": 10451, + "\u0120Small": 10452, + "\u0120weakness": 10453, + "\u0120ramp": 10454, + "itudes": 10455, + "jud": 10456, + "\u0120brilliant": 10457, + "\u0120accus": 10458, + "source": 10459, + "\u0120800": 10460, + "\u0120Evil": 10461, + "Sw": 10462, + "\u0120homeless": 10463, + "week": 10464, + "iens": 10465, + "rics": 10466, + "\u0120Third": 10467, + "TO": 10468, + "\u0120organic": 10469, + "\u0120presentation": 10470, + "agh": 10471, + "\u0120Download": 10472, + "vation": 10473, + "\u0120assembly": 10474, + "orable": 10475, + "holders": 10476, + "\u0120Bernie": 10477, + "\u0120Help": 10478, + "\u0120tong": 10479, + "\u0120Fight": 10480, + "\u0120beach": 10481, + "Book": 10482, + "\u0120Lic": 10483, + "\u0120rush": 10484, + "\u0120Round": 10485, + "oup": 10486, + "\u0120Marx": 10487, + "\u0120calculated": 10488, + "\u0120Devil": 10489, + "\u0120Sarah": 10490, + "\u0120occasionally": 10491, + "\u0120bullet": 10492, + "Available": 10493, + "gate": 10494, + "\u012091": 10495, + "\u0120hosp": 10496, + "\u0120promises": 10497, + "\u0120HIV": 10498, + "\u0120Stadium": 10499, + "\u0120Stock": 10500, + "\u0120Corporation": 10501, + "gage": 10502, + "NG": 10503, + "\u0120Credit": 10504, + "\u0120sne": 10505, + "ibl": 10506, + "\u0120accum": 10507, + "such": 10508, + "\u0120terrorists": 10509, + "\u0120consciousness": 10510, + "\u0120Zh": 10511, + "\u0120drama": 10512, + "oola": 10513, + "piration": 10514, + "\u0120labour": 10515, + "\u0120Nin": 10516, + "\u0120utter": 10517, + "\u0120democratic": 10518, + "\u0120assass": 10519, + "ilation": 10520, + "\u0120gest": 10521, + "\u0120abroad": 10522, + "\u0120metab": 10523, + "\u0120sorts": 10524, + "\u0120flav": 10525, + "UB": 10526, + "\u0120mg": 10527, + "\u0120Nothing": 10528, + "\u0120Od": 10529, + "\u0120musical": 10530, + "2009": 10531, + "\u0120drops": 10532, + "ocated": 10533, + "ateral": 10534, + "000000": 10535, + "\u0120gre": 10536, + "\u0120equality": 10537, + "\u0120burden": 10538, + "\u0120vig": 10539, + "\u0120Leader": 10540, + "------------": 10541, + "\u0120ceremony": 10542, + "\u0120fighter": 10543, + "\u0120actors": 10544, + "\u0120\u00e6": 10545, + "aman": 10546, + "Fi": 10547, + "\u0120align": 10548, + "puter": 10549, + "\u0120elder": 10550, + "\u0120NSA": 10551, + "\u0120representation": 10552, + "\u0120Ontario": 10553, + "ITH": 10554, + "usalem": 10555, + "\u0120harassment": 10556, + "itzer": 10557, + "\u0120symp": 10558, + "\u0120boxes": 10559, + "\u0120DR": 10560, + "\u0120manifest": 10561, + "atre": 10562, + "\u0120^": 10563, + "\u0120dies": 10564, + "leton": 10565, + "\u0120missions": 10566, + "ethe": 10567, + "\u0120resolve": 10568, + "\u0120followers": 10569, + "\u0120asc": 10570, + "\u0120km": 10571, + "lord": 10572, + "ammed": 10573, + "\u0120silent": 10574, + "\u0120Associated": 10575, + "\u0120timing": 10576, + "\u0120prisoners": 10577, + "\u0120Kings": 10578, + "\u0120Five": 10579, + "\u0120tower": 10580, + "\u0120approaches": 10581, + "\u0120precisely": 10582, + "\u0120bureau": 10583, + "\u0120Mother": 10584, + "\u0120Iss": 10585, + "\u0120keyboard": 10586, + "itual": 10587, + "\u0120funded": 10588, + "\u0120staying": 10589, + "\u0120psychological": 10590, + "\u0120mile": 10591, + "\u0120Leon": 10592, + "\u0120Barb": 10593, + "will": 10594, + "\u0120wider": 10595, + "\u0120Atlantic": 10596, + "\u0120till": 10597, + "\u0120Rome": 10598, + "rot": 10599, + "\u0120accompan": 10600, + "\u0120flour": 10601, + "aco": 10602, + "World": 10603, + "\u0120Express": 10604, + "\u0120Yu": 10605, + "Cor": 10606, + "\u0120pleased": 10607, + "party": 10608, + "\u0120pointing": 10609, + "\u0120inflation": 10610, + "\u0120roy": 10611, + "\u0120),": 10612, + "ainer": 10613, + "\u0120wedding": 10614, + "ormon": 10615, + "\u0120requiring": 10616, + "\u0120qualified": 10617, + "\u0120segment": 10618, + "END": 10619, + "\u0120sizes": 10620, + "eals": 10621, + "\u0120corrupt": 10622, + "assador": 10623, + "\u0120celeb": 10624, + "\u0120dreams": 10625, + "\u0120Mess": 10626, + "\u0120checking": 10627, + "\u0120Version": 10628, + "\u0120preparing": 10629, + "\u0120actively": 10630, + "\u0120Diff": 10631, + "\u0120lux": 10632, + "\u0120Winter": 10633, + "acteria": 10634, + "\u0120NE": 10635, + "\u0120deputy": 10636, + "\u0120transgender": 10637, + "\u0120summary": 10638, + "\u0120inher": 10639, + "eries": 10640, + "char": 10641, + "\u0120Yan": 10642, + "\u0120knock": 10643, + "\u0120Path": 10644, + "\u0120lip": 10645, + "roller": 10646, + "\u0120impression": 10647, + "\u0120celebrate": 10648, + "\u0120slide": 10649, + "\u0120guests": 10650, + "\u0120clip": 10651, + "FS": 10652, + "\u0120savings": 10653, + "\u0120captain": 10654, + "\u0120legacy": 10655, + "\u0120Denver": 10656, + "\u0120wounded": 10657, + "taboola": 10658, + "ACT": 10659, + "\u0120pursue": 10660, + "\u0120oxy": 10661, + "\u0120q": 10662, + "\u0120semi": 10663, + "\u0120Need": 10664, + "\u0120Affairs": 10665, + "\u0120obsc": 10666, + "\u0120checked": 10667, + "\u0120dual": 10668, + "Code": 10669, + "\u0120MD": 10670, + "lem": 10671, + "ulty": 10672, + "\u0120\u00c2\u00a9": 10673, + "\u0120Elizabeth": 10674, + "\u0120centuries": 10675, + "arded": 10676, + "src": 10677, + "\u0120evident": 10678, + "ennis": 10679, + "atin": 10680, + "\u0120unemployment": 10681, + "\u0120Mario": 10682, + "\u0120intim": 10683, + "Christ": 10684, + "\u0120biological": 10685, + "\u0120soldier": 10686, + "\u0120Added": 10687, + "\u0120math": 10688, + "\u0120Gil": 10689, + "\u0120bias": 10690, + "\u0120dating": 10691, + "\u0120Ocean": 10692, + "\u0120mice": 10693, + "Mus": 10694, + "hire": 10695, + "\u0120Tes": 10696, + "Server": 10697, + "limited": 10698, + "Size": 10699, + "\u0120meters": 10700, + "\u0120rocket": 10701, + "essee": 10702, + "\u0120certificate": 10703, + "\u0120Iranian": 10704, + "ASS": 10705, + "\u0120grid": 10706, + "Dec": 10707, + "\u0120rolling": 10708, + "commun": 10709, + "\u0120Sweden": 10710, + "bury": 10711, + "\u0120tissue": 10712, + "\u0120racism": 10713, + "\u0120Local": 10714, + "\u0120mystery": 10715, + "\u0120examine": 10716, + "\u0120stem": 10717, + "\u0120sits": 10718, + "\u0120hoped": 10719, + "oting": 10720, + "\u0120dialogue": 10721, + "\u0120persu": 10722, + "Watch": 10723, + "lay": 10724, + "MAN": 10725, + "\u0120chronic": 10726, + "\u0120Portland": 10727, + "market": 10728, + "\u0120SEC": 10729, + "\u0120parallel": 10730, + "\u0120scandal": 10731, + "\u0120carries": 10732, + "\u0120phenomenon": 10733, + "human": 10734, + "acker": 10735, + "\u0120Ox": 10736, + "\u0120retirement": 10737, + "tainment": 10738, + "ovie": 10739, + "\u0120Gear": 10740, + "\u0120duties": 10741, + "\u0120dose": 10742, + "\u0120scroll": 10743, + "MB": 10744, + "inf": 10745, + "\u0120sauce": 10746, + "\u0120landscape": 10747, + "reddit": 10748, + "\u0120Championship": 10749, + "\u0120Reddit": 10750, + "alid": 10751, + "\u0120coin": 10752, + "\u0120overs": 10753, + "\u0120posting": 10754, + "about": 10755, + "\u0120fel": 10756, + "andy": 10757, + "\u0120bold": 10758, + "\u0120focusing": 10759, + "effect": 10760, + "GR": 10761, + "\u0120deemed": 10762, + "\u0120recommendations": 10763, + "\u0120stepped": 10764, + "\u0120voter": 10765, + "\u0120Deep": 10766, + "\u0120Instagram": 10767, + "\u0120moderate": 10768, + "\u0120Maryland": 10769, + "\u0120restricted": 10770, + "\u0120MB": 10771, + "\u0120Chall": 10772, + "\u0120tob": 10773, + "\u0120cir": 10774, + "\u0120Occ": 10775, + "\u0120Ever": 10776, + "\u0120collaps": 10777, + "INFO": 10778, + "=-": 10779, + "\u0120Pict": 10780, + "\u0120Account": 10781, + "nc": 10782, + "\u0120ought": 10783, + "\u0120export": 10784, + "\u0120drunk": 10785, + "('": 10786, + "\u0120wise": 10787, + "\u0120Mort": 10788, + "necess": 10789, + "\u0120ancest": 10790, + "\u0120Incre": 10791, + "\u0120frequent": 10792, + "mir": 10793, + "\u0120interpretation": 10794, + "\u0120dependent": 10795, + "\u0120coins": 10796, + "\u0120Bol": 10797, + "Video": 10798, + "\u0120Justin": 10799, + "\u0120fatal": 10800, + "\u0120cooking": 10801, + "\u0120confusion": 10802, + "ipher": 10803, + "\u0120custody": 10804, + "\u0120Morgan": 10805, + "omach": 10806, + "\u0120Governor": 10807, + "\u0120restaurants": 10808, + "eling": 10809, + "\u0120acknowledged": 10810, + "\u0120ther": 10811, + "\u0120genes": 10812, + "ching": 10813, + "Hey": 10814, + "\u0120tactics": 10815, + "\u0120Mexican": 10816, + "\u0120vend": 10817, + "\u0120hes": 10818, + "quer": 10819, + "\u0120noting": 10820, + "\u0120Cameron": 10821, + "\u0120targeting": 10822, + "rock": 10823, + "\u0120credits": 10824, + "\u0120emotions": 10825, + "\u0120representatives": 10826, + "news": 10827, + "\u0120legislative": 10828, + "\u0120removing": 10829, + "\u0120tweeted": 10830, + "\u0120Carter": 10831, + "\u0120Fixed": 10832, + "\u0120forcing": 10833, + "\u0120speaker": 10834, + "\u0120males": 10835, + "\u0120Vietnam": 10836, + "lined": 10837, + "\u0120concepts": 10838, + "\u0120voices": 10839, + "oir": 10840, + "\u0120Trib": 10841, + "Whe": 10842, + "\u0120Jerusalem": 10843, + "\u0120Sant": 10844, + "\u0120cul": 10845, + "\u0120lady": 10846, + "\u0120Hawai": 10847, + "\u0120arts": 10848, + "\u0120Inn": 10849, + "\u0120Machine": 10850, + "\u0120Emperor": 10851, + "\u0120slot": 10852, + "gly": 10853, + "\u0120Process": 10854, + "III": 10855, + "\u0120athletes": 10856, + "\u0120Temple": 10857, + "\u0120Represent": 10858, + "\u0120presc": 10859, + "\u0120tons": 10860, + "\u0120golden": 10861, + "\u0120punch": 10862, + "\u0120GR": 10863, + "iverpool": 10864, + "\u0120enact": 10865, + "\u0120lobby": 10866, + "\u0120mos": 10867, + "\u0120picking": 10868, + "\u0120lifetime": 10869, + "\u0120cognitive": 10870, + "Each": 10871, + "zo": 10872, + "\u0120dub": 10873, + "\u0120consists": 10874, + "oln": 10875, + "\u0120festival": 10876, + "amous": 10877, + "\u0120intellig": 10878, + "words": 10879, + "\u0120Smart": 10880, + "\u0120dele": 10881, + "\u0120lapt": 10882, + "\u0120magical": 10883, + "\u0120Sin": 10884, + "bus": 10885, + "urities": 10886, + "ighth": 10887, + "\u0120Ruby": 10888, + "\u0120Sure": 10889, + "olving": 10890, + "\u0120jun": 10891, + "OST": 10892, + "\u0120imposed": 10893, + "\u0120astron": 10894, + "\u0120correl": 10895, + "\u0120NS": 10896, + "\u0120Kit": 10897, + "\u0120Future": 10898, + "burn": 10899, + "\u0120immune": 10900, + "ocus": 10901, + "\u0120courses": 10902, + "\u0120String": 10903, + "\u0120lean": 10904, + "\u0120ghost": 10905, + "\u0120outcomes": 10906, + "\u0120expense": 10907, + "\u0120everyday": 10908, + "\u0120acceptable": 10909, + "Ah": 10910, + "\u0120equipped": 10911, + "\u0120orange": 10912, + "FR": 10913, + "\u0120Dutch": 10914, + "Though": 10915, + "\u0120Rank": 10916, + "QU": 10917, + "\u0120Roberts": 10918, + "what": 10919, + "rend": 10920, + "\u0120disappear": 10921, + "\u0120spawn": 10922, + "\u0120Lam": 10923, + "ois": 10924, + "\u0120deserve": 10925, + "\u0120minimal": 10926, + "\u0120nervous": 10927, + "\u0120Would": 10928, + "\u0120rook": 10929, + "\u0120Vancouver": 10930, + "\u0120resign": 10931, + "shire": 10932, + "\u0120Works": 10933, + "\u0120Build": 10934, + "\u0120affordable": 10935, + "\u0120Gary": 10936, + "\u0120Arena": 10937, + "\u0120hanging": 10938, + "\u0120implications": 10939, + "\u0120Song": 10940, + "\u0120maintaining": 10941, + "\u0120guards": 10942, + "CON": 10943, + "\u0120derived": 10944, + "\u0120executed": 10945, + "\u0120theories": 10946, + "\u0120quoted": 10947, + "\u0120Andre": 10948, + "oga": 10949, + "seless": 10950, + "info": 10951, + "\u0120Belg": 10952, + "\u0120tears": 10953, + "\u0120Surv": 10954, + "\u0120birthday": 10955, + "igious": 10956, + "immer": 10957, + "\u0120spectrum": 10958, + "\u0120architecture": 10959, + "\u0120recruit": 10960, + "arma": 10961, + "Table": 10962, + "\u0120monsters": 10963, + "\u0120Gov": 10964, + "\u0120destination": 10965, + "\u0120attractive": 10966, + "\u0120foss": 10967, + "\u0120Moreover": 10968, + "\u0120presents": 10969, + "THE": 10970, + "\u0120reply": 10971, + "pton": 10972, + "\u0120cum": 10973, + "\u0120delight": 10974, + "\u0120affects": 10975, + "\u0120donations": 10976, + "\u0120Toy": 10977, + "\u0120Him": 10978, + "MENT": 10979, + "\u0120overcome": 10980, + "itched": 10981, + "\u0120Fantasy": 10982, + "\u0120Hat": 10983, + "\u0120Beast": 10984, + "bott": 10985, + "\u0120investigations": 10986, + "Run": 10987, + "\u0120hunting": 10988, + "di": 10989, + "fund": 10990, + "\u0120sessions": 10991, + "estyle": 10992, + "\u0120portray": 10993, + "oids": 10994, + "Yeah": 10995, + "\u0120communicate": 10996, + "\u0120comedy": 10997, + "\u0120Yang": 10998, + "\u0120belt": 10999, + "\u0120Marine": 11000, + "\u0120predicted": 11001, + "Play": 11002, + "\u0120importantly": 11003, + "\u0120remarkable": 11004, + "\u0120eliminate": 11005, + "David": 11006, + "\u0120bind": 11007, + "VID": 11008, + "\u0120advocates": 11009, + "\u0120Gaza": 11010, + "imp": 11011, + "DB": 11012, + "\u0120Na": 11013, + "\u0120Similar": 11014, + "IES": 11015, + "\u0120charity": 11016, + "vas": 11017, + "math": 11018, + "\u0120\u00e2\u0138": 11019, + "oker": 11020, + "ndum": 11021, + "\u0120caps": 11022, + "\u0120Hal": 11023, + "2000": 11024, + "ean": 11025, + "\u0120fleet": 11026, + "\u0120recre": 11027, + "Right": 11028, + "\u0120sleeping": 11029, + "ijing": 11030, + "kind": 11031, + "\u0120designated": 11032, + "\u00c3\u00a4": 11033, + "\u0120animation": 11034, + "kee": 11035, + "\u0120Introdu": 11036, + "\u0120/>": 11037, + "\u0120delayed": 11038, + "\u0120tremend": 11039, + "\u0120curious": 11040, + "Use": 11041, + "\u0120lect": 11042, + "dam": 11043, + "\u0120innovation": 11044, + "\u0120Points": 11045, + "\u0120loading": 11046, + "\u0120dispute": 11047, + "ctic": 11048, + "irds": 11049, + "\u0120BY": 11050, + "\u0120nurs": 11051, + "\u0120Value": 11052, + "IONS": 11053, + "\u0120Hum": 11054, + "\u0120template": 11055, + "mers": 11056, + "\u0120appearances": 11057, + "\u0120Entertainment": 11058, + "\u0120translation": 11059, + "\u0120sake": 11060, + "\u0120beneath": 11061, + "\u0120inhib": 11062, + "\u0120euro": 11063, + "abetes": 11064, + "\u0120studying": 11065, + "\u0120Mas": 11066, + "\u0120perceived": 11067, + "\u0120examined": 11068, + "\u0120eager": 11069, + "\u0120coaches": 11070, + "\u0120imper": 11071, + "chi": 11072, + "\u0120produces": 11073, + "\").": 11074, + "\u0120Everyone": 11075, + "\u0120municip": 11076, + "\u0120girlfriend": 11077, + "\u0120hire": 11078, + "\u0120Vice": 11079, + "\u0120suitable": 11080, + "opy": 11081, + "\u0120inequ": 11082, + "\u0120Duke": 11083, + "fish": 11084, + "first": 11085, + "\u0120Obs": 11086, + "\u0120interior": 11087, + "\u0120Bruce": 11088, + "\u0120Ry": 11089, + "\u0120analys": 11090, + "\u0120considerable": 11091, + "\u0120forecast": 11092, + "\u0120fert": 11093, + "orship": 11094, + "\u0120Drug": 11095, + "\u0120ALL": 11096, + ":\"": 11097, + "thur": 11098, + "\u0120Mail": 11099, + "\u0120ballot": 11100, + "\u0120instantly": 11101, + "\u0120Channel": 11102, + "\u0120picks": 11103, + "\u01201989": 11104, + "\u0120tent": 11105, + "oli": 11106, + "\u0120civilian": 11107, + "bling": 11108, + "ello": 11109, + "bu": 11110, + "\u0120inch": 11111, + "\u0120logo": 11112, + "\u0120cooperation": 11113, + "\u0120walks": 11114, + "\u0120investments": 11115, + "\u0120imprison": 11116, + "\u0120Festival": 11117, + "\u0120Ky": 11118, + "\u0120legally": 11119, + "\u0120gri": 11120, + "charg": 11121, + "Sl": 11122, + "\u0120threatening": 11123, + "duction": 11124, + "flow": 11125, + "\u0120dismissed": 11126, + "ibraries": 11127, + "cap": 11128, + "ele": 11129, + "\u0120McG": 11130, + "\u0120Harvard": 11131, + "\u0120Conservative": 11132, + "\u0120CBS": 11133, + "png": 11134, + "\u0120roots": 11135, + "\u0120Having": 11136, + "umbled": 11137, + "\u0120Fun": 11138, + "\\/": 11139, + "\u0120Search": 11140, + "plex": 11141, + "\u0120discussing": 11142, + "\u0120continu": 11143, + "\u0120Tai": 11144, + "\u0120Wik": 11145, + "Free": 11146, + "fit": 11147, + "\u0120refuse": 11148, + "\u0120managing": 11149, + "\u0120synd": 11150, + "ipedia": 11151, + "walk": 11152, + "\u0120professionals": 11153, + "\u0120guidance": 11154, + "\u0120universities": 11155, + "\u0120assemb": 11156, + "untu": 11157, + "Finally": 11158, + "ASE": 11159, + "\u0120Auto": 11160, + "\u0120Had": 11161, + "\u0120anniversary": 11162, + "LD": 11163, + "\u0120Dur": 11164, + "\u0120Ultimate": 11165, + "ihad": 11166, + "product": 11167, + "\u0120transit": 11168, + "\u0120restore": 11169, + "\u0120explaining": 11170, + "\u0120asset": 11171, + "\u0120transferred": 11172, + "\u0120burst": 11173, + "apolis": 11174, + "\u0120Magazine": 11175, + "\u0120Cra": 11176, + "\u0120BR": 11177, + "gged": 11178, + "\u0120HE": 11179, + "Mich": 11180, + "bet": 11181, + "\u0120Lady": 11182, + "ylum": 11183, + "erves": 11184, + "\u0120meets": 11185, + "white": 11186, + "Log": 11187, + "\u0120corresponding": 11188, + "\u0120insisted": 11189, + "GG": 11190, + "\u0120surrounded": 11191, + "\u0120tens": 11192, + "\u0120lane": 11193, + "\u0120coinc": 11194, + "home": 11195, + "\u0120existed": 11196, + "ected": 11197, + "\u0120Double": 11198, + "lamm": 11199, + "\u0120skept": 11200, + "exp": 11201, + "\u0120perception": 11202, + "iev": 11203, + "\u0120Being": 11204, + "oft": 11205, + "\u0120adopt": 11206, + ".:": 11207, + "];": 11208, + "Windows": 11209, + "\u0120satellite": 11210, + "ASH": 11211, + "\u0120infant": 11212, + "description": 11213, + "\u0120Meanwhile": 11214, + "cm": 11215, + "oca": 11216, + "\u0120Treat": 11217, + "actor": 11218, + "\u0120tobacco": 11219, + "\u0120Norm": 11220, + "emption": 11221, + "\u0120flesh": 11222, + "\u0120je": 11223, + "oop": 11224, + "\u0120Heaven": 11225, + "\u0120beating": 11226, + "anim": 11227, + "\u0120gathering": 11228, + "\u0120cultiv": 11229, + "GO": 11230, + "abe": 11231, + "\u0120Jonathan": 11232, + "\u0120Safety": 11233, + "\u0120badly": 11234, + "prot": 11235, + "\u0120choosing": 11236, + "\u0120contacted": 11237, + "\u0120quit": 11238, + "\u0120distur": 11239, + "\u0120stir": 11240, + "\u0120token": 11241, + "Det": 11242, + "\u0120Pa": 11243, + "\u0120functionality": 11244, + "003": 11245, + "some": 11246, + "\u0120limitations": 11247, + "\u0120meth": 11248, + "build": 11249, + "config": 11250, + "NT": 11251, + "rell": 11252, + "blem": 11253, + "\u0120Mom": 11254, + "\u0120veterans": 11255, + "\u0120Hu": 11256, + "\u0120trends": 11257, + "arer": 11258, + "\u0120Given": 11259, + "\u0120Caption": 11260, + "may": 11261, + "AST": 11262, + "\u0120wondering": 11263, + "\u0120Clark": 11264, + "normal": 11265, + "\u0120separated": 11266, + "\u0120desp": 11267, + "stic": 11268, + "brew": 11269, + "\u0120relating": 11270, + "\u0120Nik": 11271, + "\u0120Farm": 11272, + "\u0120enthusi": 11273, + "good": 11274, + "deb": 11275, + "\u0120activist": 11276, + "\u0120mart": 11277, + "\u0120explosion": 11278, + "\u0120Economic": 11279, + "Link": 11280, + "\u0120insight": 11281, + "\u0120convenient": 11282, + "\u0120counterpart": 11283, + "support": 11284, + "\u0120Virt": 11285, + "agen": 11286, + "\u0120Tennessee": 11287, + "\u0120Simon": 11288, + "\u0120Award": 11289, + "OCK": 11290, + "\u0120Figure": 11291, + "\u0120overseas": 11292, + "\u0120pride": 11293, + "\u0120Cas": 11294, + "note": 11295, + "mg": 11296, + "Current": 11297, + "\u0120displays": 11298, + "content": 11299, + "\u0120traveling": 11300, + "\u0120hospitals": 11301, + "\u0120Financial": 11302, + "\u0120Past": 11303, + "\u0120defendant": 11304, + "\u0120streaming": 11305, + "mble": 11306, + "\u0120Berlin": 11307, + "uki": 11308, + "\u0120distribut": 11309, + "\u0120antib": 11310, + "\u0120chocolate": 11311, + "\u0120Castle": 11312, + "\u0120interrupt": 11313, + "\u0120Row": 11314, + "\u0120conversion": 11315, + "\u0120bugs": 11316, + "\u0120Rather": 11317, + "liest": 11318, + "LY": 11319, + "\u0120Jean": 11320, + "common": 11321, + "akh": 11322, + "\u0120130": 11323, + "otton": 11324, + "\u0120Dean": 11325, + "\u0120amendment": 11326, + "\u0120gameplay": 11327, + "\u0120Warren": 11328, + "oda": 11329, + "\u0120highlights": 11330, + "\u0120irre": 11331, + "\u0120NATO": 11332, + "\u0120balls": 11333, + "\u0120demanding": 11334, + "URE": 11335, + "\u0120Luke": 11336, + "Figure": 11337, + "stop": 11338, + "onia": 11339, + "zone": 11340, + "izers": 11341, + "\u0120WR": 11342, + "\u0120awarded": 11343, + "\u0120regulatory": 11344, + "\u0120Hart": 11345, + "\u0120SN": 11346, + "pling": 11347, + "\u0120sour": 11348, + "\u0120Pixel": 11349, + "usive": 11350, + "\u0120fet": 11351, + "\u0120Sent": 11352, + "\u0120automatic": 11353, + "\u0120fer": 11354, + "vernment": 11355, + "\u0120Khan": 11356, + "TON": 11357, + "father": 11358, + "\u0120extraordinary": 11359, + "throp": 11360, + "\u0120Python": 11361, + "\u0120GPU": 11362, + "\u0120sexually": 11363, + "\u0120desktop": 11364, + "itivity": 11365, + "\u0120Antonio": 11366, + "\u0120orient": 11367, + "\u0120ears": 11368, + "obby": 11369, + "ouses": 11370, + "vertisements": 11371, + "\u0120manufacturers": 11372, + "icient": 11373, + "minute": 11374, + "\u0120conviction": 11375, + "\u0120garden": 11376, + "public": 11377, + "\u0120satisfied": 11378, + "fold": 11379, + "OK": 11380, + "\u0120inhab": 11381, + "\u0120Think": 11382, + "\u0120programme": 11383, + "\u0120stomach": 11384, + "\u0120coordin": 11385, + "\u0120holy": 11386, + "\u0120threshold": 11387, + "\u0120rhet": 11388, + "\u0120serial": 11389, + "\u0120employers": 11390, + "\u0120Everything": 11391, + "rah": 11392, + "\u0120bother": 11393, + "\u0120brands": 11394, + "Value": 11395, + "\u0120Ted": 11396, + "\u0120Planet": 11397, + "\u0120pink": 11398, + "\u0120Furthermore": 11399, + "sa": 11400, + "PE": 11401, + "reck": 11402, + "\u0120USD": 11403, + "otte": 11404, + "\u0120&&": 11405, + "\u0120landed": 11406, + "gets": 11407, + "\u0120producers": 11408, + "\u0120healthcare": 11409, + "\u0120dominant": 11410, + "\u0120destro": 11411, + "\u0120amended": 11412, + "chron": 11413, + "\u0120fits": 11414, + "\u0120Syd": 11415, + "\u0120Authority": 11416, + "ATCH": 11417, + "\u0120fights": 11418, + "\u0120LLC": 11419, + "\u0120---": 11420, + "\u0120Corp": 11421, + "\u0120toxic": 11422, + "specific": 11423, + "\u0120Corn": 11424, + "\u0120Chel": 11425, + "\u0120telephone": 11426, + "\u0120Pant": 11427, + "\u0120mysterious": 11428, + "aunch": 11429, + "odox": 11430, + "media": 11431, + "\u0120witnesses": 11432, + "agu": 11433, + "\u0120questioned": 11434, + "\u0120Brexit": 11435, + "\u0120Remember": 11436, + "enez": 11437, + "\u0120endorse": 11438, + "iatric": 11439, + "\u0120Ident": 11440, + "\u0120ridiculous": 11441, + "110": 11442, + "\u0120prayer": 11443, + "\u0120scientist": 11444, + "\u01201950": 11445, + "\u0120Aqu": 11446, + "\u0120underground": 11447, + "\u0120UFC": 11448, + "mare": 11449, + "\u0120Later": 11450, + "wich": 11451, + "\u0120subscrib": 11452, + "\u0120hosts": 11453, + "\u0120err": 11454, + "\u0120grants": 11455, + "antom": 11456, + "\u0120summon": 11457, + "early": 11458, + "\u0120Clear": 11459, + "\u0120Prim": 11460, + "\u0120suspension": 11461, + "\u0120guaranteed": 11462, + "apper": 11463, + "\u0120rice": 11464, + "\u0120Sean": 11465, + "\u0120Shin": 11466, + "\u0120referendum": 11467, + "\u0120fled": 11468, + "rust": 11469, + "\u0120360": 11470, + "tery": 11471, + "\u0120shocked": 11472, + "BR": 11473, + "\u0120Oil": 11474, + "\u0120Allah": 11475, + "\u0120partly": 11476, + "\u0120ignor": 11477, + "\u0120transmission": 11478, + "\u0120homosexual": 11479, + "iversal": 11480, + "\u0120hopefully": 11481, + "\u00e3\u0124\u00a4": 11482, + "\u0120lesson": 11483, + "Leg": 11484, + "\u0120..": 11485, + "Yet": 11486, + "table": 11487, + "appropri": 11488, + "rett": 11489, + "\u0120boards": 11490, + "\u0120incorrect": 11491, + "\u0120bacteria": 11492, + "aru": 11493, + "amac": 11494, + "\u0120snap": 11495, + ".'\"": 11496, + "\u0120parad": 11497, + "tem": 11498, + "heart": 11499, + "\u0120availability": 11500, + "\u0120wisdom": 11501, + "\u0120(+": 11502, + "\u0120priest": 11503, + "\u0120\u00c2\u0142\u0120\u00c2\u0142": 11504, + "Open": 11505, + "\u0120span": 11506, + "\u0120parameter": 11507, + "\u0120convince": 11508, + "\u0120(%)": 11509, + "rac": 11510, + "\u0120fo": 11511, + "\u0120safely": 11512, + "\u0120converted": 11513, + "\u0120Olympic": 11514, + "\u0120reserve": 11515, + "\u0120healing": 11516, + "\u0120Mine": 11517, + "Max": 11518, + "\u0120inherent": 11519, + "\u0120Graham": 11520, + "\u0120integrated": 11521, + "Dem": 11522, + "\u0120pipeline": 11523, + "\u0120applying": 11524, + "\u0120embed": 11525, + "\u0120Charlie": 11526, + "\u0120cave": 11527, + "2008": 11528, + "\u0120consensus": 11529, + "\u0120rewards": 11530, + "Pal": 11531, + "\u0120HTML": 11532, + "\u0120popularity": 11533, + "looking": 11534, + "\u0120Sword": 11535, + "\u0120Arts": 11536, + "')": 11537, + "\u0120electron": 11538, + "clusions": 11539, + "\u0120integrity": 11540, + "\u0120exclusively": 11541, + "\u0120grace": 11542, + "\u0120torture": 11543, + "\u0120burned": 11544, + "two": 11545, + "\u0120180": 11546, + "Produ": 11547, + "\u0120entreprene": 11548, + "raphics": 11549, + "\u0120gym": 11550, + "ricane": 11551, + "\u0120Tam": 11552, + "\u0120administrative": 11553, + "\u0120manufacturer": 11554, + "\u0120vel": 11555, + "\u0120Ni": 11556, + "\u0120isolated": 11557, + "\u0120Medicine": 11558, + "\u0120backup": 11559, + "\u0120promoting": 11560, + "\u0120commander": 11561, + "\u0120flee": 11562, + "\u0120Russell": 11563, + "\u0120forgotten": 11564, + "\u0120Missouri": 11565, + "\u0120residence": 11566, + "mons": 11567, + "\u0120resemb": 11568, + "\u0120wand": 11569, + "\u0120meaningful": 11570, + "PT": 11571, + "\u0120bol": 11572, + "\u0120helic": 11573, + "\u0120wealthy": 11574, + "\u0120rifle": 11575, + "strong": 11576, + "rowing": 11577, + "plan": 11578, + "asury": 11579, + "\u00e2\u0122\u00a6.": 11580, + "\u0120expanding": 11581, + "\u0120Hamilton": 11582, + "\u0120receives": 11583, + "SI": 11584, + "eatures": 11585, + "\u0120Anim": 11586, + "REE": 11587, + "Put": 11588, + "\u0120briefly": 11589, + "rive": 11590, + "\u0120stimul": 11591, + "\u0120``(": 11592, + "\u0120__": 11593, + "\u0120chip": 11594, + "\u0120haz": 11595, + "\u0120prize": 11596, + "\u0120Things": 11597, + "ACE": 11598, + "ulin": 11599, + "dict": 11600, + "oku": 11601, + "\u0120associate": 11602, + "ockets": 11603, + "youtube": 11604, + "Story": 11605, + "ategory": 11606, + "\u0120mild": 11607, + "ailing": 11608, + "\u0120Ye": 11609, + "Orig": 11610, + "\u0120Ka": 11611, + "orig": 11612, + "\u0120propaganda": 11613, + "\u0120anonymous": 11614, + "\u0120struggled": 11615, + "\u0120outrage": 11616, + "ATED": 11617, + "\u0120Beijing": 11618, + "rary": 11619, + "\u0120leather": 11620, + "\u0120worlds": 11621, + "\u0120broader": 11622, + "125": 11623, + "idal": 11624, + "\u0120Better": 11625, + "\u0120tear": 11626, + "Ext": 11627, + "\u0120proposals": 11628, + "\u0120iter": 11629, + "\u0120Squad": 11630, + "\u0120volunt": 11631, + "mi": 11632, + "Did": 11633, + "\u0120Pu": 11634, + "pin": 11635, + "\u0120speakers": 11636, + "\u0120borders": 11637, + "\u0120figured": 11638, + "='": 11639, + "\u0120simultaneously": 11640, + "aeda": 11641, + "\u0120charging": 11642, + "\u0120urged": 11643, + "\u0120conj": 11644, + "256": 11645, + "\u0120Gordon": 11646, + "merce": 11647, + "\u0120documentary": 11648, + "Share": 11649, + "itol": 11650, + "ONE": 11651, + "\u0120Garden": 11652, + "hatt": 11653, + "\u0120Thompson": 11654, + "aneous": 11655, + "apore": 11656, + "\u0120tanks": 11657, + "\u0120lessons": 11658, + "track": 11659, + "\u0120outstanding": 11660, + "\u0120volunteers": 11661, + "\u0120spray": 11662, + "\u0120managers": 11663, + "large": 11664, + "\u0120camps": 11665, + "\u0120artificial": 11666, + "\u0120Ru": 11667, + "\u0120bags": 11668, + "thal": 11669, + "\u0120compatible": 11670, + "\u0120Blade": 11671, + "\u0120fed": 11672, + "\u0120argues": 11673, + "FI": 11674, + "\u0120unfair": 11675, + "\u0120corn": 11676, + "\u0120offset": 11677, + "\u0120directions": 11678, + "\u0120disappointed": 11679, + "\u0120Convention": 11680, + "\u0120viewing": 11681, + "ME": 11682, + "ocity": 11683, + "\u0120towns": 11684, + "\u0120layers": 11685, + "\u0120rolled": 11686, + "\u0120jumped": 11687, + "\u0120attribute": 11688, + "\u0120unnecess": 11689, + "incoln": 11690, + "\u0120suppose": 11691, + "\u0120Nether": 11692, + "cha": 11693, + "\u0120buried": 11694, + "\u0120sixth": 11695, + "Ben": 11696, + "ressing": 11697, + "OUR": 11698, + "\u0120wound": 11699, + "\u0120cycl": 11700, + "\u0120mechanisms": 11701, + "\u0120congressional": 11702, + "\u0120Element": 11703, + "\u0120agreements": 11704, + "\u0120decor": 11705, + "\u0120closest": 11706, + "\u0120Mit": 11707, + "Google": 11708, + "}}": 11709, + "\u0120mixture": 11710, + "\u0120fluid": 11711, + "Sign": 11712, + "\u0120Scholar": 11713, + "\u0120pist": 11714, + "asket": 11715, + "abling": 11716, + "\u0120racing": 11717, + "hero": 11718, + "riel": 11719, + "assy": 11720, + "\u0120cheaper": 11721, + "ben": 11722, + "\u0120vertical": 11723, + "amacare": 11724, + "\u0120Reading": 11725, + "gments": 11726, + "\u0120helicop": 11727, + "\u0120sacrifice": 11728, + "aya": 11729, + "paren": 11730, + "VA": 11731, + "\u0120Les": 11732, + "\u0120Studio": 11733, + "\u0120violations": 11734, + "\u0120Anna": 11735, + "acer": 11736, + "\u00e9\u00be": 11737, + "\u0120Rat": 11738, + "\u0120Beck": 11739, + "\u0120Dick": 11740, + "\u0120ACT": 11741, + "\u0120composition": 11742, + "\u0120texture": 11743, + "\u0120Own": 11744, + "\u0120smartphone": 11745, + "\u0120NA": 11746, + "\u0120forb": 11747, + "import": 11748, + "\u0120defending": 11749, + "ilst": 11750, + "rer": 11751, + "\u0120oh": 11752, + "\u0120Jeremy": 11753, + "\u0120banking": 11754, + "ceptions": 11755, + "\u0120respective": 11756, + "/.": 11757, + "\u0120drinks": 11758, + "\u0120Wi": 11759, + "\u0120bands": 11760, + "\u0120Liverpool": 11761, + "\u0120grip": 11762, + "\u0120Buy": 11763, + "\u0120openly": 11764, + "\u0120reviewed": 11765, + "pert": 11766, + "\u0120verify": 11767, + "\u0120Cole": 11768, + "\u0120Wales": 11769, + "MO": 11770, + "\u0120unpre": 11771, + "\u0120shelter": 11772, + "\u0120Imperial": 11773, + "\u0120gui": 11774, + "\u0120Dak": 11775, + "\u0120suggestions": 11776, + "\u0120explicitly": 11777, + "\u0120slave": 11778, + "\u0120blockchain": 11779, + "\u0120competing": 11780, + "\u0120promising": 11781, + "SON": 11782, + "\u0120soccer": 11783, + "\u0120constitution": 11784, + "429": 11785, + "\u0120distract": 11786, + "\u0120User": 11787, + "esides": 11788, + "\u0120Method": 11789, + "\u0120Tokyo": 11790, + "\u0120accompanied": 11791, + "Client": 11792, + "sur": 11793, + "alog": 11794, + "\u0120identification": 11795, + "\u0120invasion": 11796, + "asma": 11797, + "\u0120industries": 11798, + "ppers": 11799, + "\u0120subtle": 11800, + "\u0120Unit": 11801, + "natural": 11802, + "\u0120survived": 11803, + "\u0120flaw": 11804, + "\u013a\u0127": 11805, + "\u0120Holl": 11806, + "\u0120deficit": 11807, + "\u0120tutorial": 11808, + "\u0120Chance": 11809, + "\u0120arguing": 11810, + "\u0120contemporary": 11811, + "\u0120integration": 11812, + "forward": 11813, + "\u0120tum": 11814, + "itis": 11815, + "\u0120hiding": 11816, + "\u0120Domin": 11817, + "\u0120Tan": 11818, + "\u0120Building": 11819, + "\u0120Vin": 11820, + "\u0120spokesperson": 11821, + "\u0120Notes": 11822, + "\u0120emerging": 11823, + "\u0120preparation": 11824, + "\u0120prost": 11825, + "\u0120suspects": 11826, + "\u0120autonom": 11827, + "Description": 11828, + "\u0120dealt": 11829, + "\u0120Pear": 11830, + "\u0120steady": 11831, + "\u0120decreased": 11832, + "\u0120sovere": 11833, + "\u0120Clin": 11834, + "\u0120gradually": 11835, + "orses": 11836, + "\u0120WAR": 11837, + "Serv": 11838, + "\u00e3\u0124\u00a2": 11839, + "hr": 11840, + "\u0120dirty": 11841, + "\u0120Barn": 11842, + "\u0120BC": 11843, + "\u0120dil": 11844, + "\u0120calendar": 11845, + "\u0120compliance": 11846, + "\u0120chamber": 11847, + "bb": 11848, + "\u0120passenger": 11849, + "ateful": 11850, + "\u0120Title": 11851, + "\u0120Sydney": 11852, + "\u0120Got": 11853, + "\u0120darkness": 11854, + "\u0120defect": 11855, + "\u0120packed": 11856, + "assion": 11857, + "\u0120gods": 11858, + "\u0120harsh": 11859, + "ICK": 11860, + "leans": 11861, + "\u0120algorithm": 11862, + "\u0120oxygen": 11863, + "\u0120visits": 11864, + "\u0120blade": 11865, + "\u0120kilomet": 11866, + "\u0120Kentucky": 11867, + "\u0120killer": 11868, + "Pack": 11869, + "enny": 11870, + "\u0120divine": 11871, + "\u0120nomination": 11872, + "being": 11873, + "\u0120engines": 11874, + "\u0120cats": 11875, + "\u0120buffer": 11876, + "\u0120Phill": 11877, + "\u0120traff": 11878, + "AGE": 11879, + "\u0120tongue": 11880, + "\u0120radiation": 11881, + "erer": 11882, + "mem": 11883, + "\u0120Explicit": 11884, + "\u00e9\u00be\u012f": 11885, + "\u0120couples": 11886, + "\u0120physics": 11887, + "\u0120McK": 11888, + "\u0120politically": 11889, + "awks": 11890, + "\u0120Bloom": 11891, + "\u0120worship": 11892, + "eger": 11893, + "uter": 11894, + "\u0120FO": 11895, + "\u0120mathemat": 11896, + "\u0120sentenced": 11897, + "\u0120disk": 11898, + "\u0120Marg": 11899, + "\u0120/*": 11900, + "PI": 11901, + "\u0120optional": 11902, + "\u0120babies": 11903, + "\u0120seeds": 11904, + "\u0120Scottish": 11905, + "\u0120thy": 11906, + "]]": 11907, + "\u0120Hitler": 11908, + "PH": 11909, + "ngth": 11910, + "\u0120recovered": 11911, + "inge": 11912, + "\u0120powder": 11913, + "\u0120lips": 11914, + "\u0120designer": 11915, + "\u0120disorders": 11916, + "\u0120courage": 11917, + "\u0120chaos": 11918, + "\"},{\"": 11919, + "\u0120carrier": 11920, + "bably": 11921, + "High": 11922, + "\u0120RT": 11923, + "esity": 11924, + "len": 11925, + "\u0120routes": 11926, + "uating": 11927, + "Fil": 11928, + "NOT": 11929, + "wall": 11930, + "sburgh": 11931, + "\u0120engaging": 11932, + "\u0120JavaScript": 11933, + "orer": 11934, + "lihood": 11935, + "\u0120unions": 11936, + "\u0120Federation": 11937, + "\u0120Tesla": 11938, + "\u0120completion": 11939, + "\u0120Ta": 11940, + "\u0120privilege": 11941, + "\u0120Orange": 11942, + "\u0120neur": 11943, + "parency": 11944, + "\u0120bones": 11945, + "\u0120titled": 11946, + "\u0120prosecutors": 11947, + "\u0120ME": 11948, + "\u0120engineer": 11949, + "\u0120Universe": 11950, + "\u0120Hig": 11951, + "nie": 11952, + "oard": 11953, + "\u0120hearts": 11954, + "\u0120Gre": 11955, + "ussion": 11956, + "\u0120ministry": 11957, + "\u0120penet": 11958, + "\u0120Nut": 11959, + "\u0120Ow": 11960, + "\u0120XP": 11961, + "instein": 11962, + "\u0120bulk": 11963, + "System": 11964, + "icism": 11965, + "\u0120Marketable": 11966, + "\u0120preval": 11967, + "\u0120poster": 11968, + "\u0120attending": 11969, + "urable": 11970, + "\u0120licensed": 11971, + "\u0120Gh": 11972, + "etry": 11973, + "\u0120Tradable": 11974, + "\u0120blast": 11975, + "\u00e0\u00a4": 11976, + "\u0120Titan": 11977, + "elled": 11978, + "die": 11979, + "Have": 11980, + "\u0120Flame": 11981, + "\u0120profound": 11982, + "\u0120participating": 11983, + "\u0120anime": 11984, + "\u0120Ess": 11985, + "\u0120specify": 11986, + "\u0120regarded": 11987, + "\u0120Spell": 11988, + "\u0120sons": 11989, + "owned": 11990, + "\u0120merc": 11991, + "\u0120experimental": 11992, + "lando": 11993, + "hs": 11994, + "\u0120Dungeon": 11995, + "inos": 11996, + "\u0120comply": 11997, + "\u0120Systems": 11998, + "arth": 11999, + "\u0120seized": 12000, + "local": 12001, + "\u0120Girls": 12002, + "udo": 12003, + "oned": 12004, + "\u0120Fle": 12005, + "\u0120constructed": 12006, + "\u0120hosted": 12007, + "\u0120scared": 12008, + "actic": 12009, + "\u0120Islands": 12010, + "\u0120MORE": 12011, + "\u0120bless": 12012, + "\u0120blocking": 12013, + "\u0120chips": 12014, + "\u0120evac": 12015, + "Ps": 12016, + "\u0120corporation": 12017, + "\u0120ox": 12018, + "\u0120lighting": 12019, + "\u0120neighbors": 12020, + "\u0120Ub": 12021, + "aro": 12022, + "\u0120beef": 12023, + "\u0120Uber": 12024, + "Facebook": 12025, + "armed": 12026, + "itate": 12027, + "\u0120Rating": 12028, + "\u0120Quick": 12029, + "\u0120occupied": 12030, + "\u0120aims": 12031, + "\u0120Additionally": 12032, + "\u0120Interest": 12033, + "\u0120dramatically": 12034, + "\u0120heal": 12035, + "\u0120painting": 12036, + "\u0120engineers": 12037, + "MM": 12038, + "\u0120Must": 12039, + "\u0120quantity": 12040, + "Paul": 12041, + "\u0120earnings": 12042, + "\u0120Posts": 12043, + "stra": 12044, + "\u00e3\u0125\u00bc\u00e3\u0125": 12045, + "\u0120stance": 12046, + "\u0120dropping": 12047, + "script": 12048, + "\u0120dressed": 12049, + "Make": 12050, + "\u0120justify": 12051, + "\u0120Ltd": 12052, + "\u0120prompted": 12053, + "\u0120scrut": 12054, + "\u0120speeds": 12055, + "\u0120Giants": 12056, + "omer": 12057, + "\u0120Editor": 12058, + "\u0120describing": 12059, + "\u0120Lie": 12060, + "mented": 12061, + "\u0120nowhere": 12062, + "ocaly": 12063, + "\u0120instruction": 12064, + "fortable": 12065, + "\u0120entities": 12066, + "\u0120cm": 12067, + "\u0120Natural": 12068, + "\u0120inquiry": 12069, + "\u0120pressed": 12070, + "izont": 12071, + "forced": 12072, + "\u0120raises": 12073, + "\u0120Netflix": 12074, + "\u0120Side": 12075, + "\u0120outer": 12076, + "\u0120amongst": 12077, + "ims": 12078, + "owski": 12079, + "\u0120climb": 12080, + "never": 12081, + "\u0120combine": 12082, + "ding": 12083, + "\u0120compr": 12084, + "\u0120significance": 12085, + "\u0120remembered": 12086, + "\u0120Nevada": 12087, + "\u0120Tel": 12088, + "\u0120Scar": 12089, + "\u0120Warriors": 12090, + "\u0120Jane": 12091, + "\u0120coup": 12092, + "bas": 12093, + "\u0120terminal": 12094, + ",-": 12095, + "OH": 12096, + "\u0120tension": 12097, + "\u0120wings": 12098, + "\u0120Myster": 12099, + "\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 12100, + "\u0120Unlike": 12101, + "valid": 12102, + "vironments": 12103, + "\u0120Ali": 12104, + "\u0120naked": 12105, + "books": 12106, + "\u0120Mun": 12107, + "\u0120Gulf": 12108, + "\u0120density": 12109, + "\u0120dimin": 12110, + "\u0120desperate": 12111, + "\u0120presidency": 12112, + "\u01201986": 12113, + "hy": 12114, + "IND": 12115, + "\u0120unlock": 12116, + "imens": 12117, + "\u0120handled": 12118, + "\u0120Eb": 12119, + "\u0120disappeared": 12120, + "\u0120genre": 12121, + "\u01201988": 12122, + "\u0120determination": 12123, + "Stream": 12124, + "iko": 12125, + "apters": 12126, + "\u0120acknowledge": 12127, + "Jan": 12128, + "\u0120capitalism": 12129, + "Pat": 12130, + "\u01202020": 12131, + "\u0120painful": 12132, + "\u0120curve": 12133, + "\u0120bombs": 12134, + "storm": 12135, + "\u0120Metal": 12136, + "encer": 12137, + "\u0120Fig": 12138, + "\u0120Aaron": 12139, + "anches": 12140, + "\u0120inspiration": 12141, + "\u0120exhaust": 12142, + "tains": 12143, + "ashi": 12144, + "\u0120descript": 12145, + "\u0120ritual": 12146, + "\u0120Chelsea": 12147, + "\u0120promotion": 12148, + "\u0120Hung": 12149, + "\u0120Ward": 12150, + "iva": 12151, + "\u0120ET": 12152, + "\u0120toss": 12153, + "allow": 12154, + "\u0120Francis": 12155, + "Dep": 12156, + "\u0120happiness": 12157, + "\u0120Glass": 12158, + "\u0120beta": 12159, + "\u0120strengthen": 12160, + "NE": 12161, + "oa": 12162, + "\u0120buttons": 12163, + "\u0120Murray": 12164, + "\u0120kicked": 12165, + "Quest": 12166, + "\u0120Talk": 12167, + "\u0120Several": 12168, + "\u0120Zero": 12169, + "\u0120drone": 12170, + "ulk": 12171, + "\u0120cam": 12172, + "\u0120Mobile": 12173, + "\u0120preventing": 12174, + "\u0120retro": 12175, + "\u0120Ax": 12176, + "\u0120cruel": 12177, + "\u0120float": 12178, + ".),": 12179, + "\u0120filing": 12180, + "\u0120Grant": 12181, + "\u0120Bor": 12182, + "\u0120rib": 12183, + "\u0120championship": 12184, + "\u0120Merc": 12185, + "\u0120styles": 12186, + "\u0120cake": 12187, + "\u0120builds": 12188, + "\u0120Self": 12189, + "iox": 12190, + "\u0120epic": 12191, + "oyd": 12192, + "Bel": 12193, + "\u0120Stew": 12194, + ".(": 12195, + "ahu": 12196, + "\u0120Beyond": 12197, + "\u0120outs": 12198, + "\u0120solo": 12199, + "\u0120Tree": 12200, + "\u0120preserve": 12201, + "\u0120tub": 12202, + "ARE": 12203, + "roc": 12204, + "\u0120Impro": 12205, + "\u0120Wright": 12206, + "\u0120bund": 12207, + "\u0120traged": 12208, + "\u0120occasional": 12209, + "bian": 12210, + "Second": 12211, + "rons": 12212, + "\u0120interactions": 12213, + "formed": 12214, + "sing": 12215, + "\u0120owns": 12216, + "\u0120hockey": 12217, + "General": 12218, + "\u0120logical": 12219, + "\u0120expend": 12220, + "\u0120escal": 12221, + "\u0120Griff": 12222, + "\u0120Crown": 12223, + "\u0120Reserve": 12224, + "\u0120stopping": 12225, + "\u0120excuse": 12226, + "second": 12227, + "\u0120operated": 12228, + "\u0120reaches": 12229, + "\u0120Malays": 12230, + "\u0120pollution": 12231, + "\u0120Brooklyn": 12232, + "\u0120delete": 12233, + "\u0120hash": 12234, + "Block": 12235, + "aha": 12236, + "\u00e2\u0122\u00b3": 12237, + "\u0120shorter": 12238, + "piece": 12239, + ">>>": 13163, + "\u0120Mormon": 13164, + "tor": 13165, + "\u0120particles": 13166, + "\u0120Bart": 13167, + "ryption": 13168, + "\u0120admin": 13169, + "\u0120squee": 13170, + "VIDIA": 13171, + "\u0120creator": 13172, + "iameter": 13173, + "icular": 13174, + "NBC": 13175, + "\u0120grabbed": 13176, + "\u0120nodd": 13177, + "\u0120rated": 13178, + "\u0120rotation": 13179, + "\u0120grasp": 13180, + "\u0120excessive": 13181, + "\u0120EC": 13182, + "\u0120Whit": 13183, + "\u0120inventory": 13184, + "aults": 13185, + "\u0120FB": 13186, + "\u0120ecosystem": 13187, + "\u0120billions": 13188, + "\u0120venture": 13189, + "named": 13190, + "\u0120defender": 13191, + "oute": 13192, + "Instead": 13193, + "irable": 13194, + "War": 13195, + "\u0120assumption": 13196, + "\u0120bite": 13197, + "\u0120earthqu": 13198, + "tail": 13199, + "space": 13200, + "\u0120gifts": 13201, + "boys": 13202, + "\u0120inevitable": 13203, + "\u0120structural": 13204, + "\u0120beneficial": 13205, + "\u0120compelling": 13206, + "hole": 13207, + "ervation": 13208, + "\u0120coat": 13209, + "oj": 13210, + "incarn": 13211, + "\u0120Years": 13212, + "\u0120determining": 13213, + "\u0120rhetoric": 13214, + "\u0120boundaries": 13215, + "\u0120whites": 13216, + "Ant": 13217, + "addy": 13218, + ")-": 13219, + "raham": 13220, + "etermin": 13221, + "\u0120harvest": 13222, + "\u0120Conc": 13223, + "\u0120laptop": 13224, + "\u0120Match": 13225, + "\u0120enjoying": 13226, + "cca": 13227, + "ollar": 13228, + "\u0120trips": 13229, + "\u0120addiction": 13230, + "\u0120Sak": 13231, + "\u0120powered": 13232, + "\u0120cous": 13233, + "\u0120Russians": 13234, + "iere": 13235, + "\u0120retrie": 13236, + "quality": 13237, + "\u0120differ": 13238, + "\u0120kingdom": 13239, + "\u0120Laur": 13240, + "\u0120Capitol": 13241, + "\u0120conclusions": 13242, + "\u0120Altern": 13243, + "\u0120Nav": 13244, + "\u0120transparent": 13245, + "BER": 13246, + "Group": 13247, + "\u0120Complete": 13248, + "\u0120infer": 13249, + "\u0120intrig": 13250, + "\u0120insane": 13251, + "RO": 13252, + "ophob": 13253, + "isen": 13254, + "qual": 13255, + "Michael": 13256, + "\u0120museum": 13257, + "\u0120Pope": 13258, + "\u0120reset": 13259, + "rative": 13260, + "five": 13261, + "\u0120aggreg": 13262, + "ittees": 13263, + "ository": 13264, + "\u0120carb": 13265, + "\u0120Record": 13266, + "\u0120decides": 13267, + "\u0120Fix": 13268, + "\u0120exceptions": 13269, + "\u0120Commissioner": 13270, + "uns": 13271, + "\u0120Environmental": 13272, + "\u0120legendary": 13273, + "istence": 13274, + "\u0120tunnel": 13275, + "km": 13276, + "\u0120insult": 13277, + "\u0120troll": 13278, + "\u0120shake": 13279, + "\u0120detention": 13280, + "ques": 13281, + "\u0120Chrome": 13282, + "\u0120Files": 13283, + "\u0120subt": 13284, + "\u0120prospects": 13285, + "\u0120prol": 13286, + "render": 13287, + "proof": 13288, + "\u0120performances": 13289, + "Str": 13290, + "\u0120href": 13291, + "ername": 13292, + "\u0120achievement": 13293, + "\u0120fut": 13294, + "Full": 13295, + "\u0120Leban": 13296, + "google": 13297, + "\u00e3\u0125\u012a": 13298, + "ampa": 13299, + "Maybe": 13300, + "\u0120projected": 13301, + "\u0120Emb": 13302, + "\u0120colleg": 13303, + "\u0120awards": 13304, + "\u0120\u00e2\u0136": 13305, + "Gold": 13306, + "\u0120Blake": 13307, + "\u0120Raj": 13308, + "ifting": 13309, + "\u0120pending": 13310, + "\u0120instinct": 13311, + "\u0120developments": 13312, + "Connect": 13313, + "\u0120Mand": 13314, + "\u0120WITH": 13315, + "\u0120Philippines": 13316, + "profile": 13317, + "\u0120altogether": 13318, + "\u0120Bund": 13319, + "\u0120TD": 13320, + "oooo": 13321, + "amped": 13322, + "iph": 13323, + "\u0120steam": 13324, + "\u0120oldest": 13325, + "\u0120detection": 13326, + "ulpt": 13327, + "\u0120\u00e7": 13328, + "\u0120Wayne": 13329, + "2006": 13330, + "fa": 13331, + "\u0120circles": 13332, + "\u0120Fu": 13333, + "\u0120donors": 13334, + "appropriate": 13335, + "\u0120Dakota": 13336, + "jamin": 13337, + "\u0120motivated": 13338, + "\u0120purchases": 13339, + "\u0120Louisiana": 13340, + "\u0120Spl": 13341, + "\u0120globe": 13342, + "\u0120105": 13343, + "zip": 13344, + "call": 13345, + "\u0120departments": 13346, + "\u0120sustainable": 13347, + "105": 13348, + "\u0120OP": 13349, + "ifiers": 13350, + "\u0120prevented": 13351, + "\u0120incomp": 13352, + "\u0120Commander": 13353, + "\u0120dominated": 13354, + "\u0120\u00c2\u00bb": 13355, + "\u0120invested": 13356, + "\u0120complexity": 13357, + "\u0120incl": 13358, + "\u0120ensuring": 13359, + "\u0120realm": 13360, + "ync": 13361, + "\u0120Independent": 13362, + "rained": 13363, + "\u0120Jen": 13364, + "\u0120Flight": 13365, + "\u0120athe": 13366, + "\u0120speculation": 13367, + "\u0120TE": 13368, + "ocate": 13369, + "tic": 13370, + "\u0120plaint": 13371, + "herry": 13372, + "\u0120toy": 13373, + "\u0120111": 13374, + "\u0120plates": 13375, + "status": 13376, + "\u0120Isa": 13377, + "\u0120devoted": 13378, + "Cop": 13379, + "\u0120ES": 13380, + "255": 13381, + "urrency": 13382, + "Main": 13383, + "\u0120slaves": 13384, + "\u0120pepper": 13385, + "\u0120quotes": 13386, + "\u0120ceiling": 13387, + "\u0120Fish": 13388, + "\u0120transformation": 13389, + "\u0120fraction": 13390, + "\u0120advantages": 13391, + "\u0120toile": 13392, + "\u0120stunning": 13393, + "\u0120moist": 13394, + "breaking": 13395, + "si": 13396, + "\u0120Location": 13397, + "\u0120Medium": 13398, + "\u0120texts": 13399, + "\u0120ugly": 13400, + "\u0120bio": 13401, + ".\u00e2\u0122\u0136": 13402, + "\u0120Based": 13403, + "\u0120trains": 13404, + "\u0120Wing": 13405, + "\u0120Ancient": 13406, + "\u0120Records": 13407, + "\u0120Hope": 13408, + "Special": 13409, + "adesh": 13410, + "obi": 13411, + "[/": 13412, + "\u0120temporarily": 13413, + "Ver": 13414, + "hu": 13415, + "oser": 13416, + "\u0120overnight": 13417, + "\u0120mamm": 13418, + "\u0120Treasury": 13419, + "\u0120Venezuel": 13420, + "\u0120Mega": 13421, + "\u0120tar": 13422, + "\u0120expects": 13423, + "black": 13424, + "orph": 13425, + "\\\\\\\\": 13426, + "\u0120acceptance": 13427, + "\u0120radar": 13428, + "sis": 13429, + "\u0120junior": 13430, + "\u0120frames": 13431, + "\u0120observation": 13432, + "acies": 13433, + "Power": 13434, + "\u0120Advanced": 13435, + "Mag": 13436, + "ologically": 13437, + "\u0120Mechan": 13438, + "\u0120sentences": 13439, + "\u0120analysts": 13440, + "aughters": 13441, + "forcement": 13442, + "\u0120vague": 13443, + "\u0120clause": 13444, + "\u0120directors": 13445, + "\u0120evaluate": 13446, + "\u0120cabinet": 13447, + "Matt": 13448, + "\u0120Classic": 13449, + "Ang": 13450, + "\u0120cler": 13451, + "\u0120Buck": 13452, + "\u0120researcher": 13453, + "\u0120160": 13454, + "\u0120poorly": 13455, + "\u0120experiencing": 13456, + "\u0120Ped": 13457, + "\u0120Manhattan": 13458, + "\u0120freed": 13459, + "\u0120themes": 13460, + "advant": 13461, + "\u0120nin": 13462, + "\u0120praise": 13463, + "104": 13464, + "\u0120Libya": 13465, + "best": 13466, + "\u0120trusted": 13467, + "\u0120cease": 13468, + "\u0120dign": 13469, + "Direct": 13470, + "\u0120bombing": 13471, + "\u0120migration": 13472, + "\u0120Sciences": 13473, + "\u0120municipal": 13474, + "\u0120Average": 13475, + "\u0120glory": 13476, + "\u0120revealing": 13477, + "\u0120arena": 13478, + "\u0120uncertainty": 13479, + "\u0120battlefield": 13480, + "iao": 13481, + "God": 13482, + "\u0120cinem": 13483, + "rape": 13484, + "elle": 13485, + "apons": 13486, + "\u0120listing": 13487, + "\u0120waited": 13488, + "\u0120spotted": 13489, + "keley": 13490, + "\u0120Audio": 13491, + "eor": 13492, + "arding": 13493, + "idding": 13494, + "igma": 13495, + "\u0120Neg": 13496, + "\u0120lone": 13497, + "\u0120----": 13498, + "exe": 13499, + "deg": 13500, + "\u0120transf": 13501, + "\u0120wash": 13502, + "\u0120slavery": 13503, + "\u0120exploring": 13504, + "\u0120WW": 13505, + "atson": 13506, + "\u0120encl": 13507, + "lies": 13508, + "\u0120Creek": 13509, + "\u0120wooden": 13510, + "Manager": 13511, + "\u0120Brand": 13512, + "ummy": 13513, + "\u0120Arthur": 13514, + "\u0120bureaucr": 13515, + "\u0120blend": 13516, + "arians": 13517, + "Further": 13518, + "\u0120supposedly": 13519, + "\u0120winds": 13520, + "\u01201979": 13521, + "\u0120gravity": 13522, + "\u0120analyses": 13523, + "\u0120Travel": 13524, + "\u0120Veter": 13525, + "\u0120dumb": 13526, + "\u0120alternate": 13527, + "gal": 13528, + "\u0120consumed": 13529, + "\u0120effectiveness": 13530, + ".''": 13531, + "\u0120paths": 13532, + "onda": 13533, + "LA": 13534, + "\u0120Strong": 13535, + "\u0120enables": 13536, + "\u0120escaped": 13537, + "\u0120\"\"": 13538, + "\u0120112": 13539, + "\u01201983": 13540, + "\u0120smiled": 13541, + "\u0120tendency": 13542, + "Fire": 13543, + "\u0120pars": 13544, + "\u0120Roc": 13545, + "\u0120lake": 13546, + "\u0120fitness": 13547, + "\u0120Ath": 13548, + "\u0120Horn": 13549, + "\u0120hier": 13550, + "\u0120impose": 13551, + "mother": 13552, + "\u0120pension": 13553, + "icut": 13554, + "borne": 13555, + "iciary": 13556, + "._": 13557, + "\u0120SU": 13558, + "\u0120polar": 13559, + "isy": 13560, + "engu": 13561, + "itialized": 13562, + "ATA": 13563, + "write": 13564, + "\u0120exercises": 13565, + "\u0120Diamond": 13566, + "otypes": 13567, + "\u0120harmful": 13568, + "onz": 13569, + "\u0120printing": 13570, + "story": 13571, + "\u0120expertise": 13572, + "\u0120Ger": 13573, + "\u0120tragedy": 13574, + "\u0120Fly": 13575, + "\u0120divid": 13576, + "ampire": 13577, + "stock": 13578, + "Mem": 13579, + "\u0120reign": 13580, + "\u0120unve": 13581, + "\u0120amend": 13582, + "\u0120Prophet": 13583, + "\u0120mutual": 13584, + "\u0120Fac": 13585, + "\u0120replacing": 13586, + "Har": 13587, + "\u0120Circuit": 13588, + "\u0120throat": 13589, + "\u0120Shot": 13590, + "\u0120batteries": 13591, + "\u0120toll": 13592, + "\u0120addressing": 13593, + "\u0120Medicaid": 13594, + "\u0120pupp": 13595, + "\u0120Nar": 13596, + "olk": 13597, + "\u0120equity": 13598, + "MR": 13599, + "\u0120Hispan": 13600, + "\u0120Large": 13601, + "mid": 13602, + "Dev": 13603, + "\u0120exped": 13604, + "\u0120demo": 13605, + "\u0120Marshall": 13606, + "ergus": 13607, + "\u0120fiber": 13608, + "\u0120divorce": 13609, + "\u0120Create": 13610, + "\u0120slower": 13611, + "\u0120Parker": 13612, + "\u0120Student": 13613, + "\u0120Training": 13614, + "Return": 13615, + "\u0120Tru": 13616, + "\u0120cub": 13617, + "\u0120Reached": 13618, + "\u0120panic": 13619, + "\u0120quarters": 13620, + "\u0120rect": 13621, + "\u0120treating": 13622, + "\u0120rats": 13623, + "\u0120Christianity": 13624, + "oler": 13625, + "\u0120sacred": 13626, + "\u0120declare": 13627, + "ulative": 13628, + "eting": 13629, + "\u0120delivering": 13630, + "estone": 13631, + "\u0120tel": 13632, + "\u0120Larry": 13633, + "\u0120meta": 13634, + "accept": 13635, + "artz": 13636, + "\u0120Roger": 13637, + "handed": 13638, + "\u0120header": 13639, + "\u0120trapped": 13640, + "\u0120Century": 13641, + "\u0120knocked": 13642, + "\u0120Oxford": 13643, + "\u0120survivors": 13644, + "bot": 13645, + "\u0120demonstration": 13646, + "\u0120dirt": 13647, + "\u0120assists": 13648, + "OME": 13649, + "\u0120Draft": 13650, + "ortunate": 13651, + "folio": 13652, + "pered": 13653, + "usters": 13654, + "gt": 13655, + "\u0120Lock": 13656, + "\u0120judicial": 13657, + "verted": 13658, + "\u0120secured": 13659, + "outing": 13660, + "\u0120Books": 13661, + "\u0120hosting": 13662, + "\u0120lifted": 13663, + "length": 13664, + "\u0120jer": 13665, + "\u0120wheels": 13666, + "\u0120Range": 13667, + "umbnails": 13668, + "\u0120diagnosis": 13669, + "tech": 13670, + "\u0120Stewart": 13671, + "\u0120Pract": 13672, + "\u0120nationwide": 13673, + "\u0120dear": 13674, + "\u0120obligations": 13675, + "\u0120grows": 13676, + "\u0120mandatory": 13677, + "\u0120suspicious": 13678, + "!'": 13679, + "Apr": 13680, + "Great": 13681, + "\u0120mortgage": 13682, + "\u0120prosecutor": 13683, + "\u0120editorial": 13684, + "\u0120Kr": 13685, + "\u0120processed": 13686, + "ungle": 13687, + "\u0120flexibility": 13688, + "Earlier": 13689, + "\u0120Cart": 13690, + "\u0120Sug": 13691, + "\u0120focuses": 13692, + "\u0120startup": 13693, + "\u0120breach": 13694, + "\u0120Tob": 13695, + "cycle": 13696, + "\u00e3\u0122\u012e": 13697, + "rose": 13698, + "\u0120bizarre": 13699, + "\u00e3\u0122\u012f": 13700, + "\u0120vegetables": 13701, + "$$": 13702, + "\u0120retreat": 13703, + "oshi": 13704, + "\u0120Shop": 13705, + "\u0120Ground": 13706, + "\u0120Stop": 13707, + "\u0120Hawaii": 13708, + "\u0120Ay": 13709, + "Perhaps": 13710, + "\u0120Beaut": 13711, + "uffer": 13712, + "enna": 13713, + "\u0120productivity": 13714, + "Fixed": 13715, + "control": 13716, + "\u0120absent": 13717, + "\u0120Campaign": 13718, + "Green": 13719, + "\u0120identifying": 13720, + "\u0120regret": 13721, + "\u0120promoted": 13722, + "\u0120Seven": 13723, + "\u0120eru": 13724, + "neath": 13725, + "aughed": 13726, + "\u0120Pin": 13727, + "\u0120Living": 13728, + "Cost": 13729, + "omatic": 13730, + "mega": 13731, + "\u0120Nig": 13732, + "ocy": 13733, + "\u0120inbox": 13734, + "\u0120empire": 13735, + "\u0120horizont": 13736, + "\u0120branches": 13737, + "\u0120metaph": 13738, + "Active": 13739, + "edi": 13740, + "\u0120Film": 13741, + "\u0120Something": 13742, + "\u0120mods": 13743, + "incial": 13744, + "\u0120Original": 13745, + "Gen": 13746, + "\u0120spirits": 13747, + "\u0120earning": 13748, + "Hist": 13749, + "\u0120riders": 13750, + "\u0120sacrific": 13751, + "MT": 13752, + "\u0120VA": 13753, + "\u0120Salt": 13754, + "\u0120occupation": 13755, + "\u0120Mi": 13756, + "\u0120disg": 13757, + "lict": 13758, + "\u0120nit": 13759, + "\u0120nodes": 13760, + "eem": 13761, + "\u0120Pier": 13762, + "\u0120hatred": 13763, + "psy": 13764, + "\u00e3\u0125\u012b": 13765, + "\u0120theater": 13766, + "\u0120sophisticated": 13767, + "\u0120defended": 13768, + "\u0120besides": 13769, + "\u0120thoroughly": 13770, + "\u0120Medicare": 13771, + "\u0120blamed": 13772, + "arently": 13773, + "\u0120crying": 13774, + "FOR": 13775, + "priv": 13776, + "\u0120singing": 13777, + "\u0120Il": 13778, + "\u0120cute": 13779, + "oided": 13780, + "olitical": 13781, + "\u0120Neuro": 13782, + "\u00e5\u00a4": 13783, + "\u0120donation": 13784, + "\u0120Eagles": 13785, + "\u0120Give": 13786, + "Tom": 13787, + "\u0120substantially": 13788, + "\u0120License": 13789, + "\u0120Ja": 13790, + "\u0120grey": 13791, + "\u0120Animal": 13792, + "\u0120ER": 13793, + "\u0120Und": 13794, + "\u0120keen": 13795, + "\u0120conclude": 13796, + "\u0120Mississippi": 13797, + "Engine": 13798, + "\u0120Studios": 13799, + "Press": 13800, + "overs": 13801, + "llers": 13802, + "\u0120350": 13803, + "\u0120Rangers": 13804, + "\u0120rou": 13805, + "erto": 13806, + "Ep": 13807, + "issa": 13808, + "ivan": 13809, + "\u0120seal": 13810, + "\u0120Regist": 13811, + "display": 13812, + "\u0120weaken": 13813, + "uum": 13814, + "\u0120Commons": 13815, + "\u0120Say": 13816, + "\u0120cultures": 13817, + "\u0120laughed": 13818, + "\u0120slip": 13819, + "\u0120treatments": 13820, + "izable": 13821, + "mart": 13822, + "\u0120Rice": 13823, + "\u0120beast": 13824, + "\u0120obesity": 13825, + "\u0120Laure": 13826, + "iga": 13827, + "Which": 13828, + "holder": 13829, + "\u0120elderly": 13830, + "\u0120pays": 13831, + "\u0120complained": 13832, + "\u0120crop": 13833, + "\u0120proc": 13834, + "\u0120explosive": 13835, + "\u0120Fan": 13836, + "\u0120Arsenal": 13837, + "Author": 13838, + "eful": 13839, + "\u0120meals": 13840, + "\u0120(-": 13841, + "idays": 13842, + "\u0120imagination": 13843, + "\u0120annually": 13844, + "\u0120ms": 13845, + "asures": 13846, + "Head": 13847, + "ikh": 13848, + "matic": 13849, + "\u0120boyfriend": 13850, + "\u0120Computer": 13851, + "\u0120bump": 13852, + "\u0120surge": 13853, + "\u0120Craig": 13854, + "\u0120Kirk": 13855, + "Del": 13856, + "mediate": 13857, + "\u0120scenarios": 13858, + "\u0120Mut": 13859, + "\u0120Stream": 13860, + "\u0120competitors": 13861, + "\u00d9\u0126": 13862, + "\u0120Stanford": 13863, + "\u0120Resources": 13864, + "azed": 13865, + "bage": 13866, + "\u0120organis": 13867, + "\u0120Release": 13868, + "\u0120separately": 13869, + "\u0120habits": 13870, + "\u0120measurements": 13871, + "\u0120Close": 13872, + "\u0120accompany": 13873, + "\u0120gly": 13874, + "\u0120tang": 13875, + "\u0120Rou": 13876, + "\u0120plugin": 13877, + "\u0120convey": 13878, + "\u0120Challenge": 13879, + "oots": 13880, + "jan": 13881, + "\u0120curs": 13882, + "\u0120Relations": 13883, + "keeper": 13884, + "\u0120approaching": 13885, + "ping": 13886, + "Speaking": 13887, + "\u0120arrangement": 13888, + "\u0120VI": 13889, + "arettes": 13890, + "\u0120affecting": 13891, + "\u0120permits": 13892, + "because": 13893, + "\u0120useless": 13894, + "\u0120Hus": 13895, + "!!!!": 13896, + "\u0120destroying": 13897, + "Unfortunately": 13898, + "\u0120fascinating": 13899, + "Sem": 13900, + "\u0120electoral": 13901, + "\u0120transparency": 13902, + "\u0120Chaos": 13903, + "\u0120volunteer": 13904, + "\u0120statistical": 13905, + "\u0120activated": 13906, + "rox": 13907, + "Web": 13908, + "HE": 13909, + "\u0120Hampshire": 13910, + "isive": 13911, + "Map": 13912, + "\u0120trash": 13913, + "\u0120Lawrence": 13914, + "stick": 13915, + "Cr": 13916, + "\u0120rings": 13917, + "EXT": 13918, + "\u0120operational": 13919, + "opes": 13920, + "Does": 13921, + "\u0120Evans": 13922, + "\u0120witnessed": 13923, + "Port": 13924, + "\u0120launching": 13925, + "econom": 13926, + "wear": 13927, + "\u0120Particip": 13928, + "umm": 13929, + "cules": 13930, + "\u0120RAM": 13931, + "\u0120Tun": 13932, + "\u0120assured": 13933, + "\u0120binary": 13934, + "\u0120betray": 13935, + "\u0120exploration": 13936, + "\u0120Fel": 13937, + "\u0120admission": 13938, + "itated": 13939, + "Sy": 13940, + "\u0120avoided": 13941, + "\u0120Simulator": 13942, + "\u0120celebrated": 13943, + "\u0120Electric": 13944, + "\u00a5\u0140": 13945, + "\u0120cluster": 13946, + "itzerland": 13947, + "health": 13948, + "Line": 13949, + "\u0120Nash": 13950, + "aton": 13951, + "\u0120spare": 13952, + "\u0120enterprise": 13953, + "\u0120DIS": 13954, + "cludes": 13955, + "\u0120flights": 13956, + "\u0120regards": 13957, + "\u0120\u00c3\u0139": 13958, + "half": 13959, + "\u0120trucks": 13960, + "\u0120contacts": 13961, + "\u0120uncons": 13962, + "\u0120Climate": 13963, + "\u0120immense": 13964, + "NEW": 13965, + "occ": 13966, + "ective": 13967, + "\u0120embod": 13968, + "\u0120patrol": 13969, + "\u0120beside": 13970, + "\u0120viable": 13971, + "\u0120creep": 13972, + "\u0120triggered": 13973, + "verning": 13974, + "\u0120comparable": 13975, + "ql": 13976, + "\u0120gaining": 13977, + "asses": 13978, + "\u0120();": 13979, + "\u0120Grey": 13980, + "\u0120MLS": 13981, + "sized": 13982, + "\u0120prosper": 13983, + "\"?": 13984, + "\u0120polling": 13985, + "\u0120shar": 13986, + "\u0120RC": 13987, + "\u0120firearm": 13988, + "orient": 13989, + "\u0120fence": 13990, + "\u0120variations": 13991, + "giving": 13992, + "\u0120Pi": 13993, + "ospel": 13994, + "\u0120pledge": 13995, + "\u0120cure": 13996, + "\u0120spy": 13997, + "\u0120violated": 13998, + "\u0120rushed": 13999, + "\u0120stroke": 14000, + "\u0120Blog": 14001, + "sels": 14002, + "\u0120Ec": 14003, + ",''": 14004, + "\u0120pale": 14005, + "\u0120Collins": 14006, + "terror": 14007, + "\u0120Canadians": 14008, + "\u0120tune": 14009, + "\u0120laboratory": 14010, + "\u0120nons": 14011, + "tarian": 14012, + "\u0120disability": 14013, + "\u0120Gam": 14014, + "\u0120singer": 14015, + "alg": 14016, + "\u0120Senior": 14017, + "\u0120traded": 14018, + "\u0120Warrior": 14019, + "\u0120infring": 14020, + "\u0120Franklin": 14021, + "\u0120strain": 14022, + "\u0120Swedish": 14023, + "\u0120seventh": 14024, + "\u0120Benn": 14025, + "\u0120Tell": 14026, + "\u0120syndrome": 14027, + "\u0120wondered": 14028, + "iden": 14029, + "++++": 14030, + "igo": 14031, + "\u0120purple": 14032, + "\u0120journalism": 14033, + "\u0120rebel": 14034, + "\u0120fu": 14035, + "blog": 14036, + "\u0120invite": 14037, + "rencies": 14038, + "\u0120Contact": 14039, + "Israel": 14040, + "\u0120Content": 14041, + "\u0120cheer": 14042, + "\u0120bedroom": 14043, + "\u0120Engineering": 14044, + "\u0120Queens": 14045, + "\u0120dwell": 14046, + "\u0120PlayStation": 14047, + "\u0120Dim": 14048, + "\u0120Colon": 14049, + "lr": 14050, + "\u0120operates": 14051, + "\u0120motivation": 14052, + "USA": 14053, + "astered": 14054, + "Core": 14055, + "\u0120Truth": 14056, + "olo": 14057, + "OSE": 14058, + "\u0120Memory": 14059, + "\u0120predec": 14060, + "\u0120anarch": 14061, + "\u01201920": 14062, + "\u0120Yam": 14063, + "\u00c3\u00a8": 14064, + "bid": 14065, + "\u0120grateful": 14066, + "\u0120excitement": 14067, + "\u0120treasure": 14068, + "\u0120longest": 14069, + "ctive": 14070, + "\u0120deserves": 14071, + "\u0120reserves": 14072, + "\u0120cops": 14073, + "\u0120Ottawa": 14074, + "\u0120Egyptian": 14075, + "anked": 14076, + "\u0120artif": 14077, + "\u0120hypothesis": 14078, + ":/": 14079, + "\u0120purchasing": 14080, + "\u0120lovely": 14081, + "HP": 14082, + "\u0120divide": 14083, + "\u0120strictly": 14084, + "\u0120questioning": 14085, + "\u0120taxpayers": 14086, + "\u0120Joy": 14087, + "\u0120rolls": 14088, + "\u0120Heavy": 14089, + "\u0120ports": 14090, + "\u0120magnetic": 14091, + "\u0120inflamm": 14092, + "\u0120brush": 14093, + "tics": 14094, + "\u00e2\u012a\u0134": 14095, + "\u0120bottles": 14096, + "ppy": 14097, + "\u0120padd": 14098, + "\u00e3\u0124\u00af": 14099, + "million": 14100, + "\u0120devastating": 14101, + "\u0120compiled": 14102, + "\u0120medication": 14103, + "\u0120twelve": 14104, + "\u0120Perry": 14105, + "Space": 14106, + "imb": 14107, + "your": 14108, + "\u0120leaked": 14109, + "\u0120Tar": 14110, + "\u0120unity": 14111, + "\u0120infected": 14112, + "\u0120traveled": 14113, + "IDE": 14114, + "\u0120McDonald": 14115, + "txt": 14116, + "\u0120Princ": 14117, + "\u0120interven": 14118, + "\u0120Taiwan": 14119, + "\u0120Pow": 14120, + "\u0120bearing": 14121, + "\u0120Thread": 14122, + "\u0120zones": 14123, + "izards": 14124, + "unks": 14125, + "Chapter": 14126, + "llor": 14127, + "\u0120\u00c2\u00b7": 14128, + "\u0120wounds": 14129, + "\u0120discretion": 14130, + "\u0120succeeded": 14131, + "iking": 14132, + "\u0120iconic": 14133, + "Call": 14134, + "\u0120screening": 14135, + "\u0120Mis": 14136, + "icts": 14137, + "\u0120ministers": 14138, + "\u0120separation": 14139, + "Player": 14140, + "\u0120bip": 14141, + "\u0120beloved": 14142, + "\u0120counting": 14143, + "\u0120Eye": 14144, + "around": 14145, + "inging": 14146, + "\u0120tablet": 14147, + "\u0120offence": 14148, + "inance": 14149, + "have": 14150, + "\u0120Info": 14151, + "\u0120Ninja": 14152, + "\u0120protective": 14153, + "\u0120Cass": 14154, + "Mac": 14155, + "\u0120Quality": 14156, + "North": 14157, + "\u0120ic": 14158, + "\u0120Cuba": 14159, + "\u0120Chronicle": 14160, + "\u0120Property": 14161, + "\u0120fastest": 14162, + "otos": 14163, + "\u0120Germ": 14164, + "OWN": 14165, + "\u0120boom": 14166, + "\u0120Stanley": 14167, + "erguson": 14168, + "\u0120clever": 14169, + "\u0120enters": 14170, + "mode": 14171, + "terior": 14172, + "\u0120Sens": 14173, + "\u0120linear": 14174, + "ARK": 14175, + "\u0120comparing": 14176, + "\u0120purely": 14177, + "\u0120safer": 14178, + "\u0120Potter": 14179, + "\u0120cups": 14180, + "RT": 14181, + "\u0120gluc": 14182, + "\u0120attributed": 14183, + "\u0120dupl": 14184, + "\u0120Pap": 14185, + "\u0120precious": 14186, + "\u0120pa": 14187, + "ictionary": 14188, + "\u0120Tig": 14189, + "\u0120Too": 14190, + "olutions": 14191, + "stan": 14192, + "\u0120robots": 14193, + "\u0120lobb": 14194, + "\u0120statute": 14195, + "\u0120prevention": 14196, + "western": 14197, + "160": 14198, + "\u0120Active": 14199, + "\u0120Maria": 14200, + "hal": 14201, + "None": 14202, + "ellar": 14203, + "\u0120KB": 14204, + "\u0120Partners": 14205, + "\u0120Single": 14206, + "\u0120Following": 14207, + "ango": 14208, + "acious": 14209, + "\u0120thou": 14210, + "\u0120kg": 14211, + "\u0120influential": 14212, + "\u0120Friends": 14213, + "Sur": 14214, + "ainted": 14215, + "\u0120forums": 14216, + "\u0120starter": 14217, + "\u0120citizenship": 14218, + "\u0120Election": 14219, + "onge": 14220, + "otation": 14221, + "osph": 14222, + ";;;;": 14223, + "utical": 14224, + "pur": 14225, + "eren": 14226, + "\u0120accusations": 14227, + "bitious": 14228, + "abbit": 14229, + "\u0120Ord": 14230, + "Posted": 14231, + "irk": 14232, + "\u0120sensitivity": 14233, + "iche": 14234, + "\u0120Amy": 14235, + "\u0120Fab": 14236, + "\u0120summit": 14237, + "\u0120pedest": 14238, + "\u0120rubber": 14239, + "\u0120agricultural": 14240, + "\u0120cancel": 14241, + "AE": 14242, + "\u0120inaug": 14243, + "\u0120contam": 14244, + "\u0120firmly": 14245, + "iw": 14246, + "stage": 14247, + "\u0120Kan": 14248, + "\u0120tier": 14249, + "\u0120invention": 14250, + "\u0120translated": 14251, + "\u0120Rules": 14252, + "Box": 14253, + "Twitter": 14254, + "IDS": 14255, + "\u0120pizza": 14256, + "\u0120debug": 14257, + "\u0120Drop": 14258, + "vs": 14259, + "\u0120horses": 14260, + "big": 14261, + "\u0120boring": 14262, + "\u0120hood": 14263, + "\u0120McCain": 14264, + "atched": 14265, + "\u0120Bros": 14266, + "\u0120skip": 14267, + "\u0120essay": 14268, + "stat": 14269, + "\u0120Legends": 14270, + "\u0120ammunition": 14271, + "auc": 14272, + "\u0120shooter": 14273, + "\u0120unh": 14274, + "\u0120supplied": 14275, + "\u0120generic": 14276, + "\u0120SK": 14277, + "iban": 14278, + "yrics": 14279, + "\u0120255": 14280, + "\u0120climbing": 14281, + "Former": 14282, + "\u0120flip": 14283, + "\u0120jumping": 14284, + "\u0120frustration": 14285, + "\u0120Terry": 14286, + "\u0120neighborhoods": 14287, + "\u0120median": 14288, + "bean": 14289, + "\u0120brains": 14290, + "Following": 14291, + "\u0120shaped": 14292, + "\u0120draws": 14293, + "\u0120altered": 14294, + "Jack": 14295, + "\u0120recipes": 14296, + "\u0120skilled": 14297, + "wealth": 14298, + "achi": 14299, + "election": 14300, + "\u0120behaviors": 14301, + "deals": 14302, + "\u0120Until": 14303, + "Fe": 14304, + "\u0120declaration": 14305, + "marks": 14306, + "\u0120Between": 14307, + "celona": 14308, + "\u0120reson": 14309, + "\u0120bubble": 14310, + "Among": 14311, + "\u0120imperial": 14312, + "GS": 14313, + "\u0120feminist": 14314, + "2005": 14315, + "\u0120Kyle": 14316, + "\u0120accounting": 14317, + "\u0120Tele": 14318, + "\u0120Tyr": 14319, + "\u0120connecting": 14320, + "\u0120rehab": 14321, + "\u0120Pred": 14322, + "sim": 14323, + "\u0120meantime": 14324, + "\u0120physician": 14325, + "MW": 14326, + "\u0120Campbell": 14327, + "\u0120Brandon": 14328, + "\u0120contributing": 14329, + "\u0120Rule": 14330, + "\u0120Weight": 14331, + "\u0120Nap": 14332, + "\u0120interactive": 14333, + "\u0120vag": 14334, + "\u0120helmet": 14335, + "\u0120Comb": 14336, + "four": 14337, + "\u0120shipped": 14338, + "\u0120completing": 14339, + "\u0120PD": 14340, + "PDATE": 14341, + "\u0120spreading": 14342, + "\u0120scary": 14343, + "erving": 14344, + "\u0120Gas": 14345, + "\u0120frank": 14346, + "school": 14347, + "\u0120romantic": 14348, + "\u0120stabil": 14349, + "Rob": 14350, + "\u0120accurately": 14351, + "\u0120acute": 14352, + "\u0120Hann": 14353, + "\u0120symbols": 14354, + "\u0120civilization": 14355, + "\u0120AW": 14356, + "\u0120lightning": 14357, + "\u0120considers": 14358, + "\u0120venue": 14359, + "\u0120\u00d7": 14360, + "\u0120oven": 14361, + "\u0120SF": 14362, + "his": 14363, + "\u0120nu": 14364, + "\u0120Learn": 14365, + "\u0120peoples": 14366, + "\u0120std": 14367, + "\u0120slee": 14368, + "\u0120slic": 14369, + "\u0120Statistics": 14370, + "\u0120corners": 14371, + "\u0120Baker": 14372, + "\u0120:)": 14373, + "mentation": 14374, + "olver": 14375, + "\u0120laughing": 14376, + "\u0120Todd": 14377, + "onde": 14378, + "\u0120Hills": 14379, + "\u0120nuts": 14380, + "\u0120Woman": 14381, + "plane": 14382, + "\u0120liver": 14383, + "\u0120Inside": 14384, + "Sorry": 14385, + "\u0120agrees": 14386, + "\u0120fundament": 14387, + "\u0120Fisher": 14388, + "\u0120auction": 14389, + "\u0120threads": 14390, + "glas": 14391, + "\u0120Basic": 14392, + "\u0120Nat": 14393, + "\u0120lacking": 14394, + "\u0120celebration": 14395, + "ju": 14396, + "\u0120silly": 14397, + "Euro": 14398, + "\u0120tatt": 14399, + "ighty": 14400, + "controlled": 14401, + "Test": 14402, + "\u0120Singh": 14403, + "\u0120rage": 14404, + "\u0120rhyth": 14405, + "offic": 14406, + "\u0120Phantom": 14407, + "\u0120headlines": 14408, + "\u0120responding": 14409, + "\u0120Morning": 14410, + "\u0120vitamin": 14411, + "\u0120boots": 14412, + "\u0120Site": 14413, + "alin": 14414, + "pi": 14415, + "\u0120viral": 14416, + "\u0120UC": 14417, + "DER": 14418, + "\u0120Sex": 14419, + "\u0120stocks": 14420, + "current": 14421, + "\u0120churches": 14422, + "\u0120Rare": 14423, + "\u0120Murphy": 14424, + "\u0120denial": 14425, + "\u0120Gaming": 14426, + "\u0120toug": 14427, + "\u0120nick": 14428, + "\u0120makers": 14429, + "\u0120Ronald": 14430, + "\u0120generous": 14431, + "\u0120Doc": 14432, + "\u0120Morris": 14433, + "\u0120transformed": 14434, + "\u0120Normal": 14435, + "\u0120104": 14436, + "\u0120Kickstarter": 14437, + "\u0120Upon": 14438, + "Online": 14439, + "\u0120IRS": 14440, + "\u0120wrap": 14441, + "\u0120loving": 14442, + "\u0120arrives": 14443, + "\u0120Due": 14444, + "\u0120heter": 14445, + "\u0120Made": 14446, + "\u0120rental": 14447, + "\u0120belongs": 14448, + "\u0120attorneys": 14449, + "\u0120crops": 14450, + "\u0120matched": 14451, + "ulum": 14452, + "oline": 14453, + "109": 14454, + "\u0120dispar": 14455, + "\u0120buyers": 14456, + "\u0120Cambridge": 14457, + "\u0120ethics": 14458, + "roups": 14459, + "\u0120justified": 14460, + "\u0120marginal": 14461, + "\u0120respected": 14462, + "winning": 14463, + "\u0120nodded": 14464, + "\u0120Serge": 14465, + "\u0120Former": 14466, + "Craft": 14467, + "################": 14468, + "\u0120Warner": 14469, + "\u0120dash": 14470, + "ete": 14471, + "\u0120entert": 14472, + "\u0120Escape": 14473, + "outheast": 14474, + "\u0120knees": 14475, + "\u0120Bomb": 14476, + "\u0120rug": 14477, + "Pass": 14478, + "\u0120attitudes": 14479, + "government": 14480, + "\u0120Prior": 14481, + "\u0120qualities": 14482, + "\u0120notification": 14483, + "\u0120Phone": 14484, + "lie": 14485, + "\u0120anticipated": 14486, + "\u0120Combat": 14487, + "\u0120Barry": 14488, + "\u01201982": 14489, + "Users": 14490, + "oner": 14491, + "\u0120computing": 14492, + "\u0120Connecticut": 14493, + "\u0120lesser": 14494, + "\u0120peers": 14495, + "\u0120Cu": 14496, + "\u0120technically": 14497, + "\u0120submission": 14498, + "\u0120Universal": 14499, + "\u0120manually": 14500, + "ourge": 14501, + "\u0120respondents": 14502, + "\u0120BTC": 14503, + "\u0120Host": 14504, + "\u0120fare": 14505, + "\u0120Bird": 14506, + "\u0120receipt": 14507, + "also": 14508, + "\u0120jack": 14509, + "\u0120agriculture": 14510, + "\u0120skull": 14511, + "\u0120!=": 14512, + "\u0120passive": 14513, + "\u0120CI": 14514, + "\u0120societies": 14515, + "\u0120reminded": 14516, + "\u0120interference": 14517, + "Buy": 14518, + "\u0120\u00e2\u013e": 14519, + "gon": 14520, + "\u0120scrutiny": 14521, + "\u0120Witch": 14522, + "\u0120conducting": 14523, + "\u0120\u00e3\u0125": 14524, + "\u0120exchanges": 14525, + "\u0120Mitchell": 14526, + "\u0120inhabit": 14527, + "\u0120twist": 14528, + "BD": 14529, + "\u0120wherever": 14530, + "groupon": 14531, + "\u0120jokes": 14532, + "\u0120Benjamin": 14533, + "\u0120Random": 14534, + "frame": 14535, + "\u0120Lions": 14536, + "\u0120highlighted": 14537, + "\u0120Arkansas": 14538, + "Ent": 14539, + "\u0120pile": 14540, + "\u0120prelim": 14541, + "gs": 14542, + "minded": 14543, + "\u0120felony": 14544, + "\u0120GA": 14545, + "\u0120Luck": 14546, + "\u0120practically": 14547, + "\u0120Bos": 14548, + "\u0120actress": 14549, + "Dam": 14550, + "\u0120Bou": 14551, + "\u0120visa": 14552, + "\u0120embedded": 14553, + "\u0120hybrid": 14554, + "\u0120earliest": 14555, + "\u0120sooner": 14556, + "social": 14557, + "\u0120HA": 14558, + "\u0120steep": 14559, + "\u0120disadvant": 14560, + "\u0120exploit": 14561, + "\u0120Egg": 14562, + "\u0120Ultra": 14563, + "\u0120necessity": 14564, + "Local": 14565, + "iege": 14566, + "\u0120dated": 14567, + "\u0120masses": 14568, + "\u0120subscription": 14569, + "pless": 14570, + "\u0120anonym": 14571, + "\u0120presumably": 14572, + "Blue": 14573, + "Their": 14574, + "asketball": 14575, + "\u0120Philip": 14576, + "\u0120comed": 14577, + "loaded": 14578, + "rane": 14579, + "\u0120reflection": 14580, + "China": 14581, + "\u0120extends": 14582, + "\u0120forming": 14583, + "\u0120unders": 14584, + "2001": 14585, + "\u0120grat": 14586, + "\u0120concentrations": 14587, + "\u0120insulin": 14588, + "\u0120secular": 14589, + "\u0120whilst": 14590, + "\u0120winners": 14591, + "Advertisements": 14592, + "\u0120deliberately": 14593, + "\u0120Working": 14594, + "\u0120sink": 14595, + "etics": 14596, + "dale": 14597, + "\u0120mandate": 14598, + "\u0120gram": 14599, + "\u0120vacation": 14600, + "\u0120warnings": 14601, + "ripp": 14602, + "\u0120THAT": 14603, + "\u0120commentary": 14604, + "\u0120intu": 14605, + "\u0120aest": 14606, + "\u0120reasoning": 14607, + "\u0120breakdown": 14608, + "\u0120Zombie": 14609, + "\u0120-->": 14610, + "\u0120Political": 14611, + "cott": 14612, + "\u0120thrust": 14613, + "\u0120technological": 14614, + "\u0120deciding": 14615, + "\u0120trafficking": 14616, + "Long": 14617, + "Welcome": 14618, + "prising": 14619, + "\u0120Communications": 14620, + "\u0120endors": 14621, + "\u0120swift": 14622, + "\u0120metabol": 14623, + "coins": 14624, + "resa": 14625, + "\u0120HTTP": 14626, + "\u0120enroll": 14627, + "\u0120Happy": 14628, + "usr": 14629, + "intage": 14630, + "\u0120[\"": 14631, + "uably": 14632, + "\u0120Material": 14633, + "\u0120repeal": 14634, + "Sept": 14635, + "kh": 14636, + "\u0120Modi": 14637, + "\u0120underneath": 14638, + "\u0120IL": 14639, + "shore": 14640, + "\u0120diagnosed": 14641, + "aceutical": 14642, + "\u0120shower": 14643, + "aux": 14644, + "\u0120Switch": 14645, + "\u0120Strength": 14646, + "\u0120jihad": 14647, + "national": 14648, + "\u0120trauma": 14649, + "ussy": 14650, + "oni": 14651, + "\u0120consolid": 14652, + "\u0120calories": 14653, + "\u0120Flynn": 14654, + "agged": 14655, + "168": 14656, + "\u0120Pink": 14657, + "\u0120fulfill": 14658, + "\u0120chains": 14659, + "\u0120notably": 14660, + "\u0120AV": 14661, + "Life": 14662, + "\u0120Chuck": 14663, + "mus": 14664, + "\u0120Urban": 14665, + "\u0120Hend": 14666, + "\u0120deposit": 14667, + "\u0120Sad": 14668, + "\u0120affair": 14669, + "ORK": 14670, + "ieval": 14671, + "\u0120FDA": 14672, + "\u0120trop": 14673, + "\u0120Overall": 14674, + "\u0120virtue": 14675, + "\u0120satisfaction": 14676, + "aund": 14677, + "\u0120lun": 14678, + "\u0120Switzerland": 14679, + "\u0120Operation": 14680, + "process": 14681, + "\u0120shook": 14682, + "\u0120counties": 14683, + "leased": 14684, + "\u0120Charlotte": 14685, + "112": 14686, + "\u0120transcript": 14687, + "\u0120redd": 14688, + "push": 14689, + "\u0120Hey": 14690, + "\u0120Analysis": 14691, + "[\"": 14692, + "\u0120alternatives": 14693, + "ardless": 14694, + "\u0120eleph": 14695, + "\u0120prejud": 14696, + "\u0120Leaf": 14697, + "Having": 14698, + "\u0120Hub": 14699, + "\u0120expressions": 14700, + "\u0120Volume": 14701, + "\u0120shocking": 14702, + "\u0120Reds": 14703, + "\u0120readily": 14704, + "\u0120planets": 14705, + "adata": 14706, + "\u0120collapsed": 14707, + "\u0120Madrid": 14708, + "\u0120irrit": 14709, + "ipper": 14710, + "\u0120Enc": 14711, + "\u0120Wire": 14712, + "\u0120buzz": 14713, + "\u0120GP": 14714, + "asha": 14715, + "\u0120accidentally": 14716, + "uru": 14717, + "\u0120frustrated": 14718, + "\u0120SA": 14719, + "\u0120hungry": 14720, + "\u0120Huff": 14721, + "\u0120labels": 14722, + "anto": 14723, + "\u0120EP": 14724, + "\u0120barriers": 14725, + ")|": 14726, + "\u0120Berkeley": 14727, + "\u0120Jets": 14728, + "\u0120pairs": 14729, + "\u0120Lan": 14730, + "James": 14731, + "\u0120Bear": 14732, + "\u0120humor": 14733, + "\u0120Liberty": 14734, + "\u0120magnitude": 14735, + "\u0120aging": 14736, + "\u0120Mason": 14737, + "\u0120friendship": 14738, + "umbling": 14739, + "\u0120emerge": 14740, + "\u0120newspapers": 14741, + "\u0120ambitious": 14742, + "\u0120Richards": 14743, + "aternal": 14744, + "\u01201981": 14745, + "\u0120cookies": 14746, + "\u0120sculpt": 14747, + "\u0120pursuit": 14748, + "Location": 14749, + "\u0120scripts": 14750, + "pc": 14751, + "\u0120arrangements": 14752, + "\u0120diameter": 14753, + "\u0120loses": 14754, + "amation": 14755, + "\u0120liqu": 14756, + "\u0120Jake": 14757, + "arette": 14758, + "\u0120understands": 14759, + "\u0120Zen": 14760, + "vm": 14761, + "\u0120approve": 14762, + "\u0120wip": 14763, + "\u0120ultra": 14764, + "\u0120intend": 14765, + "\u0120DI": 14766, + "ascular": 14767, + "\u0120stays": 14768, + "\u0120Kor": 14769, + "\u0120Kl": 14770, + "\u0120investing": 14771, + "La": 14772, + "\u0120believing": 14773, + "bad": 14774, + "mouth": 14775, + "\u0120taxpayer": 14776, + "\u00e3\u0125\u0125": 14777, + "\u0120Quebec": 14778, + "\u0120lap": 14779, + "\u0120Swiss": 14780, + "drop": 14781, + "\u0120drain": 14782, + "iri": 14783, + "etc": 14784, + "ften": 14785, + "\u0120Nex": 14786, + "\u0120straw": 14787, + "\u0120screaming": 14788, + "\u0120counted": 14789, + "\u0120damaging": 14790, + "\u0120ambassador": 14791, + "century": 14792, + "\u0120prox": 14793, + "\u0120arrests": 14794, + "uv": 14795, + "ilateral": 14796, + "\u0120Charg": 14797, + "\u0120prescribed": 14798, + "\u0120independently": 14799, + "\u0120fierce": 14800, + "\u0120Baby": 14801, + "\u0120brave": 14802, + "\u0120suits": 14803, + "=>": 14804, + "\u0120baseline": 14805, + "\u0120Rate": 14806, + "\u0120islands": 14807, + "\u0120((": 14808, + "green": 14809, + "ixels": 14810, + "\u0120namely": 14811, + "\u0120Village": 14812, + "than": 14813, + "amy": 14814, + "Version": 14815, + "gmail": 14816, + "entials": 14817, + "\u0120Sud": 14818, + "\u0120Melbourne": 14819, + "\u0120arriving": 14820, + "\u0120quantum": 14821, + "eff": 14822, + "ropolitan": 14823, + "Tri": 14824, + "\u0120funeral": 14825, + "\u0120IR": 14826, + "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 14827, + "\u0120Cob": 14828, + "itably": 14829, + "\u0120turb": 14830, + "\u0120combo": 14831, + "Review": 14832, + "\u0120deployment": 14833, + "uity": 14834, + "\u0120Bott": 14835, + "\u0120invisible": 14836, + "\u0120rendering": 14837, + "\u0120unlocked": 14838, + "\u0120aqu": 14839, + "\u0120Vladimir": 14840, + "\u0120pad": 14841, + "\u0120Brain": 14842, + "\u0120Legacy": 14843, + "dragon": 14844, + "\u0120Kurdish": 14845, + "\u0120sounded": 14846, + "\u0120detained": 14847, + "\u0120DM": 14848, + "gary": 14849, + "\u0120daughters": 14850, + "\u0120disturbing": 14851, + "uka": 14852, + "\u0120Parad": 14853, + "\u0120tast": 14854, + "\u0120unfortunate": 14855, + "\u0120ul": 14856, + "emin": 14857, + "\u0120attendance": 14858, + "trl": 14859, + "\u0120parks": 14860, + "\u0120Memorial": 14861, + "\u0120Alice": 14862, + "othy": 14863, + "guard": 14864, + "\u0120Dise": 14865, + "\u0120Shan": 14866, + "\u0120Forum": 14867, + "Rich": 14868, + "\u0120shifted": 14869, + "uez": 14870, + "\u0120lighter": 14871, + "\u0120Magn": 14872, + "\u0120cod": 14873, + "Sch": 14874, + "hammad": 14875, + "Pub": 14876, + "350": 14877, + "\u0120Pokemon": 14878, + "\u0120prototype": 14879, + "\u0120unre": 14880, + "Base": 14881, + "\u0120Students": 14882, + "\u0120Reply": 14883, + "\u0120Communist": 14884, + "\u0120gau": 14885, + "\u0120Tyler": 14886, + "IZ": 14887, + "\u0120participated": 14888, + "\u0120suprem": 14889, + "\u0120Details": 14890, + "\u0120vessels": 14891, + "rod": 14892, + "\u0120tribe": 14893, + "keep": 14894, + "\u0120assumptions": 14895, + "\u0120pound": 14896, + "\u0120crude": 14897, + "\u0120Available": 14898, + "\u0120swimming": 14899, + "\u0120inclusion": 14900, + "\u0120advances": 14901, + "culation": 14902, + "\u0120conservation": 14903, + "\u0120overd": 14904, + "\u0120Buffalo": 14905, + "Article": 14906, + "edge": 14907, + "\u0120awa": 14908, + "\u0120Madison": 14909, + "\u0120sidew": 14910, + "\u0120catast": 14911, + "\u0120Krist": 14912, + "ucle": 14913, + "\u0120Highway": 14914, + "\u0120Terror": 14915, + "\u0120activation": 14916, + "\u0120unconscious": 14917, + "\u0120Satan": 14918, + "\u0120Susan": 14919, + "illery": 14920, + "\u0120arranged": 14921, + "iop": 14922, + "\u0120rumors": 14923, + "urring": 14924, + "think": 14925, + "\u0120Keith": 14926, + "\u0120Kind": 14927, + "\u0120avoiding": 14928, + "byn": 14929, + "nut": 14930, + "\u0120Speaker": 14931, + "rus": 14932, + "names": 14933, + "\u0120guilt": 14934, + "\u0120Olympics": 14935, + "\u0120sail": 14936, + "\u0120Mes": 14937, + "levant": 14938, + "\u0120Columbus": 14939, + "aft": 14940, + "City": 14941, + "South": 14942, + "\u0120Harvey": 14943, + "\u0120Pun": 14944, + "Several": 14945, + "\u0120mentally": 14946, + "\u0120impress": 14947, + "mount": 14948, + "\u0120Ubuntu": 14949, + "\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136": 14950, + "\u0120Superman": 14951, + "\u0120MPs": 14952, + "\u0120intentions": 14953, + "\u0120Racing": 14954, + "\u0120likelihood": 14955, + "\u0120240": 14956, + "Total": 14957, + "\u0120toys": 14958, + "\u0120Watson": 14959, + "\u0120urge": 14960, + "Lear": 14961, + "\u0120Paper": 14962, + "\u0120occurring": 14963, + "\u0120Beng": 14964, + "\u0120Cert": 14965, + "\u0120stones": 14966, + "Tim": 14967, + "\u0120Twin": 14968, + "zb": 14969, + "\u0120Dynam": 14970, + "\u0120politician": 14971, + "kens": 14972, + "\u0120Enterprise": 14973, + "UTERS": 14974, + "\u0120abol": 14975, + "\u0120refresh": 14976, + "\u0120arbitrary": 14977, + "pection": 14978, + "\u0120troubles": 14979, + "\u0120});": 14980, + "tv": 14981, + "\u0120pilots": 14982, + "\u0120distribute": 14983, + "\u0120audit": 14984, + "\u0120pause": 14985, + "original": 14986, + "\u0120rivals": 14987, + "\u00c2\u00a3": 14988, + "Fig": 14989, + "TL": 14990, + "abil": 14991, + "rying": 14992, + "Lin": 14993, + "ioned": 14994, + "lon": 14995, + "\u0120fancy": 14996, + "\u0120crashed": 14997, + "\u0120tract": 14998, + "\u0120shed": 14999, + "\u0120consume": 15000, + "Based": 15001, + "download": 15002, + "init": 15003, + "\u0120voltage": 15004, + "Introdu": 15005, + "\u0120condemned": 15006, + "\u0120Finance": 15007, + "respect": 15008, + "\u0120excluded": 15009, + "\u0120establishing": 15010, + "heric": 15011, + "\u0120heritage": 15012, + "\u0120spectacular": 15013, + "\u0120unst": 15014, + "\u0120Snowden": 15015, + "\u0120Lane": 15016, + "San": 15017, + "\u0120protections": 15018, + "struction": 15019, + "incinn": 15020, + "\u0120macro": 15021, + "Custom": 15022, + "iosity": 15023, + "\u0120esp": 15024, + "\u0120functioning": 15025, + "\u0120mush": 15026, + "\u0120puzzle": 15027, + "\u0120ethical": 15028, + "Mal": 15029, + "\u0120governing": 15030, + "\u0120Ferguson": 15031, + "\u0120restored": 15032, + "\u0120stressed": 15033, + "\u0120Counter": 15034, + "\u0120Kas": 15035, + "clip": 15036, + "ANS": 15037, + "\u0120seiz": 15038, + "UK": 15039, + "byss": 15040, + "oldown": 15041, + "api": 15042, + "\u0120permanently": 15043, + "ounters": 15044, + "West": 15045, + "Through": 15046, + "Light": 15047, + "atoes": 15048, + "\u0120neat": 15049, + "\u0120cord": 15050, + "urer": 15051, + "\u0120severely": 15052, + "\u0120Aven": 15053, + "\u0120interrog": 15054, + "\u0120triple": 15055, + "Given": 15056, + "Number": 15057, + "\u0120arise": 15058, + "\u0120sher": 15059, + "plant": 15060, + "\u0120flower": 15061, + "\u0120Cou": 15062, + "\u0120ate": 15063, + "\u0120newer": 15064, + "bul": 15065, + "\u0120meanwhile": 15066, + "\u0120Lair": 15067, + "\u0120adjustment": 15068, + "\u0120Copyright": 15069, + "\u0120divers": 15070, + "iological": 15071, + "\u0120gamers": 15072, + "oat": 15073, + "\u0120historically": 15074, + "\u0120analog": 15075, + "\u0120longtime": 15076, + "\u0120prescription": 15077, + "\u0120Mist": 15078, + "\u0120Hyper": 15079, + "\u0120Maine": 15080, + "\u0120Deity": 15081, + "\u0120multipl": 15082, + "\u0120Reincarn": 15083, + "\u0120Hyd": 15084, + "\u0120Pic": 15085, + "Sil": 15086, + "rants": 15087, + "\u0120Cris": 15088, + ".;": 15089, + "({": 15090, + "ependence": 15091, + "\u0120recy": 15092, + "ateur": 15093, + "\u0120quad": 15094, + "\u0120glob": 15095, + "\u0120conced": 15096, + "team": 15097, + "\u0120capitalist": 15098, + "\u0120Lot": 15099, + "\u0120royal": 15100, + "\u0120Cyber": 15101, + "\u0120blacks": 15102, + "metic": 15103, + "riv": 15104, + "\u0120Danny": 15105, + "\u0120spo": 15106, + "\u0120RO": 15107, + "\u0120animated": 15108, + "rypted": 15109, + "\u0120Deputy": 15110, + "\u0120rendered": 15111, + "FE": 15112, + "\u0120streak": 15113, + "\u0120clouds": 15114, + "\u0120Doug": 15115, + "~~~~~~~~": 15116, + "\u0120discour": 15117, + "\u0120Veh": 15118, + "\u0120psychology": 15119, + "\u0120Journey": 15120, + "\u0120crystal": 15121, + "\u0120Frost": 15122, + "\u0120suspicion": 15123, + "\u0120relate": 15124, + "orus": 15125, + "\u0120Crypt": 15126, + "\u0120NVIDIA": 15127, + "comed": 15128, + "uting": 15129, + "incinnati": 15130, + "\u0120vulnerability": 15131, + "ostic": 15132, + "\u0120isolation": 15133, + "\u0120cooling": 15134, + "\u0120Coalition": 15135, + "\u0120119": 15136, + "Four": 15137, + "\u0120Deal": 15138, + "\u0120\u00e2\u012b": 15139, + "semble": 15140, + "rament": 15141, + "\u0120Barcelona": 15142, + "\u0120102": 15143, + "\u0120cocaine": 15144, + "ocalypse": 15145, + "Feb": 15146, + "ogenic": 15147, + "\u0120mutation": 15148, + "\u0120cryptoc": 15149, + "\u0120Kel": 15150, + "\u0120Git": 15151, + "ais": 15152, + "\u0120sisters": 15153, + "ANK": 15154, + "\u0120activate": 15155, + "Ter": 15156, + "\u0120dread": 15157, + "ylon": 15158, + "\u0120propri": 15159, + "Aust": 15160, + "\u0120Default": 15161, + "\u0120outdoor": 15162, + "\u0120sheer": 15163, + "ceive": 15164, + "\u0120gently": 15165, + "\u00d0\u00be": 15166, + "Program": 15167, + "\u0120\u00e2\u0128\u0134": 15168, + "\u0120vegan": 15169, + "\u0120Crus": 15170, + "\u0120responsibilities": 15171, + "\u0120HR": 15172, + "OLD": 15173, + "\u0120prevents": 15174, + "\u0120stiff": 15175, + "\u0120Were": 15176, + "\u0120athletic": 15177, + "\u0120Score": 15178, + "\u0120):": 15179, + "\u0120columns": 15180, + "\u0120Loc": 15181, + "available": 15182, + "\u0120Fram": 15183, + "\u0120Sessions": 15184, + "\u0120companion": 15185, + "\u0120packs": 15186, + "140": 15187, + "\u0120Knights": 15188, + "\u0120fart": 15189, + "\u0120streams": 15190, + "\u0120shore": 15191, + "\u0120appeals": 15192, + "\u0120Performance": 15193, + "haul": 15194, + "\u0120Stra": 15195, + "\u0120Nag": 15196, + "103": 15197, + "\u0120Transportation": 15198, + "BB": 15199, + "Ev": 15200, + "zan": 15201, + "Public": 15202, + "\u0120twin": 15203, + "ulsion": 15204, + "Mult": 15205, + "\u0120electro": 15206, + "\u0120statue": 15207, + "ationally": 15208, + "\u0120Nort": 15209, + "\u0120inspection": 15210, + "/*": 15211, + "igue": 15212, + "\u0120compassion": 15213, + "\u0120Tales": 15214, + "\u0120Stein": 15215, + "\u0120Screen": 15216, + "\u0120Bug": 15217, + "\u0120Lion": 15218, + "girl": 15219, + "\u0120withdrawal": 15220, + "\u0120objectives": 15221, + "\u0120bloody": 15222, + "\u0120preliminary": 15223, + "\u0120jacket": 15224, + "\u0120dimensions": 15225, + "\u0120Cool": 15226, + "\u0120Occup": 15227, + "\u0120wreck": 15228, + "\u0120doubled": 15229, + "anking": 15230, + "\u01201975": 15231, + "\u0120glasses": 15232, + "\u0120Wang": 15233, + "prov": 15234, + "Path": 15235, + "connected": 15236, + "\u0120Multi": 15237, + "\u0120Norway": 15238, + "agonist": 15239, + "\u0120feared": 15240, + "\u0120touching": 15241, + "\u0120arguably": 15242, + "\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af": 15243, + "\u0120NCAA": 15244, + "chem": 15245, + "\u0120spat": 15246, + "\u0120WWE": 15247, + "\u0120Cel": 15248, + "igger": 15249, + "\u0120attacker": 15250, + "\u0120Join": 15251, + "object": 15252, + "etta": 15253, + "\u0120eliminated": 15254, + "det": 15255, + "\u0120destruct": 15256, + "\u0120Lucas": 15257, + "ctuary": 15258, + "180": 15259, + "\u0120Brady": 15260, + "\u0120Blues": 15261, + "Bay": 15262, + "aukee": 15263, + "\u0120timeline": 15264, + "\u0120delegates": 15265, + "written": 15266, + "ufficient": 15267, + "\u0120shapes": 15268, + "Copyright": 15269, + "ouble": 15270, + "service": 15271, + "\u0120pione": 15272, + "\u0120colleges": 15273, + "\u0120rows": 15274, + "\u0120spite": 15275, + "\u0120assessed": 15276, + "360": 15277, + "\u0120lease": 15278, + "\u0120confidential": 15279, + "cker": 15280, + "\u0120Manning": 15281, + "\u0120Voice": 15282, + "\u0120sealed": 15283, + "\u0120calculate": 15284, + "NO": 15285, + "\u0120Assistant": 15286, + "\u0120teenager": 15287, + "ulent": 15288, + "atherine": 15289, + "\u0120mock": 15290, + "\u0120diamond": 15291, + "\u0120fest": 15292, + "\u0120switched": 15293, + "\u0120resume": 15294, + "\u0120Puerto": 15295, + "\u0120lanes": 15296, + "iration": 15297, + "\u0120Similarly": 15298, + "\u0120rod": 15299, + "\u0120Sel": 15300, + "\u0120Palace": 15301, + "\u0120Limited": 15302, + "eous": 15303, + "\u0120variant": 15304, + "\u0120ward": 15305, + "\u0120))": 15306, + "Show": 15307, + "OOK": 15308, + "Alex": 15309, + "\u0120Nep": 15310, + "bris": 15311, + "\u0120Wikipedia": 15312, + "\u0120exceptional": 15313, + "\u0120manages": 15314, + "\u0120Draw": 15315, + "Again": 15316, + "\u0120copper": 15317, + "utt": 15318, + "\u0120exports": 15319, + "\u0120portfolio": 15320, + "\u0120elevated": 15321, + "Rated": 15322, + "\u0120Otherwise": 15323, + "\u0120Tact": 15324, + "\u0120Shel": 15325, + "\u0120TX": 15326, + "\"\u00e2\u0122\u0136": 15327, + "\u0120resur": 15328, + "\u0120Wa": 15329, + "venant": 15330, + "\u0120monetary": 15331, + "people": 15332, + "Email": 15333, + "\u0120fifty": 15334, + "\u0120Sweet": 15335, + "\u0120Malaysia": 15336, + "\u0120confusing": 15337, + "\u0120Rio": 15338, + "uda": 15339, + "utenant": 15340, + "\");": 15341, + "\u0120praised": 15342, + "\u0120volumes": 15343, + "turn": 15344, + "\u0120mature": 15345, + "\u0120nonprofit": 15346, + "\u0120passionate": 15347, + "\u0120Private": 15348, + "\u0120103": 15349, + "\u0120descend": 15350, + "\u00e7\u00a5\u0140": 15351, + "uffy": 15352, + "headed": 15353, + "Whether": 15354, + "rien": 15355, + "zech": 15356, + "beit": 15357, + "\u0120chrom": 15358, + "\u0120McM": 15359, + "\u0120dancing": 15360, + "\u0120eleg": 15361, + "\u0120Noticed": 15362, + "115": 15363, + "\u0120advocacy": 15364, + "ENTS": 15365, + "ambling": 15366, + "\u0120Minor": 15367, + "\u0120Finn": 15368, + "\u0120priorities": 15369, + "\u0120thereof": 15370, + "\u0120Stage": 15371, + "\u0120Rogers": 15372, + "\u0120substitute": 15373, + "\u0120Jar": 15374, + "\u0120Jefferson": 15375, + "\u0120lightly": 15376, + "102": 15377, + "\u0120Lisa": 15378, + "uits": 15379, + "ysical": 15380, + "\u0120shifts": 15381, + "\u0120drones": 15382, + "\u0120workplace": 15383, + "\u0120resid": 15384, + "ensed": 15385, + "ahn": 15386, + "\u0120preferences": 15387, + "server": 15388, + "\u0120debates": 15389, + "doc": 15390, + "\u0120Gods": 15391, + "\u0120helicopter": 15392, + "\u0120honour": 15393, + "\u0120considerably": 15394, + "eded": 15395, + "\u0120Female": 15396, + "\u0120Anne": 15397, + "\u0120reun": 15398, + "\u0120Face": 15399, + "\u0120Hallow": 15400, + "\u0120Budget": 15401, + "\u0120condemn": 15402, + "\u0120tender": 15403, + "Prof": 15404, + "ocratic": 15405, + "\u0120Turner": 15406, + "\u0120Agric": 15407, + "\u01201976": 15408, + "\u0120apt": 15409, + "disc": 15410, + "\u0120Fighter": 15411, + "\u0120Aur": 15412, + "\u0120garbage": 15413, + "input": 15414, + "\u0120Karl": 15415, + "\u0120Oliver": 15416, + "\u0120Language": 15417, + "kn": 15418, + "Non": 15419, + "\u0120Clar": 15420, + "\u0120traditions": 15421, + "\u0120advertisement": 15422, + "\u0120Sor": 15423, + "\u0120archive": 15424, + "\u0120villages": 15425, + "750": 15426, + "\u0120implementing": 15427, + "waukee": 15428, + "\u0120dietary": 15429, + "\u0120switching": 15430, + "Republic": 15431, + "\u0120velocity": 15432, + "\u0120cit": 15433, + "\u0120Awards": 15434, + "\u0120financing": 15435, + "\u0120lasted": 15436, + ")]": 15437, + "\u0120reminder": 15438, + "Person": 15439, + "\u0120precision": 15440, + "\u0120designers": 15441, + "\u0120Fried": 15442, + "\u0120Border": 15443, + "\u0120tragic": 15444, + "\u0120wield": 15445, + "\u0120initiatives": 15446, + "\u0120Tank": 15447, + "wer": 15448, + "\u0120joins": 15449, + "Ro": 15450, + "inery": 15451, + "\u0120arrow": 15452, + "\u0120generating": 15453, + "founder": 15454, + "\u0120searches": 15455, + "\u0120randomly": 15456, + "Access": 15457, + "\u0120batch": 15458, + "\u0120posed": 15459, + "lat": 15460, + "\u0120pursuing": 15461, + "asa": 15462, + "\u0120testified": 15463, + "forming": 15464, + "\u0120Shar": 15465, + "wiki": 15466, + "\u0120Either": 15467, + "Sometimes": 15468, + "\u0120senators": 15469, + "\u0120Johnny": 15470, + "\u0120Taliban": 15471, + "\u0120GPS": 15472, + "\":\"/": 15473, + "\u00e3\u0123\u00ae\u00e5": 15474, + "\u0120analyzed": 15475, + "\u0120Rubio": 15476, + "\u0120Movement": 15477, + "opard": 15478, + "iii": 15479, + "Stand": 15480, + "fight": 15481, + "\u0120ignoring": 15482, + "iang": 15483, + "\u0120GN": 15484, + "soever": 15485, + "\u0120STAT": 15486, + "\u0120refusing": 15487, + "\u0120sweat": 15488, + "\u0120bay": 15489, + "PORT": 15490, + "irmed": 15491, + "aky": 15492, + "\u0120dispro": 15493, + "\u0120labeled": 15494, + "\u0120108": 15495, + "Hello": 15496, + "\u0120pleasant": 15497, + "aba": 15498, + "\u0120triumph": 15499, + "\u0120aboard": 15500, + "\u0120incom": 15501, + "\u0120Crow": 15502, + "lett": 15503, + "\u0120folk": 15504, + "\u0120chase": 15505, + "``": 15506, + "\u0120Brus": 15507, + "\u0120teens": 15508, + "cue": 15509, + "\u0120terrain": 15510, + "hyd": 15511, + "ilight": 15512, + "ORY": 15513, + "Support": 15514, + "ews": 15515, + "lli": 15516, + "raints": 15517, + "\u0120Cand": 15518, + "\u0120abused": 15519, + "achment": 15520, + "larg": 15521, + "Bas": 15522, + "\u0120Cancer": 15523, + "\u01201978": 15524, + "\u0120supporter": 15525, + "access": 15526, + "\u0120Termin": 15527, + "\u0120Tampa": 15528, + "\u0120ANY": 15529, + "\u0120newest": 15530, + "\u0120Criminal": 15531, + "edu": 15532, + "\u01201930": 15533, + "\u0120admits": 15534, + "\u0120ende": 15535, + "\u0120failures": 15536, + "urate": 15537, + "fulness": 15538, + "cycl": 15539, + "\u0120Subject": 15540, + "\u0120infinite": 15541, + "three": 15542, + "WA": 15543, + "pit": 15544, + "\u0120Install": 15545, + "Rad": 15546, + "iliation": 15547, + "GM": 15548, + "\u0120continent": 15549, + "\u0120accommodate": 15550, + "\u0120Clay": 15551, + "\u0120pup": 15552, + "\u0120Function": 15553, + "\u0120hammer": 15554, + "\u0120Alberta": 15555, + "\u0120revised": 15556, + "\u0120minorities": 15557, + "\u0120measurement": 15558, + "Connell": 15559, + "\u0120disable": 15560, + "\u0120Mix": 15561, + "Incre": 15562, + "\u0120fork": 15563, + "\u0120Rosen": 15564, + "\u0120implies": 15565, + "umblr": 15566, + "ANG": 15567, + "\u0120proteins": 15568, + "\u0120aggression": 15569, + "\u0120facilitate": 15570, + "SN": 15571, + "\u0120illegally": 15572, + "uer": 15573, + "\u0120academ": 15574, + "\u0120puzz": 15575, + "\u0120Shift": 15576, + "pay": 15577, + "ollo": 15578, + "\u0120audiences": 15579, + "Build": 15580, + "\u0120noble": 15581, + "\u0120syntax": 15582, + "\u00e2\u013a\u0127": 15583, + "\u0120beam": 15584, + "\u0120Bed": 15585, + "\u0120Ald": 15586, + "\u0120origins": 15587, + "video": 15588, + "\u01201977": 15589, + "\u0120Assault": 15590, + "\u0120garage": 15591, + "Team": 15592, + "\u0120verdict": 15593, + "\u0120dwar": 15594, + "\u0120Virtual": 15595, + "event": 15596, + "Keep": 15597, + "\u0120sentiment": 15598, + "\u0120wildlife": 15599, + "shirt": 15600, + "\u0120burg": 15601, + "\u0120recommendation": 15602, + "represent": 15603, + "\u0120gallery": 15604, + "owners": 15605, + "\u0120scholar": 15606, + "\u0120convenience": 15607, + "\u0120Swift": 15608, + "\u0120convinc": 15609, + "Cap": 15610, + "\u0120warfare": 15611, + "\u0120Visual": 15612, + "\u0120constitute": 15613, + "\u0120abort": 15614, + "\u0120Weather": 15615, + "\u0120Looking": 15616, + "\u0120Hem": 15617, + "\u0120martial": 15618, + "\u0120incoming": 15619, + "etition": 15620, + "\u0120tolerance": 15621, + "\u0120Created": 15622, + "\u0120flows": 15623, + "\u0120Elder": 15624, + "\u0120souls": 15625, + "\u0120foul": 15626, + "\u0120Pain": 15627, + "\u0120CAN": 15628, + "\u0120220": 15629, + "bc": 15630, + "hend": 15631, + "\u0120genius": 15632, + "Real": 15633, + "\u0120Wr": 15634, + "ometer": 15635, + "pad": 15636, + "\u0120limiting": 15637, + "\u0120Si": 15638, + "\u0120Lore": 15639, + "\u0120Adventures": 15640, + "\u0120varied": 15641, + "Disc": 15642, + "fin": 15643, + "\u0120Personal": 15644, + "Chris": 15645, + "\u0120invented": 15646, + "\u0120dive": 15647, + "\u0120Rise": 15648, + "\u0120oz": 15649, + "\u0120Comics": 15650, + "\u0120expose": 15651, + "\u0120Reb": 15652, + "letters": 15653, + "site": 15654, + "imated": 15655, + "\u0120hacking": 15656, + "\u0120educated": 15657, + "\u0120Nobody": 15658, + "\u0120depri": 15659, + "\u0120incentive": 15660, + "\u00e3\u0124\u00b7": 15661, + "\u0120oversight": 15662, + "\u0120tribes": 15663, + "\u0120Belgium": 15664, + "\u0120licensing": 15665, + "ourt": 15666, + "Product": 15667, + "ahl": 15668, + "\u0120Gem": 15669, + "\u0120specialist": 15670, + "\u0120cra": 15671, + "anners": 15672, + "\u0120Corbyn": 15673, + "\u01201973": 15674, + "READ": 15675, + "\u0120summar": 15676, + "\u0120overlook": 15677, + "\u0120Application": 15678, + "\u0120inappropriate": 15679, + "\u0120downloaded": 15680, + "Que": 15681, + "\u0120Bears": 15682, + "\u0120thumb": 15683, + "\u0120Character": 15684, + "\u0120Reincarnated": 15685, + "\u0120Sid": 15686, + "\u0120demonstrates": 15687, + "sky": 15688, + "\u0120Bloomberg": 15689, + "\u0120Array": 15690, + "\u0120Results": 15691, + "\u0120Fourth": 15692, + "\u0120EDT": 15693, + "\u0120Oscar": 15694, + "cend": 15695, + "\u0120106": 15696, + "\u0120NULL": 15697, + "\u0120HERE": 15698, + "match": 15699, + "\u0120Brun": 15700, + "\u0120glucose": 15701, + "ieg": 15702, + "egu": 15703, + "\u0120certified": 15704, + "\u0120relie": 15705, + "\u0120humanitarian": 15706, + "\u0120prayers": 15707, + "King": 15708, + "\u0120nan": 15709, + "hou": 15710, + "108": 15711, + "ulu": 15712, + "\u0120renewable": 15713, + "\u0120distinguish": 15714, + "\u0120dense": 15715, + "\u0120Vent": 15716, + "\u0120Package": 15717, + "\u0120Boss": 15718, + "\u0120editors": 15719, + "\u0120migr": 15720, + "Tra": 15721, + "\u0120Peters": 15722, + "\u0120Arctic": 15723, + "2004": 15724, + "\u0120Cape": 15725, + "\u0120locally": 15726, + "\u0120lasting": 15727, + "\u0120handy": 15728, + ".).": 15729, + "Pan": 15730, + "\u0120RES": 15731, + "Index": 15732, + "\u0120tensions": 15733, + "\u0120formerly": 15734, + "\u0120ideological": 15735, + "\u0120sensors": 15736, + "\u0120dealers": 15737, + "\u0120defines": 15738, + "Sk": 15739, + "\u0120proceeds": 15740, + "\u0120proxy": 15741, + "azines": 15742, + "\u0120Bash": 15743, + "\u0120Pad": 15744, + "\u0120Craft": 15745, + "ealous": 15746, + "\u0120sheets": 15747, + "ometry": 15748, + "June": 15749, + "clock": 15750, + "TT": 15751, + "\u0120Theatre": 15752, + "\u0120Buzz": 15753, + "\u0120chapters": 15754, + "\u0120millenn": 15755, + "\u0120dough": 15756, + "\u0120Congressional": 15757, + "\u0120imagined": 15758, + "avior": 15759, + "\u0120clinic": 15760, + "\u01201945": 15761, + "\u0120holder": 15762, + "root": 15763, + "olester": 15764, + "\u0120restart": 15765, + "BN": 15766, + "\u0120Hamas": 15767, + "\u0120Job": 15768, + "\u0120orb": 15769, + "\u0120ram": 15770, + "\u0120disclose": 15771, + "\u0120translate": 15772, + "\u0120immigrant": 15773, + "\u0120annoying": 15774, + "\u0120treaty": 15775, + "anium": 15776, + "\u0120Tea": 15777, + "\u0120Legion": 15778, + "\u0120crowds": 15779, + "\u0120Bec": 15780, + "\u0120Aer": 15781, + "ohyd": 15782, + "Bro": 15783, + "Looking": 15784, + "\u0120lbs": 15785, + "\u0120aggress": 15786, + "\u0120seam": 15787, + "\u0120intercept": 15788, + "\u0120MI": 15789, + "mercial": 15790, + "activ": 15791, + "\u0120Cit": 15792, + "\u0120dimension": 15793, + "\u0120consistency": 15794, + "\u0120rushing": 15795, + "\u0120Douglas": 15796, + "\u0120trim": 15797, + "Install": 15798, + "icker": 15799, + "\u0120shy": 15800, + "106": 15801, + "\u0120mentions": 15802, + "pelled": 15803, + "\u0120Tak": 15804, + "cost": 15805, + "\u0120classroom": 15806, + "\u0120fortune": 15807, + "driven": 15808, + "\u0120unle": 15809, + "\u0120Wheel": 15810, + "\u0120investor": 15811, + "\u0120Masters": 15812, + "kit": 15813, + "\u0120associations": 15814, + "\u0120Evolution": 15815, + "oping": 15816, + "uscript": 15817, + "\u0120provincial": 15818, + "\u0120Walter": 15819, + "avi": 15820, + "SO": 15821, + "\u0120unlimited": 15822, + "English": 15823, + "\u0120Cards": 15824, + "\u0120Ebola": 15825, + "nered": 15826, + "\u0120revenge": 15827, + "\u0120outright": 15828, + "umper": 15829, + "\u0120fitting": 15830, + "\u0120Solid": 15831, + "\u0120formally": 15832, + "\u0120problematic": 15833, + "\u0120hazard": 15834, + "\u0120encryption": 15835, + "\u0120straightforward": 15836, + "\u0120AK": 15837, + "\u0120pse": 15838, + "\u0120Orb": 15839, + "\u0120Chamber": 15840, + "\u0120Mak": 15841, + "Contents": 15842, + "\u0120loyalty": 15843, + "\u0120lyrics": 15844, + "\u0120Sym": 15845, + "\u0120welcomed": 15846, + "\u0120cooked": 15847, + "\u0120monop": 15848, + "\u0120nurse": 15849, + "\u0120misleading": 15850, + "\u0120eternal": 15851, + "\u0120shifting": 15852, + "\u0120+=": 15853, + "Vis": 15854, + "\u0120institutional": 15855, + "illary": 15856, + "\u0120pant": 15857, + "VERT": 15858, + "\u0120ACC": 15859, + "\u0120Enh": 15860, + "\u0120incon": 15861, + "\u0120REUTERS": 15862, + "\u0120donated": 15863, + "\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6": 15864, + "Intern": 15865, + "\u0120exhibit": 15866, + "\u0120tire": 15867, + "\u0120Ric": 15868, + "\u0120Champion": 15869, + "\u0120Muhammad": 15870, + "NING": 15871, + "\u0120Soccer": 15872, + "\u0120mobility": 15873, + "\u0120varying": 15874, + "\u0120Movie": 15875, + "\u0120lord": 15876, + "oak": 15877, + "Field": 15878, + "\u0120vector": 15879, + "usions": 15880, + "\u0120scrap": 15881, + "\u0120enabling": 15882, + "make": 15883, + "Tor": 15884, + ".*": 15885, + "||": 15886, + "\u0120Website": 15887, + "\u0120NPC": 15888, + "\u0120socialist": 15889, + "\u0120Billy": 15890, + "\u0120Additional": 15891, + "\u0120cargo": 15892, + "\u0120farms": 15893, + "\u0120Soon": 15894, + "\u0120Prize": 15895, + "\u0120midnight": 15896, + "\u0120900": 15897, + "seen": 15898, + "\u0120Spot": 15899, + "\u0120sheep": 15900, + "\u0120sponsored": 15901, + "\u0120Hi": 15902, + "\u0120Jump": 15903, + "\u01201967": 15904, + "Microsoft": 15905, + "\u0120Agent": 15906, + "\u0120charts": 15907, + "dir": 15908, + "\u0120adjacent": 15909, + "\u0120tricks": 15910, + "\u0120manga": 15911, + "\u0120exagger": 15912, + "/>": 15913, + "football": 15914, + "\u0120FCC": 15915, + "GC": 15916, + "\u0120Tier": 15917, + "andra": 15918, + "OUND": 15919, + "%),": 15920, + "\u0120fruits": 15921, + "VC": 15922, + "\u0120AA": 15923, + "Rober": 15924, + "\u0120midst": 15925, + "\u00e2\u0139": 15926, + "anka": 15927, + "\u0120legislature": 15928, + "\u0120Neil": 15929, + "\u0120tourists": 15930, + "\"\"": 15931, + "\u0120Warning": 15932, + "\u0120Nevertheless": 15933, + "\u0120Official": 15934, + "\u0120Whatever": 15935, + "\u0120mold": 15936, + "\u0120drafted": 15937, + "\u0120substances": 15938, + "\u0120breed": 15939, + "\u0120tags": 15940, + "\u0120Task": 15941, + "\u0120verb": 15942, + "\u0120manufactured": 15943, + "comments": 15944, + "\u0120Polish": 15945, + "Prov": 15946, + "\u0120determines": 15947, + "Obama": 15948, + "kers": 15949, + "\u0120utterly": 15950, + "\u0120sect": 15951, + "sche": 15952, + "\u0120Gates": 15953, + "\u0120Chap": 15954, + "\u0120aluminum": 15955, + "\u0120zombie": 15956, + "\u0120Touch": 15957, + "\u0120UP": 15958, + "\u0120satisfy": 15959, + "\u0120predomin": 15960, + "ascript": 15961, + "\u0120elaborate": 15962, + "\u01201968": 15963, + "\u0120measuring": 15964, + "\u0120Vari": 15965, + "anyahu": 15966, + "\u0120sir": 15967, + "ulates": 15968, + "idges": 15969, + "ickets": 15970, + "\u0120Spencer": 15971, + "TM": 15972, + "oubted": 15973, + "\u0120prey": 15974, + "\u0120installing": 15975, + "\u0120Cab": 15976, + "reed": 15977, + "reated": 15978, + "Supp": 15979, + "\u0120wrist": 15980, + "\u0120Kerry": 15981, + "107": 15982, + "\u0120Kle": 15983, + "\u0120Rachel": 15984, + "\u0120cotton": 15985, + "\u0120ARE": 15986, + "\u0120Ele": 15987, + "Control": 15988, + "\u0120loads": 15989, + "\u0120Dod": 15990, + "anas": 15991, + "bone": 15992, + "\u0120classical": 15993, + "\u0120Regional": 15994, + "\u0120Integ": 15995, + "VM": 15996, + "\u0120desires": 15997, + "\u0120autism": 15998, + "supported": 15999, + "\u0120Message": 16000, + "\u0120compact": 16001, + "writer": 16002, + "\u0120109": 16003, + "\u0120Hurricane": 16004, + "cision": 16005, + "\u0120cycles": 16006, + "\u0120drill": 16007, + "\u0120colleague": 16008, + "\u0120maker": 16009, + "German": 16010, + "\u0120mistaken": 16011, + "Sun": 16012, + "\u0120Gay": 16013, + "\u0120whatsoever": 16014, + "\u0120sells": 16015, + "\u0120Airl": 16016, + "liv": 16017, + "\u0120Option": 16018, + "\u0120solved": 16019, + "\u0120sectors": 16020, + "\u0120horizontal": 16021, + "\u0120equation": 16022, + "\u0120Skill": 16023, + "\u0120Bio": 16024, + "gement": 16025, + "\u0120Snap": 16026, + "\u0120Legal": 16027, + "\u0120trademark": 16028, + "\u0120makeup": 16029, + "\u0120assembled": 16030, + "\u0120saves": 16031, + "\u0120Halloween": 16032, + "\u0120Vermont": 16033, + "\u0120FROM": 16034, + "\u0120farming": 16035, + "\u0120Podcast": 16036, + "acceptable": 16037, + "\u0120Higher": 16038, + "\u0120asleep": 16039, + "ullivan": 16040, + "\u0120referen": 16041, + "\u0120Lev": 16042, + "\u0120bullets": 16043, + "oko": 16044, + "HC": 16045, + "\u0120stairs": 16046, + "\u0120maintains": 16047, + "\u0120Lower": 16048, + "\u0120Vi": 16049, + "\u0120marine": 16050, + "\u0120acres": 16051, + "\u0120coordinator": 16052, + "\u0120Joh": 16053, + "\u0120counterparts": 16054, + "\u0120Brothers": 16055, + "\u0120indict": 16056, + "bra": 16057, + "\u0120chunk": 16058, + "\u0120cents": 16059, + "Home": 16060, + "\u0120Month": 16061, + "\u0120accordingly": 16062, + "ifles": 16063, + "\u0120Germans": 16064, + "\u0120Syn": 16065, + "Hub": 16066, + "\u0120eyeb": 16067, + "\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122": 16068, + "\u0120ranges": 16069, + "\u0120Holland": 16070, + "\u0120Robot": 16071, + "fc": 16072, + "Mike": 16073, + "\u0120plasma": 16074, + "\u0120swap": 16075, + "\u0120athlete": 16076, + "\u0120Rams": 16077, + ",'\"": 16078, + "\u0120infections": 16079, + "\u0120corrid": 16080, + "\u0120vib": 16081, + "\u0120patches": 16082, + "\u0120traditionally": 16083, + "\u0120revelation": 16084, + "\u0120sweep": 16085, + "\u0120glance": 16086, + "\u0120inex": 16087, + "2003": 16088, + "\u0120Raw": 16089, + "working": 16090, + "osures": 16091, + "\u0120Dat": 16092, + "\u0120Lynch": 16093, + "\u0120leverage": 16094, + "\u0120Reid": 16095, + "\u0120correlation": 16096, + "iances": 16097, + "avascript": 16098, + "\u0120repository": 16099, + "retty": 16100, + "\u01201972": 16101, + "240": 16102, + "\u0120oun": 16103, + "pol": 16104, + "\u0120Reed": 16105, + "\u0120tactical": 16106, + "isite": 16107, + "Apple": 16108, + "\u0120Quinn": 16109, + "\u0120raped": 16110, + "illo": 16111, + "Europe": 16112, + "\u0120algorithms": 16113, + "\u0120Rodrig": 16114, + "iu": 16115, + "\u0120illum": 16116, + "\u0120fame": 16117, + "\u0120introducing": 16118, + "\u0120delays": 16119, + "\u0120Raiders": 16120, + "\u0120whistle": 16121, + "\u0120novels": 16122, + "\u0120Really": 16123, + "\u0120deriv": 16124, + "\u0120publications": 16125, + "\u0120Neither": 16126, + "\u0120Commerce": 16127, + "\u0120aston": 16128, + "language": 16129, + "Notes": 16130, + "\u0120Roth": 16131, + "\u0120Fear": 16132, + "\u0120mate": 16133, + "\u0120parade": 16134, + "\u0120QB": 16135, + "\u0120maneu": 16136, + "\u0120Cincinnati": 16137, + "mitting": 16138, + "\u0120waist": 16139, + "\u0120Rew": 16140, + "\u0120discont": 16141, + "\u00d0\u00b0": 16142, + "\u0120staring": 16143, + "\u0120alias": 16144, + "\u0120securities": 16145, + "\u0120toilet": 16146, + "\u0120Jedi": 16147, + "\u0120unlaw": 16148, + "vised": 16149, + "////////": 16150, + "](": 16151, + "\u0120Weiss": 16152, + "\u0120prest": 16153, + "\u0120Compan": 16154, + "\u0120memo": 16155, + "\u0120Grace": 16156, + "July": 16157, + "\u0120Elite": 16158, + "center": 16159, + "\u0120Stay": 16160, + "\u0120galaxy": 16161, + "\u0120tooth": 16162, + "\u0120Settings": 16163, + "\u0120subjected": 16164, + "\u00e3\u0124\u00a6": 16165, + "\u0120lineback": 16166, + "\u0120retailers": 16167, + "\u0120Want": 16168, + "\u0120dangers": 16169, + "Air": 16170, + "\u0120voluntary": 16171, + "eway": 16172, + "\u0120interpreted": 16173, + "otine": 16174, + "\u00c3\u00a7": 16175, + "\u0120pel": 16176, + "Service": 16177, + "\u0120Eventually": 16178, + "\u0120careers": 16179, + "\u0120threaten": 16180, + "\u0120memor": 16181, + "\u0120Bradley": 16182, + "ancies": 16183, + "sn": 16184, + "\u0120Unknown": 16185, + "National": 16186, + "\u0120shadows": 16187, + "ailand": 16188, + "\u0120Dash": 16189, + "Everyone": 16190, + "izzard": 16191, + "March": 16192, + "=(": 16193, + "\u0120pulls": 16194, + "\u0120stranger": 16195, + "\u0120backwards": 16196, + "\u0120Bernard": 16197, + "imensional": 16198, + "\u0120chron": 16199, + "\u0120theoretical": 16200, + "ktop": 16201, + "\u0120ware": 16202, + "\u0120Investig": 16203, + "\u0120Initi": 16204, + "\u0120Operations": 16205, + "oven": 16206, + "ocide": 16207, + "*/": 16208, + "\u0120flames": 16209, + "\u0120Cash": 16210, + "shit": 16211, + "\u0120cab": 16212, + "\u0120Analy": 16213, + "\u0120Seah": 16214, + "\u0120defining": 16215, + "\u0120ordering": 16216, + "\u0120immun": 16217, + "\u0120persistent": 16218, + "ACH": 16219, + "Russian": 16220, + "mans": 16221, + "\u0120hind": 16222, + "\u0120photography": 16223, + "\u00c2\u00a9": 16224, + "\u0120hug": 16225, + "\u0120107": 16226, + "\u0120Hence": 16227, + "iots": 16228, + "udeau": 16229, + "\u0120subsidies": 16230, + "\u0120routinely": 16231, + "\u0120Device": 16232, + "itic": 16233, + "\u0120disgust": 16234, + "lander": 16235, + "\u01201940": 16236, + "\u0120assignment": 16237, + "\u0120Besides": 16238, + "wick": 16239, + "\u0120Dust": 16240, + "usc": 16241, + "structed": 16242, + "111": 16243, + "develop": 16244, + "\u0120fond": 16245, + "\u0120intersection": 16246, + "\u0120dignity": 16247, + "\u0120commissioner": 16248, + "Without": 16249, + "reach": 16250, + "\u0120cartoon": 16251, + "\u0120scales": 16252, + "\u00e3\u0125\u0143": 16253, + "FIG": 16254, + "\u0120surveys": 16255, + "\u0120Indonesia": 16256, + "\u0120artwork": 16257, + "\u0120unch": 16258, + "\u0120cycling": 16259, + "unct": 16260, + "auer": 16261, + "orate": 16262, + "\u0120Obviously": 16263, + "\u0120characterized": 16264, + "feld": 16265, + "\u0120affirm": 16266, + "\u0120innings": 16267, + "\u0120\u00e9": 16268, + "\u0120aliens": 16269, + "\u0120cloth": 16270, + "etooth": 16271, + "\u0120Certain": 16272, + "\u00c2\u00a7": 16273, + "\u0120digest": 16274, + "know": 16275, + "\u0120XL": 16276, + "\u0120predictions": 16277, + "\u0120din": 16278, + "WAR": 16279, + "\u0120aftermath": 16280, + "Example": 16281, + "\u0120Success": 16282, + "\u0120Thr": 16283, + "IGN": 16284, + "\u0120miner": 16285, + "Bus": 16286, + "\u0120clarity": 16287, + "heimer": 16288, + "\u0120OUT": 16289, + "\u0120Send": 16290, + "\u0120Circle": 16291, + "\u0120Diet": 16292, + "\u0120pronounced": 16293, + "\u0120creators": 16294, + "\u0120earthquake": 16295, + "attery": 16296, + "geons": 16297, + "\u0120od": 16298, + "\u0120laying": 16299, + "orp": 16300, + "Ult": 16301, + "project": 16302, + "\u0120undermin": 16303, + "\u0120sequel": 16304, + "Sam": 16305, + "\u0120Darkness": 16306, + "\u0120reception": 16307, + "bull": 16308, + "YS": 16309, + "\u0120Vir": 16310, + "\u0120sequences": 16311, + "\u0120Coin": 16312, + "\u0120outfit": 16313, + "\u0120Wait": 16314, + "119": 16315, + "\u0120delivers": 16316, + "......": 16317, + "\u0120blown": 16318, + "\u0120Esc": 16319, + "\u0120Math": 16320, + "perm": 16321, + "\u0120Ul": 16322, + "\u0120glim": 16323, + "\u0120facial": 16324, + "\u0120greenhouse": 16325, + "\u0120tokens": 16326, + "/-": 16327, + "\u0120Annual": 16328, + "\u0120ONE": 16329, + "\u0120teenage": 16330, + "\u0120Physical": 16331, + "\u0120Lang": 16332, + "\u0120Celt": 16333, + "\u0120sued": 16334, + "ividually": 16335, + "\u0120patience": 16336, + "chair": 16337, + "regular": 16338, + "\u0120aug": 16339, + "inv": 16340, + "except": 16341, + "\u0120Lil": 16342, + "\u0120nest": 16343, + "fd": 16344, + "sum": 16345, + "\u0120Chase": 16346, + "Russia": 16347, + "\u0120Jennifer": 16348, + "\u0120offseason": 16349, + "Overall": 16350, + "Fore": 16351, + "\u0120riot": 16352, + "Aud": 16353, + "former": 16354, + "\u0120defenders": 16355, + "\u0120CT": 16356, + "iotic": 16357, + "ribly": 16358, + "\u0120automated": 16359, + "\u0120penis": 16360, + "\u0120insist": 16361, + "\u0120diagram": 16362, + "\u0120SQL": 16363, + "\u0120Garc": 16364, + "\u0120witch": 16365, + "client": 16366, + "ierra": 16367, + "ambers": 16368, + "\u0120recount": 16369, + "far": 16370, + "Very": 16371, + "osterone": 16372, + "\u0120appreciated": 16373, + "\u0120Perfect": 16374, + "Section": 16375, + "\u0120doses": 16376, + "ocaust": 16377, + "\u0120costly": 16378, + "\u0120grams": 16379, + "\u0120Shi": 16380, + "\u0120wrestling": 16381, + "\u01201971": 16382, + "\u0120trophy": 16383, + "\u0120nerve": 16384, + "\u0120Kaz": 16385, + "\u0120Experience": 16386, + "\u0120pledged": 16387, + "\u0120playback": 16388, + "\u0120creativity": 16389, + "bye": 16390, + "\u0120attackers": 16391, + "\u0120holders": 16392, + "\u0120Coach": 16393, + "\u0120PhD": 16394, + "\u0120transfers": 16395, + "\u0120colored": 16396, + "\u0120Hindu": 16397, + "\u0120drown": 16398, + "\u0120listened": 16399, + "\u0120WA": 16400, + "iasm": 16401, + "PO": 16402, + "\u0120appealing": 16403, + "\u0120disclosed": 16404, + "\u0120Chicken": 16405, + "agging": 16406, + "\u0120pleaded": 16407, + "\u0120navigation": 16408, + "\u0120Returns": 16409, + "\u0120[[": 16410, + "ROR": 16411, + "EA": 16412, + "\u0120photographer": 16413, + "\u0120Rider": 16414, + "ippers": 16415, + "\u0120slice": 16416, + "\u0120erect": 16417, + "\u0120hed": 16418, + "issance": 16419, + "\u0120Vikings": 16420, + "urious": 16421, + "\u0120appet": 16422, + "oubtedly": 16423, + "Child": 16424, + "\u0120authentic": 16425, + "oos": 16426, + "\u0120Making": 16427, + "\u0120announcing": 16428, + "\u0120bod": 16429, + "\u0120meter": 16430, + "\u0120Nine": 16431, + "\u0120Rogue": 16432, + "\u0120workforce": 16433, + "\u0120renewed": 16434, + "\u0120organisations": 16435, + "acs": 16436, + "PLE": 16437, + "Short": 16438, + "\u0120compounds": 16439, + "\u0120Visit": 16440, + "\u0120envelop": 16441, + "earth": 16442, + "\u0120supportive": 16443, + "ggle": 16444, + "\u0120Brussels": 16445, + "\u0120Guild": 16446, + "Create": 16447, + "REL": 16448, + "\u0120averaged": 16449, + "\u01201969": 16450, + "riages": 16451, + "\u0120lengthy": 16452, + "\u0120forgot": 16453, + "Okay": 16454, + "\u0120Erd": 16455, + "\u0120dealer": 16456, + "\u0120recession": 16457, + "DD": 16458, + "\u0120desperately": 16459, + "\u0120hunger": 16460, + "\u0120sticks": 16461, + "\u0120mph": 16462, + "\u0120Faith": 16463, + "\u0120intentionally": 16464, + "\u0120demol": 16465, + "ueller": 16466, + "\u0120Sale": 16467, + "\u0120debris": 16468, + "spring": 16469, + "\u0120leap": 16470, + ">>>>": 16471, + "\u0120containers": 16472, + "selling": 16473, + "ranean": 16474, + "attering": 16475, + "\u0120commented": 16476, + "\u0120CM": 16477, + "onut": 16478, + "\u0120woods": 16479, + "especially": 16480, + "\u0120organize": 16481, + "ivic": 16482, + "\u0120Woods": 16483, + "anga": 16484, + "squ": 16485, + "\u0120maj": 16486, + "amon": 16487, + "\u0120axis": 16488, + "\u01201974": 16489, + "\u0120Denmark": 16490, + "\u0120warrior": 16491, + "\u0120Pand": 16492, + "\u0120outlined": 16493, + "\u0120BO": 16494, + "insula": 16495, + "zilla": 16496, + "ebook": 16497, + "\u0120dare": 16498, + "\u0120searched": 16499, + "\u0120navigate": 16500, + "Sn": 16501, + "writing": 16502, + "\u0120united": 16503, + "Japan": 16504, + "\u0120Hebrew": 16505, + "\u0120flame": 16506, + "\u0120relies": 16507, + "\u0120catching": 16508, + "\u0120Sho": 16509, + "\u0120imprisonment": 16510, + "\u0120pockets": 16511, + "\u0120closure": 16512, + "\u0120Fam": 16513, + "tim": 16514, + "adequ": 16515, + "Activity": 16516, + "\u0120recruiting": 16517, + "\u0120WATCH": 16518, + "\u0120Argentina": 16519, + "dest": 16520, + "\u0120apologize": 16521, + "oro": 16522, + "\u0120lacks": 16523, + "\u0120tuned": 16524, + "\u0120Griffin": 16525, + "\u0120infamous": 16526, + "\u0120celebrity": 16527, + "sson": 16528, + "\u0120----------------------------------------------------------------": 16529, + "\u0120Isis": 16530, + "\u0120Display": 16531, + "\u0120credibility": 16532, + "\u0120economies": 16533, + "\u0120headline": 16534, + "\u0120Cowboys": 16535, + "\u0120indef": 16536, + "\u0120lately": 16537, + "\u0120incentives": 16538, + "button": 16539, + "\u0120Mob": 16540, + "Aut": 16541, + "\u0120resigned": 16542, + "\u0120Om": 16543, + "camp": 16544, + "\u0120profiles": 16545, + "\u0120schemes": 16546, + "olphins": 16547, + "ayed": 16548, + "Clinton": 16549, + "enh": 16550, + "\u0120Yahoo": 16551, + "\u0120abst": 16552, + "\u0120ank": 16553, + "suits": 16554, + "\u0120wished": 16555, + "\u0120Marco": 16556, + "udden": 16557, + "\u0120sphere": 16558, + "\u0120Bishop": 16559, + "\u0120incorporated": 16560, + "\u0120Plant": 16561, + "114": 16562, + "\u0120hated": 16563, + "pic": 16564, + "\u0120donate": 16565, + "\u0120lined": 16566, + "\u0120beans": 16567, + "\u0120stealing": 16568, + "\u0120costume": 16569, + "\u0120sheriff": 16570, + "\u0120forty": 16571, + "\u0120intact": 16572, + "\u0120adapted": 16573, + "\u0120travelling": 16574, + "bart": 16575, + "\u0120nicely": 16576, + "\u0120dried": 16577, + "\u0120scal": 16578, + "osity": 16579, + "NOTE": 16580, + "\u0120Bh": 16581, + "\u0120Broncos": 16582, + "\u0120Ign": 16583, + "\u0120intimate": 16584, + "\u0120chemistry": 16585, + "\u0120optimal": 16586, + "Deb": 16587, + "\u0120Generation": 16588, + "\u0120],": 16589, + "ichi": 16590, + "\u0120Wii": 16591, + "\u0120YOUR": 16592, + "ventions": 16593, + "Write": 16594, + "\u0120popul": 16595, + "unning": 16596, + "\u0120Wor": 16597, + "Vol": 16598, + "\u0120queen": 16599, + "heads": 16600, + "KK": 16601, + "\u0120analyze": 16602, + "opic": 16603, + "earchers": 16604, + "\u0120dot": 16605, + "legraph": 16606, + "astically": 16607, + "\u0120upgrades": 16608, + "\u0120cares": 16609, + "\u0120extending": 16610, + "\u0120freeze": 16611, + "\u0120inability": 16612, + "\u0120organs": 16613, + "\u0120pretend": 16614, + "\u0120outlet": 16615, + "113": 16616, + "olan": 16617, + "\u0120Mall": 16618, + "uling": 16619, + "talk": 16620, + "\u0120expressing": 16621, + "\u0120Always": 16622, + "\u0120Begin": 16623, + "files": 16624, + "\u0120licenses": 16625, + "%%": 16626, + "\u0120Mitt": 16627, + "\u0120filters": 16628, + "\u0120Milwaukee": 16629, + "GN": 16630, + "\u0120unfold": 16631, + "Mo": 16632, + "\u0120nutrition": 16633, + "ppo": 16634, + "Bo": 16635, + "\u0120founding": 16636, + "\u0120undermine": 16637, + "\u0120easiest": 16638, + "\u0120Czech": 16639, + "\u0120Mack": 16640, + "\u0120sexuality": 16641, + "\u0120Nixon": 16642, + "Win": 16643, + "\u0120Arn": 16644, + "\u0120Kin": 16645, + "\u00e3\u0124\u00a3": 16646, + "icer": 16647, + "\u0120fortun": 16648, + "\u0120surfaces": 16649, + "aghd": 16650, + "\u0120carriers": 16651, + "\u0120PART": 16652, + "\u0120Tib": 16653, + "\u0120interval": 16654, + "\u0120frustrating": 16655, + "\u0120Ship": 16656, + "\u0120Armed": 16657, + "ffe": 16658, + "\u0120boats": 16659, + "\u0120Abraham": 16660, + "inis": 16661, + "\u0120suited": 16662, + "thread": 16663, + "iov": 16664, + "abul": 16665, + "\u0120Venezuela": 16666, + "\u0120tom": 16667, + "super": 16668, + "\u0120castle": 16669, + "although": 16670, + "ioxide": 16671, + "eches": 16672, + "\u0120evolutionary": 16673, + "\u0120negotiate": 16674, + "\u0120confronted": 16675, + "Remember": 16676, + "\u0120170": 16677, + "Such": 16678, + "\u0120911": 16679, + "mult": 16680, + "\u0120Abyss": 16681, + "urry": 16682, + "kees": 16683, + "spec": 16684, + "\u0120Barbara": 16685, + "\u0120belonging": 16686, + "\u0120villain": 16687, + "istani": 16688, + "\u0120accountable": 16689, + "\u0120portions": 16690, + "\u0120Decl": 16691, + "Ur": 16692, + "\u0120Kate": 16693, + "gre": 16694, + "\u0120magazines": 16695, + "UCK": 16696, + "\u0120regulate": 16697, + "omon": 16698, + "\u0120Almost": 16699, + "\u0120overview": 16700, + "\u0120scram": 16701, + "\u0120loot": 16702, + "\u0120Fitz": 16703, + "\u0120characteristic": 16704, + "\u0120Snake": 16705, + "say": 16706, + "\u0120Rico": 16707, + "\u0120trait": 16708, + "\u0120Joined": 16709, + "aucus": 16710, + "\u0120adaptation": 16711, + "\u0120Airlines": 16712, + "\u0120archae": 16713, + "\u0120Ide": 16714, + "\u0120bikes": 16715, + "\u0120literary": 16716, + "\u0120influences": 16717, + "\u0120Used": 16718, + "Creat": 16719, + "\u0120plea": 16720, + "\u0120Defence": 16721, + "\u0120Assass": 16722, + "\u0120pond": 16723, + "ULT": 16724, + ")\"": 16725, + "\u0120evaluated": 16726, + "\u0120obtaining": 16727, + "\u0120demographic": 16728, + "\u0120vigil": 16729, + "aley": 16730, + "\u0120spouse": 16731, + "\u0120Seahawks": 16732, + "respons": 16733, + "\u0120Belt": 16734, + "umatic": 16735, + "\u0120rises": 16736, + "runner": 16737, + "\u0120Michelle": 16738, + "\u0120potent": 16739, + "race": 16740, + "\u0120PAC": 16741, + "Find": 16742, + "olesterol": 16743, + "ISS": 16744, + "\u0120Introduced": 16745, + "resses": 16746, + "ignment": 16747, + "Os": 16748, + "\u0120Tu": 16749, + "\u0120Dex": 16750, + "icides": 16751, + "\u0120sparked": 16752, + "\u0120Laura": 16753, + "\u0120Bryant": 16754, + "\u0120smiling": 16755, + "\u0120Nexus": 16756, + "\u0120defendants": 16757, + "\u0120Catal": 16758, + "\u0120dishes": 16759, + "shaped": 16760, + "\u0120prolong": 16761, + "mt": 16762, + "($": 16763, + "\u00e3\u0122\u0124": 16764, + "\u0120calculations": 16765, + "\u0120Same": 16766, + "\u0120piv": 16767, + "HH": 16768, + "\u0120cancelled": 16769, + "\u0120grin": 16770, + "\u0120territories": 16771, + "istically": 16772, + "Come": 16773, + "\u0120Parent": 16774, + "Project": 16775, + "\u0120neglig": 16776, + "\u0120Privacy": 16777, + "\u0120ammo": 16778, + "LECT": 16779, + "olutely": 16780, + "\u0120Epic": 16781, + "\u0120misunder": 16782, + "wal": 16783, + "April": 16784, + "mos": 16785, + "pathy": 16786, + "\u0120Carson": 16787, + "\u0120albums": 16788, + "\u0120Easy": 16789, + "\u0120pistol": 16790, + "<<": 16791, + "\u0120\\(": 16792, + "target": 16793, + "help": 16794, + "\u0120interpre": 16795, + "conscious": 16796, + "\u0120Housing": 16797, + "\u0120Joint": 16798, + "127": 16799, + "\u0120beers": 16800, + "science": 16801, + "\u0120Firefox": 16802, + "effective": 16803, + "\u0120Cabin": 16804, + "\u0120Okay": 16805, + "\u0120Applic": 16806, + "\u0120spacecraft": 16807, + "\u0120SR": 16808, + "vet": 16809, + "\u0120Strange": 16810, + "SB": 16811, + "\u0120corps": 16812, + "iberal": 16813, + "efficient": 16814, + "\u0120prevalence": 16815, + "\u0120economists": 16816, + "118": 16817, + "Thread": 16818, + "ordable": 16819, + "ODE": 16820, + "\u0120Cant": 16821, + "=-=-": 16822, + "ifiable": 16823, + "\u0120Around": 16824, + "\u0120pole": 16825, + "\u0120willingness": 16826, + "CLA": 16827, + "\u0120Kid": 16828, + "\u0120complement": 16829, + "\u0120scattered": 16830, + "\u0120inmates": 16831, + "\u0120bleeding": 16832, + "every": 16833, + "\u0120queue": 16834, + "\u0120Train": 16835, + "\u0120hij": 16836, + "\u0120melee": 16837, + "pleted": 16838, + "\u0120digit": 16839, + "\u0120gem": 16840, + "official": 16841, + "\u0120lifting": 16842, + "\u00d0\u00b5": 16843, + "Requ": 16844, + "itutes": 16845, + "\u0120packaging": 16846, + "\u0120Workers": 16847, + "hran": 16848, + "\u0120Lebanon": 16849, + "olesc": 16850, + "\u0120punished": 16851, + "\u0120Juan": 16852, + "\u0120jam": 16853, + "\u0120Document": 16854, + "\u0120mapping": 16855, + "icates": 16856, + "\u0120inevitably": 16857, + "\u0120vanilla": 16858, + "\u0120Ton": 16859, + "\u0120watches": 16860, + "\u0120leagues": 16861, + "\u0120initiated": 16862, + "degree": 16863, + "portion": 16864, + "\u0120recalls": 16865, + "\u0120ruin": 16866, + "\u0120melt": 16867, + "IAN": 16868, + "\u0120hem": 16869, + "Exp": 16870, + "\u0120baking": 16871, + "\u0120Colomb": 16872, + "atible": 16873, + "\u0120radius": 16874, + "plug": 16875, + "\u0120IF": 16876, + "etically": 16877, + "\u0120fict": 16878, + "HER": 16879, + "\u0120Tap": 16880, + "atinum": 16881, + "\u0120ink": 16882, + "\u0120coh": 16883, + "\u0120Wizard": 16884, + "both": 16885, + "tex": 16886, + "\u0120spends": 16887, + "\u0120Currently": 16888, + "\u0120Pit": 16889, + "\u0120neurons": 16890, + "ignt": 16891, + "\u0120rall": 16892, + "\u0120buses": 16893, + "building": 16894, + "\u0120adjustments": 16895, + "\u0120cried": 16896, + "iblical": 16897, + "atted": 16898, + "\u0120Zion": 16899, + "\u0120Matter": 16900, + "\u0120meditation": 16901, + "\u0120Dennis": 16902, + "\u0120ours": 16903, + "\u0120Tab": 16904, + "\u0120rankings": 16905, + "ortal": 16906, + "\u0120advers": 16907, + "\u0120surrender": 16908, + "\u0120Gob": 16909, + "cium": 16910, + "omas": 16911, + "imeter": 16912, + "\u0120multiplayer": 16913, + "\u0120heroin": 16914, + "\u0120optimistic": 16915, + "\u0120indicator": 16916, + "\u0120Brig": 16917, + "\u0120grocery": 16918, + "\u0120applicant": 16919, + "\u0120Rocket": 16920, + "vid": 16921, + "Exception": 16922, + "pent": 16923, + "\u0120organizing": 16924, + "\u0120encounters": 16925, + "\u0120TOD": 16926, + "\u0120jewel": 16927, + "Save": 16928, + "\u0120Christie": 16929, + "\u0120heating": 16930, + "\u0120lazy": 16931, + "\u0120CP": 16932, + "\u0120cousin": 16933, + "Config": 16934, + "\u0120regener": 16935, + "\u0120nearest": 16936, + "\u0120achieving": 16937, + "ENS": 16938, + "throw": 16939, + "\u0120Richmond": 16940, + "antle": 16941, + "2002": 16942, + "\u0120anten": 16943, + "bird": 16944, + "133": 16945, + "\u0120narc": 16946, + "raint": 16947, + "unny": 16948, + "\u0120Hispanic": 16949, + "ournaments": 16950, + "\u0120prophe": 16951, + "\u0120Thailand": 16952, + "\u0120Ti": 16953, + "\u0120injection": 16954, + "\u0120inherit": 16955, + "ravis": 16956, + "\u0120medi": 16957, + "\u0120whoever": 16958, + "\u0120DEBUG": 16959, + "GP": 16960, + "\u0120Hud": 16961, + "Card": 16962, + "prom": 16963, + "\u0120por": 16964, + "\u0120overhead": 16965, + "Law": 16966, + "\u0120violate": 16967, + "\u0120heated": 16968, + "\u0120descriptions": 16969, + "\u0120achievements": 16970, + "\u0120Beer": 16971, + "\u0120Quant": 16972, + "Was": 16973, + "\u0120eighth": 16974, + "\u0120Iv": 16975, + "\u0120specialized": 16976, + "UPDATE": 16977, + "\u0120Delta": 16978, + "Pop": 16979, + "Jul": 16980, + "\u0120Ask": 16981, + "ophy": 16982, + "\u0120newsletters": 16983, + "\u0120Tool": 16984, + "\u0120gard": 16985, + "\u0120Confeder": 16986, + "\u0120GMT": 16987, + "\u0120Abbott": 16988, + "\u0120immunity": 16989, + "\u0120VM": 16990, + "Islam": 16991, + "\u0120implicit": 16992, + "wd": 16993, + "\u01201944": 16994, + "ravity": 16995, + "ometric": 16996, + "\u0120surviving": 16997, + "urai": 16998, + "\u0120Prison": 16999, + "\u0120rust": 17000, + "\u0120Sketch": 17001, + "\u0120bees": 17002, + "\u0120Theory": 17003, + "\u0120merit": 17004, + "Tex": 17005, + "chat": 17006, + "\u0120mim": 17007, + "\u0120paste": 17008, + "\u0120Koch": 17009, + "\u0120ignorance": 17010, + "\u0120Shoot": 17011, + "\u0120basement": 17012, + "United": 17013, + "\u0120Advis": 17014, + "height": 17015, + "\u0120foster": 17016, + "\u0120detain": 17017, + "information": 17018, + "\u0120neural": 17019, + "';": 17020, + "\u0120proves": 17021, + "allery": 17022, + "\u0120invitation": 17023, + "umbers": 17024, + "\u0120cattle": 17025, + "\u0120bicycle": 17026, + "zi": 17027, + "\u0120consultant": 17028, + "\u0120apology": 17029, + "\u0120Tiger": 17030, + "\u0120123": 17031, + "999": 17032, + "\u0120individually": 17033, + "rt": 17034, + "igion": 17035, + "\u0120Brazilian": 17036, + "\u0120disturb": 17037, + "\u0120entrepreneurs": 17038, + "\u0120forests": 17039, + "cerpt": 17040, + "plates": 17041, + "pher": 17042, + "clipse": 17043, + "\u0120twitter": 17044, + "\u0120acids": 17045, + "ographical": 17046, + "hum": 17047, + "\u0120Bald": 17048, + "ifully": 17049, + "\u0120compiler": 17050, + "\u0120DA": 17051, + "\u0120donor": 17052, + "asi": 17053, + "\u0120tribal": 17054, + "lash": 17055, + "\u0120Config": 17056, + "\u0120applicants": 17057, + "\u0120salaries": 17058, + "135": 17059, + "Putin": 17060, + "\u0120Focus": 17061, + "irs": 17062, + "\u0120misconduct": 17063, + "\u0120Haz": 17064, + "\u0120eaten": 17065, + "Mobile": 17066, + "Muslim": 17067, + "\u0120Marcus": 17068, + "viol": 17069, + "\u0120favorable": 17070, + "\u0120stub": 17071, + "adin": 17072, + "\u0120Hob": 17073, + "\u0120faithful": 17074, + "\u0120electronics": 17075, + "\u0120vacuum": 17076, + "wait": 17077, + "backed": 17078, + "economic": 17079, + "dist": 17080, + "\u0120tenure": 17081, + "\u0120sincere": 17082, + "\u0120Together": 17083, + "\u0120Wave": 17084, + "\u0120progression": 17085, + "\u0120denying": 17086, + "\u0120distress": 17087, + "braska": 17088, + "third": 17089, + "\u0120mixing": 17090, + "\u0120colonial": 17091, + "\u0120privately": 17092, + "\u0120unrest": 17093, + "aternity": 17094, + "\u0120premises": 17095, + "anti": 17096, + "gregation": 17097, + "\u0120licence": 17098, + "\u0120Hind": 17099, + "\u0120Samuel": 17100, + "\u0120convincing": 17101, + "\u0120Ace": 17102, + "\u0120Rust": 17103, + "\u0120Netanyahu": 17104, + "\u0120handles": 17105, + "\u0120Patch": 17106, + "oriented": 17107, + "aho": 17108, + "\u0120Gonz": 17109, + "\u0120hackers": 17110, + "claimer": 17111, + "\u0120customs": 17112, + "\u0120Gran": 17113, + "fighters": 17114, + "\u0120luc": 17115, + "\u0120manuscript": 17116, + "arenthood": 17117, + "\u0120devil": 17118, + "\u0120warriors": 17119, + "\u0120offenders": 17120, + "William": 17121, + "\u0120holidays": 17122, + "\u0120nightmare": 17123, + "\u0120lever": 17124, + "ifferent": 17125, + "Stat": 17126, + "\u0120exhibition": 17127, + "puted": 17128, + "\u0120Pure": 17129, + "\u0120alpha": 17130, + "\u0120enthusiasm": 17131, + "\u0120Representatives": 17132, + "EAR": 17133, + "\u0120Typ": 17134, + "\u0120wheat": 17135, + "\u0120Alf": 17136, + "\u0120correction": 17137, + "\u0120evangel": 17138, + "ATT": 17139, + "Miss": 17140, + "\u0120soup": 17141, + "\u0120implied": 17142, + "param": 17143, + "\u0120sexy": 17144, + "\u0120Lux": 17145, + "\u0120republic": 17146, + "patch": 17147, + "ablish": 17148, + "\u0120icons": 17149, + "\u0120fathers": 17150, + "\u0120GET": 17151, + "\u0120Carib": 17152, + "\u0120regulated": 17153, + "\u0120Cohen": 17154, + "\u0120Bobby": 17155, + "\u0120ner": 17156, + "\u0120bent": 17157, + "ventory": 17158, + "\u0120Along": 17159, + "\u0120EST": 17160, + "\u0120Wallace": 17161, + "\u0120murders": 17162, + "rise": 17163, + "kell": 17164, + "\u0120Commonwealth": 17165, + "\u0120nasty": 17166, + "eta": 17167, + "\u0120MIT": 17168, + "\u0120administered": 17169, + "\u0120genuinely": 17170, + "Editor": 17171, + "nick": 17172, + "\u0120hydro": 17173, + "********************************": 17174, + "\u0120Ble": 17175, + "\u0120fines": 17176, + "\u0120gorge": 17177, + "ausible": 17178, + "rh": 17179, + "\u0120apple": 17180, + "mentioned": 17181, + "\u0120rope": 17182, + "otyp": 17183, + "HR": 17184, + "\u0120disappointing": 17185, + "\u0120cage": 17186, + "nik": 17187, + "\u0120doubts": 17188, + "\u0120FREE": 17189, + "prints": 17190, + "\u0120MUST": 17191, + "\u0120vendors": 17192, + "\u0120Inqu": 17193, + "\u0120liberals": 17194, + "\u0120contractor": 17195, + "\u0120upside": 17196, + "children": 17197, + "\u0120tricky": 17198, + "\u0120regulators": 17199, + "charged": 17200, + "liter": 17201, + "\u0120***": 17202, + "\u0120rebell": 17203, + "lang": 17204, + "\u0120locals": 17205, + "\u0120physicians": 17206, + "\u0120hey": 17207, + "arse": 17208, + "tm": 17209, + "\u0120Lex": 17210, + "\u0120behavioral": 17211, + "successful": 17212, + "FX": 17213, + "\u0120brick": 17214, + "ovic": 17215, + "\u0120conform": 17216, + "\u0120reviewing": 17217, + "\u0120insights": 17218, + "\u0120biology": 17219, + "\u0120Remove": 17220, + "\u0120Extra": 17221, + "\u0120committing": 17222, + "induced": 17223, + "ignty": 17224, + "igm": 17225, + "\u0120atomic": 17226, + "Common": 17227, + "\u0120EM": 17228, + "\u0120Pere": 17229, + "\u0120Items": 17230, + "eh": 17231, + "\u0120preserved": 17232, + "\u0120Hood": 17233, + "\u0120prisoner": 17234, + "\u0120bankruptcy": 17235, + "\u0120gren": 17236, + "ushes": 17237, + "\u0120exploitation": 17238, + "\u0120signatures": 17239, + "\u0120finan": 17240, + "],\"": 17241, + "\u0120MR": 17242, + "\u0120meg": 17243, + "remlin": 17244, + "\u0120musicians": 17245, + "\u0120selecting": 17246, + "\u0120examining": 17247, + "INK": 17248, + "lated": 17249, + "Hi": 17250, + "\u0120artic": 17251, + "\u0120pets": 17252, + "\u0120impair": 17253, + "\u0120MAN": 17254, + "\u0120tablets": 17255, + "include": 17256, + "Range": 17257, + "\u0120caut": 17258, + "\u0120logs": 17259, + "\u0120mounting": 17260, + "\u0120unaware": 17261, + "\u0120dynamics": 17262, + "\u0120Palestine": 17263, + "\u0120Quarter": 17264, + "\u0120Purple": 17265, + "\u0120ma": 17266, + "\u0120Import": 17267, + "\u0120collections": 17268, + "ciation": 17269, + "\u0120successor": 17270, + "\u0120clone": 17271, + "\u0120aiming": 17272, + "\u0120possessed": 17273, + "\u0120sticking": 17274, + "\u0120shaking": 17275, + "\u0120locate": 17276, + "\u0120Hockey": 17277, + "Turn": 17278, + "170": 17279, + "\u0120fifteen": 17280, + "\u0120Harrison": 17281, + "\u0120continuously": 17282, + "\u0120TC": 17283, + "\u0120Valent": 17284, + "\u0120Rescue": 17285, + "\u0120bypass": 17286, + "amount": 17287, + "\u0120mast": 17288, + "\u0120protects": 17289, + "\u0120artistic": 17290, + "\u0120sometime": 17291, + "\u0120shoe": 17292, + "\u0120shouted": 17293, + "ificant": 17294, + "etitive": 17295, + "\u0120Register": 17296, + "\u0120Jin": 17297, + "\u0120concentrated": 17298, + "lington": 17299, + "onies": 17300, + "\u0120generator": 17301, + "yrim": 17302, + "\u0120Armen": 17303, + "\u0120clearing": 17304, + "ido": 17305, + "\u0120TW": 17306, + "alph": 17307, + "\u0120ladies": 17308, + "Hard": 17309, + "\u0120dialog": 17310, + "\u0120inputs": 17311, + "\u00e6\u013e": 17312, + "\u0120poses": 17313, + "\u0120slots": 17314, + "\u0120Premium": 17315, + "\u0120leaks": 17316, + "\u0120bosses": 17317, + "\u0120113": 17318, + "course": 17319, + "Acc": 17320, + "\u0120Newton": 17321, + "\u0120Austria": 17322, + "\u0120Mage": 17323, + "\u0120teaches": 17324, + "abad": 17325, + "\u0120wears": 17326, + "\u0120cyl": 17327, + "\u0120curse": 17328, + "\u0120Sales": 17329, + "\u0120Wings": 17330, + "\u0120psy": 17331, + "\u0120gaps": 17332, + "\u0120Iceland": 17333, + "\u0120Pinterest": 17334, + "\u0120landlord": 17335, + "\u0120definitions": 17336, + "\u0120Ker": 17337, + "\u0120sufficiently": 17338, + "\u0120Pence": 17339, + "\u0120Architect": 17340, + "\u0120surpass": 17341, + "\u0120114": 17342, + "\u0120superhero": 17343, + "\u0120Disease": 17344, + "\u0120priests": 17345, + "\u0120Culture": 17346, + "\u0120definitive": 17347, + "\u0120secretly": 17348, + "\u0120Dance": 17349, + "install": 17350, + "chief": 17351, + "\u0120Jessica": 17352, + "Would": 17353, + "Updated": 17354, + "\u0120locker": 17355, + "\u0120Kay": 17356, + "\u0120memorial": 17357, + "\u00e8\u00a6": 17358, + "fat": 17359, + "\u0120disgu": 17360, + "\u0120flavors": 17361, + "\u0120Baseball": 17362, + "\u0120Resistance": 17363, + "\u0120kicks": 17364, + "\u0120env": 17365, + "\u0120teenagers": 17366, + "Dark": 17367, + "\u0120CAR": 17368, + "\u0120halt": 17369, + "\u0120LG": 17370, + "\u0120Gabriel": 17371, + "\u0120fever": 17372, + "\u0120satur": 17373, + "\u0120mall": 17374, + "\u0120affiliate": 17375, + "\u0120Sleep": 17376, + "\u0120Specific": 17377, + "\u0120Vel": 17378, + "\u0120jar": 17379, + "\u0120Sacred": 17380, + "\u0120Edwards": 17381, + "\u0120ACL": 17382, + "\u0120retained": 17383, + "\u0120Giant": 17384, + "\u0120limitation": 17385, + "inces": 17386, + "\u0120refusal": 17387, + "\u0120Tale": 17388, + "\u0120Butler": 17389, + "\u0120accidents": 17390, + "\u0120CSS": 17391, + "\u0120imported": 17392, + "\u0120Copy": 17393, + "\u00ce\u00b1": 17394, + "ERT": 17395, + "zel": 17396, + "\u0120divisions": 17397, + "hots": 17398, + "\u0120Alb": 17399, + "\u0120DS": 17400, + "Loader": 17401, + "Washington": 17402, + "atisf": 17403, + "\u0120Creative": 17404, + "\\.": 17405, + "\u0120Autom": 17406, + "redict": 17407, + "\u0120receptor": 17408, + "\u0120Carlos": 17409, + "Method": 17410, + "oka": 17411, + "\u0120malicious": 17412, + "\u0120stepping": 17413, + ",[": 17414, + "\u0120Dad": 17415, + "\u0120attraction": 17416, + "\u0120Effects": 17417, + "\u0120Pirate": 17418, + "\u0120Cer": 17419, + "\u0120Industry": 17420, + "\u0120Rud": 17421, + "\u0120charter": 17422, + "\u0120dining": 17423, + "\u0120insists": 17424, + "\u0120configure": 17425, + "\u0120(#": 17426, + "\u0120Simple": 17427, + "\u0120Scroll": 17428, + "UTC": 17429, + "175": 17430, + "\u0120Kon": 17431, + "\u0120marketplace": 17432, + "\u0120\u00e3\u0124": 17433, + "\u0120refres": 17434, + "\u0120gates": 17435, + "erred": 17436, + "\u0120Pod": 17437, + "\u0120behave": 17438, + "Frank": 17439, + "node": 17440, + "\u0120endorsed": 17441, + "hett": 17442, + "asive": 17443, + "\u0120Homeland": 17444, + "\u0120rides": 17445, + "\u0120Leave": 17446, + "erness": 17447, + "\u0120flooding": 17448, + "AFP": 17449, + "\u0120risen": 17450, + "\u0120continually": 17451, + "\u0120unanim": 17452, + "\u0120Contract": 17453, + "\u0120Pas": 17454, + "\u0120guided": 17455, + "\u0120Chile": 17456, + "bd": 17457, + "\u0120succ": 17458, + "ptic": 17459, + "\u0120committees": 17460, + "\u0120Luther": 17461, + "\u0120Anyone": 17462, + "\u0120sab": 17463, + "124": 17464, + "\u0120pixel": 17465, + "\u0120Bak": 17466, + "\u0120Tag": 17467, + "\u0120Bennett": 17468, + "Enter": 17469, + "small": 17470, + "\u0120Presidential": 17471, + "\u0120pul": 17472, + "\u0120contrace": 17473, + "archive": 17474, + "\u0120coastal": 17475, + "\u0120Kids": 17476, + "192": 17477, + "\u00e2\u0122\u00b2": 17478, + "icky": 17479, + "INGTON": 17480, + "\u0120wolf": 17481, + "\u0120Stalin": 17482, + "Tur": 17483, + "idget": 17484, + "amas": 17485, + "\u0120Unless": 17486, + "\u0120sponsor": 17487, + "\u0120morph": 17488, + "\u0120Choose": 17489, + "\u0120runner": 17490, + "\u0120unbel": 17491, + "\u0120mud": 17492, + "\u0120Mana": 17493, + "\u0120dubbed": 17494, + "\u0120godd": 17495, + "urers": 17496, + "window": 17497, + "\u0120relied": 17498, + "\u0120celebrating": 17499, + "osc": 17500, + "\u0120135": 17501, + "\u0120lobbying": 17502, + "\u0120incomplete": 17503, + "\u0120restriction": 17504, + "\u0120incap": 17505, + "itus": 17506, + "\u0120expectation": 17507, + "\u0120Apollo": 17508, + "\u0120intens": 17509, + "\u0120sync": 17510, + "GH": 17511, + "\u0120manipulation": 17512, + "BY": 17513, + "\u0120spear": 17514, + "\u0120breasts": 17515, + "\u0120volcan": 17516, + "ilia": 17517, + "Material": 17518, + "\u0120formats": 17519, + "\u0120Bast": 17520, + "\u0120parliamentary": 17521, + "\u0120snake": 17522, + "\u0120servants": 17523, + "\u0120Trudeau": 17524, + "\u0120Grim": 17525, + "\u0120Arabic": 17526, + "\u0120SCP": 17527, + "\u0120Boys": 17528, + "station": 17529, + "\u0120prospective": 17530, + "orde": 17531, + "initialized": 17532, + "\u0120bored": 17533, + "ABLE": 17534, + "\u0120accessed": 17535, + "\u0120taxi": 17536, + "\u0120Shell": 17537, + "aiden": 17538, + "ursed": 17539, + "inates": 17540, + "\u0120Insurance": 17541, + "\u0120Pete": 17542, + "September": 17543, + "650": 17544, + "\u0120adventures": 17545, + "\u0120Cover": 17546, + "\u0120tribute": 17547, + "\u0120sketch": 17548, + "\u0120empower": 17549, + "\u0120\u00d8": 17550, + "\u0120Glenn": 17551, + "\u0120Daw": 17552, + "=\\\"": 17553, + "\u0120Politics": 17554, + "\u0120guides": 17555, + "\u0120dioxide": 17556, + "\u0120Gore": 17557, + "\u0120Bright": 17558, + "\u0120Sierra": 17559, + "\u0120valued": 17560, + "cond": 17561, + "\u0120pointer": 17562, + "Select": 17563, + "\u0120risky": 17564, + "\u0120absorb": 17565, + "images": 17566, + "\u0120refuses": 17567, + "\u0120bonuses": 17568, + "___": 17569, + "\u0120hilar": 17570, + "\u0120Features": 17571, + "220": 17572, + "\u0120Collector": 17573, + "Foot": 17574, + "\u01201964": 17575, + "culus": 17576, + "\u0120dawn": 17577, + "\u0120workout": 17578, + "\u0120LO": 17579, + "\u0120philosophical": 17580, + "\u0120Sandy": 17581, + "\u0120Youth": 17582, + "\u0120liable": 17583, + "Af": 17584, + "blue": 17585, + "\u0120overturn": 17586, + "lessness": 17587, + "\u0120Tribune": 17588, + "\u0120Ing": 17589, + "\u0120factories": 17590, + "\u0120catches": 17591, + "\u0120prone": 17592, + "\u0120matrix": 17593, + "\u0120login": 17594, + "\u0120inacc": 17595, + "\u0120exert": 17596, + "sys": 17597, + "\u0120needle": 17598, + "\u0120Qur": 17599, + "\u0120notified": 17600, + "oulder": 17601, + "tx": 17602, + "\u0120reminds": 17603, + "\u0120publishers": 17604, + "\u0120nort": 17605, + "\u0120git": 17606, + "\u0120flies": 17607, + "\u0120Emily": 17608, + "\u0120flowing": 17609, + "\u0120Alien": 17610, + "\u0120Strateg": 17611, + "\u0120hardest": 17612, + "\u0120modification": 17613, + "API": 17614, + "\u0120MY": 17615, + "\u0120crashes": 17616, + "stairs": 17617, + "number": 17618, + "\u0120urging": 17619, + "channel": 17620, + "\u0120Falcon": 17621, + "\u0120inhabitants": 17622, + "\u0120terrifying": 17623, + "\u0120utilize": 17624, + "\u0120banner": 17625, + "\u0120cigarettes": 17626, + "\u0120senses": 17627, + "\u0120Holmes": 17628, + "\u0120practition": 17629, + "\u0120Phillips": 17630, + "otto": 17631, + "\u0120compile": 17632, + "Model": 17633, + "\u0120Ko": 17634, + "\u0120[]": 17635, + "Americans": 17636, + "\u0120Terms": 17637, + "\u0120medications": 17638, + "\u0120Ana": 17639, + "\u0120fundamentally": 17640, + "\u0120Notice": 17641, + "\u0120weaker": 17642, + "\u01200000": 17643, + "\u0120garlic": 17644, + "\u0120outbreak": 17645, + "\u0120economist": 17646, + "\u0120Birth": 17647, + "\u0120obstacles": 17648, + "arcer": 17649, + "\u0120Orthodox": 17650, + "\u0120placebo": 17651, + "\u0120Crew": 17652, + "aspberry": 17653, + "\u0120Angels": 17654, + "\u0120discharge": 17655, + "\u0120destructive": 17656, + "117": 17657, + "\u0120Rising": 17658, + "\u0120dairy": 17659, + "late": 17660, + "\u0120collision": 17661, + "\u0120Tigers": 17662, + "eanor": 17663, + "ocumented": 17664, + "\u0120Invalid": 17665, + "\u0120dont": 17666, + "\u0120Liter": 17667, + "\u0120Va": 17668, + "\u0120hydrogen": 17669, + "\u0120variants": 17670, + "\u0120Browns": 17671, + "\u01201965": 17672, + "\u0120indigenous": 17673, + "\u0120trades": 17674, + "\u0120remainder": 17675, + "\u0120swept": 17676, + "\u0120Impact": 17677, + "\u0120redist": 17678, + "\u0120unint": 17679, + "graduate": 17680, + "\u00e3\u0125\u0137": 17681, + "\u0120WILL": 17682, + "\u00e3\u0123\u00ae\u00e7": 17683, + "\u0120Critical": 17684, + "\u0120fisher": 17685, + "\u0120vicious": 17686, + "\u0120reversed": 17687, + "Year": 17688, + "\u0120Sox": 17689, + "\u0120shootings": 17690, + "\u0120filming": 17691, + "\u0120touchdowns": 17692, + "aires": 17693, + "mel": 17694, + "\u0120grandfather": 17695, + "\u0120affection": 17696, + "ingle": 17697, + "\u0120overly": 17698, + "Additional": 17699, + "\u0120supreme": 17700, + "\u0120Grad": 17701, + "\u0120sporting": 17702, + "\u0120mercy": 17703, + "\u0120Brooks": 17704, + "ounty": 17705, + "\u0120performs": 17706, + "\u0120tightly": 17707, + "\u0120demons": 17708, + "\u0120killings": 17709, + "\u0120faction": 17710, + "\u0120Nova": 17711, + "auts": 17712, + "\u0120undoubtedly": 17713, + "arin": 17714, + "\u0120underway": 17715, + "rak": 17716, + "\u0120liv": 17717, + "\u0120Region": 17718, + "\u0120briefing": 17719, + "sers": 17720, + "cloud": 17721, + "\u0120Mik": 17722, + "usp": 17723, + "\u0120prediction": 17724, + "azor": 17725, + "\u0120portable": 17726, + "\u0120Gand": 17727, + "\u0120presenting": 17728, + "\u01201080": 17729, + "\u00c2\u00bb": 17730, + "ushi": 17731, + "\u0120Spark": 17732, + "thereum": 17733, + "\u0120justification": 17734, + "\u0120Ny": 17735, + "\u0120contractors": 17736, + "mingham": 17737, + "\u0120Style": 17738, + "\u00e5\u0127": 17739, + "\u0120Chronicles": 17740, + "\u0120Picture": 17741, + "\u0120proving": 17742, + "\u0120wives": 17743, + "sett": 17744, + "\u0120molecules": 17745, + "\u0120Fairy": 17746, + "\u0120consisting": 17747, + "\u0120pier": 17748, + "alone": 17749, + "inition": 17750, + "\u0120nucle": 17751, + "json": 17752, + "\u0120gotta": 17753, + "\u0120mobil": 17754, + "\u0120verbal": 17755, + "arium": 17756, + "\u0120monument": 17757, + "ucked": 17758, + "\u0120256": 17759, + "Tech": 17760, + "minecraft": 17761, + "\u0120Track": 17762, + "\u0120tile": 17763, + "\u0120compatibility": 17764, + "asis": 17765, + "\u0120sadd": 17766, + "\u0120instructed": 17767, + "\u0120Mueller": 17768, + "\u0120lethal": 17769, + "\u0120hormone": 17770, + "\u0120orche": 17771, + "else": 17772, + "\u0120skelet": 17773, + "\u0120entertaining": 17774, + "\u0120minimize": 17775, + "again": 17776, + "\u0120undergo": 17777, + "\u0120constraints": 17778, + "\u0120cigarette": 17779, + "\u0120Islamist": 17780, + "\u0120travels": 17781, + "\u0120Panthers": 17782, + "lings": 17783, + "Care": 17784, + "\u0120lawsuits": 17785, + "uras": 17786, + "\u0120cryst": 17787, + "\u0120lowered": 17788, + "\u0120aerial": 17789, + "\u0120combinations": 17790, + "\u0120haun": 17791, + "\u0120cha": 17792, + "\u0120vine": 17793, + "\u0120quantities": 17794, + "\u0120linking": 17795, + "bank": 17796, + "\u0120soy": 17797, + "Bill": 17798, + "\u0120Angela": 17799, + "\u0120recipient": 17800, + "\u0120Protest": 17801, + "\u0120socket": 17802, + "\u0120solidarity": 17803, + "\u0120\u00e2\u0128": 17804, + "mill": 17805, + "\u0120varies": 17806, + "\u0120Pakistani": 17807, + "Dragon": 17808, + "\u0120une": 17809, + "\u0120horizon": 17810, + "\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142": 17811, + "\u0120provinces": 17812, + "\u0120frankly": 17813, + "\u0120enacted": 17814, + "notes": 17815, + "['": 17816, + "\u0120192": 17817, + "ocracy": 17818, + "\u0120endorsement": 17819, + "\u0120overtime": 17820, + "True": 17821, + "Lab": 17822, + "licted": 17823, + "\u0120DNC": 17824, + "\u0120beats": 17825, + "\u0120Jamie": 17826, + "152": 17827, + "\u0120INT": 17828, + "Contact": 17829, + "\u0120accounted": 17830, + "hash": 17831, + "\u0120Packers": 17832, + "pires": 17833, + "\u0120lesbian": 17834, + "\u0120amendments": 17835, + "\u0120hopeful": 17836, + "\u0120Finland": 17837, + "\u0120spotlight": 17838, + "\u0120configured": 17839, + "\u0120troubled": 17840, + "\u0120gaze": 17841, + "\u0120Calgary": 17842, + "\u0120reliability": 17843, + "\u0120insurg": 17844, + "swer": 17845, + "buy": 17846, + "\u0120Skin": 17847, + "\u0120pixels": 17848, + "\u0120handgun": 17849, + "\u0120paras": 17850, + "\u0120categor": 17851, + "\u0120EL": 17852, + "\u0120Rex": 17853, + "Indeed": 17854, + "\u0120kinda": 17855, + "\u0120conjunction": 17856, + "\u0120Bryan": 17857, + "\u0120Manufact": 17858, + "yang": 17859, + "Plus": 17860, + "SQL": 17861, + "ishment": 17862, + "\u0120dominate": 17863, + "\u0120nail": 17864, + "\u0120oath": 17865, + "\u0120erupt": 17866, + "\u0120Fine": 17867, + "itbart": 17868, + "\u0120Chip": 17869, + "\u0120Abd": 17870, + "\u0120Nam": 17871, + "\u0120buyer": 17872, + "\u0120dissent": 17873, + "Leaks": 17874, + "Contin": 17875, + "\u0120rider": 17876, + "\u0120Someone": 17877, + "\u0120illusion": 17878, + "cin": 17879, + "\u0120Boeing": 17880, + "\u0120inadequ": 17881, + "ovation": 17882, + "iants": 17883, + "\u0120rebuild": 17884, + "450": 17885, + "\u0120Destiny": 17886, + "SW": 17887, + "\u0120Till": 17888, + "Hit": 17889, + "iaz": 17890, + "\u0120Bangl": 17891, + "achers": 17892, + "\u0120Reform": 17893, + "\u0120segments": 17894, + "\u0120systematic": 17895, + "dc": 17896, + "\u0120Conservatives": 17897, + "\u0120portal": 17898, + "hor": 17899, + "\u0120Dragonbound": 17900, + "\u0120dragged": 17901, + "omo": 17902, + "\u0120thee": 17903, + "advert": 17904, + "\u0120Reports": 17905, + "\u0120Et": 17906, + "\u0120barrels": 17907, + "August": 17908, + "\u0120comparisons": 17909, + "\u0120hex": 17910, + "\u0120anthrop": 17911, + "\"[": 17912, + "borough": 17913, + "abi": 17914, + "\u0120pictured": 17915, + "playing": 17916, + "\u0120Address": 17917, + "\u0120Mirror": 17918, + "Smith": 17919, + "\u0120tires": 17920, + "\u0120NPR": 17921, + "AAAA": 17922, + "\u0120classification": 17923, + "\u0120Than": 17924, + "\u0120Harm": 17925, + "\u0120RA": 17926, + "\u0120rejection": 17927, + "mination": 17928, + "\u0120ranged": 17929, + "\u0120Falls": 17930, + "DI": 17931, + "Host": 17932, + "\u00e3\u0124\u00b4": 17933, + "\u0120Example": 17934, + "listed": 17935, + "thirds": 17936, + "\u0120safegu": 17937, + "brand": 17938, + "\u0120probable": 17939, + "Canada": 17940, + "ITION": 17941, + "\u0120Qaeda": 17942, + "\u0120chick": 17943, + "\u0120imports": 17944, + "hit": 17945, + "loc": 17946, + "WW": 17947, + "\u0120blew": 17948, + "\u0120anytime": 17949, + "\u0120wholes": 17950, + "iked": 17951, + "\u0120calculation": 17952, + "create": 17953, + "\u0120Ori": 17954, + "\u0120upgraded": 17955, + "\u0120appar": 17956, + "utory": 17957, + "\u0120Mol": 17958, + "Brit": 17959, + "\u0120Jong": 17960, + "INAL": 17961, + "\u0120Starting": 17962, + "\u0120dice": 17963, + "urtle": 17964, + "\u0120relying": 17965, + "closure": 17966, + "\u0120profitable": 17967, + "\u0120slaughter": 17968, + "\u0120Manual": 17969, + "caster": 17970, + "\u0120\"$": 17971, + "\u0120feather": 17972, + "\u0120Simply": 17973, + "ieves": 17974, + "\u0120deterior": 17975, + "\u0120PCI": 17976, + "\u0120stamp": 17977, + "\u0120flaws": 17978, + "\u0120shade": 17979, + "hammer": 17980, + "\u0120passport": 17981, + "\u0120conting": 17982, + "amel": 17983, + "\u0120observers": 17984, + "\u0120neglect": 17985, + "\u0120RB": 17986, + "\u0120Brotherhood": 17987, + "\u0120skeptical": 17988, + "family": 17989, + "usk": 17990, + "\u0120emotionally": 17991, + "\u00e2\u013b": 17992, + "\u0120Beta": 17993, + "asonable": 17994, + "idity": 17995, + "\u0120Mul": 17996, + "\u0120kicking": 17997, + "\u0120Carm": 17998, + "ollah": 17999, + "VERTIS": 18000, + "\u0120Athen": 18001, + "\u0120ladder": 18002, + "\u0120Bullet": 18003, + "\u00e5\u00a3": 18004, + "0001": 18005, + "\u0120Wildlife": 18006, + "\u0120Mask": 18007, + "\u0120Nan": 18008, + "Rev": 18009, + "\u0120unacceptable": 18010, + "legal": 18011, + "\u0120crowded": 18012, + "agi": 18013, + "\u0120Cox": 18014, + "je": 18015, + "\u0120morality": 18016, + "\u0120fuels": 18017, + "\u0120cables": 18018, + "\u0120mankind": 18019, + "\u0120Caribbean": 18020, + "\u0120anchor": 18021, + "\u0120byte": 18022, + "\u0120Often": 18023, + "\u0120Oz": 18024, + "\u0120crafted": 18025, + "\u0120historian": 18026, + "\u0120Wu": 18027, + "\u0120towers": 18028, + "\u0120Citizens": 18029, + "\u0120helm": 18030, + "\u0120credentials": 18031, + "\u0120singular": 18032, + "\u0120Jesse": 18033, + "\u0120tackles": 18034, + "\u0120contempt": 18035, + "\u0120afore": 18036, + "\u0120Shadows": 18037, + "\u0120nil": 18038, + "\u0120urgent": 18039, + "apple": 18040, + "blood": 18041, + "\u0120von": 18042, + "\u0120offline": 18043, + "\u0120breathe": 18044, + "\u0120jumps": 18045, + "\u0120irrelevant": 18046, + "oxic": 18047, + "omal": 18048, + "important": 18049, + "Jim": 18050, + "\u0120gloves": 18051, + "arming": 18052, + "depth": 18053, + "\u0120talents": 18054, + "ookie": 18055, + "\u0120SB": 18056, + "\u0120palm": 18057, + "uffs": 18058, + "esta": 18059, + "IGH": 18060, + "\u0120canon": 18061, + "\u0120Verizon": 18062, + "\u0120Ple": 18063, + "\u0120coupled": 18064, + "velt": 18065, + "\u0120fundraising": 18066, + "\u0120Getting": 18067, + "\u0120DLC": 18068, + "\u0120mathematical": 18069, + "\u0120HS": 18070, + "\u0120Cardinals": 18071, + "telling": 18072, + "\u0120sponsors": 18073, + "\u0120\u00cf": 18074, + "\u0120Bulls": 18075, + "option": 18076, + "\u0120propose": 18077, + "\u0120memorable": 18078, + "\u0120embraced": 18079, + "\u0120declining": 18080, + "Health": 18081, + "eda": 18082, + "\u0120};": 18083, + "\u0120spam": 18084, + "mile": 18085, + "\u0120pitcher": 18086, + "\u0120Eight": 18087, + "\u0120caring": 18088, + "utic": 18089, + "role": 18090, + "\u0120airline": 18091, + "ernandez": 18092, + "\u0120Athlet": 18093, + "\u0120certification": 18094, + "uxe": 18095, + "riger": 18096, + "\u0120empir": 18097, + "\u0120sensation": 18098, + "\u0120dism": 18099, + "\u0120bolt": 18100, + "\u0120evolve": 18101, + "House": 18102, + "\u0120consultation": 18103, + "\u0120Duty": 18104, + "\u0120touches": 18105, + "\u0120Nathan": 18106, + "\u0120faint": 18107, + "had": 18108, + "\"(": 18109, + "\u0120Consumer": 18110, + "\u0120Extreme": 18111, + "\u0120127": 18112, + "\u0120Herm": 18113, + "\u0120Sacrament": 18114, + "izoph": 18115, + "\u0120anxious": 18116, + "ulously": 18117, + "\u0120socially": 18118, + "\u0120UTC": 18119, + "\u0120solving": 18120, + "\u0120Letter": 18121, + "History": 18122, + "educ": 18123, + "Price": 18124, + "));": 18125, + "\u0120reload": 18126, + "amic": 18127, + "\u0120pork": 18128, + "\u0120discourse": 18129, + "\u0120tournaments": 18130, + "airo": 18131, + "\u0120Kur": 18132, + "\u0120Costa": 18133, + "\u0120violating": 18134, + "\u0120interfere": 18135, + "\u0120recreational": 18136, + "uffle": 18137, + "\u0120speeches": 18138, + "\u0120needing": 18139, + "\u0120remembers": 18140, + "\u0120credited": 18141, + "nia": 18142, + "focused": 18143, + "amera": 18144, + "\u0120bru": 18145, + "umbs": 18146, + "\u0120Cuban": 18147, + "\u0120preceding": 18148, + "\u0120nonsense": 18149, + "acial": 18150, + "\u0120smartphones": 18151, + "\u0120Stories": 18152, + "Sports": 18153, + "\u0120Emergency": 18154, + "ouncing": 18155, + "efined": 18156, + "\u0120ber": 18157, + "\u0120consulting": 18158, + "\u0120masters": 18159, + "heastern": 18160, + ".\"[": 18161, + "\u0120Running": 18162, + "\u0120suscept": 18163, + "\u0120Feng": 18164, + "America": 18165, + "prises": 18166, + "stitial": 18167, + "\u0120Weekly": 18168, + "\u0120Greater": 18169, + "modules": 18170, + "ifter": 18171, + "Graphics": 18172, + "uler": 18173, + "\u0120wholly": 18174, + "\u0120suppress": 18175, + "\u0120concealed": 18176, + "\u0120happily": 18177, + "\u0120accepts": 18178, + "\u0120Enjoy": 18179, + "\u0120rivers": 18180, + "\u0120Except": 18181, + "225": 18182, + "\u0120NHS": 18183, + "\u0120McConnell": 18184, + "\u0120pussy": 18185, + "ferred": 18186, + "utable": 18187, + "\u0120attain": 18188, + "\u0120>=": 18189, + "\u0120deposits": 18190, + "rophic": 18191, + "\u0120notorious": 18192, + "\u0120Shaw": 18193, + "ilitation": 18194, + "\u0120epidemic": 18195, + "allic": 18196, + "\u0120smallest": 18197, + "ovich": 18198, + "\u0120accessories": 18199, + "perties": 18200, + "\u0120surplus": 18201, + "\u0120Mech": 18202, + "\u0120ambig": 18203, + "\u0120Immigration": 18204, + "\u0120chim": 18205, + "eval": 18206, + "\u0120practicing": 18207, + "\u0120Mystery": 18208, + "\u0120domains": 18209, + "\u0120Silicon": 18210, + "apps": 18211, + "\u0120kilometers": 18212, + "ea": 18213, + "\u0120Smash": 18214, + "\u0120warranty": 18215, + "\u0120nost": 18216, + "sil": 18217, + "rev": 18218, + "Jon": 18219, + "\u0120Dublin": 18220, + "\u0120tastes": 18221, + "\u0120bout": 18222, + "great": 18223, + "error": 18224, + "\u0120switches": 18225, + "\u0120Bapt": 18226, + "DO": 18227, + "oki": 18228, + "\u0120sourced": 18229, + "produ": 18230, + "\u0120attachment": 18231, + "\u0120Issue": 18232, + "\u0120Question": 18233, + "Join": 18234, + "\u0120fitted": 18235, + "\u0120unlawful": 18236, + "^^": 18237, + "erek": 18238, + "\u0120authentication": 18239, + "\u0120stole": 18240, + "\u0120accountability": 18241, + "label": 18242, + "Search": 18243, + "\u0120albeit": 18244, + "atican": 18245, + "funded": 18246, + "\u0120Adding": 18247, + "\u0120IQ": 18248, + "\u0120submar": 18249, + "lit": 18250, + "aque": 18251, + "\u0120Learning": 18252, + "\u0120integer": 18253, + "Master": 18254, + "\u0120Chrom": 18255, + "\u0120premier": 18256, + "Op": 18257, + "\u0120Liu": 18258, + "\u0120blessed": 18259, + "\u0120Globe": 18260, + "\u0120Response": 18261, + "\u0120legitim": 18262, + "\u0120Merkel": 18263, + "\u0120disposal": 18264, + "\u00c2\u00b4": 18265, + "\u0120gauge": 18266, + "peat": 18267, + "\u0120induced": 18268, + "\u0120questionable": 18269, + "arthy": 18270, + "\u0120Vit": 18271, + "\u0120Feed": 18272, + "Until": 18273, + "Ut": 18274, + "worthy": 18275, + "RY": 18276, + "\u0120Herald": 18277, + "\u0120Hammer": 18278, + "\u0120medal": 18279, + "\u0120Rivers": 18280, + "\u0120Hack": 18281, + "\u0120clarify": 18282, + "\u0120tracked": 18283, + "\u0120autonomous": 18284, + "\u0120tenant": 18285, + "\u0120Qatar": 18286, + "erie": 18287, + "\u0120grim": 18288, + "\u0120Monitor": 18289, + "\u0120resistant": 18290, + "\u0120Spec": 18291, + "\u0120Wells": 18292, + "NAS": 18293, + "148": 18294, + "\u0120miners": 18295, + "iotics": 18296, + "\u0120misses": 18297, + "116": 18298, + "gian": 18299, + "git": 18300, + "\u0120Eyes": 18301, + "pres": 18302, + "\u0120graduated": 18303, + "\u0120angel": 18304, + "\u0120synchron": 18305, + "\u0120efficiently": 18306, + "\u0120transmitted": 18307, + "Harry": 18308, + "\u0120globally": 18309, + "ENCE": 18310, + "\u0120Montana": 18311, + "raged": 18312, + "\u0120Prevention": 18313, + "\u0120piss": 18314, + "\u0120Ll": 18315, + "\u0120shelf": 18316, + "\u0120BJP": 18317, + "\u0120Testament": 18318, + "\u0120Late": 18319, + "iker": 18320, + "\u0120Happ": 18321, + "\u0120Julian": 18322, + "hall": 18323, + "\u0120spont": 18324, + "\u0120shutdown": 18325, + "\u0120inconsistent": 18326, + "\u0120subscribers": 18327, + "\u0120skeleton": 18328, + "\u0120Nebraska": 18329, + "\u0120inspire": 18330, + "\u0120Void": 18331, + "Feed": 18332, + "\u0120angles": 18333, + "\u0120Springs": 18334, + "\u0120benchmark": 18335, + "\u0120vaccines": 18336, + "izophren": 18337, + "sexual": 18338, + "uffed": 18339, + "\u0120shine": 18340, + "\u0120Kath": 18341, + "\u0120gesture": 18342, + "inea": 18343, + "\u0120rip": 18344, + "\u0120oppression": 18345, + "\u0120conscience": 18346, + "bt": 18347, + "\u0120Lum": 18348, + "\u0120incidence": 18349, + "\u0120Fa": 18350, + "wr": 18351, + "\u0120mineral": 18352, + "\u0120Spurs": 18353, + "alky": 18354, + "\u0120thunder": 18355, + "\u0120opio": 18356, + "Being": 18357, + "\u0120Palm": 18358, + "\u0120wasted": 18359, + "\u0120lb": 18360, + "iaries": 18361, + "\u0120Initiative": 18362, + "\u0120curric": 18363, + "\u0120marker": 18364, + "\u0120McL": 18365, + "\u0120extensions": 18366, + "\u0120Pv": 18367, + "\u0120Arms": 18368, + "\u0120offerings": 18369, + "\u0120defenses": 18370, + "\u0120vendor": 18371, + "\u0120contradict": 18372, + "\u0120Colin": 18373, + "\u0120reddit": 18374, + "\u0120peripher": 18375, + "122": 18376, + "\u0120sins": 18377, + "Edit": 18378, + "ICT": 18379, + "Soft": 18380, + "\u0120Shah": 18381, + "\u0120administrator": 18382, + "\u0120Trip": 18383, + "\u0120pornography": 18384, + "\u0120tuition": 18385, + "inence": 18386, + "\u0120Progress": 18387, + "\u0120catalog": 18388, + "\u0120suite": 18389, + "\u0120hike": 18390, + "\u0120reproductive": 18391, + "engine": 18392, + "\u0120drought": 18393, + "\u0120Noah": 18394, + "\u0120230": 18395, + "\u0120dude": 18396, + "\u0120relaxed": 18397, + "\u0120partition": 18398, + "\u0120participant": 18399, + "\u0120telesc": 18400, + "\u0120feas": 18401, + "\u0120FF": 18402, + "owner": 18403, + "\u0120sweeping": 18404, + "\u0120lenses": 18405, + "\u0120matchup": 18406, + "\u0120Repl": 18407, + "ournals": 18408, + "\u0120credible": 18409, + "\u0120grandmother": 18410, + "\u0120thermal": 18411, + "\u0120subscribing": 18412, + "\u0120identities": 18413, + "colm": 18414, + "UCT": 18415, + "\u0120reluctant": 18416, + "users": 18417, + "\u0120Cort": 18418, + "\u0120assisted": 18419, + "OSS": 18420, + "ATIONS": 18421, + "ISH": 18422, + "\u0120pharmaceutical": 18423, + "icable": 18424, + "adian": 18425, + "\u0120Sonic": 18426, + "\u0120Fury": 18427, + "\u0120Mong": 18428, + "AH": 18429, + "\u0120Psychology": 18430, + "\u0120phosph": 18431, + "\u0120treats": 18432, + "\u0143\u0136": 18433, + "\u0120steadily": 18434, + "\u0120Hello": 18435, + "\u0120relates": 18436, + "\u0120clue": 18437, + "Expl": 18438, + "auth": 18439, + "\u0120revision": 18440, + "\u0120eld": 18441, + "osion": 18442, + "\u0120bron": 18443, + "144": 18444, + "rikes": 18445, + "\u0120mines": 18446, + "\u0120blanket": 18447, + "\u0120Fail": 18448, + "eled": 18449, + "\u0120Imagine": 18450, + "\u0120Planned": 18451, + "aic": 18452, + "Request": 18453, + "Mad": 18454, + "\u0120Horse": 18455, + "\u0120Eagle": 18456, + "\u0120capac": 18457, + "157": 18458, + "\u0120ling": 18459, + "\u0120Nice": 18460, + "\u0120Parenthood": 18461, + "minster": 18462, + "ogs": 18463, + "ensitive": 18464, + "Nothing": 18465, + "\u0120carn": 18466, + "Fin": 18467, + "\u0120PE": 18468, + "\u0120rifles": 18469, + "\u0120LP": 18470, + "Sand": 18471, + "\u0120guiActive": 18472, + "\u0120tourist": 18473, + "CNN": 18474, + "\u0120unveiled": 18475, + "\u0120predecessor": 18476, + "}{": 18477, + "uber": 18478, + "\u0120offshore": 18479, + "\u0120optical": 18480, + "\u0120Rot": 18481, + "\u0120Pearl": 18482, + "eton": 18483, + "\u0120stared": 18484, + "\u0120farther": 18485, + "atility": 18486, + "contin": 18487, + "\u0120Gy": 18488, + "\u0120Foster": 18489, + "\u0120Coc": 18490, + "rients": 18491, + "\u0120designing": 18492, + "\u0120Economy": 18493, + "ONG": 18494, + "Women": 18495, + "\u0120Nancy": 18496, + "erver": 18497, + "\u0120mascul": 18498, + "\u0120casualties": 18499, + "\u0120225": 18500, + "\u0120Sullivan": 18501, + "\u0120Choice": 18502, + "\u0120aster": 18503, + "ws": 18504, + "\u0120hotels": 18505, + "\u0120considerations": 18506, + "\u0120couch": 18507, + "\u0120Strip": 18508, + "\u0120Gn": 18509, + "\u0120manipulate": 18510, + "lied": 18511, + "\u0120synthetic": 18512, + "\u0120assaulted": 18513, + "\u0120offenses": 18514, + "\u0120Drake": 18515, + "\u0120impe": 18516, + "October": 18517, + "\u0120Heritage": 18518, + "hl": 18519, + "\u0120Blair": 18520, + "Unlike": 18521, + "\u0120grief": 18522, + "\u0120450": 18523, + "\u0120opted": 18524, + "\u0120resignation": 18525, + "ilo": 18526, + "\u0120verse": 18527, + "\u0120Tomb": 18528, + "\u0120upt": 18529, + "\u0120aired": 18530, + "\u0120Hook": 18531, + "\u0120MLB": 18532, + "\u0120assumes": 18533, + "outed": 18534, + "\u0120Vers": 18535, + "\u0120inferior": 18536, + "\u0120bundle": 18537, + "\u0120DNS": 18538, + "ographer": 18539, + "\u0120multip": 18540, + "\u0120Souls": 18541, + "\u0120illustrated": 18542, + "\u0120tactic": 18543, + "\u0120dressing": 18544, + "\u0120duo": 18545, + "Conf": 18546, + "\u0120relent": 18547, + "\u0120cant": 18548, + "\u0120scarce": 18549, + "\u0120candy": 18550, + "\u0120CF": 18551, + "\u0120affiliated": 18552, + "\u0120sprint": 18553, + "ylan": 18554, + "\u0120Garcia": 18555, + "\u0120junk": 18556, + "Print": 18557, + "exec": 18558, + "Crit": 18559, + "\u0120portrait": 18560, + "iries": 18561, + "\u0120OFF": 18562, + "\u0120disputes": 18563, + "WR": 18564, + "Love": 18565, + "\u00e3\u0123\u0126": 18566, + "\u0120Reyn": 18567, + "\u0120hipp": 18568, + "opath": 18569, + "\u0120floors": 18570, + "\u0120Feel": 18571, + "\u0120worries": 18572, + "\u0120settlements": 18573, + "\u0120Pos": 18574, + "\u0120mosque": 18575, + "\u0120finals": 18576, + "\u0120crushed": 18577, + "\u0120Probably": 18578, + "\u0120Bot": 18579, + "\u0120Mans": 18580, + "\u0120Period": 18581, + "\u0120sovereignty": 18582, + "\u0120seller": 18583, + "\u0120apost": 18584, + "\u0120amateur": 18585, + "\u0120dorm": 18586, + "\u0120consuming": 18587, + "\u0120armour": 18588, + "\u0120Roose": 18589, + "\u0120intensive": 18590, + "\u0120eliminating": 18591, + "\u0120Sunni": 18592, + "\u0120Aleppo": 18593, + "jin": 18594, + "\u0120advise": 18595, + "pal": 18596, + "\u0120Halo": 18597, + "\u0120descent": 18598, + "\u0120simpler": 18599, + "\u0120booth": 18600, + "STR": 18601, + "Later": 18602, + "\u0120Cave": 18603, + "===": 18604, + "\u0120mol": 18605, + "\u0120fist": 18606, + "\u0120shotgun": 18607, + "supp": 18608, + "\u0120robbery": 18609, + "Effect": 18610, + "\u0120obscure": 18611, + "\u0120Professional": 18612, + "\u0120embassy": 18613, + "\u0120militant": 18614, + "\u0120incarcer": 18615, + "\u0120generates": 18616, + "\u0120launches": 18617, + "\u0120administrators": 18618, + "\u0120shaft": 18619, + "\u0120circular": 18620, + "\u0120freshman": 18621, + "\u0120Wes": 18622, + "\u0120Joel": 18623, + "\u0120Drew": 18624, + "\u0120Duncan": 18625, + "\u0120Apparently": 18626, + "sight": 18627, + "\u0120Internal": 18628, + "\u0120Individual": 18629, + "\u0120FE": 18630, + "\u0120bore": 18631, + "\u0120Mt": 18632, + "\u0120broadly": 18633, + "\u0120Options": 18634, + "ountain": 18635, + "ipes": 18636, + "\u0120Videos": 18637, + "204": 18638, + "\u0120hills": 18639, + "\u0120simulation": 18640, + "\u0120disappointment": 18641, + "itan": 18642, + "\u0120Laboratory": 18643, + "\u0120upward": 18644, + "\u0120boundary": 18645, + "\u0120darker": 18646, + "hart": 18647, + "\u0120dominance": 18648, + "Cong": 18649, + "\u0120Oracle": 18650, + "\u0120Lords": 18651, + "\u0120scholarship": 18652, + "\u0120Vincent": 18653, + "ede": 18654, + "\u0120Rah": 18655, + "\u0120encourages": 18656, + "rov": 18657, + "\u0120quo": 18658, + "\u0120premise": 18659, + "\u0120Crisis": 18660, + "\u0120Holocaust": 18661, + "\u0120rhythm": 18662, + "\u0120metric": 18663, + "club": 18664, + "\u0120transported": 18665, + "\u0120nod": 18666, + "\u0120Pist": 18667, + "\u0120ancestors": 18668, + "\u0120Freder": 18669, + "thumbnails": 18670, + "\u0120CE": 18671, + "OND": 18672, + "Phil": 18673, + "venge": 18674, + "\u0120Products": 18675, + "castle": 18676, + "\u0120qualifying": 18677, + "\u0120Karen": 18678, + "VERTISEMENT": 18679, + "\u0120mighty": 18680, + "\u0120explanations": 18681, + "\u0120fixing": 18682, + "Di": 18683, + "\u0120declaring": 18684, + "\u0120anonymity": 18685, + "\u0120juven": 18686, + "\u0120Nord": 18687, + "\u0120Doom": 18688, + "\u0120Actually": 18689, + "Ok": 18690, + "phis": 18691, + "\u0120Desert": 18692, + "\u0120116": 18693, + "IK": 18694, + "\u0120FM": 18695, + "\u0120incomes": 18696, + "VEL": 18697, + "okers": 18698, + "\u0120pecul": 18699, + "\u0120lightweight": 18700, + "gue": 18701, + "\u0120accent": 18702, + "\u0120increment": 18703, + "\u0120Chan": 18704, + "\u0120complaining": 18705, + "\u0120Baghd": 18706, + "\u0120midfielder": 18707, + "\u0120overhaul": 18708, + "Process": 18709, + "\u0120Hollow": 18710, + "\u0120Titans": 18711, + "Small": 18712, + "manuel": 18713, + "\u0120Unity": 18714, + "\u0120Events": 18715, + "Sty": 18716, + "\u0120disproportion": 18717, + "nesty": 18718, + "enes": 18719, + "\u0120Cod": 18720, + "\u0120demonstrations": 18721, + "\u0120Crimson": 18722, + "\u0120OH": 18723, + "\u0120enrolled": 18724, + "\u0120cel": 18725, + "\u0120Brett": 18726, + "\u0120aide": 18727, + "\u0120heels": 18728, + "\u0120broadband": 18729, + "\u0120marking": 18730, + "\u0120wizard": 18731, + "\u0120NJ": 18732, + "\u0120Chiefs": 18733, + "\u0120ingredient": 18734, + "\u0120dug": 18735, + "\u0120Shut": 18736, + "urchase": 18737, + "endor": 18738, + "\u0120farmer": 18739, + "\u0120Goldman": 18740, + "129": 18741, + "155": 18742, + "Order": 18743, + "\u0120lion": 18744, + "iably": 18745, + "\u0120stain": 18746, + "array": 18747, + "ilitary": 18748, + "\u0120FAQ": 18749, + "\u0120exploded": 18750, + "\u0120McCarthy": 18751, + "\u0120Tweet": 18752, + "\u0120Greens": 18753, + "eking": 18754, + "ln": 18755, + "ensen": 18756, + "\u0120motorcycle": 18757, + "\u0120particle": 18758, + "\u0120cholesterol": 18759, + "Bron": 18760, + "\u0120stair": 18761, + "\u0120oxid": 18762, + "\u0120desirable": 18763, + "ibles": 18764, + "\u0120theor": 18765, + "forcing": 18766, + "\u0120promotional": 18767, + "ovo": 18768, + "boot": 18769, + "\u0120Bonus": 18770, + "rawling": 18771, + "\u0120shortage": 18772, + "\u0120Psy": 18773, + "\u0120recruited": 18774, + "\u0120infants": 18775, + "\u0120testosterone": 18776, + "\u0120deduct": 18777, + "\u0120distinctive": 18778, + "\u0120firmware": 18779, + "built": 18780, + "145": 18781, + "\u0120explored": 18782, + "\u0120factions": 18783, + "\u0120vide": 18784, + "\u0120tattoo": 18785, + "\u0120financially": 18786, + "\u0120fatigue": 18787, + "\u0120proceeding": 18788, + "constitutional": 18789, + "\u0120miser": 18790, + "\u0120chairs": 18791, + "gging": 18792, + "ipple": 18793, + "\u0120dent": 18794, + "\u0120disreg": 18795, + "\u00e7\u0136": 18796, + "stant": 18797, + "llo": 18798, + "bps": 18799, + "akening": 18800, + "\u0120abnormal": 18801, + "\u0120ERA": 18802, + "\u00e5\u00a3\u00ab": 18803, + "\u0120HBO": 18804, + "\u0120MAR": 18805, + "\u0120concess": 18806, + "\u0120servant": 18807, + "\u0120aspir": 18808, + "lav": 18809, + "\u0120Panel": 18810, + "amo": 18811, + "\u0120precip": 18812, + "\u0120recordings": 18813, + "\u0120proceeded": 18814, + "\u0120colony": 18815, + "\u0120Tang": 18816, + "ablo": 18817, + "\u0120stripped": 18818, + "Left": 18819, + "too": 18820, + "\u0120potatoes": 18821, + "\u0120finest": 18822, + "%).": 18823, + "\u0120crap": 18824, + "\u0120Zach": 18825, + "abases": 18826, + "\u0120Goth": 18827, + "\u0120billionaire": 18828, + "wolf": 18829, + "\u0120sanction": 18830, + "SK": 18831, + "\u0120logged": 18832, + "Po": 18833, + "eyed": 18834, + "unal": 18835, + "\u0120cricket": 18836, + "\u0120armies": 18837, + "\u0120uncovered": 18838, + "Cloud": 18839, + "\u00c3\u00b3n": 18840, + "\u0120rebounds": 18841, + "\u0120mes": 18842, + "Oper": 18843, + "Pac": 18844, + "\u0120nationally": 18845, + "\u0120inserted": 18846, + "pict": 18847, + "\u0120governance": 18848, + "\u00d0\u00b8": 18849, + "\u0120privileges": 18850, + "GET": 18851, + "\u0120favorites": 18852, + "imity": 18853, + "\u0120lover": 18854, + "them": 18855, + "empl": 18856, + "\u0120gorgeous": 18857, + "Ann": 18858, + "\u0120slipped": 18859, + "\u0120veto": 18860, + "Bob": 18861, + "\u0120slim": 18862, + "ucc": 18863, + "\u0120Fame": 18864, + "uddenly": 18865, + "\u0120denies": 18866, + "\u0120Maur": 18867, + "\u0120distances": 18868, + "\u0120wanna": 18869, + "tar": 18870, + "\u0120SER": 18871, + "\u0120\u00e2\u012a": 18872, + "\u0120lemon": 18873, + "athetic": 18874, + "\u0120literal": 18875, + "\u0120distinguished": 18876, + "\u0120answering": 18877, + "GI": 18878, + "\u0120religions": 18879, + "\u0120Philos": 18880, + "\u0120Lay": 18881, + "\u0120compos": 18882, + "irements": 18883, + "\u0120Kos": 18884, + "inez": 18885, + "rolling": 18886, + "\u0120youngest": 18887, + "andise": 18888, + "\u0120Born": 18889, + "\u0120altar": 18890, + "amina": 18891, + "\u0120Boot": 18892, + "voc": 18893, + "\u0120digging": 18894, + "\u0120pressures": 18895, + "\u0120len": 18896, + "264": 18897, + "\u0120assassination": 18898, + "\u0120Birmingham": 18899, + "\u0120Myth": 18900, + "\u0120sovereign": 18901, + "\u0120Artist": 18902, + "\u0120Photograph": 18903, + "\u0120depicted": 18904, + "\u0120dispens": 18905, + "orthy": 18906, + "\u0120ambul": 18907, + "integ": 18908, + "\u0120Cele": 18909, + "\u0120Tibet": 18910, + "\u0120hierarchy": 18911, + "\u0120cu": 18912, + "\u0120preseason": 18913, + "\u0120Peterson": 18914, + "\u0120colours": 18915, + "\u0120worrying": 18916, + "\u0120backers": 18917, + "\u0120Palmer": 18918, + "\u0120\u00ce\u00bc": 18919, + "\u0120contributor": 18920, + "\u0120hearings": 18921, + "\u0120urine": 18922, + "\u0120\u00d9": 18923, + "ourgeois": 18924, + "Similar": 18925, + "\u0120Zimmer": 18926, + "something": 18927, + "\u0120USC": 18928, + "\u0120strengths": 18929, + "\u0120FI": 18930, + "\u0120logging": 18931, + "Asked": 18932, + "\u0120Thai": 18933, + "inqu": 18934, + "\u0120Walt": 18935, + "\u0120crews": 18936, + "itism": 18937, + "301": 18938, + "\u0120sharply": 18939, + "umed": 18940, + "\u0120redirect": 18941, + "rators": 18942, + "Inf": 18943, + "\u0120Weapons": 18944, + "\u0120teasp": 18945, + "1999": 18946, + "Live": 18947, + "\u0120Especially": 18948, + "\u0120Ster": 18949, + "\u0120Veterans": 18950, + "\u0120intro": 18951, + "otherapy": 18952, + "\u0120malware": 18953, + "\u0120breeding": 18954, + "\u0120molecular": 18955, + "\u0120Route": 18956, + "\u0120Comment": 18957, + "ochem": 18958, + "\u0120ain": 18959, + "Season": 18960, + "\u0120linebacker": 18961, + "\u00c4\u00ab": 18962, + "\u0120Economics": 18963, + "esar": 18964, + "\u0120Lives": 18965, + "\u0120Emma": 18966, + "\u0120kin": 18967, + "\u0120Territ": 18968, + "\u0120planted": 18969, + "oton": 18970, + "\u0120Butter": 18971, + "\u0120Spons": 18972, + "PER": 18973, + "\u0120dungeon": 18974, + "\u0120symbolic": 18975, + "\u0120filmed": 18976, + "\u0120diets": 18977, + "\u0120concludes": 18978, + "\u0120certainty": 18979, + "\u0120Format": 18980, + "\u0120strangers": 18981, + "format": 18982, + "\u0120Phase": 18983, + "\u0120copied": 18984, + "\u0120metres": 18985, + "lda": 18986, + "\u0120Users": 18987, + "\u0120deliberate": 18988, + "\u0120washed": 18989, + "\u0120Lance": 18990, + "imation": 18991, + "\u0120improper": 18992, + "\u0120Genesis": 18993, + "ickr": 18994, + "\u0120Kush": 18995, + "\u0120realise": 18996, + "\u0120embarrassing": 18997, + "alking": 18998, + "bucks": 18999, + "\u0120verified": 19000, + "\u0120outline": 19001, + "years": 19002, + "\u0120Income": 19003, + "202": 19004, + "\u0120zombies": 19005, + "Final": 19006, + "\u0120Millenn": 19007, + "\u0120modifications": 19008, + "\u0120Vision": 19009, + "\u0120Moses": 19010, + "verb": 19011, + "iterranean": 19012, + "\u0120Jet": 19013, + "\u0120naval": 19014, + "\u0120Agg": 19015, + "\u0120url": 19016, + "\u0120victories": 19017, + "\u0120nonetheless": 19018, + "\u0120injust": 19019, + "\u0120Fact": 19020, + "\u00e7\u013c": 19021, + "\u0120insufficient": 19022, + "review": 19023, + "facebook": 19024, + "\u0120negotiating": 19025, + "\u0120guarantees": 19026, + "imen": 19027, + "utenberg": 19028, + "\u0120gambling": 19029, + "\u0120congr": 19030, + "Loading": 19031, + "\u0120nevertheless": 19032, + "\u0120presidents": 19033, + "\u0120Industrial": 19034, + "\u0120118": 19035, + "\u0120poured": 19036, + "\u0120Tory": 19037, + "\u0120175": 19038, + "\u0120:=": 19039, + "Scott": 19040, + "angered": 19041, + "Tok": 19042, + "\u0120organizers": 19043, + "Mat": 19044, + "\u0120Growth": 19045, + "\u0120adul": 19046, + "\u0120ensures": 19047, + "\u0120117": 19048, + "\u00e9\u00be\u012f\u00e5": 19049, + "\u0120massacre": 19050, + "\u0120grades": 19051, + "before": 19052, + "ADVERTISEMENT": 19053, + "\u0120Slow": 19054, + "\u0120MMA": 19055, + "\u00e2\u0122\u0136\"": 19056, + "\u0120Vatican": 19057, + "Qaeda": 19058, + "\u0120owe": 19059, + "6666": 19060, + "\u0120Sorry": 19061, + "\u0120Grass": 19062, + "\u0120backgrounds": 19063, + "\u0120exhausted": 19064, + "\u0120clan": 19065, + "\u0120compromised": 19066, + "\u0120Elf": 19067, + "\u0120Isaac": 19068, + "enson": 19069, + "Invest": 19070, + "IFA": 19071, + "\u0120interrupted": 19072, + "\u00e3\u0125\u012b\u00e3\u0125\u00a9": 19073, + "\u0120twisted": 19074, + "\u0120Dragons": 19075, + "Mode": 19076, + "\u0120Kremlin": 19077, + "\u0120fertil": 19078, + "heres": 19079, + "phan": 19080, + "\u0120Node": 19081, + "fed": 19082, + "\u0120Orc": 19083, + "\u0120unwilling": 19084, + "Cent": 19085, + "\u0120priorit": 19086, + "\u0120graduates": 19087, + "\u0120subjective": 19088, + "\u0120issuing": 19089, + "\u0120Lt": 19090, + "\u0120viewer": 19091, + "\u0120woke": 19092, + "Thus": 19093, + "brook": 19094, + "\u0120depressed": 19095, + "\u0120bracket": 19096, + "\u0120Gor": 19097, + "\u0120Fighting": 19098, + "\u0120striker": 19099, + "Report": 19100, + "\u0120Portugal": 19101, + "\u0120neo": 19102, + "wed": 19103, + "199": 19104, + "\u0120fleeing": 19105, + "shadow": 19106, + "identified": 19107, + "USE": 19108, + "Steam": 19109, + "\u0120stretched": 19110, + "\u0120revelations": 19111, + "arted": 19112, + "\u0120Dw": 19113, + "\u0120alignment": 19114, + "eston": 19115, + "\u0120Jared": 19116, + "Sep": 19117, + "\u0120blogs": 19118, + "update": 19119, + "gom": 19120, + "risk": 19121, + "\u0120clash": 19122, + "\u0120Hour": 19123, + "\u0120runtime": 19124, + "\u0120unwanted": 19125, + "\u0120scam": 19126, + "\u0120rack": 19127, + "\u0120enlight": 19128, + "onest": 19129, + "\u0120Ferr": 19130, + "\u0120convictions": 19131, + "\u0120piano": 19132, + "\u0120circulation": 19133, + "\u0120Welcome": 19134, + "\u0120backlash": 19135, + "\u0120Wade": 19136, + "\u0120receivers": 19137, + "otive": 19138, + "Jeff": 19139, + "\u0120networking": 19140, + "\u0120Prep": 19141, + "\u0120Explorer": 19142, + "\u0120lecture": 19143, + "\u0120uploaded": 19144, + "\u0120Meat": 19145, + "BLE": 19146, + "\u0120Nazis": 19147, + "\u0120Synd": 19148, + "stud": 19149, + "roots": 19150, + "rians": 19151, + "\u0120portrayed": 19152, + "\u0120??": 19153, + "\u0120Buddha": 19154, + "sun": 19155, + "Robert": 19156, + "\u0120Complex": 19157, + "\u0120oversee": 19158, + "\u0120stealth": 19159, + "Title": 19160, + "\u0120Jobs": 19161, + "\u0120Kum": 19162, + "\u0120appreciation": 19163, + "\u0120MOD": 19164, + "\u0120basics": 19165, + "\u0120clips": 19166, + "\u0120nursing": 19167, + "\u0120proposition": 19168, + "\u0120realised": 19169, + "\u0120NYC": 19170, + "\u0120allocated": 19171, + "rium": 19172, + "aran": 19173, + "\u0120Production": 19174, + "\u0120Vote": 19175, + "\u0120smugg": 19176, + "\u0120hunter": 19177, + "azer": 19178, + "\u0120Changes": 19179, + "\u0120fluct": 19180, + "yon": 19181, + "Array": 19182, + "\u0120kits": 19183, + "Water": 19184, + "\u0120uncommon": 19185, + "\u0120resting": 19186, + "ells": 19187, + "would": 19188, + "\u0120pursued": 19189, + "\u0120assertion": 19190, + "ometown": 19191, + "\u0120Mosul": 19192, + "\u0120Platform": 19193, + "iolet": 19194, + "\u0120shareholders": 19195, + "\u0120trails": 19196, + "Pay": 19197, + "\u0120Enforcement": 19198, + "types": 19199, + "\u0120Anonymous": 19200, + "\u0120satisfying": 19201, + "ilogy": 19202, + "\u0120('": 19203, + "wave": 19204, + "city": 19205, + "Steve": 19206, + "\u0120confrontation": 19207, + "\u0120Eld": 19208, + "Capt": 19209, + "ahan": 19210, + "htm": 19211, + "\u0120Ctrl": 19212, + "ONS": 19213, + "230": 19214, + "ifa": 19215, + "holding": 19216, + "\u0120delicate": 19217, + "\u0120jaw": 19218, + "\u0120Going": 19219, + "orum": 19220, + "Sal": 19221, + "\u0120dull": 19222, + "\u0120Beth": 19223, + "\u0120prisons": 19224, + "\u0120ego": 19225, + "\u0120Elsa": 19226, + "avorite": 19227, + "\u0120Gang": 19228, + "\u0120Nuclear": 19229, + "\u0120spider": 19230, + "atsu": 19231, + "\u0120sampling": 19232, + "\u0120absorbed": 19233, + "\u0120Pharm": 19234, + "ieth": 19235, + "\u0120bucket": 19236, + "\u0120Recomm": 19237, + "OF": 19238, + "\u0120Factory": 19239, + "ANCE": 19240, + "\u0120bacter": 19241, + "Has": 19242, + "\u0120Observ": 19243, + "121": 19244, + "\u0120premiere": 19245, + "Develop": 19246, + "\u0120currencies": 19247, + "Cast": 19248, + "\u0120accompanying": 19249, + "\u0120Nashville": 19250, + "\u0120fatty": 19251, + "\u0120Brend": 19252, + "\u0120locks": 19253, + "\u0120centered": 19254, + "\u0120UT": 19255, + "aughs": 19256, + "orie": 19257, + "\u0120Affordable": 19258, + "vance": 19259, + "DL": 19260, + "emet": 19261, + "\u0120throne": 19262, + "\u0120Bluetooth": 19263, + "\u0120naming": 19264, + "ifts": 19265, + "ADE": 19266, + "\u0120corrected": 19267, + "\u0120promptly": 19268, + "\u0120STR": 19269, + "\u0120genome": 19270, + "\u0120cope": 19271, + "\u0120valley": 19272, + "\u0120rounded": 19273, + "\u0120Kend": 19274, + "alion": 19275, + "pers": 19276, + "\u0120tourism": 19277, + "\u0120stark": 19278, + "vl": 19279, + "\u0120blowing": 19280, + "\u0120Schedule": 19281, + "std": 19282, + "\u0120unhappy": 19283, + "\u0120litigation": 19284, + "cedes": 19285, + "\u0120android": 19286, + "\u0120integral": 19287, + "erers": 19288, + "uded": 19289, + "tax": 19290, + "\u0120reiter": 19291, + "\u0120Motors": 19292, + "ociated": 19293, + "\u0120wonders": 19294, + "\u0120Apost": 19295, + "ucking": 19296, + "\u0120Roosevelt": 19297, + "fram": 19298, + "\u0120yields": 19299, + "\u0120constitutes": 19300, + "awk": 19301, + "Interest": 19302, + "\u0120interim": 19303, + "\u0120breakthrough": 19304, + "\u0120Cher": 19305, + "\u0120prosec": 19306, + "\u0120Dj": 19307, + "\u0120MT": 19308, + "Resp": 19309, + "\u0120PT": 19310, + "\u0120sperm": 19311, + "edit": 19312, + "BT": 19313, + "Linux": 19314, + "country": 19315, + "league": 19316, + "\u0120dick": 19317, + "\u0120oct": 19318, + "\u0120inserting": 19319, + "\u0120scra": 19320, + "\u0120Brewing": 19321, + "\u01201966": 19322, + "\u0120runners": 19323, + "\u0120plun": 19324, + "idy": 19325, + "\u0120Dian": 19326, + "\u0120dysfunction": 19327, + "\u0120exclusion": 19328, + "\u0120disgr": 19329, + "\u0120incorporate": 19330, + "\u0120reconc": 19331, + "\u0120nominated": 19332, + "\u0120Archer": 19333, + "draw": 19334, + "achelor": 19335, + "\u0120writings": 19336, + "\u0120shallow": 19337, + "\u0120hast": 19338, + "\u0120BMW": 19339, + "\u0120RS": 19340, + "\u0120thigh": 19341, + "\u01201963": 19342, + "\u0120lamb": 19343, + "\u0120favored": 19344, + "agle": 19345, + "\u0120cooler": 19346, + "\u0120Hours": 19347, + "\u0120GU": 19348, + "\u0120Origin": 19349, + "\u0120glimpse": 19350, + "--------------------": 19351, + "Lim": 19352, + "\u0120cheek": 19353, + "\u0120jealous": 19354, + "-'": 19355, + "\u0120harness": 19356, + "\u0120Poison": 19357, + "\u0120disabilities": 19358, + "neapolis": 19359, + "\u0120outlook": 19360, + "\u0120notify": 19361, + "\u0120Indianapolis": 19362, + "\u0120abrupt": 19363, + "nsic": 19364, + "\u0120encrypted": 19365, + "\u0120forfe": 19366, + "reath": 19367, + "\u0120rabb": 19368, + "\u0120foundations": 19369, + "\u0120compliment": 19370, + "\u0120Interview": 19371, + "\u0120Swe": 19372, + "\u0120adolesc": 19373, + "\u0120monitors": 19374, + "\u0120Sacramento": 19375, + "\u0120timely": 19376, + "\u0120contempl": 19377, + "\u0120positioned": 19378, + "\u0120posters": 19379, + "phies": 19380, + "iovascular": 19381, + "void": 19382, + "\u0120Fifth": 19383, + "\u0120investigative": 19384, + "OUN": 19385, + "\u0120integrate": 19386, + "\u0120INC": 19387, + "isha": 19388, + "iblings": 19389, + "\u0120Request": 19390, + "\u0120Rodriguez": 19391, + "\u0120slides": 19392, + "\u0120DX": 19393, + "\u0120feminism": 19394, + "\u0120datas": 19395, + "\u0120bend": 19396, + "irus": 19397, + "\u0120Nigeria": 19398, + "Fox": 19399, + "Change": 19400, + "\u0120airplane": 19401, + "\u0120Laden": 19402, + "\u0120publicity": 19403, + "ixty": 19404, + "\u0120commitments": 19405, + "\u0120aggregate": 19406, + "\u0120displaying": 19407, + "\u0120Arrow": 19408, + "\u0120122": 19409, + "\u0120respects": 19410, + "android": 19411, + "six": 19412, + "\u0120Sha": 19413, + "\u0120restoration": 19414, + ")\\": 19415, + "WS": 19416, + "oys": 19417, + "\u0120illustrate": 19418, + "without": 19419, + "126": 19420, + "\u0120\u00e2\u0136\u0124": 19421, + "\u0120pickup": 19422, + "nels": 19423, + "\u0120....": 19424, + "food": 19425, + "\u0120Fen": 19426, + ")?": 19427, + "\u0120phenomena": 19428, + "\u0120companions": 19429, + "\u0120Write": 19430, + "\u0120spill": 19431, + "\u0120bridges": 19432, + "\u0120Updated": 19433, + "\u0120Fo": 19434, + "\u0120insects": 19435, + "ASHINGTON": 19436, + "\u0120scare": 19437, + "iltr": 19438, + "\u0120Zhang": 19439, + "\u0120severity": 19440, + "\u0120indul": 19441, + "149": 19442, + "\u0120Coffee": 19443, + "\u0120norms": 19444, + "\u0120pulse": 19445, + "\u0120FT": 19446, + "\u0120horrific": 19447, + "\u0120Destroy": 19448, + "\u0120JSON": 19449, + "\u0120olive": 19450, + "\u0120discusses": 19451, + "Rest": 19452, + "Elect": 19453, + "\u0120Winn": 19454, + "\u0120Surviv": 19455, + "\u0120Hait": 19456, + "Sure": 19457, + "oped": 19458, + "\u0120rooted": 19459, + "\u0120Ske": 19460, + "\u0120Bronze": 19461, + "\u0120lol": 19462, + "Default": 19463, + "\u0120commodity": 19464, + "redited": 19465, + "\u0120libertarian": 19466, + "\u0120forbidden": 19467, + "\u0120gran": 19468, + "\u00e0\u00a8": 19469, + "\u0120lag": 19470, + "enz": 19471, + "drive": 19472, + "\u0120mathematics": 19473, + "\u0120wires": 19474, + "\u0120critically": 19475, + "\u0120carbohyd": 19476, + "\u0120Chancellor": 19477, + "\u0120Eddie": 19478, + "\u0120banning": 19479, + "\u0120Fri": 19480, + "\u0120complications": 19481, + "etric": 19482, + "\u0120Bangladesh": 19483, + "\u0120bandwidth": 19484, + "Stop": 19485, + "\u0120Originally": 19486, + "\u0120halfway": 19487, + "ynasty": 19488, + "shine": 19489, + "\u0120tales": 19490, + "rities": 19491, + "avier": 19492, + "\u0120spinning": 19493, + "\u0120WHO": 19494, + "\u0120neighbourhood": 19495, + "bach": 19496, + "\u0120commerce": 19497, + "\u0120Sle": 19498, + "BU": 19499, + "\u0120entrepreneur": 19500, + "\u0120peculiar": 19501, + "\u0120Comments": 19502, + "fre": 19503, + "320": 19504, + "ICS": 19505, + "\u0120imagery": 19506, + "\u0120Canon": 19507, + "\u0120Electronic": 19508, + "short": 19509, + "((": 19510, + "Dig": 19511, + "\u0120commem": 19512, + "uced": 19513, + "\u0120inclined": 19514, + "\u0120Summon": 19515, + "\u0120cliff": 19516, + "\u0120Mediterranean": 19517, + "\u0120poetry": 19518, + "\u0120prosperity": 19519, + "\u0120Rece": 19520, + "\u0120pills": 19521, + "member": 19522, + "\u0120finale": 19523, + "unc": 19524, + "\u0120Gig": 19525, + "\u00e4\u00bd": 19526, + "\u0120lod": 19527, + "\u0120backward": 19528, + "-+": 19529, + "\u0120Forward": 19530, + "\u0120thri": 19531, + "sure": 19532, + "\u0120soap": 19533, + "\u0120FX": 19534, + "RES": 19535, + "\u0120Sexual": 19536, + "oulos": 19537, + "\u0120foolish": 19538, + "\u0120righteous": 19539, + "\u0120coff": 19540, + "terrorism": 19541, + "ustain": 19542, + "oter": 19543, + "\u0120abuses": 19544, + "next": 19545, + "\u0120abusive": 19546, + "\u0120thereafter": 19547, + "\u0120prohibition": 19548, + "\u0120SUP": 19549, + "\u0120dip": 19550, + "\u0120ripped": 19551, + "\u0120inherited": 19552, + "\u0120bats": 19553, + "stru": 19554, + "GT": 19555, + "\u0120flawed": 19556, + "phabet": 19557, + "\u0120fog": 19558, + "doors": 19559, + "\u0120imaging": 19560, + "\u0120digits": 19561, + "\u0120Hungary": 19562, + "\u0120arrog": 19563, + "\u0120teachings": 19564, + "\u0120protocols": 19565, + "\u0120Banks": 19566, + "\u00e0\u00b8": 19567, + "pound": 19568, + "\u0120Curt": 19569, + ".\")": 19570, + "./": 19571, + "\u0120exemption": 19572, + "endix": 19573, + "\u0120Mull": 19574, + "\u0120improves": 19575, + "\u0120Gamer": 19576, + "dimensional": 19577, + "Icon": 19578, + "\u0120Margaret": 19579, + "Status": 19580, + "dates": 19581, + "\u0120intends": 19582, + "\u0120depict": 19583, + "\u0120parked": 19584, + "Joe": 19585, + "\u0120Marines": 19586, + "chnology": 19587, + "!).": 19588, + "\u0120judged": 19589, + "\u0120weights": 19590, + "Ray": 19591, + "\u0120apartments": 19592, + "hester": 19593, + "\u0120reinforce": 19594, + "\u0120offender": 19595, + "occup": 19596, + "\u0120sore": 19597, + "ept": 19598, + "\u0120PHP": 19599, + "\u0120Brow": 19600, + "\u0120authorization": 19601, + "\u0120Risk": 19602, + "\u0120Delaware": 19603, + "\u0120QU": 19604, + "\u0120notifications": 19605, + "\u0120sunlight": 19606, + "\u0120exclude": 19607, + "dat": 19608, + "\u0120mesh": 19609, + "\u0120Sudan": 19610, + "\u0120belonged": 19611, + "\u0120subway": 19612, + "\u0120noon": 19613, + "\u0120Interior": 19614, + "olics": 19615, + "\u0120Lakers": 19616, + "\u0120coding": 19617, + "Disclaimer": 19618, + "Calif": 19619, + "Old": 19620, + "\u0120disl": 19621, + "?????": 19622, + "\u0120confirms": 19623, + "\u0120recruitment": 19624, + "\u0120homicide": 19625, + "Consider": 19626, + "\u0120Jeffrey": 19627, + "fty": 19628, + "};": 19629, + "\u0120objection": 19630, + "doing": 19631, + "\u0120Leo": 19632, + "Want": 19633, + "\u0120glow": 19634, + "\u0120Clarke": 19635, + "\u0120Norman": 19636, + "\u0120verification": 19637, + "\u0120packet": 19638, + "\u0120Formula": 19639, + "\u0120plag": 19640, + "esville": 19641, + "\u0120shouting": 19642, + "\u0120ov": 19643, + "\u0120REC": 19644, + "\u0120Bub": 19645, + "\u0120ninth": 19646, + "\u0120energ": 19647, + "\u0120validity": 19648, + "\u0120ups": 19649, + "jack": 19650, + "\u0120neighboring": 19651, + "\u0120Nec": 19652, + "eworks": 19653, + "\u0120Hab": 19654, + "arez": 19655, + "\u0120spine": 19656, + "\u0120eventual": 19657, + "\u0120Leaders": 19658, + "\u0120Carn": 19659, + "\u0120probation": 19660, + "\u0120romance": 19661, + "msg": 19662, + "\u0120Mechanical": 19663, + "ERY": 19664, + "Rock": 19665, + "\u0120partisan": 19666, + "Node": 19667, + "assets": 19668, + "minent": 19669, + "\u0120foreigners": 19670, + "\u0120testify": 19671, + "\u0120Usually": 19672, + "lords": 19673, + "\u0120Gren": 19674, + "\u0120Powell": 19675, + "BIL": 19676, + "\u0120sr": 19677, + "\u0120addict": 19678, + "\u0120shells": 19679, + "\u0120sigh": 19680, + "\u0120Yale": 19681, + "ternity": 19682, + "\u0120750": 19683, + "EU": 19684, + "\u0120Rifle": 19685, + "\u0120patron": 19686, + "ema": 19687, + "\u0120Bannon": 19688, + "anity": 19689, + "\u0120tropical": 19690, + "\u0120VII": 19691, + "cross": 19692, + "Everything": 19693, + "\u0120ISO": 19694, + "\u0120humble": 19695, + "assing": 19696, + "\u0120FIG": 19697, + "\u0120updating": 19698, + "yson": 19699, + "\u0120calcium": 19700, + "\u0120competent": 19701, + "\u0120steering": 19702, + "Prot": 19703, + "\u0120SY": 19704, + "\u0120Finals": 19705, + "\u0120Rug": 19706, + "159": 19707, + "137": 19708, + "\u0120Golf": 19709, + "\u0120126": 19710, + "\u0120accommodation": 19711, + "\u0120Hughes": 19712, + "\u0120aesthetic": 19713, + "artisan": 19714, + "\u0120Twilight": 19715, + "\u0120prince": 19716, + "\u0120Agriculture": 19717, + "\u0120Disco": 19718, + "\u0120precedent": 19719, + "\u0120typing": 19720, + "authorized": 19721, + "Option": 19722, + "\u0120Aub": 19723, + "lishes": 19724, + "acht": 19725, + "mag": 19726, + "Peter": 19727, + "\u0120UFO": 19728, + "monton": 19729, + "\u0120Lith": 19730, + "\u0120arom": 19731, + "\u0120securing": 19732, + "\u0120confined": 19733, + "private": 19734, + "\u0120swords": 19735, + "\u0120markers": 19736, + "\u0120metabolic": 19737, + "select": 19738, + "\u0120Curse": 19739, + "\u0120Ot": 19740, + "gressive": 19741, + "\u0120incumb": 19742, + "\u0120Saga": 19743, + "\u0120priced": 19744, + "\u0120clearance": 19745, + "Content": 19746, + "\u0120drilling": 19747, + "\u0120notices": 19748, + "\u0120bourgeois": 19749, + "\u0120vest": 19750, + "\u0120cookie": 19751, + "\u0120Guardians": 19752, + "rys": 19753, + "inyl": 19754, + "\u0120124": 19755, + "\u0120plausible": 19756, + "ongh": 19757, + "\u0120Odin": 19758, + "\u0120conception": 19759, + "\u0120Yuk": 19760, + "\u0120Baghdad": 19761, + "\u0120Flag": 19762, + "Austral": 19763, + "\u0120IBM": 19764, + "\u0120internationally": 19765, + "\u0120WikiLeaks": 19766, + "IED": 19767, + "\u0120cyn": 19768, + "\u0120chooses": 19769, + "\u0120Pill": 19770, + "\u0120combining": 19771, + "\u0120radi": 19772, + "\u0120Mohammed": 19773, + "defense": 19774, + "atching": 19775, + "Subject": 19776, + "iciency": 19777, + "Frame": 19778, + "\u0120{\"": 19779, + "\u0120chess": 19780, + "\u0120timer": 19781, + "190": 19782, + "\u0120tin": 19783, + "\u0120ordinance": 19784, + "emetery": 19785, + "\u0120accusing": 19786, + "\u0120noticeable": 19787, + "\u0120centres": 19788, + "\u0120lid": 19789, + "\u0120Mills": 19790, + "imgur": 19791, + "\u0120zoom": 19792, + "ergic": 19793, + "\u0120compression": 19794, + "prim": 19795, + "find": 19796, + "\u0120surg": 19797, + "\u0120pand": 19798, + "\u0120Kee": 19799, + "\u0120Chad": 19800, + "cellence": 19801, + "oyle": 19802, + "\u0120socialism": 19803, + "\u0120Travis": 19804, + "\u0120MHz": 19805, + "\u0120guild": 19806, + "ALLY": 19807, + "\u0120Subscribe": 19808, + "\u0120Related": 19809, + "\u0120occurrence": 19810, + "itching": 19811, + "\u0120fictional": 19812, + "\u0120crush": 19813, + "\u0120EA": 19814, + "cod": 19815, + "mix": 19816, + "\u0120Triple": 19817, + "\u0120retrieve": 19818, + "\u0120stimulus": 19819, + "\u0120psychiat": 19820, + "\u0120Door": 19821, + "\u0120homosexuality": 19822, + "\u0120elementary": 19823, + "\u0120cellular": 19824, + "idian": 19825, + "\u0120Laun": 19826, + "\u0120intriguing": 19827, + "\u0120foam": 19828, + "\u0120Bass": 19829, + "idi": 19830, + "itsu": 19831, + "\u0120assure": 19832, + "\u0120congrat": 19833, + "\u0120businessman": 19834, + "\u0120Boost": 19835, + "close": 19836, + "\u0120lied": 19837, + "\u0120sciences": 19838, + "\u0120Omega": 19839, + "\u0120Graphics": 19840, + "\u0120<=": 19841, + "spoken": 19842, + "\u0120connectivity": 19843, + "Saturday": 19844, + "\u0120Avengers": 19845, + "\u0120toggle": 19846, + "\u0120ankle": 19847, + "\u0120nationalist": 19848, + "model": 19849, + "\u0120Pool": 19850, + "ophobia": 19851, + "Var": 19852, + "\u0120Mons": 19853, + "atories": 19854, + "\u0120aggressively": 19855, + "Clear": 19856, + "Forge": 19857, + "acters": 19858, + "\u0120hedge": 19859, + "\u0120pipes": 19860, + "\u0120blunt": 19861, + "\u0120sq": 19862, + "\u0120remotely": 19863, + "Wed": 19864, + "asers": 19865, + "\u0120refriger": 19866, + "\u0120tiles": 19867, + "\u0120rescued": 19868, + "\u0120comprised": 19869, + "insky": 19870, + "\u0120manif": 19871, + "avanaugh": 19872, + "\u0120prolifer": 19873, + "\u0120aligned": 19874, + "xml": 19875, + "\u0120triv": 19876, + "\u0120coordination": 19877, + "\u0120PER": 19878, + "\u0120Quote": 19879, + "134": 19880, + "bf": 19881, + "\u0120Saw": 19882, + "\u0120termination": 19883, + "\u0120190": 19884, + "\u0120additions": 19885, + "\u0120trio": 19886, + "\u0120projections": 19887, + "\u0120positively": 19888, + "\u0120inclusive": 19889, + "\u0120membr": 19890, + "1990": 19891, + "older": 19892, + "\u0120practiced": 19893, + "inkle": 19894, + "Arch": 19895, + "\u0120starters": 19896, + "arius": 19897, + "\u0120intermediate": 19898, + "\u0120Benef": 19899, + "\u0120Killer": 19900, + "\u0120interventions": 19901, + "\u0120Kil": 19902, + "\u0120Flying": 19903, + "Inv": 19904, + "\u0120premature": 19905, + "\u0120psychiatric": 19906, + "\u0120indie": 19907, + "\u0120collar": 19908, + "\u0120Rainbow": 19909, + "afi": 19910, + "\u0120disruption": 19911, + "\u0120FOX": 19912, + "casting": 19913, + "\u0120misdem": 19914, + "cro": 19915, + "\u0120wipe": 19916, + "ardon": 19917, + "\u0120bast": 19918, + "\u0120Tommy": 19919, + "\u0120Representative": 19920, + "\u0120belly": 19921, + "\u0120PO": 19922, + "\u0120Breitbart": 19923, + "132": 19924, + "\u0120messaging": 19925, + "Should": 19926, + "References": 19927, + "\u0120GRE": 19928, + "istical": 19929, + "LP": 19930, + "\u0120Cav": 19931, + "\u0120Crazy": 19932, + "\u0120intuitive": 19933, + "keeping": 19934, + "\u0120Moss": 19935, + "\u0120discontin": 19936, + "\u0120Module": 19937, + "\u0120unrelated": 19938, + "\u0120Practice": 19939, + "\u0120Transport": 19940, + "\u0120statistically": 19941, + "orns": 19942, + "\u0120sized": 19943, + "pu": 19944, + "\u0120caf": 19945, + "\u0120Worlds": 19946, + "\u0120Rodgers": 19947, + "\u0120Lun": 19948, + "\u0120Comic": 19949, + "living": 19950, + "\u0120cared": 19951, + "\u0120climbed": 19952, + "){": 19953, + "\u0120consisted": 19954, + "\u0120medieval": 19955, + "folk": 19956, + "\u0120hacked": 19957, + "\u0120dire": 19958, + "\u0120Hermione": 19959, + "\u0120tended": 19960, + "ceans": 19961, + "Daniel": 19962, + "went": 19963, + "\u0120legislators": 19964, + "\u0120redes": 19965, + "games": 19966, + "\u0120gn": 19967, + "amiliar": 19968, + "\u0120++": 19969, + "ggy": 19970, + "threat": 19971, + "\u0120magnet": 19972, + "\u0120perceive": 19973, + "\u0120zip": 19974, + "\u0120indictment": 19975, + "\u0120critique": 19976, + "gard": 19977, + "\u0120Safe": 19978, + "\u0120Cream": 19979, + "\u0120advent": 19980, + "oba": 19981, + "\u0120vowed": 19982, + "ousands": 19983, + "\u0120ski": 19984, + "\u0120abortions": 19985, + "uart": 19986, + "\u0120stunned": 19987, + "\u0120advancing": 19988, + "\u0120lacked": 19989, + "\u0120\\\"": 19990, + "\u0120schizophren": 19991, + "\u0120elegant": 19992, + "\u0120conferences": 19993, + "\u0120canceled": 19994, + "\u0120Hudson": 19995, + "\u0120Hopefully": 19996, + "\u0120trump": 19997, + "\u0120frequencies": 19998, + "\u0120meteor": 19999, + "\u0120Junior": 20000, + "\u0120Fleet": 20001, + "\u0120Malcolm": 20002, + "\u0120Tools": 20003, + "\u0120........": 20004, + "\u0120hobby": 20005, + "\u0120Europeans": 20006, + "\u01201500": 20007, + "\u0120Into": 20008, + "\u0120sway": 20009, + "\u0120Appro": 20010, + "\u0120Compl": 20011, + "Community": 20012, + "\u0120tide": 20013, + "\u0120Summit": 20014, + "\u00e4\u00bb": 20015, + "\u0120intervals": 20016, + "\u0120Ether": 20017, + "\u0120habitat": 20018, + "\u0120Stevens": 20019, + "lishing": 20020, + "\u0120Domain": 20021, + "\u0120triggers": 20022, + "\u0120chasing": 20023, + "\u0120charm": 20024, + "\u0120Flower": 20025, + "itored": 20026, + "\u0120blessing": 20027, + "\u0120textures": 20028, + "Five": 20029, + "\u0120liquor": 20030, + "RP": 20031, + "FIN": 20032, + "\u01201962": 20033, + "CAR": 20034, + "Unknown": 20035, + "\u0120resil": 20036, + "\u0120Lily": 20037, + "\u0120abundance": 20038, + "\u0120predictable": 20039, + "rar": 20040, + "\u0120bullshit": 20041, + "leen": 20042, + "chet": 20043, + "Mor": 20044, + "Much": 20045, + "\u00e4\u00b9": 20046, + "\u0120emphasized": 20047, + "\u0120crust": 20048, + "\u0120primitive": 20049, + "\u0120enjoyable": 20050, + "\u0120Pictures": 20051, + "\u0120teammate": 20052, + "pler": 20053, + "\u0120Tol": 20054, + "\u0120Kane": 20055, + "\u0120summoned": 20056, + "thy": 20057, + "rama": 20058, + "\u0120Honda": 20059, + "\u0120realizing": 20060, + "\u0120quicker": 20061, + "\u0120concentrate": 20062, + "clear": 20063, + "\u0120210": 20064, + "\u0120Erdogan": 20065, + "aris": 20066, + "\u0120responds": 20067, + "\u0120BI": 20068, + "\u0120eligibility": 20069, + "\u0120pushes": 20070, + "\u0120Idaho": 20071, + "\u0120aggrav": 20072, + "\u0120ruins": 20073, + "urations": 20074, + "\u0120bans": 20075, + "\u0120anat": 20076, + "share": 20077, + "\u0120grind": 20078, + "hin": 20079, + "umen": 20080, + "\u0120utilities": 20081, + "\u0120Yankees": 20082, + "\u0120databases": 20083, + "\u0120DD": 20084, + "\u0120displaced": 20085, + "\u0120dependencies": 20086, + "\u0120stimulation": 20087, + "hun": 20088, + "houses": 20089, + "\u0120Pretty": 20090, + "\u0120Ravens": 20091, + "\u0120TODAY": 20092, + "\u0120associates": 20093, + "\u0120therape": 20094, + "cled": 20095, + "\u0120deer": 20096, + "\u0120repairs": 20097, + "rentice": 20098, + "\u0120receptors": 20099, + "\u0120remed": 20100, + "\u0120Ce": 20101, + "\u0120marriages": 20102, + "\u0120ballots": 20103, + "\u0120Soldier": 20104, + "\u0120hilarious": 20105, + "opl": 20106, + "138": 20107, + "\u0120inherently": 20108, + "\u0120ignorant": 20109, + "\u0120bounce": 20110, + "\u0120Easter": 20111, + "RELATED": 20112, + "\u0120Currency": 20113, + "EV": 20114, + "\u00e3\u0125\u0140": 20115, + "\u0120Lead": 20116, + "\u0120deceased": 20117, + "Brien": 20118, + "\u0120Musk": 20119, + "JS": 20120, + "\u0120merge": 20121, + "hearted": 20122, + "creat": 20123, + "mitt": 20124, + "mund": 20125, + "\u0120\u00e2\u0122\u012d": 20126, + "\u0120Bag": 20127, + "\u0120projection": 20128, + "\u0120java": 20129, + "\u0120Standards": 20130, + "\u0120Leonard": 20131, + "\u0120coconut": 20132, + "\u0120Population": 20133, + "\u0120traject": 20134, + "\u0120imply": 20135, + "\u0120curiosity": 20136, + "\u0120DB": 20137, + "\u0120Fresh": 20138, + "\u0120Por": 20139, + "\u0120heavier": 20140, + "neys": 20141, + "gomery": 20142, + "\u0120deserved": 20143, + "\u0120phrases": 20144, + "\u0120GC": 20145, + "\u0120yeast": 20146, + "desc": 20147, + "Death": 20148, + "\u0120reboot": 20149, + "\u0120metadata": 20150, + "ICAL": 20151, + "\u0120repay": 20152, + "\u0120Independence": 20153, + "\u0120suburban": 20154, + "icals": 20155, + "\u0120atop": 20156, + "\u0120allocation": 20157, + "generation": 20158, + "\u0120Gram": 20159, + "\u0120moisture": 20160, + "\u0120pine": 20161, + "\u0120Liberals": 20162, + "\u0120aides": 20163, + "\u0120underest": 20164, + "\u0120Berry": 20165, + "\u0120ceremon": 20166, + "370": 20167, + "astrous": 20168, + "\u0120Pirates": 20169, + "\u0120tense": 20170, + "\u0120Industries": 20171, + "\u0120Appeals": 20172, + "\u0120Near": 20173, + "\u0120\u00e8\u00a3\u0131\u00e7": 20174, + "\u0120lovers": 20175, + "\u0120CAP": 20176, + "\u0120Craw": 20177, + "\u0120giants": 20178, + "\u0120efficacy": 20179, + "Element": 20180, + "\u0120Behavior": 20181, + "\u0120Toyota": 20182, + "\u0120intest": 20183, + "Priv": 20184, + "AI": 20185, + "\u0120maneuver": 20186, + "\u0120perfection": 20187, + "\u0120bang": 20188, + "paper": 20189, + "rill": 20190, + "George": 20191, + "border": 20192, + "inters": 20193, + "\u0120Seth": 20194, + "\u0120clues": 20195, + "\u0120Levi": 20196, + "\u0120Revenue": 20197, + "147": 20198, + "\u0120vapor": 20199, + "\u0120fortunate": 20200, + "\u0120threatens": 20201, + "\u0120vet": 20202, + "\u0120dependency": 20203, + "ersed": 20204, + "article": 20205, + "\u0120Blizzard": 20206, + "\u0120chlor": 20207, + "\u0120minus": 20208, + "\u0120Bills": 20209, + "\u0120cryptocurrency": 20210, + "\u0120metabolism": 20211, + "tering": 20212, + "\u0120pestic": 20213, + "steps": 20214, + "\u0120Treasure": 20215, + "racted": 20216, + "\u0120Constant": 20217, + "\u0120temp": 20218, + "139": 20219, + "\u0120Detective": 20220, + "urally": 20221, + "\u0120recovering": 20222, + "\u0120cortex": 20223, + "\u0120144": 20224, + "closed": 20225, + "\u0120prejudice": 20226, + "aunted": 20227, + "\u0120storms": 20228, + "\u0120NOW": 20229, + "\u0120machinery": 20230, + "Address": 20231, + "\u0120compelled": 20232, + "270": 20233, + "\u0120despair": 20234, + "bane": 20235, + "\u0120vegetable": 20236, + "\u0120beds": 20237, + "Learn": 20238, + "\u0120colorful": 20239, + "\u0120spike": 20240, + "\u0120margins": 20241, + "\u0120sympathy": 20242, + "\u0120workshop": 20243, + "\u0120CBC": 20244, + "Sat": 20245, + "\u0120burns": 20246, + "\u0120Gender": 20247, + "\u0120129": 20248, + "\u0120Cable": 20249, + "\u0120debts": 20250, + "\u0120Theresa": 20251, + "\u0120reflecting": 20252, + "\u0120airst": 20253, + "\u0120rim": 20254, + "ramid": 20255, + "\u0120weaknesses": 20256, + "Writ": 20257, + "oggle": 20258, + "ti": 20259, + "\u0120Charge": 20260, + "\u0120weighed": 20261, + "\u0120(.": 20262, + "\u0120laughter": 20263, + "\u0120router": 20264, + "\u0120Democracy": 20265, + "Dear": 20266, + "\u0120hasht": 20267, + "\u0120dy": 20268, + "\u0120hints": 20269, + "running": 20270, + "\u0120finishes": 20271, + "arus": 20272, + "Mass": 20273, + "result": 20274, + "ascus": 20275, + "\u0120vintage": 20276, + "\u0120conqu": 20277, + "\u0120wildly": 20278, + "acist": 20279, + "\u0120lingu": 20280, + "\u0120protagonist": 20281, + "strom": 20282, + "teenth": 20283, + "\u0120Solo": 20284, + "mac": 20285, + "filled": 20286, + "\u0120renown": 20287, + "itives": 20288, + "\u0120motive": 20289, + "\u0120Antar": 20290, + "\u0120Mann": 20291, + "\u0120Adjust": 20292, + "\u0120rockets": 20293, + "\u0120troubling": 20294, + "ei": 20295, + "\u0120organisms": 20296, + "assis": 20297, + "Christian": 20298, + "\u0120145": 20299, + "\u0120Hass": 20300, + "\u0120swall": 20301, + "\u0120wax": 20302, + "\u0120Survival": 20303, + "VS": 20304, + "\u0120Murd": 20305, + "vd": 20306, + "standard": 20307, + "\u0120dragons": 20308, + "\u0120acceleration": 20309, + "rational": 20310, + "final": 20311, + "\u0120paired": 20312, + "\u0120Ethereum": 20313, + "\u0120interfaces": 20314, + "\u0120resent": 20315, + "\u0120artifacts": 20316, + "\u00c5\u00ab": 20317, + "arel": 20318, + "\u0120competitor": 20319, + "\u0120Nicholas": 20320, + "\u0120Surface": 20321, + "cpp": 20322, + "\u0120Tot": 20323, + "\u0120economically": 20324, + "\u0120organised": 20325, + "\u0120enforced": 20326, + "inho": 20327, + "\u0120varieties": 20328, + "\u0120abdom": 20329, + "\u0120Bailey": 20330, + "idav": 20331, + "\u0120Salv": 20332, + "paid": 20333, + "\u0120altitude": 20334, + "essert": 20335, + "\u0120Gutenberg": 20336, + "area": 20337, + "opoulos": 20338, + "\u0120professors": 20339, + "iggs": 20340, + "\u0120Fate": 20341, + "hey": 20342, + "\u01203000": 20343, + "Dist": 20344, + "\u0120twins": 20345, + "cill": 20346, + "\u0120Maps": 20347, + "\u0120traps": 20348, + "\u0120weed": 20349, + "\u0120Kiss": 20350, + "\u0120yoga": 20351, + "\u0120recipients": 20352, + "\u0120Westminster": 20353, + "\u0120pools": 20354, + "\u0120Walmart": 20355, + "188": 20356, + "\u0120Schools": 20357, + "attack": 20358, + "\u0120ARM": 20359, + "paragraph": 20360, + "Warning": 20361, + "jl": 20362, + "\u0120selfish": 20363, + "anchez": 20364, + "\u0120Heights": 20365, + "Fre": 20366, + "\u0120Soph": 20367, + "\u0120--------------------------------": 20368, + "tml": 20369, + "333": 20370, + "\u0120raids": 20371, + "\u0120satellites": 20372, + "KEY": 20373, + "\u0120lasts": 20374, + "\u00d1\u0124": 20375, + "Ins": 20376, + "\u0120Dame": 20377, + "\u0120unpredict": 20378, + "///": 20379, + "ghai": 20380, + "\u0120artillery": 20381, + "\u0120cruise": 20382, + "\u0120gel": 20383, + "\u0120Cabinet": 20384, + "\u0120blows": 20385, + "\u0120Esp": 20386, + "\u0120proximity": 20387, + "othe": 20388, + "\u0120Skills": 20389, + "\u0120Upper": 20390, + "obo": 20391, + "\u0120NDP": 20392, + "\u0120enjoys": 20393, + "\u0120repeating": 20394, + "\u0120Construction": 20395, + "\u0120Questions": 20396, + "Hillary": 20397, + "\u0120uint": 20398, + "\u0120processors": 20399, + "\u0120Gibson": 20400, + "\u0120Multiple": 20401, + "qa": 20402, + "\u0120Bom": 20403, + "\u0120Miles": 20404, + "ventional": 20405, + "\u0120hurts": 20406, + "skin": 20407, + "\u0120AIDS": 20408, + "\u0120advisers": 20409, + "\u0120Root": 20410, + "\u0120methodology": 20411, + "\u0120Dale": 20412, + "\u0120deton": 20413, + "\u0120Knowledge": 20414, + "sequently": 20415, + "\u0120121": 20416, + "\u0120connects": 20417, + "Cy": 20418, + "\u0120Danger": 20419, + "\u0120contributors": 20420, + "\u0120Bent": 20421, + "\u0120brass": 20422, + "\u0120Guns": 20423, + "into": 20424, + "\u0120Fortune": 20425, + "\u0120broker": 20426, + "balance": 20427, + "\u0120lengths": 20428, + "\u0120vic": 20429, + "\u0120averaging": 20430, + "\u0120appropriately": 20431, + "\u0120Camera": 20432, + "\u0120sandwich": 20433, + "\u0120CDC": 20434, + "\u0120coordinate": 20435, + "\u0120navig": 20436, + "\u0120goodness": 20437, + "laim": 20438, + "\u0120brake": 20439, + "\u0120extremist": 20440, + "\u0120Wake": 20441, + "\u0120Mend": 20442, + "\u0120Tiny": 20443, + "\u0120COL": 20444, + "\u0120RF": 20445, + "\u0120Dual": 20446, + "\u0120Wine": 20447, + "Case": 20448, + "\u0120refined": 20449, + "\u0120lamp": 20450, + "Lead": 20451, + "\u0120bapt": 20452, + "\u0120Carb": 20453, + "\u0120Sadd": 20454, + "\u0120Minneapolis": 20455, + "PDF": 20456, + "Early": 20457, + "\u0120Hidden": 20458, + "Its": 20459, + "\u0120TIME": 20460, + "\u0120pap": 20461, + "\u0120commissioned": 20462, + "\u0120Few": 20463, + "\u0120Colts": 20464, + "\u0120Bren": 20465, + "\u0120bothered": 20466, + "\u0120likewise": 20467, + "Exper": 20468, + "\u0120Schw": 20469, + "cry": 20470, + "nn": 20471, + "\u0120Mitch": 20472, + "imon": 20473, + "MG": 20474, + "bm": 20475, + "UMP": 20476, + "rays": 20477, + "\u0120registry": 20478, + "\u0120270": 20479, + "achine": 20480, + "rella": 20481, + "anting": 20482, + "00000": 20483, + "\u0120ruined": 20484, + "spot": 20485, + "\u0120ta": 20486, + "\u0120maximize": 20487, + "\u0120inconven": 20488, + "Dead": 20489, + "Human": 20490, + "Enabled": 20491, + "\u0120Marie": 20492, + "\u0120chill": 20493, + "\u0120Paradise": 20494, + "\u0120starring": 20495, + "\u0120Latino": 20496, + "\u0120Protocol": 20497, + "\u0120EVER": 20498, + "\u0120suppliers": 20499, + "message": 20500, + "\u0120Brock": 20501, + "\u0120serum": 20502, + "\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a": 20503, + "\u0120encomp": 20504, + "\u0120ambition": 20505, + "uese": 20506, + "\u0120arrows": 20507, + "Andrew": 20508, + "\u0120antenna": 20509, + "\u01201961": 20510, + "\u0120Bark": 20511, + "\u0120bool": 20512, + "\u00e3\u0124\u00aa": 20513, + "\u0120Storage": 20514, + "\u0120railway": 20515, + "\u0120tougher": 20516, + "\u0120Cad": 20517, + "\u0120washing": 20518, + "Py": 20519, + "']": 20520, + "embed": 20521, + "\u0120Memphis": 20522, + "ackle": 20523, + "\u0120famously": 20524, + "\u0120Fortunately": 20525, + "ovies": 20526, + "\u0120mindset": 20527, + "\u0120sneak": 20528, + "\u0120Dh": 20529, + "RAW": 20530, + "\u0120Simpson": 20531, + "\u0120livest": 20532, + "\u0120landmark": 20533, + "\u0120cement": 20534, + "Low": 20535, + "\u0120thrilled": 20536, + "\u0120Course": 20537, + "inel": 20538, + "\u0120chuck": 20539, + "idate": 20540, + "global": 20541, + "\u0120whit": 20542, + "\u0120\u00ef\u00bf\u00bd": 20543, + "adays": 20544, + "ski": 20545, + "\u0120SV": 20546, + "\u0120viruses": 20547, + "306": 20548, + "\u0120Respons": 20549, + "\u0120theaters": 20550, + "\u0120Branch": 20551, + "\u0120Geneva": 20552, + "\u0120MK": 20553, + "\u0120unbeliev": 20554, + "\u0120communist": 20555, + "Original": 20556, + "\u0120Received": 20557, + "\u0120Transfer": 20558, + "\u0120Arg": 20559, + "Input": 20560, + "\u0120Strategy": 20561, + "\u0120palace": 20562, + "thening": 20563, + "Dri": 20564, + "\u0120sentencing": 20565, + "umbnail": 20566, + "\u0120pins": 20567, + "recy": 20568, + "\u0120siblings": 20569, + "Getting": 20570, + "\u0120BU": 20571, + "\u0120Northwest": 20572, + "\u0120prolonged": 20573, + "\u0120Sakura": 20574, + "Comb": 20575, + "\u0120Bour": 20576, + "\u0120inadequate": 20577, + "\u0120Kash": 20578, + "\u0120username": 20579, + "\u0120Improve": 20580, + "\u0120battling": 20581, + "\u0120MAC": 20582, + "\u0120curriculum": 20583, + "\u0120soda": 20584, + "\u0120Cannon": 20585, + "\u0120sensible": 20586, + "spons": 20587, + "December": 20588, + "\u0120wicked": 20589, + "\u0120Pengu": 20590, + "\u0120dictators": 20591, + "\u0120Hearts": 20592, + "ogyn": 20593, + "\u0120similarities": 20594, + "\u0120Stats": 20595, + "\u0120hollow": 20596, + "itations": 20597, + "\":[": 20598, + "\u0120hover": 20599, + "\u0120Listen": 20600, + "sch": 20601, + "Sund": 20602, + "\u0120cad": 20603, + "\u0120Parks": 20604, + "\u0120lur": 20605, + "\u0120hype": 20606, + "\u0120Lem": 20607, + "NAME": 20608, + "isure": 20609, + "Friday": 20610, + "\u0120shoots": 20611, + "\u0120closes": 20612, + "\u0120db": 20613, + "\u0120Ridge": 20614, + "\u0120Different": 20615, + "\u0120replies": 20616, + "\u0120Broadway": 20617, + "opers": 20618, + "\u0120intoler": 20619, + "\u0120Zeus": 20620, + "akespe": 20621, + "\u0120proprietary": 20622, + "\u0120requesting": 20623, + "\u0120controllers": 20624, + "\u0120MIN": 20625, + "imedia": 20626, + "becca": 20627, + "\u0120expans": 20628, + "\u0120oils": 20629, + "Bot": 20630, + "\u0120Chand": 20631, + "\u0120printer": 20632, + "\u0120topped": 20633, + "\u0120POL": 20634, + "\u0120Earlier": 20635, + "Social": 20636, + "avin": 20637, + "\u0120decreases": 20638, + "\u0120Seb": 20639, + "\u0120specifications": 20640, + "\u0120Blast": 20641, + "\u0120Kurt": 20642, + "\u0120freel": 20643, + "Brown": 20644, + "\u0120dilig": 20645, + "roe": 20646, + "\u0120Problem": 20647, + "\u0120Quad": 20648, + "\u0120decentral": 20649, + "\u0120Vector": 20650, + "anut": 20651, + "\u0120plugins": 20652, + "\u0120Gregory": 20653, + "\u0120fucked": 20654, + "elines": 20655, + "\u0120Ambassador": 20656, + "take": 20657, + "\u0120cleans": 20658, + "ongyang": 20659, + "Anonymous": 20660, + "stro": 20661, + "\"}": 20662, + "aline": 20663, + "\u0120Odd": 20664, + "\u0120Eug": 20665, + "216": 20666, + "\u0120boil": 20667, + "\u0120Powers": 20668, + "\u0120nurses": 20669, + "Obviously": 20670, + "\u0120Technical": 20671, + "\u0120exceeded": 20672, + "ORS": 20673, + "\u0120extremists": 20674, + "\u0120traces": 20675, + "expl": 20676, + "\u0120comr": 20677, + "\u0120Sach": 20678, + ")/": 20679, + "\u0120masks": 20680, + "\u0120sci": 20681, + "Bon": 20682, + "\u0120regression": 20683, + "wegian": 20684, + "\u0120advisor": 20685, + "itures": 20686, + "\u0120Vo": 20687, + "example": 20688, + "\u0120Instruct": 20689, + "\u0120siege": 20690, + "\u0120reductions": 20691, + "ptr": 20692, + "\u0120statutory": 20693, + "\u0120removes": 20694, + "\u0120puck": 20695, + "redits": 20696, + "\u0120bee": 20697, + "\u0120salad": 20698, + "\u0120promotions": 20699, + "\u0120Joshua": 20700, + "withstanding": 20701, + "ETH": 20702, + "\u0120Cha": 20703, + "imus": 20704, + "\u0120expenditure": 20705, + "aunting": 20706, + "\u0120delighted": 20707, + "\u0120155": 20708, + "beh": 20709, + "\u0120carpet": 20710, + "\u0120Spart": 20711, + "\u0120jungle": 20712, + "lists": 20713, + "\u0120bullying": 20714, + "\u0120Nobel": 20715, + "\u0120Glen": 20716, + "\u0120referenced": 20717, + "\u0120introduces": 20718, + "sein": 20719, + "\u0120chopped": 20720, + "glass": 20721, + "\u0120Wrest": 20722, + "\u0120neutrality": 20723, + "\u0120\u00e2\u013b": 20724, + "\u0120investigator": 20725, + "\u0120shelves": 20726, + "\u0120unconstitutional": 20727, + "\u0120reproduction": 20728, + "\u0120merchant": 20729, + "mia": 20730, + "\u0120metrics": 20731, + "\u0120explosives": 20732, + "\u0120Sonia": 20733, + "\u0120bodily": 20734, + "\u0120thickness": 20735, + "\u0120predominantly": 20736, + "\u0120Ability": 20737, + "\u0120monitored": 20738, + "ICH": 20739, + "\u0120].": 20740, + "\u0120Martinez": 20741, + "\u0120visibility": 20742, + "\u0120queries": 20743, + "\u0120genocide": 20744, + "\u0120Warfare": 20745, + "Query": 20746, + "\u0120studios": 20747, + "\u0120embry": 20748, + "\u0120corridor": 20749, + "\u0120cleaned": 20750, + "complete": 20751, + "\u0120MH": 20752, + "\u0120enrollment": 20753, + "INGS": 20754, + "\u0120impacted": 20755, + "\u0120disastrous": 20756, + "\u0120Yun": 20757, + "\u0120Claire": 20758, + "\u0120Basically": 20759, + "yt": 20760, + "usterity": 20761, + "\u0120indirectly": 20762, + "wik": 20763, + "\u0120dod": 20764, + "\u0120Carr": 20765, + "\u0120amp": 20766, + "\u0120prohibit": 20767, + "\u0120Initial": 20768, + "\u0120Rd": 20769, + "iji": 20770, + "\u0120educate": 20771, + "corn": 20772, + "iott": 20773, + "\u0120Beauty": 20774, + "\u0120detective": 20775, + "\u0120Conn": 20776, + "since": 20777, + "\u0120stagger": 20778, + "\u0120obese": 20779, + "\u0120bree": 20780, + "ologic": 20781, + "isse": 20782, + "walker": 20783, + "\u0120blades": 20784, + "\u0120lawful": 20785, + "func": 20786, + "\u0120Behind": 20787, + "\u0120appetite": 20788, + "\u0120(*": 20789, + "\u0120tennis": 20790, + "\u0120offspring": 20791, + "\u0120jets": 20792, + "\u0120structured": 20793, + "\u0120aforementioned": 20794, + "Nov": 20795, + "\u0120scaling": 20796, + "fill": 20797, + "\u0120stew": 20798, + "\u0120curb": 20799, + "\u0120Stephan": 20800, + "edIn": 20801, + "SF": 20802, + "obic": 20803, + "\u00e9\u0143\u0136": 20804, + "oug": 20805, + "\u0120MM": 20806, + "\u0120genetically": 20807, + "opez": 20808, + "136": 20809, + "\u0120umb": 20810, + "ancers": 20811, + "\u0120cohort": 20812, + "\u0120merchandise": 20813, + "\u0120imposing": 20814, + "\u0120Legislature": 20815, + "\u0120Archive": 20816, + "ivia": 20817, + "\u0120Naval": 20818, + "\u0120offences": 20819, + "\u0120miracle": 20820, + "\u0120snapped": 20821, + "\u0120foes": 20822, + "\u0120extensively": 20823, + "\u0120Raf": 20824, + "\u0120cater": 20825, + "edience": 20826, + "Kit": 20827, + "\u0120Bin": 20828, + "\u0120recommends": 20829, + "\u0120Cities": 20830, + "\u0120rigid": 20831, + "\u0120READ": 20832, + "\u0120Noble": 20833, + "\u0120Tian": 20834, + "\u0120certificates": 20835, + "antis": 20836, + "oiler": 20837, + "\u0120Buddhist": 20838, + "did": 20839, + "\u0120surveyed": 20840, + "\u0120downward": 20841, + "\u0120prints": 20842, + "\u0120Motion": 20843, + "ronics": 20844, + "\u0120Sans": 20845, + "ossibly": 20846, + "uctions": 20847, + "\u0120colonies": 20848, + "\u0120Danish": 20849, + "unit": 20850, + "\u0120spoil": 20851, + "\u0120advisory": 20852, + "berries": 20853, + "Plan": 20854, + "\u0120specification": 20855, + "ophers": 20856, + "\u0120Resource": 20857, + "\u0120shirts": 20858, + "prisingly": 20859, + "communications": 20860, + "\u0120trivial": 20861, + "\u0120mentioning": 20862, + "isexual": 20863, + "\u0120supplements": 20864, + "\u0120supervision": 20865, + "BP": 20866, + "vor": 20867, + "\u0120wit": 20868, + "\u0120cooldown": 20869, + "\u0120plaintiff": 20870, + "\u0120Reviews": 20871, + "\u0120Sri": 20872, + "\u0120Mint": 20873, + "\u0120Sugar": 20874, + "\u0120afterward": 20875, + "\u0120Priest": 20876, + "\u0120Investment": 20877, + "ogene": 20878, + "\u0120Taking": 20879, + "\u0120stretching": 20880, + "\u0120inflammation": 20881, + "\u0120Tehran": 20882, + "\u0120lining": 20883, + "\u0120freezing": 20884, + "\u0120Entity": 20885, + "\u0120inspiring": 20886, + "special": 20887, + "price": 20888, + "\u0120sue": 20889, + "\u0120Porter": 20890, + "ounge": 20891, + "ETA": 20892, + "\u0120Derek": 20893, + "\u0120Luis": 20894, + "uo": 20895, + "ymph": 20896, + "\u0120exterior": 20897, + "ihil": 20898, + "\u0120Ashley": 20899, + "inator": 20900, + "\u0120nutrients": 20901, + "\u0120Thrones": 20902, + "\u0120finances": 20903, + "\u0120Inspect": 20904, + "\u0120specially": 20905, + "\u0120Required": 20906, + "\u0120PTS": 20907, + "\u0120Violence": 20908, + "ointed": 20909, + "shots": 20910, + "\u0120excerpt": 20911, + "coon": 20912, + "INS": 20913, + "\u0120Gri": 20914, + "\u0120recognised": 20915, + "Week": 20916, + "Young": 20917, + "\u0120vom": 20918, + "isle": 20919, + "\u0120Curry": 20920, + "\u0120Buddh": 20921, + "\u0120notebook": 20922, + "\u0120durable": 20923, + "/?": 20924, + "\u0120Gad": 20925, + "\u0120Pupp": 20926, + "\u0120forgive": 20927, + "park": 20928, + "\u0120personalities": 20929, + "analysis": 20930, + "clamation": 20931, + "\u0120elevator": 20932, + "\u0120warehouse": 20933, + "\u0120Role": 20934, + "unn": 20935, + "\u0120illustration": 20936, + "\u0120Scan": 20937, + "\u0120atmospheric": 20938, + "Import": 20939, + "ANC": 20940, + "ricted": 20941, + "fu": 20942, + "010": 20943, + "\u0120arche": 20944, + "\u0120rewarded": 20945, + "akespeare": 20946, + "\u0120internally": 20947, + "\u0120RBI": 20948, + "alker": 20949, + "\u0120elephant": 20950, + "owitz": 20951, + "\u0120Pizza": 20952, + "\u0120bipartisan": 20953, + "\u00c3\u00a9s": 20954, + "\u0120slowed": 20955, + "\u0120Stark": 20956, + "\u0120override": 20957, + "OUS": 20958, + "\u0120320": 20959, + "undreds": 20960, + "\u0120Deck": 20961, + "\u0120Census": 20962, + "bee": 20963, + "146": 20964, + "otor": 20965, + "\u0120ip": 20966, + "\u0120ub": 20967, + "ocations": 20968, + "\u0120Button": 20969, + "rice": 20970, + "\u0120cripp": 20971, + "fff": 20972, + "\u0120originated": 20973, + "\u0120overwhelmed": 20974, + "appa": 20975, + "\u0120foremost": 20976, + "\u00e2\u0122\u0133": 20977, + "\u0120LEG": 20978, + "release": 20979, + "eatured": 20980, + "atches": 20981, + "\u0120reps": 20982, + "\u0120lending": 20983, + "\u0120Reference": 20984, + "\u0120Client": 20985, + "165": 20986, + "venth": 20987, + "Complete": 20988, + "\u0120Patrol": 20989, + "\u0120sworn": 20990, + "cam": 20991, + "\u0120shuttle": 20992, + "\u0120Ralph": 20993, + "\u0120hometown": 20994, + "-,": 20995, + "onal": 20996, + "\u0120BP": 20997, + "\u00e5\u0131": 20998, + "\u0120persuade": 20999, + "\u0120Alexand": 21000, + "\u0120combines": 21001, + "\u0120vivid": 21002, + "\u0120Lag": 21003, + "\u0120encoding": 21004, + "\u0120salvation": 21005, + "wen": 21006, + "\u0120Recovery": 21007, + "iya": 21008, + "University": 21009, + "\u0120Biden": 21010, + "\u0120budgets": 21011, + "\u0120Texans": 21012, + "fits": 21013, + "\u0120honored": 21014, + "\u0120python": 21015, + "TD": 21016, + "###": 21017, + "clone": 21018, + "\u0120blink": 21019, + "\u0120Liquid": 21020, + "\u0120unemployed": 21021, + "\u0120clashes": 21022, + "\u0120Counsel": 21023, + "\u0120directing": 21024, + "\u0120punct": 21025, + "\u0120Falcons": 21026, + "\u0120shark": 21027, + "\u0120Damascus": 21028, + "\u0120jeans": 21029, + "\u0120embark": 21030, + "\u0120seize": 21031, + "\u0120upwards": 21032, + "280": 21033, + "\u0120Ez": 21034, + "\u0120Anything": 21035, + "\u0120exotic": 21036, + "lower": 21037, + "\u0120Creator": 21038, + "\u0120Um": 21039, + "\u0120suburbs": 21040, + "berger": 21041, + "\u0120Wend": 21042, + "\u0120mint": 21043, + "\u0120XX": 21044, + "\u0120Dro": 21045, + "\u0120suffers": 21046, + "\u0120herb": 21047, + "tree": 21048, + "\u0120fragile": 21049, + "\u0120flooded": 21050, + "\u0120Alcohol": 21051, + "olean": 21052, + "nyder": 21053, + "\u0120KO": 21054, + "Fram": 21055, + "\u0120136": 21056, + "\u0120owed": 21057, + "\u0120Melee": 21058, + "\u0120Hash": 21059, + "\u0120whisk": 21060, + "\u0120sudo": 21061, + "rr": 21062, + "Quick": 21063, + "appro": 21064, + "\u0120ii": 21065, + "\u0120Examples": 21066, + "hee": 21067, + "\u0120promotes": 21068, + "perature": 21069, + "kar": 21070, + "\u0120Honor": 21071, + "\u0120sodium": 21072, + "\u0120Lif": 21073, + "rosso": 21074, + "intendent": 21075, + "\u0120correspondent": 21076, + "Found": 21077, + "secret": 21078, + "\u0120identifies": 21079, + "agne": 21080, + "\u0120lou": 21081, + "\u0120PP": 21082, + "\u0120coincidence": 21083, + "move": 21084, + "\u0120militia": 21085, + "\u0120infiltr": 21086, + "\u0120Primary": 21087, + "\u0120pitching": 21088, + "\u0120Ib": 21089, + "\u0120GOOD": 21090, + "\u00e3\u0124\u00b8": 21091, + "\u0120Wizards": 21092, + "iral": 21093, + "\u0120Venus": 21094, + "RR": 21095, + "\u0120\u00e2\u0122\u0137": 21096, + "\u0120Casey": 21097, + "\u0120sadly": 21098, + "\u0120admire": 21099, + "\u0120embarrassed": 21100, + "cb": 21101, + "Mel": 21102, + "\u0120tubes": 21103, + "\u0120beautifully": 21104, + "\u0120Queensland": 21105, + "Below": 21106, + "rez": 21107, + "quet": 21108, + "pleasant": 21109, + "\u0120\u00c2\u00ab": 21110, + "Camp": 21111, + "\u0120decisive": 21112, + "1998": 21113, + "\u0120Lamb": 21114, + "utton": 21115, + "hn": 21116, + "\u0120Jagu": 21117, + "aunder": 21118, + "\u0120Cord": 21119, + "\u0120clerk": 21120, + "\u0120caffe": 21121, + "\u0120wiped": 21122, + "\u0120reim": 21123, + "\u0120Mountains": 21124, + "\u0120imprisoned": 21125, + "\u0120develops": 21126, + "\u0120Pra": 21127, + "\u0120modeling": 21128, + "Anyone": 21129, + "ancel": 21130, + "\u0120Sit": 21131, + "\u0120shields": 21132, + "\u0120lawn": 21133, + "\u0120cardiovascular": 21134, + "\u0120demonstrating": 21135, + "\u0120parse": 21136, + "\u0120Israelis": 21137, + "\u0120euros": 21138, + "143": 21139, + "\u0120glorious": 21140, + "inski": 21141, + "ecd": 21142, + "\u0120conditioning": 21143, + "\u0120helpless": 21144, + "\u0120microsc": 21145, + "\u0120Harbor": 21146, + "\u0120stakes": 21147, + "\u0120260": 21148, + "\u0120unequ": 21149, + "\u0120Floyd": 21150, + "\u0120damp": 21151, + "\u0120apparatus": 21152, + "\u0120Laws": 21153, + "\u0120counters": 21154, + "\u0120induce": 21155, + "atable": 21156, + "\u0120Ahmed": 21157, + "\u0120slam": 21158, + "November": 21159, + "\u0120persist": 21160, + "\u0120imminent": 21161, + "\u00c3\u00a1n": 21162, + "\u0120shred": 21163, + "\u0120phases": 21164, + "\u0120Edmonton": 21165, + "\u0120Armstrong": 21166, + "\u0120Meet": 21167, + "\u0120Kitty": 21168, + "\u00d1\u0122": 21169, + "circ": 21170, + "\u0120Adult": 21171, + "\u0120arose": 21172, + "\u0120Xen": 21173, + "Dan": 21174, + "gow": 21175, + "\u0120superf": 21176, + "\u0120Admir": 21177, + "\u0120endure": 21178, + "\u0120keyword": 21179, + "yrus": 21180, + "\u0120yarn": 21181, + "\u0120pathway": 21182, + "\u0120Hopkins": 21183, + "midt": 21184, + "\u0120censorship": 21185, + "dependent": 21186, + "\u0120instructor": 21187, + "Sources": 21188, + "\u0120toe": 21189, + "\u0120balloon": 21190, + "Nob": 21191, + "\u0120swear": 21192, + "\u0120Castro": 21193, + "\u0120gloss": 21194, + "\u0120Kavanaugh": 21195, + "\u0120remarkably": 21196, + "Photos": 21197, + "\u0120Nom": 21198, + "\u0120Southeast": 21199, + "yers": 21200, + "\u0120validation": 21201, + "\u0120cannon": 21202, + "\u0120Victory": 21203, + "\u0120Pierre": 21204, + "\u0120cautious": 21205, + "Audio": 21206, + "\u0120fetch": 21207, + "\u0120Gift": 21208, + "\u0120Hyp": 21209, + "\u0120remedy": 21210, + "ZE": 21211, + "\u0120scent": 21212, + "\u0120beard": 21213, + "\u0120Rut": 21214, + "-\"": 21215, + "\u0120patents": 21216, + "Hy": 21217, + "\u0120unjust": 21218, + "\u0120potato": 21219, + "\u0120forthcoming": 21220, + "\u0120chef": 21221, + "\u0120Rift": 21222, + "affe": 21223, + "\u0120ROM": 21224, + "\u0120Launch": 21225, + "\u0120pads": 21226, + "\u0120Neo": 21227, + "\u0120onset": 21228, + "\u0120squeeze": 21229, + "safe": 21230, + "\u0120prefix": 21231, + "\u0120TM": 21232, + "\u0120Nearly": 21233, + "\u0120Clinical": 21234, + "\u0120Mental": 21235, + "otiation": 21236, + "\u0120Unic": 21237, + "antry": 21238, + "\u0120Cir": 21239, + "\u0120epit": 21240, + "\u00c3\u00a6": 21241, + "\u0120extracted": 21242, + "versely": 21243, + "riad": 21244, + "\u0120strains": 21245, + "\u0120tops": 21246, + "\u0120poem": 21247, + "\u0120Randy": 21248, + "\u0120Maple": 21249, + "THER": 21250, + "upiter": 21251, + "\u0120SSD": 21252, + "\u013c\u00e9": 21253, + "\u0120uncon": 21254, + "pering": 21255, + "\u0120slept": 21256, + "iners": 21257, + "\u0120underwater": 21258, + "\u0120Evidence": 21259, + "gone": 21260, + "205": 21261, + "\u0120historians": 21262, + "\u0120synthesis": 21263, + "\u0120frog": 21264, + "basketball": 21265, + "\u0120vibrant": 21266, + "\u0120subord": 21267, + "\u0120365": 21268, + "\u0120Dial": 21269, + "\u0120cooperate": 21270, + "HAHA": 21271, + "\u0120greeted": 21272, + "158": 21273, + "\u0120jazz": 21274, + "\u0120intox": 21275, + "\u0120Walking": 21276, + "\u0120supervisor": 21277, + "\u0120Fusion": 21278, + "\u0120Mercedes": 21279, + "send": 21280, + "Ham": 21281, + "sd": 21282, + "nl": 21283, + "\u0120tours": 21284, + "\u0120FIFA": 21285, + "\u0120culp": 21286, + "gd": 21287, + "304": 21288, + "\u0120pleas": 21289, + "\u0120illustrates": 21290, + "\u0120Colombia": 21291, + "\u0120highlighting": 21292, + "\u0120Summary": 21293, + "\u0120exposing": 21294, + "\u0120Dru": 21295, + "\u0120irony": 21296, + "ritional": 21297, + "\u0120Carroll": 21298, + "\u0120Ellis": 21299, + "Pict": 21300, + "\u0120Rapt": 21301, + "\u0120adapter": 21302, + "\u0120unm": 21303, + "\u0120corpse": 21304, + "\u0120celebrities": 21305, + "Den": 21306, + "atum": 21307, + "\u0120Apocalypse": 21308, + "\u0120Wag": 21309, + "lining": 21310, + "\u0120hormones": 21311, + "Rub": 21312, + "\u0120Xi": 21313, + "\u0120Vaults": 21314, + "208": 21315, + "alkyrie": 21316, + "inosaur": 21317, + "\u0120feeds": 21318, + "vity": 21319, + "\u0120defeating": 21320, + "Wait": 21321, + "\u0120emphasize": 21322, + "\u0120Steelers": 21323, + "yrinth": 21324, + "leys": 21325, + "\u0120Whenever": 21326, + "Currently": 21327, + "\u0120Clock": 21328, + "\u0120collectively": 21329, + "anyon": 21330, + "\u0120JP": 21331, + "\u0120mentality": 21332, + "\u0120downloads": 21333, + "\u0120surroundings": 21334, + "\u0120Barnes": 21335, + "\u0120flagship": 21336, + "\u0120indicators": 21337, + "\u0120grapp": 21338, + "January": 21339, + "\u0120Elemental": 21340, + "\u0120Athena": 21341, + "ibal": 21342, + "\u0120sights": 21343, + "\u0120capita": 21344, + "\u0120Treaty": 21345, + "\u0120voiced": 21346, + "\u0120Gaz": 21347, + "lette": 21348, + "\u0120ya": 21349, + "\u0120expired": 21350, + "Legend": 21351, + "Hot": 21352, + "nature": 21353, + "\u0120unstable": 21354, + "\u0120280": 21355, + "\u00c3\u00ba": 21356, + "Comment": 21357, + "ALE": 21358, + "\u0120quests": 21359, + "\u0120handler": 21360, + "nis": 21361, + "\u0120versatile": 21362, + "\u0120conceal": 21363, + "engeance": 21364, + "\u0120Interactive": 21365, + "\u0120obsessed": 21366, + "\u0120Dogs": 21367, + "\u0120cracked": 21368, + "Sound": 21369, + "sv": 21370, + "\u0120Dylan": 21371, + "roads": 21372, + "fx": 21373, + "\u0120Catholics": 21374, + "\u0120Hag": 21375, + "\u0120slammed": 21376, + "\u0120glowing": 21377, + "sale": 21378, + "\u0120tissues": 21379, + "\u0120Chi": 21380, + "nee": 21381, + "\u0120cher": 21382, + "sic": 21383, + "urrection": 21384, + "\u0120bacon": 21385, + "ulatory": 21386, + ").\"": 21387, + "\u0120irregular": 21388, + "FORM": 21389, + "assed": 21390, + "\u0120intentional": 21391, + "\u0120compensate": 21392, + "\u0120Speaking": 21393, + "\u0120Sets": 21394, + "153": 21395, + "\u0120conventions": 21396, + "bands": 21397, + "emade": 21398, + "\u0120ecc": 21399, + "\u0120Winston": 21400, + "\u0120Assassin": 21401, + "\u0120Belgian": 21402, + "\u0120dependence": 21403, + "\u0120niche": 21404, + "\u0120bark": 21405, + "\u0120Jazz": 21406, + "\u0120disadvantage": 21407, + "\u0120gasoline": 21408, + "\u0120165": 21409, + "\u00e7\u013c\u0126": 21410, + "essa": 21411, + "module": 21412, + "angular": 21413, + "OY": 21414, + "\u0120Treatment": 21415, + "itas": 21416, + "olation": 21417, + "\u0120Arnold": 21418, + "\u0120feud": 21419, + "\u0120Nest": 21420, + "\u0120theatre": 21421, + "ewater": 21422, + "\u0120minors": 21423, + "olicy": 21424, + "\u0120Haven": 21425, + "division": 21426, + "\u0120trunk": 21427, + "Far": 21428, + "\u0120Pull": 21429, + "\u0120capturing": 21430, + "\u01201800": 21431, + "\u0120Teen": 21432, + "\u0120exempl": 21433, + "\u0120clinics": 21434, + "\u0120Burg": 21435, + "\u0120substit": 21436, + "\u0120payload": 21437, + "\u0120Lav": 21438, + "\u0120Troy": 21439, + "\u0120Witness": 21440, + "\u0120fragments": 21441, + "\u0120passwords": 21442, + "\u0120gospel": 21443, + "\u0120Gin": 21444, + "\u0120tenants": 21445, + "olith": 21446, + "Six": 21447, + "Previous": 21448, + "\u0120Ages": 21449, + "\u0120Darwin": 21450, + "\u0120blat": 21451, + "\u0120empathy": 21452, + "smith": 21453, + "bag": 21454, + "\u0120Echo": 21455, + "\u0120Camb": 21456, + "\u0120Madd": 21457, + "\u0120Boo": 21458, + "\u0120rede": 21459, + "\u0120Burning": 21460, + "\u0120smoothly": 21461, + "\u0120Adrian": 21462, + "\u0120Vampire": 21463, + "\u0120Monsters": 21464, + "steam": 21465, + "Style": 21466, + "Ma": 21467, + "rea": 21468, + "\u0120Dwar": 21469, + "alyst": 21470, + "ursor": 21471, + "\u0120elimination": 21472, + "\u0120crypto": 21473, + "cht": 21474, + "\u0120Eternal": 21475, + "\u00e2\u0122\u00a6]": 21476, + "\u0120Sorce": 21477, + "Ill": 21478, + "NER": 21479, + "\u0120uh": 21480, + "Conclusion": 21481, + "wage": 21482, + "\u0120respir": 21483, + "\u0120reminis": 21484, + "hetical": 21485, + "\u0120gy": 21486, + "\u0120utilized": 21487, + "icidal": 21488, + "\u01201900": 21489, + "\u0120hunters": 21490, + "\u0120Swan": 21491, + "\u0120React": 21492, + "\u0120visitor": 21493, + "\u0120Thanksgiving": 21494, + "308": 21495, + "Posts": 21496, + "\u0120hips": 21497, + "1997": 21498, + "omers": 21499, + "\u0120knocking": 21500, + "\u0120Vehicle": 21501, + "\u0120til": 21502, + "\u0120138": 21503, + "\u0120mi": 21504, + "\u0120Investigation": 21505, + "\u0120Kenya": 21506, + "\u0120casino": 21507, + "\u0120motives": 21508, + "\u0120regain": 21509, + "rex": 21510, + "\u0120weekends": 21511, + "\u0120stabbed": 21512, + "boro": 21513, + "\u0120exploited": 21514, + "\u0120HAVE": 21515, + "\u0120Television": 21516, + "cock": 21517, + "\u0120preparations": 21518, + "\u0120endeav": 21519, + "\u0120Remote": 21520, + "\u0120Maker": 21521, + "\u0120Produ": 21522, + "\u0120Evan": 21523, + "\u0120informational": 21524, + "\u0120Louisville": 21525, + "154": 21526, + "\u0120Dreams": 21527, + "\u0120plots": 21528, + "\u0120Runner": 21529, + "\u0120hurting": 21530, + "\u0120academy": 21531, + "\u0120Montgomery": 21532, + "nm": 21533, + "\u0120Lanc": 21534, + "\u0120Alz": 21535, + "210": 21536, + "elong": 21537, + "\u0120retailer": 21538, + "\u0120arising": 21539, + "\u0120rebellion": 21540, + "\u0120blonde": 21541, + "played": 21542, + "\u0120instrumental": 21543, + "Cross": 21544, + "\u0120retention": 21545, + "\u0120therapeutic": 21546, + "\u0120seas": 21547, + "\u0120infantry": 21548, + "\u0120Clint": 21549, + "\u0120prompting": 21550, + "\u0120bitch": 21551, + "\u0120stems": 21552, + "\u0120Kra": 21553, + "\u0120thesis": 21554, + "\u0120Bog": 21555, + "rued": 21556, + "\u0120kings": 21557, + "\u0120clay": 21558, + "ificent": 21559, + "\u0120YES": 21560, + "\u0120Thing": 21561, + "\u0120Cubs": 21562, + "veyard": 21563, + "elsh": 21564, + "inarily": 21565, + "\u0120Ey": 21566, + "\u0120Rolling": 21567, + "\u0120evolving": 21568, + "India": 21569, + "\u0120recognizes": 21570, + "\u0120graduation": 21571, + "isers": 21572, + "\u0120fertility": 21573, + "\u0120Milan": 21574, + "Command": 21575, + "\u0120boxing": 21576, + "\u01201943": 21577, + "\u0120gluten": 21578, + "\u0120Emir": 21579, + "\u0120idol": 21580, + "\u0120conceived": 21581, + "\u0120Creation": 21582, + "Merit": 21583, + "uddy": 21584, + "ussions": 21585, + "\u0120Lieutenant": 21586, + "ietal": 21587, + "\u0120unchanged": 21588, + "\u0120Scale": 21589, + "\u0120Crimea": 21590, + "balls": 21591, + "atorial": 21592, + "\u0120depths": 21593, + "\u0120empirical": 21594, + "\u0120transm": 21595, + "\u0120unsafe": 21596, + "missible": 21597, + "comfort": 21598, + "156": 21599, + "\u0120mechanic": 21600, + "002": 21601, + "lins": 21602, + "\u0120smoked": 21603, + "Pos": 21604, + "\u0120slowing": 21605, + "\u0120lav": 21606, + "Texas": 21607, + "\u0120cheating": 21608, + "\u0120Metropolitan": 21609, + "ethyl": 21610, + "\u0120discovering": 21611, + "asse": 21612, + "\u0120pencil": 21613, + "\u0120Pyongyang": 21614, + "\u0120closet": 21615, + "\u0120Sheet": 21616, + "\u0120Entry": 21617, + "oustic": 21618, + "\u0120myst": 21619, + "erate": 21620, + "ariat": 21621, + "\u0120minerals": 21622, + "\u0120musician": 21623, + "\u0120Pul": 21624, + "\u0120Maz": 21625, + "249": 21626, + "\u0120permissions": 21627, + "\u0120iv": 21628, + "enary": 21629, + "ickers": 21630, + "\u0120Bing": 21631, + "hea": 21632, + "enable": 21633, + "\u0120griev": 21634, + "\u0120asserted": 21635, + "\u0120Colonel": 21636, + "\u0120affidav": 21637, + "wo": 21638, + "\u0120seated": 21639, + "\u0120Ride": 21640, + "\u0120paintings": 21641, + "\u0120Pix": 21642, + "\u0120137": 21643, + "ishi": 21644, + "umbai": 21645, + "gotten": 21646, + "\u0120Earl": 21647, + "\u0120inning": 21648, + "\u0120census": 21649, + "\u0120travelled": 21650, + "\u0120Consult": 21651, + "185": 21652, + "bind": 21653, + "\u0120simplicity": 21654, + "\u0120overlooked": 21655, + "\u0120Helpful": 21656, + "\u0120monkey": 21657, + "\u0120overwhelmingly": 21658, + "Blood": 21659, + "\u0120Flint": 21660, + "\u0120Jama": 21661, + "\u0120Present": 21662, + "\u0120Rage": 21663, + "\u0120TA": 21664, + "ptive": 21665, + "\u0120turnout": 21666, + "wald": 21667, + "\u0120Dolphins": 21668, + "\u0120VPN": 21669, + "\u0120onion": 21670, + "\u0120crafting": 21671, + "mma": 21672, + "\u0120Mercury": 21673, + "\u0120arrange": 21674, + "\u0120alerts": 21675, + "\u0120OT": 21676, + "zbollah": 21677, + "\u0120gases": 21678, + "\u0120Richardson": 21679, + "sal": 21680, + "lar": 21681, + "\u0120frost": 21682, + "\u0120lowering": 21683, + "\u0120acclaim": 21684, + "\u0120startups": 21685, + "\u0120Gain": 21686, + "essment": 21687, + "\u0120guardian": 21688, + "\u00e4\u00ba\u00ba": 21689, + "\u0120Pie": 21690, + "\u0120Links": 21691, + "\u0120merits": 21692, + "\u0120awake": 21693, + "\u0120parental": 21694, + "\u0120exceeds": 21695, + "\u0120idle": 21696, + "\u0120Pilot": 21697, + "\u0120eBay": 21698, + "\u0120Accept": 21699, + "ipeg": 21700, + "Cam": 21701, + "\u0120Kot": 21702, + "\u0120traders": 21703, + "olitics": 21704, + "unker": 21705, + "\u0120Pale": 21706, + "osi": 21707, + "anmar": 21708, + "\u01201947": 21709, + "\u0120Fell": 21710, + "estial": 21711, + "itating": 21712, + "GF": 21713, + "\u0120Sr": 21714, + "ifted": 21715, + "\u0120connector": 21716, + "\u0120Bone": 21717, + "illes": 21718, + "260": 21719, + "hma": 21720, + "\u0120overlap": 21721, + "\u0120GitHub": 21722, + "\u0120cleaner": 21723, + "\u0120Baptist": 21724, + "\u0120WAS": 21725, + "\u0120lungs": 21726, + "\u00d1\u0123": 21727, + "\u0120BUT": 21728, + "\u0120cite": 21729, + "\u0120pitched": 21730, + "reatment": 21731, + "\u0120trophies": 21732, + "\u0120Nu": 21733, + "386": 21734, + "\u0120Pride": 21735, + "\u0120attendees": 21736, + "[]": 21737, + "179": 21738, + "\u0120spatial": 21739, + "\u0120prizes": 21740, + "\u0120Religion": 21741, + "\u0120showcase": 21742, + "\u0120Category": 21743, + "vidia": 21744, + "Target": 21745, + "Property": 21746, + "?,": 21747, + "\u0120fusion": 21748, + "pie": 21749, + "\u0120UCLA": 21750, + "\u0120soundtrack": 21751, + "\u0120princess": 21752, + "\u0120Caval": 21753, + "should": 21754, + "\u0120limbs": 21755, + "Background": 21756, + "\u0120lonely": 21757, + "\u0120cores": 21758, + "\u0120Tail": 21759, + "sheet": 21760, + "\u0120132": 21761, + "Ra": 21762, + "\u00e3\u0124\u00ab": 21763, + "\u0120Bolt": 21764, + "\u0120booked": 21765, + "\u0120administer": 21766, + "\u0120equals": 21767, + "wy": 21768, + "\u0120observing": 21769, + "\u0120Baron": 21770, + "\u0120Adobe": 21771, + "\u0120virgin": 21772, + "\u0120Socialist": 21773, + "Move": 21774, + "ghazi": 21775, + "\u0120Linda": 21776, + "212": 21777, + "\u0120brewing": 21778, + "\u0120merchants": 21779, + "burse": 21780, + "\u0120divor": 21781, + "\u0120metals": 21782, + "\u0120Ner": 21783, + "\u0120sums": 21784, + "\u0120Enemy": 21785, + "\u0120envision": 21786, + "\u0120granting": 21787, + "\u0120Honey": 21788, + "\u0120Skyrim": 21789, + "\u0120socio": 21790, + "graded": 21791, + "\u0120selective": 21792, + "WASHINGTON": 21793, + "\u01201948": 21794, + "\u0120Sirius": 21795, + "\u0120Gross": 21796, + "activity": 21797, + "\u0120Ivan": 21798, + "\u0120furious": 21799, + "BSD": 21800, + "\u0120Previous": 21801, + "\u0120responsive": 21802, + "\u0120charitable": 21803, + "\u0120leaning": 21804, + "\u0120Pew": 21805, + "\u0120violates": 21806, + "\\\\\\\\\\\\\\\\": 21807, + "\u0120Coming": 21808, + "wire": 21809, + "\u0120poet": 21810, + "\u0120resolutions": 21811, + "command": 21812, + "\u0120Portuguese": 21813, + "\u0120nickname": 21814, + "\u0120deaf": 21815, + "February": 21816, + "\u0120recognise": 21817, + "\u0120entirety": 21818, + "\u0120seasonal": 21819, + "placed": 21820, + "\u0120Telegraph": 21821, + "\u0120microphone": 21822, + "ouring": 21823, + "\u0120grains": 21824, + "\u0120governed": 21825, + "\u0120postp": 21826, + "\u0120Waters": 21827, + "inement": 21828, + "\u0120undocumented": 21829, + "\u0120Comcast": 21830, + "\u0120fox": 21831, + "\u0120assaults": 21832, + "reon": 21833, + "many": 21834, + "\u0120Jenkins": 21835, + "\u0120Anyway": 21836, + "\u0120assessments": 21837, + "\u0120downs": 21838, + "\u0120Mouse": 21839, + "\u0120superb": 21840, + "kt": 21841, + "\u0120Dow": 21842, + "\u0120taxation": 21843, + "401": 21844, + "\u0120smiles": 21845, + "\u0120undertaken": 21846, + "\u0120exh": 21847, + "\u0120enthusiastic": 21848, + "\u0120twent": 21849, + "\u0120governmental": 21850, + "\u0120autonomy": 21851, + "\u0120Technologies": 21852, + "\u0120Chain": 21853, + "\u0120prevalent": 21854, + "fb": 21855, + "\u0120nicotine": 21856, + "ogram": 21857, + "job": 21858, + "\u0120awaiting": 21859, + "\u0120Menu": 21860, + "\u0120deputies": 21861, + "kov": 21862, + "ishops": 21863, + "Button": 21864, + "\u0120Shanghai": 21865, + "\u0120diesel": 21866, + "\u0120Duck": 21867, + "Ryan": 21868, + "\u0120PCs": 21869, + "NF": 21870, + "jury": 21871, + "ente": 21872, + "\u0120inaccurate": 21873, + "eddy": 21874, + "Whatever": 21875, + "\u0120showc": 21876, + "\u0120Nad": 21877, + "odus": 21878, + "etr": 21879, + "\u0120plaintiffs": 21880, + "\u0120WOR": 21881, + "\u0120Assange": 21882, + "\u0120privat": 21883, + "\u0120premiums": 21884, + "\u0120tam": 21885, + "URL": 21886, + "\u0120elites": 21887, + "\u0120Ranger": 21888, + "ottenham": 21889, + "\u0120Hoff": 21890, + "\u0120Athens": 21891, + "\u0120definite": 21892, + "\u0120sighed": 21893, + "\u0120evenly": 21894, + "211": 21895, + "\u0120Amber": 21896, + "akia": 21897, + "\u0120mailing": 21898, + "\u0120crashing": 21899, + "\u0120Confederate": 21900, + "rugged": 21901, + "Wal": 21902, + "\u0120Depths": 21903, + "\u0120juvenile": 21904, + "\u0120reactor": 21905, + "Introduction": 21906, + "\u0120Deluxe": 21907, + "1995": 21908, + "\u0120Sanchez": 21909, + "\u0120Mead": 21910, + "ivable": 21911, + ":-": 21912, + "\u0120Planning": 21913, + "\u0120Trap": 21914, + "quin": 21915, + "\u0120Protect": 21916, + "vered": 21917, + "Information": 21918, + "\u0120kidney": 21919, + "innamon": 21920, + "las": 21921, + "\u0120policing": 21922, + "\u0120tolerate": 21923, + "\u0120Qi": 21924, + "\u0120biased": 21925, + "Fort": 21926, + "\u0120Ki": 21927, + "save": 21928, + "\u0120privileged": 21929, + "\u0120beasts": 21930, + "\u0120Glas": 21931, + "\u0120Cinem": 21932, + "\u0120comeback": 21933, + "Sunday": 21934, + "\u0120extinction": 21935, + "hops": 21936, + "\u0120transmit": 21937, + "\u0120doubles": 21938, + "\u0120Flat": 21939, + "167": 21940, + "\u0120disputed": 21941, + "\u0120injustice": 21942, + "foo": 21943, + "Vict": 21944, + "roleum": 21945, + "\u0120Julie": 21946, + "Context": 21947, + "\u0120Rarity": 21948, + "issue": 21949, + "Component": 21950, + "\u0120counseling": 21951, + "anne": 21952, + "dark": 21953, + "\u0120objections": 21954, + "uilt": 21955, + "\u0120gast": 21956, + "\u0120plac": 21957, + "\u0120unused": 21958, + "\u00e3\u0125\u0129": 21959, + "\u0120Trial": 21960, + "\u0120Jas": 21961, + "hedral": 21962, + "obb": 21963, + "\u0120temporal": 21964, + "\u0120PRO": 21965, + "\u0120NW": 21966, + "\u0120Anniversary": 21967, + "Large": 21968, + "\u0120therm": 21969, + "\u0120david": 21970, + "\u0120systemic": 21971, + "\u0120Shir": 21972, + "mut": 21973, + "\u0120Nept": 21974, + "address": 21975, + "\u0120scanning": 21976, + "\u0120understandable": 21977, + "\u0120canvas": 21978, + "Cat": 21979, + "\u0120Zoo": 21980, + "\u0120angels": 21981, + "LO": 21982, + "\u0120Statement": 21983, + "\u0120Sig": 21984, + "ovable": 21985, + "\u0120Away": 21986, + "sharing": 21987, + "ocrats": 21988, + "stated": 21989, + "\u0120weighing": 21990, + "Nor": 21991, + "wild": 21992, + "Bey": 21993, + "\u0120astonishing": 21994, + "\u0120Reynolds": 21995, + "\u0120opener": 21996, + "\u0120trainer": 21997, + "\u0120surgical": 21998, + "pn": 21999, + "\u0120adjusting": 22000, + "wheel": 22001, + "\u0120frown": 22002, + "ervative": 22003, + "\u0120suspend": 22004, + "Within": 22005, + "tein": 22006, + "\u0120obstacle": 22007, + "\u0120liberties": 22008, + "ymes": 22009, + "\u0120uranium": 22010, + "ansom": 22011, + "anol": 22012, + "uba": 22013, + "\u0120Loss": 22014, + "\u0120arous": 22015, + "\u0120Henderson": 22016, + "Wow": 22017, + "spl": 22018, + "cur": 22019, + "\u0120\u00c2\u0143": 22020, + "\u0120theirs": 22021, + "Damage": 22022, + "\u0120downloading": 22023, + "\u0120discern": 22024, + "\u0120Sto": 22025, + "\u0120Fla": 22026, + "\u0120hath": 22027, + "\u0120Aj": 22028, + "\u0120unpleasant": 22029, + "European": 22030, + "expensive": 22031, + "\u0120screenshot": 22032, + "\u0120UV": 22033, + "\u0120allied": 22034, + "\u0120Persian": 22035, + "\u0120monopoly": 22036, + "\u0120atom": 22037, + "\u0120Redskins": 22038, + "\"><": 22039, + "\u0120cancell": 22040, + "\u0120cinema": 22041, + "131": 22042, + "fair": 22043, + "\u0120Alfred": 22044, + "\u0120duck": 22045, + "args": 22046, + "223": 22047, + "\u0120ISI": 22048, + "\u0120signaling": 22049, + "inar": 22050, + "\u0120laughs": 22051, + "\u0120forwards": 22052, + "\u0120reckless": 22053, + "\u0120listeners": 22054, + "ativity": 22055, + "\u0120vastly": 22056, + "nant": 22057, + "Less": 22058, + "\u0120Hunting": 22059, + "\u0120Scientific": 22060, + "ITED": 22061, + "\u0120knight": 22062, + "\u0120HTC": 22063, + "usa": 22064, + "tmp": 22065, + "\u0120rude": 22066, + "\u0120Legendary": 22067, + "\u0120arises": 22068, + "Bad": 22069, + "\u0120Claim": 22070, + "peg": 22071, + "\u0120realities": 22072, + "Think": 22073, + "\u0120\u00c2\u00b0": 22074, + "\u0120rode": 22075, + "\u0120strive": 22076, + "\u0120anecd": 22077, + "\u0120shorts": 22078, + "\u0120hypothes": 22079, + "\u0120coordinated": 22080, + "\u0120Gandhi": 22081, + "\u0120FPS": 22082, + "RED": 22083, + "\u0120susceptible": 22084, + "\u0120shrink": 22085, + "\u0120Chart": 22086, + "Help": 22087, + "\u0120ion": 22088, + "deep": 22089, + "ribes": 22090, + "\u0120Kai": 22091, + "\u0120Customer": 22092, + "Summary": 22093, + "\u0120cough": 22094, + "wife": 22095, + "\u0120lend": 22096, + "\u0120positioning": 22097, + "\u0120lottery": 22098, + "\u0120Canyon": 22099, + "\u0120fade": 22100, + "\u0120bronze": 22101, + "\u0120Kenny": 22102, + "\u0120boasts": 22103, + "\u0120Enhanced": 22104, + "record": 22105, + "\u0120emergence": 22106, + "\u0120akin": 22107, + "\u0120Bert": 22108, + "itous": 22109, + "\u00e2\u0138\u0133": 22110, + "\u0120stip": 22111, + "\u0120exchanged": 22112, + "omore": 22113, + "alsh": 22114, + "\u0120reservoir": 22115, + "\u0120standpoint": 22116, + "WM": 22117, + "\u0120initiate": 22118, + "\u0120decay": 22119, + "\u0120brewery": 22120, + "\u0120terribly": 22121, + "\u0120mortal": 22122, + "levard": 22123, + "\u0120revis": 22124, + "NI": 22125, + "elo": 22126, + "\u0120confess": 22127, + "\u0120MSNBC": 22128, + "\u0120submissions": 22129, + "Controller": 22130, + "\u0120202": 22131, + "\u0120Ruth": 22132, + "});": 22133, + "\u0120Azure": 22134, + "\u0120.\"": 22135, + "206": 22136, + "\u0120Marketing": 22137, + "\u0120laund": 22138, + "iencies": 22139, + "\u0120renowned": 22140, + "\u0120Trou": 22141, + "\u0120NGO": 22142, + "blems": 22143, + "\u0120terrified": 22144, + "\u0120warns": 22145, + "\u0120pert": 22146, + "\u0120unsure": 22147, + "480": 22148, + "alez": 22149, + "ultz": 22150, + "\u0120Outside": 22151, + "\u0120styl": 22152, + "\u0120Underground": 22153, + "\u0120panc": 22154, + "\u0120dictionary": 22155, + "\u0120foe": 22156, + "riminal": 22157, + "\u0120Norwegian": 22158, + "\u0120jailed": 22159, + "\u0120maternal": 22160, + "\u00c3\u00a9e": 22161, + "\u0120Lucy": 22162, + "cop": 22163, + "Cho": 22164, + "\u0120unsigned": 22165, + "\u0120Zelda": 22166, + "\u0120Insider": 22167, + "\u0120Continued": 22168, + "\u0120133": 22169, + "\u0120Naruto": 22170, + "\u0120Majority": 22171, + "169": 22172, + "\u0120Wo": 22173, + "\u00e3\u0124\u0135": 22174, + "\u0120pastor": 22175, + "\u0120informal": 22176, + "\u00d0\u00bd": 22177, + "anthrop": 22178, + "join": 22179, + "\u00e3\u0123\u0139": 22180, + "itational": 22181, + "NP": 22182, + "\u0120Writing": 22183, + "fn": 22184, + "\u0120Bever": 22185, + "195": 22186, + "\u0120yelling": 22187, + "\u0120drastically": 22188, + "\u0120eject": 22189, + "\u0120neut": 22190, + "\u0120thrive": 22191, + "\u0120Frequ": 22192, + "oux": 22193, + "\u0120possesses": 22194, + "\u0120Senators": 22195, + "\u0120DES": 22196, + "\u0120Shakespeare": 22197, + "\u0120Franco": 22198, + "\u0120LB": 22199, + "uchi": 22200, + "\u0120incarn": 22201, + "\u0120founders": 22202, + "Function": 22203, + "\u0120brightness": 22204, + "\u0120BT": 22205, + "\u0120whale": 22206, + "\u0120Theater": 22207, + "mass": 22208, + "\u0120Doll": 22209, + "Something": 22210, + "\u0120echoed": 22211, + "\u0120Hex": 22212, + "crit": 22213, + "afia": 22214, + "\u0120goddess": 22215, + "\u0120eleven": 22216, + "\u0120Preview": 22217, + "\u0120Aurora": 22218, + "\u0120401": 22219, + "ulsive": 22220, + "\u0120Logan": 22221, + "inburgh": 22222, + "\u0120Centers": 22223, + "\u0120ONLY": 22224, + "\u0120Aid": 22225, + "\u0120paradox": 22226, + "\u0120hurd": 22227, + "\u0120LC": 22228, + "Due": 22229, + "court": 22230, + "\u0120offended": 22231, + "\u0120evaluating": 22232, + "\u0120Matthews": 22233, + "\u0120tomb": 22234, + "\u0120payroll": 22235, + "\u0120extraction": 22236, + "\u0120Hands": 22237, + "ifi": 22238, + "\u0120supernatural": 22239, + "\u0120COMM": 22240, + "]=": 22241, + "dogs": 22242, + "\u0120512": 22243, + "\u0120Meeting": 22244, + "Richard": 22245, + "\u0120Maximum": 22246, + "\u0120ideals": 22247, + "Things": 22248, + "mand": 22249, + "\u0120Regardless": 22250, + "\u0120humili": 22251, + "buffer": 22252, + "Little": 22253, + "\u0120Dani": 22254, + "\u0120Nak": 22255, + "\u0120liberation": 22256, + "\u0120Abe": 22257, + "\u0120OL": 22258, + "\u0120stuffed": 22259, + "aca": 22260, + "inda": 22261, + "raphic": 22262, + "\u0120mosqu": 22263, + "\u0120campaigning": 22264, + "\u0120occupy": 22265, + "Squ": 22266, + "rina": 22267, + "\u0120Wel": 22268, + "\u0120VS": 22269, + "\u0120physic": 22270, + "\u0120puls": 22271, + "rint": 22272, + "oaded": 22273, + "ETF": 22274, + "\u0120Archives": 22275, + "\u0120venues": 22276, + "hner": 22277, + "\u0120Turbo": 22278, + "\u0120lust": 22279, + "\u0120appealed": 22280, + "quez": 22281, + "ilib": 22282, + "\u0120Timothy": 22283, + "\u0120omn": 22284, + "dro": 22285, + "\u0120obsession": 22286, + "\u0120Savage": 22287, + "1996": 22288, + "Global": 22289, + "Jes": 22290, + "214": 22291, + "\u0120sliding": 22292, + "\u0120disappro": 22293, + "\u0120Magical": 22294, + "\u0120voluntarily": 22295, + "gb": 22296, + "aney": 22297, + "\u0120prophet": 22298, + "\u0120Rein": 22299, + "\u0120Julia": 22300, + "\u0120Worth": 22301, + "aurus": 22302, + "\u0120bounds": 22303, + "ieu": 22304, + ")))": 22305, + "\u0120crore": 22306, + "\u0120Citizen": 22307, + "Sky": 22308, + "\u0120columnist": 22309, + "\u0120seekers": 22310, + "ondo": 22311, + "ISA": 22312, + "\u0120Length": 22313, + "\u0120nostalg": 22314, + "\u0120newcom": 22315, + "\u0120detrim": 22316, + "entric": 22317, + "375": 22318, + "\u0120GE": 22319, + "\u0120autop": 22320, + "\u0120academics": 22321, + "AppData": 22322, + "\u0120Shen": 22323, + "\u0120idiot": 22324, + "\u0120Transit": 22325, + "\u0120teaspoon": 22326, + "Wil": 22327, + "KO": 22328, + "\u0120Comedy": 22329, + ">,": 22330, + "\u0120populated": 22331, + "WD": 22332, + "\u0120pigs": 22333, + "\u0120Oculus": 22334, + "\u0120sympathetic": 22335, + "\u0120marathon": 22336, + "198": 22337, + "\u0120seizure": 22338, + "sided": 22339, + "\u0120dop": 22340, + "irtual": 22341, + "Land": 22342, + "\u0120Floor": 22343, + "osaurs": 22344, + "...]": 22345, + "\u0120los": 22346, + "\u0120subsidiary": 22347, + "EY": 22348, + "\u0120Parts": 22349, + "\u0120Stef": 22350, + "\u0120Judiciary": 22351, + "\u0120134": 22352, + "\u0120mirrors": 22353, + "\u0120ket": 22354, + "times": 22355, + "\u0120neurolog": 22356, + "\u0120cav": 22357, + "\u0120Guest": 22358, + "\u0120tumor": 22359, + "scill": 22360, + "\u0120Lloyd": 22361, + "Est": 22362, + "\u0120clearer": 22363, + "\u0120stereotypes": 22364, + "\u0120dur": 22365, + "nothing": 22366, + "Reddit": 22367, + "\u0120negotiated": 22368, + "------------------------": 22369, + "235": 22370, + "\u0120flown": 22371, + "\u0120Seoul": 22372, + "\u0120Resident": 22373, + "\u0120SCH": 22374, + "\u0120disappearance": 22375, + "\u0120Vince": 22376, + "grown": 22377, + "\u0120grabs": 22378, + "ril": 22379, + "\u0120Infinite": 22380, + "\u0120Twenty": 22381, + "\u0120pedestrian": 22382, + "\u0120jersey": 22383, + "\u0120Fur": 22384, + "\u0120Infinity": 22385, + "\u0120Elliott": 22386, + "\u0120mentor": 22387, + "\u0120morally": 22388, + "\u0120obey": 22389, + "secure": 22390, + "iffe": 22391, + "\u0120antibiotics": 22392, + "angled": 22393, + "\u0120Freeman": 22394, + "\u0120Introduction": 22395, + "Jun": 22396, + "\u0120marsh": 22397, + "icans": 22398, + "\u0120EVENTS": 22399, + "ochond": 22400, + "Wall": 22401, + "iculty": 22402, + "\u0120misdemeanor": 22403, + "\u0120ly": 22404, + "Thomas": 22405, + "\u0120Resolution": 22406, + "\u0120animations": 22407, + "\u0120Dry": 22408, + "\u0120intercourse": 22409, + "\u0120Newcastle": 22410, + "\u0120Hog": 22411, + "\u0120Equipment": 22412, + "177": 22413, + "\u0120territorial": 22414, + "\u0120archives": 22415, + "203": 22416, + "Filter": 22417, + "\u0120Munich": 22418, + "\u0120commanded": 22419, + "\u0120Wand": 22420, + "\u0120pitches": 22421, + "\u0120Croat": 22422, + "\u0120ratios": 22423, + "\u0120Mits": 22424, + "\u0120accumulated": 22425, + "\u0120Specifically": 22426, + "\u0120gentleman": 22427, + "acerb": 22428, + "\u0120penn": 22429, + "\u0120aka": 22430, + "\u0120Fuk": 22431, + "\u0120intervene": 22432, + "\u0120Refuge": 22433, + "\u0120Alzheimer": 22434, + "\u0120succession": 22435, + "ohan": 22436, + "does": 22437, + "Lord": 22438, + "\u0120separat": 22439, + "\u0120correspondence": 22440, + "\u0120shiny": 22441, + "Prior": 22442, + "\u0120sulf": 22443, + "\u0120miserable": 22444, + "\u0120dedication": 22445, + "().": 22446, + "\u0120specialists": 22447, + "\u0120defects": 22448, + "\u0120Cult": 22449, + "\u0120Xia": 22450, + "\u0120jeopard": 22451, + "\u0120Ore": 22452, + "Ability": 22453, + "\u0120lear": 22454, + "\u0120ambitions": 22455, + "\u0120BMI": 22456, + "\u0120Arabs": 22457, + "\u01201942": 22458, + "\u0120preservation": 22459, + "ificate": 22460, + "\u0120ashamed": 22461, + "loss": 22462, + "\u0120Restaur": 22463, + "\u0120resemble": 22464, + "\u0120enrich": 22465, + "\u0120KN": 22466, + "\u0120Clan": 22467, + "float": 22468, + "\u0120playable": 22469, + "ITT": 22470, + "\u0120harmony": 22471, + "arrison": 22472, + "\u0120Weinstein": 22473, + "were": 22474, + "\u0120poisoning": 22475, + "\u0120Comput": 22476, + "\u0120WordPress": 22477, + "major": 22478, + "\u0120Valve": 22479, + "Fan": 22480, + "\u0120Throw": 22481, + "\u0120Romans": 22482, + "\u0120Depression": 22483, + "ados": 22484, + "\u0120tortured": 22485, + "\u0120balancing": 22486, + "bottom": 22487, + "\u0120acquiring": 22488, + "\u0120Monte": 22489, + "ardi": 22490, + "\u0120aura": 22491, + "\u0120##": 22492, + "\u0120Standing": 22493, + "\u0120Atlas": 22494, + "CF": 22495, + "\u0120intrins": 22496, + "\u0120Benghazi": 22497, + "\u0120camping": 22498, + "\u0120tapped": 22499, + "blade": 22500, + "strous": 22501, + "\u0120Rabb": 22502, + "\u0120Written": 22503, + "tip": 22504, + "\u0120Neigh": 22505, + "sterdam": 22506, + "\u0120Allow": 22507, + "\u0120Healing": 22508, + "\u0120Rhod": 22509, + "num": 22510, + "\u0120caffeine": 22511, + "\u0120Percent": 22512, + "\u0120boo": 22513, + "\u0120apples": 22514, + "305": 22515, + "\u0120welcoming": 22516, + "\u0120applaud": 22517, + "\u0120austerity": 22518, + "\u00c2\u00b1": 22519, + "\u0120Reality": 22520, + "efe": 22521, + "\u00e5\u00ae": 22522, + "\u0120sucks": 22523, + "\u0120tabs": 22524, + "\u0120PayPal": 22525, + "\u0120backpack": 22526, + "\u0120gifted": 22527, + "abulary": 22528, + "\u0120Scout": 22529, + "irteen": 22530, + "\u0120chin": 22531, + "\u0120omitted": 22532, + "\u0120negatively": 22533, + "\u0120accessing": 22534, + "\u0120Earn": 22535, + "\u0120ambulance": 22536, + "\u0120headphones": 22537, + "\u0120205": 22538, + "\u0120Refresh": 22539, + "president": 22540, + "\u0120Kitchen": 22541, + "\u0120Entered": 22542, + "\u0120Snyder": 22543, + "005": 22544, + "omical": 22545, + "\u0120borrowed": 22546, + "\u0120Nem": 22547, + "\u0120aviation": 22548, + "\u0120stall": 22549, + "rimination": 22550, + "\u0120uniforms": 22551, + "itime": 22552, + "\u0120Simmons": 22553, + "energy": 22554, + "ablished": 22555, + "yy": 22556, + "qualified": 22557, + "\u0120rallies": 22558, + "\u0120Stuart": 22559, + "flight": 22560, + "\u0120gangs": 22561, + "rag": 22562, + "\u0120vault": 22563, + "lux": 22564, + "\u0120Compar": 22565, + "\u0120designation": 22566, + "209": 22567, + "\u0120Jos": 22568, + "dollar": 22569, + "zero": 22570, + "\u0120wells": 22571, + "303": 22572, + "\u0120constituents": 22573, + "\u0120heck": 22574, + "\u0120cows": 22575, + "\u0120commanders": 22576, + "\u0120differential": 22577, + "\u0120Catherine": 22578, + "299": 22579, + "\u0120valve": 22580, + "\u0120brace": 22581, + "\u0120perspectives": 22582, + "cert": 22583, + "fact": 22584, + "icularly": 22585, + "\u0120McN": 22586, + "planes": 22587, + "\u0120intric": 22588, + "\u0120peas": 22589, + "ovan": 22590, + "\u0120tossed": 22591, + "retch": 22592, + "\u0120Lopez": 22593, + "\u0120unfamiliar": 22594, + "death": 22595, + "\u0120Apart": 22596, + "\u0120Chang": 22597, + "\u0120relieved": 22598, + "rophe": 22599, + "\u0120airports": 22600, + "\u0120freak": 22601, + "util": 22602, + "Mill": 22603, + "\u0120Chin": 22604, + "\u0120Owen": 22605, + "male": 22606, + "\u0120Broken": 22607, + "\u0120Winds": 22608, + "rob": 22609, + "rising": 22610, + "\u0120firefighters": 22611, + "\u0120authoritarian": 22612, + "\u0120148": 22613, + "Bitcoin": 22614, + "external": 22615, + "\u0120browsers": 22616, + "ichever": 22617, + "orian": 22618, + "\u0120unb": 22619, + "\u0120poke": 22620, + "\u0120Zot": 22621, + "Mid": 22622, + "\u0120Popular": 22623, + "\u0120covert": 22624, + "\u0120contributes": 22625, + "\u0120650": 22626, + "\u0120contention": 22627, + "Gate": 22628, + "\u0120consoles": 22629, + "\u0120chromos": 22630, + "\u0120IX": 22631, + "\u0120visually": 22632, + "\u0120Eisen": 22633, + "\u0120jewelry": 22634, + "\u0120delegation": 22635, + "\u0120accelerate": 22636, + "\u0120Riley": 22637, + "\u0120slope": 22638, + "\u0120indoor": 22639, + "itially": 22640, + "\u0120hugely": 22641, + "\u0120tunnels": 22642, + "\u0120fined": 22643, + "\u0120directive": 22644, + "\u0120forehead": 22645, + "ustomed": 22646, + "\u0120skate": 22647, + "Music": 22648, + "gas": 22649, + "\u0120recognizing": 22650, + "ambo": 22651, + "\u0120overweight": 22652, + "\u0120Grade": 22653, + "\u00d9\u012c": 22654, + "\u0120sounding": 22655, + "\u0120locking": 22656, + "\u0120REM": 22657, + "Store": 22658, + "\u0120excav": 22659, + "\u0120Likewise": 22660, + "\u0120Lights": 22661, + "\u0120elbow": 22662, + "\u0120Supply": 22663, + "wic": 22664, + "\u0120handsome": 22665, + "1994": 22666, + "Coll": 22667, + "\u0120adequately": 22668, + "\u0120Associate": 22669, + "\u0120strips": 22670, + "\u0120crackdown": 22671, + "\u0120marvel": 22672, + "\u0120Kun": 22673, + "\u0120passages": 22674, + "@@@@": 22675, + "\u0120Tall": 22676, + "\u0120thoughtful": 22677, + "namese": 22678, + "\u0120prostitution": 22679, + "business": 22680, + "\u0120ballistic": 22681, + "personal": 22682, + "cig": 22683, + "izational": 22684, + "Round": 22685, + "\u0120\u00c2\u0142\u0120\u00c2\u0142\u0120\u00c2\u0142\u0120\u00c2\u0142": 22686, + "\u0120Coleman": 22687, + "\u0120admitting": 22688, + "\u0120Plug": 22689, + "\u0120bitcoins": 22690, + "\u0120Suz": 22691, + "\u0120fairness": 22692, + "\u0120supplier": 22693, + "\u0120catastrophic": 22694, + "\u0120Helen": 22695, + "oqu": 22696, + "Marc": 22697, + "\u0120Articles": 22698, + "gie": 22699, + "\u0120endangered": 22700, + "\u0120destiny": 22701, + "\u0120Volt": 22702, + "olia": 22703, + "axis": 22704, + "\u0120cheat": 22705, + "\u0120unified": 22706, + "ICO": 22707, + "quote": 22708, + "302": 22709, + "\u0120Sed": 22710, + "\u0120suppression": 22711, + "\u0120analyzing": 22712, + "\u0120squat": 22713, + "\u0120figuring": 22714, + "\u0120coordinates": 22715, + "\u0120chunks": 22716, + "\u01201946": 22717, + "\u0120subp": 22718, + "\u0120wiki": 22719, + "\u0120Forbes": 22720, + "\u0120Jupiter": 22721, + "\u0120Erik": 22722, + "imer": 22723, + "\u0120Commercial": 22724, + "\\)": 22725, + "\u0120legitimacy": 22726, + "\u0120dental": 22727, + "\u0120Mean": 22728, + "\u0120deficits": 22729, + "550": 22730, + "Originally": 22731, + "\u0120Horror": 22732, + "\u0120contamination": 22733, + "llah": 22734, + "\u0120confisc": 22735, + "\u0120Clare": 22736, + "TB": 22737, + "\u0120Failed": 22738, + "aned": 22739, + "\u0120ruler": 22740, + "\u0120Controller": 22741, + "\u0120feminists": 22742, + "Fix": 22743, + "gay": 22744, + "207": 22745, + "\u0120rabbit": 22746, + "Third": 22747, + "owntown": 22748, + "\u0120glue": 22749, + "\u0120volatile": 22750, + "\u0120shining": 22751, + "\u0120foll": 22752, + "\u0120impaired": 22753, + "\u0120supers": 22754, + "\u00e6\u012a": 22755, + "\u0120clutch": 22756, + "\u013c\u00e9\u0128\u0134": 22757, + "\u0120prolet": 22758, + "\u0120(!": 22759, + "\u0120yelled": 22760, + "\u0120Kiev": 22761, + "\u0120Ern": 22762, + "\u0120Shock": 22763, + "KB": 22764, + "\u0120situated": 22765, + "query": 22766, + "\u0120Nas": 22767, + "\u0120annex": 22768, + "character": 22769, + "\u0120Holiday": 22770, + "\u0120automation": 22771, + "\u0120Jill": 22772, + "\u0120Remastered": 22773, + "\u0120linem": 22774, + "\u0120wilderness": 22775, + "\u0120Horizon": 22776, + "\u0120Guinea": 22777, + "AZ": 22778, + "\u0120mainland": 22779, + "\u0120secrecy": 22780, + "LEASE": 22781, + "\u0120punk": 22782, + "\u0120Province": 22783, + "(),": 22784, + "Speed": 22785, + "\u0120handing": 22786, + "\u0120Sebast": 22787, + "Sir": 22788, + "rase": 22789, + "\u0120journals": 22790, + "\u0120congest": 22791, + "\u0120Tut": 22792, + "irrel": 22793, + "\u0120schizophrenia": 22794, + "\u0120misogyn": 22795, + "healthy": 22796, + "Iron": 22797, + "\u0120reacted": 22798, + "-$": 22799, + "252": 22800, + "\u0120plural": 22801, + "\u0120plum": 22802, + "\u0120bargain": 22803, + "\u0120grounded": 22804, + "finder": 22805, + "\u0120disse": 22806, + "\u0120Laz": 22807, + "OOD": 22808, + "\u0120atroc": 22809, + "Factory": 22810, + "\u0120minions": 22811, + "\u0120ori": 22812, + "\u0120Brave": 22813, + "\u0120PRE": 22814, + "\u0120Myanmar": 22815, + "\u0120Hod": 22816, + "\u0120expedition": 22817, + "\u0120explode": 22818, + "\u0120Coord": 22819, + "\u0120extr": 22820, + "\u0120Brief": 22821, + "\u0120ADHD": 22822, + "\u0120hardcore": 22823, + "feeding": 22824, + "\u0120dile": 22825, + "\u0120Fruit": 22826, + "\u0120vaccination": 22827, + "\u0120Mao": 22828, + "osphere": 22829, + "\u0120contests": 22830, + "-|": 22831, + "\u0120fren": 22832, + "isphere": 22833, + "Rom": 22834, + "\u0120Sharp": 22835, + "\u0120Trend": 22836, + "\u0120disconnect": 22837, + "\u00e2\u0122\u00a2\u00e2\u0122\u00a2": 22838, + "\u0120persecution": 22839, + "Earth": 22840, + "\u0120healthier": 22841, + "384": 22842, + "\u0120cob": 22843, + "\u0120Trinity": 22844, + "OWS": 22845, + "ANN": 22846, + "\u0120specialty": 22847, + "\u0120gru": 22848, + "\u0120cooperative": 22849, + "why": 22850, + "Starting": 22851, + "\u0120Issues": 22852, + "stre": 22853, + "ensor": 22854, + "\u0120185": 22855, + "Adv": 22856, + "!?": 22857, + "\u0120Revel": 22858, + "emia": 22859, + "\u0120Hulk": 22860, + "\u0120celebrations": 22861, + "\u0120Sou": 22862, + "raud": 22863, + "\u0120Klein": 22864, + "\u0120unreal": 22865, + "context": 22866, + "\u0120partnerships": 22867, + "\u0120adopting": 22868, + "tical": 22869, + "\u0120splash": 22870, + "\u0120Hezbollah": 22871, + "category": 22872, + "cyclop": 22873, + "xton": 22874, + "\u0120Dot": 22875, + "urdy": 22876, + "tz": 22877, + "\u0120envelope": 22878, + "\u0120NL": 22879, + "\u00e2\u0137": 22880, + "\u0120wherein": 22881, + "Spec": 22882, + "184": 22883, + "\u0120telev": 22884, + "aliation": 22885, + "\u0120myths": 22886, + "\u00e5\u00b0": 22887, + "\u0120rigorous": 22888, + "\u0120communicating": 22889, + "\u0120observer": 22890, + "\u0120rehe": 22891, + "\u0120Wash": 22892, + "\u0120apologized": 22893, + "\u0120Tin": 22894, + "\u0120expenditures": 22895, + "workers": 22896, + "document": 22897, + "\u0120hesitate": 22898, + "\u0120Lenin": 22899, + "\u0120unpredictable": 22900, + "\u0120renewal": 22901, + "cler": 22902, + "okia": 22903, + "\u0120CONT": 22904, + "\u0120postseason": 22905, + "Tokens": 22906, + "\u0120exacerb": 22907, + "\u0120betting": 22908, + "\u0120147": 22909, + "\u0120elevation": 22910, + "Wood": 22911, + "\u0120Solomon": 22912, + "194": 22913, + "004": 22914, + "output": 22915, + "\u0120redund": 22916, + "\u0120Mumbai": 22917, + "\u0120pH": 22918, + "\u0120reproduce": 22919, + "\u0120Duration": 22920, + "MAX": 22921, + "\u0120bog": 22922, + "CBS": 22923, + "\u0120Balance": 22924, + "\u0120Sgt": 22925, + "\u0120Recent": 22926, + "\u0120cd": 22927, + "\u0120popped": 22928, + "\u0120incompet": 22929, + "prop": 22930, + "ayan": 22931, + "guy": 22932, + "Pacific": 22933, + "\u0120tyr": 22934, + "\u0120{{": 22935, + "\u0120Mystic": 22936, + "\u0120Dana": 22937, + "\u0120masturb": 22938, + "\u0120geometry": 22939, + "\u00c3\u00a2": 22940, + "\u0120Correct": 22941, + "\u0120trajectory": 22942, + "\u0120distracted": 22943, + "\u0120foo": 22944, + "\u0120Welsh": 22945, + "Luc": 22946, + "mith": 22947, + "\u0120rugby": 22948, + "\u0120respiratory": 22949, + "\u0120triangle": 22950, + "\u0120215": 22951, + "\u0120undergraduate": 22952, + "\u0120Superior": 22953, + "changing": 22954, + "_-": 22955, + "\u0120rightly": 22956, + "\u0120referee": 22957, + "\u0120lucrative": 22958, + "\u0120unauthorized": 22959, + "\u0120resembles": 22960, + "\u0120GNU": 22961, + "\u0120Derby": 22962, + "\u0120pathways": 22963, + "\u0120Led": 22964, + "\u0120endurance": 22965, + "\u0120stint": 22966, + "\u0120collector": 22967, + "Fast": 22968, + "\u0120dots": 22969, + "\u0120nationals": 22970, + "\u0120Securities": 22971, + "\u0120whip": 22972, + "Param": 22973, + "\u0120learns": 22974, + "Magic": 22975, + "\u0120detailing": 22976, + "moon": 22977, + "\u0120broadcasting": 22978, + "\u0120baked": 22979, + "265": 22980, + "holm": 22981, + "\u0120Sah": 22982, + "\u0120Hussein": 22983, + "\u0120Courtesy": 22984, + "174": 22985, + "\u0120146": 22986, + "\u0120geographic": 22987, + "peace": 22988, + "\u0120judging": 22989, + "\u0120Stern": 22990, + "Bur": 22991, + "\u0120storyline": 22992, + "Gun": 22993, + "\u0120Stick": 22994, + "245": 22995, + "307": 22996, + "\u00e3\u0124\u00b4\u00e3\u0125\u00b3": 22997, + "\u0120Administrator": 22998, + "\u0120burnt": 22999, + "\u0120pave": 23000, + "choes": 23001, + "Exec": 23002, + "\u0120campuses": 23003, + "Result": 23004, + "\u0120mutations": 23005, + "\u0120Charter": 23006, + "\u0120captures": 23007, + "\u0120compares": 23008, + "\u0120badge": 23009, + "Scient": 23010, + "\u0120erad": 23011, + "iery": 23012, + "oi": 23013, + "ettes": 23014, + "\u0120Estate": 23015, + "\u0120strap": 23016, + "\u0120proudly": 23017, + "\u0120fried": 23018, + "\u0120withdrawn": 23019, + "\u0120Voy": 23020, + "phony": 23021, + "Items": 23022, + "\u0120Pierce": 23023, + "bard": 23024, + "\u0120annotation": 23025, + "anton": 23026, + "illon": 23027, + "Impro": 23028, + "...)": 23029, + "\u0120happier": 23030, + "------": 23031, + "adjust": 23032, + "\u0120staffers": 23033, + "\u0120activism": 23034, + "\u0120perf": 23035, + "\u0120alright": 23036, + "Need": 23037, + "\u0120commence": 23038, + "\u0120opioid": 23039, + "\u0120Amanda": 23040, + "Es": 23041, + "\u0120Pars": 23042, + "\u0120Kaw": 23043, + "Works": 23044, + "248": 23045, + "\u0120indo": 23046, + "tc": 23047, + "endant": 23048, + "\u0120Moto": 23049, + "\u0120legalization": 23050, + "OTE": 23051, + "\u0120tasked": 23052, + "\u0120tsp": 23053, + "\u0120ACTIONS": 23054, + "166": 23055, + "\u0120refreshing": 23056, + "\u0120NR": 23057, + "\u0120Perez": 23058, + "\u0120infringement": 23059, + "SY": 23060, + "Listen": 23061, + "inning": 23062, + "ku": 23063, + "\u0120rotate": 23064, + "program": 23065, + "arah": 23066, + "Design": 23067, + "\u0120(\u00c2\u00a3": 23068, + "\u0120storing": 23069, + "\u0120warrants": 23070, + "\u0120judgement": 23071, + "\u0120Brist": 23072, + "usually": 23073, + "photo": 23074, + "\u0120Ran": 23075, + "\u0120Pine": 23076, + "\u0120outrageous": 23077, + "\u0120Valentine": 23078, + "luence": 23079, + "\u0120Everybody": 23080, + "Altern": 23081, + "\u0120relevance": 23082, + "\u0120terminated": 23083, + "\u0120dessert": 23084, + "\u0120fulfilled": 23085, + "\u0120prosecuted": 23086, + "\u0120Words": 23087, + "\u0120migrant": 23088, + "\u0120cultivation": 23089, + "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 23090, + "idelity": 23091, + "\u0120Vern": 23092, + "\u0120Login": 23093, + "\u0120metaphor": 23094, + "\u0120Tip": 23095, + "\u0120recruits": 23096, + "\u0120Pig": 23097, + "ribing": 23098, + "\u0120enthusiasts": 23099, + "exper": 23100, + "\u0120frightening": 23101, + "\u0120Hair": 23102, + "anson": 23103, + "strate": 23104, + "\u0120hi": 23105, + "Height": 23106, + "\u0120owning": 23107, + "none": 23108, + "\u0120dislike": 23109, + "\u0120knives": 23110, + "pherd": 23111, + "\u0120loudly": 23112, + "\u0120APIs": 23113, + "Display": 23114, + "\u0120Lac": 23115, + "\u0120USS": 23116, + "abl": 23117, + "verages": 23118, + "Jew": 23119, + "\u0120172": 23120, + "\u0120Historical": 23121, + "atoon": 23122, + "\u0120Physics": 23123, + "intern": 23124, + "\u0120warmth": 23125, + "\u0120topp": 23126, + "DM": 23127, + "\u0120gunman": 23128, + "\u0120emperor": 23129, + "odi": 23130, + "\u00e3\u0125\u00a3": 23131, + "inatory": 23132, + "\u0120Rib": 23133, + "\u0120131": 23134, + "\u0120Saturn": 23135, + "\u0120Shining": 23136, + "\u0120waking": 23137, + "Quotes": 23138, + "\u0120comedian": 23139, + "enberg": 23140, + "\u00c2\u00bd": 23141, + "\u0120believers": 23142, + "\u0120paperwork": 23143, + "custom": 23144, + "\u0120lev": 23145, + "\u0120lament": 23146, + "\u0120pouring": 23147, + "222": 23148, + "political": 23149, + "\u0120Supplement": 23150, + "maid": 23151, + "\u0120cruelty": 23152, + "\u0120tread": 23153, + "ysics": 23154, + "Aw": 23155, + "rites": 23156, + "\u0120modifier": 23157, + "\u0120Position": 23158, + "Adam": 23159, + "lb": 23160, + "ubs": 23161, + "\u0120imperfect": 23162, + "\u0120clusters": 23163, + "\u0120Engineer": 23164, + "\u0120Cherry": 23165, + "\u0120inauguration": 23166, + "\u0120Sau": 23167, + "\u0120embodiment": 23168, + "\u0120Uncle": 23169, + "\u0120overr": 23170, + "\u0120explosions": 23171, + "cule": 23172, + "\u0120Princeton": 23173, + "\u0120Andrea": 23174, + "\u0120incorrectly": 23175, + "\u0120earnest": 23176, + "\u0120pilgr": 23177, + "\u0120Sprint": 23178, + "\u0120sleeve": 23179, + "\u0120hears": 23180, + "\u0120Amazing": 23181, + "\u0120browsing": 23182, + "agin": 23183, + "\u0120homeland": 23184, + "\u0120haw": 23185, + "\u0120diving": 23186, + "istered": 23187, + "178": 23188, + "\u0120bargaining": 23189, + "\u0120Arcade": 23190, + "\u0120delegate": 23191, + "terson": 23192, + "................................................................": 23193, + "\u0120Jacksonville": 23194, + "275": 23195, + "\u0120stagn": 23196, + "\u0120adam": 23197, + "\u0120Sherman": 23198, + "CB": 23199, + "\u0120suburb": 23200, + "\u0120Foods": 23201, + "\u0120converting": 23202, + "\u0120Arist": 23203, + "\u0120chambers": 23204, + "love": 23205, + "\u0120amino": 23206, + "\u0120Gan": 23207, + "\u0120madness": 23208, + "mc": 23209, + "\u0120USE": 23210, + "defined": 23211, + "\u0120ultr": 23212, + "indust": 23213, + "\u0120wolves": 23214, + "lance": 23215, + "Additionally": 23216, + "\u0120cracks": 23217, + "asia": 23218, + "\u0120Reason": 23219, + "\u0120Pump": 23220, + "\u0120accidental": 23221, + "\u0120Laser": 23222, + "\u0120Rid": 23223, + "\u0120initialized": 23224, + "elli": 23225, + "\u0120unnamed": 23226, + "\u0120noun": 23227, + "\u0120Passed": 23228, + "\u0120hostage": 23229, + "\u0120Ethiop": 23230, + "shirts": 23231, + "\u0120unrel": 23232, + "\u0120Embassy": 23233, + "\u01201941": 23234, + "\u0120atoms": 23235, + "\u0120purported": 23236, + "164": 23237, + "\u0120Fi": 23238, + "\u0120gallons": 23239, + "\u0120Monica": 23240, + "\u0120pg": 23241, + "enment": 23242, + "\u0120sorted": 23243, + "\u0120Gospel": 23244, + "\u0120heights": 23245, + "\u0120traced": 23246, + "\u0120undergoing": 23247, + "Shell": 23248, + "\u0120sacks": 23249, + "\u0120proportions": 23250, + "\u0120halluc": 23251, + "Font": 23252, + "acet": 23253, + "\u0120warmer": 23254, + "\u0120INTER": 23255, + "\u0120grabbing": 23256, + "Plug": 23257, + "\u0120realization": 23258, + "\u0120Burke": 23259, + "\u0120enchant": 23260, + "ATER": 23261, + "\u0120Seed": 23262, + "\u0120abundant": 23263, + "FM": 23264, + "\u0120civic": 23265, + "Vs": 23266, + "isi": 23267, + "\u0120vow": 23268, + "\u0120reper": 23269, + "\u0120Partnership": 23270, + "\u0120penetration": 23271, + "\u0120axe": 23272, + "\u0120shattered": 23273, + "\u0120Zombies": 23274, + "\u0120vinyl": 23275, + "\u0120Alert": 23276, + "eon": 23277, + "\u0120obliged": 23278, + "\u0120Illust": 23279, + "\u0120Plaza": 23280, + "\u0120Frontier": 23281, + "\u0120davidjl": 23282, + "\u0120Serial": 23283, + "\u0120Hav": 23284, + "\u0120Nutrition": 23285, + "Bi": 23286, + "\u0120\u00e2\u0138\u012a": 23287, + "\u0120Jays": 23288, + "linux": 23289, + "\u0120hurry": 23290, + "\u0120voy": 23291, + "\u0120hopeless": 23292, + "\u0120Stealth": 23293, + "\u0120\u00e3\u0123": 23294, + "essors": 23295, + "ttle": 23296, + "borg": 23297, + "\u0120Safari": 23298, + "fell": 23299, + "\u0120wary": 23300, + "due": 23301, + "\u0120Above": 23302, + "Ha": 23303, + "ELL": 23304, + "\u0120notor": 23305, + "\u0120Won": 23306, + "Too": 23307, + "\u0120occupations": 23308, + "\u0120possessions": 23309, + "\u0120inviting": 23310, + "\u0120predators": 23311, + "\u0120accelerated": 23312, + "\u0120157": 23313, + "uterte": 23314, + "\u0120Cube": 23315, + "east": 23316, + "account": 23317, + "Give": 23318, + "\u0120transplant": 23319, + "redients": 23320, + "idable": 23321, + "\u0120screenshots": 23322, + "\u0120Gund": 23323, + "\u0120FS": 23324, + "\u0120travelers": 23325, + "\u0120sensory": 23326, + "\u0120Fiat": 23327, + "\u0120Rockets": 23328, + "\u0130\u012d": 23329, + "_{": 23330, + "Friend": 23331, + "\u0120charming": 23332, + "ALS": 23333, + "\u0120enjoyment": 23334, + "mph": 23335, + "\u01205000": 23336, + "\u0120REG": 23337, + "\u00d9\u0128": 23338, + "bia": 23339, + "\u0120compilation": 23340, + "rost": 23341, + "\u0120VP": 23342, + "\u0120Schne": 23343, + "2019": 23344, + "\u0120copying": 23345, + "MORE": 23346, + "\u0120Flore": 23347, + "falls": 23348, + "215": 23349, + "total": 23350, + "\u0120disciples": 23351, + "double": 23352, + "\u0120exceeding": 23353, + "\u0120smashed": 23354, + "\u0120conceptual": 23355, + "\u0120Romania": 23356, + "\u0120Brent": 23357, + "\u0120ICE": 23358, + "\u0120Tou": 23359, + "\u0120grap": 23360, + "\u0120nails": 23361, + "189": 23362, + "\u00e3\u0125\u013a": 23363, + "\u0120procure": 23364, + "eur": 23365, + "\u0120confirming": 23366, + "\u0120Cec": 23367, + "awi": 23368, + "\u0120Eden": 23369, + "\u0120ng": 23370, + "\u0120engineered": 23371, + "atics": 23372, + "\u0120hooked": 23373, + "\u0120disgusting": 23374, + "\u0120Murder": 23375, + "\u00e3\u0124\u00bf": 23376, + "Library": 23377, + "\u0120168": 23378, + "Almost": 23379, + "hematic": 23380, + "Menu": 23381, + "\u0120Notre": 23382, + "\u0120Jur": 23383, + "\u0120kidnapped": 23384, + "\u0120hacker": 23385, + "\u0120Jade": 23386, + "\u0120creepy": 23387, + "\u0120drawings": 23388, + "\u0120Sponsor": 23389, + "\u0120cyclists": 23390, + "\u0120Goblin": 23391, + "\u0120optimized": 23392, + "\u0120staged": 23393, + "\u0120McD": 23394, + "between": 23395, + "Age": 23396, + "eno": 23397, + "Sex": 23398, + "\u0120Wide": 23399, + "nings": 23400, + "avis": 23401, + "\u0120incapable": 23402, + "\u0120Kob": 23403, + "\u0120rewarding": 23404, + "\u0120Lone": 23405, + "olescent": 23406, + "\u0120contracted": 23407, + "\u0120sticky": 23408, + "Jose": 23409, + "Ball": 23410, + "fest": 23411, + "\u0120Input": 23412, + "\u0120Recently": 23413, + "\u0120tomat": 23414, + "square": 23415, + "Application": 23416, + "\u0120nitrogen": 23417, + "\u0120duplicate": 23418, + "\u0120Recon": 23419, + "\u0120Dear": 23420, + "London": 23421, + "\u0120intra": 23422, + "\u0120dock": 23423, + "\u0120outreach": 23424, + "\u0120Million": 23425, + "\u0120mammals": 23426, + "ampton": 23427, + "VAL": 23428, + "\u0120snaps": 23429, + "\u0120dos": 23430, + "\u0120Whole": 23431, + "\u0120Ready": 23432, + "Try": 23433, + "\u0120Winnipeg": 23434, + "earance": 23435, + "\u0120incurred": 23436, + "renched": 23437, + "\u0120NSW": 23438, + "ilot": 23439, + "raine": 23440, + "\u0120cube": 23441, + "got": 23442, + "\u0120runway": 23443, + "etermined": 23444, + "\u0120Hawks": 23445, + "\u0120survivor": 23446, + "\u0120Wish": 23447, + "\u0120Din": 23448, + "\u0120DEF": 23449, + "\u0120Vault": 23450, + "187": 23451, + "\u0120mushrooms": 23452, + "\u0120crisp": 23453, + "bey": 23454, + "\u0120Discovery": 23455, + "\u0120developmental": 23456, + "\u0120paradigm": 23457, + "\u0120chaotic": 23458, + "\u0120Tsu": 23459, + "\u0120333": 23460, + "bons": 23461, + "\u0120bacterial": 23462, + "\u0120commits": 23463, + "\u0120cosmic": 23464, + "\u0120mega": 23465, + "ocative": 23466, + "\u0120Paint": 23467, + "ophobic": 23468, + "\u0120vain": 23469, + "\u0120carved": 23470, + "\u0120Thief": 23471, + "\u0120Gul": 23472, + "owship": 23473, + "\u0120cites": 23474, + "\u0120Edinburgh": 23475, + "\u0120diminished": 23476, + "\u0120acknowledges": 23477, + "\u0120Kills": 23478, + "\u0120microw": 23479, + "\u0120Hera": 23480, + "\u0120seniors": 23481, + "\u0120whereby": 23482, + "Hop": 23483, + "atron": 23484, + "\u0120unavailable": 23485, + "\u0120Nate": 23486, + "\u0120480": 23487, + "\u0120slated": 23488, + "\u0120Rebecca": 23489, + "\u0120Battery": 23490, + "\u0120grammar": 23491, + "\u0120headset": 23492, + "\u0120cursor": 23493, + "\u0120excluding": 23494, + "anye": 23495, + "aundering": 23496, + "ebin": 23497, + "\u0120feasible": 23498, + "\u0120Publishing": 23499, + "\u0120Labs": 23500, + "\u0120Cliff": 23501, + "\u0120Ferrari": 23502, + "\u0120pac": 23503, + "visible": 23504, + "marked": 23505, + "pell": 23506, + "\u0120polite": 23507, + "\u0120staggering": 23508, + "\u0120Galactic": 23509, + "\u0120superst": 23510, + "\u0120paran": 23511, + "\u0120Officers": 23512, + "\u00e3\u0122\u0123": 23513, + "\u0120specifics": 23514, + "ulus": 23515, + "239": 23516, + "\u0120Paste": 23517, + "AMP": 23518, + "\u0120Panama": 23519, + "\u0120Delete": 23520, + "anguard": 23521, + "restrial": 23522, + "\u0120heroic": 23523, + "\u0120Dy": 23524, + "\u00d8\u00a7\u00d9\u0126": 23525, + "\u0120incumbent": 23526, + "\u0120crunch": 23527, + "tro": 23528, + "\u0120scoop": 23529, + "\u0120blogger": 23530, + "\u0120sellers": 23531, + "uren": 23532, + "\u0120medicines": 23533, + "\u0120Caps": 23534, + "\u0120Animation": 23535, + "oxy": 23536, + "\u0120outward": 23537, + "\u0120inquiries": 23538, + "229": 23539, + "\u0120psychologist": 23540, + "\u0120Sask": 23541, + "evil": 23542, + "\u0120contaminated": 23543, + "\u00e3\u0124\u00a8": 23544, + "herence": 23545, + "\u0120branded": 23546, + "\u0120Abdul": 23547, + "zh": 23548, + "\u0120paragraphs": 23549, + "\u0120mins": 23550, + "\u0120correlated": 23551, + "erb": 23552, + "\u0120impart": 23553, + "\u0120milestone": 23554, + "\u0120Solutions": 23555, + "otle": 23556, + "\u0120undercover": 23557, + "\u0120marched": 23558, + "\u0120Chargers": 23559, + "fax": 23560, + "\u0120Secrets": 23561, + "\u0120ruth": 23562, + "weather": 23563, + "\u0120feminine": 23564, + "\u0120sham": 23565, + "\u0120prestigious": 23566, + "iggins": 23567, + "\u0120sung": 23568, + "history": 23569, + "ettle": 23570, + "ggie": 23571, + "\u0120outdated": 23572, + "oland": 23573, + "\u0120perceptions": 23574, + "\u0120Session": 23575, + "\u0120Dodgers": 23576, + "uj": 23577, + "\u0120END": 23578, + "Doc": 23579, + "\u0120deficiency": 23580, + "Grand": 23581, + "\u0120Joker": 23582, + "\u0120retrospect": 23583, + "\u0120diagnostic": 23584, + "\u0120harmless": 23585, + "\u0120rogue": 23586, + "\u0120Aval": 23587, + "Equ": 23588, + "\u0120transc": 23589, + "\u0120Robertson": 23590, + "\u0120Depending": 23591, + "\u0120Burns": 23592, + "ivo": 23593, + "\u0120hostility": 23594, + "Features": 23595, + "\u0135\u013a": 23596, + "\u0120discomfort": 23597, + "\u0120LCD": 23598, + "specified": 23599, + "\u0120Expect": 23600, + "340": 23601, + "\u0120imperative": 23602, + "\u0120Regular": 23603, + "Chinese": 23604, + "\u0120statewide": 23605, + "\u0120symm": 23606, + "\u0120loops": 23607, + "\u0120autumn": 23608, + "Nick": 23609, + "\u0120shaping": 23610, + "\u0120quot": 23611, + "\u0120cherry": 23612, + "\u0120Crossref": 23613, + "\u00e8\u00a6\u013c\u00e9\u0128\u0134": 23614, + "Standard": 23615, + "heed": 23616, + "\u0120Dell": 23617, + "\u0120Vietnamese": 23618, + "\u0120ost": 23619, + "\u0120Valkyrie": 23620, + "OA": 23621, + "Assad": 23622, + "\u0120rebound": 23623, + "\u0120Traffic": 23624, + "places": 23625, + "\u00e6\u013a": 23626, + "\u0120Buc": 23627, + "172": 23628, + "\u0120shelters": 23629, + "\u0120insisting": 23630, + "\u0120Certainly": 23631, + "\u0120Kenneth": 23632, + "\u0120TCP": 23633, + "\u0120penal": 23634, + "\u0120Replay": 23635, + "heard": 23636, + "\u0120dialect": 23637, + "iza": 23638, + "\u0120FY": 23639, + "itcher": 23640, + "\u0120DL": 23641, + "\u0120spiral": 23642, + "\u0120quarterbacks": 23643, + "\u0120hull": 23644, + "\u0120google": 23645, + "\u0120todd": 23646, + "\u0120Sterling": 23647, + "\u0120Plate": 23648, + "\u0120spying": 23649, + "mbol": 23650, + "\u0120Realm": 23651, + "\u0120Proced": 23652, + "\u0120Crash": 23653, + "\u0120terminate": 23654, + "\u0120protesting": 23655, + "Center": 23656, + "guided": 23657, + "\u0120uncover": 23658, + "\u0120boycott": 23659, + "\u0120realizes": 23660, + "sound": 23661, + "\u0120pretending": 23662, + "\u0120Vas": 23663, + "1980": 23664, + "\u0120framed": 23665, + "\u0120139": 23666, + "\u0120descended": 23667, + "\u0120rehabilitation": 23668, + "\u0120borrowing": 23669, + "\u0120Buch": 23670, + "\u0120blur": 23671, + "Ron": 23672, + "\u0120Frozen": 23673, + "enza": 23674, + "Chief": 23675, + "\u0120Poor": 23676, + "\u0120translates": 23677, + "MIN": 23678, + "\u0120212": 23679, + "JECT": 23680, + "\u0120erupted": 23681, + "\u0120successes": 23682, + "SEC": 23683, + "\u0120plague": 23684, + "\u0120gems": 23685, + "doms": 23686, + "\u0120stretches": 23687, + "\u0120Spy": 23688, + "\u0120storytelling": 23689, + "Credit": 23690, + "\u0120Push": 23691, + "\u0120traction": 23692, + "\u0120ineffective": 23693, + "\u0120Luna": 23694, + "\u0120tapes": 23695, + "\u0120analytics": 23696, + "ercise": 23697, + "\u0120programmes": 23698, + "\u0120Carbon": 23699, + "\u0120behold": 23700, + "heavy": 23701, + "\u0120Conservation": 23702, + "\u0120FIR": 23703, + "\u0120sack": 23704, + "termin": 23705, + "ricks": 23706, + "\u0120housed": 23707, + "\u0120unusually": 23708, + "Ice": 23709, + "\u0120executing": 23710, + "\u0120Moroc": 23711, + "eday": 23712, + "\u0120editions": 23713, + "\u0120smarter": 23714, + "\u0120BA": 23715, + "\u0120outlaw": 23716, + "\u0120vanished": 23717, + "iba": 23718, + "ALSE": 23719, + "\u0120Silva": 23720, + "238": 23721, + "Could": 23722, + "\u0120philosopher": 23723, + "\u0120evacuated": 23724, + "Secret": 23725, + "142": 23726, + "\u0120visas": 23727, + "\u00e3\u0124\u00ac": 23728, + "\u0120Malt": 23729, + "\u0120Clearly": 23730, + "\u0120Niger": 23731, + "\u0120Cairo": 23732, + "\u0120Fist": 23733, + "380": 23734, + "\u0120XML": 23735, + "auto": 23736, + "itant": 23737, + "\u0120reinforced": 23738, + "Record": 23739, + "\u0120Survivor": 23740, + "GHz": 23741, + "\u0120screws": 23742, + "parents": 23743, + "\u0120oceans": 23744, + "mares": 23745, + "\u0120brakes": 23746, + "vasive": 23747, + "\u0120hello": 23748, + "\u0120SIM": 23749, + "rimp": 23750, + "\u0120ore": 23751, + "\u0120Armour": 23752, + "247": 23753, + "\u0120terrific": 23754, + "\u0120tones": 23755, + "141": 23756, + "\u0120Minutes": 23757, + "Episode": 23758, + "\u0120curves": 23759, + "\u0120inflammatory": 23760, + "\u0120batting": 23761, + "\u0120Beautiful": 23762, + "Lay": 23763, + "\u0120unpop": 23764, + "vable": 23765, + "\u0120riots": 23766, + "\u0120Tactics": 23767, + "baugh": 23768, + "\u0120Cock": 23769, + "\u0120orgasm": 23770, + "\u0120Sas": 23771, + "\u0120constructor": 23772, + "etz": 23773, + "Gov": 23774, + "\u0120antagon": 23775, + "\u0120theat": 23776, + "\u0120deeds": 23777, + "hao": 23778, + "cuts": 23779, + "\u0120McCl": 23780, + "\u0120um": 23781, + "\u0120Scientists": 23782, + "\u0120grassroots": 23783, + "yssey": 23784, + "\"]=>": 23785, + "\u0120surfaced": 23786, + "\u0120shades": 23787, + "\u0120neighbours": 23788, + "\u0120advertis": 23789, + "oya": 23790, + "\u0120merged": 23791, + "Upon": 23792, + "\u0120gad": 23793, + "\u0120anticipate": 23794, + "Anyway": 23795, + "\u0120slogan": 23796, + "\u0120disrespect": 23797, + "Iran": 23798, + "\u0120TB": 23799, + "acted": 23800, + "\u0120subpoen": 23801, + "mediately": 23802, + "OOOO": 23803, + "\u0120waiver": 23804, + "\u0120vulnerabilities": 23805, + "ottesville": 23806, + "\u0120Huffington": 23807, + "Josh": 23808, + "\u0120DH": 23809, + "Monday": 23810, + "\u0120Ellen": 23811, + "Know": 23812, + "xon": 23813, + "items": 23814, + "228": 23815, + "\u0120fills": 23816, + "\u0120Nike": 23817, + "\u0120cumulative": 23818, + "andals": 23819, + "Ir": 23820, + "\u0120\u00ec": 23821, + "\u0120friction": 23822, + "igator": 23823, + "\u0120scans": 23824, + "\u0120Vienna": 23825, + "ldom": 23826, + "\u0120performers": 23827, + "Prim": 23828, + "\u0120bidding": 23829, + "Mur": 23830, + "\u0120leaned": 23831, + "\u0120Prix": 23832, + "alks": 23833, + "\u0120[\u00e2\u0122\u00a6]": 23834, + "\u0120Twitch": 23835, + "\u0120Developer": 23836, + "\u0120Gir": 23837, + "\u0120callback": 23838, + "Abstract": 23839, + "\u0120accustomed": 23840, + "\u0120freedoms": 23841, + "\u0120PG": 23842, + "uracy": 23843, + "\u0120lump": 23844, + "isman": 23845, + ",,,,": 23846, + "1992": 23847, + "\u0120RED": 23848, + "\u0120worm": 23849, + "Match": 23850, + "\u0120Platinum": 23851, + "IJ": 23852, + "\u0120Owner": 23853, + "Trivia": 23854, + "compl": 23855, + "\u0120newborn": 23856, + "\u0120fantas": 23857, + "Own": 23858, + "\u01201959": 23859, + "\u0120sympath": 23860, + "\u0120ubiqu": 23861, + "\u0120outputs": 23862, + "\u0120allev": 23863, + "\u0120prag": 23864, + "Kevin": 23865, + "\u0120favors": 23866, + "\u0120burial": 23867, + "\u0120nurt": 23868, + "solete": 23869, + "cache": 23870, + "\u0120156": 23871, + "\u0120unlocks": 23872, + "techn": 23873, + "Making": 23874, + "\u0120conquer": 23875, + "adic": 23876, + "\u00e6\u0138": 23877, + "\u0120elf": 23878, + "\u0120electorate": 23879, + "\u0120Kurds": 23880, + "\u0120Stack": 23881, + "\u0120Samurai": 23882, + "\u0120\u00e2\u013a\u0127": 23883, + "\u0120{}": 23884, + "\u0120Said": 23885, + "\u0120Fallout": 23886, + "\u0120kindness": 23887, + "\u0120Customs": 23888, + "\u0120Boulevard": 23889, + "\u0120helicopters": 23890, + "otics": 23891, + "\u0120Veget": 23892, + "comment": 23893, + "\u0120criticised": 23894, + "\u0120polished": 23895, + "\u0120Remix": 23896, + "\u0120Cultural": 23897, + "\u0120recons": 23898, + "\u0120doi": 23899, + "atem": 23900, + "Screen": 23901, + "\u0120barred": 23902, + "Comments": 23903, + "\u0120Generally": 23904, + "\u0120slap": 23905, + "720": 23906, + "Vari": 23907, + "pine": 23908, + "\u0120empt": 23909, + "\u0120hats": 23910, + "\u0120Playing": 23911, + "lab": 23912, + "average": 23913, + "forms": 23914, + "\u0120Cotton": 23915, + "\u0120cans": 23916, + "\u0120DON": 23917, + "\u0120Somalia": 23918, + "Crypt": 23919, + "\u0120Increases": 23920, + "Ever": 23921, + "modern": 23922, + "\u0120surgeon": 23923, + "3000": 23924, + "\u0120randomized": 23925, + "================================================================": 23926, + "Bern": 23927, + "impl": 23928, + "\u0120COR": 23929, + "\u0120proclaim": 23930, + "thouse": 23931, + "\u0120toes": 23932, + "\u0120ample": 23933, + "\u0120preserving": 23934, + "\u0120disbel": 23935, + "grand": 23936, + "Besides": 23937, + "\u0120silk": 23938, + "\u0120Pattern": 23939, + "hm": 23940, + "\u0120enterprises": 23941, + "\u0120affidavit": 23942, + "\u0120Advisory": 23943, + "\u0120advertised": 23944, + "\u0120Religious": 23945, + "sections": 23946, + "psych": 23947, + "\u0120Fields": 23948, + "aways": 23949, + "\u0120hashtag": 23950, + "\u0120Nightmare": 23951, + "\u0120vampire": 23952, + "\u0120forensic": 23953, + "rossover": 23954, + "nar": 23955, + "\u0120navy": 23956, + "\u0120vacant": 23957, + "\u0120Duel": 23958, + "\u0120hallway": 23959, + "\u0120facebook": 23960, + "identally": 23961, + "\u0120NRA": 23962, + "\u0120matt": 23963, + "\u0120hurricane": 23964, + "\u0120Kirby": 23965, + "\u0120Puzzle": 23966, + "\u0120skirt": 23967, + "oust": 23968, + "dullah": 23969, + "\u0120analogy": 23970, + "inion": 23971, + "\u0120tomatoes": 23972, + "\u0120NV": 23973, + "\u0120Peak": 23974, + "\u0120Meyer": 23975, + "\u0120appointments": 23976, + "\u0120masc": 23977, + "\u0120alley": 23978, + "rehend": 23979, + "\u0120charities": 23980, + "\u0120undo": 23981, + "\u0120destinations": 23982, + "\u0120Testing": 23983, + "\">\"": 24618, + "cats": 24619, + "*.": 24620, + "\u0120gestures": 24621, + "general": 24622, + "League": 24623, + "\u0120packets": 24624, + "\u0120Inspector": 24625, + "\u0120Berg": 24626, + "\u0120fraudulent": 24627, + "\u0120criticize": 24628, + "Fun": 24629, + "\u0120blaming": 24630, + "ndra": 24631, + "\u0120slash": 24632, + "\u0120Eston": 24633, + "\u0120proposing": 24634, + "\u0120whales": 24635, + "\u0120therapist": 24636, + "\u0120subset": 24637, + "\u0120leisure": 24638, + "ELD": 24639, + "\u0120CVE": 24640, + "\u0120Activity": 24641, + "\u0120culmin": 24642, + "shop": 24643, + "\u0120DAY": 24644, + "ischer": 24645, + "\u0120Admiral": 24646, + "\u0120Attacks": 24647, + "\u01201958": 24648, + "\u0120memoir": 24649, + "\u0120folded": 24650, + "\u0120sexist": 24651, + "\u0120153": 24652, + "\u0120LI": 24653, + "\u0120readings": 24654, + "\u0120embarrassment": 24655, + "\u0120Employment": 24656, + "wart": 24657, + "chin": 24658, + "\u0120continuation": 24659, + "lia": 24660, + "Recently": 24661, + "\u0120duel": 24662, + "\u0120evacuation": 24663, + "\u0120Kashmir": 24664, + "\u0120disposition": 24665, + "\u0120Rig": 24666, + "\u0120bolts": 24667, + "\u0120insurers": 24668, + "467": 24669, + "Mex": 24670, + "\u0120retaliation": 24671, + "\u0120misery": 24672, + "\u0120unreasonable": 24673, + "raining": 24674, + "Imm": 24675, + "\u0120PU": 24676, + "emer": 24677, + "\u0120genital": 24678, + "\u00e3\u0124\u00b3": 24679, + "\u0120Candy": 24680, + "\u0120onions": 24681, + "\u0120Patt": 24682, + "liner": 24683, + "\u0120conceded": 24684, + "\u0120fa": 24685, + "\u0120forc": 24686, + "\u0120Hernandez": 24687, + "\u0120Geoff": 24688, + "debian": 24689, + "\u0120Teams": 24690, + "\u0120cries": 24691, + "\u0120homeowners": 24692, + "237": 24693, + "ABC": 24694, + "\u0120stitch": 24695, + "\u0120statistic": 24696, + "\u0120headers": 24697, + "\u0120Biology": 24698, + "\u0120motors": 24699, + "\u0120GEN": 24700, + "\u0120Lip": 24701, + "\u0120hates": 24702, + "\u0120heel": 24703, + "Self": 24704, + "ipl": 24705, + "EDIT": 24706, + "orting": 24707, + "\u0120annot": 24708, + "\u0120Speech": 24709, + "oldemort": 24710, + "\u0120Javascript": 24711, + "\u0120LeBron": 24712, + "\u0120footprint": 24713, + "\u0120fn": 24714, + "\u0120seizures": 24715, + "nas": 24716, + "hide": 24717, + "\u01201954": 24718, + "\u0120Bee": 24719, + "\u0120Declaration": 24720, + "\u0120Katie": 24721, + "\u0120reservations": 24722, + "NR": 24723, + "female": 24724, + "\u0120saturated": 24725, + "\u0120biblical": 24726, + "\u0120trolls": 24727, + "Device": 24728, + "photos": 24729, + "\u0120drums": 24730, + "\u00e3\u0125\u012b\u00e3\u0125\u00a9\u00e3\u0124\u00b4\u00e3\u0125\u00b3": 24731, + "Night": 24732, + "fighter": 24733, + "\u0120Hak": 24734, + "riber": 24735, + "\u0120cush": 24736, + "\u0120disciplinary": 24737, + "baum": 24738, + "\u0120GH": 24739, + "\u0120Schmidt": 24740, + "ilibrium": 24741, + "\u0120sixty": 24742, + "\u0120Kushner": 24743, + "rots": 24744, + "\u0120pund": 24745, + "\u0120Rac": 24746, + "\u0120springs": 24747, + "\u0120conve": 24748, + "Business": 24749, + "Fall": 24750, + "\u0120qualifications": 24751, + "\u0120verses": 24752, + "\u0120narciss": 24753, + "\u0120Koh": 24754, + "\u0120Wow": 24755, + "\u0120Charlottesville": 24756, + "edo": 24757, + "\u0120interrogation": 24758, + "\u0120Wool": 24759, + "365": 24760, + "Brian": 24761, + "\u0120\u00e2\u013e\u0135": 24762, + "\u0120alleges": 24763, + "onds": 24764, + "idation": 24765, + "\u0120Jackie": 24766, + "yu": 24767, + "\u0120lakes": 24768, + "\u0120worthwhile": 24769, + "\u0120crystals": 24770, + "\u0120Juda": 24771, + "\u0120comprehend": 24772, + "\u0120flush": 24773, + "\u0120absorption": 24774, + "\u0120OC": 24775, + "\u0120frightened": 24776, + "\u0120Chocolate": 24777, + "Martin": 24778, + "\u0120buys": 24779, + "\u0120bucks": 24780, + "\u0120appell": 24781, + "\u0120Championships": 24782, + "\u0120listener": 24783, + "\u0120Defensive": 24784, + "\u0120cz": 24785, + "uds": 24786, + "\u0120Mate": 24787, + "\u0120replay": 24788, + "\u0120decorated": 24789, + "\u0120sunk": 24790, + "\u0120VIP": 24791, + "\u0120Ank": 24792, + "\u0120195": 24793, + "aaaa": 24794, + "Nobody": 24795, + "\u0120Milk": 24796, + "\u0120Gur": 24797, + "\u0120Mk": 24798, + "\u0120Sara": 24799, + "\u0120seating": 24800, + "\u0120Wid": 24801, + "Track": 24802, + "\u0120employs": 24803, + "\u0120gigantic": 24804, + "APP": 24805, + "\u00e3\u0124\u00a7": 24806, + "inventory": 24807, + "\u0120towel": 24808, + "atche": 24809, + "lasting": 24810, + "\u0120TL": 24811, + "\u0120latency": 24812, + "\u0120kne": 24813, + "Ber": 24814, + "meaning": 24815, + "\u0120upheld": 24816, + "\u0120playground": 24817, + "\u0120mant": 24818, + "Side": 24819, + "\u0120stereo": 24820, + "\u0120northwest": 24821, + "\u0120exceptionally": 24822, + "\u0120rays": 24823, + "\u0120recurring": 24824, + "Drive": 24825, + "\u0120upright": 24826, + "\u0120abduct": 24827, + "\u0120Marathon": 24828, + "\u0120goodbye": 24829, + "\u0120alphabet": 24830, + "hp": 24831, + "\u0120courtroom": 24832, + "rington": 24833, + "othing": 24834, + "Tag": 24835, + "\u0120diplomats": 24836, + "\u0120barbar": 24837, + "\u0120Aqua": 24838, + "183": 24839, + "3333": 24840, + "\u0120maturity": 24841, + "\u0120instability": 24842, + "\u0120Apache": 24843, + "\u0120===": 24844, + "\u0120fasting": 24845, + "\u0120Grid": 24846, + "ModLoader": 24847, + "\u0120152": 24848, + "Abs": 24849, + "\u0120Operating": 24850, + "etti": 24851, + "\u0120acquaint": 24852, + "Donnell": 24853, + "\u0120Kem": 24854, + "\u0120Forge": 24855, + "\u0120armored": 24856, + "Mil": 24857, + "\u0120philosophers": 24858, + "invest": 24859, + "Players": 24860, + "\u00e2\u012a": 24861, + "\u0120myriad": 24862, + "\u0120comrades": 24863, + "Rot": 24864, + "\u0120remembering": 24865, + "\u0120corresponds": 24866, + "\u0120programmers": 24867, + "\u0120Lynn": 24868, + "\u0120olig": 24869, + "\u0120coherent": 24870, + "ynchron": 24871, + "\u0120Chemical": 24872, + "\u0120jugg": 24873, + "pair": 24874, + "posts": 24875, + "Eye": 24876, + "\u0120Inner": 24877, + "\u0120semester": 24878, + "ottest": 24879, + "\u0120Emirates": 24880, + "ricanes": 24881, + "orously": 24882, + "mits": 24883, + "\u0120Wis": 24884, + "\u0120dodge": 24885, + "location": 24886, + "\u0120faded": 24887, + "Amazon": 24888, + "\u0120Proceed": 24889, + "\u0120INFO": 24890, + "journal": 24891, + "\u0120Truck": 24892, + "Ten": 24893, + "\u0120217": 24894, + "\u0120statutes": 24895, + "mobile": 24896, + "\u0120Types": 24897, + "Recomm": 24898, + "buster": 24899, + "pex": 24900, + "\u0120legends": 24901, + "\u0120headache": 24902, + "faced": 24903, + "\u0120WiFi": 24904, + "ifty": 24905, + "\u0120HER": 24906, + "\u0120circuits": 24907, + "ERROR": 24908, + "226": 24909, + "olin": 24910, + "\u0120cylinder": 24911, + "ospace": 24912, + "ikers": 24913, + "Prem": 24914, + "Quant": 24915, + "\u0120conflicting": 24916, + "\u0120slightest": 24917, + "\u0120forged": 24918, + "ionage": 24919, + "Stephen": 24920, + "\u0120Kub": 24921, + "\u0120Opportun": 24922, + "\u0120Heal": 24923, + "\u0120blo": 24924, + "\u0120rulers": 24925, + "\u0120huh": 24926, + "\u0120submarine": 24927, + "fy": 24928, + "asser": 24929, + "\u0120allowance": 24930, + "\u0120Kasich": 24931, + "\u0120Tas": 24932, + "\u0120Australians": 24933, + "ForgeModLoader": 24934, + "\u0120\u00e2\u0128\u0133": 24935, + "\u0120Matrix": 24936, + "amins": 24937, + "\u01201200": 24938, + "\u0120Acqu": 24939, + "236": 24940, + "Document": 24941, + "\u0120Breaking": 24942, + "193": 24943, + "\u0120Subst": 24944, + "\u0120Roller": 24945, + "\u0120Properties": 24946, + "\u0120NI": 24947, + "tier": 24948, + "\u0120crushing": 24949, + "\u0120advocating": 24950, + "Furthermore": 24951, + "keepers": 24952, + "\u0120sexism": 24953, + "xd": 24954, + "\u0120caller": 24955, + "\u0120Sense": 24956, + "chieve": 24957, + "\u0120TF": 24958, + "\u0120fueled": 24959, + "\u0120reminiscent": 24960, + "\u0120obsess": 24961, + "urst": 24962, + "\u0120uphold": 24963, + "\u0120Fans": 24964, + "hetics": 24965, + "\u0120\u00e2\u0139": 24966, + "\u0120Bath": 24967, + "\u0120beverage": 24968, + "\u0120oscill": 24969, + "254": 24970, + "\u0120poles": 24971, + "\u0120gradual": 24972, + "\u0120exting": 24973, + "\u0120Suff": 24974, + "\u0120Suddenly": 24975, + "\u0120liking": 24976, + "\u01201949": 24977, + "unciation": 24978, + "amination": 24979, + "\u0120Omar": 24980, + "\u0120LV": 24981, + "\u0120Consequently": 24982, + "\u0120synthes": 24983, + "\u0120GIF": 24984, + "\u0120pains": 24985, + "\u0120interacting": 24986, + "uously": 24987, + "incre": 24988, + "\u0120rumor": 24989, + "\u0120Scientology": 24990, + "197": 24991, + "\u0120Zig": 24992, + "\u0120spelling": 24993, + "\u0120ASS": 24994, + "\u0120extingu": 24995, + "mson": 24996, + "\u0120gh": 24997, + "\u0120remarked": 24998, + "\u0120Strategic": 24999, + "\u0120MON": 25000, + "\u00e5\u00a5": 25001, + "gae": 25002, + "\u0120WHAT": 25003, + "Eric": 25004, + "\u0120Campus": 25005, + "\u0120methane": 25006, + "\u0120imagin": 25007, + "JUST": 25008, + "\u0120Alm": 25009, + "XT": 25010, + "iq": 25011, + "\u0120RSS": 25012, + "\u0120wrongdoing": 25013, + "atta": 25014, + "\u0120bigot": 25015, + "\u0120demonstrators": 25016, + "\u0120Calvin": 25017, + "\u0120Villa": 25018, + "\u0120membrane": 25019, + "\u0120Awesome": 25020, + "\u0120benefic": 25021, + "268": 25022, + "\u0120magnificent": 25023, + "\u0120Lots": 25024, + "Greg": 25025, + "\u0120Boris": 25026, + "\u0120detainees": 25027, + "\u0120Herman": 25028, + "\u0120whispered": 25029, + "\u0120awe": 25030, + "Professor": 25031, + "funding": 25032, + "\u0120physiological": 25033, + "\u0120Destruction": 25034, + "\u0120limb": 25035, + "\u0120manipulated": 25036, + "\u0120bubbles": 25037, + "\u0120pseud": 25038, + "\u0120hydra": 25039, + "\u0120Bristol": 25040, + "\u0120stellar": 25041, + "\u0120Expansion": 25042, + "\u0120Kell": 25043, + "\u0120Interestingly": 25044, + "\u0120mans": 25045, + "\u0120dragging": 25046, + "\u0120ecological": 25047, + "\u0120Fit": 25048, + "\u0120gent": 25049, + "\u0120benefited": 25050, + "\u0120Haiti": 25051, + "\u0120polyg": 25052, + "\u00e3\u0125\u0130": 25053, + "\u01202030": 25054, + "\u0120prow": 25055, + "\u0120reconstruction": 25056, + "\u0120wast": 25057, + "\u0120psychic": 25058, + "\u0120Greeks": 25059, + "Handler": 25060, + "162": 25061, + "\u0120Pulse": 25062, + "\u0120solicit": 25063, + "\u0120sys": 25064, + "\u0120influx": 25065, + "\u0120Gentle": 25066, + "percent": 25067, + "\u0120proliferation": 25068, + "\u0120taxable": 25069, + "\u0120disregard": 25070, + "\u0120escaping": 25071, + "\u0120ginger": 25072, + "\u0120withstand": 25073, + "\u0120devastated": 25074, + "\u0120Dew": 25075, + "series": 25076, + "\u0120injected": 25077, + "elaide": 25078, + "\u0120turnover": 25079, + "heat": 25080, + "\u013b\u0124": 25081, + "Happy": 25082, + "\u0120Silent": 25083, + "\u00e3\u0124\u0143": 25084, + "ivism": 25085, + "\u0120irrational": 25086, + "AMA": 25087, + "\u0120reef": 25088, + "rub": 25089, + "\u0120162": 25090, + "\u0120bankers": 25091, + "\u0120Ethics": 25092, + "vv": 25093, + "\u0120criticisms": 25094, + "Kn": 25095, + "186": 25096, + "Movie": 25097, + "\u0120Tories": 25098, + "\u0120nood": 25099, + "\u0120distortion": 25100, + "False": 25101, + "odore": 25102, + "\u0120tasty": 25103, + "Research": 25104, + "\u0120UID": 25105, + "-)": 25106, + "\u0120divorced": 25107, + "\u0120MU": 25108, + "\u0120Hayes": 25109, + "\u0120Isn": 25110, + "iani": 25111, + "\u0120HQ": 25112, + "\u0120\"#": 25113, + "ignant": 25114, + "\u0120traumatic": 25115, + "\u0120Ling": 25116, + "Hun": 25117, + "\u0120sabot": 25118, + "online": 25119, + "random": 25120, + "\u0120renamed": 25121, + "rared": 25122, + "KA": 25123, + "dead": 25124, + "\u00c3\u00a9t": 25125, + "\u0120Assistance": 25126, + "\u0120seaf": 25127, + "++++++++": 25128, + "\u0120seldom": 25129, + "\u0120Webb": 25130, + "\u0120boolean": 25131, + "ulet": 25132, + "\u0120refrain": 25133, + "\u0120DIY": 25134, + "rule": 25135, + "\u0120shutting": 25136, + "\u0120utilizing": 25137, + "loading": 25138, + "\u0120Param": 25139, + "coal": 25140, + "ooter": 25141, + "\u0120attracting": 25142, + "\u0120Dol": 25143, + "\u0120hers": 25144, + "agnetic": 25145, + "\u0120Reach": 25146, + "imo": 25147, + "\u0120discarded": 25148, + "\u0120Pip": 25149, + "015": 25150, + "\u00c3\u00bcr": 25151, + "\u0120mug": 25152, + "Imagine": 25153, + "COL": 25154, + "\u0120cursed": 25155, + "\u0120Shows": 25156, + "\u0120Curtis": 25157, + "\u0120Sachs": 25158, + "speaking": 25159, + "\u0120Vista": 25160, + "\u0120Framework": 25161, + "ongo": 25162, + "\u0120subreddit": 25163, + "\u0120crus": 25164, + "\u0120Oval": 25165, + "Row": 25166, + "growing": 25167, + "\u0120installment": 25168, + "\u0120glac": 25169, + "\u0120Advance": 25170, + "ECK": 25171, + "\u0120LGBTQ": 25172, + "LEY": 25173, + "\u0120acet": 25174, + "\u0120successive": 25175, + "\u0120Nicole": 25176, + "\u01201957": 25177, + "Quote": 25178, + "\u0120circumstance": 25179, + "ackets": 25180, + "\u0120142": 25181, + "ortium": 25182, + "\u0120guessed": 25183, + "\u0120Frame": 25184, + "\u0120perpetrators": 25185, + "\u0120Aviation": 25186, + "\u0120Bench": 25187, + "\u0120handc": 25188, + "Ap": 25189, + "\u01201956": 25190, + "259": 25191, + "rand": 25192, + "NetMessage": 25193, + "din": 25194, + "urtles": 25195, + "hig": 25196, + "\u0120VIII": 25197, + "ffiti": 25198, + "\u0120Swords": 25199, + "bial": 25200, + "\u0120kidnapping": 25201, + "device": 25202, + "\u0120barn": 25203, + "\u0120Eli": 25204, + "aucas": 25205, + "Send": 25206, + "Constructed": 25207, + "\u0120\u00c2\u00bd": 25208, + "\u0120needles": 25209, + "\u0120advertisements": 25210, + "\u0120vou": 25211, + "\u0120exhibited": 25212, + "\u0120Fortress": 25213, + "Ask": 25214, + "Berry": 25215, + "TYPE": 25216, + "\u0120cancers": 25217, + "umping": 25218, + "\u0120Territory": 25219, + "\u0120prud": 25220, + "\u0120nas": 25221, + "\u0120atheist": 25222, + "\u0120balances": 25223, + "\u00e3\u0123\u0141": 25224, + "\u0120Shawn": 25225, + "&&": 25226, + "\u0120landsc": 25227, + "\u0120RGB": 25228, + "\u0120petty": 25229, + "\u0120excellence": 25230, + "\u0120translations": 25231, + "\u0120parcel": 25232, + "\u0120Chev": 25233, + "East": 25234, + "\u0120Output": 25235, + "imi": 25236, + "\u0120ambient": 25237, + "\u0120Threat": 25238, + "\u0120villains": 25239, + "\u0120550": 25240, + "ICA": 25241, + "\u0120taller": 25242, + "\u0120leaking": 25243, + "cup": 25244, + "\u0120polish": 25245, + "\u0120infectious": 25246, + "\u0120KC": 25247, + "\u0120@@": 25248, + "background": 25249, + "\u0120bureaucracy": 25250, + "\u0120Sai": 25251, + "unless": 25252, + "itious": 25253, + "\u0120Skype": 25254, + "Atl": 25255, + "IDENT": 25256, + "008": 25257, + "\u0120hypocr": 25258, + "\u0120pitchers": 25259, + "\u0120guessing": 25260, + "\u0120FINAL": 25261, + "Between": 25262, + "\u0120villagers": 25263, + "\u0120252": 25264, + "fashion": 25265, + "\u0120Tunis": 25266, + "Beh": 25267, + "\u0120Exc": 25268, + "\u0120MID": 25269, + "288": 25270, + "\u0120Haskell": 25271, + "196": 25272, + "\u0120NOR": 25273, + "\u0120specs": 25274, + "\u0120invari": 25275, + "\u0120glut": 25276, + "\u0120Cars": 25277, + "\u0120impulse": 25278, + "\u0120honors": 25279, + "gel": 25280, + "\u0120jurisdictions": 25281, + "\u0120Bundle": 25282, + "ulas": 25283, + "California": 25284, + "\u0120Increase": 25285, + "\u0120pear": 25286, + "\u0120singles": 25287, + "\u0120cues": 25288, + "\u0120underwent": 25289, + "\u0120WS": 25290, + "\u0120exaggerated": 25291, + "\u0120dubious": 25292, + "\u0120flashing": 25293, + "LOG": 25294, + ")].": 25295, + "Journal": 25296, + "tg": 25297, + "Van": 25298, + "\u0120Istanbul": 25299, + "\u0120Insp": 25300, + "\u0120Franken": 25301, + "Draw": 25302, + "\u0120sadness": 25303, + "\u0120ironic": 25304, + "\u0120Fry": 25305, + "xc": 25306, + "\u0120164": 25307, + "isch": 25308, + "Way": 25309, + "\u0120Protestant": 25310, + "horn": 25311, + "\u0120unaff": 25312, + "\u0120Viv": 25313, + "illas": 25314, + "\u0120Productions": 25315, + "\u0120Hogan": 25316, + "\u0120perimeter": 25317, + "\u0120Sisters": 25318, + "\u0120spontaneous": 25319, + "\u0120downside": 25320, + "\u0120descendants": 25321, + "\u0120orn": 25322, + "worm": 25323, + "Japanese": 25324, + "\u01201955": 25325, + "\u0120151": 25326, + "\u0120Doing": 25327, + "elsen": 25328, + "umbles": 25329, + "\u0120radically": 25330, + "\u0120Drum": 25331, + "\u0120Bach": 25332, + "\u0120liabilities": 25333, + "\u0120OB": 25334, + "\u0120Elementary": 25335, + "\u0120meme": 25336, + "ynes": 25337, + "\u0120fingerprint": 25338, + "\u0120Grab": 25339, + "\u0120undertake": 25340, + "Members": 25341, + "\u0120Reader": 25342, + "\u0120Sims": 25343, + "god": 25344, + "\u0120hypothetical": 25345, + "scient": 25346, + "\u0120AJ": 25347, + "\u0120charism": 25348, + "\u0120admissions": 25349, + "\u0120Missile": 25350, + "trade": 25351, + "\u0120exercising": 25352, + "\u0120Background": 25353, + "Written": 25354, + "\u0120vocals": 25355, + "whether": 25356, + "\u0120vi": 25357, + "\u0120Winner": 25358, + "\u0120litter": 25359, + "\u0120Shooting": 25360, + "STEM": 25361, + "\u00e3\u0124\u00a1": 25362, + "\u0120AFL": 25363, + "\u0120variability": 25364, + "\u0120eats": 25365, + "\u0120DPS": 25366, + "brow": 25367, + "\u0120elephants": 25368, + "\u0120strat": 25369, + "\u0120\u00c5": 25370, + "\u0120settlers": 25371, + "Matthew": 25372, + "\u0120inadvert": 25373, + "HI": 25374, + "\u0120IMF": 25375, + "\u0120Goal": 25376, + "\u0120nerves": 25377, + "Johnson": 25378, + "eye": 25379, + "ablishment": 25380, + "Thursday": 25381, + "BILITY": 25382, + "Had": 25383, + "amoto": 25384, + "hetamine": 25385, + "eps": 25386, + "\u0120mitochond": 25387, + "\u0120compressed": 25388, + "\u0120Trevor": 25389, + "\u0120Animals": 25390, + "Tool": 25391, + "Lock": 25392, + "\u0120tweak": 25393, + "\u0120pinch": 25394, + "\u0120cancellation": 25395, + "Pot": 25396, + "\u0120focal": 25397, + "\u0120Astron": 25398, + "173": 25399, + "\u0120ASC": 25400, + "\u0120OTHER": 25401, + "umni": 25402, + "\u0120demise": 25403, + "dl": 25404, + "\u00d9\u0127": 25405, + "Semitism": 25406, + "\u0120cracking": 25407, + "\u0120collaborative": 25408, + "\u0120explores": 25409, + "sql": 25410, + "\u0120herbs": 25411, + "\u0120configurations": 25412, + "mis": 25413, + "\u0120Result": 25414, + "acey": 25415, + "\u0120Smoke": 25416, + "\u0120sanct": 25417, + "elia": 25418, + "\u0120degener": 25419, + "\u0120deepest": 25420, + "\u0120screamed": 25421, + "\u0120nap": 25422, + "Software": 25423, + "\u0120STAR": 25424, + "EF": 25425, + "\u0120Xin": 25426, + "sponsored": 25427, + "manship": 25428, + "233": 25429, + "\u0120primaries": 25430, + "\u0120filtering": 25431, + "\u0120assemble": 25432, + "mil": 25433, + "\u0120Myers": 25434, + "bows": 25435, + "\u0120punched": 25436, + "Mic": 25437, + "\u0120innovations": 25438, + "\u0120func": 25439, + "ando": 25440, + "\u0120fracking": 25441, + "\u0120Vul": 25442, + "\u00d0\u00be\u00d0": 25443, + "oshop": 25444, + "\u0120Immun": 25445, + "\u0120settling": 25446, + "\u0120adolescents": 25447, + "\u0120rebuilding": 25448, + "\u0120transforming": 25449, + "\u0120parole": 25450, + "\u0120harbor": 25451, + "\u0120booking": 25452, + "otional": 25453, + "ongevity": 25454, + "\u0120Yo": 25455, + "bug": 25456, + "\u0120emerges": 25457, + "\u0120Methods": 25458, + "\u0120Chu": 25459, + "Pres": 25460, + "\u0120Dungeons": 25461, + "\u0120trailing": 25462, + "\u0120Rum": 25463, + "\u0120Hugh": 25464, + "\u00e5\u00a4\u00a9": 25465, + "\u0120Era": 25466, + "\u0120Battles": 25467, + "Results": 25468, + "\u0120Trading": 25469, + "\u0120versa": 25470, + "css": 25471, + "axies": 25472, + "heet": 25473, + "\u0120greed": 25474, + "1989": 25475, + "\u0120gardens": 25476, + "\u0120contingent": 25477, + "Park": 25478, + "\u0120Leafs": 25479, + "hook": 25480, + "robe": 25481, + "\u0120diplomacy": 25482, + "\u0120Fuel": 25483, + "\u0120Invasion": 25484, + "\u0120upgrading": 25485, + "Male": 25486, + "\u0120elic": 25487, + "\u0120relentless": 25488, + "\u0120Covenant": 25489, + "apesh": 25490, + "\u0120Trop": 25491, + "Ty": 25492, + "production": 25493, + "arty": 25494, + "\u0120punches": 25495, + "ako": 25496, + "cyclopedia": 25497, + "\u0120Rabbit": 25498, + "\u0120HDMI": 25499, + "\u0120141": 25500, + "\u0120foil": 25501, + "ItemImage": 25502, + "\u0120FG": 25503, + "\u0120implementations": 25504, + "\u0120Pom": 25505, + "ixtures": 25506, + "\u0120await": 25507, + "\u0120330": 25508, + "amus": 25509, + "\u0120umbrella": 25510, + "\u0120foresee": 25511, + "separ": 25512, + "\u0120circumcision": 25513, + "\u0120peripheral": 25514, + "Say": 25515, + "\u0120Expert": 25516, + "Inc": 25517, + "\u0120withdrew": 25518, + "\u0120Anders": 25519, + "fried": 25520, + "\u0120radioactive": 25521, + "\u0120Opening": 25522, + "\u0120boarding": 25523, + "\u0120ND": 25524, + "\u0120overthrow": 25525, + "Activ": 25526, + "WP": 25527, + "\u0120Acts": 25528, + "\u00d7\u013b": 25529, + "\u0120motions": 25530, + "vic": 25531, + "\u0120Mighty": 25532, + "\u0120Defender": 25533, + "aer": 25534, + "\u0120thankful": 25535, + "\u0120Killing": 25536, + "\u0120Bris": 25537, + "moil": 25538, + "\u0120predicting": 25539, + "266": 25540, + "choice": 25541, + "\u0120killers": 25542, + "\u0120incub": 25543, + "\u0120Chest": 25544, + "athering": 25545, + "\u0120proclaimed": 25546, + "flower": 25547, + "ossom": 25548, + "umbledore": 25549, + "\u0120Cycling": 25550, + "\u0120Occupy": 25551, + "AGES": 25552, + "Pen": 25553, + "\u0120Yug": 25554, + "\u0120packaged": 25555, + "\u0120heightened": 25556, + "cot": 25557, + "stack": 25558, + "Cond": 25559, + "\u0120stamps": 25560, + "mage": 25561, + "\u0120persuaded": 25562, + "\u0120ensl": 25563, + "\u0120Cardinal": 25564, + "\u0120solitary": 25565, + "\u0120possessing": 25566, + "\u0120Cork": 25567, + "\u0120evid": 25568, + "\u0120Tay": 25569, + "\u0120blues": 25570, + "\u0120extremism": 25571, + "\u0120lunar": 25572, + "\u0120clown": 25573, + "Techn": 25574, + "\u0120festivals": 25575, + "\u0120PvP": 25576, + "\u0120Lar": 25577, + "\u0120consequently": 25578, + "present": 25579, + "\u0120someday": 25580, + "\u00e7\u0130\u012d": 25581, + "\u0120Meteor": 25582, + "\u0120touring": 25583, + "culture": 25584, + "\u0120beaches": 25585, + "Ship": 25586, + "cause": 25587, + "\u0120Flood": 25588, + "\u00e3\u0125\u00af": 25589, + "\u0120purity": 25590, + "those": 25591, + "\u0120emission": 25592, + "bolt": 25593, + "\u0120chord": 25594, + "\u0120Scripture": 25595, + "Lu": 25596, + "\u0120${": 25597, + "created": 25598, + "Others": 25599, + "258": 25600, + "\u0120elemental": 25601, + "\u0120annoyed": 25602, + "\u0120AE": 25603, + "dan": 25604, + "\u0120Sag": 25605, + "Researchers": 25606, + "\u0120fairy": 25607, + "\u00e2\u0122\u0135\u00e2\u0122\u0135": 25608, + "============": 25609, + "Smart": 25610, + "GGGG": 25611, + "\u0120skeletons": 25612, + "\u0120pupils": 25613, + "linked": 25614, + "\u0120urgency": 25615, + "enabled": 25616, + "\u0120Fuck": 25617, + "\u0120councill": 25618, + "rab": 25619, + "UAL": 25620, + "TI": 25621, + "\u0120lifes": 25622, + "\u0120confessed": 25623, + "Bug": 25624, + "\u0120harmon": 25625, + "\u0120CONFIG": 25626, + "\u0120Neutral": 25627, + "Double": 25628, + "\u0120staple": 25629, + "\u0120SHA": 25630, + "British": 25631, + "\u0120SNP": 25632, + "ATOR": 25633, + "oco": 25634, + "\u0120swinging": 25635, + "gex": 25636, + "oleon": 25637, + "plain": 25638, + "\u0120Missing": 25639, + "\u0120Trophy": 25640, + "vari": 25641, + "ranch": 25642, + "\u0120301": 25643, + "440": 25644, + "0000000000000000": 25645, + "\u0120restoring": 25646, + "\u0120haul": 25647, + "ucing": 25648, + "nerg": 25649, + "\u0120futures": 25650, + "\u0120strategist": 25651, + "question": 25652, + "\u0120lateral": 25653, + "\u0120Bard": 25654, + "\u0120sor": 25655, + "\u0120Rhodes": 25656, + "\u0120Downtown": 25657, + "?????-": 25658, + "\u0120Lit": 25659, + "\u0120Bened": 25660, + "\u0120coil": 25661, + "street": 25662, + "\u0120Portal": 25663, + "FILE": 25664, + "\u0120Gru": 25665, + "*,": 25666, + "231": 25667, + "neum": 25668, + "\u0120sucked": 25669, + "\u0120rapper": 25670, + "\u0120tendencies": 25671, + "\u0120Lauren": 25672, + "cellaneous": 25673, + "267": 25674, + "\u0120browse": 25675, + "\u0120overc": 25676, + "header": 25677, + "oise": 25678, + "\u0120beet": 25679, + "\u0120Gle": 25680, + "Stay": 25681, + "\u0120mum": 25682, + "\u0120typed": 25683, + "\u0120discounts": 25684, + "Talk": 25685, + "\u0120Og": 25686, + "existing": 25687, + "\u0120Sell": 25688, + "uph": 25689, + "CI": 25690, + "\u0120Austrian": 25691, + "\u0120Warm": 25692, + "\u0120dismissal": 25693, + "\u0120averages": 25694, + "camera": 25695, + "\u0120allegiance": 25696, + "LAN": 25697, + "=\"#": 25698, + "\u0120commentators": 25699, + "\u0120Setting": 25700, + "\u0120Midwest": 25701, + "\u0120pharmac": 25702, + "\u0120EXP": 25703, + "\u0120stainless": 25704, + "Chicago": 25705, + "\u0120tan": 25706, + "244": 25707, + "\u0120countryside": 25708, + "\u0120Vac": 25709, + "295": 25710, + "\u0120pinned": 25711, + "\u0120crises": 25712, + "\u0120standardized": 25713, + "Task": 25714, + "\u0120Jail": 25715, + "\u0120Docker": 25716, + "colored": 25717, + "forth": 25718, + "\"},": 25719, + "\u0120patrons": 25720, + "\u0120spice": 25721, + "\u0120mourn": 25722, + "\u0120Mood": 25723, + "\u0120laundry": 25724, + "\u0120equip": 25725, + "\u0120Mole": 25726, + "yll": 25727, + "\u0120THC": 25728, + "nation": 25729, + "\u0120Sherlock": 25730, + "\u0120issu": 25731, + "\u0120Kre": 25732, + "\u0120Americas": 25733, + "\u0120AAA": 25734, + "\u0120systematically": 25735, + "\u0120contra": 25736, + "\u0120Sally": 25737, + "\u0120rationale": 25738, + "\u0120carriage": 25739, + "\u0120peaks": 25740, + "\u0120contradiction": 25741, + "ensation": 25742, + "\u0120Failure": 25743, + "\u0120props": 25744, + "\u0120namespace": 25745, + "\u0120cove": 25746, + "fields": 25747, + "\u00e3\u0124\u012d": 25748, + "\u0120wool": 25749, + "\u0120Catch": 25750, + "\u0120presumed": 25751, + "\u0120Diana": 25752, + "ragon": 25753, + "igi": 25754, + "\u0120hamm": 25755, + "\u0120stunt": 25756, + "\u0120GUI": 25757, + "\u0120Observatory": 25758, + "\u0120Shore": 25759, + "\u0120smells": 25760, + "annah": 25761, + "\u0120cockpit": 25762, + "\u0120Duterte": 25763, + "850": 25764, + "\u0120oppressed": 25765, + "breaker": 25766, + "\u0120Contribut": 25767, + "\u0120Peru": 25768, + "\u0120Monsanto": 25769, + "\u0120Attempt": 25770, + "\u0120commanding": 25771, + "\u0120fridge": 25772, + "\u0120Rin": 25773, + "\u0120Chess": 25774, + "uality": 25775, + "\u0120ol": 25776, + "Republican": 25777, + "\u0120Glory": 25778, + "\u0120WIN": 25779, + ".......": 25780, + "agent": 25781, + "reading": 25782, + "\u0120inh": 25783, + "Jones": 25784, + "\u0120clicks": 25785, + "alan": 25786, + "\u0120[];": 25787, + "\u0120Majesty": 25788, + "\u0120Ced": 25789, + "opus": 25790, + "atel": 25791, + "\u00c3\u00aa": 25792, + "ARC": 25793, + "\u0120Ecuador": 25794, + "\u00e3\u0125\u0142": 25795, + "\u0120Kuro": 25796, + "\u0120rituals": 25797, + "\u0120captive": 25798, + "\u0120ounce": 25799, + "\u0120disagreement": 25800, + "\u0120slog": 25801, + "fuel": 25802, + "Pet": 25803, + "Mail": 25804, + "\u0120exercised": 25805, + "\u0120solic": 25806, + "\u0120rainfall": 25807, + "\u0120devotion": 25808, + "\u0120Assessment": 25809, + "\u0120robotic": 25810, + "options": 25811, + "\u0120RP": 25812, + "\u0120Families": 25813, + "\u0120Flames": 25814, + "\u0120assignments": 25815, + "007": 25816, + "akedown": 25817, + "\u0120vocabulary": 25818, + "Reilly": 25819, + "\u0120caval": 25820, + "gars": 25821, + "\u0120suppressed": 25822, + "\u0120SET": 25823, + "\u0120Johns": 25824, + "\u0120warp": 25825, + "broken": 25826, + "\u0120statues": 25827, + "\u0120advocated": 25828, + "\u0120275": 25829, + "\u0120peril": 25830, + "omorph": 25831, + "\u0120Femin": 25832, + "perfect": 25833, + "\u0120hatch": 25834, + "Lib": 25835, + "512": 25836, + "\u0120lifelong": 25837, + "313": 25838, + "\u0120cheeks": 25839, + "\u0120numbered": 25840, + "\u0120Mug": 25841, + "Body": 25842, + "ravel": 25843, + "Weight": 25844, + "\u0120Jak": 25845, + "\u0120Heath": 25846, + "\u0120kissing": 25847, + "\u0120JUST": 25848, + "\u0120waving": 25849, + "upload": 25850, + "\u0120insider": 25851, + "\u0120Progressive": 25852, + "\u0120Filter": 25853, + "tta": 25854, + "\u0120Beam": 25855, + "\u0120violently": 25856, + "ipation": 25857, + "\u0120skepticism": 25858, + "\u01201918": 25859, + "\u0120Annie": 25860, + "\u0120SI": 25861, + "\u0120genetics": 25862, + "\u0120onboard": 25863, + "atl": 25864, + "\u0120Friedman": 25865, + "\u0120Bri": 25866, + "ceptive": 25867, + "\u0120pirate": 25868, + "\u0120Reporter": 25869, + "278": 25870, + "\u0120mythology": 25871, + "\u0120eclipse": 25872, + "\u0120skins": 25873, + "\u0120glyph": 25874, + "ingham": 25875, + "Files": 25876, + "Cour": 25877, + "women": 25878, + "\u0120regimes": 25879, + "\u0120photographed": 25880, + "Kat": 25881, + "\u0120MAX": 25882, + "Officials": 25883, + "\u0120unexpectedly": 25884, + "\u0120impressions": 25885, + "Front": 25886, + ";;;;;;;;": 25887, + "\u0120supremacy": 25888, + "\u0120sang": 25889, + "\u0120aggravated": 25890, + "\u0120abruptly": 25891, + "\u0120Sector": 25892, + "\u0120excuses": 25893, + "\u0120costing": 25894, + "idepress": 25895, + "Stack": 25896, + "\u0120RNA": 25897, + "obil": 25898, + "\u0120ghosts": 25899, + "ldon": 25900, + "atibility": 25901, + "Topics": 25902, + "\u0120reimburse": 25903, + "\u0120HM": 25904, + "\u0120Deg": 25905, + "\u0120thief": 25906, + "yet": 25907, + "ogenesis": 25908, + "leaning": 25909, + "\u0120Kol": 25910, + "\u0120Basketball": 25911, + "\u0120fi": 25912, + "\u0120Seeing": 25913, + "\u0120recycling": 25914, + "\u0120[-": 25915, + "Congress": 25916, + "\u0120lectures": 25917, + "Psy": 25918, + "\u0120nep": 25919, + "\u0120maid": 25920, + "\u0120oriented": 25921, + "AX": 25922, + "\u0120respectful": 25923, + "rene": 25924, + "flush": 25925, + "\u0120Unloaded": 25926, + "request": 25927, + "grid": 25928, + "\u0120Alternatively": 25929, + "\u0120Hugo": 25930, + "\u0120decree": 25931, + "\u0120Buddhism": 25932, + "andum": 25933, + "Android": 25934, + "\u0120Congo": 25935, + "\u0120Joyce": 25936, + "\u0120acknowledging": 25937, + "hesive": 25938, + "\u0120Tomorrow": 25939, + "\u0120Hiro": 25940, + "thren": 25941, + "\u0120Maced": 25942, + "\u0120hoax": 25943, + "\u0120Increased": 25944, + "\u0120Pradesh": 25945, + "Wild": 25946, + "______": 25947, + "161": 25948, + "\u0120aunt": 25949, + "\u0120distributing": 25950, + "\u0120Tucker": 25951, + "\u0120SSL": 25952, + "\u0120Wolves": 25953, + "Building": 25954, + "oult": 25955, + "\u0120Luo": 25956, + "\u0120Yas": 25957, + "\u0120Spir": 25958, + "\u0120Shape": 25959, + "\u0120Cambod": 25960, + "\u0120IPv": 25961, + "\u0120ml": 25962, + "\u0120extrad": 25963, + "390": 25964, + "\u0120Penny": 25965, + "dream": 25966, + "\u0120stationed": 25967, + "optional": 25968, + "eworthy": 25969, + ".": 26700, + "\u0120Workshop": 26701, + "\u0120Retail": 26702, + "\u0120Avatar": 26703, + "625": 26704, + "Na": 26705, + "\u0120VC": 26706, + "\u0120Secure": 26707, + "MY": 26708, + "1988": 26709, + "ossip": 26710, + "\u0120prostate": 26711, + "\u0120unden": 26712, + "\u0120gamer": 26713, + "\u0120Contents": 26714, + "\u0120Warhammer": 26715, + "\u0120Sentinel": 26716, + "310": 26717, + "\u0120segregation": 26718, + "\u0120Flex": 26719, + "\u0120MAY": 26720, + "\u0120drills": 26721, + "\u0120Drugs": 26722, + "Islamic": 26723, + "\u0120spur": 26724, + "\u0120cafe": 26725, + "\u0120imaginary": 26726, + "\u0120guiding": 26727, + "\u0120swings": 26728, + "\u0120Theme": 26729, + "oby": 26730, + "\u0120nud": 26731, + "\u0120begging": 26732, + "\u0120strongh": 26733, + "\u0120rejecting": 26734, + "\u0120pedestrians": 26735, + "\u0120Prospect": 26736, + "Rare": 26737, + "sle": 26738, + "\u0120concessions": 26739, + "\u0120Constitutional": 26740, + "\u0120beams": 26741, + "\u0120fibers": 26742, + "poon": 26743, + "\u0120instincts": 26744, + "property": 26745, + "\u0120BIG": 26746, + "Sanders": 26747, + "imates": 26748, + "\u0120coating": 26749, + "\u0120corpses": 26750, + "\u0120TRUE": 26751, + "checked": 26752, + "\u0120166": 26753, + "Ash": 26754, + "\u0120JS": 26755, + "\u0120Fiction": 26756, + "\u0120communal": 26757, + "\u0120energetic": 26758, + "oooooooo": 26759, + "\u0120nowadays": 26760, + "ILD": 26761, + "ibo": 26762, + "\u0120SUV": 26763, + "Ren": 26764, + "\u0120dwelling": 26765, + "Silver": 26766, + "\u0120tally": 26767, + "\u0120Moving": 26768, + "\u0120coward": 26769, + "\u0120generals": 26770, + "\u0120horns": 26771, + "\u0120circulated": 26772, + "\u0120robbed": 26773, + "\u0120Unlimited": 26774, + "\u0120harassed": 26775, + "\u0120inhibit": 26776, + "\u0120composer": 26777, + "\u0120Spotify": 26778, + "\u0120spreads": 26779, + "364": 26780, + "\u0120suicidal": 26781, + "\u0120noises": 26782, + "\u0120Stur": 26783, + "\u0120saga": 26784, + "\u0120Kag": 26785, + "iso": 26786, + "\u0120theoretically": 26787, + "Money": 26788, + "\u0120similarity": 26789, + "\u0120sliced": 26790, + "utils": 26791, + "inges": 26792, + "\"-": 26793, + "\u0120anth": 26794, + "\u0120imped": 26795, + "Module": 26796, + "Throughout": 26797, + "\u0120menus": 26798, + "committee": 26799, + "andi": 26800, + "obj": 26801, + "inav": 26802, + "fired": 26803, + "\u0120Abdullah": 26804, + "\u0120undead": 26805, + "\u0120fonts": 26806, + "Hold": 26807, + "ENG": 26808, + "\u0120sustainability": 26809, + "\u0120flick": 26810, + "\u0120razor": 26811, + "\u0120Fest": 26812, + "\u0120Characters": 26813, + "\u0120wording": 26814, + "\u0120populist": 26815, + "\u0120criticizing": 26816, + "\u0120muse": 26817, + "vine": 26818, + "\u0120cardboard": 26819, + "\u0120kindly": 26820, + "\u0120fringe": 26821, + "\u0120Theft": 26822, + "icultural": 26823, + "\u0120governors": 26824, + "\u0120\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 26825, + "\u0120163": 26826, + "\u0120timeout": 26827, + "\u0120Auth": 26828, + "Children": 26829, + "AU": 26830, + "\u0120redemption": 26831, + "\u0120Alger": 26832, + "\u01201914": 26833, + "\u0120waved": 26834, + "\u0120astronauts": 26835, + "ograms": 26836, + "\u0120swamp": 26837, + "\u0120Finnish": 26838, + "\u0120candle": 26839, + "\u0120tonnes": 26840, + "utm": 26841, + "\u0120ray": 26842, + "\u0120spun": 26843, + "\u0120fearful": 26844, + "articles": 26845, + "\u0120caus": 26846, + "orically": 26847, + "\u0120Requires": 26848, + "\u0120Gol": 26849, + "\u0120pope": 26850, + "\u0120inaugural": 26851, + "\u0120gle": 26852, + "ADA": 26853, + "\u0120ISIL": 26854, + "\u0120Offensive": 26855, + "\u0120watchdog": 26856, + "\u0120balcon": 26857, + "entity": 26858, + "\u0120Hoo": 26859, + "\u0120gallon": 26860, + "ACC": 26861, + "\u0120doubling": 26862, + "\u0120implication": 26863, + "\u0120Sight": 26864, + "\u0120doctr": 26865, + "-------": 26866, + "\u0120\\\\": 26867, + "\u0120malt": 26868, + "Roll": 26869, + "\u0120\u00e2\u012b\u00a5": 26870, + "\u0120recap": 26871, + "adding": 26872, + "uces": 26873, + "\u0120Bend": 26874, + "figure": 26875, + "\u0120turkey": 26876, + "\u0120societal": 26877, + "\u0120Tickets": 26878, + "\u0120commercially": 26879, + "\u0120spicy": 26880, + "\u0120216": 26881, + "\u0120Ramp": 26882, + "\u0120superiority": 26883, + "\u00c3\u00af": 26884, + "\u0120Tracker": 26885, + "Carl": 26886, + "\u0120Coy": 26887, + "\u0120Patriot": 26888, + "\u0120consulted": 26889, + "\u0120listings": 26890, + "\u0120slew": 26891, + "reenshot": 26892, + "\u0120Gone": 26893, + "\u0120[...]": 26894, + "309": 26895, + "\u0120hottest": 26896, + "\u00d8\u00b1": 26897, + "\u0120rocky": 26898, + "\u0120Diaz": 26899, + "\u0120massage": 26900, + "\u0120paraly": 26901, + "\u0120pony": 26902, + "Az": 26903, + "\u0120cartridge": 26904, + "\u0120NZ": 26905, + "\u0120snack": 26906, + "\u0120Lamar": 26907, + "plement": 26908, + "\u0120Leslie": 26909, + "\u0120mater": 26910, + "\u0120snipp": 26911, + "246": 26912, + "\u0120jointly": 26913, + "\u0120Brisbane": 26914, + "\u0120iPod": 26915, + "\u0120pumping": 26916, + "\u0120goat": 26917, + "\u0120Sharon": 26918, + "ealing": 26919, + "\u0120coron": 26920, + "\u0120anomal": 26921, + "rahim": 26922, + "\u0120Connection": 26923, + "\u0120sculpture": 26924, + "\u0120scheduling": 26925, + "\u0120Daddy": 26926, + "athing": 26927, + "\u0120eyebrows": 26928, + "\u0120curved": 26929, + "\u0120sentiments": 26930, + "\u0120drafting": 26931, + "Drop": 26932, + "([": 26933, + "\u0120nominal": 26934, + "\u0120Leadership": 26935, + "\u0120Grow": 26936, + "\u0120176": 26937, + "\u0120constructive": 26938, + "ivation": 26939, + "\u0120corrupted": 26940, + "gerald": 26941, + "\u0120Cros": 26942, + "\u0120Chester": 26943, + "\u0120Lap": 26944, + "\u00e3\u0123\u00aa": 26945, + "OTH": 26946, + "DATA": 26947, + "\u0120almond": 26948, + "probably": 26949, + "Imp": 26950, + "\u0120feast": 26951, + "\u0120Warcraft": 26952, + "Flor": 26953, + "\u0120checkpoint": 26954, + "\u0120transcription": 26955, + "\u0120204": 26956, + "\u0120tweaks": 26957, + "\u0120relieve": 26958, + "Science": 26959, + "\u0120performer": 26960, + "Zone": 26961, + "\u0120turmoil": 26962, + "igated": 26963, + "hibit": 26964, + "\u0120Cafe": 26965, + "themed": 26966, + "\u0120fluor": 26967, + "bench": 26968, + "\u0120decom": 26969, + "\u0120Unt": 26970, + "\u0120Barrett": 26971, + "\u0120Facts": 26972, + "\u0120tasting": 26973, + "\u0120PTSD": 26974, + "\u0120Seal": 26975, + "\u0120Judaism": 26976, + "\u0120Dynamic": 26977, + "\u0120Cors": 26978, + "Ve": 26979, + "\u0120Ming": 26980, + "\u0120Transform": 26981, + "von": 26982, + "\u0120Defenders": 26983, + "\u0120Tactical": 26984, + "\u0120Von": 26985, + "\u0120Univers": 26986, + "\u0120distorted": 26987, + "\u0120Breath": 26988, + "?'\"": 26989, + "\u0120agon": 26990, + "\u0120Deadly": 26991, + "\u0120lan": 26992, + "\u0120Cycle": 26993, + "orned": 26994, + "\u0120reliably": 26995, + "\u0120glor": 26996, + "\u0120Monkey": 26997, + "\u00e3\u0125\u00a1": 26998, + "\u0120adren": 26999, + "\u0120microwave": 27000, + "\u0120Alban": 27001, + "ircraft": 27002, + "digit": 27003, + "smart": 27004, + "\u0120Dread": 27005, + "\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af": 27006, + "{{": 27007, + "\u0120Rochester": 27008, + "\u0120simplified": 27009, + "\u0120inflicted": 27010, + "\u0120takeover": 27011, + "\u0120yourselves": 27012, + "aditional": 27013, + "\u0120muscular": 27014, + "KS": 27015, + "\u0120ingen": 27016, + "Tax": 27017, + "\u0120Feature": 27018, + "277": 27019, + "\u0120cruc": 27020, + "\u0120crate": 27021, + "\u0120unidentified": 27022, + "\u0120acclaimed": 27023, + "\u0120Manga": 27024, + "\u0120Frances": 27025, + "\u0120Nepal": 27026, + "\u0120Gerald": 27027, + "\u0120Kuwait": 27028, + "\u0120slain": 27029, + "\u0120Heb": 27030, + "\u0120Goku": 27031, + "\u00e3\u0123\u00ae\u00e6": 27032, + "286": 27033, + "Mrs": 27034, + "\u0120Cody": 27035, + "\u0120Sanctuary": 27036, + "016": 27037, + "\u0120dismant": 27038, + "\u0120dataset": 27039, + "\u0120Hond": 27040, + "buck": 27041, + "\u0120Patterson": 27042, + "\u0120palette": 27043, + "\u0120GD": 27044, + "icol": 27045, + "\u0120Lodge": 27046, + "\u0120planetary": 27047, + "akin": 27048, + "\u0120Registered": 27049, + "abwe": 27050, + "\u0120Petersburg": 27051, + "\u0120hailed": 27052, + "\u0120Piece": 27053, + "Sche": 27054, + "\u0120DOJ": 27055, + "\u0120enumer": 27056, + "181": 27057, + "\u0120Observer": 27058, + "\u0120Bold": 27059, + "founded": 27060, + "commerce": 27061, + "\u0120exploits": 27062, + "\u0120Finding": 27063, + "URN": 27064, + "\u0120Sne": 27065, + "\u0120Acid": 27066, + "ayette": 27067, + "\u0120Values": 27068, + "\u0120drastic": 27069, + "\u0120architectural": 27070, + "\u0120\".": 27071, + "\u00d7\u0137": 27072, + "umped": 27073, + "\u0120wrapping": 27074, + "\u0120widow": 27075, + "\u0120Slayer": 27076, + "lace": 27077, + "once": 27078, + "Germany": 27079, + "avoid": 27080, + "\u0120temples": 27081, + "PAR": 27082, + "\u00c3\u00b4": 27083, + "\u0120Lucifer": 27084, + "\u0120Flickr": 27085, + "lov": 27086, + "forces": 27087, + "\u0120scouting": 27088, + "\u0120louder": 27089, + "tesy": 27090, + "\u0120beforehand": 27091, + "\u00c4\u0135": 27092, + "\u0120Neon": 27093, + "\u0120Wol": 27094, + "\u0120Typically": 27095, + "\u0120Politico": 27096, + "-+-+": 27097, + "\u0120builder": 27098, + "\u0120derive": 27099, + "Kill": 27100, + "\u0120poker": 27101, + "\u0120ambiguous": 27102, + "\u0120lifts": 27103, + "\u0120cyt": 27104, + "\u0120ribs": 27105, + "oodle": 27106, + "\u0120Sounds": 27107, + "hair": 27108, + "\u0120Syndrome": 27109, + "tf": 27110, + "\u0120proportional": 27111, + "uid": 27112, + "\u0120pertaining": 27113, + "\u0120Kindle": 27114, + "\u0120Negro": 27115, + "\u0120reiterated": 27116, + "\u0120Tonight": 27117, + "oths": 27118, + "\u0120Cornell": 27119, + "\u0120owing": 27120, + "\u0120208": 27121, + "elfare": 27122, + "ocating": 27123, + "\u0120Birds": 27124, + "Subscribe": 27125, + "\u0120essays": 27126, + "\u0120burdens": 27127, + "\u0120illustrations": 27128, + "arious": 27129, + "ERAL": 27130, + "\u0120Calcul": 27131, + "\u0120xen": 27132, + "\u0120LinkedIn": 27133, + "\u0120Jung": 27134, + "\u0120redesign": 27135, + "Connor": 27136, + "296": 27137, + "\u0120reversal": 27138, + "\u0120Adelaide": 27139, + "\u0120LL": 27140, + "\u0120sinking": 27141, + "\u0120gum": 27142, + "USH": 27143, + "capt": 27144, + "\u0120Grimm": 27145, + "\u0120footsteps": 27146, + "\u0120CBD": 27147, + "ispers": 27148, + "\u0120prose": 27149, + "Wednesday": 27150, + "\u0120Movies": 27151, + "edin": 27152, + "\u0120overturned": 27153, + "\u0120contentious": 27154, + "USB": 27155, + "~~~~~~~~~~~~~~~~": 27156, + "\u0120Copper": 27157, + "\u0120pointless": 27158, + "NV": 27159, + "values": 27160, + "olphin": 27161, + "dain": 27162, + "\u0120deposited": 27163, + "\u0120GW": 27164, + "\u0120preceded": 27165, + "\u0120Cla": 27166, + "\u0120Golem": 27167, + "\u0120Nim": 27168, + "\u0120\u00ce\u00b2": 27169, + "\u0120Engineers": 27170, + "middle": 27171, + "\u0120flatt": 27172, + "operative": 27173, + "\u0120councils": 27174, + "imbabwe": 27175, + "elin": 27176, + "\u0120stressful": 27177, + "\u0120LD": 27178, + "\u0120resh": 27179, + "lake": 27180, + "\u0120wheelchair": 27181, + "\u0120Alternative": 27182, + "\u0120optimize": 27183, + "operation": 27184, + "\u0120peek": 27185, + "\u0120oneself": 27186, + "igil": 27187, + "\u0120transitions": 27188, + "opathy": 27189, + "blank": 27190, + "\u0120169": 27191, + "171": 27192, + "________________________________________________________________": 27193, + "\u0120laundering": 27194, + "Enc": 27195, + "\u0120DEC": 27196, + "\u0120workouts": 27197, + "\u0120spikes": 27198, + "\u0120dinosaurs": 27199, + "\u0120discriminatory": 27200, + "Pool": 27201, + "Rather": 27202, + "385": 27203, + "RNA": 27204, + "testers": 27205, + "eto": 27206, + "\u0120Identity": 27207, + "\u0120vein": 27208, + "\u0120Burton": 27209, + "\u0120arcade": 27210, + "420": 27211, + "Ultimately": 27212, + "\u0120Sadly": 27213, + "\u00c3\u00b0": 27214, + "pill": 27215, + "\u0120cubic": 27216, + "\u0120Spectrum": 27217, + "these": 27218, + "states": 27219, + "\u0120unofficial": 27220, + "hawks": 27221, + "\u0120EVERY": 27222, + "\u0120rainbow": 27223, + "\u0120incarceration": 27224, + "anding": 27225, + "\u0120syll": 27226, + "\u0120Everton": 27227, + "\u0120179": 27228, + "\u0120Serbia": 27229, + "\u0120189": 27230, + "meter": 27231, + "\u0120Mickey": 27232, + "\u0120antiqu": 27233, + "\u0120factual": 27234, + "neck": 27235, + "\u0120Nare": 27236, + "norm": 27237, + "must": 27238, + "\u0120highways": 27239, + "\u0120glam": 27240, + "\u0120dividing": 27241, + "\u0120Squadron": 27242, + "\u0120Martha": 27243, + "\u0120births": 27244, + "Cover": 27245, + "////////////////": 27246, + "\u0120Wong": 27247, + "Phot": 27248, + "\u0120ALS": 27249, + "rio": 27250, + "\u0120Nonetheless": 27251, + "\u0120Lemon": 27252, + "\u0120206": 27253, + "\u0120EE": 27254, + "\u0120derivative": 27255, + "\u0120WWII": 27256, + "vote": 27257, + "\u0120therein": 27258, + "\u0120separating": 27259, + "446": 27260, + "sync": 27261, + "\u0120Streets": 27262, + "\u0120ratt": 27263, + "\u0120municipality": 27264, + "\u0120Shortly": 27265, + "\u0120monk": 27266, + "),\"": 27267, + "\u0120scrub": 27268, + "\u0120operatives": 27269, + "Neither": 27270, + "Place": 27271, + "\u0120Limit": 27272, + "Female": 27273, + "\u0120Actor": 27274, + "Character": 27275, + "\u0120constituted": 27276, + "357": 27277, + "\u0120protested": 27278, + "\u0120Straw": 27279, + "\u0120Height": 27280, + "ilda": 27281, + "\u0120Typh": 27282, + "\u0120floods": 27283, + "\u0120cosmetic": 27284, + "WAY": 27285, + "perture": 27286, + "upon": 27287, + "tons": 27288, + "essing": 27289, + "\u0120Pocket": 27290, + "\u0120rooft": 27291, + "\u0120Caucas": 27292, + "\u0120antidepress": 27293, + "\u0120incompatible": 27294, + "ECD": 27295, + "\u0120opera": 27296, + "\u0120Contest": 27297, + "\u0120generators": 27298, + "lime": 27299, + "Defense": 27300, + "1987": 27301, + "forum": 27302, + "\u0120savage": 27303, + "\u0120Hungarian": 27304, + "nz": 27305, + "\u0120metallic": 27306, + "\u0120expelled": 27307, + "\u0120residency": 27308, + "\u0120dresses": 27309, + "666": 27310, + "\u0120Clement": 27311, + "fires": 27312, + "Category": 27313, + "\u0120geek": 27314, + "alis": 27315, + "\u0120cemetery": 27316, + "educated": 27317, + "\u0120crawl": 27318, + "\u0120Unable": 27319, + "\u0120Tyson": 27320, + "akis": 27321, + "\u0120pardon": 27322, + "\u0120Wra": 27323, + "\u0120strengthened": 27324, + "\u0120Fors": 27325, + "335": 27326, + "\u0120HC": 27327, + "\u0120Mond": 27328, + "\u0120visuals": 27329, + "\u0120Beatles": 27330, + "ettlement": 27331, + "\u0120\u00ef": 27332, + "gro": 27333, + "\u0120bash": 27334, + "\u0120poorest": 27335, + "\u0120excel": 27336, + "\u0120aspirations": 27337, + "\u0120Municip": 27338, + "ensible": 27339, + "\u0120ceremonies": 27340, + "\u0120intimidation": 27341, + "\u0120CONTR": 27342, + "beck": 27343, + "\u0120Kap": 27344, + "asu": 27345, + "\u0120trademarks": 27346, + "\u0120Sew": 27347, + "\u0120Competition": 27348, + "network": 27349, + "\u0120Arri": 27350, + "\u0120Tet": 27351, + "Roaming": 27352, + "WC": 27353, + "Dat": 27354, + "\u0120sob": 27355, + "\u0120pairing": 27356, + "\u0120overdose": 27357, + "SAY": 27358, + "aber": 27359, + "\u0120revolt": 27360, + "\u0120Fah": 27361, + "acting": 27362, + "eq": 27363, + "estation": 27364, + "Fight": 27365, + "\u0120Marks": 27366, + "273": 27367, + "\u0120178": 27368, + "Raw": 27369, + "\u00e3\u0123\u012d": 27370, + "349": 27371, + "blocks": 27372, + "\u0120verge": 27373, + "estine": 27374, + "\u0120Podesta": 27375, + "\u0120invasive": 27376, + "\u0120profoundly": 27377, + "\u0120Ao": 27378, + "each": 27379, + "\u0120lest": 27380, + "interpret": 27381, + "\u0120shrinking": 27382, + "\u0120errone": 27383, + "\u0120chees": 27384, + "lys": 27385, + "\u0120Ivy": 27386, + "\u0120Directory": 27387, + "\u0120hinted": 27388, + "VICE": 27389, + "\u0120contacting": 27390, + "\u0120Gent": 27391, + "hei": 27392, + "\u0120labeling": 27393, + "\u0120mercury": 27394, + "\u0120Lite": 27395, + "\u0120expires": 27396, + "\u0120destabil": 27397, + "ritis": 27398, + "cu": 27399, + "\u0120feathers": 27400, + "\u0120steer": 27401, + "\u0120programmed": 27402, + "\u0120Vader": 27403, + "Going": 27404, + "\u0120Elim": 27405, + "\u0120yo": 27406, + "\u0120Miche": 27407, + "\u0120203": 27408, + "\u0120sleeves": 27409, + "\u0120bully": 27410, + "\u0120Humans": 27411, + "368": 27412, + "\u0120compress": 27413, + "\u0120Banner": 27414, + "ARS": 27415, + "\u0120awhile": 27416, + "\u0120calib": 27417, + "\u0120sponsorship": 27418, + "\u0120Difficulty": 27419, + "\u0120Papers": 27420, + "\u0120identifier": 27421, + "}.": 27422, + "\u0120yog": 27423, + "\u0120Shia": 27424, + "\u0120cleanup": 27425, + "\u0120vibe": 27426, + "introdu": 27427, + "imming": 27428, + "Australia": 27429, + "\u0120outlines": 27430, + "\u0120Youtube": 27431, + "train": 27432, + "\u0120Makes": 27433, + "\u0120deported": 27434, + "\u0120centr": 27435, + "\u0120Dug": 27436, + "\u0120Boulder": 27437, + "\u0120Buffy": 27438, + "\u0120injunction": 27439, + "\u0120Harley": 27440, + "\u0120Groups": 27441, + "\u0120Dumbledore": 27442, + "\u0120Clara": 27443, + "\u0120\"-": 27444, + "\u0120sacrificed": 27445, + "eph": 27446, + "Shadow": 27447, + "ibling": 27448, + "\u0120freelance": 27449, + "\u0120evidently": 27450, + "phal": 27451, + "\u0120retains": 27452, + "Mir": 27453, + "\u0120finite": 27454, + "dar": 27455, + "\u0120Cous": 27456, + "\u0120repaired": 27457, + "\u0120periodic": 27458, + "\u0120championships": 27459, + "\u0120asteroid": 27460, + "blind": 27461, + "\u0120expressly": 27462, + "\u0120Astros": 27463, + "\u0120scaled": 27464, + "\u0120geographical": 27465, + "\u0120Rapids": 27466, + "Enjoy": 27467, + "\u0120elastic": 27468, + "\u0120Mohamed": 27469, + "Market": 27470, + "begin": 27471, + "\u0120discovers": 27472, + "\u0120telecommunications": 27473, + "\u0120scanner": 27474, + "\u0120enlarge": 27475, + "\u0120sharks": 27476, + "\u0120psychedel": 27477, + "\u0120Rouge": 27478, + "\u0120snapshot": 27479, + "isine": 27480, + "XP": 27481, + "\u0120pesticides": 27482, + "\u0120LSD": 27483, + "\u0120Distribution": 27484, + "really": 27485, + "\u0120degradation": 27486, + "\u0120disguise": 27487, + "\u0120biom": 27488, + "\u0120EXT": 27489, + "\u0120equations": 27490, + "\u0120hazards": 27491, + "\u0120Compared": 27492, + ")*": 27493, + "\u0120virtues": 27494, + "\u0120elders": 27495, + "\u0120enhancing": 27496, + "\u0120Across": 27497, + "eros": 27498, + "angling": 27499, + "\u0120combust": 27500, + "ucci": 27501, + "\u0120concussion": 27502, + "\u0120contraception": 27503, + "\u0120Kang": 27504, + "\u0120expresses": 27505, + "\u0120aux": 27506, + "\u0120Pione": 27507, + "\u0120exhibits": 27508, + "Debug": 27509, + "OTAL": 27510, + "\u0120Already": 27511, + "\u0120Wheeler": 27512, + "\u0120expands": 27513, + "?:": 27514, + "\u0120reconciliation": 27515, + "\u0120pirates": 27516, + "\u0120purse": 27517, + "\u0120discourage": 27518, + "\u0120spectacle": 27519, + "Rank": 27520, + "\u0120wraps": 27521, + "\u0120Thought": 27522, + "\u0120impending": 27523, + "Opp": 27524, + "\u0120Anglo": 27525, + "\u0120EUR": 27526, + "\u0120screwed": 27527, + "retched": 27528, + "\u0120encouragement": 27529, + "models": 27530, + "\u0120confuse": 27531, + "mmm": 27532, + "\u0120Vitamin": 27533, + "\u00e2\u0138\u0133\u00e2\u0138\u0133": 27534, + "Cru": 27535, + "\u0120knights": 27536, + "\u0120discard": 27537, + "\u0120bishops": 27538, + "\u0120Wear": 27539, + "\u0120Garrett": 27540, + "kan": 27541, + "\u00e3\u0125\u0141": 27542, + "\u0120masculine": 27543, + "capital": 27544, + "\u0120Aus": 27545, + "\u0120fatally": 27546, + "thanks": 27547, + "\u0120AU": 27548, + "\u0120Gut": 27549, + "1200": 27550, + "\u012000000000": 27551, + "\u0120surrog": 27552, + "\u0120BIOS": 27553, + "raits": 27554, + "\u0120Watts": 27555, + "\u0120resurrection": 27556, + "\u0120Electoral": 27557, + "\u0120Tips": 27558, + "4000": 27559, + "\u0120nutrient": 27560, + "\u0120depicting": 27561, + "\u0120sprink": 27562, + "\u0120muff": 27563, + "\u0120LIM": 27564, + "\u0120Sample": 27565, + "psc": 27566, + "ibi": 27567, + "generated": 27568, + "\u0120specimens": 27569, + "\u0120dissatisf": 27570, + "\u0120tailored": 27571, + "\u0120holdings": 27572, + "\u0120Monthly": 27573, + "\u0120Eat": 27574, + "poons": 27575, + "\u0120nec": 27576, + "\u0120Cage": 27577, + "\u0120Lotus": 27578, + "\u0120Lantern": 27579, + "\u0120frontier": 27580, + "\u0120pensions": 27581, + "\u0120joked": 27582, + "\u0120Hardy": 27583, + "=-=-=-=-": 27584, + "rade": 27585, + "UID": 27586, + "\u0120rails": 27587, + "\u0120emit": 27588, + "\u0120slate": 27589, + "\u0120smug": 27590, + "\u0120spit": 27591, + "\u0120Calls": 27592, + "\u0120Jacobs": 27593, + "feat": 27594, + "\u0120UE": 27595, + "\u0120restruct": 27596, + "\u0120regeneration": 27597, + "\u0120energies": 27598, + "\u0120Connor": 27599, + "OHN": 27600, + "\u0120Cheese": 27601, + "\u0120ger": 27602, + "\u0120resurrect": 27603, + "management": 27604, + "NW": 27605, + "\u0120presently": 27606, + "\u0120Bruins": 27607, + "Member": 27608, + "\u0120Mang": 27609, + "idan": 27610, + "\u0120boosting": 27611, + "wyn": 27612, + "+.": 27613, + "requisite": 27614, + "\u0120NYPD": 27615, + "\u0120Megan": 27616, + "\u0120Conditions": 27617, + "\u0120pics": 27618, + "nesium": 27619, + "\u0120Rash": 27620, + "\u0120174": 27621, + "\u0120Ducks": 27622, + "\u0120embro": 27623, + "zu": 27624, + "onian": 27625, + "religious": 27626, + "\u0120craz": 27627, + "\u0120ACA": 27628, + "\u0120Zucker": 27629, + "EMA": 27630, + "\u0120Pros": 27631, + "Weapon": 27632, + "\u0120Knox": 27633, + "\u0120Arduino": 27634, + "\u0120stove": 27635, + "\u0120heavens": 27636, + "\u0120Purchase": 27637, + "\u0120herd": 27638, + "\u0120fundraiser": 27639, + "Digital": 27640, + "5000": 27641, + "\u0120proponents": 27642, + "/\u00e2\u0122\u012d": 27643, + "\u0120jelly": 27644, + "\u0120Visa": 27645, + "\u0120monks": 27646, + "\u0120advancement": 27647, + "\u0120Wer": 27648, + "\u0120187": 27649, + "eus": 27650, + "ertility": 27651, + "\u0120fetal": 27652, + "\u01201936": 27653, + "Lo": 27654, + "\u0120outfits": 27655, + "\u0120staircase": 27656, + "bomb": 27657, + "\u0120customized": 27658, + "clair": 27659, + "Tree": 27660, + "\u0120mapped": 27661, + "\u0120Considering": 27662, + "\u0120Torres": 27663, + "\u0120methyl": 27664, + "\u0120approximate": 27665, + "\u0120doom": 27666, + "\u0120Hansen": 27667, + "\u0120crossover": 27668, + "\u0120standalone": 27669, + "\u00e4\u00bc": 27670, + "\u0120invites": 27671, + "\u0120graveyard": 27672, + "\u0120hp": 27673, + "DonaldTrump": 27674, + "\u0120escort": 27675, + "Gar": 27676, + "\u0120predecessors": 27677, + "\u0120hay": 27678, + "\u0120enzyme": 27679, + "\u0120Straight": 27680, + "visors": 27681, + "Ing": 27682, + "aneously": 27683, + "\u0120Applied": 27684, + "\u0120fec": 27685, + "\u0120Durant": 27686, + "\u0120outspoken": 27687, + "orb": 27688, + "\u0120zeal": 27689, + "\u0120disgrace": 27690, + "').": 27691, + "\u0120Cheng": 27692, + "289": 27693, + "\u0120Rena": 27694, + "\u0120Suicide": 27695, + "294": 27696, + "\u0120outraged": 27697, + "\u0120Newman": 27698, + "\u0120Nvidia": 27699, + "\u0120Aber": 27700, + "\u0120Bers": 27701, + "\u0120recreation": 27702, + "Window": 27703, + "\u0120DP": 27704, + "xe": 27705, + "\u0120pedoph": 27706, + "\u0120fallout": 27707, + "amboo": 27708, + "\u0120presentations": 27709, + "\u0120Apps": 27710, + "\u0120html": 27711, + "345": 27712, + "\u0120XXX": 27713, + "\u0120rubbing": 27714, + "\u0120Leather": 27715, + "\u0120humidity": 27716, + "seys": 27717, + "established": 27718, + "\u0120Units": 27719, + "646": 27720, + "\u0120respectable": 27721, + "Auto": 27722, + "\u0120thriving": 27723, + "\u0120Innovation": 27724, + "angs": 27725, + "Extra": 27726, + "regulation": 27727, + "298": 27728, + "pick": 27729, + "Examples": 27730, + "\u0120CJ": 27731, + "Attack": 27732, + "\u0120dracon": 27733, + "LT": 27734, + "\u0120sticker": 27735, + "rers": 27736, + "\u0120sunny": 27737, + "Iss": 27738, + "regulated": 27739, + "dim": 27740, + "\u0120Abstract": 27741, + "\u0120husbands": 27742, + "Office": 27743, + "omination": 27744, + "itars": 27745, + "ANGE": 27746, + "ascal": 27747, + "\u0120Kris": 27748, + "\u0120Infantry": 27749, + "\u0120malf": 27750, + "\u0120Athe": 27751, + "\u0120Rally": 27752, + "balanced": 27753, + "........................": 27754, + "OUP": 27755, + "\u0120molecule": 27756, + "metics": 27757, + "\u0120Split": 27758, + "\u0120Instructions": 27759, + "\u0120Nights": 27760, + "cards": 27761, + "\u0120tug": 27762, + "\u0120cone": 27763, + "\u00e5\u0143": 27764, + "\u0120tx": 27765, + "\u0120Discussion": 27766, + "\u0120catastrophe": 27767, + "ppe": 27768, + "gio": 27769, + "\u0120communism": 27770, + "\u0120halted": 27771, + "\u0120Guant": 27772, + "clean": 27773, + "\u0120Sched": 27774, + "\u0120Kanye": 27775, + "\u0120wander": 27776, + "\u0120Seriously": 27777, + "\u0120188": 27778, + "ennial": 27779, + "follow": 27780, + "productive": 27781, + "\u0120Flow": 27782, + "\u0120Sail": 27783, + "\u0120craw": 27784, + "\u0120simulations": 27785, + "oru": 27786, + "angles": 27787, + "\u0120Nolan": 27788, + "\u0120menstru": 27789, + "470": 27790, + "\u0120207": 27791, + "aja": 27792, + "\u0120casually": 27793, + "boarding": 27794, + "\u0120222": 27795, + "ovy": 27796, + "\u0120Numbers": 27797, + "umat": 27798, + "OE": 27799, + "287": 27800, + "\u0120Clemson": 27801, + "\u0120certs": 27802, + "\u0120slid": 27803, + "\u0120Tribe": 27804, + "\u0120toast": 27805, + "\u0120fortunes": 27806, + "\u0120fals": 27807, + "\u0120Committees": 27808, + "\u0120gp": 27809, + "\u0120fiery": 27810, + "\u0120Nets": 27811, + "\u0120Anime": 27812, + "Package": 27813, + "\u0120Compare": 27814, + "laughter": 27815, + "infect": 27816, + "\u0120atrocities": 27817, + "\u0120justices": 27818, + "\u0120insults": 27819, + "\u0120Vernon": 27820, + "\u0120shaken": 27821, + "\u0120persona": 27822, + "estamp": 27823, + "367": 27824, + "brain": 27825, + "\u0120experimenting": 27826, + "Ken": 27827, + "\u0120Electronics": 27828, + "\u0120161": 27829, + "domain": 27830, + "\u0120graphical": 27831, + "bishop": 27832, + "\u0120whopping": 27833, + "\u0120Evangel": 27834, + "\u0120advertisers": 27835, + "\u0120Spear": 27836, + "\u0120bids": 27837, + "\u0120destroys": 27838, + "utz": 27839, + "\u0120undersc": 27840, + "\u0120ADD": 27841, + "\u0120ants": 27842, + "\u0120Cum": 27843, + "ipples": 27844, + "\u0120Fill": 27845, + "\u0120glanced": 27846, + "\u0120indicted": 27847, + "\u0120Eff": 27848, + "\u0120miscon": 27849, + "\u0120Desktop": 27850, + "\u0120abide": 27851, + "\u00e3\u0125\u0122": 27852, + "\u0120Io": 27853, + "\u0120Coul": 27854, + "\u0120capsule": 27855, + "\u0120Chrys": 27856, + "MON": 27857, + "\u0120undes": 27858, + "\u0120IRA": 27859, + "\u0120citation": 27860, + "\u0120dictate": 27861, + "\u0120Networks": 27862, + "\u0120Conflict": 27863, + "\u0120Stuff": 27864, + "xa": 27865, + "isec": 27866, + "\u0120Chemistry": 27867, + "\u0120quarterly": 27868, + "Williams": 27869, + "anan": 27870, + "Opt": 27871, + "\u0120Alexandria": 27872, + "outheastern": 27873, + "\u0120Springfield": 27874, + "\u0120Blacks": 27875, + "\u0120geography": 27876, + "242": 27877, + "\u0120utmost": 27878, + "\u0120Exxon": 27879, + "abouts": 27880, + "EVA": 27881, + "\u0120Enable": 27882, + "\u0120Barr": 27883, + "\u0120disagreed": 27884, + "\u0120Cyprus": 27885, + "\u0120dementia": 27886, + "\u0120labs": 27887, + "\u0120ubiquitous": 27888, + "\u0120LOVE": 27889, + "\u0120consolidated": 27890, + "sr": 27891, + "\u0120creamy": 27892, + "\u0120Timber": 27893, + "Regardless": 27894, + "\u0120Certificate": 27895, + "\u0120\"...": 27896, + "ogenous": 27897, + "Captain": 27898, + "\u0120insulting": 27899, + "\u0120Soros": 27900, + "\u0120Instr": 27901, + "\u0120Bulgaria": 27902, + "better": 27903, + "\u0120sucking": 27904, + "\u0120Davidson": 27905, + "atz": 27906, + "\u0120collateral": 27907, + "gif": 27908, + "\u0120plagued": 27909, + "\u0120Cancel": 27910, + "\u0120Gardner": 27911, + "RB": 27912, + "\u0120sixteen": 27913, + "Remove": 27914, + "uristic": 27915, + "cook": 27916, + "Rod": 27917, + "\u0120comprising": 27918, + "fle": 27919, + ")\u00e2\u0122\u0136": 27920, + "\u0120Viking": 27921, + "growth": 27922, + "agonal": 27923, + "\u0120srf": 27924, + "afety": 27925, + "mot": 27926, + "Nearly": 27927, + "stown": 27928, + "\u0120Factor": 27929, + "\u0120automobile": 27930, + "\u0120procedural": 27931, + "mask": 27932, + "ampires": 27933, + "\u0120disappears": 27934, + "jab": 27935, + "315": 27936, + "\u01201951": 27937, + "needed": 27938, + "\u0120daring": 27939, + "leader": 27940, + "\u0120podium": 27941, + "\u0120unhealthy": 27942, + "\u0120mund": 27943, + "\u0120pyramid": 27944, + "ocre": 27945, + "\u0120kissed": 27946, + "\u0120dreamed": 27947, + "\u0120Fantastic": 27948, + "\u0120Gly": 27949, + "\u00e5\u012c": 27950, + "\u0120greatness": 27951, + "\u0120spices": 27952, + "\u0120metropolitan": 27953, + "\u0120compuls": 27954, + "iets": 27955, + "1016": 27956, + "\u0120Sham": 27957, + "\u0120Pyr": 27958, + "flies": 27959, + "\u0120Midnight": 27960, + "\u0120swallowed": 27961, + "\u0120genres": 27962, + "\u0120Lucky": 27963, + "\u0120Rewards": 27964, + "\u0120dispatch": 27965, + "\u0120IPA": 27966, + "\u0120Apply": 27967, + "\u0120aven": 27968, + "alities": 27969, + "312": 27970, + "things": 27971, + "\u0120().": 27972, + "\u0120mates": 27973, + "\u0120Sz": 27974, + "\u0120COP": 27975, + "olate": 27976, + "OFF": 27977, + "\u0120recharge": 27978, + "caps": 27979, + "\u0120Yorker": 27980, + "icone": 27981, + "\u0120galaxies": 27982, + "ileaks": 27983, + "Dave": 27984, + "\u0120Puzz": 27985, + "\u0120Celtic": 27986, + "\u0120AFC": 27987, + "276": 27988, + "\u0120Sons": 27989, + "\u0120affirmative": 27990, + "Hor": 27991, + "\u0120tutorials": 27992, + "\u0120CITY": 27993, + "\u0120Rosa": 27994, + "\u0120Extension": 27995, + "Series": 27996, + "\u0120fats": 27997, + "\u0120rab": 27998, + "lis": 27999, + "\u0120unic": 28000, + "\u0120eve": 28001, + "\u0120Spin": 28002, + "\u0120adulthood": 28003, + "typ": 28004, + "\u0120sectarian": 28005, + "\u0120checkout": 28006, + "\u0120Cycl": 28007, + "Single": 28008, + "\u0120martyr": 28009, + "\u0120chilling": 28010, + "888": 28011, + "oufl": 28012, + "\u0120];": 28013, + "\u0120congestion": 28014, + "mk": 28015, + "\u0120Whereas": 28016, + "\u01201938": 28017, + "urrencies": 28018, + "erion": 28019, + "\u0120boast": 28020, + "\u0120Patients": 28021, + "\u0120chap": 28022, + "\u0120BD": 28023, + "realDonaldTrump": 28024, + "\u0120examines": 28025, + "hov": 28026, + "\u0120startling": 28027, + "\u0120Babylon": 28028, + "wid": 28029, + "omew": 28030, + "brance": 28031, + "\u0120Odyssey": 28032, + "wig": 28033, + "\u0120torch": 28034, + "\u0120Vox": 28035, + "\u0120Moz": 28036, + "\u0120Troll": 28037, + "\u0120Ans": 28038, + "Similarly": 28039, + "\u0120Ful": 28040, + "006": 28041, + "Unless": 28042, + "\u0120Alone": 28043, + "stead": 28044, + "\u0120Publisher": 28045, + "rights": 28046, + "tu": 28047, + "\u0120Doesn": 28048, + "\u0120professionally": 28049, + "\u0120clo": 28050, + "icz": 28051, + "\u0120steals": 28052, + "\u0120\u00e1": 28053, + "1986": 28054, + "\u0120sturdy": 28055, + "\u0120Johann": 28056, + "\u0120medals": 28057, + "\u0120filings": 28058, + "\u0120Fraser": 28059, + "done": 28060, + "\u0120multinational": 28061, + "\u0120feder": 28062, + "\u0120worthless": 28063, + "\u0120pest": 28064, + "Yesterday": 28065, + "ankind": 28066, + "\u0120gays": 28067, + "\u0120borne": 28068, + "\u0120POS": 28069, + "Picture": 28070, + "\u0120percentages": 28071, + "251": 28072, + "rame": 28073, + "\u0120potions": 28074, + "AMD": 28075, + "\u0120Lebanese": 28076, + "\u0120rang": 28077, + "\u0120LSU": 28078, + "ongs": 28079, + "\u0120peninsula": 28080, + "\u0120Clause": 28081, + "ALK": 28082, + "oha": 28083, + "\u0120MacBook": 28084, + "\u0120unanimous": 28085, + "\u0120lenders": 28086, + "\u0120hangs": 28087, + "\u0120franchises": 28088, + "orers": 28089, + "\u0120Updates": 28090, + "\u0120isolate": 28091, + "andro": 28092, + "Soon": 28093, + "\u0120disruptive": 28094, + "\u0120Surve": 28095, + "\u0120stitches": 28096, + "\u0120Scorp": 28097, + "\u0120Dominion": 28098, + "\u0120supplying": 28099, + "Arg": 28100, + "\u0120turret": 28101, + "\u0120Luk": 28102, + "\u0120brackets": 28103, + "*)": 28104, + "\u0120Revolutionary": 28105, + "\u0120Honest": 28106, + "\u0120noticing": 28107, + "\u0120Shannon": 28108, + "\u0120afforded": 28109, + "\u0120tha": 28110, + "\u0120Janet": 28111, + "!--": 28112, + "\u0120Narendra": 28113, + "\u0120Plot": 28114, + "Hol": 28115, + "sever": 28116, + "eenth": 28117, + "\u0120obstruction": 28118, + "\u01201024": 28119, + "staff": 28120, + "jas": 28121, + "orget": 28122, + "scenes": 28123, + "laughs": 28124, + "\u0120Fargo": 28125, + "crime": 28126, + "\u0120orchestr": 28127, + "\u0120delet": 28128, + "iliary": 28129, + "rieved": 28130, + "\u0120militar": 28131, + "\u0120Greene": 28132, + "\u00e2\u0139\u0131": 28133, + "\u00e3\u0123\u00a6": 28134, + "\u0120Guards": 28135, + "\u0120unleashed": 28136, + "\u0120Weber": 28137, + "\u0120adjustable": 28138, + "\u0120caliber": 28139, + "\u0120motivations": 28140, + "\u0120\u00c3\u0142": 28141, + "mAh": 28142, + "\u0120Lanka": 28143, + "handle": 28144, + "\u0120pent": 28145, + "\u0120Rav": 28146, + "\u0120Angular": 28147, + "\u0120Kau": 28148, + "umbing": 28149, + "\u0120philanthrop": 28150, + "\u0120dehyd": 28151, + "\u0120toxicity": 28152, + "eer": 28153, + "\u0120YORK": 28154, + "witz": 28155, + "\u00e5\u00bc": 28156, + "\u0120IE": 28157, + "community": 28158, + "\u0120AH": 28159, + "\u0120retali": 28160, + "\u0120massively": 28161, + "\u0120Daniels": 28162, + "\u0120DEL": 28163, + "\u0120carcin": 28164, + "Url": 28165, + "\u0120routing": 28166, + "\u0120NPCs": 28167, + "\u0120RAF": 28168, + "ryce": 28169, + "\u0120waived": 28170, + "\u0120Guatem": 28171, + "Everybody": 28172, + "\u0120covenant": 28173, + "\u0120173": 28174, + "\u0120relaxing": 28175, + "\u0120quart": 28176, + "almost": 28177, + "\u0120guarded": 28178, + "\u0120Soldiers": 28179, + "\u0120PLAY": 28180, + "\u0120outgoing": 28181, + "LAND": 28182, + "\u0120rewrite": 28183, + "\u0120MOV": 28184, + "\u0120Imper": 28185, + "\u0120Solution": 28186, + "\u0120phenomenal": 28187, + "\u0120longevity": 28188, + "\u0120impat": 28189, + "\u0120Nissan": 28190, + "irie": 28191, + "\u0120odor": 28192, + "\u0120Zar": 28193, + "oks": 28194, + "\u0120militias": 28195, + "\u0120SPEC": 28196, + "\u0120tolerated": 28197, + "arser": 28198, + "\u0120Bradford": 28199, + "+,": 28200, + "\u0120surreal": 28201, + "sf": 28202, + "Canadian": 28203, + "\u0120resemblance": 28204, + "\u0120carbohydrate": 28205, + "VIEW": 28206, + "\u0120accessory": 28207, + "meal": 28208, + "largest": 28209, + "iegel": 28210, + "Someone": 28211, + "\u0120toughest": 28212, + "oso": 28213, + "\u0120funnel": 28214, + "\u0120condemnation": 28215, + "luent": 28216, + "\u0120wired": 28217, + "\u0120Sunset": 28218, + "Jesus": 28219, + "\u0120PST": 28220, + "\u0120Pages": 28221, + "\u0120Tycoon": 28222, + "\u0120PF": 28223, + "\u0120selections": 28224, + "\u0120\u00e0\u00a4": 28225, + "partisan": 28226, + "\u0120highs": 28227, + "\u0120Rune": 28228, + "\u0120crafts": 28229, + "lead": 28230, + "\u0120Parents": 28231, + "\u0120reclaim": 28232, + "eker": 28233, + "\u0120Allied": 28234, + "aeper": 28235, + "\u0120looming": 28236, + "\u0120beneficiaries": 28237, + "\u0120Hull": 28238, + "Students": 28239, + "Jewish": 28240, + "dj": 28241, + "\u0120pact": 28242, + "template": 28243, + "\u0120Officials": 28244, + "\u0120Baylor": 28245, + "\u0120hemp": 28246, + "\u0120youths": 28247, + "\u0120Levels": 28248, + "\u0120Xiao": 28249, + "\u0120Ches": 28250, + "\u0120endeavor": 28251, + "\u0120Removed": 28252, + "\u0120hippocamp": 28253, + "Hell": 28254, + "\u00e3\u0124\u012c": 28255, + "805": 28256, + "\u0120dinosaur": 28257, + "\u0120Wrath": 28258, + "\u0120Indonesian": 28259, + "\u0120calculator": 28260, + "\u0120Dictionary": 28261, + "\u0120420": 28262, + "\u0120MAG": 28263, + "(_": 28264, + "!,": 28265, + "tarians": 28266, + "\u0120restricting": 28267, + "racuse": 28268, + "\u0120weekday": 28269, + "OUNT": 28270, + "\u0120shrugged": 28271, + "leground": 28272, + "\u0120bald": 28273, + "\u0120Doctors": 28274, + "\u0120touted": 28275, + "\u0120Maxwell": 28276, + "\u0120214": 28277, + "\u0120diplomat": 28278, + "\u0120repression": 28279, + "\u0120constituency": 28280, + "vice": 28281, + "ranked": 28282, + "\u0120Napoleon": 28283, + "gang": 28284, + "\u0120Forever": 28285, + "tun": 28286, + "\u0120bulb": 28287, + "\u0120PDT": 28288, + "\u0120Cisco": 28289, + "VEN": 28290, + "\u0120resumed": 28291, + "Steven": 28292, + "\u0120Manitoba": 28293, + "\u0120fabulous": 28294, + "\u0120Agents": 28295, + "1984": 28296, + "\u0120amusing": 28297, + "\u0120Mysteries": 28298, + "\u0120orthodox": 28299, + "floor": 28300, + "\u0120questionnaire": 28301, + "\u0120penetrate": 28302, + "\u0120filmmakers": 28303, + "\u0120Unc": 28304, + "\u0120stamped": 28305, + "\u0120thirteen": 28306, + "\u0120outfield": 28307, + "\u0120forwarded": 28308, + "\u0120appra": 28309, + "\u0120aided": 28310, + "try": 28311, + "\u0120unfocused": 28312, + "\u0120Liz": 28313, + "\u0120Wendy": 28314, + "\u0120Scene": 28315, + "Charg": 28316, + "\u0120rejects": 28317, + "\u0120leftist": 28318, + "\u0120Providence": 28319, + "\u0120Brid": 28320, + "regn": 28321, + "\u0120prophecy": 28322, + "\u0120LIVE": 28323, + "499": 28324, + "\u0120forge": 28325, + "\u0120FML": 28326, + "\u0120intrinsic": 28327, + "\u0120Frog": 28328, + "\u0120wont": 28329, + "\u0120Holt": 28330, + "\u0120famed": 28331, + "CLUS": 28332, + "aepernick": 28333, + "\u0120Hate": 28334, + "\u0120Cay": 28335, + "\u0120registering": 28336, + "ortality": 28337, + "ropy": 28338, + "ocalyptic": 28339, + "aan": 28340, + "nav": 28341, + "\u0120fascist": 28342, + "IFIED": 28343, + "\u0120implicated": 28344, + "\u0120Resort": 28345, + "\u0120Chandler": 28346, + "\u0120Brick": 28347, + "Pin": 28348, + "ysc": 28349, + "Usage": 28350, + "\u0120Helm": 28351, + "usra": 28352, + "\u00e2\u013a\u0127\u00e2\u013a\u0127": 28353, + "\u0120Abbas": 28354, + "\u0120unanimously": 28355, + "\u0120keeper": 28356, + "\u0120addicted": 28357, + "???": 28358, + "\u0120helmets": 28359, + "\u0120antioxid": 28360, + "apsed": 28361, + "808": 28362, + "giene": 28363, + "\u0120waits": 28364, + "\u0120minion": 28365, + "raved": 28366, + "\u0120Porsche": 28367, + "\u0120dreaming": 28368, + "\u0120171": 28369, + "\u0120Cain": 28370, + "\u0120unfor": 28371, + "asso": 28372, + "\u0120Configuration": 28373, + "kun": 28374, + "hardt": 28375, + "\u0120nested": 28376, + "\u0120LDS": 28377, + "LES": 28378, + "\u0120tying": 28379, + "enos": 28380, + "\u0120cue": 28381, + "\u0120Marqu": 28382, + "skirts": 28383, + "\u0120clicked": 28384, + "\u0120expiration": 28385, + "\u0120Accordingly": 28386, + "\u0120WC": 28387, + "\u0120blessings": 28388, + "\u0120addictive": 28389, + "\u0120Narr": 28390, + "yx": 28391, + "\u0120Jaguars": 28392, + "\u0120rents": 28393, + "\u0120Siber": 28394, + "\u0120tipped": 28395, + "ousse": 28396, + "\u0120Fitzgerald": 28397, + "\u0120hierarch": 28398, + "outine": 28399, + "\u0120wavelength": 28400, + ">.": 28401, + "chid": 28402, + "\u0120Processing": 28403, + "/+": 28404, + "ranking": 28405, + "Easy": 28406, + "\u0120Construct": 28407, + "\u0120tet": 28408, + "insured": 28409, + "HUD": 28410, + "\u0120quoting": 28411, + "\u0120communicated": 28412, + "inx": 28413, + "\u0120inmate": 28414, + "\u0120erected": 28415, + "\u0120Absolutely": 28416, + "\u0120Surely": 28417, + "\u0120unim": 28418, + "\u0120Throne": 28419, + "heid": 28420, + "\u0120claws": 28421, + "\u0120superstar": 28422, + "\u0120Lenn": 28423, + "\u0120Whis": 28424, + "Uk": 28425, + "abol": 28426, + "\u0120sket": 28427, + "\u0120Niet": 28428, + "\u0120perks": 28429, + "\u0120affinity": 28430, + "\u0120openings": 28431, + "phasis": 28432, + "\u0120discriminate": 28433, + "Tip": 28434, + "vc": 28435, + "\u0120grinding": 28436, + "\u0120Jenny": 28437, + "\u0120asthma": 28438, + "holes": 28439, + "\u0120Homer": 28440, + "\u0120registers": 28441, + "\u0120Glad": 28442, + "\u0120creations": 28443, + "\u0120lithium": 28444, + "\u0120applause": 28445, + "until": 28446, + "Justice": 28447, + "\u0120Turks": 28448, + "\u0120scandals": 28449, + "\u0120bake": 28450, + "tank": 28451, + "Mech": 28452, + "\u0120Means": 28453, + "\u0120Maid": 28454, + "Republicans": 28455, + "isal": 28456, + "windows": 28457, + "\u0120Santos": 28458, + "\u0120vegetation": 28459, + "338": 28460, + "tri": 28461, + "\u0120flux": 28462, + "insert": 28463, + "\u0120clarified": 28464, + "\u0120mortg": 28465, + "\u0120Chim": 28466, + "\u0120Tort": 28467, + "\u0120disclaim": 28468, + "metal": 28469, + "\u0120Aside": 28470, + "\u0120induction": 28471, + "\u0120infl": 28472, + "\u0120atheists": 28473, + "amph": 28474, + "\u0120ether": 28475, + "\u0120Vital": 28476, + "\u0120Built": 28477, + "Mind": 28478, + "\u0120weaponry": 28479, + "SET": 28480, + "\u0120186": 28481, + "admin": 28482, + "gam": 28483, + "contract": 28484, + "afa": 28485, + "\u0120derivatives": 28486, + "\u0120snacks": 28487, + "\u0120churn": 28488, + "Econom": 28489, + "\u0120capped": 28490, + "\u0120Understanding": 28491, + "\u0120Hers": 28492, + "\u0120Iz": 28493, + "\u0120duct": 28494, + "IENT": 28495, + "aughty": 28496, + "\u0120\u00e2\u013e\u0136": 28497, + "\u0120NP": 28498, + "\u0120sailing": 28499, + "Initialized": 28500, + "\u0120ted": 28501, + "\u0120reactors": 28502, + "\u0120Lomb": 28503, + "\u0120choke": 28504, + "\u0120Worm": 28505, + "\u0120admiration": 28506, + "\u0120swung": 28507, + "ensibly": 28508, + "\u0120rash": 28509, + "\u0120Goals": 28510, + "\u0120Important": 28511, + "Shot": 28512, + "\u0120Ras": 28513, + "\u0120trainers": 28514, + "\u0120Bun": 28515, + "Working": 28516, + "\u0120harmed": 28517, + "\u0120Pandora": 28518, + "\u0120LTE": 28519, + "\u0120mushroom": 28520, + "\u0120CHAR": 28521, + "\u0120Fee": 28522, + "\u0120Moy": 28523, + "Born": 28524, + "oliberal": 28525, + "\u0120Martial": 28526, + "\u0120gentlemen": 28527, + "\u0120lingering": 28528, + "Official": 28529, + "\u0120graffiti": 28530, + "\u0120Names": 28531, + "Der": 28532, + "\u0120quint": 28533, + "istrate": 28534, + "azeera": 28535, + "\u0120NOTICE": 28536, + "\u0120Florence": 28537, + "\u0120payable": 28538, + "\u0120depicts": 28539, + "\u0120Species": 28540, + "Heart": 28541, + "\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122": 28542, + "\u0120enclosed": 28543, + "Increases": 28544, + "Daily": 28545, + "\u0120Lis": 28546, + "\u0120enactment": 28547, + "\u0120Bacon": 28548, + "\u0120Steele": 28549, + "demand": 28550, + "\u0120183": 28551, + "\u0120mouths": 28552, + "\u0120stranded": 28553, + "\u0120enhancement": 28554, + "011": 28555, + "\u0120Whats": 28556, + "\u0120healed": 28557, + "eny": 28558, + "\u0120Rab": 28559, + "\u0120340": 28560, + "\u0120Labyrinth": 28561, + "roach": 28562, + "\u0120Yosh": 28563, + "\u0120Clippers": 28564, + "\u0120concerts": 28565, + "Internet": 28566, + "355": 28567, + "\u0120stickers": 28568, + "\u0120termed": 28569, + "\u0120Axe": 28570, + "\u0120grandparents": 28571, + "France": 28572, + "\u0120Clim": 28573, + "\u0120Uh": 28574, + "ulic": 28575, + "\u0120thrill": 28576, + "centric": 28577, + "\u0120Overview": 28578, + "\u0120Conduct": 28579, + "\u0120substantive": 28580, + "\u0120182": 28581, + "mur": 28582, + "\u0120stray": 28583, + "\u0120Coff": 28584, + "\u0120repetitive": 28585, + "\u0120Forgotten": 28586, + "\u0120qualification": 28587, + "ewitness": 28588, + "\u0120Zimbabwe": 28589, + "\u0120simulated": 28590, + "\u0120JD": 28591, + "253": 28592, + "\u0120Ware": 28593, + "\u0120unsc": 28594, + "Times": 28595, + "\u0120summons": 28596, + "\u0120disconnected": 28597, + "\u0120184": 28598, + "cius": 28599, + "\u0120Gujar": 28600, + "odka": 28601, + "\u0120erase": 28602, + "\u0120Tobacco": 28603, + "elected": 28604, + "\u0120uncont": 28605, + "\u0120Shepard": 28606, + "\u0120Lamp": 28607, + "\u0120alerted": 28608, + "\u0120operative": 28609, + "arna": 28610, + "uint": 28611, + "\u0120negligence": 28612, + "acements": 28613, + "\u0120supra": 28614, + "\u0120prevail": 28615, + "\u0120Shark": 28616, + "\u0120belts": 28617, + "\u00e3\u0123\u00ab": 28618, + "\u0120tighter": 28619, + "Engineers": 28620, + "\u0120inactive": 28621, + "\u0120exponent": 28622, + "\u0120Willie": 28623, + "aples": 28624, + "\u0120heir": 28625, + "\u0120Hits": 28626, + "iann": 28627, + "\u0120Says": 28628, + "\u0120currents": 28629, + "\u0120Bengal": 28630, + "\u0120arist": 28631, + "Buffer": 28632, + "\u0120breeze": 28633, + "\u0120Wesley": 28634, + "Cola": 28635, + "\u0120pronoun": 28636, + "\u0120deed": 28637, + "\u0120Kling": 28638, + "\u0120oft": 28639, + "\u0120inflict": 28640, + "\u0120punishing": 28641, + "\u0120nm": 28642, + "iku": 28643, + "ODUCT": 28644, + "014": 28645, + "\u0120subsidy": 28646, + "\u0120DEA": 28647, + "\u0120Herbert": 28648, + "\u0120Jal": 28649, + "Bank": 28650, + "\u0120deferred": 28651, + "\u0120shipment": 28652, + "Bott": 28653, + "\u0120alle": 28654, + "bearing": 28655, + "HTML": 28656, + "Offline": 28657, + "\u0120213": 28658, + "\u0120scrolling": 28659, + "\u0120scanned": 28660, + "\u0120Libyan": 28661, + "\u0120TOP": 28662, + "chrom": 28663, + "dt": 28664, + "column": 28665, + "PsyNetMessage": 28666, + "Zero": 28667, + "\u0120torso": 28668, + "050": 28669, + "\u00e2\u0137\u0132": 28670, + "\u0120imperson": 28671, + "\u0120Schwartz": 28672, + "udic": 28673, + "\u0120pissed": 28674, + "\u0120Sapp": 28675, + "257": 28676, + "\u0120ISPs": 28677, + "ogl": 28678, + "\u0120supervised": 28679, + "\u0120adolescent": 28680, + "\u0120attained": 28681, + "\u0120Delivery": 28682, + "\u0120Bunny": 28683, + "\u01201937": 28684, + "\u0120miniature": 28685, + "\u0120os": 28686, + "\u0120370": 28687, + "608": 28688, + "\u0120Mourinho": 28689, + "\u0120innate": 28690, + "\u0120tempo": 28691, + "\u0120NM": 28692, + "\u0120Fallen": 28693, + "009": 28694, + "\u0120provocative": 28695, + "Streamer": 28696, + "\u0120Benedict": 28697, + "\u0120Bolshe": 28698, + "\u0120turtle": 28699, + "\u0120PCB": 28700, + "\u0120Equal": 28701, + "Director": 28702, + "\u0120Rend": 28703, + "\u0120fluids": 28704, + "Authorities": 28705, + "\u0120cousins": 28706, + "requency": 28707, + "\u0120Neighbor": 28708, + "sets": 28709, + "shared": 28710, + "Charles": 28711, + "password": 28712, + "\u0120gears": 28713, + "\u0120211": 28714, + "\u0120Hardware": 28715, + "rika": 28716, + "\u0120upstream": 28717, + "Hom": 28718, + "\u0120disproportionately": 28719, + "ivities": 28720, + "\u0120undefined": 28721, + "\u0120electrons": 28722, + "\u0120commemor": 28723, + "Eventually": 28724, + "\u0120><": 28725, + "\u0120irresponsible": 28726, + "218": 28727, + "\u0120Released": 28728, + "\u0120OVER": 28729, + "\u0120IGN": 28730, + "\u0120Bread": 28731, + "stellar": 28732, + "\u0120Sage": 28733, + "tted": 28734, + "damage": 28735, + "edition": 28736, + "\u0120Prec": 28737, + "\u0120lime": 28738, + "\u0120confinement": 28739, + "\u0120calorie": 28740, + "weapon": 28741, + "\u0120differing": 28742, + "\u0120Sina": 28743, + "mys": 28744, + "amd": 28745, + "\u0120intricate": 28746, + "kk": 28747, + "\u0120PAT": 28748, + "\u00c3\u00a3o": 28749, + "stones": 28750, + "links": 28751, + "\u0120ranch": 28752, + "Semitic": 28753, + "\u0120differentiate": 28754, + "\u0120Singer": 28755, + "occupied": 28756, + "\u0120fortress": 28757, + "cmd": 28758, + "\u0120interception": 28759, + "\u0120Ankara": 28760, + "\u0120rept": 28761, + "\u0120Solitaire": 28762, + "\u0120remake": 28763, + "pred": 28764, + "\u0120dared": 28765, + "autions": 28766, + "\u0120BACK": 28767, + "Running": 28768, + "\u0120debugging": 28769, + "\u0120graphs": 28770, + "399": 28771, + "\u0120Nigel": 28772, + "\u0120bun": 28773, + "\u0120pillow": 28774, + "\u0120progressed": 28775, + "fashioned": 28776, + "\u0120obedience": 28777, + "ERN": 28778, + "\u0120rehears": 28779, + "Cell": 28780, + "tl": 28781, + "Sher": 28782, + "\u0120herald": 28783, + "\u0120Payment": 28784, + "\u0120Cory": 28785, + "\u0120Dept": 28786, + "\u0120repent": 28787, + "\u0120Weak": 28788, + "uckland": 28789, + "\u0120pleasing": 28790, + "\u0120shortages": 28791, + "\u0120jurors": 28792, + "\u0120Kab": 28793, + "qqa": 28794, + "Anti": 28795, + "\u0120wow": 28796, + "\u0120RCMP": 28797, + "\u0120tsun": 28798, + "\u0120Sic": 28799, + "\u0120comprises": 28800, + "\u0120spies": 28801, + "\u0120precinct": 28802, + "nu": 28803, + "\u0120urges": 28804, + "\u0120timed": 28805, + "\u0120stripes": 28806, + "\u0120Boots": 28807, + "\u0120yen": 28808, + "Advanced": 28809, + "\u0120discrete": 28810, + "\u0120Archangel": 28811, + "employment": 28812, + "Diff": 28813, + "\u0120monuments": 28814, + "\u0120209": 28815, + "worker": 28816, + "\u0120196": 28817, + "\u0120Ig": 28818, + "utterstock": 28819, + "TPS": 28820, + "Jac": 28821, + "\u0120homelessness": 28822, + "\u0120commentator": 28823, + "\u0120racially": 28824, + "fing": 28825, + "seed": 28826, + "Ele": 28827, + "ellation": 28828, + "\u0120ethanol": 28829, + "\u0120parish": 28830, + "\u0120Dong": 28831, + "\u0120Awakening": 28832, + "\u0120deviation": 28833, + "\u0120Bearing": 28834, + "\u0120Tsuk": 28835, + "\u0120recess": 28836, + "\u0120lymph": 28837, + "\u0120Cannabis": 28838, + "\u00e5\u013e": 28839, + "\u0120NEWS": 28840, + "\u0120dra": 28841, + "\u0120Stefan": 28842, + "\u0120Wrong": 28843, + "\u0120SAM": 28844, + "\u0120loosely": 28845, + "\u0120interpreter": 28846, + "\u0120Plain": 28847, + "Government": 28848, + "\u0120bigotry": 28849, + "\u0120grenades": 28850, + "avez": 28851, + "pictured": 28852, + "\u0120mandated": 28853, + "\u0120Monk": 28854, + "\u0120Pedro": 28855, + "\u0120lava": 28856, + "274": 28857, + "\u0120cynical": 28858, + "\u0120Scrolls": 28859, + "locks": 28860, + "Mp": 28861, + "\u0120congregation": 28862, + "ornings": 28863, + "phil": 28864, + "\u0120Ibid": 28865, + "\u0120ferv": 28866, + "\u0120disappearing": 28867, + "\u0120arrogant": 28868, + "syn": 28869, + "\u0120Maver": 28870, + "\u0120Suit": 28871, + "241": 28872, + "\u0120abbre": 28873, + "ackers": 28874, + "Pa": 28875, + "\u0120Yel": 28876, + "Whenever": 28877, + "\u0120235": 28878, + "\u0120Vine": 28879, + "\u0120Anat": 28880, + "\u0120extinct": 28881, + "LET": 28882, + "\u0120executable": 28883, + "VERS": 28884, + "oxide": 28885, + "DNA": 28886, + "\u0120Prel": 28887, + "\u0120resentment": 28888, + "\u0120comprise": 28889, + "\u0120Aviv": 28890, + "\u0120interceptions": 28891, + "\u0120prolific": 28892, + "INA": 28893, + "\u0120Erin": 28894, + "thought": 28895, + "219": 28896, + "\u0120Psychiatry": 28897, + "unky": 28898, + "chemist": 28899, + "Ho": 28900, + "\u0120McCoy": 28901, + "\u0120bricks": 28902, + "Los": 28903, + "rily": 28904, + "\u0120USSR": 28905, + "\u0120rud": 28906, + "\u0120laud": 28907, + "\u0120Wise": 28908, + "\u0120Emerald": 28909, + "\u0120revived": 28910, + "\u0120damned": 28911, + "\u0120Repair": 28912, + "idem": 28913, + "ctica": 28914, + "\u0120patriarch": 28915, + "\u0120Nurs": 28916, + "meg": 28917, + "\u0120cheapest": 28918, + "reements": 28919, + "empty": 28920, + "\u0120Celebr": 28921, + "\u0120deprivation": 28922, + "chanted": 28923, + "\u0120Thumbnails": 28924, + "Energy": 28925, + "\u0120Ethan": 28926, + "\u0120Qing": 28927, + "\u0120opposes": 28928, + "WIND": 28929, + "vik": 28930, + "\u0120Mau": 28931, + "\u0120SUB": 28932, + "667": 28933, + "GRE": 28934, + "\u0120Volunte": 28935, + "nton": 28936, + "Cook": 28937, + "\u00e5\u0132": 28938, + "esque": 28939, + "\u0120plummet": 28940, + "\u0120suing": 28941, + "\u0120pronounce": 28942, + "\u0120resisting": 28943, + "\u0120Fishing": 28944, + "\u0120Trials": 28945, + "\u0120yell": 28946, + "\u0120310": 28947, + "\u0120induct": 28948, + "\u0120personalized": 28949, + "often": 28950, + "Reb": 28951, + "EMBER": 28952, + "\u0120viewpoint": 28953, + "\u0120existential": 28954, + "())": 28955, + "remove": 28956, + "MENTS": 28957, + "lasses": 28958, + "\u0120evapor": 28959, + "\u0120aisle": 28960, + "meta": 28961, + "\u0120reflective": 28962, + "\u0120entitlement": 28963, + "\u0120devised": 28964, + "music": 28965, + "ascade": 28966, + "\u0120winding": 28967, + "offset": 28968, + "\u0120accessibility": 28969, + "kered": 28970, + "Better": 28971, + "\u0120Johnston": 28972, + "thinking": 28973, + "Snow": 28974, + "\u0120Croatia": 28975, + "\u0120Atomic": 28976, + "271": 28977, + "348": 28978, + "\u0120textbook": 28979, + "\u0120Sixth": 28980, + "\u0120\u00d8\u00a7\u00d9\u0126": 28981, + "\u0120slider": 28982, + "\u0120Burger": 28983, + "bol": 28984, + "Sync": 28985, + "\u0120grandchildren": 28986, + "\u0120cerv": 28987, + "+)": 28988, + "\u0120eternity": 28989, + "\u0120tweeting": 28990, + "\u0120speculative": 28991, + "\u0120pivotal": 28992, + "\u0120WP": 28993, + "\u0120TER": 28994, + "ynamic": 28995, + "\u0120upl": 28996, + "\u0120Cats": 28997, + "perhaps": 28998, + "\u0120classmates": 28999, + "\u0120blatant": 29000, + "'-": 29001, + "\u0120lakh": 29002, + "antine": 29003, + "\u0120Borg": 29004, + "iom": 29005, + "/(": 29006, + "\u0120Athletic": 29007, + "\u0120sar": 29008, + "OTA": 29009, + "\u0120Hoffman": 29010, + "Nevertheless": 29011, + "\u0120adorable": 29012, + "\u0120spawned": 29013, + "Associated": 29014, + "\u0120Domestic": 29015, + "\u0120implant": 29016, + "\u0120Luxem": 29017, + "\u0120Kens": 29018, + "\u0120pumps": 29019, + "\u0120SAT": 29020, + "Attributes": 29021, + "509": 29022, + "avour": 29023, + "\u0120centralized": 29024, + "\u0120TN": 29025, + "\u0120freshly": 29026, + "\u0120Achieve": 29027, + "\u0120outsiders": 29028, + "herty": 29029, + "\u0120Ree": 29030, + "\u0120Towers": 29031, + "\u0120Dart": 29032, + "akable": 29033, + "\u0120mp": 29034, + "\u0120Heavenly": 29035, + "\u0120ripe": 29036, + "\u0120Caroline": 29037, + "ryan": 29038, + "\u0120classics": 29039, + "\u0120retiring": 29040, + "\u0120228": 29041, + "\u0120ah": 29042, + "\u0120dealings": 29043, + "\u0120punching": 29044, + "\u0120Chapman": 29045, + "Options": 29046, + "maxwell": 29047, + "volume": 29048, + "\u0120stal": 29049, + "\u0120exported": 29050, + "\u0120Quite": 29051, + "\u0120numerical": 29052, + "Burn": 29053, + "Fact": 29054, + "\u0120Keystone": 29055, + "\u0120trending": 29056, + "\u0120altering": 29057, + "\u0120Africans": 29058, + "478": 29059, + "\u0120MN": 29060, + "\u0120Knock": 29061, + "\u0120temptation": 29062, + "\u0120prestige": 29063, + "Overview": 29064, + "\u0120Traditional": 29065, + "\u0120Bahrain": 29066, + "Private": 29067, + "\u0120HOU": 29068, + "\u0120barr": 29069, + "\u0120Tat": 29070, + "Cube": 29071, + "USD": 29072, + "\u0120Grande": 29073, + "\u0120Gat": 29074, + "\u0120Flo": 29075, + "\u0120resides": 29076, + "\u0120indec": 29077, + "volent": 29078, + "\u0120perpetual": 29079, + "ubes": 29080, + "\u0120worldview": 29081, + "\u0120Quantum": 29082, + "\u0120filtered": 29083, + "\u0120ensu": 29084, + "orgetown": 29085, + "ERSON": 29086, + "\u0120Mild": 29087, + "379": 29088, + "OTT": 29089, + "\u00c3\u00a5": 29090, + "\u0120vitamins": 29091, + "\u0120ribbon": 29092, + "\u0120sincerely": 29093, + "\u0120Hin": 29094, + "\u0120eighteen": 29095, + "\u0120contradictory": 29096, + "\u0120glaring": 29097, + "\u0120expectancy": 29098, + "\u0120conspir": 29099, + "\u0120monstrous": 29100, + "\u0120380": 29101, + "reci": 29102, + "\u0120handic": 29103, + "\u0120pumped": 29104, + "\u0120indicative": 29105, + "\u0120rapp": 29106, + "\u0120avail": 29107, + "\u0120LEGO": 29108, + "\u0120Marijuana": 29109, + "1985": 29110, + "erton": 29111, + "\u0120twentieth": 29112, + "################################": 29113, + "\u0120Swamp": 29114, + "\u0120valuation": 29115, + "\u0120affiliates": 29116, + "adjusted": 29117, + "\u0120Facility": 29118, + "262": 29119, + "\u0120enzymes": 29120, + "itudinal": 29121, + "\u0120imprint": 29122, + "Site": 29123, + "\u0120installer": 29124, + "\u0120TRA": 29125, + "mology": 29126, + "linear": 29127, + "\u0120Collective": 29128, + "igating": 29129, + "\u0120Token": 29130, + "\u0120speculated": 29131, + "KN": 29132, + "\u0120Cly": 29133, + "ority": 29134, + "\u0120defer": 29135, + "\u0120inspectors": 29136, + "approved": 29137, + "RM": 29138, + "\u0120Suns": 29139, + "\u0120informing": 29140, + "\u0120Syracuse": 29141, + "ibli": 29142, + "765": 29143, + "\u0120glove": 29144, + "\u0120authorize": 29145, + "\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6": 29146, + "\u0120Cruise": 29147, + "\u0120contracting": 29148, + "shell": 29149, + "IFE": 29150, + "\u0120Jewel": 29151, + "pract": 29152, + "\u0120Photoshop": 29153, + "\u0120Knowing": 29154, + "harm": 29155, + "\u0120attractions": 29156, + "adan": 29157, + "etus": 29158, + "018": 29159, + "wagen": 29160, + "Alt": 29161, + "\u0120multiply": 29162, + "\u0120equilibrium": 29163, + ":{": 29164, + "\u0120Fighters": 29165, + "\u0120Edgar": 29166, + "\u0120fourteen": 29167, + "Govern": 29168, + "\u0120misuse": 29169, + "\u0120abusing": 29170, + "\u0120ancestry": 29171, + "ramer": 29172, + "644": 29173, + "\u0120worms": 29174, + "\u0120thicker": 29175, + "\u0120Combine": 29176, + "\u0120peasants": 29177, + "\u0120vind": 29178, + "\u0120conquest": 29179, + "\u0120mocked": 29180, + "\u0120cinnamon": 29181, + "\u0120Cald": 29182, + "\u0120Gallup": 29183, + "\u0120avoidance": 29184, + "\u0120incarnation": 29185, + "\u0120Strat": 29186, + "\u0120tasted": 29187, + "enta": 29188, + "\u0120Neal": 29189, + "pared": 29190, + "\u0120terminology": 29191, + "jection": 29192, + "Scientists": 29193, + "\u0120INS": 29194, + "\u0120Dee": 29195, + "\u0120directories": 29196, + "Road": 29197, + "\u0120Shap": 29198, + "bright": 29199, + "\u0120Directors": 29200, + "\u0120Column": 29201, + "\u0120bob": 29202, + "\u0120preferably": 29203, + "\u0120glitch": 29204, + "furt": 29205, + "\u0120eg": 29206, + "idis": 29207, + "CBC": 29208, + "\u0120surrendered": 29209, + "\u0120testament": 29210, + "336": 29211, + "uggest": 29212, + "\u0120Nil": 29213, + "another": 29214, + "\u0120pathetic": 29215, + "\u0120Donna": 29216, + "\u0120218": 29217, + "\u0120Avery": 29218, + "\u0120whiskey": 29219, + "\u0120fixture": 29220, + "\u0120Conquest": 29221, + "\u0120bets": 29222, + "Occ": 29223, + "\u0120Leicester": 29224, + "].\"": 29225, + "\u0120));": 29226, + "\u0120flashes": 29227, + "456": 29228, + "\u0120masked": 29229, + "gebra": 29230, + "\u0120computed": 29231, + "chel": 29232, + "auder": 29233, + "\u0120defeats": 29234, + "\u0120Liberation": 29235, + "\u0120Osama": 29236, + "\u0120Vive": 29237, + "Changes": 29238, + "Channel": 29239, + "\u0120tariffs": 29240, + "\u0120mage": 29241, + "\u0120Sax": 29242, + "\u0120inadvertently": 29243, + "\u0120CRE": 29244, + "\u0120Reaper": 29245, + "inky": 29246, + "grading": 29247, + "\u0120stereotyp": 29248, + "\u0120curl": 29249, + "\u0120FANT": 29250, + "\u0120frameworks": 29251, + "Mom": 29252, + "\u0120Anch": 29253, + "\u0120flavour": 29254, + "carbon": 29255, + "\u0120permitting": 29256, + "letcher": 29257, + "\u0120Mozilla": 29258, + "\u0120Parking": 29259, + "\u0120Champ": 29260, + "Scroll": 29261, + "\u0120murderer": 29262, + "\u0120rested": 29263, + "\u0120owes": 29264, + "\u0120Poss": 29265, + "ADD": 29266, + "IFF": 29267, + "resolution": 29268, + "\u0120Mining": 29269, + "\u0120comparative": 29270, + "Dim": 29271, + "\u0120neighbouring": 29272, + "\u0120AST": 29273, + "\u0120Toxic": 29274, + "\u0120biases": 29275, + "\u0120gunfire": 29276, + "urous": 29277, + "\u0120Moment": 29278, + "1983": 29279, + "\u0120pervasive": 29280, + "ttp": 29281, + "\u0120Normally": 29282, + "rir": 29283, + "Sarah": 29284, + "\u0120Albany": 29285, + "\u0120unsett": 29286, + "\u0120SMS": 29287, + "ipers": 29288, + "layer": 29289, + "\u0120Whites": 29290, + "uple": 29291, + "\u0120turbo": 29292, + "\u0120Leeds": 29293, + "\u0120thats": 29294, + "\u0120Miner": 29295, + "MER": 29296, + "\u0120Reign": 29297, + "\u0120perme": 29298, + "\u0120Blitz": 29299, + "\u01201934": 29300, + "\u0120intimidating": 29301, + "tube": 29302, + "\u0120eccentric": 29303, + "abolic": 29304, + "boxes": 29305, + "\u0120Associates": 29306, + "votes": 29307, + "\u0120simulate": 29308, + "umbo": 29309, + "astery": 29310, + "\u0120shipments": 29311, + "FFFF": 29312, + "anth": 29313, + "\u0120seasoned": 29314, + "\u0120experimentation": 29315, + "\u00e2\u0138\u0142": 29316, + "laws": 29317, + "Meet": 29318, + "iddles": 29319, + "antics": 29320, + "Rating": 29321, + "ISIS": 29322, + "hift": 29323, + "\u0120fronts": 29324, + "buf": 29325, + "017": 29326, + "\u0120unatt": 29327, + "\u0120Dil": 29328, + "leases": 29329, + "\u0120Gardens": 29330, + "777": 29331, + "touch": 29332, + "vell": 29333, + "458": 29334, + "\u0120=====": 29335, + "saving": 29336, + "\u0120erosion": 29337, + "\u0120Quin": 29338, + "\u0120earns": 29339, + "\u0120accomplishment": 29340, + "\u0120Wei": 29341, + "\u0120<[": 29342, + "_____": 29343, + "\u0120irrig": 29344, + "\u0120Teddy": 29345, + "\u0120conquered": 29346, + "\u0120Armored": 29347, + "\u0120asserts": 29348, + "\u0120manipulating": 29349, + "r\u00c3\u00a9": 29350, + "\u0120transcripts": 29351, + "Gallery": 29352, + "\u0120plotting": 29353, + "Neil": 29354, + "\u0120betrayal": 29355, + "loader": 29356, + "\u0120Sul": 29357, + "\u0120displacement": 29358, + "\u0120royalty": 29359, + "\u0120WI": 29360, + "heit": 29361, + "\u0120Devices": 29362, + "allel": 29363, + "\u0120municipalities": 29364, + "\u0120canal": 29365, + "Stars": 29366, + "\u0120UAE": 29367, + "\u0120\"\u00e2\u0122\u00a6": 29368, + "\u0120CU": 29369, + "above": 29370, + "\u0120resonance": 29371, + "\u0120guiActiveUn": 29372, + "added": 29373, + "\u0120Braves": 29374, + "\u0120Ibn": 29375, + "\u0120hereby": 29376, + "\u0120BRE": 29377, + "\u0120shareholder": 29378, + "\u0120Hir": 29379, + "\u0120Ji": 29380, + "\u0120strangely": 29381, + "\u0120admired": 29382, + "\u0120plight": 29383, + "\u0120bachelor": 29384, + "\u0120Pole": 29385, + "ciplinary": 29386, + "Tony": 29387, + "\u0120Armenian": 29388, + "\u0120unman": 29389, + "\u0120Zionist": 29390, + "Stage": 29391, + "iscover": 29392, + "\u0120automotive": 29393, + "\u0120sidelines": 29394, + "\u0120slick": 29395, + "\u0120Renaissance": 29396, + "\u0120FUN": 29397, + "Images": 29398, + "\u0120Haj": 29399, + "\u0120ping": 29400, + "\u0120shortcut": 29401, + "\u0120Blvd": 29402, + "\u0120Looks": 29403, + "\u0120bursts": 29404, + "\u0120clamp": 29405, + "\u0120mish": 29406, + "\u0120sorting": 29407, + "\u0120patriot": 29408, + "\u0120correctness": 29409, + "\u0120Scandinav": 29410, + "\u0120Cavaliers": 29411, + "python": 29412, + "azar": 29413, + "\u0120375": 29414, + "\u0120Jaune": 29415, + "409": 29416, + "\u0120detrimental": 29417, + "\u0120stabbing": 29418, + "\u0120poisoned": 29419, + "\u0120fountain": 29420, + "ocent": 29421, + "orst": 29422, + "\u0120Mari": 29423, + "\u0120rains": 29424, + "\u0120Overs": 29425, + "\u0120Institution": 29426, + "udget": 29427, + "AMY": 29428, + "tale": 29429, + "\u0120KR": 29430, + "\u0120Prices": 29431, + "\u0120headaches": 29432, + "\u0120landsl": 29433, + "\u0120Aura": 29434, + "Bonus": 29435, + "\u0120Zhao": 29436, + "\u0120Hip": 29437, + "\u0120hops": 29438, + "\u0120Kurdistan": 29439, + "\u0120exploiting": 29440, + "ryn": 29441, + "\u0120hypocrisy": 29442, + "opening": 29443, + "\u0120gunshot": 29444, + "\u0120wed": 29445, + "interstitial": 29446, + "Interstitial": 29447, + "\u0120amen": 29448, + "Breaking": 29449, + "\u0120marketed": 29450, + "Wire": 29451, + "\u0120Crowd": 29452, + "Continue": 29453, + "\u0120Known": 29454, + "\u0120Effective": 29455, + "orean": 29456, + "izons": 29457, + "Joseph": 29458, + "\u0120escalation": 29459, + "username": 29460, + "\u0120curtain": 29461, + "ATES": 29462, + "\u0120PAR": 29463, + "\u0120Miy": 29464, + "\u0120counterfe": 29465, + "lene": 29466, + "\u0120contenders": 29467, + "daily": 29468, + "\u0120Asc": 29469, + "\u0120Phillip": 29470, + "mostly": 29471, + "\u0120filename": 29472, + "hene": 29473, + "\u0120resembling": 29474, + "\u0120staging": 29475, + "\u0120Chloe": 29476, + "\u0120wiring": 29477, + "Hon": 29478, + "\u0120Renew": 29479, + "ottage": 29480, + "\u0120Hybrid": 29481, + "much": 29482, + "\u0120strokes": 29483, + "\u0120policymakers": 29484, + "APTER": 29485, + "\u0120Arkham": 29486, + "plot": 29487, + "\u0120assistants": 29488, + "\u0120deport": 29489, + "\u0120Sega": 29490, + "\u0120influenza": 29491, + "\u0120Cursed": 29492, + "\u0120Kobe": 29493, + "\u0120skinny": 29494, + "Provider": 29495, + "\u0120Rip": 29496, + "\u0120incremental": 29497, + "products": 29498, + "BF": 29499, + "\u0120dome": 29500, + "\u0120Credits": 29501, + "\u0120losers": 29502, + "ints": 29503, + "\u0120Betty": 29504, + "\u0120Talent": 29505, + "\u0120DAM": 29506, + "Lv": 29507, + "Ess": 29508, + "\u0120dens": 29509, + "temp": 29510, + "Judge": 29511, + "odic": 29512, + "\u0120'(": 29513, + "URES": 29514, + "etsk": 29515, + "VO": 29516, + "\u0120retrieved": 29517, + "\u0120architects": 29518, + "\u00d9\u0129": 29519, + "\u0120ethic": 29520, + "\u0120Secondary": 29521, + "stocks": 29522, + "adia": 29523, + "\u0120325": 29524, + "\u0120Opinion": 29525, + "\u0120simultaneous": 29526, + "\u0120dizz": 29527, + "ulp": 29528, + "\u0120smuggling": 29529, + "ippery": 29530, + "Random": 29531, + "facing": 29532, + "\u0120Das": 29533, + "\u0120stockp": 29534, + "\u0120disclosures": 29535, + "pointer": 29536, + "\u0120coral": 29537, + "\u0120Selection": 29538, + "\u0120Pike": 29539, + "ivalent": 29540, + "\u0120ruthless": 29541, + "\u0120Rim": 29542, + "\u0120ensuing": 29543, + "\u0120Experiment": 29544, + "\u0120congressman": 29545, + "\u0120believer": 29546, + "\u0120unspecified": 29547, + "\u0120Mord": 29548, + "\u0120knowledgeable": 29549, + "\u0120VERY": 29550, + "TX": 29551, + "\u0120straps": 29552, + "\u0120turf": 29553, + "apeshifter": 29554, + "\u0120marital": 29555, + "\u0120flock": 29556, + "\u00e3\u0123\u0128": 29557, + "263": 29558, + "AMES": 29559, + "\u0120Opposition": 29560, + "\u0120treasures": 29561, + "\u0120GOD": 29562, + "\u0120modeled": 29563, + "\u0120WORLD": 29564, + "\u0120([": 29565, + "\u0120Usage": 29566, + "HF": 29567, + "\u0120$(": 29568, + "ussed": 29569, + "\u0120pioneer": 29570, + "Eight": 29571, + "parse": 29572, + "bread": 29573, + "ritz": 29574, + "\u0120Miranda": 29575, + "\u0120Kant": 29576, + "++)": 29577, + "oren": 29578, + "\u0120provoked": 29579, + "\u0120breeds": 29580, + "\u0120Includes": 29581, + "\u0120Pastebin": 29582, + "\u0120Flip": 29583, + "Java": 29584, + "\u0120brink": 29585, + "\u0120rumored": 29586, + "\u0120unseen": 29587, + "\u0120garnered": 29588, + "\u0120Defin": 29589, + "alted": 29590, + "\u0120tattoos": 29591, + "\u0120hesitation": 29592, + "isitions": 29593, + "\u0120Weaver": 29594, + "\u0120Reporting": 29595, + "\u0120therapies": 29596, + "\u0120consultants": 29597, + "\u0120residual": 29598, + "\u0120Mali": 29599, + "\u0120Roma": 29600, + "iago": 29601, + "\u0120Residents": 29602, + "ubi": 29603, + "\u0120remedies": 29604, + "\u0120adaptive": 29605, + "\u0120Alive": 29606, + "\u0120Barcl": 29607, + "\u0120wallets": 29608, + "crypt": 29609, + "etermination": 29610, + "\u0120Pelosi": 29611, + "\u0120slipping": 29612, + "otonin": 29613, + "\u0120alliances": 29614, + "patrick": 29615, + "iris": 29616, + "\u0120orth": 29617, + "\u0120Perkins": 29618, + "\u0120DeV": 29619, + "\u0120Gets": 29620, + "\u0120drying": 29621, + "gee": 29622, + "forest": 29623, + "\u0120Forget": 29624, + "orem": 29625, + "339": 29626, + "\u0120vaguely": 29627, + "\u0120Dion": 29628, + "\u0120Porn": 29629, + "\u0120HOW": 29630, + "\u0120pneum": 29631, + "\u0120rubble": 29632, + "\u0120Taste": 29633, + "encia": 29634, + "\u0120Gel": 29635, + "\u0120dst": 29636, + "\u0120245": 29637, + "\u0120Morocco": 29638, + "inflamm": 29639, + "\u0120Twins": 29640, + "\u0120bots": 29641, + "daughter": 29642, + "\u0120Balk": 29643, + "\u0120brethren": 29644, + "\u0120logos": 29645, + "\u0120gobl": 29646, + "fps": 29647, + "\u0120subdivision": 29648, + "\u0120pawn": 29649, + "\u0120squeezed": 29650, + "\u0120morale": 29651, + "\u0120DW": 29652, + "'\"": 29653, + "\u0120knot": 29654, + "ooky": 29655, + "\u0120divisive": 29656, + "\u0120boosted": 29657, + "chy": 29658, + "\u00e3\u0125\u0132": 29659, + "ifact": 29660, + "\u0120newcomers": 29661, + "\u0120Wrestling": 29662, + "\u0120scouts": 29663, + "wolves": 29664, + "Rat": 29665, + "\u0120nineteenth": 29666, + "\u0120Osborne": 29667, + "Stats": 29668, + "\u0120empowered": 29669, + "\u0120psychopath": 29670, + "\u0120OEM": 29671, + "uggage": 29672, + "\u0120PK": 29673, + "\u0120Mohammad": 29674, + "Pak": 29675, + "\u0120anarchists": 29676, + "\u0120Extract": 29677, + "esthes": 29678, + "\u0120Stockholm": 29679, + "loo": 29680, + "\u0120Graph": 29681, + "\u0120deploying": 29682, + "\u0120Stranger": 29683, + "\u0120Mold": 29684, + "\u0120staffer": 29685, + "\u0120discounted": 29686, + "uckle": 29687, + "please": 29688, + "\u0120Landing": 29689, + "\u00c3\u0143a": 29690, + "\u0120193": 29691, + "\u0120ante": 29692, + "\u0120repetition": 29693, + "\u0120+/-": 29694, + "\u0120parody": 29695, + "\u0120lively": 29696, + "AAA": 29697, + "\u0120Horus": 29698, + "\u0120pits": 29699, + "inders": 29700, + "LOC": 29701, + "\u0120Venice": 29702, + "406": 29703, + "\u0120Discover": 29704, + "\u00e2\u0128": 29705, + "ellectual": 29706, + "\u0120pens": 29707, + "\u0120eyel": 29708, + "iguous": 29709, + "Impl": 29710, + "\u0120joking": 29711, + "\u0120inval": 29712, + "\u0120Belfast": 29713, + "\u0120creditors": 29714, + "\u0120Skywalker": 29715, + "ovsky": 29716, + "\u0120ceasefire": 29717, + "\u0120seals": 29718, + "isoft": 29719, + ")).": 29720, + "\u0120Felix": 29721, + "ITS": 29722, + "\u0120tresp": 29723, + "\u0120Blockchain": 29724, + "eware": 29725, + "\u0120Schwar": 29726, + "enne": 29727, + "mounted": 29728, + "\u0120Beacon": 29729, + "lesh": 29730, + "\u0120immensely": 29731, + "\u0120cheering": 29732, + "Employ": 29733, + "scene": 29734, + "ishly": 29735, + "atchewan": 29736, + "\u0120Nicolas": 29737, + "\u0120drained": 29738, + "\u0120Exit": 29739, + "\u0120Azerb": 29740, + "jun": 29741, + "\u0120floated": 29742, + "uania": 29743, + "Deep": 29744, + "\u0120superv": 29745, + "\u0120mystical": 29746, + "\u0120Dollar": 29747, + "\u0120Apostle": 29748, + "\u0120REL": 29749, + "\u0120Provided": 29750, + "\u0120Bucks": 29751, + "\u00e3\u0125\u00b4": 29752, + "cutting": 29753, + "\u0120enhancements": 29754, + "\u0120Penguins": 29755, + "\u0120Isaiah": 29756, + "\u0120jerk": 29757, + "\u0120Wyn": 29758, + "\u0120stalled": 29759, + "\u0120cryptocurrencies": 29760, + "\u0120Roland": 29761, + "single": 29762, + "\u0120lumin": 29763, + "\u0120Fellow": 29764, + "\u0120Capacity": 29765, + "\u0120Kazakh": 29766, + "WN": 29767, + "\u0120financed": 29768, + "389": 29769, + "\u0120tid": 29770, + "\u0120collusion": 29771, + "\u0120Myr": 29772, + "\u00ee\u0122": 29773, + "Senator": 29774, + "\u0120pediatric": 29775, + "\u0120neatly": 29776, + "\u0120sandwiches": 29777, + "\u0120Architecture": 29778, + "\u0120tucked": 29779, + "\u0120balcony": 29780, + "\u0120earthquakes": 29781, + "quire": 29782, + "Future": 29783, + "\u0120hefty": 29784, + "\u00e9\u0139": 29785, + "\u0120specializes": 29786, + "\u0120stresses": 29787, + "\u0120sender": 29788, + "\u0120misunderstanding": 29789, + "\u0120epile": 29790, + "\u0120provoke": 29791, + "\u0120Colors": 29792, + "\u0120dismay": 29793, + "uko": 29794, + "[_": 29795, + "586": 29796, + "neutral": 29797, + "\u0120donating": 29798, + "\u0120Randall": 29799, + "Multi": 29800, + "\u0120conveniently": 29801, + "\u0120Sung": 29802, + "\u0120Coca": 29803, + "\u0120tents": 29804, + "\u0120Acceler": 29805, + "\u0120partnered": 29806, + "272": 29807, + "irming": 29808, + "\u0120BAS": 29809, + "sometimes": 29810, + "\u0120objected": 29811, + "ubric": 29812, + "posed": 29813, + "LCS": 29814, + "grass": 29815, + "\u0120attributable": 29816, + "VIS": 29817, + "Israeli": 29818, + "\u0120repeats": 29819, + "\u0120RM": 29820, + "vag": 29821, + "uta": 29822, + "inous": 29823, + "\u0120inert": 29824, + "\u0120Miguel": 29825, + "\u00e6\u0143": 29826, + "\u0120Hawaiian": 29827, + "Board": 29828, + "\u0120artific": 29829, + "\u0120Azerbai": 29830, + "asio": 29831, + "\u0120Rent": 29832, + "AIN": 29833, + "\u0120appliances": 29834, + "\u0120nationality": 29835, + "\u0120asshole": 29836, + "\u0120Neb": 29837, + "\u0120notch": 29838, + "hani": 29839, + "\u0120Bride": 29840, + "Availability": 29841, + "\u0120intercepted": 29842, + "\u0120continental": 29843, + "\u0120swelling": 29844, + "\u0120Perspect": 29845, + "bies": 29846, + ".<": 29847, + "ithmetic": 29848, + "\u0120Lara": 29849, + "\u0120tempting": 29850, + "addr": 29851, + "\u0120overseeing": 29852, + "clad": 29853, + "\u0120DV": 29854, + "\u0120Gingrich": 29855, + "\u0120mun": 29856, + "\u0120Appropri": 29857, + "\u0120alterations": 29858, + "\u0120Patreon": 29859, + "\u0120havoc": 29860, + "\u0120disciplines": 29861, + "\u0120notoriously": 29862, + "akuya": 29863, + "ieri": 29864, + "?).": 29865, + "\u0120Went": 29866, + "\u0120silicon": 29867, + "\u0120tremb": 29868, + "Container": 29869, + "Known": 29870, + "\u0120mortar": 29871, + "este": 29872, + "icka": 29873, + "Arthur": 29874, + "\u0120Previously": 29875, + "\u0120Marty": 29876, + "\u0120sparse": 29877, + "gins": 29878, + "\u0120inward": 29879, + "\u0120Participant": 29880, + "Copy": 29881, + "\u0120Misc": 29882, + "\u0120antibiotic": 29883, + "\u0120Retro": 29884, + "\u0120elusive": 29885, + "\u0120assail": 29886, + "\u0120Battalion": 29887, + "\u0120Bought": 29888, + "\u0120diminish": 29889, + "\u0120Europa": 29890, + "session": 29891, + "\u0120Dangerous": 29892, + "iesel": 29893, + "\u0120disbelief": 29894, + "\u0120blasts": 29895, + "extreme": 29896, + "\u0120Boyd": 29897, + "\u0120Projects": 29898, + "\u0120Guys": 29899, + "\u0120undergone": 29900, + "\u0120grill": 29901, + "\u0120Dwight": 29902, + "\u0120197": 29903, + "USER": 29904, + "\u0120filesystem": 29905, + "\u0120clocks": 29906, + "Taylor": 29907, + "\u0120wrapper": 29908, + "\u0120folding": 29909, + "ousand": 29910, + "\u0120Philippine": 29911, + "ATIONAL": 29912, + "\u0120Perth": 29913, + "\u0120ashes": 29914, + "\u0120accumulate": 29915, + "\u0120Gateway": 29916, + "Shop": 29917, + "orkshire": 29918, + "Han": 29919, + "\u0120Barrel": 29920, + "\u0120Leh": 29921, + "\u0120XV": 29922, + "\u0120whim": 29923, + "\u0120repo": 29924, + "\u0120CG": 29925, + "\u0120Mam": 29926, + "\u0120incorporating": 29927, + "\u0120bailout": 29928, + "\u0120linguistic": 29929, + "\u0120disinteg": 29930, + "CLE": 29931, + "\u0120cinematic": 29932, + "\u0120Fiber": 29933, + "Syn": 29934, + "ilion": 29935, + "\u0120Compos": 29936, + "chens": 29937, + "\u0120neoc": 29938, + "\u0120boiled": 29939, + "FINE": 29940, + "ono": 29941, + "uncle": 29942, + "iken": 29943, + "\u0120BM": 29944, + "\u00ce\u00b9": 29945, + "\u0120receipts": 29946, + "\u0120disposed": 29947, + "\u0120Thirty": 29948, + "\u0120Rough": 29949, + "\u0120ABS": 29950, + "\u0120notwithstanding": 29951, + "ollen": 29952, + "#$": 29953, + "\u0120unreliable": 29954, + "\u0120bloom": 29955, + "\u0120mediocre": 29956, + "\u0120tram": 29957, + "\u0120Tasman": 29958, + "\u0120shakes": 29959, + "\u0120manifesto": 29960, + "\u0120MW": 29961, + "\u0120satisfactory": 29962, + "\u0120shores": 29963, + "\u0120computation": 29964, + "\u0120assertions": 29965, + "ormons": 29966, + "arag": 29967, + "abit": 29968, + "Democrats": 29969, + "\u0120Loot": 29970, + "\u0120Volks": 29971, + "haired": 29972, + "\u0120gravitational": 29973, + "Sing": 29974, + "\u0120Miz": 29975, + "\u0120throttle": 29976, + "\u0120tyranny": 29977, + "\u0120Views": 29978, + "\u0120robber": 29979, + "\u0120Minority": 29980, + "\u0120shrine": 29981, + "scope": 29982, + "purpose": 29983, + "\u0120nucleus": 29984, + "ourcing": 29985, + "\u0120USDA": 29986, + "\u0120DHS": 29987, + "wra": 29988, + "\u0120Bowie": 29989, + "Scale": 29990, + "\u0120BEL": 29991, + "xi": 29992, + "Iter": 29993, + "\u0120(),": 29994, + "wright": 29995, + "\u0120sailors": 29996, + "oused": 29997, + "NASA": 29998, + "\u0120Proof": 29999, + "\u0120Mineral": 30000, + "token": 30001, + "\u0120FD": 30002, + "Rew": 30003, + "\u0120ell": 30004, + "630": 30005, + "\u0120chancellor": 30006, + "\u0120Gos": 30007, + "\u0120amounted": 30008, + "\u0120Recre": 30009, + "omez": 30010, + "\u0120Optim": 30011, + "\u0120Olive": 30012, + "\u0120tracker": 30013, + "owler": 30014, + "\u0120Unique": 30015, + "Root": 30016, + "\u0120maritime": 30017, + "\u0120Quran": 30018, + "\u0120Adapt": 30019, + "\u0120ecosystems": 30020, + "\u0120Repeat": 30021, + "\u0120Soy": 30022, + "\u0120IMP": 30023, + "\u0120graduating": 30024, + "andem": 30025, + "Pur": 30026, + "\u0120Reset": 30027, + "\u0120Trick": 30028, + "\u0120Philly": 30029, + "\u0120Tue": 30030, + "\u0120Malaysian": 30031, + "\u0120climax": 30032, + "\u0120bury": 30033, + "\u0120conspic": 30034, + "\u0120Southampton": 30035, + "\u0120Flowers": 30036, + "\u0120escorted": 30037, + "\u0120Educational": 30038, + "\u0120IRC": 30039, + "\u0120brutally": 30040, + "eating": 30041, + "\u0120pillar": 30042, + "\u0120Sang": 30043, + "\u0120Jude": 30044, + "arling": 30045, + "\u0120Amnesty": 30046, + "\u0120reminding": 30047, + "\u0120Administrative": 30048, + "hesda": 30049, + "\u0120flashed": 30050, + "\u0120PBS": 30051, + "perate": 30052, + "feature": 30053, + "\u0120swipe": 30054, + "\u0120graves": 30055, + "oultry": 30056, + "261": 30057, + "breaks": 30058, + "\u0120Guer": 30059, + "\u0120shrimp": 30060, + "\u0120Voting": 30061, + "quist": 30062, + "\u0120analytical": 30063, + "\u0120tablespoons": 30064, + "\u0120SOU": 30065, + "\u0120researched": 30066, + "\u0120disrupted": 30067, + "\u0120jour": 30068, + "\u0120replica": 30069, + "\u0120cartoons": 30070, + "bians": 30071, + "})": 30072, + "copy": 30073, + "Got": 30074, + "ouched": 30075, + "PUT": 30076, + "\u0120swarm": 30077, + "notations": 30078, + "said": 30079, + "\u0120rebuilt": 30080, + "\u0120collaborate": 30081, + "\u0120raging": 30082, + "\u0120nar": 30083, + "\u0120demographics": 30084, + "\u0120DDR": 30085, + "\u0120distrust": 30086, + "ossier": 30087, + "\u0120Kro": 30088, + "\u0120pumpkin": 30089, + "\u0120regrets": 30090, + "\u0120fatalities": 30091, + "\u0120Lens": 30092, + "\u0120Ole": 30093, + "pd": 30094, + "\u0120puppet": 30095, + "\u0120Outlook": 30096, + "\u0120Stam": 30097, + "Ol": 30098, + "Fair": 30099, + "UU": 30100, + "\u0120rewritten": 30101, + "\u00c4\u00b1": 30102, + "\u0120fascinated": 30103, + "\u0120vectors": 30104, + "\u0120tribunal": 30105, + "uay": 30106, + "\u0120Mats": 30107, + "\u0120Coins": 30108, + "[[": 30109, + "\u0120181": 30110, + "\u0120renders": 30111, + "\u0120Kaepernick": 30112, + "\u0120espionage": 30113, + "\u0120summ": 30114, + "\u0120ditch": 30115, + "Account": 30116, + "\u0120spreadsheet": 30117, + "\u0120mutant": 30118, + "past": 30119, + "407": 30120, + "\u0120dye": 30121, + "\u0120initiation": 30122, + "\u01204000": 30123, + "\u0120punishable": 30124, + "\u0120thinner": 30125, + "\u0120Khal": 30126, + "\u0120intermedi": 30127, + "Dun": 30128, + "\u0120Gotham": 30129, + "\u0120eagerly": 30130, + "\u0120vaginal": 30131, + "powers": 30132, + "VW": 30133, + "\u0120WATCHED": 30134, + "\u0120predator": 30135, + "amsung": 30136, + "\u0120disparity": 30137, + "\u0120[*": 30138, + "\u0120amph": 30139, + "\u0120outskirts": 30140, + "\u0120Spirits": 30141, + "\u0120skeletal": 30142, + "\u00d0\u00bb": 30143, + "\u0120Rear": 30144, + "\u0120issuance": 30145, + "\u0120Logic": 30146, + "released": 30147, + "ZZ": 30148, + "\u0120Bound": 30149, + "Entry": 30150, + "\u0120exits": 30151, + "isol": 30152, + "\u0120Founder": 30153, + "\u0120wre": 30154, + "\u0120Greenland": 30155, + "\u0120MMO": 30156, + "taker": 30157, + "INC": 30158, + "\u00e3\u0123\u00be": 30159, + "\u0120hourly": 30160, + "henko": 30161, + "\u0120fantasies": 30162, + "\u0120disob": 30163, + "\u0120demolition": 30164, + "\u00e3\u0125\u012d": 30165, + "\u0120enlisted": 30166, + "ratulations": 30167, + "\u0120misguided": 30168, + "\u0120ensured": 30169, + "\u0120discouraged": 30170, + "mort": 30171, + "\u0120flank": 30172, + "\u0120cess": 30173, + "\u0120reacts": 30174, + "\u0120Sere": 30175, + "sensitive": 30176, + "\u0120Serpent": 30177, + "assad": 30178, + "\u0120247": 30179, + "\u0120calmly": 30180, + "busters": 30181, + "\u0120bleed": 30182, + "\u0120Stro": 30183, + "\u0120amusement": 30184, + "\u0120Antarctica": 30185, + "\u0120scept": 30186, + "\u0120Gaw": 30187, + "aq": 30188, + "asonic": 30189, + "\u0120sprawling": 30190, + "native": 30191, + "aturated": 30192, + "\u0120Battlefield": 30193, + "IVERS": 30194, + "EB": 30195, + "\u0120Gems": 30196, + "\u0120Northwestern": 30197, + "\u0120Films": 30198, + "\u0120Automatic": 30199, + "\u0120apprehend": 30200, + "\u00e3\u0123\u00a8": 30201, + "\u0120guiName": 30202, + "\u0120backend": 30203, + "\u0120evidenced": 30204, + "geant": 30205, + "012": 30206, + "\u0120Siege": 30207, + "\u0120externalTo": 30208, + "\u0120unfocusedRange": 30209, + "\u0120guiActiveUnfocused": 30210, + "\u0120guiIcon": 30211, + "\u0120externalToEVA": 30212, + "\u0120externalToEVAOnly": 30213, + "Fri": 30214, + "chard": 30215, + "enaries": 30216, + "\u0120chiefs": 30217, + "\u0120cf": 30218, + "\u0120HUD": 30219, + "\u0120corrobor": 30220, + "\u0120dB": 30221, + "\u0120Taken": 30222, + "\u0120Patricia": 30223, + "rail": 30224, + "\u0120Charm": 30225, + "\u0120Libertarian": 30226, + "rieve": 30227, + "Personal": 30228, + "\u0120OUR": 30229, + "geries": 30230, + "\u0120dumping": 30231, + "\u0120neurological": 30232, + "itimate": 30233, + "\u0120Clintons": 30234, + "rafted": 30235, + "\u0120Molly": 30236, + "\u0120terminals": 30237, + "register": 30238, + "\u0120flare": 30239, + "\u0120encoded": 30240, + "\u0120autopsy": 30241, + "pel": 30242, + "machine": 30243, + "\u0120exemptions": 30244, + "\u0120Royals": 30245, + "distance": 30246, + "\u0120drafts": 30247, + "\u0120lame": 30248, + "\u0120Cunning": 30249, + "\u0120spouses": 30250, + "\u0120Markets": 30251, + "\u0120Carrier": 30252, + "\u0120implying": 30253, + "\u0120Yak": 30254, + "sid": 30255, + "\u0120loser": 30256, + "\u0120vigilant": 30257, + "\u0120impeachment": 30258, + "\u0120augmented": 30259, + "\u0120Employees": 30260, + "\u0120unintended": 30261, + "ternally": 30262, + "\u0120Watt": 30263, + "\u0120recognizable": 30264, + "essim": 30265, + "\u00e6\u013f": 30266, + "\u0120coated": 30267, + "rha": 30268, + "\u0120lieutenant": 30269, + "\u0120Legislation": 30270, + "published": 30271, + "444": 30272, + "013": 30273, + "\u0120ideally": 30274, + "\u0120Password": 30275, + "\u0120simplify": 30276, + "\u0120Meta": 30277, + "\u0120MRI": 30278, + "\u0120pleading": 30279, + "organized": 30280, + "handler": 30281, + "\u0120unravel": 30282, + "correct": 30283, + "\u0120icy": 30284, + "\u0120paranoid": 30285, + "\u0120passer": 30286, + "\u0120inspections": 30287, + "ofer": 30288, + "\u0120Healthcare": 30289, + "283": 30290, + "\u0120Brut": 30291, + "iola": 30292, + "forge": 30293, + "\u0120Medieval": 30294, + "MSN": 30295, + "ievers": 30296, + "\u0120Programming": 30297, + "\u00e5\u012b": 30298, + "\u0120223": 30299, + "mu": 30300, + "\u0120CLE": 30301, + "uga": 30302, + "\u0120shoppers": 30303, + "\u0120informative": 30304, + "\u0120Plans": 30305, + "\u0120supplementation": 30306, + "\u0120Tests": 30307, + "tyard": 30308, + "ocytes": 30309, + "\u0120Vega": 30310, + "\u0120Gujarat": 30311, + "ermanent": 30312, + "Except": 30313, + "\u0120LOT": 30314, + "alla": 30315, + "\u0120Cumm": 30316, + "\u0120Osw": 30317, + "\u0120venom": 30318, + "\u0120Debt": 30319, + "\u0120DOWN": 30320, + "\u0120reunion": 30321, + "\u0120muc": 30322, + "\u0120Relief": 30323, + "\u0120geop": 30324, + "\u0120\u00f0\u0141\u013a": 30325, + "alogue": 30326, + "Anth": 30327, + "echo": 30328, + "\u0120corros": 30329, + "\u0120replication": 30330, + "\u0120Blazing": 30331, + "\u0120Daughter": 30332, + "\u0120inflic": 30333, + "\u0120Lindsey": 30334, + "\u00d9\u012a": 30335, + "284": 30336, + "Exit": 30337, + "\u0120gloom": 30338, + "TAIN": 30339, + "\u0120undermining": 30340, + "\u0120advising": 30341, + "hidden": 30342, + "\u0120overflow": 30343, + "\u0120gor": 30344, + "urdue": 30345, + "\u0120echoes": 30346, + "enhagen": 30347, + "\u0120impuls": 30348, + "drug": 30349, + "cash": 30350, + "\u0120async": 30351, + "\u0120mirac": 30352, + "atts": 30353, + "punk": 30354, + "\u0120pivot": 30355, + "\u0120Legislative": 30356, + "\u0120bloggers": 30357, + "\u0120Claw": 30358, + "sburg": 30359, + "dyl": 30360, + "\u0120Recommend": 30361, + "\u0120verte": 30362, + "\u0120prohibiting": 30363, + "\u0120Panther": 30364, + "Jonathan": 30365, + "\u0120omin": 30366, + "\u0120hateful": 30367, + "281": 30368, + "\u0120Orche": 30369, + "\u0120Murdoch": 30370, + "downs": 30371, + "\u0120asymm": 30372, + "GER": 30373, + "Always": 30374, + "\u0120informs": 30375, + "\u0120WM": 30376, + "\u0120Pony": 30377, + "\u0120Appendix": 30378, + "\u0120Arlington": 30379, + "Jam": 30380, + "\u0120medicinal": 30381, + "\u0120Slam": 30382, + "ITIES": 30383, + "\u0120reaff": 30384, + "\u0120Ri": 30385, + "FG": 30386, + "Spring": 30387, + "bool": 30388, + "\u0120thighs": 30389, + "\u0120markings": 30390, + "\u0120Raqqa": 30391, + "\u0120Lak": 30392, + "poll": 30393, + "tsky": 30394, + "\u0120Morty": 30395, + "\u0120Definition": 30396, + "\u0120debunk": 30397, + "endered": 30398, + "\u0120Leone": 30399, + "avers": 30400, + "\u0120mortgages": 30401, + "Apparently": 30402, + "Nic": 30403, + "haus": 30404, + "\u0120Thousands": 30405, + "auld": 30406, + "\u0120mash": 30407, + "shoot": 30408, + "\u0120diarr": 30409, + "\u0120consciously": 30410, + "Hero": 30411, + "eas": 30412, + "\u0120Naturally": 30413, + "\u0120Destroyer": 30414, + "\u0120dashboard": 30415, + "services": 30416, + "Rog": 30417, + "\u0120millennials": 30418, + "\u0120invade": 30419, + "-(": 30420, + "\u0120commissions": 30421, + "\u0120Auckland": 30422, + "\u0120broadcasts": 30423, + "\u0120frontal": 30424, + "\u0120crank": 30425, + "\u0120Historic": 30426, + "\u0120rumours": 30427, + "CTV": 30428, + "\u0120steril": 30429, + "\u0120booster": 30430, + "rocket": 30431, + "\u00e3\u0124\u00bc": 30432, + "utsche": 30433, + "\u0120PI": 30434, + "\u0120233": 30435, + "\u0120Producer": 30436, + "\u0120Analytics": 30437, + "\u0120invaluable": 30438, + "\u0120unintention": 30439, + "\u0120CY": 30440, + "\u0120scrutin": 30441, + "\u0120gigg": 30442, + "\u0120engulf": 30443, + "\u0120proletariat": 30444, + "\u0120hacks": 30445, + "\u0120Hew": 30446, + "arak": 30447, + "\u0120Slime": 30448, + "ielding": 30449, + "agher": 30450, + "\u0120Elliot": 30451, + "\u0120telecom": 30452, + "\u0120219": 30453, + "ultan": 30454, + "\u0120Arbor": 30455, + "\u0120Scouts": 30456, + "Ban": 30457, + "\u0120lifespan": 30458, + "\u0120blasp": 30459, + "388": 30460, + "\u0120judiciary": 30461, + "\u0120Continental": 30462, + "asking": 30463, + "McC": 30464, + "LED": 30465, + "\u0120baggage": 30466, + "\u0120Sorcerer": 30467, + "\u0120remnants": 30468, + "\u0120Griffith": 30469, + "etsu": 30470, + "\u0120Subaru": 30471, + "\u0120Personality": 30472, + "designed": 30473, + "ushima": 30474, + "agnar": 30475, + "\u0120recoil": 30476, + "\u0120passions": 30477, + "\\\":": 30478, + "\u0120tee": 30479, + "\u0120abolition": 30480, + "\u0120Creating": 30481, + "jac": 30482, + "\u0120194": 30483, + "019": 30484, + "\u0120pillars": 30485, + "riched": 30486, + "/\"": 30487, + "tk": 30488, + "\u0120livelihood": 30489, + "\u0120roasted": 30490, + "ahon": 30491, + "\u0120Hutch": 30492, + "assert": 30493, + "\u0120dividend": 30494, + "\u0120knit": 30495, + "\u0120daunting": 30496, + "\u0120disturbance": 30497, + "\u0120shale": 30498, + "\u0120cultivated": 30499, + "\u0120refrigerator": 30500, + "LB": 30501, + "\u0120NET": 30502, + "\u0120commercials": 30503, + "\u0120thinkers": 30504, + "455": 30505, + "\u0120chop": 30506, + "Broad": 30507, + "\u0120suspicions": 30508, + "\u0120tagged": 30509, + "lifting": 30510, + "\u0120stylish": 30511, + "\u0120Shields": 30512, + "Shortly": 30513, + "\u0120tails": 30514, + "Auth": 30515, + "STE": 30516, + "\u0120GAME": 30517, + "\u0120seism": 30518, + "\u0120Kis": 30519, + "ologne": 30520, + "\u0120cowork": 30521, + "\u0120forcibly": 30522, + "\u0120thyroid": 30523, + "\u0120PB": 30524, + "ANE": 30525, + "married": 30526, + "horse": 30527, + "\u0120polymer": 30528, + "\u0120Chal": 30529, + "odor": 30530, + "DEBUG": 30531, + "\u0120Context": 30532, + "\u0120bliss": 30533, + "\u0120pinpoint": 30534, + "\u0120Mathemat": 30535, + "legram": 30536, + "\u0120Weekend": 30537, + "\u0120labelled": 30538, + "\u0120bart": 30539, + "itles": 30540, + "\u0120estrogen": 30541, + "\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136": 30542, + "\"'": 30543, + "\u0120visibly": 30544, + "\u0120outsider": 30545, + "aida": 30546, + "Area": 30547, + "\u0120dissemin": 30548, + "\u0120dishonest": 30549, + "\u0120Closed": 30550, + "\u0120Bulletin": 30551, + "\u0120Ramsey": 30552, + "sword": 30553, + "\u0120XI": 30554, + "ourced": 30555, + "Same": 30556, + "346": 30557, + "\u0120Repe": 30558, + "\u0120Kou": 30559, + "cake": 30560, + "emis": 30561, + "Cache": 30562, + "\u0120Meaning": 30563, + "\u0120Enlight": 30564, + "onomy": 30565, + "\u0120manifestation": 30566, + "sworth": 30567, + "Jay": 30568, + "\u0120chore": 30569, + "\u00c3\u00b6r": 30570, + "Dream": 30571, + "\u0120sanctioned": 30572, + "\u0120culturally": 30573, + "\u0120Ara": 30574, + "Nav": 30575, + "\u0120theological": 30576, + "\u0120strut": 30577, + "\u0120VO": 30578, + "\u0120Handbook": 30579, + "\u0120constructing": 30580, + "\u0120\u00c2\u00b6": 30581, + "\u0120Benefits": 30582, + "\u0120Psychological": 30583, + "sac": 30584, + "\u00e5\u00b8": 30585, + "policy": 30586, + "\u0120Matters": 30587, + "\u0120Reported": 30588, + "\u0120Byte": 30589, + "\u0120vitro": 30590, + "\u0120Maiden": 30591, + "\u0120lam": 30592, + "\u0120Jennings": 30593, + "\u0120garment": 30594, + "\u0120Rutgers": 30595, + "\u0120Stafford": 30596, + "\u0120Wellington": 30597, + "\u0120intermitt": 30598, + "\u0120npm": 30599, + "\u0120ordeal": 30600, + "\u0120plugged": 30601, + "ooming": 30602, + "inished": 30603, + "framework": 30604, + "\u0120timber": 30605, + "\u0120cass": 30606, + "\u0120850": 30607, + "iless": 30608, + "\u0120Redux": 30609, + "768": 30610, + "Stre": 30611, + "\u0120surpassed": 30612, + "whel": 30613, + "\u0120parallels": 30614, + "\u0120veil": 30615, + "\u0120GI": 30616, + "\u0120REST": 30617, + "\u0120readiness": 30618, + "sort": 30619, + "\u0120modifying": 30620, + "\u0120Slate": 30621, + "ruff": 30622, + "\u0120marble": 30623, + "\u0120infrared": 30624, + "\u0120auditor": 30625, + "\u0120FANTASY": 30626, + "\u0120Poverty": 30627, + "\u0120SPD": 30628, + "\u0120\"(": 30629, + "Ky": 30630, + "RAY": 30631, + "\u0120executions": 30632, + "\u0120Beverly": 30633, + "\u0120Marxism": 30634, + "\u0120Burst": 30635, + "\u0120Kali": 30636, + "estones": 30637, + "Clearly": 30638, + "Ell": 30639, + "\u00e3\u0123\u00a7": 30640, + "\u0120Proceedings": 30641, + "Token": 30642, + "IFIC": 30643, + "\u00c3\u00b1a": 30644, + "Central": 30645, + "\u0120Haley": 30646, + "\u0120Drama": 30647, + "\u0120formations": 30648, + "ORN": 30649, + "Books": 30650, + "\u0120dominating": 30651, + "\u0120Flyers": 30652, + "\u0120Companion": 30653, + "\u0120disciplined": 30654, + "\u0120Yugoslav": 30655, + "\u0120Spells": 30656, + "\u0120vengeance": 30657, + "\u0120landlords": 30658, + "Len": 30659, + "\u0120Ogre": 30660, + "anoia": 30661, + "\u0120piercing": 30662, + "\u0120congreg": 30663, + "\u0120scorer": 30664, + "obia": 30665, + "\u0120nickel": 30666, + "\u0120Learns": 30667, + "\u0120rejo": 30668, + "\u0120masterpiece": 30669, + "Flash": 30670, + "\u0120inhabited": 30671, + "\u0120OpenGL": 30672, + "\u0120Dud": 30673, + "\u0120ICO": 30674, + "\u0120arter": 30675, + "\u0120plur": 30676, + "\u0120mastery": 30677, + "\u0120longstanding": 30678, + "sted": 30679, + "\u0120wines": 30680, + "\u0120televised": 30681, + "\u0120Shrine": 30682, + "\u0120Bayern": 30683, + "\u0120\u00e2\u0135\u013a": 30684, + "\u0120enclosure": 30685, + "john": 30686, + "\u0120prophets": 30687, + "\u0120Resurrection": 30688, + "\u0120Orders": 30689, + "\u0120uneven": 30690, + "rals": 30691, + "\u0120dwind": 30692, + "\u0120Lah": 30693, + "\u0120Sloven": 30694, + "378": 30695, + "\u0120insistence": 30696, + "affle": 30697, + "\u0120Clone": 30698, + "\u0120hardship": 30699, + "\u0120Congressman": 30700, + "\u0120plead": 30701, + "\u0120reviewers": 30702, + "\u0120cured": 30703, + "\u01201935": 30704, + "asley": 30705, + "fake": 30706, + "\u0120Thinking": 30707, + "ydia": 30708, + "PART": 30709, + "\u0120Dota": 30710, + "oit": 30711, + "\u0120whipped": 30712, + "\u0120bouncing": 30713, + "\u0120Hispanics": 30714, + "comings": 30715, + "\u0120cannabin": 30716, + "\u0120Chambers": 30717, + "\u0120Zack": 30718, + "Optional": 30719, + "\u0120coats": 30720, + "\u0120prowess": 30721, + "\u0120Norton": 30722, + "\u0120plainly": 30723, + "\u0120freight": 30724, + "\u0120inhibition": 30725, + "\u0120clam": 30726, + "\u0120303": 30727, + "kef": 30728, + "aleigh": 30729, + "Luke": 30730, + "\u0120psycho": 30731, + "atorium": 30732, + "MED": 30733, + "\u0120treaties": 30734, + "\u0120indisc": 30735, + "\u0120dc": 30736, + "OPS": 30737, + "\u0120resilient": 30738, + "\u0120Interstate": 30739, + "\u0120slack": 30740, + "\u0120mundane": 30741, + "\u0120establishes": 30742, + "359": 30743, + "\u0120strained": 30744, + "\u0120nond": 30745, + "Sus": 30746, + "\u0120caste": 30747, + "arate": 30748, + "ieving": 30749, + "\u0120unfairly": 30750, + "\u0120parser": 30751, + "onial": 30752, + "ursive": 30753, + "Via": 30754, + "\u0120Otto": 30755, + "\u0120Authorities": 30756, + "stroke": 30757, + "KR": 30758, + "\u0120Mercy": 30759, + "\u0120furnished": 30760, + "\u0120outset": 30761, + "\u0120metic": 30762, + "1982": 30763, + "olithic": 30764, + "\u0120Tent": 30765, + "ogical": 30766, + "\u0120Aircraft": 30767, + "\u0120hides": 30768, + "\u0120Became": 30769, + "\u0120educators": 30770, + "reaching": 30771, + "\u0120volatility": 30772, + "\u0120toddler": 30773, + "\u0120NASCAR": 30774, + "\u0120Twelve": 30775, + "\u0120Highlights": 30776, + "\u0120grape": 30777, + "\u0120splits": 30778, + "\u0120peasant": 30779, + "\u0120reneg": 30780, + "\u0120MSI": 30781, + "Temp": 30782, + "stars": 30783, + "\u0120trek": 30784, + "\u0120Hyde": 30785, + "binding": 30786, + "\u0120realism": 30787, + "\u0120oxide": 30788, + "\u0120Hos": 30789, + "\u0120mounts": 30790, + "\u0120biting": 30791, + "\u0120collapsing": 30792, + "\u0120postal": 30793, + "\u0120museums": 30794, + "\u0120detached": 30795, + "\u0120respecting": 30796, + "\u0120monopol": 30797, + "\u0120workflow": 30798, + "\u0120Cake": 30799, + "Template": 30800, + "\u0120Organisation": 30801, + "\u0120persistence": 30802, + "369": 30803, + "Coming": 30804, + "Brad": 30805, + "\u0120redundant": 30806, + "\u0120GTA": 30807, + "\u0120bending": 30808, + "\u0120revoked": 30809, + "\u0120offending": 30810, + "\u0120framing": 30811, + "\u0120printf": 30812, + "Commun": 30813, + "members": 30814, + "Outside": 30815, + "\u0120construed": 30816, + "\u0120coded": 30817, + "FORE": 30818, + "\u0120chast": 30819, + "Chat": 30820, + "Indian": 30821, + "\u0120Yard": 30822, + "?!\"": 30823, + "\u0120Ports": 30824, + "\u0120Xavier": 30825, + "\u0120RET": 30826, + "'.\"": 30827, + "\u0120Boat": 30828, + "ivated": 30829, + "icht": 30830, + "umerable": 30831, + "Ds": 30832, + "\u0120Dunn": 30833, + "\u0120coffin": 30834, + "\u0120securely": 30835, + "\u0120Raptors": 30836, + "\u0120Bes": 30837, + "Installation": 30838, + "\u0120inception": 30839, + "\u0120Healthy": 30840, + "endants": 30841, + "\u0120psychologists": 30842, + "\u0120Sheikh": 30843, + "cultural": 30844, + "\u0120BlackBerry": 30845, + "shift": 30846, + "Fred": 30847, + "oche": 30848, + "\u0120cakes": 30849, + "\u0120SEO": 30850, + "\u0120Gian": 30851, + "\u0120Asians": 30852, + "ogging": 30853, + "element": 30854, + "\u0120pundits": 30855, + "\u0120Vaugh": 30856, + "\u0120Gavin": 30857, + "\u0120hitter": 30858, + "\u0120drowned": 30859, + "\u0120chalk": 30860, + "\u0120Zika": 30861, + "\u0120measles": 30862, + "802": 30863, + "\u00e2\u0122\u00a6..": 30864, + "\u0120AWS": 30865, + "]\"": 30866, + "\u0120distort": 30867, + "\u0120Mast": 30868, + "\u0120antibodies": 30869, + "\u0120Mash": 30870, + "Memory": 30871, + "\u0120Uganda": 30872, + "\u0120Prob": 30873, + "\u0120vomiting": 30874, + "\u0120Turns": 30875, + "\u0120occupying": 30876, + "\u0120evasion": 30877, + "\u0120Therapy": 30878, + "\u0120promo": 30879, + "\u0120electr": 30880, + "\u0120blueprint": 30881, + "\u0120Dre": 30882, + "priced": 30883, + "\u0120Depot": 30884, + "\u0120alleviate": 30885, + "\u0120Somali": 30886, + "marg": 30887, + "nine": 30888, + "\u0120nostalgia": 30889, + "\u0120Shepherd": 30890, + "\u0120cavalry": 30891, + "\u0120torped": 30892, + "\u0120Bloody": 30893, + "xb": 30894, + "\u0120sank": 30895, + "\u0120goalt": 30896, + "reportprint": 30897, + "embedreportprint": 30898, + "cloneembedreportprint": 30899, + "\u0120Initially": 30900, + "\u0120Fischer": 30901, + "\u0120noteworthy": 30902, + "cern": 30903, + "\u0120inefficient": 30904, + "rawdownload": 30905, + "rawdownloadcloneembedreportprint": 30906, + "cation": 30907, + "\u0120Dynasty": 30908, + "lag": 30909, + "DES": 30910, + "\u0120distinctly": 30911, + "\u0120Estonia": 30912, + "\u0120openness": 30913, + "\u0120gossip": 30914, + "ruck": 30915, + "Width": 30916, + "\u0120Ibrahim": 30917, + "\u0120petroleum": 30918, + "\u0120avatar": 30919, + "\u0120Hed": 30920, + "atha": 30921, + "\u0120Hogwarts": 30922, + "\u0120caves": 30923, + "678": 30924, + "\u0120safeguard": 30925, + "\u0120Mog": 30926, + "isson": 30927, + "\u0120Durham": 30928, + "slaught": 30929, + "\u0120Graduate": 30930, + "\u0120subconscious": 30931, + "\u0120Excellent": 30932, + "\u0120Dum": 30933, + "-----": 30934, + "\u0120piles": 30935, + "\u0120WORK": 30936, + "\u0120Garn": 30937, + "\u0120Fol": 30938, + "\u0120ATM": 30939, + "\u0120avoids": 30940, + "\u0120Tul": 30941, + "\u0120bleak": 30942, + "ELY": 30943, + "ivist": 30944, + "lightly": 30945, + "Pers": 30946, + "\u0120Dob": 30947, + "\u0120LS": 30948, + "\u0120insanity": 30949, + "\u00ce\u00b5": 30950, + "atalie": 30951, + "Enlarge": 30952, + "\u0120twists": 30953, + "\u0120faulty": 30954, + "\u0120piracy": 30955, + "\u0120impover": 30956, + "\u0120rugged": 30957, + "\u0120Fashion": 30958, + "\u0120sands": 30959, + "'?": 30960, + "swick": 30961, + "\u0120natives": 30962, + "\u0120hen": 30963, + "\u0120Noise": 30964, + "\u00e3\u0125\u0139": 30965, + "\u0120greens": 30966, + "\u0120freezer": 30967, + "\u0120dynasty": 30968, + "\u0120Fathers": 30969, + "\u0120Newark": 30970, + "\u0120archaeological": 30971, + "\u0120ot": 30972, + "obar": 30973, + "\u0120blockade": 30974, + "\u0120allerg": 30975, + "LV": 30976, + "\u0120debit": 30977, + "\u0120RFC": 30978, + "\u0120Milton": 30979, + "\u0120Pressure": 30980, + "\u0120willingly": 30981, + "\u0120disproportionate": 30982, + "\u0120oppressive": 30983, + "\u0120diamonds": 30984, + "\u0120belongings": 30985, + "1970": 30986, + "\u0120bells": 30987, + "\u0120imperialism": 30988, + "\u0120227": 30989, + "\u0120exploding": 30990, + "\u0120Eclipse": 30991, + "\u01201919": 30992, + "\u0120rant": 30993, + "\u0120nominations": 30994, + "347": 30995, + "\u0120peacefully": 30996, + "rica": 30997, + "\u0120FUCK": 30998, + "\u0120vibration": 30999, + "malink": 31000, + "\u0120ropes": 31001, + "\u0120Ivanka": 31002, + "\u0120Brewery": 31003, + "\u0120Booker": 31004, + "\u0120Owens": 31005, + "goers": 31006, + "Services": 31007, + "\u0120Snape": 31008, + "\u0120191": 31009, + "395": 31010, + "\u0120299": 31011, + "justice": 31012, + "\u0120bri": 31013, + "\u0120discs": 31014, + "\u0120prominently": 31015, + "\u0120vulgar": 31016, + "\u0120skipping": 31017, + "lves": 31018, + "\u0120tsunami": 31019, + "374": 31020, + "\u0120Urug": 31021, + "\u0120Eid": 31022, + "recated": 31023, + "phen": 31024, + "\u0120faults": 31025, + "\u0120Started": 31026, + "950": 31027, + "\u0120pi": 31028, + "\u0120detector": 31029, + "\u0120bastard": 31030, + "\u0120validated": 31031, + "SpaceEngineers": 31032, + "OURCE": 31033, + "\u0120(~": 31034, + "\u0120unsur": 31035, + "\u0120affirmed": 31036, + "\u0120fascism": 31037, + "\u0120resolving": 31038, + "\u0120Chavez": 31039, + "\u0120Cyn": 31040, + "\u0120detract": 31041, + "Lost": 31042, + "\u0120rigged": 31043, + "\u0120homage": 31044, + "\u0120Bruno": 31045, + "555": 31046, + "eca": 31047, + "\u0120presses": 31048, + "\u0120humour": 31049, + "\u0120spacing": 31050, + "\u0120'/": 31051, + "olkien": 31052, + "Coun": 31053, + "OPER": 31054, + "Tre": 31055, + "Son": 31056, + "\u0120Cambodia": 31057, + "ierre": 31058, + "mong": 31059, + "ozy": 31060, + "\u0120liquidity": 31061, + "\u0120Soviets": 31062, + "\u0120Fernando": 31063, + "\u0120229": 31064, + "\u0120slug": 31065, + "\u0120Catalan": 31066, + "electric": 31067, + "\u0120scenery": 31068, + "\u0120Hearth": 31069, + "\u0120constrained": 31070, + "\u0120goalie": 31071, + "\u0120Guidelines": 31072, + "\u0120Ammo": 31073, + "\u0120Pearson": 31074, + "\u0120taxed": 31075, + "\u0120fetus": 31076, + "Response": 31077, + "\u0120Alexis": 31078, + "thia": 31079, + "Guy": 31080, + "\u0120reconstruct": 31081, + "\u0120extremes": 31082, + "\u0120concluding": 31083, + "\u0120Peg": 31084, + "ooks": 31085, + "\u0120deductions": 31086, + "Rose": 31087, + "\u0120groundbreaking": 31088, + "\u0120Targ": 31089, + "\u00e3\u0125\u0123": 31090, + "\u0120Reve": 31091, + "resource": 31092, + "\u0120moons": 31093, + "\u0120electromagnetic": 31094, + "\u0120amidst": 31095, + "\u0120Viktor": 31096, + "NESS": 31097, + "BACK": 31098, + "\u0120commute": 31099, + "\u0120Anaheim": 31100, + "\u0120fluctuations": 31101, + "640": 31102, + "\u0120noodles": 31103, + "\u0120Copenhagen": 31104, + "\u0120Tide": 31105, + "\u0120Grizz": 31106, + "\u0120SEE": 31107, + "\u0120pipelines": 31108, + "\u0120scars": 31109, + "endo": 31110, + "agus": 31111, + "\u0120ETF": 31112, + "/#": 31113, + "\u0120Become": 31114, + "448": 31115, + "\u0120visc": 31116, + "\u0120Recommended": 31117, + "\u0120jumper": 31118, + "\u0120cognition": 31119, + "\u0120assassin": 31120, + "\u0120witnessing": 31121, + "\u0120Setup": 31122, + "\u0120lac": 31123, + "vim": 31124, + "ISM": 31125, + "pages": 31126, + "SSL": 31127, + "358": 31128, + "\u0120adject": 31129, + "industrial": 31130, + "lore": 31131, + "chery": 31132, + "\u0120glitter": 31133, + "\u0120calf": 31134, + "Florida": 31135, + "\u0120spoilers": 31136, + "\u0120succeeds": 31137, + "\u0120chanting": 31138, + "\u0120slogans": 31139, + "\u0120Tracy": 31140, + "Visit": 31141, + "rology": 31142, + "\u0120mornings": 31143, + "\u0120lineage": 31144, + "\u0120sip": 31145, + "\u0120intensely": 31146, + "\u0120flourish": 31147, + "\u0120Sleeping": 31148, + "\u0120Fem": 31149, + "orpor": 31150, + "\u0120Klan": 31151, + "\u0120Darth": 31152, + "hack": 31153, + "\u0120Nielsen": 31154, + "\u0120tumors": 31155, + "\u0120procurement": 31156, + "\u0120Yorkshire": 31157, + "\u0120raided": 31158, + "KY": 31159, + "Anna": 31160, + "\u0120//[": 31161, + "\u0120Disorder": 31162, + "\u0120Mustang": 31163, + "\u0120Wen": 31164, + "\u0120Trying": 31165, + "sq": 31166, + "\u0120deliveries": 31167, + "\u0120shutter": 31168, + "\u0120cerebral": 31169, + "\u0120bipolar": 31170, + "\u0120CN": 31171, + "lass": 31172, + "jet": 31173, + "\u0120debating": 31174, + ">:": 31175, + "\u0120eagle": 31176, + "grades": 31177, + "\u0120Dixon": 31178, + "UGC": 31179, + "MAS": 31180, + "\u0120Draco": 31181, + "\u0120Machines": 31182, + "affer": 31183, + "\u0120eman": 31184, + "\u00c2\u00b2": 31185, + "pron": 31186, + "\u0120Gym": 31187, + "\u0120comparatively": 31188, + "\u0120Tribunal": 31189, + "PRO": 31190, + "\u0120lex": 31191, + "\u0120fertile": 31192, + "\u0120depressing": 31193, + "\u0120superficial": 31194, + "essential": 31195, + "\u0120Hunters": 31196, + "gp": 31197, + "\u0120prominence": 31198, + "Liber": 31199, + "\u0120Ancest": 31200, + "otechnology": 31201, + "\u0120mocking": 31202, + "\u0120Traff": 31203, + "\u0138\u013c": 31204, + "Medium": 31205, + "Iraq": 31206, + "\u0120psychiatrist": 31207, + "Quantity": 31208, + "\u0120Lect": 31209, + "\u0120noisy": 31210, + "520": 31211, + "GY": 31212, + "\u0120slapped": 31213, + "\u0120MTV": 31214, + "\u0120para": 31215, + "pull": 31216, + "Multiple": 31217, + "asher": 31218, + "\u0120nour": 31219, + "\u0120Seg": 31220, + "Spell": 31221, + "vous": 31222, + "ordial": 31223, + "Senior": 31224, + "\u0120Goldberg": 31225, + "\u0120Plasma": 31226, + "need": 31227, + "\u0120messenger": 31228, + "eret": 31229, + "\u0120teamed": 31230, + "\u0120literacy": 31231, + "\u0120Leah": 31232, + "\u0120Doyle": 31233, + "\u0120emitted": 31234, + "UX": 31235, + "\u0120evade": 31236, + "\u0120maze": 31237, + "\u0120wrongly": 31238, + "\u0120Lars": 31239, + "\u0120stereotype": 31240, + "\u0120pledges": 31241, + "\u0120aroma": 31242, + "\u0120MET": 31243, + "\u0120acre": 31244, + "\u0120OD": 31245, + "\u0120ff": 31246, + "\u0120breweries": 31247, + "\u0120Hilton": 31248, + "undle": 31249, + "\u0120Kak": 31250, + "\u0120Thankfully": 31251, + "\u0120Canucks": 31252, + "inctions": 31253, + "\u0120Appears": 31254, + "\u0120coer": 31255, + "\u0120undermined": 31256, + "rovers": 31257, + "Andre": 31258, + "\u0120blaze": 31259, + "umers": 31260, + "\u0120famine": 31261, + "amphetamine": 31262, + "ulkan": 31263, + "Amount": 31264, + "\u0120desperation": 31265, + "wikipedia": 31266, + "development": 31267, + "\u0120Corinth": 31268, + "ussia": 31269, + "Jackson": 31270, + "LI": 31271, + "Native": 31272, + "Rs": 31273, + "Ohio": 31274, + "\u0120Kathleen": 31275, + "Fortunately": 31276, + "\u0120attendant": 31277, + "\u0120Preferred": 31278, + "\u0120Didn": 31279, + "\u0120Vs": 31280, + "Mis": 31281, + "\u0120respondent": 31282, + "\u0120boun": 31283, + "stable": 31284, + "\u0120paved": 31285, + "\u0120unexpl": 31286, + "\u0120Cheney": 31287, + "LM": 31288, + "\u0120Cull": 31289, + "blown": 31290, + "\u0120confronting": 31291, + "ocese": 31292, + "serving": 31293, + "Wi": 31294, + "\u0120Lithuania": 31295, + "anni": 31296, + "\u0120stalk": 31297, + "hd": 31298, + "\u0120vener": 31299, + "APH": 31300, + "ynchronous": 31301, + "URR": 31302, + "umably": 31303, + "historic": 31304, + "Half": 31305, + "Hay": 31306, + "\u0120resilience": 31307, + "spection": 31308, + "\u0120abandoning": 31309, + "Obs": 31310, + "\u0120Debbie": 31311, + "\u0120gradient": 31312, + "\u0120Plaint": 31313, + "\u0120Canal": 31314, + "ARCH": 31315, + "\u0120expansive": 31316, + "\u0120fung": 31317, + "\u0120bounced": 31318, + "Und": 31319, + "\u0120precautions": 31320, + "\u0120clarification": 31321, + "\u0120dagger": 31322, + "\u0120grips": 31323, + "\u0120\u00c2\u00b5": 31324, + "\u0120Rivera": 31325, + "\u0120Undead": 31326, + "isites": 31327, + "\u0120FIRST": 31328, + "\u00c3\u00b1o": 31329, + "audi": 31330, + "\u0120hostages": 31331, + "\u0120compliant": 31332, + "\u0120alumni": 31333, + "Seven": 31334, + "\u0120cybersecurity": 31335, + "either": 31336, + "Collect": 31337, + "\u0120invariably": 31338, + "\u0120Soci": 31339, + "\u0120lawmaker": 31340, + "\u0120ale": 31341, + "\u0120Personally": 31342, + "Nazi": 31343, + "\u0120customization": 31344, + "\u0120Proc": 31345, + "\u0120Saskatchewan": 31346, + "eaturing": 31347, + "\u0120spared": 31348, + "\u0120discontinued": 31349, + "\u0120computational": 31350, + "\u0120Motorola": 31351, + "\u0120supremacist": 31352, + "governmental": 31353, + "\u0120paradise": 31354, + "\u0120Downing": 31355, + "\u0120Nikon": 31356, + "\u0120catalyst": 31357, + "berra": 31358, + "Toronto": 31359, + "875": 31360, + "beta": 31361, + "\u0120Macron": 31362, + "\u0120unrealistic": 31363, + "vector": 31364, + "\u0120Vehicles": 31365, + "itiveness": 31366, + "\u0120RV": 31367, + "\u0120Colbert": 31368, + "sin": 31369, + "oji": 31370, + "entin": 31371, + "\u0120Krish": 31372, + "hello": 31373, + "ffield": 31374, + "oky": 31375, + "\u0120Tate": 31376, + "\u0120maple": 31377, + "\u0120aids": 31378, + "chemical": 31379, + "334": 31380, + "nuts": 31381, + "\u0120Warp": 31382, + "\u0120xx": 31383, + "\u0120Robb": 31384, + "umerous": 31385, + "_-_": 31386, + "ftime": 31387, + "\u0120VW": 31388, + "\u0120winger": 31389, + "\u0120Dome": 31390, + "tools": 31391, + "\u0120PV": 31392, + "\u0120Georgetown": 31393, + "\u0120geared": 31394, + "\u0120jihadists": 31395, + "\u0120cp": 31396, + "\u0120steroids": 31397, + "Mother": 31398, + "clerosis": 31399, + "\u0120DRM": 31400, + "nesia": 31401, + "\u0120linger": 31402, + "\u0120immersive": 31403, + "\u0120COUN": 31404, + "\u0120outweigh": 31405, + "ensual": 31406, + "Band": 31407, + "\u0120transforms": 31408, + "matched": 31409, + "psons": 31410, + "\u0120Judicial": 31411, + "factor": 31412, + "\u0120referral": 31413, + "\u0120oddly": 31414, + "\u0120Wenger": 31415, + "Bring": 31416, + "\u0120Bows": 31417, + "602": 31418, + "ICLE": 31419, + "\u0120lions": 31420, + "\u0120Academic": 31421, + "\u0120Thorn": 31422, + "\u0120Raider": 31423, + "kefeller": 31424, + "Storage": 31425, + "Lower": 31426, + "\u0120Ort": 31427, + "\u0120Equality": 31428, + "ALT": 31429, + "\u0120SOC": 31430, + "Types": 31431, + "\u0120lyn": 31432, + "\u0120Asset": 31433, + "coat": 31434, + "TPP": 31435, + "CVE": 31436, + "\u0120Pioneer": 31437, + "application": 31438, + "Modern": 31439, + "\u0120HK": 31440, + "Environment": 31441, + "Alright": 31442, + "Rain": 31443, + "IPP": 31444, + "\u0120Shiite": 31445, + "\u0120mound": 31446, + "\u0120Abilities": 31447, + "condition": 31448, + "Staff": 31449, + "\u0120competence": 31450, + "\u0120Moor": 31451, + "\u0120Diablo": 31452, + "\u0120withheld": 31453, + "\u0120ostensibly": 31454, + "\u0120Brom": 31455, + "\u0120msg": 31456, + "\u0120denomin": 31457, + "\u0120References": 31458, + "\u0120FP": 31459, + "\u0120plunged": 31460, + "\u0120pamph": 31461, + "moving": 31462, + "central": 31463, + "\u0120downright": 31464, + "\u0120fading": 31465, + "Tal": 31466, + "Typ": 31467, + "\u0120Thy": 31468, + "ukes": 31469, + "ithe": 31470, + "\u0120ove": 31471, + "\u0120battled": 31472, + "\u0120seafood": 31473, + "\u0120figur": 31474, + "\u0120RD": 31475, + "crop": 31476, + "\u0120squads": 31477, + "{\\": 31478, + "\u00e0\u00b9": 31479, + "\u0120Eh": 31480, + "\u0120interviewing": 31481, + "\u0120Qin": 31482, + "\u0120aspiring": 31483, + "PLIC": 31484, + "\u0120clauses": 31485, + "\u0120Gast": 31486, + "\u0120Nir": 31487, + "\u0120luggage": 31488, + "\u0120hose": 31489, + "\u0120systemd": 31490, + "\u0120descending": 31491, + "\u0120Revised": 31492, + "\u0120Rails": 31493, + "align": 31494, + "709": 31495, + "337": 31496, + "\u0120fug": 31497, + "charging": 31498, + "tags": 31499, + "\u0120uter": 31500, + "kish": 31501, + "WARNING": 31502, + "490": 31503, + "profits": 31504, + "\u0120voyage": 31505, + "\u0120ace": 31506, + "\u0120Vanguard": 31507, + "\u0120Tanks": 31508, + "\u0120Muk": 31509, + "\u0120226": 31510, + "Safe": 31511, + "Armor": 31512, + "\u0120volcanic": 31513, + "\u0120womb": 31514, + "\u0120MIL": 31515, + "\u0120beginner": 31516, + "\u0120Recogn": 31517, + "\u0120AAP": 31518, + "PLAY": 31519, + ")!": 31520, + "\u0120detecting": 31521, + "cn": 31522, + "\u0120breaches": 31523, + "Basically": 31524, + "\u0120Pag": 31525, + "\u0120Municipal": 31526, + "\u0120Indie": 31527, + "\u0120Laf": 31528, + "\u0120Disable": 31529, + "\u0120Olson": 31530, + "\u0120restrained": 31531, + "\u0120rulings": 31532, + "\u0120humane": 31533, + "events": 31534, + "\u0120Cinema": 31535, + "displayText": 31536, + "\u0120Hatch": 31537, + "actionDate": 31538, + "onnaissance": 31539, + "\u0120assaulting": 31540, + "\u0120Lug": 31541, + "CHAT": 31542, + "\u0120vigorous": 31543, + "\u0120Perse": 31544, + "\u0120intolerance": 31545, + "\u0120Snapchat": 31546, + "\u0120Sharks": 31547, + "\u0120dummy": 31548, + "\u0120Diagn": 31549, + "\u0120Guitar": 31550, + "imeters": 31551, + "403": 31552, + "REG": 31553, + "Ax": 31554, + "\u0120separates": 31555, + "\u0120Mahm": 31556, + "\u0120tv": 31557, + "jah": 31558, + "OOL": 31559, + "Circ": 31560, + "\u0120Windsor": 31561, + "ussian": 31562, + "\u0120intuition": 31563, + "\u0120disdain": 31564, + "\u0120Donovan": 31565, + "\u0120221": 31566, + "Emb": 31567, + "\u0120condemning": 31568, + "\u0120generosity": 31569, + "zzy": 31570, + "\u0120panties": 31571, + "\u0120Prevent": 31572, + "ActionCode": 31573, + "ANA": 31574, + "342": 31575, + "externalActionCode": 31576, + "\u0120specifying": 31577, + "\u0120crystall": 31578, + "Jere": 31579, + "\u0120rupt": 31580, + "\u0120Apprentice": 31581, + "\u0120profiling": 31582, + "\u00d0\u00ba": 31583, + "Strike": 31584, + "\u0120sideline": 31585, + "\u0120obligated": 31586, + "\u0120occult": 31587, + "\u0120bureaucratic": 31588, + "antically": 31589, + "rupted": 31590, + "negative": 31591, + "\u0120Ethiopia": 31592, + "\u0120Civic": 31593, + "\u0120insiders": 31594, + "eligible": 31595, + "\u0120TVs": 31596, + "\u0120BAR": 31597, + "\u0120TI": 31598, + "iologist": 31599, + "\u0120AIR": 31600, + "\u0120substituted": 31601, + "Arab": 31602, + "\u0120Saul": 31603, + "\u0120Yog": 31604, + "prem": 31605, + "\u0120builders": 31606, + "\u0120stationary": 31607, + "\u0120doubtful": 31608, + "\u0120vigorously": 31609, + "\u0120thrilling": 31610, + "Physical": 31611, + "\u0120Carey": 31612, + "\u0120Hydra": 31613, + "geoning": 31614, + "\u0120Sly": 31615, + "yton": 31616, + "\u0120borrowers": 31617, + "\u0120Parkinson": 31618, + "\u0120\u00eb": 31619, + "\u0120Jamaica": 31620, + "\u0120satir": 31621, + "\u0120insurgents": 31622, + "\u0120Firm": 31623, + "\u0120isot": 31624, + "\u0120Karn": 31625, + "ourning": 31626, + "akens": 31627, + "docs": 31628, + "little": 31629, + "\u0120Monaco": 31630, + "CLASS": 31631, + "Turkey": 31632, + "Ly": 31633, + "\u0120Conan": 31634, + "assic": 31635, + "\u0120starred": 31636, + "\u0120Pacers": 31637, + "eties": 31638, + "\u0120tipping": 31639, + "Moon": 31640, + "\u0120Rw": 31641, + "same": 31642, + "\u0120cavity": 31643, + "\u0120goof": 31644, + "\u0120Zo": 31645, + "Shock": 31646, + "ummer": 31647, + "\u0120emphasizes": 31648, + "\u0120regrett": 31649, + "\u0120novelty": 31650, + "\u0120envy": 31651, + "\u0120Passive": 31652, + "rw": 31653, + "505": 31654, + "\u0120indifferent": 31655, + "\u0120Rica": 31656, + "\u0120Himself": 31657, + "\u0120Freddie": 31658, + "\u0120adip": 31659, + "\u00e4\u00b8\u0122": 31660, + "\u0120breakout": 31661, + "\u0120hurried": 31662, + "\u0120Huang": 31663, + "\u0120Disk": 31664, + "\u0120roaming": 31665, + "?????-?????-": 31666, + "UV": 31667, + "\u0120Ricky": 31668, + "\u0120Sigma": 31669, + "\u0120marginalized": 31670, + "\u0120edits": 31671, + "\u0120304": 31672, + "memory": 31673, + "\u0120specimen": 31674, + "293": 31675, + "\u00e3\u0123\u00af": 31676, + "\u0120vertically": 31677, + "\u0120audition": 31678, + "\u0120Heck": 31679, + "\u0120caster": 31680, + "\u0120Holdings": 31681, + "adal": 31682, + "\u0120Cron": 31683, + "\u0120Liam": 31684, + "\u0120deflect": 31685, + "Pick": 31686, + "\u0120Debug": 31687, + "REF": 31688, + "\u0120versatility": 31689, + "othes": 31690, + "classified": 31691, + "\u0120Mahar": 31692, + "\u0120Hort": 31693, + "Counter": 31694, + "stasy": 31695, + "noticed": 31696, + "331": 31697, + "\u0120Shim": 31698, + "fuck": 31699, + "\u0120Bie": 31700, + "\u0120airing": 31701, + "\u0120Protein": 31702, + "\u0120Holding": 31703, + "\u0120spectators": 31704, + "iliated": 31705, + "\u0120Thatcher": 31706, + "nosis": 31707, + "\u00e3\u0125\u00bc\u00e3\u0125\u00b3": 31708, + "Tele": 31709, + "Boston": 31710, + "\u0120Templ": 31711, + "stay": 31712, + "\u0120declarations": 31713, + "479": 31714, + "Volume": 31715, + "\u0120Designer": 31716, + "\u0120Overwatch": 31717, + "idae": 31718, + "\u0120onwards": 31719, + "\u0120nets": 31720, + "\u0120Manila": 31721, + "particularly": 31722, + "\u0120politic": 31723, + "oother": 31724, + "\u0120portraits": 31725, + "\u0120pavement": 31726, + "cffff": 31727, + "\u0120saints": 31728, + "\u0120beginners": 31729, + "ESPN": 31730, + "\u0120shortcomings": 31731, + "\u00e2\u0137\u0132\u00e2\u0137\u0132": 31732, + "\u0120comet": 31733, + "\u0120Organic": 31734, + "quel": 31735, + "\u0120hospitalized": 31736, + "Break": 31737, + "\u0120peel": 31738, + "dylib": 31739, + "aspx": 31740, + "urances": 31741, + "\u0120TIM": 31742, + "Pg": 31743, + "\u0120readable": 31744, + "\u0120Malik": 31745, + "\u0120muzzle": 31746, + "\u0120benchmarks": 31747, + "dal": 31748, + "\u0120Vacc": 31749, + "\u0120Hicks": 31750, + "609": 31751, + "\u0120Biblical": 31752, + "heng": 31753, + "\u0120overload": 31754, + "\u0120Civilization": 31755, + "\u0120immoral": 31756, + "\u0120fries": 31757, + "\u00e3\u0124\u0134": 31758, + "\u0120reproduced": 31759, + "\u0120formulation": 31760, + "jug": 31761, + "irez": 31762, + "gear": 31763, + "\u0120coached": 31764, + "MpServer": 31765, + "\u0120SJ": 31766, + "\u0120Kw": 31767, + "Init": 31768, + "deal": 31769, + "\u0120Oro": 31770, + "\u0120Loki": 31771, + "\u0120Songs": 31772, + "\u0120232": 31773, + "\u0120Louise": 31774, + "asionally": 31775, + "\u0120uncond": 31776, + "ollywood": 31777, + "\u0120progressives": 31778, + "\u0120Enough": 31779, + "\u0120Doe": 31780, + "\u0120wreckage": 31781, + "\u0120brushed": 31782, + "\u0120BaseType": 31783, + "\u0120zoning": 31784, + "ishable": 31785, + "hetically": 31786, + "\u0120Caucus": 31787, + "\u0120Hue": 31788, + "\u0120karma": 31789, + "\u0120Sporting": 31790, + "\u0120trader": 31791, + "\u0120seeming": 31792, + "\u0120Capture": 31793, + "430": 31794, + "bish": 31795, + "\u0120tunes": 31796, + "\u0120indoors": 31797, + "\u0120Sphere": 31798, + "\u0120Dancing": 31799, + "TERN": 31800, + "\u0120nob": 31801, + "\u0120GST": 31802, + "maps": 31803, + "\u0120peppers": 31804, + "Fit": 31805, + "\u0120oversees": 31806, + "\u0120Rabbi": 31807, + "\u0120Ruler": 31808, + "vertising": 31809, + "office": 31810, + "xxx": 31811, + "\u0120raft": 31812, + "Changed": 31813, + "\u0120textbooks": 31814, + "Links": 31815, + "\u0120Omn": 31816, + "\u00e3\u0122\u0133": 31817, + "\u0120inconvenience": 31818, + "\u0120Donetsk": 31819, + "=~": 31820, + "\u0120implicitly": 31821, + "\u0120boosts": 31822, + "\u0120Bones": 31823, + "\u0120Boom": 31824, + "Courtesy": 31825, + "\u0120sensational": 31826, + "ANY": 31827, + "\u0120greedy": 31828, + "eden": 31829, + "\u0120inexper": 31830, + "\u0120Ler": 31831, + "\u0120Vale": 31832, + "\u0120tighten": 31833, + "\u0120EAR": 31834, + "\u0120Num": 31835, + "\u0120ancestor": 31836, + "Sent": 31837, + "\u0120Horde": 31838, + "urgical": 31839, + "allah": 31840, + "\u0120sap": 31841, + "amba": 31842, + "\u0120Spread": 31843, + "twitch": 31844, + "\u0120grandson": 31845, + "\u0120fracture": 31846, + "\u0120moderator": 31847, + "\u0120Seventh": 31848, + "\u0120Reverse": 31849, + "\u0120estimation": 31850, + "Choose": 31851, + "\u0120parach": 31852, + "\u0120barric": 31853, + "\u00e3\u0122\u0132": 31854, + "\u0120compass": 31855, + "\u0120allergic": 31856, + "\u00e2\u0122\u0137": 31857, + "OTHER": 31858, + "errilla": 31859, + "\u0120wagon": 31860, + "\u0120zinc": 31861, + "\u0120rubbed": 31862, + "\u0120Fuller": 31863, + "\u0120Luxembourg": 31864, + "\u0120Hoover": 31865, + "\u0120liar": 31866, + "\u0120Evening": 31867, + "\u0120Cobb": 31868, + "esteem": 31869, + "\u0120selector": 31870, + "\u0120Brawl": 31871, + "isance": 31872, + "\u0120Ek": 31873, + "\u0120troop": 31874, + "\u0120guts": 31875, + "\u0120Appeal": 31876, + "\u0120Tibetan": 31877, + "\u0120routines": 31878, + "\u0120Ment": 31879, + "\u0120summarized": 31880, + "steamapps": 31881, + "\u0120tranqu": 31882, + "\u01201929": 31883, + "oran": 31884, + "\u0120Authent": 31885, + "\u0120gmaxwell": 31886, + "\u0120apprehens": 31887, + "\u0120poems": 31888, + "\u0120sausage": 31889, + "\u0120Webster": 31890, + "urus": 31891, + "\u0120themed": 31892, + "\u0120lounge": 31893, + "\u0120charger": 31894, + "Spoiler": 31895, + "\u0120spilled": 31896, + "hog": 31897, + "\u0120Sunder": 31898, + "\u0120Ain": 31899, + "\u0120Angry": 31900, + "\u0120disqual": 31901, + "\u0120Frequency": 31902, + "\u0120Ethernet": 31903, + "\u0120helper": 31904, + "Percent": 31905, + "\u0120horrifying": 31906, + "\u0120ail": 31907, + "\u0120Allan": 31908, + "EEE": 31909, + "\u0120Crossing": 31910, + "449": 31911, + "\u0120holog": 31912, + "\u0120Puzzles": 31913, + "\u0120Goes": 31914, + "erenn": 31915, + "604": 31916, + "\u00e3\u0123\u0131": 31917, + "\u0120Rafael": 31918, + "\u0120atten": 31919, + "\u0120Emanuel": 31920, + "\u0120upro": 31921, + "\u0120Susp": 31922, + "Psych": 31923, + "\u0120Trainer": 31924, + "\u0120NES": 31925, + "\u0120Hunts": 31926, + "becue": 31927, + "\u0120counselor": 31928, + "Rule": 31929, + "\u0120toxins": 31930, + "\u0120banners": 31931, + "rifice": 31932, + "\u0120greeting": 31933, + "\u0120frenzy": 31934, + "\u0120allocate": 31935, + "\u0120*)": 31936, + "expr": 31937, + "503": 31938, + "\u0120Chick": 31939, + "\u0120Torn": 31940, + "\u0120consolidation": 31941, + "\u0120Fletcher": 31942, + "switch": 31943, + "frac": 31944, + "clips": 31945, + "\u0120McKin": 31946, + "\u0120Lunar": 31947, + "Month": 31948, + "ITCH": 31949, + "\u0120scholarly": 31950, + "raped": 31951, + "398": 31952, + "\u01201910": 31953, + "\u0120egreg": 31954, + "\u0120insecure": 31955, + "\u0120victorious": 31956, + "cffffcc": 31957, + "\u0120singled": 31958, + "\u0120elves": 31959, + "\u0120Wond": 31960, + "burst": 31961, + "\u0120camoufl": 31962, + "\u0120BLACK": 31963, + "\u0120conditioned": 31964, + "\u00e7\u012b": 31965, + "answered": 31966, + "\u0120compulsory": 31967, + "ascist": 31968, + "\u0120podcasts": 31969, + "\u0120Frankfurt": 31970, + "bnb": 31971, + "\u0120neoliberal": 31972, + "\u0120Keyboard": 31973, + "\u0120Belle": 31974, + "warm": 31975, + "\u0120trusts": 31976, + "\u0120insured": 31977, + "\u0120Bucc": 31978, + "usable": 31979, + "607": 31980, + "\u0120Plains": 31981, + "\u01201890": 31982, + "\u0120sabotage": 31983, + "\u0120lodged": 31984, + "felt": 31985, + "\u0120ga": 31986, + "\u0120Narc": 31987, + "\u0120Salem": 31988, + "\u0120seventy": 31989, + "\u0120Blank": 31990, + "pocket": 31991, + "\u0120whisper": 31992, + "\u0120mating": 31993, + "omics": 31994, + "\u0120Salman": 31995, + "\u0120Kad": 31996, + "\u0120angered": 31997, + "\u0120collisions": 31998, + "\u0120extraordinarily": 31999, + "\u0120coercion": 32000, + "Ghost": 32001, + "birds": 32002, + "\u00e8\u0122": 32003, + "kok": 32004, + "\u0120permissible": 32005, + "avorable": 32006, + "\u0120pointers": 32007, + "\u0120dissip": 32008, + "aci": 32009, + "\u0120theatrical": 32010, + "\u0120Cosmic": 32011, + "\u0120forgetting": 32012, + "\u0120finalized": 32013, + "\u00e5\u00a4\u00a7": 32014, + "yout": 32015, + "library": 32016, + "\u0120booming": 32017, + "\u0120Believe": 32018, + "\u0120Teacher": 32019, + "\u0120Liv": 32020, + "\u0120GOODMAN": 32021, + "\u0120Dominican": 32022, + "ORED": 32023, + "\u0120Parties": 32024, + "\u0120precipitation": 32025, + "\u0120Slot": 32026, + "Roy": 32027, + "\u0120Combined": 32028, + "\u0120integrating": 32029, + "\u0120chrome": 32030, + "\u0120intestinal": 32031, + "\u0120Rebell": 32032, + "\u0120matchups": 32033, + "\u0120blockbuster": 32034, + "\u0120Loren": 32035, + "\u0120Levy": 32036, + "\u0120preaching": 32037, + "\u0120Sending": 32038, + "\u0120Purpose": 32039, + "rax": 32040, + "fif": 32041, + "\u0120authoritative": 32042, + "\u0120PET": 32043, + "astical": 32044, + "\u0120dishon": 32045, + "\u0120chatting": 32046, + "\u0120\"$:/": 32047, + "Connection": 32048, + "\u0120recreate": 32049, + "\u0120delinqu": 32050, + "\u0120broth": 32051, + "\u0120Dirty": 32052, + "\u0120Admin": 32053, + "zman": 32054, + "\u0120scholarships": 32055, + "\u0120253": 32056, + "contact": 32057, + "alsa": 32058, + "767": 32059, + "creen": 32060, + "abbage": 32061, + "\u01201915": 32062, + "\u0120blended": 32063, + "\u0120alarmed": 32064, + "Language": 32065, + "356": 32066, + "\u0120blends": 32067, + "\u0120Changed": 32068, + "Wolf": 32069, + "\u0120hepat": 32070, + "Creating": 32071, + "\u0120persecut": 32072, + "\u0120sweetness": 32073, + "arte": 32074, + "\u0120forfeiture": 32075, + "\u0120Roberto": 32076, + "impro": 32077, + "NFL": 32078, + "\u0120Magnet": 32079, + "Detailed": 32080, + "\u0120insignificant": 32081, + "\u0120POLIT": 32082, + "\u0120BBQ": 32083, + "\u0120CPS": 32084, + "\u0120seaw": 32085, + "aminer": 32086, + "mL": 32087, + "endif": 32088, + "finals": 32089, + "\u0120265": 32090, + "uish": 32091, + "\u0120})": 32092, + "\u0120Problems": 32093, + "\u0120emblem": 32094, + "\u0120seriousness": 32095, + "\u0120parsing": 32096, + "\u0120substitution": 32097, + "\u0120pressured": 32098, + "\u0120recycled": 32099, + "aleb": 32100, + "Ruby": 32101, + "\u0120proficiency": 32102, + "Driver": 32103, + "\u0120Wester": 32104, + ":'": 32105, + "AFTA": 32106, + "\u0120mantle": 32107, + "\u0120Clayton": 32108, + "flag": 32109, + "\u0120practitioner": 32110, + "covered": 32111, + "\u0120Struct": 32112, + "addafi": 32113, + "425": 32114, + "\u0120Township": 32115, + "\u0120Hydro": 32116, + "Louis": 32117, + "343": 32118, + "\u0120condo": 32119, + "\u0120Tao": 32120, + "\u0120utilization": 32121, + "\u0120nausea": 32122, + "\u0120Dems": 32123, + "ridges": 32124, + "pause": 32125, + "\u0120formulas": 32126, + "\u0120challenger": 32127, + "376": 32128, + "\u0120defective": 32129, + "\u0120Railway": 32130, + "\u0120PubMed": 32131, + "\u0120yogurt": 32132, + "lbs": 32133, + "\u0120Norfolk": 32134, + "OPE": 32135, + "\u0120Moody": 32136, + "\u0120distributor": 32137, + "\u0120scrolls": 32138, + "\u0120extracts": 32139, + "Stan": 32140, + "\u0120viability": 32141, + "\u0120exposes": 32142, + "\u0120starvation": 32143, + "\u0120Steps": 32144, + "\u0120Dodd": 32145, + "few": 32146, + "STD": 32147, + "332": 32148, + "\u0120closures": 32149, + "\u0120complementary": 32150, + "\u0120Sasha": 32151, + "umpy": 32152, + "\u0120monet": 32153, + "\u0120articulate": 32154, + "\u0120Doct": 32155, + "killer": 32156, + "\u0120scrim": 32157, + "\u0120264": 32158, + "\u0120prostitutes": 32159, + "\u0120severed": 32160, + "\u0120attachments": 32161, + "\u0120cooled": 32162, + "Lev": 32163, + "\u0120Falk": 32164, + "fail": 32165, + "\u0120policeman": 32166, + "\u0120Dag": 32167, + "\u0120prayed": 32168, + "\u0120Kernel": 32169, + "\u0120clut": 32170, + "\u0120cath": 32171, + "\u0120anomaly": 32172, + "Storm": 32173, + "emaker": 32174, + "\u0120Breakfast": 32175, + "uli": 32176, + "oire": 32177, + "JJ": 32178, + "hz": 32179, + "Operation": 32180, + "\u0120Sick": 32181, + "354": 32182, + "\u0120Guatemala": 32183, + "Rate": 32184, + "\u0120exposures": 32185, + "faces": 32186, + "\u0120Archae": 32187, + "raf": 32188, + "\u0120Mia": 32189, + "\u01202025": 32190, + "\u0120opaque": 32191, + "\u0120disguised": 32192, + "\u0120Headquarters": 32193, + "Sah": 32194, + "\u0120pots": 32195, + "978": 32196, + "\u0120Malf": 32197, + "\u0120frowned": 32198, + "\u0120poisonous": 32199, + "\u0120Convers": 32200, + "eeks": 32201, + "\u0120crab": 32202, + ".\"\"": 32203, + "\u0120treason": 32204, + "\u0120ranc": 32205, + "\u0120escalating": 32206, + "\u0120warr": 32207, + "\u0120mobs": 32208, + "\u0120lamps": 32209, + "\u0120Sunshine": 32210, + "\u0120Brunswick": 32211, + "Phones": 32212, + "\u0120spelled": 32213, + "\u0120Skip": 32214, + "\u01202050": 32215, + "\u01201911": 32216, + "\u0120Pluto": 32217, + "\u0120Amend": 32218, + "\u0120meats": 32219, + "387": 32220, + "\u0120stomp": 32221, + "\u0120Zhou": 32222, + "\u0120Leviathan": 32223, + "\u0120Hazard": 32224, + "adv": 32225, + "\u0120Orwell": 32226, + "\u0120aloud": 32227, + "\u0120bumper": 32228, + "\u0120Anarch": 32229, + "ubuntu": 32230, + "\u0120Serious": 32231, + "fitting": 32232, + "\u0120Optional": 32233, + "\u0120Cecil": 32234, + "REAM": 32235, + "\u0120serotonin": 32236, + "\u0120cultivate": 32237, + "agogue": 32238, + "}\\": 32239, + "\u0120mosques": 32240, + "\u0120Sunny": 32241, + "\u0120reactive": 32242, + "revolution": 32243, + "\u0120Lup": 32244, + "\u0120Fedora": 32245, + "\u0120defenseman": 32246, + "\u0120VID": 32247, + "istine": 32248, + "\u0120drowning": 32249, + "\u0120Broadcasting": 32250, + "\u0120thriller": 32251, + "\u0120Scy": 32252, + "\u0120accelerating": 32253, + "\u0120directs": 32254, + "odied": 32255, + "bike": 32256, + "duration": 32257, + "\u0120painfully": 32258, + "Redd": 32259, + "\u0120productions": 32260, + "\u0120gag": 32261, + "\u0120whist": 32262, + "\u0120sock": 32263, + "\u0120infinitely": 32264, + "\u0120Concern": 32265, + "\u0120Citadel": 32266, + "\u0120lieu": 32267, + "\u0120candles": 32268, + "ogeneous": 32269, + "arger": 32270, + "\u0120heavenly": 32271, + "inflammatory": 32272, + "Performance": 32273, + "Cs": 32274, + "ructose": 32275, + "azaki": 32276, + "\u0120pessim": 32277, + "\u0120inference": 32278, + "\u0120powd": 32279, + "\u0120Zoe": 32280, + "\u0120paints": 32281, + "\u0120dazz": 32282, + "pta": 32283, + "-----------": 32284, + "\u0120inspir": 32285, + "\u0120Experimental": 32286, + "\u0120Knife": 32287, + "regor": 32288, + "bors": 32289, + "\u0120showers": 32290, + "romeda": 32291, + "\u0120saint": 32292, + "\u0120benign": 32293, + "\u0120Jiang": 32294, + "\u0120envisioned": 32295, + "\u0120shroud": 32296, + "IFT": 32297, + "HO": 32298, + "\u0120shuff": 32299, + "\u0120ICC": 32300, + "\u0120segreg": 32301, + "\u0120revisit": 32302, + "ighthouse": 32303, + "Li": 32304, + "\u0120substrate": 32305, + "\u0120Seas": 32306, + "\u0120Reward": 32307, + "\u0120Hep": 32308, + "\u0120Brass": 32309, + "sbm": 32310, + "\u0120eliminates": 32311, + "\u0120stamina": 32312, + "\u0120VAT": 32313, + "\u0120Loan": 32314, + "\u0120constraint": 32315, + "\u0120appropriated": 32316, + "\u0120pes": 32317, + "\u0120ALE": 32318, + "ranging": 32319, + "\u0120404": 32320, + "392": 32321, + "\u0120intellectuals": 32322, + "achu": 32323, + "\u0120restructuring": 32324, + "\u0120Levin": 32325, + "\u0120runes": 32326, + "\u0120delightful": 32327, + "\u0120carbohydrates": 32328, + "\u0120Models": 32329, + "\u0120Expo": 32330, + "\u0120transporting": 32331, + "alloc": 32332, + "\u0120ringing": 32333, + "Samsung": 32334, + "\u0120scarcely": 32335, + "\u0120URLs": 32336, + "\u0120MAS": 32337, + "\u0120prototypes": 32338, + "\u0120narrator": 32339, + "\u0120CPUs": 32340, + "cdn": 32341, + "\u0120Barton": 32342, + "\u0120decidedly": 32343, + "\u0120Shu": 32344, + "ixir": 32345, + "ocious": 32346, + "\u0120Myst": 32347, + "Nintendo": 32348, + "\u0120reuse": 32349, + "\u0120forgiven": 32350, + "Few": 32351, + "inical": 32352, + "nat": 32353, + "\u0120seamless": 32354, + "\u0120Eva": 32355, + "\u0120EVE": 32356, + "\u0120JO": 32357, + "landers": 32358, + "\u0120softer": 32359, + "negie": 32360, + "\u0120transient": 32361, + "\u0120orbital": 32362, + "\u0120fulfil": 32363, + "\u0120Kom": 32364, + "Hopefully": 32365, + "\u0120dynamically": 32366, + "\u0120Hunger": 32367, + "\u00e5\u013d": 32368, + "\u0120Armenia": 32369, + "elman": 32370, + "berto": 32371, + "\u0120pige": 32372, + "\u0120IDs": 32373, + "limit": 32374, + "\u0120veins": 32375, + "\u0120soaring": 32376, + "packs": 32377, + "Golden": 32378, + "\u0120Crab": 32379, + "istor": 32380, + "\u0120RPM": 32381, + "\u0120$$": 32382, + "gression": 32383, + "\u0120jihadist": 32384, + "\u0120gamble": 32385, + "\u0120careg": 32386, + "\u0120inflated": 32387, + "Face": 32388, + "\u0120Firearms": 32389, + "\u0120Emmanuel": 32390, + "\u00e2\u013f": 32391, + "\u0120shocks": 32392, + "grab": 32393, + "\u0120splend": 32394, + "\u0120HPV": 32395, + "abortion": 32396, + "Above": 32397, + "Entity": 32398, + "players": 32399, + "\u0120commenced": 32400, + "ulence": 32401, + "\u0120fulfillment": 32402, + "\u0120embodiments": 32403, + "\u0120Welfare": 32404, + "\u0120hail": 32405, + "\u0120<@": 32406, + "tten": 32407, + "\u0120catcher": 32408, + "\u0120Jazeera": 32409, + "\u0120volcano": 32410, + "\u0120stabilize": 32411, + "\u0120Handler": 32412, + "\u0120intensified": 32413, + "\u0120Abrams": 32414, + "\u0120humiliation": 32415, + "paced": 32416, + "605": 32417, + "\u0120CentOS": 32418, + "Specific": 32419, + "\u0120heed": 32420, + "\u0120CAM": 32421, + "\u0120Galile": 32422, + "Die": 32423, + "\u0120abolished": 32424, + "\u0120Thomson": 32425, + "\u0120Teachers": 32426, + "\u0120Wass": 32427, + "jong": 32428, + "\u0120ISBN": 32429, + "\u0120Allies": 32430, + "shake": 32431, + "\u00e5\u00b7": 32432, + "vict": 32433, + "Howard": 32434, + "\u0120deem": 32435, + "\u0120exceedingly": 32436, + "\u0120Smartstocks": 32437, + "ibe": 32438, + "\u0120doorway": 32439, + "\u0120competed": 32440, + "igmat": 32441, + "\u0120nationalists": 32442, + "\u0120groom": 32443, + "\u0120Keen": 32444, + "\u0120disposable": 32445, + "decl": 32446, + "\u0120Tolkien": 32447, + "\u0120Scheme": 32448, + "\u0120biod": 32449, + "\u0120avid": 32450, + "\u0120Elon": 32451, + "agar": 32452, + "\u0120TSA": 32453, + "Roman": 32454, + "\u0120artificially": 32455, + "\u0120advisors": 32456, + "XL": 32457, + "\u0120Inferno": 32458, + "366": 32459, + "\u0120tedious": 32460, + "\u0120Photography": 32461, + "\u0120Carrie": 32462, + "\u0120trope": 32463, + "\u0120Sandra": 32464, + "\u0120decimal": 32465, + "Queen": 32466, + "\u0120Gundam": 32467, + "\u0120OM": 32468, + "otech": 32469, + "NBA": 32470, + "\u01201932": 32471, + "\u0120entrenched": 32472, + "\u0120Marion": 32473, + "\u0120fraternity": 32474, + "Labour": 32475, + "Henry": 32476, + "\u0120latitude": 32477, + "Either": 32478, + "\u0120enhances": 32479, + "\u0120Potential": 32480, + "\u0120shines": 32481, + "idad": 32482, + "\u0120breadth": 32483, + "\u0120capacities": 32484, + "\u0120\u00f0\u0141\u013b\u0124": 32485, + "\u0120Bronx": 32486, + "\u0120sexes": 32487, + "\u0120differentiation": 32488, + "\u0120heavyweight": 32489, + "\u0120Taj": 32490, + "dra": 32491, + "\u0120migrate": 32492, + "\u0120exhaustion": 32493, + "\u0120RUN": 32494, + "elsius": 32495, + "\u0120Cuomo": 32496, + "\u0120guitars": 32497, + "\u0120clones": 32498, + "\u0120Somew": 32499, + "\u0120Pry": 32500, + "-------------": 32501, + "\u0120warranted": 32502, + "cycles": 32503, + "\u0120salvage": 32504, + "\u0120disks": 32505, + "RANT": 32506, + "\u0120NGOs": 32507, + "\u0120Martian": 32508, + "\":[{\"": 32509, + "\u0120addicts": 32510, + "ojure": 32511, + "illet": 32512, + "\u0120amazingly": 32513, + "artments": 32514, + "pixel": 32515, + "\u0120GPUs": 32516, + "Layout": 32517, + "\u00e8\u00a3": 32518, + "\u0120Tamil": 32519, + "\u0120Basil": 32520, + "\u0120impartial": 32521, + "\u0120Structure": 32522, + "fork": 32523, + "bryce": 32524, + "\u0120ridge": 32525, + "\u0120Hamburg": 32526, + "rious": 32527, + "\u0120blitz": 32528, + "cigarettes": 32529, + "\u0120canned": 32530, + "402": 32531, + "\u0120ironically": 32532, + "\u0120compassionate": 32533, + "\u0120Hawkins": 32534, + ".#": 32535, + "\u0120Cathedral": 32536, + "\u0120rallied": 32537, + "internal": 32538, + "\u0120quota": 32539, + "stakes": 32540, + "TEXT": 32541, + "mom": 32542, + "\u0120completes": 32543, + "\u0120238": 32544, + "\u0120shrug": 32545, + "\u00e3\u0125\u0133": 32546, + "\u0120Ninth": 32547, + "\u0120revise": 32548, + "\u0120Provider": 32549, + "\u0120treacher": 32550, + "\u0120quasi": 32551, + "\u0120PRES": 32552, + "\u0120deposition": 32553, + "\u0120confidentiality": 32554, + "issors": 32555, + "\u0120imbalance": 32556, + "\u0120spanning": 32557, + "\u0120angular": 32558, + "\u0120Cul": 32559, + "communication": 32560, + "\u0120Nora": 32561, + "\u0120Genius": 32562, + "opter": 32563, + "\u0120sacked": 32564, + "Spot": 32565, + "\u0120finely": 32566, + "\u0120CHR": 32567, + "282": 32568, + "waves": 32569, + "Palest": 32570, + "\u0120Rohing": 32571, + "NL": 32572, + "\u00e8\u00bf": 32573, + "\u0120shitty": 32574, + "\u0120Scalia": 32575, + "475": 32576, + "Progress": 32577, + "\u0120referencing": 32578, + "\u0120classrooms": 32579, + "abee": 32580, + "\u0120sod": 32581, + "hesion": 32582, + "708": 32583, + "\u0120Zuckerberg": 32584, + "\u0120Finish": 32585, + "\u0120Scotia": 32586, + "\u0120Savior": 32587, + "\u0120Installation": 32588, + "antha": 32589, + "(-": 32590, + "\u0120302": 32591, + "\u0120Punk": 32592, + "\u0120crater": 32593, + "youtu": 32594, + "\u0120roast": 32595, + "\u0120influencing": 32596, + "\u0120dup": 32597, + "\u0120JR": 32598, + "\u0120Grav": 32599, + "\u0120stature": 32600, + "\u0120bathrooms": 32601, + "Aside": 32602, + "Wiki": 32603, + "mean": 32604, + "\u0120Zak": 32605, + "\u0120Ones": 32606, + "\u0120Nath": 32607, + "\u0120hypert": 32608, + "\u0120commencement": 32609, + "Civil": 32610, + "\u0120moderately": 32611, + "\u0120distributors": 32612, + "\u0120breastfeeding": 32613, + "\u0120980": 32614, + "\u0120Sik": 32615, + "\u0120Cig": 32616, + "\u0120AMER": 32617, + "RIP": 32618, + "\u0120Career": 32619, + "usting": 32620, + "\u0120messed": 32621, + "\u0120eh": 32622, + "\u0120Jensen": 32623, + "/$": 32624, + "\u0120blackmail": 32625, + "\u0120conversions": 32626, + "\u0120scientifically": 32627, + "\u0120mantra": 32628, + "paying": 32629, + "\u0120ivory": 32630, + "\u0120Courts": 32631, + "OUGH": 32632, + "auntlet": 32633, + "Serial": 32634, + "Brow": 32635, + "\u0120Hundreds": 32636, + "323": 32637, + "\u0120pee": 32638, + "\u0120linux": 32639, + "\u0120submer": 32640, + "\u0120Principal": 32641, + "485": 32642, + "\u0120DSL": 32643, + "\u0120Cousins": 32644, + "\u0120doctrines": 32645, + "\u0120Athletics": 32646, + "\u0120315": 32647, + "\u0120Karma": 32648, + "\u0120attent": 32649, + "urger": 32650, + "\u0120prescribe": 32651, + "\u0120encaps": 32652, + "\u0120Came": 32653, + "\u0120secretive": 32654, + "\u0120Crimes": 32655, + "dn": 32656, + "Clean": 32657, + "\u0120Egyptians": 32658, + "\u0120Carpenter": 32659, + "\u0120ll": 32660, + "Hum": 32661, + "\u0120Milo": 32662, + "\u0120capitalists": 32663, + "\u0120briefed": 32664, + "Twe": 32665, + "\u0120Basin": 32666, + "elvet": 32667, + "Mos": 32668, + "\u0120plunge": 32669, + "\u0120Kaiser": 32670, + "\u0120Fuj": 32671, + "illin": 32672, + "\u0120safeguards": 32673, + "\u0120oste": 32674, + "\u0120Opportunity": 32675, + "\u0120Mafia": 32676, + "\u0120Calling": 32677, + "apa": 32678, + "urban": 32679, + "brush": 32680, + "illard": 32681, + "c\u00c3\u00a9": 32682, + "intelligence": 32683, + "\u0120Lob": 32684, + "\u0120Druid": 32685, + "\u0120smoother": 32686, + "\u0120footing": 32687, + "\u0120motorists": 32688, + "arcity": 32689, + "\u0120masculinity": 32690, + "\u0120mism": 32691, + "\u0120abdominal": 32692, + "\u0120Tavern": 32693, + "\u0120Roh": 32694, + "\u0120escapes": 32695, + "signed": 32696, + "Anthony": 32697, + "\u0120sacrificing": 32698, + "\u0120intimacy": 32699, + "\u0120anterior": 32700, + "\u0120Kod": 32701, + "\u0120motif": 32702, + "\u0120graz": 32703, + "\u0120visualization": 32704, + "\u0120guitarist": 32705, + "\u0120Trotsky": 32706, + "magic": 32707, + "Dar": 32708, + "\u0120Mori": 32709, + "\u0120wards": 32710, + "\u0120toilets": 32711, + "lest": 32712, + "\u0120teleport": 32713, + "\u0120Sundays": 32714, + "\u0120Plat": 32715, + "ETS": 32716, + "\u0120eSports": 32717, + "Patrick": 32718, + "\u0120Katherine": 32719, + "enko": 32720, + "\u0120hassle": 32721, + "\u0120Mick": 32722, + "ggles": 32723, + "\u0120hob": 32724, + "aintain": 32725, + "\u0120airborne": 32726, + "\u0120spans": 32727, + "\u0120chili": 32728, + "\u0120aperture": 32729, + "\u0120volunteered": 32730, + "\u0120Incident": 32731, + "\u0120Fres": 32732, + "\u0120Veteran": 32733, + "aughtered": 32734, + "ingo": 32735, + "\u0120uninsured": 32736, + "CLOSE": 32737, + "\u0120fuse": 32738, + "\u0120erotic": 32739, + "\u0120advertise": 32740, + "raising": 32741, + "Texture": 32742, + "\u0120attends": 32743, + "\u0120REAL": 32744, + "uddled": 32745, + "\u0120smoot": 32746, + "\u0120305": 32747, + "\u0120Willis": 32748, + "\u0120blond": 32749, + "Analysis": 32750, + "\u0120VT": 32751, + "onica": 32752, + "\u0120stronghold": 32753, + "RF": 32754, + "NM": 32755, + ".>>": 32756, + "\u0120prosperous": 32757, + "\u0120boasted": 32758, + "292": 32759, + "\u0120Manufacturing": 32760, + "PRESS": 32761, + "gren": 32762, + "\u0120pharmacy": 32763, + "\u0120Rockefeller": 32764, + "kai": 32765, + "\u0120thumbs": 32766, + "\u0120Hut": 32767, + "\u0120motherboard": 32768, + "\u0120guardians": 32769, + "\u0120Alter": 32770, + "llular": 32771, + "\u0120shack": 32772, + "\u0120wisely": 32773, + "\u0120backbone": 32774, + "erva": 32775, + "\u0120suicides": 32776, + "\u0120McGregor": 32777, + "ijah": 32778, + "Emer": 32779, + "\u0120Brav": 32780, + "\u0120designate": 32781, + "POST": 32782, + "produced": 32783, + "\u0120cleansing": 32784, + "irlwind": 32785, + "existent": 32786, + "\u0120Humph": 32787, + "\u0120Payne": 32788, + "\u0120vested": 32789, + "\u00c5\u00a1": 32790, + "\u0120stringent": 32791, + "iona": 32792, + "\u0120unsub": 32793, + "\u0120summed": 32794, + "\u0120Hercules": 32795, + "subject": 32796, + "\u0120Ragnar": 32797, + "\u0120Nos": 32798, + "\u0120characterization": 32799, + "\u0120savvy": 32800, + "\u0120Dawson": 32801, + "\u0120Casino": 32802, + "\u0120fri": 32803, + "\u0120Barrier": 32804, + "\u0120misinformation": 32805, + "\u0120insulation": 32806, + "\u0120corridors": 32807, + "\u0120airplanes": 32808, + "\u0120Noct": 32809, + "ahi": 32810, + "\u01201916": 32811, + "kb": 32812, + "armac": 32813, + "\u0120shun": 32814, + "\u0120schema": 32815, + "\u0120horrified": 32816, + "\u0120239": 32817, + "aunders": 32818, + "NB": 32819, + "iates": 32820, + "erity": 32821, + "\u0120Shard": 32822, + "\u0120rarity": 32823, + "\u0120grouped": 32824, + "\u0120Ghana": 32825, + "against": 32826, + "\u0120Biological": 32827, + "\u0120Aware": 32828, + "owell": 32829, + "\u00cf\u0126": 32830, + "\u0120Beau": 32831, + "shaw": 32832, + "Hack": 32833, + "\u0120Julius": 32834, + "USS": 32835, + "olson": 32836, + "auna": 32837, + "cru": 32838, + "\u0120Maurice": 32839, + "\u0120Ik": 32840, + "\u0120sequencing": 32841, + "\u0120radicals": 32842, + "\u0120(?,": 32843, + "virtual": 32844, + "\u0120anyways": 32845, + "\u0120reperc": 32846, + "\u0120handlers": 32847, + "\u0120hesitant": 32848, + "\u00e9\u0125": 32849, + "\u0120MF": 32850, + "plementation": 32851, + "associated": 32852, + "\u0120campaigned": 32853, + "\u0120Yue": 32854, + "utations": 32855, + "\u0120Yoga": 32856, + "\u0120simmer": 32857, + "\u0120rods": 32858, + "\u0120melody": 32859, + "\u0120convoy": 32860, + "videos": 32861, + "\u0120screened": 32862, + "Neg": 32863, + "ochemical": 32864, + "\u0120())": 32865, + "\u0120ultras": 32866, + "\u0120antip": 32867, + "\u0120Islanders": 32868, + "704": 32869, + "\u0120fetish": 32870, + "\u0120ridiculously": 32871, + "\u0120Kart": 32872, + "\u0120mitochondrial": 32873, + "\u0120interfering": 32874, + "Builder": 32875, + "\u0120overfl": 32876, + "\u0120acne": 32877, + "\u0120Mud": 32878, + "\u0120Kerr": 32879, + "flex": 32880, + "\u0120Postal": 32881, + "\u0120Baltic": 32882, + "477": 32883, + "\u0120Persons": 32884, + "ourage": 32885, + "HB": 32886, + "\u0120Muse": 32887, + "\u0120Immortal": 32888, + "\u0120Driving": 32889, + "\u0120petitions": 32890, + "\u0120subscript": 32891, + "\u0120sorce": 32892, + "\u0120Processor": 32893, + "uton": 32894, + "Sony": 32895, + "\u0120phon": 32896, + "\u0120raced": 32897, + "\u0120Anthrop": 32898, + "\u0120daytime": 32899, + "\u0120Exercise": 32900, + "Adding": 32901, + "\u0120engages": 32902, + "\u0120Qualcomm": 32903, + "\u0120miracles": 32904, + "\u0120memes": 32905, + "\u0120Drink": 32906, + "\u0120Orioles": 32907, + "\u0120hairs": 32908, + "\u0120Polar": 32909, + "athom": 32910, + "\u0120slippery": 32911, + "\u0120Remy": 32912, + "\u0120caramel": 32913, + "\u0120YEAR": 32914, + "\u0120alk": 32915, + "Ign": 32916, + "aution": 32917, + "\u0120Merlin": 32918, + "\u0120Cran": 32919, + "\u0120apologies": 32920, + "\u0120410": 32921, + "\u0120outing": 32922, + "\u0120Memories": 32923, + "appointed": 32924, + "\u0120countered": 32925, + "uld": 32926, + "posing": 32927, + "\u0120firewall": 32928, + "\u0120Wast": 32929, + "\u0120Wet": 32930, + "worked": 32931, + "seller": 32932, + "\u0120repealed": 32933, + "ereo": 32934, + "assuming": 32935, + "BLIC": 32936, + "mite": 32937, + "\u0120CEOs": 32938, + "\u0120Chapel": 32939, + "elligent": 32940, + "________________________": 32941, + "Dog": 32942, + "\u0120wart": 32943, + "\u0120subscriber": 32944, + "sports": 32945, + "\u0120begged": 32946, + "\u0120MV": 32947, + "\u0120semif": 32948, + "ethical": 32949, + "\u0120preach": 32950, + "\u0120revital": 32951, + "\u0120punitive": 32952, + "\u0120shortcuts": 32953, + "\u0120instituted": 32954, + "\u0120Warsaw": 32955, + "\u0120abdomen": 32956, + "\u0120KING": 32957, + "\u0120superintendent": 32958, + "\u0120fry": 32959, + "\u0120Geo": 32960, + "TOR": 32961, + "\u0120contradictions": 32962, + "aptic": 32963, + "\u0120landscapes": 32964, + "bugs": 32965, + "\u0120clust": 32966, + "\u0120volley": 32967, + "cribed": 32968, + "\u0120tandem": 32969, + "\u0120robes": 32970, + "WHAT": 32971, + "\u0120promoter": 32972, + "\u0120eloqu": 32973, + "reviewed": 32974, + "\u0120DK": 32975, + "\u0120Plato": 32976, + "\u0120fps": 32977, + "Tank": 32978, + "\u0120Derrick": 32979, + "\u0120prioritize": 32980, + "asper": 32981, + "\u0120Honduras": 32982, + "\u0120Completed": 32983, + "nec": 32984, + "\u0120mog": 32985, + "nir": 32986, + "\u0120Mayo": 32987, + "DEF": 32988, + "stall": 32989, + "inness": 32990, + "\u0120Volkswagen": 32991, + "\u0120precaution": 32992, + "\u0120Mell": 32993, + "iak": 32994, + "istries": 32995, + "\u0120248": 32996, + "\u0120overlapping": 32997, + "Senate": 32998, + "\u0120Enhance": 32999, + "resy": 33000, + "racial": 33001, + "ORTS": 33002, + "\u0120Mormons": 33003, + "Strong": 33004, + "\u0120Coch": 33005, + "Mexico": 33006, + "\u0120Maduro": 33007, + "\u0120jars": 33008, + "\u0120cane": 33009, + "Wik": 33010, + "olla": 33011, + "ifference": 33012, + "\u0120physicist": 33013, + "\u0120Maggie": 33014, + "\u0120285": 33015, + "\u0120depiction": 33016, + "\u0120McLaren": 33017, + "Ju": 33018, + "\u0120slows": 33019, + "\u0120commissioners": 33020, + "\u0120Willow": 33021, + "\u0120Explos": 33022, + "hovah": 33023, + "\u0120technician": 33024, + "\u0120homicides": 33025, + "\u0120Flav": 33026, + "\u0120Truman": 33027, + "\u012010000": 33028, + "uctor": 33029, + "\u0120shader": 33030, + "Newsletter": 33031, + "457": 33032, + "\u0120rever": 33033, + "\u0120hardened": 33034, + "\u0120whereabouts": 33035, + "\u0120redevelop": 33036, + "\u0120carbs": 33037, + "\u0120travers": 33038, + "\u0120squirrel": 33039, + "\u0120follower": 33040, + "\u0120sings": 33041, + "508": 33042, + "\u0120rabbits": 33043, + "emonium": 33044, + "\u0120documenting": 33045, + "\u0120misunderstood": 33046, + ")'": 33047, + "Rick": 33048, + "ggies": 33049, + "\u0120premie": 33050, + "\u0120skating": 33051, + "\u0120passports": 33052, + "\u0120fists": 33053, + "ageddon": 33054, + "Haw": 33055, + "ACP": 33056, + "080": 33057, + "\u0120Thoughts": 33058, + "\u0120Carlson": 33059, + "\u0120priesthood": 33060, + "hua": 33061, + "\u0120dungeons": 33062, + "\u0120Loans": 33063, + "\u0120antis": 33064, + "\u0120familiarity": 33065, + "\u0120Sabb": 33066, + "opal": 33067, + "\u0120Ink": 33068, + "strike": 33069, + "\u0120cram": 33070, + "\u0120legalized": 33071, + "\u0120cuisine": 33072, + "\u0120fibre": 33073, + "Travel": 33074, + "\u0120Monument": 33075, + "ODY": 33076, + "ethy": 33077, + "\u0120interstate": 33078, + "\u0120PUR": 33079, + "emporary": 33080, + "\u0120Arabian": 33081, + "developed": 33082, + "\u0120saddle": 33083, + "\u0120github": 33084, + "\u0120Offer": 33085, + "\u0120ISP": 33086, + "rolet": 33087, + "\u0120SUPER": 33088, + "\u0120Denis": 33089, + "\u0120multiplier": 33090, + "\u0120stirred": 33091, + "Interestingly": 33092, + "\u0120customary": 33093, + "\u0120billed": 33094, + "hex": 33095, + "\u0120multiplied": 33096, + "\u0120flipping": 33097, + "\u0120Crosby": 33098, + "\u0120fundamentals": 33099, + "iae": 33100, + "\u0120Played": 33101, + "\u0120Atom": 33102, + "amazon": 33103, + "\u0120Flam": 33104, + "eez": 33105, + "activated": 33106, + "\u0120tablespoon": 33107, + "\u0120liberalism": 33108, + "\u0120Palin": 33109, + "\u0120Patel": 33110, + "Num": 33111, + "\u0120TAM": 33112, + "\u0120surn": 33113, + "\u0120Reloaded": 33114, + "\u0120coined": 33115, + "\"],": 33116, + "\u0120Clash": 33117, + "\u0120Agu": 33118, + "\u0120pragmatic": 33119, + "\u0120Activate": 33120, + "\u0120802": 33121, + "\u0120trailers": 33122, + "\u0120silhou": 33123, + "\u0120probes": 33124, + "\u0120circus": 33125, + "\u0120Bain": 33126, + "\u0120Lindsay": 33127, + "\u0120Abbey": 33128, + "Delivery": 33129, + "\u0120concession": 33130, + "\u0120gastro": 33131, + "\u0120Sprite": 33132, + "\u00c4\u0141": 33133, + "andel": 33134, + "\u0120gimm": 33135, + "\u0120autobi": 33136, + "\u0120Turtle": 33137, + "\u0120wonderfully": 33138, + "\u0120Haram": 33139, + "\u0120Worldwide": 33140, + "\u0120Handle": 33141, + "\u0120theorists": 33142, + "\u0120sleek": 33143, + "\u0120Zhu": 33144, + "ographically": 33145, + "EGA": 33146, + "\u0120Owners": 33147, + "aths": 33148, + "\u0120Antarctic": 33149, + "natal": 33150, + "=\"\"": 33151, + "flags": 33152, + "````": 33153, + "\u0120sul": 33154, + "Kh": 33155, + "\u0120potassium": 33156, + "\u0120lineman": 33157, + "\u0120cereal": 33158, + "\u0120Seasons": 33159, + "\u01202022": 33160, + "\u0120mathematic": 33161, + "\u0120astronomers": 33162, + "professional": 33163, + "\u0120fares": 33164, + "cknowled": 33165, + "\u0120chi": 33166, + "\u0120youngsters": 33167, + "\u0120mistakenly": 33168, + "\u0120hemisphere": 33169, + "\u0120Divinity": 33170, + "rone": 33171, + "\u0120\",": 33172, + "rings": 33173, + "\u0120attracts": 33174, + "vana": 33175, + "\u00e5\u00b9": 33176, + "CAP": 33177, + "\u0120playlist": 33178, + "\u0120porch": 33179, + "\u00e3\u0123\u00a3": 33180, + "\u0120incorporates": 33181, + "\u0120soak": 33182, + "\u0120asserting": 33183, + "\u0120Terrorism": 33184, + "\u0120Pablo": 33185, + "Ja": 33186, + "cester": 33187, + "\u0120fearing": 33188, + "\u0120Prayer": 33189, + "\u0120escalated": 33190, + "GW": 33191, + "\u0120robe": 33192, + "\u0120Brighton": 33193, + "acists": 33194, + "\u0120Symphony": 33195, + "\u0120Dwarf": 33196, + "\u0120Parade": 33197, + "\u0120Lego": 33198, + "\u0120inexpl": 33199, + "\u0120lords": 33200, + "leaf": 33201, + "RAG": 33202, + "liber": 33203, + "\u0120cigars": 33204, + "\u0120Jehovah": 33205, + "606": 33206, + "WINDOWS": 33207, + "\u0120Liberia": 33208, + "ebus": 33209, + "Heavy": 33210, + "\u0120lubric": 33211, + "\u0120RW": 33212, + "anguages": 33213, + "\u0120narrowed": 33214, + "computer": 33215, + "\u0120Ember": 33216, + "\u0120murdering": 33217, + "\u0120downstream": 33218, + "\u0120Tuls": 33219, + "\u0120Tables": 33220, + "Topic": 33221, + "\u0120Accuracy": 33222, + "=/": 33223, + "lost": 33224, + "\u0120Rei": 33225, + "\u0120progresses": 33226, + "bear": 33227, + "\u0120establishments": 33228, + "Justin": 33229, + "\u0120Peach": 33230, + "\u0120Gomez": 33231, + "\u00e5\u00bf": 33232, + "\u0120Triangle": 33233, + "Ident": 33234, + "\u0120Hive": 33235, + "Resources": 33236, + "\u0120mixes": 33237, + "\u0120Assuming": 33238, + "Mu": 33239, + "\u0120hypoc": 33240, + "\u0120sane": 33241, + "\u0120Wan": 33242, + "idious": 33243, + "Success": 33244, + "\u0120io": 33245, + "Angel": 33246, + "\u0120dangerously": 33247, + "\u0120Creature": 33248, + "WORK": 33249, + ":[": 33250, + "\u0120Katrina": 33251, + "Listener": 33252, + "Miller": 33253, + "\u0120Idlib": 33254, + "hang": 33255, + "\u0120circumvent": 33256, + "href": 33257, + "\u0120celestial": 33258, + "\u0120Weeks": 33259, + "\u0120Pug": 33260, + "\u0120Dalton": 33261, + "\u0120subpoena": 33262, + "uku": 33263, + "\u0120persisted": 33264, + "pei": 33265, + "olding": 33266, + "\u0120Documents": 33267, + "\u0120Hast": 33268, + "\u0120CENT": 33269, + "\u0120primer": 33270, + "\u0120synonymous": 33271, + "\u0120nib": 33272, + "ombs": 33273, + "\u0120notation": 33274, + "\u0120Dish": 33275, + "\u0120Atmosp": 33276, + "\u0120forbid": 33277, + "\u0120ANG": 33278, + "pattern": 33279, + "los": 33280, + "\u0120projectiles": 33281, + "brown": 33282, + ".\",": 33283, + "\u0120Venom": 33284, + "\u0120fiercely": 33285, + "ublished": 33286, + "\u0120Uran": 33287, + "\u0120Nicarag": 33288, + "410": 33289, + "\u0120CAL": 33290, + "OTOS": 33291, + "\u0120Miracle": 33292, + "\u0120Enchant": 33293, + "\u0120guarding": 33294, + "append": 33295, + "Attach": 33296, + "\u0120leveled": 33297, + "\u0120condoms": 33298, + "ihilation": 33299, + "649": 33300, + "\u0120nightmares": 33301, + "\u0120THEY": 33302, + "\u0120START": 33303, + "\u0120Kinn": 33304, + "\u0120roommate": 33305, + "\u0120hygiene": 33306, + "opping": 33307, + "Job": 33308, + "\u0120lvl": 33309, + "\u0120VER": 33310, + "\u0120Keeping": 33311, + "abetic": 33312, + "\u0120formatting": 33313, + "erala": 33314, + "\u0120revisions": 33315, + "\u0120resurg": 33316, + "Tel": 33317, + "\u0120Goodman": 33318, + "353": 33319, + "pod": 33320, + "\u0120indisp": 33321, + "\u0120Translation": 33322, + "\u0120gown": 33323, + "\u0120Mund": 33324, + "\u0120cis": 33325, + "\u0120bystand": 33326, + "collect": 33327, + "\u0120Punjab": 33328, + "actively": 33329, + "\u0120Gamb": 33330, + "tell": 33331, + "\u0120importing": 33332, + "gencies": 33333, + "\u0120locom": 33334, + "\u0120Brill": 33335, + "Holy": 33336, + "\u0120Berger": 33337, + "\u0120showdown": 33338, + "\u0120responders": 33339, + "ILY": 33340, + "\u0120takedown": 33341, + "leted": 33342, + "\u0120mattered": 33343, + "\u0120predictive": 33344, + "\u0120overlay": 33345, + "GPU": 33346, + "\u0120Vick": 33347, + "\u0120conveyed": 33348, + "Tab": 33349, + "peer": 33350, + "Scan": 33351, + "\u0120defensively": 33352, + "vae": 33353, + "\u0120approving": 33354, + "\u0120tiers": 33355, + "\u0120Via": 33356, + "querade": 33357, + "\u0120Saudis": 33358, + "\u0120demolished": 33359, + "\u0120Prophe": 33360, + "\u0120mono": 33361, + "\u0120hospitality": 33362, + "HAM": 33363, + "\u0120Ariel": 33364, + "MOD": 33365, + "\u0120Torah": 33366, + "\u0120blah": 33367, + "\u0120Belarus": 33368, + "erential": 33369, + "\u0120Tuc": 33370, + "\u0120banker": 33371, + "397": 33372, + "\u0120mosquit": 33373, + "\u0120Scientist": 33374, + "\u0120Musical": 33375, + "\u0120hust": 33376, + "Shift": 33377, + "\u0120torment": 33378, + "\u0120standoff": 33379, + "Educ": 33380, + "\u0120Fog": 33381, + "\u0120amplifier": 33382, + "Shape": 33383, + "Instance": 33384, + "\u0120Critics": 33385, + "\u0120daemon": 33386, + "Houston": 33387, + "\u0120mattress": 33388, + "\u0120IDF": 33389, + "\u0120obscene": 33390, + "\u0120Amer": 33391, + "hetti": 33392, + "\u0120compiling": 33393, + "352": 33394, + "verett": 33395, + "\u0120Reduction": 33396, + "istration": 33397, + "\u0120Blessed": 33398, + "\u0120Bachelor": 33399, + "316": 33400, + "\u0120prank": 33401, + "\u0120Vulcan": 33402, + "dding": 33403, + "\u0120mourning": 33404, + "\u0120Quint": 33405, + "\u0120Blaster": 33406, + "testing": 33407, + "\u0120sediment": 33408, + ">>>": 33409, + "\u0120Eternity": 33410, + "\u0120WHERE": 33411, + "\u0120Maze": 33412, + "\u0120reacting": 33413, + "\u0120Alv": 33414, + "omsday": 33415, + "\u0120CRA": 33416, + "\u0120translator": 33417, + "\u0120bogus": 33418, + "atu": 33419, + "Website": 33420, + "olls": 33421, + "\u0120baptism": 33422, + "\u0120sibling": 33423, + "\u0120Autumn": 33424, + "vez": 33425, + "\u00e3\u0123\u00ae\u00e9": 33426, + "guards": 33427, + "Georg": 33428, + "assadors": 33429, + "\u0120Freud": 33430, + "\u0120continents": 33431, + "\u0120Registry": 33432, + "Bernie": 33433, + "\u0138\u013c\u00e5\u00a3\u00ab": 33434, + "\u0120tolerant": 33435, + "\u0120UW": 33436, + "\u0120horribly": 33437, + "995": 33438, + "\u0120MIDI": 33439, + "\u0120impatient": 33440, + "ocado": 33441, + "eri": 33442, + "\u0120Worst": 33443, + "\u0120Norris": 33444, + "\u0120Talking": 33445, + "\u0120defends": 33446, + "ensable": 33447, + "\u01202021": 33448, + "\u0120anatomy": 33449, + "Lew": 33450, + "\u0120drawer": 33451, + "\u0120Canberra": 33452, + "\u0120patriotic": 33453, + "\u00e9\u00be\u012f\u00e5\u0138\u013c\u00e5\u00a3\u00ab": 33454, + "\u0120Avg": 33455, + "ARM": 33456, + "\u0120undisclosed": 33457, + "\u0120farewell": 33458, + "459": 33459, + "bable": 33460, + "\u0120Allison": 33461, + "OLOG": 33462, + "\u0120conco": 33463, + "tight": 33464, + "\u0120ACPI": 33465, + "\u0120Mines": 33466, + "lich": 33467, + "\u0120\u00e2\u0136\u013e": 33468, + "represented": 33469, + "200000": 33470, + "\u0120enthusiast": 33471, + "OTS": 33472, + "bil": 33473, + "\u0120Ingredients": 33474, + "\u0120inventor": 33475, + "\u0120MySQL": 33476, + "\u00c2\u0142\u00c2\u0142\u00c2\u0142": 33477, + "\u0120ABOUT": 33478, + "within": 33479, + "\u0120mk": 33480, + "Bul": 33481, + "\u0120Fake": 33482, + "\u0120draconian": 33483, + "Wa": 33484, + "helm": 33485, + "\u0120Terran": 33486, + "erville": 33487, + "\u0120commonplace": 33488, + "SIZE": 33489, + "\u0120\"<": 33490, + "replace": 33491, + "ographs": 33492, + "\u0120SELECT": 33493, + "incible": 33494, + "\u0120Mostly": 33495, + "\u0120Sheffield": 33496, + "\u0120IDE": 33497, + "uggle": 33498, + "\u0120citations": 33499, + "hurst": 33500, + "\u0120Unix": 33501, + "\u0120unleash": 33502, + "\u0120Piper": 33503, + "\u0120Nano": 33504, + "\u0120succumb": 33505, + "\u0120reluctance": 33506, + "\u01202500": 33507, + "\u0120Merchant": 33508, + "\u0120wiret": 33509, + "\u0120combos": 33510, + "\u0120Birthday": 33511, + "\u0120charcoal": 33512, + "\u0120UPS": 33513, + "\u0120Fairfax": 33514, + "\u0120driveway": 33515, + "\u0120Tek": 33516, + "\u0120Pitch": 33517, + "overe": 33518, + "\u0120technicians": 33519, + "\u0120Actual": 33520, + "flation": 33521, + "\u0120Fiscal": 33522, + "\u0120Empty": 33523, + "anamo": 33524, + "\u0120magnesium": 33525, + "\u0120slut": 33526, + "\u0120growers": 33527, + "Investigators": 33528, + "():": 33529, + "\u0120Satellite": 33530, + "\u0120Keynes": 33531, + "missive": 33532, + "lane": 33533, + "\u0120borough": 33534, + "344": 33535, + "\u0120TEAM": 33536, + "\u0120Bethesda": 33537, + "CV": 33538, + "hower": 33539, + "\u0120RAD": 33540, + "\u0120chant": 33541, + "\u0120Riy": 33542, + "\u0120compositions": 33543, + "\u0120mildly": 33544, + "\u0120meddling": 33545, + "\u0120agility": 33546, + "aneers": 33547, + "501": 33548, + "\u0120synth": 33549, + "linger": 33550, + "291": 33551, + "\u0120exclaimed": 33552, + "Party": 33553, + "\u0120contamin": 33554, + "\u0120Manor": 33555, + "\u0120Respond": 33556, + "\u0120praising": 33557, + "\u0120manners": 33558, + "fleet": 33559, + "Summer": 33560, + "\u0120Lynd": 33561, + "\u0120Definitely": 33562, + "grim": 33563, + "\u0120bowling": 33564, + "stri": 33565, + "\u00e7\u013d": 33566, + "ynt": 33567, + "\u0120mandates": 33568, + "DIV": 33569, + "\u0120reconcile": 33570, + "views": 33571, + "\u0120Damon": 33572, + "vette": 33573, + "Flo": 33574, + "\u0120Greatest": 33575, + "ilon": 33576, + "icia": 33577, + "\u0120portrayal": 33578, + "\u0120cushion": 33579, + "504": 33580, + "1979": 33581, + "ossal": 33582, + "Applic": 33583, + "scription": 33584, + "\u0120mitigation": 33585, + "ATS": 33586, + "pac": 33587, + "\u0120erased": 33588, + "\u0120deficiencies": 33589, + "\u0120Hollande": 33590, + "\u0120Xu": 33591, + "\u0120bred": 33592, + "\u0120pregnancies": 33593, + "femin": 33594, + "\u0120emph": 33595, + "\u0120planners": 33596, + "\u0120outper": 33597, + "uttering": 33598, + "\u0120perpetrator": 33599, + "\u0120motto": 33600, + "\u0120Ellison": 33601, + "\u0120NEVER": 33602, + "\u0120admittedly": 33603, + "ARI": 33604, + "\u0120Azerbaijan": 33605, + "\u0120millisec": 33606, + "\u0120combustion": 33607, + "\u0120Bottle": 33608, + "\u0120Lund": 33609, + "\u0120Ps": 33610, + "\u0120Dress": 33611, + "\u0120fabricated": 33612, + "\u0120battered": 33613, + "\u0120sidel": 33614, + "\u0120Notting": 33615, + "Foreign": 33616, + "\u0120Jerome": 33617, + "020": 33618, + "\u0120Arbit": 33619, + "\u0120knots": 33620, + "\u0120RIGHT": 33621, + "Moving": 33622, + "\u00e3\u0123\u013b": 33623, + "\u0120surgeries": 33624, + "\u0120courthouse": 33625, + "\u0120mastered": 33626, + "\u0120hovering": 33627, + "\u0120Bran": 33628, + "\u0120Alison": 33629, + "\u0120safest": 33630, + "military": 33631, + "\u0120bullied": 33632, + "\u0120barrage": 33633, + "Reader": 33634, + "ESE": 33635, + "\u0120Geographic": 33636, + "Tools": 33637, + "314": 33638, + "\u0120Geek": 33639, + "roth": 33640, + "glers": 33641, + "\u0120FIN": 33642, + "\u00cf\u0123": 33643, + "\u0120Aston": 33644, + "altern": 33645, + "488": 33646, + "\u0120veterin": 33647, + "Gamer": 33648, + "\u0120intel": 33649, + "renches": 33650, + "Shield": 33651, + "\u0120amnesty": 33652, + "\u0120Bhar": 33653, + "\u0120piled": 33654, + "\u0120honorable": 33655, + "\u0120Institutes": 33656, + "\u0120soaked": 33657, + "\u0120coma": 33658, + "\u0120EFF": 33659, + "341": 33660, + "bytes": 33661, + "\u0120Gmail": 33662, + "lein": 33663, + "\u0120Canadiens": 33664, + "material": 33665, + "Il": 33666, + "\u0120instructors": 33667, + "\u0120KY": 33668, + "\u0120conceive": 33669, + "ubb": 33670, + "\u0120Possible": 33671, + "\u0120easing": 33672, + "\u0120Christina": 33673, + "\u0120caric": 33674, + "\u0120HDR": 33675, + "ROM": 33676, + "\u0120shovel": 33677, + "delete": 33678, + "\u0120puff": 33679, + "\u0120Changing": 33680, + "\u0120seamlessly": 33681, + "Attribute": 33682, + "\u0120acquisitions": 33683, + "akery": 33684, + "\u0120EF": 33685, + "\u0120autistic": 33686, + "\u0120Takes": 33687, + "\u0120Powder": 33688, + "\u0120Stir": 33689, + "510": 33690, + "\u0120Bubble": 33691, + "settings": 33692, + "\u0120Fowler": 33693, + "\u0120mustard": 33694, + "\u0120moreover": 33695, + "\u0120copyrighted": 33696, + "\u0120LEDs": 33697, + "1500": 33698, + "\u00e6\u012b": 33699, + "\u0120HIS": 33700, + "enf": 33701, + "\u0120custod": 33702, + "\u0120Huck": 33703, + "Gi": 33704, + "\u0120img": 33705, + "Answer": 33706, + "Ct": 33707, + "jay": 33708, + "\u0120Infrastructure": 33709, + "\u0120federally": 33710, + "Loc": 33711, + "\u0120microbes": 33712, + "\u0120overrun": 33713, + "dds": 33714, + "otent": 33715, + "adiator": 33716, + ">>>>>>>>": 33717, + "\u0120tornado": 33718, + "\u0120adjud": 33719, + "\u0120intrigued": 33720, + "\u0120si": 33721, + "\u0120Revelation": 33722, + "progress": 33723, + "\u0120burglary": 33724, + "\u0120Saiyan": 33725, + "\u0120Kathy": 33726, + "\u0120serpent": 33727, + "\u0120Andreas": 33728, + "\u0120compel": 33729, + "essler": 33730, + "\u0120Plastic": 33731, + "\u0120Advent": 33732, + "\u0120Positive": 33733, + "\u0120Qt": 33734, + "\u0120Hindus": 33735, + "registered": 33736, + "ularity": 33737, + "\u0120righteousness": 33738, + "\u0120demonic": 33739, + "uitive": 33740, + "\u0120BDS": 33741, + "\u0120Gregg": 33742, + "cia": 33743, + "\u0120Crusade": 33744, + "\u0120Sinai": 33745, + "WARE": 33746, + "+(": 33747, + "\u0120mell": 33748, + "\u0120derail": 33749, + "yards": 33750, + "Ast": 33751, + "\u0120noticeably": 33752, + "\u0120Ober": 33753, + "Ram": 33754, + "\u0120unnoticed": 33755, + "\u0120seq": 33756, + "avage": 33757, + "Ts": 33758, + "\u0120640": 33759, + "\u0120concede": 33760, + "\u0120])": 33761, + "Fill": 33762, + "\u0120captivity": 33763, + "\u0120Improvement": 33764, + "\u0120Crusader": 33765, + "araoh": 33766, + "MAP": 33767, + "\u00e6\u0139": 33768, + "\u0120stride": 33769, + "always": 33770, + "Fly": 33771, + "Nit": 33772, + "\u0120algae": 33773, + "\u0120Cooking": 33774, + "\u0120Doors": 33775, + "Malley": 33776, + "\u0120policemen": 33777, + "\u00e3\u0123\u012f": 33778, + "\u0120astronaut": 33779, + "accessible": 33780, + "495": 33781, + "\u0120RAW": 33782, + "cliffe": 33783, + "udicrous": 33784, + "\u0120depended": 33785, + "alach": 33786, + "\u0120ventures": 33787, + "rake": 33788, + "\u0120tits": 33789, + "\u0120Hou": 33790, + "\u0120condom": 33791, + "ormonal": 33792, + "\u0120indent": 33793, + "\u0120uploading": 33794, + "Footnote": 33795, + "Important": 33796, + "\u0120271": 33797, + "\u0120mindful": 33798, + "\u0120contends": 33799, + "Cra": 33800, + "\u0120calibr": 33801, + "\u0120OECD": 33802, + "plugin": 33803, + "Fat": 33804, + "\u0120ISS": 33805, + "\u0120Dynamics": 33806, + "ansen": 33807, + "686": 33808, + "'),": 33809, + "\u0120sprite": 33810, + "\u0120handheld": 33811, + "\u0120Hipp": 33812, + "=~=~": 33813, + "Trust": 33814, + "\u0120semantics": 33815, + "\u0120Bundes": 33816, + "\u0120Reno": 33817, + "\u0120Literature": 33818, + "sense": 33819, + "Gary": 33820, + "\u0120Aeg": 33821, + "\u0120Trin": 33822, + "EEK": 33823, + "\u0120cleric": 33824, + "\u0120SSH": 33825, + "\u0120christ": 33826, + "\u0120invading": 33827, + "ibu": 33828, + "\u0120enum": 33829, + "aura": 33830, + "\u0120allege": 33831, + "\u0120Incredible": 33832, + "BBC": 33833, + "\u0120thru": 33834, + "\u0120sailed": 33835, + "\u0120emulate": 33836, + "\u0120insecurity": 33837, + "\u0120crou": 33838, + "\u0120accommodations": 33839, + "\u0120incompetent": 33840, + "\u0120slips": 33841, + "\u0120Earthqu": 33842, + "sama": 33843, + "ILLE": 33844, + "\u0120iPhones": 33845, + "asaki": 33846, + "\u0120bye": 33847, + "\u0120ard": 33848, + "\u0120extras": 33849, + "\u0120slaughtered": 33850, + "\u0120crowdfunding": 33851, + "resso": 33852, + "\u0120filib": 33853, + "\u0120ERROR": 33854, + "\u0120TLS": 33855, + "egg": 33856, + "\u0120Ital": 33857, + "\u0120enlist": 33858, + "\u0120Catalonia": 33859, + "\u0120Scots": 33860, + "\u0120sergeant": 33861, + "\u0120dissolve": 33862, + "NH": 33863, + "\u0120standings": 33864, + "rique": 33865, + "IQ": 33866, + "\u0120beneficiary": 33867, + "\u0120aquarium": 33868, + "YouTube": 33869, + "\u0120PowerShell": 33870, + "\u0120brightest": 33871, + "\u0120Warrant": 33872, + "Sold": 33873, + "Writing": 33874, + "\u0120beginnings": 33875, + "\u0120Reserved": 33876, + "\u0120Latinos": 33877, + "heading": 33878, + "\u0120440": 33879, + "\u0120rooftop": 33880, + "ATING": 33881, + "\u0120390": 33882, + "VPN": 33883, + "Gs": 33884, + "kernel": 33885, + "turned": 33886, + "\u0120preferable": 33887, + "\u0120turnovers": 33888, + "\u0120Hels": 33889, + "Sa": 33890, + "\u0120Shinji": 33891, + "veh": 33892, + "\u0120MODULE": 33893, + "Viol": 33894, + "\u0120exiting": 33895, + "\u0120jab": 33896, + "\u0120Vanilla": 33897, + "\u0120acron": 33898, + "\u0120Gap": 33899, + "bern": 33900, + "Ak": 33901, + "\u0120McGu": 33902, + "\u0120endlessly": 33903, + "\u0120Farage": 33904, + "\u0120Noel": 33905, + "Va": 33906, + "MK": 33907, + "\u0120brute": 33908, + "\u0120Kru": 33909, + "\u0120ESV": 33910, + "\u0120Olivia": 33911, + "\u00e2\u0122\u0142": 33912, + "\u0120Kaf": 33913, + "\u0120trusting": 33914, + "\u0120hots": 33915, + "324": 33916, + "\u0120malaria": 33917, + "\u0120json": 33918, + "\u0120pounding": 33919, + "ortment": 33920, + "Country": 33921, + "\u0120postponed": 33922, + "\u0120unequiv": 33923, + "?),": 33924, + "\u0120Rooney": 33925, + "udding": 33926, + "\u0120Leap": 33927, + "urrence": 33928, + "shapeshifter": 33929, + "\u0120HAS": 33930, + "osate": 33931, + "\u0120cavern": 33932, + "\u0120conservatism": 33933, + "\u0120BAD": 33934, + "\u0120mileage": 33935, + "\u0120arresting": 33936, + "Vaults": 33937, + "\u0120mixer": 33938, + "Democratic": 33939, + "\u0120Benson": 33940, + "\u0120authored": 33941, + "8000": 33942, + "\u0120proactive": 33943, + "\u0120Spiritual": 33944, + "tre": 33945, + "\u0120incarcerated": 33946, + "\u0120Sort": 33947, + "\u0120peaked": 33948, + "\u0120wielding": 33949, + "reciation": 33950, + "\u00d7\u013b\u00d7": 33951, + "Patch": 33952, + "\u0120Emmy": 33953, + "\u0120exqu": 33954, + "tto": 33955, + "\u0120Ratio": 33956, + "\u0120Picks": 33957, + "\u0120Gry": 33958, + "phant": 33959, + "\u0120fret": 33960, + "\u0120ethn": 33961, + "\u0120archived": 33962, + "%-": 33963, + "cases": 33964, + "\u0120Blaze": 33965, + "\u0120imb": 33966, + "cv": 33967, + "yss": 33968, + "imony": 33969, + "\u0120countdown": 33970, + "\u0120awakening": 33971, + "\u0120Tunisia": 33972, + "\u0120Refer": 33973, + "\u0120MJ": 33974, + "\u0120unnatural": 33975, + "\u0120Carnegie": 33976, + "izen": 33977, + "\u0120Nuggets": 33978, + "hess": 33979, + "\u0120evils": 33980, + "647": 33981, + "\u0120introductory": 33982, + "loving": 33983, + "\u0120McMahon": 33984, + "\u0120ambiguity": 33985, + "Label": 33986, + "\u0120Almighty": 33987, + "\u0120coloring": 33988, + "\u0120Claus": 33989, + "setting": 33990, + "NULL": 33991, + "\u0120Favorite": 33992, + "\u0120SIG": 33993, + ">(": 33994, + "\u0120Shiva": 33995, + "\u0120Mayer": 33996, + "\u0120stormed": 33997, + "\u0120Coverage": 33998, + "weapons": 33999, + "igham": 34000, + "\u0120unanswered": 34001, + "\u0120leve": 34002, + "\u0120coy": 34003, + "cas": 34004, + "bags": 34005, + "asured": 34006, + "Seattle": 34007, + "\u0120Santorum": 34008, + "serious": 34009, + "\u0120courageous": 34010, + "\u0120Soup": 34011, + "\u0120confiscated": 34012, + "\u0120///": 34013, + "\u0120unconventional": 34014, + "\u0120moms": 34015, + "\u0120Rohingya": 34016, + "\u0120Orchestra": 34017, + "\u0120Potion": 34018, + "\u0120discredit": 34019, + "\u0120FIL": 34020, + "fixed": 34021, + "\u0120Deer": 34022, + "doi": 34023, + "\u0120Dimension": 34024, + "\u0120bureaucrats": 34025, + "eteen": 34026, + "\u0120actionGroup": 34027, + "ohm": 34028, + "\u0120bumps": 34029, + "\u0120Utility": 34030, + "\u0120submarines": 34031, + "renheit": 34032, + "research": 34033, + "\u0120Shapiro": 34034, + "\u0120sketches": 34035, + "\u0120deceptive": 34036, + "\u0120Vil": 34037, + "esame": 34038, + "\u0120Essentially": 34039, + "\u0120rampage": 34040, + "isky": 34041, + "\u0120muttered": 34042, + "thritis": 34043, + "\u0120236": 34044, + "fet": 34045, + "bars": 34046, + "\u0120pupil": 34047, + "\u0120Thou": 34048, + "oS": 34049, + "song": 34050, + "\u0120fractured": 34051, + "\u0120revert": 34052, + "picture": 34053, + "\u0120criterion": 34054, + "usher": 34055, + "\u0120repercussions": 34056, + "\u0120Vintage": 34057, + "\u0120Superintendent": 34058, + "Officers": 34059, + "\u0120flagged": 34060, + "\u0120blames": 34061, + "\u0120inverse": 34062, + "ographers": 34063, + "\u0120makeshift": 34064, + "\u0120devoid": 34065, + "\u0120fossils": 34066, + "\u0120Aristotle": 34067, + "\u0120Funds": 34068, + "\u0120depleted": 34069, + "\u0120Flu": 34070, + "\u0120Yuan": 34071, + "\u0120woes": 34072, + "\u0120lipid": 34073, + "\u0120situ": 34074, + "requisites": 34075, + "\u0120furnish": 34076, + "\u0120Samar": 34077, + "\u0120shameful": 34078, + "\u0120adversely": 34079, + "\u0120adept": 34080, + "\u0120remorse": 34081, + "\u0120murderous": 34082, + "uckles": 34083, + "\u0120ESL": 34084, + "\u0120314": 34085, + "sent": 34086, + "\u0120redef": 34087, + "\u0120Cache": 34088, + "\u0120Purs": 34089, + "igans": 34090, + "\u0120460": 34091, + "\u0120prescriptions": 34092, + "\u0120fres": 34093, + "Fuck": 34094, + "ocrates": 34095, + "Twenty": 34096, + "\u0120Weird": 34097, + "\u0120Toggle": 34098, + "\u0120Called": 34099, + "itizens": 34100, + "\u0120poultry": 34101, + "\u0120harvesting": 34102, + "\u00e3\u0124\u00a6\u00e3\u0124\u00b9": 34103, + "Bottom": 34104, + "\u0120cautioned": 34105, + "tn": 34106, + "396": 34107, + "\u0120Nikki": 34108, + "\u0120evaluations": 34109, + "\u0120harassing": 34110, + "\u0120bindings": 34111, + "\u0120Monetary": 34112, + "\u0120hitters": 34113, + "\u0120adversary": 34114, + "unts": 34115, + "\u0120setback": 34116, + "\u0120encrypt": 34117, + "\u0120Cait": 34118, + "\u0120lows": 34119, + "enges": 34120, + "\u0120Norn": 34121, + "\u0120bulbs": 34122, + "\u0120bottled": 34123, + "\u0120Voyager": 34124, + "317": 34125, + "\u0120spheres": 34126, + "politics": 34127, + "\u0120subtract": 34128, + "\u0120sensations": 34129, + "\u0120appalling": 34130, + "\u0120316": 34131, + "\u0120environmentally": 34132, + "\u0120STEM": 34133, + "\u0120publishes": 34134, + "560": 34135, + "\u0120diligence": 34136, + "484": 34137, + "\u0120advises": 34138, + "\u0120petrol": 34139, + "\u0120imagining": 34140, + "\u0120patrols": 34141, + "\u0120Integer": 34142, + "\u0120Ashes": 34143, + "actus": 34144, + "\u0120Radiant": 34145, + "\u0120LT": 34146, + "itability": 34147, + "htaking": 34148, + "Setting": 34149, + "\u0120nuanced": 34150, + "\u0120Reef": 34151, + "\u0120Developers": 34152, + "Ni": 34153, + "pieces": 34154, + "990": 34155, + "License": 34156, + "\u0120lowers": 34157, + "\u0120Ottoman": 34158, + "327": 34159, + "ooo": 34160, + "\u0120quitting": 34161, + "markets": 34162, + "Behind": 34163, + "\u0120basin": 34164, + "\u0120docs": 34165, + "anie": 34166, + "flash": 34167, + "ctl": 34168, + "\u0120civilized": 34169, + "\u0120Fukushima": 34170, + "\"],\"": 34171, + "\u0120KS": 34172, + "\u0120Honestly": 34173, + "arat": 34174, + "\u0120constructs": 34175, + "\u0120Lans": 34176, + "\u0120Dire": 34177, + "\u0120LIKE": 34178, + "\u0120Trouble": 34179, + "\u0120withholding": 34180, + "\u0120Oblivion": 34181, + "\u0120sanity": 34182, + "anya": 34183, + "Const": 34184, + "\u0120grocer": 34185, + "\u0120Celsius": 34186, + "\u0120recounted": 34187, + "\u0120Wife": 34188, + "Border": 34189, + "atered": 34190, + "happy": 34191, + "\u0120spoiler": 34192, + "\u0120logically": 34193, + "Hall": 34194, + "\u0120succeeding": 34195, + "\u0120polymorph": 34196, + "\u0120axes": 34197, + "\u0120Shotgun": 34198, + "\u0120Slim": 34199, + "\u0120Principles": 34200, + "\u0120Leth": 34201, + "arta": 34202, + "\u0120scor": 34203, + "Screenshot": 34204, + "\u0120relaxation": 34205, + "#$#$": 34206, + "\u0120deterrent": 34207, + "iddy": 34208, + "\u0120powerless": 34209, + "\u0120lesbians": 34210, + "\u0120chords": 34211, + "\u0120Edited": 34212, + "selected": 34213, + "\u0120separatists": 34214, + "0002": 34215, + "\u0120airspace": 34216, + "\u0120turnaround": 34217, + "\u0120cunning": 34218, + "PATH": 34219, + "Poly": 34220, + "\u0120bombed": 34221, + "\u0120tion": 34222, + "xs": 34223, + "\u0120withhold": 34224, + "\u0120waged": 34225, + "\u0120Liberties": 34226, + "Flag": 34227, + "\u0120comforting": 34228, + "454": 34229, + "\u0120Iris": 34230, + "arers": 34231, + "\u0120rag": 34232, + "\u0120relocated": 34233, + "\u0120Guarant": 34234, + "\u0120strategically": 34235, + "\u0120gamma": 34236, + "uberty": 34237, + "\u0120Lockheed": 34238, + "gres": 34239, + "\u0120grilled": 34240, + "\u0120Lowe": 34241, + "stats": 34242, + "\u0120Rocks": 34243, + "\u0120sensing": 34244, + "\u0120renting": 34245, + "\u0120Geological": 34246, + "\u00d8\u00a7\u00d8": 34247, + "otrop": 34248, + "\u0120sew": 34249, + "\u0120improperly": 34250, + "486": 34251, + "\u0120\u00e2\u0138\u0142": 34252, + "\u0120starving": 34253, + "\u0120Bj": 34254, + "Discussion": 34255, + "328": 34256, + "\u0120Combo": 34257, + "\u0120Fixes": 34258, + "NAT": 34259, + "\u0120striving": 34260, + "thora": 34261, + "\u0120harvested": 34262, + "\u0120Ping": 34263, + "\u0120playful": 34264, + "\u0120avenues": 34265, + "\u0120occupational": 34266, + "\u0120wakes": 34267, + "\u0120Courier": 34268, + "\u0120drummer": 34269, + "\u0120Browser": 34270, + "\u0120Houth": 34271, + "itu": 34272, + "\u0120apparel": 34273, + "paste": 34274, + "\u0120hunted": 34275, + "\u0120Secondly": 34276, + "lain": 34277, + "XY": 34278, + "\u0120PIN": 34279, + "icons": 34280, + "\u0120cocktails": 34281, + "\u0120sizable": 34282, + "\u0120hurdles": 34283, + "estinal": 34284, + "\u0120Recreation": 34285, + "\u0120eco": 34286, + "648": 34287, + "\u0120Died": 34288, + "mint": 34289, + "\u0120fingerprints": 34290, + "\u0120dispose": 34291, + "\u0120Bosnia": 34292, + "tsy": 34293, + "2200": 34294, + "\u0120inspected": 34295, + "\u0120Fou": 34296, + "\u0120fuss": 34297, + "\u0120ambush": 34298, + "\u0120Rak": 34299, + "\u0120manifested": 34300, + "Prosecut": 34301, + "\u0120suffice": 34302, + "rences": 34303, + "\u0120compensated": 34304, + "\u0120Cyrus": 34305, + "\u0120genus": 34306, + "\u0120Wolverine": 34307, + "\u0120Trends": 34308, + "\u0120hikes": 34309, + "\u0120Seen": 34310, + "\u0120enrol": 34311, + "Cold": 34312, + "\u0120politely": 34313, + "\u0120Slav": 34314, + "\u0120Rupert": 34315, + "\u0120eyewitness": 34316, + "\u0120Alto": 34317, + "\u0120uncomp": 34318, + "\u0120posterior": 34319, + "Must": 34320, + "\u0120Herz": 34321, + "\u0120progressively": 34322, + "\u0120234": 34323, + "\u0120indifference": 34324, + "\u0120Cunningham": 34325, + "\u0120academia": 34326, + "\u0120sewer": 34327, + "\u0120astounding": 34328, + "\u0120AES": 34329, + "rather": 34330, + "\u0120eldest": 34331, + "\u0120climbs": 34332, + "\u0120Adds": 34333, + "\u0120outcry": 34334, + "\u0120contag": 34335, + "\u0120Houses": 34336, + "\u0120pept": 34337, + "\u0120Melania": 34338, + "interested": 34339, + "\u0120UCH": 34340, + "\u0120Roots": 34341, + "\u0120Hubbard": 34342, + "\u0120TBD": 34343, + "\u0120Romanian": 34344, + "filename": 34345, + "Stone": 34346, + "\u0120Impl": 34347, + "\u0120chromosome": 34348, + "Cle": 34349, + "dx": 34350, + "\u0120scrambled": 34351, + "\u0120Pt": 34352, + "\u0120242": 34353, + "OPLE": 34354, + "\u0120tremendously": 34355, + "Street": 34356, + "\u0120craving": 34357, + "\u0120bundled": 34358, + "\u0120RG": 34359, + "pipe": 34360, + "\u0120injuring": 34361, + "\u0120arcane": 34362, + "Particip": 34363, + "\u0120Heroic": 34364, + "sty": 34365, + "\u0120topping": 34366, + "\u0120Tempest": 34367, + "rentices": 34368, + "bh": 34369, + "\u0120paranoia": 34370, + "\u0120Unicode": 34371, + "\u0120egregious": 34372, + "\u0120\\'": 34373, + "\u0120Oswald": 34374, + "\u0120gravel": 34375, + "\u0120Simpsons": 34376, + "\u0120bland": 34377, + "\u0120Guantanamo": 34378, + "Writer": 34379, + "liners": 34380, + "\u0120Dice": 34381, + "JC": 34382, + "\u0120parity": 34383, + "\u0120sided": 34384, + "\u0120237": 34385, + "\u0120Pyrrha": 34386, + "atters": 34387, + "dk": 34388, + "Fine": 34389, + "compan": 34390, + "\u0120formulated": 34391, + "\u0120Idol": 34392, + "ilers": 34393, + "hemoth": 34394, + "\u0120Fav": 34395, + "\u0120intrusion": 34396, + "\u0120carrots": 34397, + "\u0120Layer": 34398, + "\u0120Hacker": 34399, + "\u0120----------------": 34400, + "\u0120moderation": 34401, + "\u00e9\u0123": 34402, + "ococ": 34403, + "\u0120characterize": 34404, + "\u0120Teresa": 34405, + "\u0120socioeconomic": 34406, + "\u0120perk": 34407, + "\u0120Participation": 34408, + "training": 34409, + "\u0120Paulo": 34410, + "phys": 34411, + "\u0120trustworthy": 34412, + "\u0120embodied": 34413, + "\u0120Merch": 34414, + "currency": 34415, + "\u0120Priority": 34416, + "\u0120teasing": 34417, + "\u0120absorbing": 34418, + "\u0120unfinished": 34419, + "\u0120Comparison": 34420, + "\u0120disple": 34421, + "writers": 34422, + "\u0120professions": 34423, + "\u0120Penguin": 34424, + "\u0120angrily": 34425, + "\u0120LINK": 34426, + "688": 34427, + "\u0120Correspond": 34428, + "\u0120prevailed": 34429, + "\u0120cartel": 34430, + "lp": 34431, + "asms": 34432, + "\u0120Redemption": 34433, + "\u0120Islamists": 34434, + "effects": 34435, + "dose": 34436, + "\u0120Latter": 34437, + "\u0120Halifax": 34438, + "\u0120vas": 34439, + "\u0120Topics": 34440, + "\u0120Named": 34441, + "advertising": 34442, + "zza": 34443, + "ICES": 34444, + "\u0120retarded": 34445, + "achable": 34446, + "\u0120Puppet": 34447, + "\u0120ItemLevel": 34448, + "\u0120retract": 34449, + "\u0120identifiable": 34450, + "Aaron": 34451, + "\u0120Buster": 34452, + "sol": 34453, + "helle": 34454, + "assemb": 34455, + "Hope": 34456, + "ranged": 34457, + "Ba": 34458, + "\u0120Purch": 34459, + "\u00e9\u0122": 34460, + "\u0120Siri": 34461, + "\u0120arrivals": 34462, + "\u01201912": 34463, + "\u0120shortened": 34464, + "\u0120312": 34465, + "\u0120discrepancy": 34466, + "\u0120Temperature": 34467, + "\u0120Walton": 34468, + "\u0120kinderg": 34469, + "polit": 34470, + "\u0120remix": 34471, + "\u0120connectors": 34472, + "\u00e3\u0125\u013a\u00e3\u0125\u00a9": 34473, + "\u0120Kazakhstan": 34474, + "dominated": 34475, + "\u0120sugars": 34476, + "imble": 34477, + "\u0120Panic": 34478, + "\u0120Demand": 34479, + "\u0120Colony": 34480, + "onen": 34481, + "\u0120MER": 34482, + "775": 34483, + "uria": 34484, + "azaar": 34485, + "\u0120Degree": 34486, + "Pri": 34487, + "\u0120sunshine": 34488, + "\u0120251": 34489, + "\u0120psychedelic": 34490, + "\u0120digitally": 34491, + "\u0120Braun": 34492, + "\u0120shimmer": 34493, + "\u0120shave": 34494, + "\u0120Telesc": 34495, + "\u0120Astral": 34496, + "\u0120Venezuelan": 34497, + "\u0120OG": 34498, + "\u0120crawling": 34499, + "Integ": 34500, + "\u0120Feather": 34501, + "\u0120unfolding": 34502, + "\u0120appropriation": 34503, + "\u0120\u00e8\u00a3\u0131\u00e8": 34504, + "\u0120Mobility": 34505, + "\u0120Ney": 34506, + "-.": 34507, + "bilt": 34508, + "LIN": 34509, + "\u0120Tube": 34510, + "\u0120Conversely": 34511, + "\u0120keyboards": 34512, + "\u0120Cao": 34513, + "\u0120overth": 34514, + "\u0120laure": 34515, + ">>\\": 34516, + "\u0120Viper": 34517, + "acha": 34518, + "Offset": 34519, + "\u0120Raleigh": 34520, + "\u0120Jae": 34521, + "Jordan": 34522, + "jp": 34523, + "\u0120totalitarian": 34524, + "Connector": 34525, + "\u0120observes": 34526, + "\u0120Spartan": 34527, + "\u0120Immediately": 34528, + "\u0120Scal": 34529, + "Cool": 34530, + "\u0120taps": 34531, + "\u0120roar": 34532, + "Past": 34533, + "\u0120chars": 34534, + "\u0120Bender": 34535, + "\u0120Sheldon": 34536, + "\u0120painter": 34537, + "\u0120beacon": 34538, + "\u0120Creatures": 34539, + "\u0120downturn": 34540, + "\u0120hinder": 34541, + "\u0120Andromeda": 34542, + "\u00c3\u013d": 34543, + "ccoli": 34544, + "\u0120Fitness": 34545, + "etrical": 34546, + "\u0120utilizes": 34547, + "\u0120senate": 34548, + "\u0120ensemble": 34549, + "\u0120cheers": 34550, + "TW": 34551, + "\u0120affluent": 34552, + "kil": 34553, + "rylic": 34554, + "ordering": 34555, + "Computer": 34556, + "\u0120gruesome": 34557, + "ostics": 34558, + "\u0120Ubisoft": 34559, + "\u0120Kelley": 34560, + "\u0120wrench": 34561, + "\u0120bourgeoisie": 34562, + "IBLE": 34563, + "\u0120Preston": 34564, + "worn": 34565, + "arist": 34566, + "reating": 34567, + "\u0120stained": 34568, + "arine": 34569, + "\u0120slime": 34570, + "ENN": 34571, + "\u0120chests": 34572, + "\u0120groundwater": 34573, + "annot": 34574, + "\u0120Tray": 34575, + "\u0120Locke": 34576, + "\u0120CTR": 34577, + "\u0120dudes": 34578, + "\u0120External": 34579, + "\u0120Decoder": 34580, + "\u0120paramed": 34581, + "\u0120Medline": 34582, + "809": 34583, + "\u0120Dinner": 34584, + "rupal": 34585, + "gz": 34586, + "\u0120Gum": 34587, + "\u0120Demo": 34588, + "jee": 34589, + "\u0120dh": 34590, + "berman": 34591, + "archs": 34592, + "\u0120enqu": 34593, + "\u0120Epstein": 34594, + "\u0120devastation": 34595, + "\u0120friendships": 34596, + "\u0120Ard": 34597, + "\u0120231": 34598, + "\u0120Rubin": 34599, + "\u0120Distance": 34600, + "\u0120spurred": 34601, + "\u0120dossier": 34602, + "\u0120overlooking": 34603, + "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\": 34604, + "Forest": 34605, + "\u0120Comes": 34606, + "\\\",": 34607, + "\u0120Iranians": 34608, + "\u0120fixtures": 34609, + "Laughs": 34610, + "\u0120curry": 34611, + "\u0120Kingston": 34612, + "\u0120squash": 34613, + "\u0120catalogue": 34614, + "\u0120abnormalities": 34615, + "\u0120digestive": 34616, + ".........": 34617, + "\u0120subordinate": 34618, + "ogly": 34619, + "\u0120249": 34620, + "Middle": 34621, + "\u0120massac": 34622, + "\u0120burgers": 34623, + "\u0120downstairs": 34624, + "\u01201931": 34625, + "394": 34626, + "\u0120VG": 34627, + "\u0120lasers": 34628, + "\u0120Sikh": 34629, + "\u0120Alexa": 34630, + "derived": 34631, + "\u0120cyclist": 34632, + "\u00e3\u0123\u00ae\u00e9\u0143\u0136": 34633, + "oneliness": 34634, + "!!!!!!!!": 34635, + "\u0120buffs": 34636, + "legate": 34637, + "\u0120raping": 34638, + "\u0120recommending": 34639, + "rored": 34640, + "\u0120multicultural": 34641, + "unique": 34642, + "\u0120businessmen": 34643, + "\u0120uneasy": 34644, + "\u0120MAP": 34645, + "\u0120dispersed": 34646, + "cipline": 34647, + "Jess": 34648, + "\u0120Kerala": 34649, + "\u00e5\u00a7": 34650, + "\u0120abstraction": 34651, + "Surv": 34652, + "Uh": 34653, + "\u0120printers": 34654, + "ija": 34655, + "owder": 34656, + "\u0120analogous": 34657, + "\u0120ASP": 34658, + "afer": 34659, + "\u0120unfolded": 34660, + "\u0120leveling": 34661, + "\u0120breached": 34662, + "\u0120Hearing": 34663, + "\u0120nat": 34664, + "\u0120translating": 34665, + "critical": 34666, + "\u0120antagonist": 34667, + "\u0120Yesterday": 34668, + "\u0120fuzzy": 34669, + "wash": 34670, + "mere": 34671, + "\u0120bewild": 34672, + "\u0120Mae": 34673, + "Virgin": 34674, + "phrase": 34675, + "\u0120signaled": 34676, + "\u0120HIGH": 34677, + "\u0120protester": 34678, + "\u0120garner": 34679, + "unknown": 34680, + "\u0120kay": 34681, + "\u0120abducted": 34682, + "\u0120stalking": 34683, + "amn": 34684, + "\u0120deserving": 34685, + "\u0120Riv": 34686, + "\u0120Jorge": 34687, + "\u0120scratching": 34688, + "\u0120Saving": 34689, + "iping": 34690, + "\u0120tease": 34691, + "\u0120missionary": 34692, + "\u0120Morrow": 34693, + "TIME": 34694, + "Present": 34695, + "\u0120chemotherapy": 34696, + "terness": 34697, + "\u0120Homes": 34698, + "\u0120Purdue": 34699, + "\u0120staunch": 34700, + "\u0120Whitney": 34701, + "\u0120THERE": 34702, + "\u00ce\u00bc": 34703, + "iatus": 34704, + "\u0120Ernest": 34705, + "\u0120Deploy": 34706, + "\u0120coveted": 34707, + "FML": 34708, + "\u0120Dialogue": 34709, + "\u0120exited": 34710, + "fruit": 34711, + "\u0120nerd": 34712, + "\":\"\",\"": 34713, + "\u0120vivo": 34714, + "ruly": 34715, + "460": 34716, + "\u0120Amen": 34717, + "rehensible": 34718, + "\u0120\u00e2\u013a": 34719, + "DIR": 34720, + "\u0120adherence": 34721, + "\u0120chew": 34722, + "\u0120Coke": 34723, + "\u0120Sergei": 34724, + "digital": 34725, + "\u0120Neck": 34726, + "gently": 34727, + "enthal": 34728, + "/)": 34729, + "\u0120weary": 34730, + "\u0120guise": 34731, + "\u0120Concord": 34732, + "\u0120Onion": 34733, + "atcher": 34734, + "\u0120binge": 34735, + "\u0120Directive": 34736, + "\u0120manned": 34737, + "ansk": 34738, + "\u0120illusions": 34739, + "\u0120billionaires": 34740, + "383": 34741, + "olyn": 34742, + "odynamic": 34743, + "\u0120Wheat": 34744, + "\u0120Alic": 34745, + "\u0120coloured": 34746, + "\u0120NAFTA": 34747, + "abo": 34748, + "\u0120macros": 34749, + "independent": 34750, + "sweet": 34751, + "\u0120spac": 34752, + "\u0120Kabul": 34753, + "\u0120\u00c4": 34754, + "eme": 34755, + "\u0120dictated": 34756, + "\u0120shouts": 34757, + "={": 34758, + "\u0120ripping": 34759, + "\u0120Shay": 34760, + "\u0120Cricket": 34761, + "directed": 34762, + "\u0120analysed": 34763, + "\u0120WARRANT": 34764, + "agons": 34765, + "\u0120Blazers": 34766, + "\u0120cheered": 34767, + "\u0120arithmetic": 34768, + "\u0120Tanz": 34769, + "373": 34770, + "\u0120Flags": 34771, + "\u0120295": 34772, + "\u0120witches": 34773, + "\u0120Included": 34774, + "\u0120Gained": 34775, + "\u0120Blades": 34776, + "Gam": 34777, + "\u0120Samantha": 34778, + "\u0120Atlantis": 34779, + "\u0120Pratt": 34780, + "\u0120spoiled": 34781, + "\u0120IB": 34782, + "\u0120Ramirez": 34783, + "Probably": 34784, + "rero": 34785, + "\u0120Ng": 34786, + "\u0120Warlock": 34787, + "tp": 34788, + "\u0120overhe": 34789, + "\u0120administrations": 34790, + "\u0120tint": 34791, + "\u0120regiment": 34792, + "\u0120pistols": 34793, + "\u0120blankets": 34794, + "\u0120epist": 34795, + "\u0120bowls": 34796, + "\u0120hydraulic": 34797, + "\u0120dean": 34798, + "\u0120jung": 34799, + "\u0120ascend": 34800, + "705": 34801, + "\u0120Santiago": 34802, + "\u00c3\u00ae": 34803, + "\u0120unavoid": 34804, + "\u0120Shaman": 34805, + "reb": 34806, + "\u0120stemming": 34807, + "998": 34808, + "\u0120MG": 34809, + "sticks": 34810, + "esthesia": 34811, + "ERO": 34812, + "\u0120morbid": 34813, + "\u0120Grill": 34814, + "\u0120Poe": 34815, + "anyl": 34816, + "\u0120deleting": 34817, + "\u0120Surveillance": 34818, + "\u0120directives": 34819, + "\u0120iterations": 34820, + "\u0120Rox": 34821, + "\u0120Milky": 34822, + "Father": 34823, + "\u0120patented": 34824, + "447": 34825, + "\u0120precursor": 34826, + "\u0120maiden": 34827, + "\u0120Phen": 34828, + "\u0120Vegan": 34829, + "\u0120Patent": 34830, + "Kelly": 34831, + "Redditor": 34832, + "\u0120nods": 34833, + "\u0120ventilation": 34834, + "\u0120Schwarz": 34835, + "\u0120wizards": 34836, + "\u0120ominous": 34837, + "\u0120Heads": 34838, + "\u0120BG": 34839, + "\u0120lumber": 34840, + "\u0120Spiel": 34841, + "\u0120isEnabled": 34842, + "\u0120ancestral": 34843, + "\u0120Ships": 34844, + "\u0120wrestler": 34845, + "phi": 34846, + "\u0120yuan": 34847, + "\u0120Rebellion": 34848, + "\u0120iceberg": 34849, + "\u0120magically": 34850, + "\u0120diversion": 34851, + "arro": 34852, + "ythm": 34853, + "\u0120Riders": 34854, + "\u0120Robbie": 34855, + "\u0120Kara": 34856, + "\u0120Maintenance": 34857, + "\u0120Herb": 34858, + "\u0120harms": 34859, + "packed": 34860, + "\u0120Feinstein": 34861, + "\u0120marrying": 34862, + "\u0120blending": 34863, + "\u0120Rates": 34864, + "\u01201880": 34865, + "\u0120wrink": 34866, + "\u0120Unch": 34867, + "\u0120Torch": 34868, + "described": 34869, + "\u0120humanoid": 34870, + "ilitating": 34871, + "\u0120Conv": 34872, + "\u0120Feld": 34873, + "IGHTS": 34874, + "\u0120whistleblower": 34875, + "ortmund": 34876, + "etsy": 34877, + "arrett": 34878, + "\u0120Mono": 34879, + "\u0120Ike": 34880, + "\u0120CNBC": 34881, + "\u0120WAY": 34882, + "\u0120MDMA": 34883, + "\u0120Individuals": 34884, + "\u0120supplemental": 34885, + "\u0120powerhouse": 34886, + "\u0120Stru": 34887, + "Focus": 34888, + "aphael": 34889, + "\u0120Colleg": 34890, + "atti": 34891, + "ZA": 34892, + "\u0120perenn": 34893, + "\u0120Signature": 34894, + "\u0120Rodney": 34895, + "\u0120cubes": 34896, + "iddled": 34897, + "\u0120Dante": 34898, + "\u0120INV": 34899, + "ilingual": 34900, + "\u0120Cth": 34901, + "\u0120sofa": 34902, + "\u0120intimidate": 34903, + "\u0120Roe": 34904, + "\u0120Diplom": 34905, + "\u0120Countries": 34906, + "ayson": 34907, + "\u0120extradition": 34908, + "\u0120disabling": 34909, + "\u0120Cardiff": 34910, + "\u0120memorandum": 34911, + "\u0120Trace": 34912, + "\u0120???": 34913, + "sector": 34914, + "\u0120Rouhani": 34915, + "\u0120Yates": 34916, + "\u0120Freeze": 34917, + "\u0120bladder": 34918, + "Motor": 34919, + "\u0120Promise": 34920, + "antasy": 34921, + "\u0120foreseeable": 34922, + "\u0120Cologne": 34923, + "container": 34924, + "\u0120Trees": 34925, + "\u0120Gors": 34926, + "\u0120Sinclair": 34927, + "\u0120barring": 34928, + "keye": 34929, + "\u0120slashed": 34930, + "\u0120Statistical": 34931, + "\u00e9\u0129": 34932, + "\u0120\u00e2\u0138\u00ba": 34933, + "Allows": 34934, + "\u0120humility": 34935, + "\u0120drilled": 34936, + "\u0120Furn": 34937, + "443": 34938, + "\u0120sewage": 34939, + "\u0120homepage": 34940, + "\u0120courtyard": 34941, + "\u0120vile": 34942, + "\u0120subsidiaries": 34943, + "ajo": 34944, + "directory": 34945, + "\u0120ammon": 34946, + "Vers": 34947, + "charges": 34948, + "\u0120}}": 34949, + "\u0120Chains": 34950, + "\u0120246": 34951, + "nob": 34952, + "\u0120percept": 34953, + "\u0120grit": 34954, + "\u0120fishermen": 34955, + "\u0120Iraqis": 34956, + "\u0120DISTR": 34957, + "\u0120FULL": 34958, + "\u0120Evaluation": 34959, + "graph": 34960, + "atial": 34961, + "\u0120cooperating": 34962, + "\u0120melan": 34963, + "\u0120enlightened": 34964, + "\u0120ali": 34965, + "tailed": 34966, + "\u0120salute": 34967, + "\u0120weakest": 34968, + "\u0120Bulldogs": 34969, + "UA": 34970, + "\u0120Alloy": 34971, + "\u0120semen": 34972, + "ocene": 34973, + "\u0120Williamson": 34974, + "spr": 34975, + ",\u00e2\u0122\u0136": 34976, + "\u0120GF": 34977, + "ittens": 34978, + "Beat": 34979, + "\u0120Junk": 34980, + "iphate": 34981, + "\u0120Farmers": 34982, + "\u0120Bitcoins": 34983, + "igers": 34984, + "dh": 34985, + "\u0120Loyal": 34986, + "payer": 34987, + "\u0120entertained": 34988, + "\u0120penned": 34989, + "\u0120coupon": 34990, + "Queue": 34991, + "\u0120weakening": 34992, + "carry": 34993, + "\u0120underestimate": 34994, + "\u0120shootout": 34995, + "\u0120charismatic": 34996, + "\u0120Procedure": 34997, + "\u0120prudent": 34998, + "inances": 34999, + "\u0120riches": 35000, + "\u0120cortical": 35001, + "\u0120strides": 35002, + "\u0120drib": 35003, + "\u0120Oilers": 35004, + "540": 35005, + "\u0120Perform": 35006, + "\u0120Bangkok": 35007, + "\u0120euth": 35008, + "SER": 35009, + "\u0120simplistic": 35010, + "tops": 35011, + "campaign": 35012, + "Quality": 35013, + "\u0120impoverished": 35014, + "\u0120Eisenhower": 35015, + "\u0120augment": 35016, + "\u0120Harden": 35017, + "\u0120intervened": 35018, + "\u0120listens": 35019, + "\u0120Kok": 35020, + "\u0120sage": 35021, + "\u0120rubbish": 35022, + "\u0120Ded": 35023, + "\u0120mull": 35024, + "pelling": 35025, + "\u0120videot": 35026, + "Production": 35027, + "DJ": 35028, + "miah": 35029, + "\u0120adaptations": 35030, + "\u0120medically": 35031, + "\u0120boarded": 35032, + "\u0120arrogance": 35033, + "\u0120scrapped": 35034, + "\u0120oppress": 35035, + "FORMATION": 35036, + "\u0120junction": 35037, + "415": 35038, + "EEEE": 35039, + "Skill": 35040, + "\u0120subdu": 35041, + "\u0120Suggest": 35042, + "\u0120Pett": 35043, + "\u0120lett": 35044, + "\u0120Manip": 35045, + "\u0120Caf": 35046, + "\u0120Cooperation": 35047, + "Ther": 35048, + "\u0120regained": 35049, + "\u00b6\u00e6": 35050, + "reflect": 35051, + "\u0120thugs": 35052, + "\u0120Shelby": 35053, + "\u0120dictates": 35054, + "\u0120Weiner": 35055, + "\u0120Hale": 35056, + "\u0120battleground": 35057, + "schild": 35058, + "\u0120condol": 35059, + "hunt": 35060, + "ositories": 35061, + "\u0120accuses": 35062, + "Filename": 35063, + "\u0120shri": 35064, + "\u0120motivate": 35065, + "\u0120reflections": 35066, + "Null": 35067, + "\u0120Lobby": 35068, + "\u00a5\u00b5": 35069, + "\u0120SATA": 35070, + "\u0120Backup": 35071, + "\u00d1\u0125": 35072, + "nin": 35073, + "\u0120Correction": 35074, + "\u0120juicy": 35075, + "utra": 35076, + "\u0120Pric": 35077, + "\u0120restraining": 35078, + "\u0120Airbnb": 35079, + "\u0120Arrest": 35080, + "\u0120appropriations": 35081, + "\u0120slopes": 35082, + "\u0120manslaughter": 35083, + "\u0120workings": 35084, + "\u0120Huss": 35085, + "\u0120Frey": 35086, + "Leave": 35087, + "\u0120Harmony": 35088, + "\u0120Feder": 35089, + "\u0120430": 35090, + "\u0120trench": 35091, + "\u0120gladly": 35092, + "\u0120bullpen": 35093, + "\u0120Gau": 35094, + "bones": 35095, + "\u0120groove": 35096, + "\u0120pretext": 35097, + "\u00e3\u0127\u012d": 35098, + "\u0120transmitter": 35099, + "\u0120Component": 35100, + "\u0120underage": 35101, + "\u0120Empires": 35102, + "Tile": 35103, + "\u0120oy": 35104, + "\u0120Marvin": 35105, + "\u0120CAS": 35106, + "\u0120bloss": 35107, + "\u0120replicated": 35108, + "\u0120Mariners": 35109, + "Marcus": 35110, + "\u0120Blocks": 35111, + "\u0120liberated": 35112, + "\u0120butterfly": 35113, + "Feel": 35114, + "\u0120fermentation": 35115, + "\u0120youtube": 35116, + "\u0120offend": 35117, + "\u0120Term": 35118, + "resist": 35119, + "\u0120cessation": 35120, + "\u0120insurgency": 35121, + "\u0120bir": 35122, + "\u0120Raise": 35123, + "595": 35124, + "\u0120hypotheses": 35125, + "502": 35126, + "\u0120plaque": 35127, + "ocrat": 35128, + "\u0120jackets": 35129, + "\u0120HuffPost": 35130, + "among": 35131, + "\u0120confer": 35132, + "487": 35133, + "\u0120Lilly": 35134, + "\u0120adapting": 35135, + "\u0120Fay": 35136, + "\u0120shoved": 35137, + "vec": 35138, + "\u0120refine": 35139, + "\u0120gon": 35140, + "\u0120gunmen": 35141, + "zai": 35142, + "\u0120Shuttle": 35143, + "\u0120Izan": 35144, + "\u01201913": 35145, + "\u0120plethora": 35146, + "\u00c2\u00b7\u00c2\u00b7": 35147, + "\u0120510": 35148, + "\u0120puberty": 35149, + "\u0120241": 35150, + "\u0120Wealth": 35151, + "\u0120Alma": 35152, + "\u0120MEM": 35153, + "\u0120Adults": 35154, + "Cas": 35155, + "prison": 35156, + "Race": 35157, + "\u0120waterproof": 35158, + "\u0120athleticism": 35159, + "\u0120capitalize": 35160, + "\u0120Juice": 35161, + "\u0120illuminated": 35162, + "\u0120Pascal": 35163, + "\u0120irritation": 35164, + "\u0120Witnesses": 35165, + "adle": 35166, + "\u0120Astro": 35167, + "\u0120fax": 35168, + "\u0120Elvis": 35169, + "Primary": 35170, + "\u0120Lich": 35171, + "\u0120Elves": 35172, + "\u0120residing": 35173, + "\u0120stumble": 35174, + "319": 35175, + "\u0120PKK": 35176, + "\u0120adversaries": 35177, + "DOS": 35178, + "\u0120Ritual": 35179, + "\u0120smear": 35180, + "\u0120arson": 35181, + "idental": 35182, + "\u0120scant": 35183, + "\u0120monarchy": 35184, + "\u0120halftime": 35185, + "\u0120residue": 35186, + "\u0120indign": 35187, + "\u0120Shaun": 35188, + "\u0120Elm": 35189, + "auri": 35190, + "Aff": 35191, + "WATCH": 35192, + "\u0120Lyon": 35193, + "helps": 35194, + "361": 35195, + "\u0120lobbyist": 35196, + "\u0120diminishing": 35197, + "\u0120outbreaks": 35198, + "\u0120goats": 35199, + "favorite": 35200, + "\u0120Nah": 35201, + "sonian": 35202, + "\u0120Booster": 35203, + "\u0120sandbox": 35204, + "\u0120Fare": 35205, + "\u0120Malta": 35206, + "\u0120attRot": 35207, + "\u0120MOR": 35208, + "lde": 35209, + "\u0120navigating": 35210, + "Touch": 35211, + "\u0120untrue": 35212, + "\u0120Disaster": 35213, + "\u0120ludicrous": 35214, + "Password": 35215, + "\u0120JFK": 35216, + "blogspot": 35217, + "416": 35218, + "\u0120UNDER": 35219, + "ernal": 35220, + "\u0120delaying": 35221, + "TOP": 35222, + "\u0120implants": 35223, + "\u0120AVG": 35224, + "\u0120Huge": 35225, + "attr": 35226, + "\u0120journalistic": 35227, + "\u0120Peyton": 35228, + "\u0120IA": 35229, + "Rap": 35230, + "goal": 35231, + "\u0120Programme": 35232, + "\u0120smashing": 35233, + "wives": 35234, + "println": 35235, + "\u0120Plague": 35236, + "inus": 35237, + "EEP": 35238, + "\u0120cruiser": 35239, + "\u0120Parish": 35240, + "uminium": 35241, + "\u0120occupants": 35242, + "\u0120Jihad": 35243, + "mop": 35244, + "\u0120pint": 35245, + "\u0120hect": 35246, + "\u0120Mecca": 35247, + "director": 35248, + "\u0120Funding": 35249, + "\u0120Mixed": 35250, + "\u0120stag": 35251, + "Tier": 35252, + "\u0120gust": 35253, + "\u0120brightly": 35254, + "orsi": 35255, + "\u0120uphill": 35256, + "RD": 35257, + "\u0120lesions": 35258, + "\u0120Bundy": 35259, + "livious": 35260, + "\u0120biologist": 35261, + "\u0120Faculty": 35262, + "\u0120Authorization": 35263, + "\u0120244": 35264, + "Allow": 35265, + "\u00ef\u00b8": 35266, + "\u0120Giul": 35267, + "\u0120pertinent": 35268, + "otaur": 35269, + "esse": 35270, + "\u0120Roof": 35271, + "\u0120unmanned": 35272, + "351": 35273, + "\u0120Shak": 35274, + "\u0120Orient": 35275, + "\u0120endanger": 35276, + "Dir": 35277, + "\u0120replen": 35278, + "edient": 35279, + "\u0120tailor": 35280, + "\u0120gadgets": 35281, + "\u0120audible": 35282, + "\u00e2\u013a\u0128": 35283, + "Nice": 35284, + "\u0120bombard": 35285, + "\u0120Rape": 35286, + "\u0120defiance": 35287, + "\u0120TWO": 35288, + "\u0120Filipino": 35289, + "\u0120unaffected": 35290, + "ervatives": 35291, + "\u0120soared": 35292, + "\u0120Bolton": 35293, + "\u0120compromising": 35294, + "\u0120Brewers": 35295, + "RAL": 35296, + "\u0120AHL": 35297, + "icycle": 35298, + "\u0120vampires": 35299, + "\u0120dipped": 35300, + "oyer": 35301, + "\u0120XIII": 35302, + "\u0120sideways": 35303, + "\u0120Waste": 35304, + "\u0120Diss": 35305, + "\u0120\u00e2\u0136\u013e\u00e2\u0136\u0122\u00e2\u0136\u0122": 35306, + "$.": 35307, + "\u0120habitats": 35308, + "\u0120Beef": 35309, + "truth": 35310, + "trained": 35311, + "split": 35312, + "Rus": 35313, + "Andy": 35314, + "\u0120Bram": 35315, + "REP": 35316, + "pid": 35317, + "\u00e8\u00a3\u0127": 35318, + "\u0120Mutant": 35319, + "Anim": 35320, + "\u0120Marina": 35321, + "\u0120futile": 35322, + "highest": 35323, + "frequency": 35324, + "\u0120epilepsy": 35325, + "\u0120coping": 35326, + "\u0120concise": 35327, + "\u0120tracing": 35328, + "\u0120SUN": 35329, + "panel": 35330, + "\u0120Sophie": 35331, + "\u0120Crowley": 35332, + "\u0120Adolf": 35333, + "\u0120Shooter": 35334, + "\u0120shaky": 35335, + "\u0120IG": 35336, + "\u0120Lies": 35337, + "\u0120Barber": 35338, + "pkg": 35339, + "\u0120uptake": 35340, + "\u0120predatory": 35341, + "ULTS": 35342, + "/**": 35343, + "\u0120intoxicated": 35344, + "\u0120Westbrook": 35345, + "odder": 35346, + "hement": 35347, + "\u0120baseman": 35348, + "APD": 35349, + "storage": 35350, + "\u0120Fifty": 35351, + "editor": 35352, + "GEN": 35353, + "UTION": 35354, + "irting": 35355, + "\u0120sewing": 35356, + "rift": 35357, + "\u0120agony": 35358, + "\u0120Sands": 35359, + "\u0120254": 35360, + "Cash": 35361, + "\u0120lodge": 35362, + "\u0120punt": 35363, + "Natural": 35364, + "\u0120Ideas": 35365, + "\u0120erroneous": 35366, + "\u0120Sensor": 35367, + "\u0120Hannity": 35368, + "\u01201921": 35369, + "\u0120mould": 35370, + "\u0120Gon": 35371, + "kaya": 35372, + "\u0120anonymously": 35373, + "\u0120KEY": 35374, + "\u0120simulator": 35375, + "Winter": 35376, + "\u0120streamed": 35377, + "507": 35378, + "?\",": 35379, + "\u0120teased": 35380, + "\u0120coefficient": 35381, + "\u0120wartime": 35382, + "\u0120THR": 35383, + "''.": 35384, + "\u0120Banking": 35385, + "mpire": 35386, + "\u0120fandom": 35387, + "\u0120lia": 35388, + "Ga": 35389, + "\u0120downhill": 35390, + "\u0120interpreting": 35391, + "Individual": 35392, + "Norm": 35393, + "\u0120jealousy": 35394, + "bitcoin": 35395, + "\u0120pleasures": 35396, + "\u0120Toys": 35397, + "\u0120Chevrolet": 35398, + "\u0120Advisor": 35399, + "IZE": 35400, + "\u0120receptions": 35401, + "706": 35402, + "Cro": 35403, + "\u0120262": 35404, + "\u0120citrus": 35405, + "iru": 35406, + "Reviewer": 35407, + "jected": 35408, + "UES": 35409, + "anz": 35410, + "1981": 35411, + "\u0120Worker": 35412, + "\u0120complied": 35413, + "orescent": 35414, + "continental": 35415, + "Ton": 35416, + "\u0120Prism": 35417, + "\u0120Sheep": 35418, + "\u0120288": 35419, + "nox": 35420, + "\u0120Vog": 35421, + "Ord": 35422, + "\u0120realms": 35423, + "tek": 35424, + "\u0120irrigation": 35425, + "\u0120bicycles": 35426, + "\u0120electronically": 35427, + "poly": 35428, + "tall": 35429, + "());": 35430, + "\u0120aesthetics": 35431, + "\u0120Integrated": 35432, + "Explore": 35433, + "\u0120dunk": 35434, + "476": 35435, + "pain": 35436, + "\u0120Jacques": 35437, + "\u0120Dmit": 35438, + "Frames": 35439, + "\u0120reunited": 35440, + "\u0120humid": 35441, + "Dro": 35442, + "Political": 35443, + "\u0120youthful": 35444, + "\u0120entails": 35445, + "\u0120mosquito": 35446, + "363": 35447, + "species": 35448, + "\u0120coordinating": 35449, + "\u0120Mayhem": 35450, + "\u0120Magnus": 35451, + "Mount": 35452, + "Improved": 35453, + "\u0120STATE": 35454, + "ATTLE": 35455, + "\u0120flowed": 35456, + "\u0120tackled": 35457, + "\u0120fashioned": 35458, + "\u0120reorgan": 35459, + "ivari": 35460, + "finger": 35461, + "\u0120reluctantly": 35462, + "etting": 35463, + "\u0120Vand": 35464, + "young": 35465, + "\u0120Garland": 35466, + "\u0120presumption": 35467, + "\u0120amenities": 35468, + "\u0120Pleasant": 35469, + "onential": 35470, + "\u0120Oxy": 35471, + "\u0120morals": 35472, + "\u0120Yah": 35473, + "Ready": 35474, + "Simon": 35475, + "Enh": 35476, + "Demon": 35477, + "\u0120clich": 35478, + "Monitor": 35479, + "\u0120DU": 35480, + "\u0120welcomes": 35481, + "\u0120standout": 35482, + "\u0120dreadful": 35483, + "\u0120bananas": 35484, + "\u0120balloons": 35485, + "hooting": 35486, + "basic": 35487, + "\u0120suffix": 35488, + "\u0120duly": 35489, + "cano": 35490, + "Chain": 35491, + "atos": 35492, + "\u0120geopolitical": 35493, + "\u0120(&": 35494, + "\u0120Gemini": 35495, + "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 35496, + "\u0120acquitted": 35497, + "Luck": 35498, + "protect": 35499, + "1024": 35500, + "\u0120scarcity": 35501, + "\u0120mindfulness": 35502, + "ecided": 35503, + "DN": 35504, + "prime": 35505, + "\u0120Presidents": 35506, + "\u0120VIDEO": 35507, + "\u0120(\u00e2\u012a\u0134": 35508, + "addock": 35509, + "NOR": 35510, + "\u0120Pru": 35511, + "pun": 35512, + "\u0120LOL": 35513, + "))))": 35514, + "\u0120Liqu": 35515, + "\u0120SAS": 35516, + "\u0120styling": 35517, + "\u0120punishments": 35518, + "\u0120numb": 35519, + "\u0120ascertain": 35520, + "\u0120Rockies": 35521, + "flu": 35522, + "Thumbnail": 35523, + "\u0120perpetrated": 35524, + "\u0120Semi": 35525, + "\u0120disarm": 35526, + "\u0120Older": 35527, + "\u0120Exception": 35528, + "\u0120exponentially": 35529, + "\u0120Communities": 35530, + "\u0120abolish": 35531, + "\u0120Partner": 35532, + "ptoms": 35533, + "\u0120777": 35534, + "\u0120Foley": 35535, + "\u0120Cases": 35536, + "\u0120grease": 35537, + "\u0120Rebirth": 35538, + "Ground": 35539, + "\u0120;)": 35540, + "\u0120Doctrine": 35541, + "ikini": 35542, + "Ye": 35543, + "\u0120Blossom": 35544, + "\u0120persists": 35545, + "bill": 35546, + "\u0120infusion": 35547, + "\u0120buddies": 35548, + "911": 35549, + "\u0120Patient": 35550, + "\u0120demos": 35551, + "\u0120acquaintance": 35552, + "\u0120Paw": 35553, + "atari": 35554, + "\u0120xml": 35555, + "\u0120fascination": 35556, + "\u0120Serve": 35557, + "\u00cf\u0124": 35558, + "branded": 35559, + "\u0120az": 35560, + "Returns": 35561, + "\u0120overshadow": 35562, + "\u0120roam": 35563, + "\u0120speedy": 35564, + "numbered": 35565, + "helial": 35566, + "\u0120disciple": 35567, + "\u0120assurances": 35568, + "given": 35569, + "pecting": 35570, + "\u0120Natalie": 35571, + "\u00e7\u0136\u00b0": 35572, + "\u0120mosquitoes": 35573, + "rotein": 35574, + "\u0120numeric": 35575, + "\u0120independents": 35576, + "\u0120transitional": 35577, + "\u0120reactionary": 35578, + "\u0120Mechdragon": 35579, + "doctor": 35580, + "\u0120shortest": 35581, + "\u0120sequential": 35582, + "\u0120Bac": 35583, + "\u0120Accounts": 35584, + "\u00e3\u0123\u012e": 35585, + "achy": 35586, + "ractive": 35587, + "\u0120Regiment": 35588, + "\u0120breathtaking": 35589, + "fficiency": 35590, + "\u0120Bates": 35591, + "\u0120311": 35592, + "\u0120wardrobe": 35593, + "fts": 35594, + "\u0120Berk": 35595, + "Simply": 35596, + "\u0120Riverside": 35597, + "ivering": 35598, + "idential": 35599, + "lucent": 35600, + "\u0120enriched": 35601, + "\u0120Conver": 35602, + "\u0120Giving": 35603, + "\u00e3\u0125\u013b": 35604, + "\u0120legalize": 35605, + "\u0120FTC": 35606, + "\u0120freaking": 35607, + "Mix": 35608, + "\u0120terrestrial": 35609, + "esian": 35610, + "cients": 35611, + "Wing": 35612, + "LOAD": 35613, + "\u0120ledge": 35614, + "\u0120Violent": 35615, + "\u0120Metall": 35616, + "\u0120308": 35617, + "\u0120southeastern": 35618, + "hetto": 35619, + "Meat": 35620, + "\u0120slowdown": 35621, + "\u0120retreated": 35622, + "Jeremy": 35623, + "endas": 35624, + "*****": 35625, + "eric": 35626, + "\u0120reins": 35627, + "oppable": 35628, + "\u0120Humanity": 35629, + "earances": 35630, + "rigan": 35631, + "Camera": 35632, + "\u0120waivers": 35633, + "soc": 35634, + "\u0120alteration": 35635, + "transform": 35636, + "\u0120Cemetery": 35637, + "506": 35638, + "\u0120indefinite": 35639, + "\u0120stimulating": 35640, + "yg": 35641, + "603": 35642, + "\u0120Sop": 35643, + "\u0120descriptive": 35644, + "Phase": 35645, + "\u0120Edmund": 35646, + "\u0120pneumonia": 35647, + "ventus": 35648, + "Amb": 35649, + "\u0120laboratories": 35650, + "\u0120Exclusive": 35651, + "ugar": 35652, + "Were": 35653, + "\u0120malfunction": 35654, + "\u0120homosexuals": 35655, + "\u0120-------": 35656, + "uni": 35657, + "\u0120turbines": 35658, + "\u0120Equity": 35659, + "Du": 35660, + "\u0120minded": 35661, + "\u0120RH": 35662, + "\u0120Blackhawks": 35663, + "\u0120feats": 35664, + "\u01201700": 35665, + "repl": 35666, + "362": 35667, + "laden": 35668, + "\u0120indispensable": 35669, + "lyss": 35670, + "tti": 35671, + "\u0120reel": 35672, + "\u0120diverted": 35673, + "\u0120likeness": 35674, + "\u0120subscriptions": 35675, + "\u0120fingert": 35676, + "\u0120filthy": 35677, + "destruct": 35678, + "draft": 35679, + "\u0120Bernardino": 35680, + "launch": 35681, + "\u0120perplex": 35682, + "\u0120SUM": 35683, + "carb": 35684, + "\u0120sweater": 35685, + "\u0120Venture": 35686, + "\u0120Jag": 35687, + "\u0120Celeb": 35688, + "\u0120Voters": 35689, + "\u0120steadfast": 35690, + "\u0120athletics": 35691, + "\u0120Hanson": 35692, + "\u0120Drac": 35693, + "Tracker": 35694, + "\u0120commend": 35695, + "\u0120Presidency": 35696, + "\u0120DID": 35697, + "informed": 35698, + "\u0120webpage": 35699, + "Pretty": 35700, + "\u0120forcefully": 35701, + "\u00e3\u0125\u0125\u00e3\u0124\u00af": 35702, + "\u0120relocation": 35703, + "\u0120satire": 35704, + "\u00e2\u012b": 35705, + "\u0120Sunderland": 35706, + "\u00e6\u0126": 35707, + "Voice": 35708, + "????????": 35709, + "\u0120informant": 35710, + "\u0120bowel": 35711, + "\u0120Uniform": 35712, + "\u0120...\"": 35713, + "\u0120purge": 35714, + "\u0120picnic": 35715, + "\u0120Umb": 35716, + "\u0120UPDATE": 35717, + "\u0120Sapphire": 35718, + "\u0120Stall": 35719, + "learn": 35720, + "\u0120objectively": 35721, + "\u0120obliter": 35722, + "\u0120loophole": 35723, + "\u0120journeys": 35724, + "\u0120omission": 35725, + "Pros": 35726, + "\u0120Sidney": 35727, + "ploma": 35728, + "\u0120sprayed": 35729, + "\u0120guru": 35730, + "\u0120traitor": 35731, + "\u0120timet": 35732, + "\u0120snapping": 35733, + "\u0120Sevent": 35734, + "urnal": 35735, + "\u0120Ukip": 35736, + "\u0120bowed": 35737, + "poral": 35738, + "liberal": 35739, + "Ros": 35740, + "Questions": 35741, + "iOS": 35742, + "\u0120summarize": 35743, + "STAT": 35744, + "\u01201850": 35745, + "apest": 35746, + "\u0120lender": 35747, + "\u0120Variable": 35748, + "bringing": 35749, + "\u0120LORD": 35750, + ",)": 35751, + "\u0120collapses": 35752, + "xiety": 35753, + "\u0120Ned": 35754, + "YD": 35755, + "\u0120Scha": 35756, + "\u0120antibody": 35757, + "\u0120disband": 35758, + "yre": 35759, + "illusion": 35760, + "\u0120rover": 35761, + "shed": 35762, + "\u0120Hirosh": 35763, + "cci": 35764, + "\u0120calam": 35765, + "\u0120Morton": 35766, + "Pinterest": 35767, + "\u01201928": 35768, + "\u0120Euras": 35769, + "ordes": 35770, + "\u0120fences": 35771, + "\u0120Inventory": 35772, + "\u0120Valencia": 35773, + "\u0120Ud": 35774, + "\u0120Tiff": 35775, + "\u0120sque": 35776, + "\u0120quotation": 35777, + "\u0120troublesome": 35778, + "erker": 35779, + "QUEST": 35780, + "\u0120Kingdoms": 35781, + "south": 35782, + "\u0120levy": 35783, + "Prince": 35784, + "\u0120Sting": 35785, + "\u0120nicknamed": 35786, + "\u0120appe": 35787, + "\u0120photographic": 35788, + "\u0120corpus": 35789, + "reference": 35790, + "\u0120Trog": 35791, + "Unt": 35792, + ")=(": 35793, + "\u0120Latvia": 35794, + "\u0120activating": 35795, + "\u0120licensee": 35796, + "\u0120disparities": 35797, + "\u0120Newsletter": 35798, + "\u00e3\u0125\u0125\u00e3\u0125\u012a": 35799, + "\u0120freeing": 35800, + "\u0120Jeep": 35801, + "\u0120Perception": 35802, + "insk": 35803, + "\u0120silicone": 35804, + "\u0120Hayden": 35805, + "Lean": 35806, + "\u0120Suzuki": 35807, + "ibrarian": 35808, + "668": 35809, + "\u0120spor": 35810, + "\u0120correlations": 35811, + "aghetti": 35812, + "\u0120tuber": 35813, + "\u0120IPCC": 35814, + "ilus": 35815, + "\u0120Vu": 35816, + "\u0120wealthiest": 35817, + "\u0120Carbuncle": 35818, + "anza": 35819, + "\u0120fooled": 35820, + "\u0120Zur": 35821, + "\u0120daddy": 35822, + "rano": 35823, + "ilian": 35824, + "\u0120knockout": 35825, + "fman": 35826, + "required": 35827, + "\u0120Wikileaks": 35828, + "\u0120Duffy": 35829, + "ONT": 35830, + "\u0120insol": 35831, + "\u0120Objects": 35832, + "\u0120bou": 35833, + "\u0120Nordic": 35834, + "\u0120Insert": 35835, + "scan": 35836, + "\u0120dancers": 35837, + "\u0120idiots": 35838, + "majority": 35839, + "\u0120Neville": 35840, + "\u0120FreeBSD": 35841, + "\u0120tart": 35842, + "panic": 35843, + "690": 35844, + "\u0120cocoa": 35845, + "\u0120sampled": 35846, + "\u0120lookup": 35847, + "Indust": 35848, + "\u0120injections": 35849, + "genre": 35850, + "\u0120au": 35851, + "\u0120roadway": 35852, + "\u0120genitals": 35853, + "Kind": 35854, + "\u0120Examiner": 35855, + "\u0120Yaz": 35856, + "Fresh": 35857, + "\u0120paralysis": 35858, + "\u0120Aluminum": 35859, + "\u0120reap": 35860, + "ok\u00c3\u00a9": 35861, + "\u0120sloppy": 35862, + "\u0120Tunnel": 35863, + "posium": 35864, + "nery": 35865, + "enic": 35866, + "\u0120herbal": 35867, + "\u0120Outer": 35868, + "\u0120Builder": 35869, + "\u0120incur": 35870, + "\u0120ideologies": 35871, + "\u0120backups": 35872, + "consuming": 35873, + "\u0120Detect": 35874, + "deck": 35875, + "\u0120KNOW": 35876, + "\u0120Gret": 35877, + "\u0120MIC": 35878, + "\u0120toughness": 35879, + "\u0120Exhibit": 35880, + "\u0120hive": 35881, + "Les": 35882, + "\u0120SCHOOL": 35883, + "\u0120Atari": 35884, + "alde": 35885, + "\u0120Null": 35886, + "andestine": 35887, + "mouse": 35888, + "\u0120brigade": 35889, + "489": 35890, + "\u0120revol": 35891, + "\u0120Lawson": 35892, + "\u0120Wah": 35893, + "opoly": 35894, + "ebted": 35895, + "\u0120Saunders": 35896, + "\u0120313": 35897, + "\u0120Winc": 35898, + "\u0120taboo": 35899, + "\u0120Helmet": 35900, + "\u0120wedge": 35901, + "chip": 35902, + "\u0120Tina": 35903, + "bg": 35904, + "\u0120infuri": 35905, + "rn": 35906, + "\u0120anomalies": 35907, + "\u0120Sync": 35908, + "\u0120Exam": 35909, + "\u0120Commit": 35910, + "\u0120Diary": 35911, + "\u0120ALSO": 35912, + "\u0120Debor": 35913, + "omedical": 35914, + "\u0120comprehension": 35915, + "655": 35916, + "\u0120empowering": 35917, + "\u0120ire": 35918, + "\u0120juices": 35919, + "\u0120ETH": 35920, + "\u0120Boxing": 35921, + "=\"/": 35922, + "\u0120facilitated": 35923, + "poke": 35924, + "\u0120Parsons": 35925, + "\u0120Moder": 35926, + "travel": 35927, + "\u0120civilizations": 35928, + "\u0120libertarians": 35929, + "\u0120rune": 35930, + "\u0120Clarks": 35931, + "athed": 35932, + "\u0120campaigners": 35933, + "\u0120Dispatch": 35934, + "\u0120Fahrenheit": 35935, + "\u0120Capcom": 35936, + "----------": 35937, + "\u0120lace": 35938, + "\u0120draining": 35939, + "\u0120liner": 35940, + "\u0120Artificial": 35941, + "\u00c3\u00a9n": 35942, + "task": 35943, + "]).": 35944, + "\u0120GMO": 35945, + "\u0120Operator": 35946, + "ordinary": 35947, + "\u0120Influence": 35948, + "\u0120Ups": 35949, + "\u0120potency": 35950, + "ussen": 35951, + "ospons": 35952, + "\u0120Swim": 35953, + "\u0120Deadline": 35954, + "Unity": 35955, + "\u0120culinary": 35956, + "\u0120enlightenment": 35957, + "\u0120wearer": 35958, + "\u0120mined": 35959, + "\u0120ply": 35960, + "\u0120incest": 35961, + "\u0120DVDs": 35962, + "Walk": 35963, + "BTC": 35964, + "Trade": 35965, + "\u0120deval": 35966, + "iband": 35967, + "\u0120Oversight": 35968, + "Palestinian": 35969, + "\u0120dart": 35970, + "\u0120mul": 35971, + "LR": 35972, + "\u0120removable": 35973, + "\u0120Realms": 35974, + "\u00ec\u013f": 35975, + "\u0120miscar": 35976, + "\u0120Vulkan": 35977, + "685": 35978, + "\u00c3\u00a8re": 35979, + "\u0120Sap": 35980, + "\u0120merging": 35981, + "\u0120Carly": 35982, + "chester": 35983, + "\u0120brisk": 35984, + "\u0120luxurious": 35985, + "\u0120Generator": 35986, + "\u0120bitterness": 35987, + "\u0120edible": 35988, + "\u0120243": 35989, + "TG": 35990, + "\u0120rectangle": 35991, + "WithNo": 35992, + "below": 35993, + "Jenn": 35994, + "\u0120darkest": 35995, + "\u0120hitch": 35996, + "\u0120dosage": 35997, + "\u0120scaven": 35998, + "\u0120Keller": 35999, + "\u0120Illustrated": 36000, + "Certainly": 36001, + "\u0120Mavericks": 36002, + "Marginal": 36003, + "\u0120diarrhea": 36004, + "\u0120enormously": 36005, + "\u0120999": 36006, + "shr": 36007, + "quart": 36008, + "\u0120adamant": 36009, + "\u0120Mew": 36010, + "\u0120renovation": 36011, + "\u0120cervical": 36012, + "\u0120Percentage": 36013, + "eners": 36014, + "\u0120Kimber": 36015, + "\u0120floats": 36016, + "\u0120dex": 36017, + "\u0120Witcher": 36018, + "\u0120Swansea": 36019, + "dm": 36020, + "\u0120salty": 36021, + "yellow": 36022, + "\u0120cape": 36023, + "\u0120Drain": 36024, + "\u0120Paula": 36025, + "\u0120Toledo": 36026, + "lesi": 36027, + "Magazine": 36028, + "\u0120Wick": 36029, + "\u0120Mn": 36030, + "\u0120Ack": 36031, + "\u0120Riding": 36032, + "ASON": 36033, + "\u0120homophobic": 36034, + "ARP": 36035, + "\u0120wandered": 36036, + "CPU": 36037, + "oodoo": 36038, + "\u0120Pipe": 36039, + "\u0120tightening": 36040, + "\u0120Butt": 36041, + "318": 36042, + "\u0120deserted": 36043, + "Session": 36044, + "\u0120facilitating": 36045, + "Jump": 36046, + "\u0120emergencies": 36047, + "OWER": 36048, + "\u0120exhaustive": 36049, + "\u0120AFTER": 36050, + "\u0120heartbeat": 36051, + "\u0120Label": 36052, + "acky": 36053, + "\u0120Certified": 36054, + "iltration": 36055, + "Ze": 36056, + "\u0120Utt": 36057, + "\u01201300": 36058, + "\u0120presume": 36059, + "\u0120Disp": 36060, + "\u0120surged": 36061, + "\u0120dolls": 36062, + "Columb": 36063, + "\u0120chimpan": 36064, + "\u0120Razor": 36065, + "\u0120ticks": 36066, + "\u0120councillor": 36067, + "\u0120pilgrimage": 36068, + "\u0120Rebels": 36069, + "\u0120QC": 36070, + "\u0120Auction": 36071, + "xia": 36072, + "ikk": 36073, + "bred": 36074, + "\u0120insertion": 36075, + "\u0120coarse": 36076, + "dB": 36077, + "SEE": 36078, + "\u0120Zap": 36079, + "\u0120Foo": 36080, + "\u0120contempor": 36081, + "\u0120Quarterly": 36082, + "otions": 36083, + "\u0120Alchemist": 36084, + "\u0120Trey": 36085, + "\u0120Duo": 36086, + "Sweet": 36087, + "804": 36088, + "\u0120Giov": 36089, + "\u0120funn": 36090, + "Nin": 36091, + "hoff": 36092, + "\u0120ramifications": 36093, + "\u01201922": 36094, + "\u0120Experts": 36095, + "azes": 36096, + "\u0120garments": 36097, + "arial": 36098, + "\u0120Nab": 36099, + "\u0120257": 36100, + "\u0120Ved": 36101, + "\u0120humorous": 36102, + "\u0120Pompe": 36103, + "\u0120nylon": 36104, + "\u0120lurking": 36105, + "\u0120Sergey": 36106, + "\u0120Mattis": 36107, + "\u0120misogyny": 36108, + "\u0120Components": 36109, + "\u0120Watching": 36110, + "\u0120Folk": 36111, + "ractical": 36112, + "Bush": 36113, + "\u0120taped": 36114, + "\u0120grouping": 36115, + "\u0120beads": 36116, + "\u01202048": 36117, + "\u0120condu": 36118, + "querque": 36119, + "Reading": 36120, + "\u0120grievances": 36121, + "Ultra": 36122, + "\u0120endpoint": 36123, + "Hig": 36124, + "\u0120Static": 36125, + "\u0120Scarborough": 36126, + "Lua": 36127, + "\u0120Messi": 36128, + "aqu": 36129, + "\u0120PsyNet": 36130, + "\u0120Rudd": 36131, + "\u0120avenue": 36132, + "vp": 36133, + "Jer": 36134, + "\u0120shady": 36135, + "\u0120Resist": 36136, + "\u0120Artemis": 36137, + "\u0120careless": 36138, + "\u0120brokers": 36139, + "\u0120temperament": 36140, + "\u0120520": 36141, + "Tags": 36142, + "\u0120Turning": 36143, + "\u0120uttered": 36144, + "\u0120pedd": 36145, + "\u0120improvised": 36146, + "\u0120:(": 36147, + "\u0120tabl": 36148, + "\u0120plains": 36149, + "1600": 36150, + "pressure": 36151, + "\u0120Essence": 36152, + "margin": 36153, + "friends": 36154, + "\u0120Restoration": 36155, + "\u0120pollut": 36156, + "\u0120Poker": 36157, + "\u0120Augustine": 36158, + "\u0120CIS": 36159, + "\u0120SEAL": 36160, + "orama": 36161, + "\u0120thwart": 36162, + "seek": 36163, + "\u0120pagan": 36164, + "\u00c2\u00ba": 36165, + "cpu": 36166, + "\u0120garn": 36167, + "\u0120assortment": 36168, + "\u0120ILCS": 36169, + "tower": 36170, + "Recommended": 36171, + "\u0120unborn": 36172, + "\u0120RandomRedditor": 36173, + "\u0120RandomRedditorWithNo": 36174, + "\u0120paralyzed": 36175, + "\u0120eruption": 36176, + "\u0120intersect": 36177, + "\u0120Stoke": 36178, + "\u0120Sco": 36179, + "Bind": 36180, + "\u00e5\u00be": 36181, + "\u0120PNG": 36182, + "\u0120Negative": 36183, + "\u0120NOAA": 36184, + "Leon": 36185, + "\u0120alloy": 36186, + "\u0120Lama": 36187, + "\u0120Diversity": 36188, + "575": 36189, + "\u0120underestimated": 36190, + "\u0120Scor": 36191, + "\u0120mural": 36192, + "\u0120busted": 36193, + "soon": 36194, + "lif": 36195, + "\u0120nonex": 36196, + "\u0120allergy": 36197, + "\u0120Underworld": 36198, + "\u0120Rays": 36199, + "\u0120Blasio": 36200, + "\u0120hrs": 36201, + "\u0120Dir": 36202, + "\u0120327": 36203, + "byter": 36204, + "\u0120replacements": 36205, + "\u0120activates": 36206, + "rived": 36207, + "MH": 36208, + "\u0120pans": 36209, + "\u0120HI": 36210, + "\u0120longitudinal": 36211, + "\u0120nuisance": 36212, + "aler": 36213, + "\u0120swell": 36214, + "\u0120Signed": 36215, + "sci": 36216, + "\u0120Isles": 36217, + "\u0120AGA": 36218, + "\u0120defiant": 36219, + "\u0120sonic": 36220, + "ocon": 36221, + "KC": 36222, + "\u0120Aim": 36223, + "tie": 36224, + "ahah": 36225, + "\u0120mL": 36226, + "DX": 36227, + "\u0120bisc": 36228, + "\u0120Billboard": 36229, + "\u0120SYSTEM": 36230, + "NEY": 36231, + "gaard": 36232, + "\u0120distressed": 36233, + "formerly": 36234, + "Alan": 36235, + "\u0120chefs": 36236, + "\u0120optics": 36237, + "\u0120Comet": 36238, + "\u0120AMC": 36239, + "\u0120redesigned": 36240, + "irmation": 36241, + "\u0120sightings": 36242, + "382": 36243, + "311": 36244, + "\u0120WB": 36245, + "\u0120contraction": 36246, + "\u0120TOTAL": 36247, + "Dual": 36248, + "\u0120startled": 36249, + "\u0120understandably": 36250, + "\u0120sunglasses": 36251, + "ETHOD": 36252, + "\u0120docker": 36253, + "\u0120surfing": 36254, + "\u0120HEL": 36255, + "\u0120Slack": 36256, + "tones": 36257, + "\u0120shalt": 36258, + "Visual": 36259, + "498": 36260, + "Department": 36261, + "cussion": 36262, + "\u0120unrestricted": 36263, + "\u0120tad": 36264, + "\u0120rename": 36265, + "employed": 36266, + "\u0120educating": 36267, + "\u0120grinned": 36268, + "bedroom": 36269, + "\u0120Activities": 36270, + "\u0120Velvet": 36271, + "\u0120SWAT": 36272, + "\u0120shuffle": 36273, + "igor": 36274, + "\u0120saturation": 36275, + "Finding": 36276, + "cream": 36277, + "icter": 36278, + "\u0120vodka": 36279, + "tracking": 36280, + "tec": 36281, + "\u0120foreground": 36282, + "iesta": 36283, + "\u0120vehement": 36284, + "\u0120ECB": 36285, + "\u0120Tie": 36286, + "Ey": 36287, + "\u0120turtles": 36288, + "\u0120Railroad": 36289, + "\u0120Katz": 36290, + "\u0120Frames": 36291, + "\u0120menace": 36292, + "\u0120Fellowship": 36293, + "\u0120Essential": 36294, + "uggish": 36295, + "\u0120drip": 36296, + "chwitz": 36297, + "\u0120Kyoto": 36298, + "sb": 36299, + "\u0120Nina": 36300, + "Parameter": 36301, + "\u0120alarms": 36302, + "\u0120Claud": 36303, + "\u0120pioneering": 36304, + "\u0120chiefly": 36305, + "\u0120Scream": 36306, + "Collection": 36307, + "\u0120thankfully": 36308, + "\u0120Ronaldo": 36309, + "\u00e5\u0143\u0132": 36310, + "strip": 36311, + "\u0120Disneyland": 36312, + "commercial": 36313, + "Seeing": 36314, + "Soul": 36315, + "\u0120evacuate": 36316, + "\u0120civ": 36317, + "\u0120Ashe": 36318, + "\u0120divides": 36319, + "\u0120Dagger": 36320, + "rehensive": 36321, + "\u0120berries": 36322, + "\u0120DF": 36323, + "\u0120sushi": 36324, + "\u0120plurality": 36325, + "WI": 36326, + "\u0120disadvantaged": 36327, + "\u0120battalion": 36328, + "obiles": 36329, + "451": 36330, + "\u0120cling": 36331, + "\u0120undeniable": 36332, + "\u0120Lounge": 36333, + "\u0120haunt": 36334, + "phe": 36335, + "\u0120quantify": 36336, + "\u0120differed": 36337, + "\u0120[*]": 36338, + "\u0120Viz": 36339, + "cum": 36340, + "slave": 36341, + "\u0120videog": 36342, + "\u0120quar": 36343, + "\u0120bundles": 36344, + "\u0120Alonso": 36345, + "tackle": 36346, + "\u0120neuronal": 36347, + "\u0120landslide": 36348, + "confirmed": 36349, + "\u0120Depth": 36350, + "\u0120renewables": 36351, + "Bear": 36352, + "\u0120Macedonia": 36353, + "\u0120jerseys": 36354, + "\u0120bunk": 36355, + "\u0120Spawn": 36356, + "\u0120Controls": 36357, + "\u0120Buchanan": 36358, + "\u0120robotics": 36359, + "\u0120emphasizing": 36360, + "\u0120Tutorial": 36361, + "hyp": 36362, + "iston": 36363, + "\u0120monumental": 36364, + "\u00e6\u00b0": 36365, + "\u0120Carry": 36366, + "\u0120tbsp": 36367, + "enance": 36368, + "Hill": 36369, + "arthed": 36370, + "\u0120rotten": 36371, + "Dean": 36372, + "\u0120twisting": 36373, + "\u0120goodwill": 36374, + "\u0120immersion": 36375, + "Living": 36376, + "\u0120brushes": 36377, + "\u0120CGI": 36378, + "\u0120Atk": 36379, + "traditional": 36380, + "\u0120phantom": 36381, + "\u0120Stamina": 36382, + "\u0120expansions": 36383, + "\u0120Marin": 36384, + "\u0120embarked": 36385, + "\u0120Eg": 36386, + "intestinal": 36387, + "\u0120PEOPLE": 36388, + "\u0120Booth": 36389, + "\u0120Appalach": 36390, + "\u0120relegated": 36391, + "VT": 36392, + "MIT": 36393, + "\u0120muster": 36394, + "\u0120withdrawing": 36395, + "\u0120microscope": 36396, + "\u0120Gathering": 36397, + "\u0120Crescent": 36398, + "\u0120Argentine": 36399, + "\u0120Decre": 36400, + "\u0120Dominic": 36401, + "\u0120buds": 36402, + "antage": 36403, + "\u0120Ion": 36404, + "\u0120widened": 36405, + "ONSORED": 36406, + "\u0120Gloves": 36407, + "iannopoulos": 36408, + "razen": 36409, + "feel": 36410, + "\u0120repayment": 36411, + "\u0120hindsight": 36412, + "\u0120REALLY": 36413, + "\u0120Pistol": 36414, + "\u0120Brah": 36415, + "\u0120watts": 36416, + "\u0120survives": 36417, + "\u0120flurry": 36418, + "issy": 36419, + "Alert": 36420, + "\u0120Uruguay": 36421, + "Phoenix": 36422, + "Slow": 36423, + "\u0120Grave": 36424, + "\u0120Fir": 36425, + "\u0120manageable": 36426, + "\u0120tariff": 36427, + "\u0120UDP": 36428, + "\u0120Pistons": 36429, + "\u0120Nigerian": 36430, + "\u0120strikeouts": 36431, + "\u0120cosmetics": 36432, + "whelming": 36433, + "fab": 36434, + "cape": 36435, + "proxy": 36436, + "\u0120rethink": 36437, + "\u0120overcoming": 36438, + "simple": 36439, + "\u0120woo": 36440, + "\u0120distracting": 36441, + "\u0120Stanton": 36442, + "\u0120Tulsa": 36443, + "\u0120Dock": 36444, + "659": 36445, + "\u0120discord": 36446, + "\u0120Emacs": 36447, + "\u0120Ves": 36448, + "\u0120ROB": 36449, + "\u0120reassuring": 36450, + "\u0120consortium": 36451, + "Muslims": 36452, + "321": 36453, + "\u0120prompts": 36454, + "sei": 36455, + "\u0120Hitch": 36456, + "imposed": 36457, + "\u0120Fool": 36458, + "\u0120indiscrim": 36459, + "wrong": 36460, + "buquerque": 36461, + "Davis": 36462, + "!]": 36463, + "\u0120timeless": 36464, + "\u0120NEED": 36465, + "\u0120pesticide": 36466, + "\u0120rallying": 36467, + "\u0120Calder": 36468, + "\u0120\u00e5\u00a4": 36469, + "\u0120xp": 36470, + "\u0120Unle": 36471, + "\u0120Export": 36472, + "luaj": 36473, + "Buff": 36474, + ")[": 36937, + "\u0120sqor": 36938, + "Saudi": 36939, + "\u0120istg": 36940, + "\u0120indulge": 36941, + "proc": 36942, + "\u0120disgusted": 36943, + "\u0120compounded": 36944, + "\u0120nem": 36945, + "\u0120schooling": 36946, + "\u0120Cure": 36947, + "processing": 36948, + "Sol": 36949, + "\u0120proverb": 36950, + "itized": 36951, + "\u0120Alvarez": 36952, + "\u0120scarf": 36953, + "\u0120rectangular": 36954, + "reve": 36955, + "\u0120hormonal": 36956, + "\u0120Stress": 36957, + "itizen": 36958, + "\u0120425": 36959, + "girls": 36960, + "\u0120Noir": 36961, + "\u0120Rapp": 36962, + "\u0120marches": 36963, + "church": 36964, + "\u0120Uses": 36965, + "\u0120405": 36966, + "\u0120Berm": 36967, + "\u0120ordinances": 36968, + "\u0120Judgment": 36969, + "Charges": 36970, + "\u0120Zin": 36971, + "\u0120dusty": 36972, + "\u0120strawberries": 36973, + "\u0120perce": 36974, + "\u0120Thur": 36975, + "\u0120Deborah": 36976, + "netflix": 36977, + "\u0120Lambert": 36978, + "\u0120amused": 36979, + "\u0120Guang": 36980, + "YOU": 36981, + "RGB": 36982, + "\u0120CCTV": 36983, + "\u0120fiat": 36984, + "rang": 36985, + "\u0120federation": 36986, + "\u0120Mant": 36987, + "\u0120Bust": 36988, + "\u0120Mare": 36989, + "respective": 36990, + "\u0120Migration": 36991, + "\u0120BIT": 36992, + "590": 36993, + "\u0120patriotism": 36994, + "\u0120outlining": 36995, + "region": 36996, + "\u0120Jos\u00c3\u00a9": 36997, + "\u0120blasting": 36998, + "\u0120Ezra": 36999, + "Bs": 37000, + "\u0120undermines": 37001, + "\u0120Smooth": 37002, + "\u0120clashed": 37003, + "radio": 37004, + "\u0120transitioning": 37005, + "\u0120Buccaneers": 37006, + "\u0120Owl": 37007, + "\u0120plugs": 37008, + "\u0120hiatus": 37009, + "\u0120Pinball": 37010, + "\u0120mig": 37011, + "\u0120Nutr": 37012, + "\u0120Wolfe": 37013, + "\u0120integers": 37014, + "\u0120orbits": 37015, + "\u0120Edwin": 37016, + "\u0120DirectX": 37017, + "bite": 37018, + "\u0120blazing": 37019, + "vr": 37020, + "Edge": 37021, + "\u0120PID": 37022, + "exit": 37023, + "\u0120Comed": 37024, + "\u0120Pathfinder": 37025, + "\u0120Guid": 37026, + "\u0120Signs": 37027, + "\u0120Zer": 37028, + "\u0120Agenda": 37029, + "\u0120reimbursement": 37030, + "Mesh": 37031, + "iPhone": 37032, + "\u0120Marcos": 37033, + "\u0120Sites": 37034, + "hate": 37035, + "enburg": 37036, + "\u0120sockets": 37037, + "pend": 37038, + "Batman": 37039, + "vir": 37040, + "\u0120SHOW": 37041, + "\u0120provisional": 37042, + "conn": 37043, + "\u0120Deaths": 37044, + "ATIVE": 37045, + "Profile": 37046, + "sym": 37047, + "JA": 37048, + "\u0120ninja": 37049, + "installed": 37050, + "idates": 37051, + "ebra": 37052, + "\u0120Omaha": 37053, + "\u0120seizing": 37054, + "\u0120Beasts": 37055, + "\u0120salts": 37056, + "Mission": 37057, + "Generally": 37058, + "\u0120Trilogy": 37059, + "heon": 37060, + "legates": 37061, + "\u0120dime": 37062, + "\u0120faire": 37063, + "parable": 37064, + "Graph": 37065, + "\u0120totaling": 37066, + "\u0120diagrams": 37067, + "\u0120Yanuk": 37068, + "plet": 37069, + "\u0120Meh": 37070, + "\u0120mythical": 37071, + "\u0120Stephens": 37072, + "autical": 37073, + "ochemistry": 37074, + "\u0120kilograms": 37075, + "\u0120elbows": 37076, + "ancock": 37077, + "\u0120BCE": 37078, + "\u0120Prague": 37079, + "\u0120improv": 37080, + "\u0120Devin": 37081, + "\u0120\"\\": 37082, + "paralle": 37083, + "\u0120supremacists": 37084, + "\u0120Billion": 37085, + "\u0120regimen": 37086, + "innacle": 37087, + "\u0120requisite": 37088, + "angan": 37089, + "\u0120Burlington": 37090, + "ainment": 37091, + "\u0120Objective": 37092, + "omsky": 37093, + "GV": 37094, + "\u0120unilateral": 37095, + "\u0120tc": 37096, + "\u0120hires": 37097, + "mental": 37098, + "\u0120involuntary": 37099, + "\u0120transpl": 37100, + "\u0120ASCII": 37101, + "\u00c2\u00a8": 37102, + "Events": 37103, + "\u0120doubted": 37104, + "\u0120Kaplan": 37105, + "\u0120Courage": 37106, + "igon": 37107, + "\u0120Managing": 37108, + "\u0120Tart": 37109, + "\u0120falsehood": 37110, + "\u0120Violet": 37111, + "\u0120airs": 37112, + "\u0120fertilizer": 37113, + "Britain": 37114, + "\u0120aquatic": 37115, + "ouf": 37116, + "Words": 37117, + "\u0120Hartford": 37118, + "\u0120evenings": 37119, + "\u0120Vengeance": 37120, + "quite": 37121, + "Gall": 37122, + "\u0120Pret": 37123, + "\u0120pdf": 37124, + "\u0120LM": 37125, + "\u0120Sochi": 37126, + "\u0120Intercept": 37127, + "920": 37128, + "\u0120profitability": 37129, + "\u0120Idle": 37130, + "\u0120MacDonald": 37131, + "\u0120Establishment": 37132, + "umsy": 37133, + "\u0120gatherings": 37134, + "\u0120Naj": 37135, + "Charlie": 37136, + "\u0120ascent": 37137, + "\u0120Protector": 37138, + "\u0120algebra": 37139, + "\u0120bios": 37140, + "forums": 37141, + "ELS": 37142, + "Introduced": 37143, + "\u0120335": 37144, + "\u0120astronomy": 37145, + "Contribut": 37146, + "\u0120Polic": 37147, + "Platform": 37148, + "\u0120containment": 37149, + "wrap": 37150, + "\u0120coronary": 37151, + "\u0120Jelly": 37152, + "manager": 37153, + "\u0120heartbreaking": 37154, + "cair": 37155, + "\u0120Chero": 37156, + "cgi": 37157, + "Medical": 37158, + "\u0120Accountability": 37159, + "!!\"": 37160, + "ophile": 37161, + "\u0120psychotic": 37162, + "\u0120Restrict": 37163, + "\u0120equitable": 37164, + "issues": 37165, + "\u01201905": 37166, + "\u0120Nek": 37167, + "cised": 37168, + "\u0120Tracking": 37169, + "\u0120ozone": 37170, + "\u0120cooker": 37171, + "rosis": 37172, + "\u0120reopen": 37173, + "\u0120infinity": 37174, + "\u0120Pharmaceutical": 37175, + "ensional": 37176, + "Attempt": 37177, + "\u0120Rory": 37178, + "Marco": 37179, + "\u0120awaits": 37180, + "HOW": 37181, + "treated": 37182, + "\u0120bolst": 37183, + "\u0120revered": 37184, + "\u0120pods": 37185, + "oppers": 37186, + "0010": 37187, + "\u0120amplitude": 37188, + "rican": 37189, + "SPONSORED": 37190, + "\u0120trousers": 37191, + "\u0120halves": 37192, + "\u0120Kaine": 37193, + "\u0120Cutler": 37194, + "\u0120AUTH": 37195, + "\u0120splendid": 37196, + "\u0120preventive": 37197, + "\u0120Dudley": 37198, + "ifacts": 37199, + "uminati": 37200, + "\u0120Yin": 37201, + "\u0120admon": 37202, + "\u0120Vag": 37203, + "\u0120inverted": 37204, + "\u0120hastily": 37205, + "\u0120Hague": 37206, + "Lyn": 37207, + "\u0120ledger": 37208, + "\u0120astronomical": 37209, + "getting": 37210, + "\u0120circa": 37211, + "\u0120Cic": 37212, + "\u0120Tennis": 37213, + "Limited": 37214, + "\u0120dru": 37215, + "\u0120BYU": 37216, + "\u0120travellers": 37217, + "\u0120pane": 37218, + "\u0120Intro": 37219, + "\u0120patiently": 37220, + "\u0120aiding": 37221, + "\u0120loos": 37222, + "\u0120Tough": 37223, + "\u0120293": 37224, + "\u0120consumes": 37225, + "SourceFile": 37226, + "\u0120\"\"\"": 37227, + "\u0120bonding": 37228, + "\u0120tilted": 37229, + "\u0120menstrual": 37230, + "\u0120Celestial": 37231, + "ULAR": 37232, + "Plugin": 37233, + "\u0120risking": 37234, + "Naz": 37235, + "\u0120Riyadh": 37236, + "\u0120accredited": 37237, + "\u0120skirm": 37238, + "\u00e9\u013d": 37239, + "\u0120examiner": 37240, + "\u0120messing": 37241, + "\u0120nearing": 37242, + "\u0120Chern": 37243, + "\u0120Beckham": 37244, + "\u0120swapped": 37245, + "\u0120goose": 37246, + "Kay": 37247, + "\u0120lofty": 37248, + "\u0120Wallet": 37249, + "\u0120['": 37250, + "\u0120apocalypse": 37251, + "\u0120bamboo": 37252, + "\u0120SPACE": 37253, + "\u0120Elena": 37254, + "\u0120306": 37255, + "acons": 37256, + "\u0120tightened": 37257, + "\u0120adolescence": 37258, + "\u0120rainy": 37259, + "\u0120vandalism": 37260, + "\u0120Newtown": 37261, + "\u0120conject": 37262, + "cakes": 37263, + "\u0120cheated": 37264, + "\u0120moderators": 37265, + "params": 37266, + "EFF": 37267, + "\u0120deceit": 37268, + "\u0120STL": 37269, + "\u0120Tanzania": 37270, + "\u0120RI": 37271, + "\u01201923": 37272, + "\u0120Exile": 37273, + "thel": 37274, + "\u0120theolog": 37275, + "\u0120quirky": 37276, + "\u0120Irvine": 37277, + "\u0120needy": 37278, + "oris": 37279, + "Um": 37280, + "Ka": 37281, + "\u0120mailbox": 37282, + "322": 37283, + "\u0120bos": 37284, + "\u0120Petra": 37285, + "KING": 37286, + "\u0120enlarged": 37287, + "Often": 37288, + "\u0120badass": 37289, + "\u0120343": 37290, + "\u0120Places": 37291, + "\u0120CAD": 37292, + "\u0120pristine": 37293, + "\u0120intervening": 37294, + "direction": 37295, + "\u0120laz": 37296, + "\u0120DSM": 37297, + "\u0120projecting": 37298, + "\u0120Funk": 37299, + "agog": 37300, + "payment": 37301, + "nov": 37302, + "\u0120chatter": 37303, + "ARB": 37304, + "\u0120examinations": 37305, + "\u0120Household": 37306, + "\u0120Gus": 37307, + "Ford": 37308, + "414": 37309, + "Boss": 37310, + "\u0120mystic": 37311, + "\u0120leaps": 37312, + "\u0120Bav": 37313, + "ulz": 37314, + "budget": 37315, + "Football": 37316, + "\u0120subsidized": 37317, + "\u0120firsthand": 37318, + "\u0120coincide": 37319, + "ocular": 37320, + "Conn": 37321, + "\u0120Collabor": 37322, + "\u0120fools": 37323, + "amura": 37324, + "ahar": 37325, + "rists": 37326, + "\u0120swollen": 37327, + "\u0120expended": 37328, + "\u0120Pau": 37329, + "sup": 37330, + "\u0120spar": 37331, + "\u0120keynote": 37332, + "suff": 37333, + "\u0120unequal": 37334, + "\u0120progressing": 37335, + "strings": 37336, + "\u0120Gamergate": 37337, + "Disney": 37338, + "\u0120Eleven": 37339, + "omnia": 37340, + "\u0120scripted": 37341, + "\u0120earners": 37342, + "brother": 37343, + "\u0120Enabled": 37344, + "\u00e6\u00b3": 37345, + "\u0120larvae": 37346, + "\u0120LOC": 37347, + "mess": 37348, + "Wilson": 37349, + "\u0120Template": 37350, + "successfully": 37351, + "\u0120paramount": 37352, + "\u0120camouflage": 37353, + "\u0120binds": 37354, + "\u0120Quiet": 37355, + "\u0120Shutterstock": 37356, + "rush": 37357, + "\u0120mascot": 37358, + "fortune": 37359, + "\u0120Colt": 37360, + "\u0120Beyon": 37361, + "habi": 37362, + "\u0120hairc": 37363, + "\u0120267": 37364, + "\u0120Deus": 37365, + "\u0120twitch": 37366, + "\u0120concentrating": 37367, + "\u0120nipples": 37368, + "cible": 37369, + "\u0120gir": 37370, + "NZ": 37371, + "Math": 37372, + "nih": 37373, + "Required": 37374, + "\u0120ponder": 37375, + "\u0120SAN": 37376, + "\u0120weddings": 37377, + "\u0120loneliness": 37378, + "NES": 37379, + "\u0120Mahjong": 37380, + "695": 37381, + "addle": 37382, + "\u0120Garner": 37383, + "\u0120COUR": 37384, + "Bridge": 37385, + "\u0120spree": 37386, + "\u0120Caldwell": 37387, + "\u0120bribery": 37388, + "\u0120\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 37389, + "plugins": 37390, + "\u0120racket": 37391, + "\u0120champagne": 37392, + "versible": 37393, + "Vote": 37394, + "\u0120modifiers": 37395, + "Mayor": 37396, + "680": 37397, + "\u0120assemblies": 37398, + "\u0120Sultan": 37399, + "\u0120Ning": 37400, + "\u0120Ladies": 37401, + "\u0120sulfur": 37402, + "\u0120orbs": 37403, + "\u0120-----": 37404, + "_______": 37405, + "\u0120Journalism": 37406, + "\u0120esports": 37407, + "\u0120lush": 37408, + "\u0120hue": 37409, + "\u0120spectral": 37410, + "Honest": 37411, + "\u00e3\u0125\u0131": 37412, + "\u0120bushes": 37413, + "\u0120reinforcement": 37414, + "\u0120reopened": 37415, + "\u0120Wheels": 37416, + "\u0120Morg": 37417, + "rieving": 37418, + "\u0120auxiliary": 37419, + "\u0120jQuery": 37420, + "\u0120BAT": 37421, + "tesque": 37422, + "\u0120vertex": 37423, + "pure": 37424, + "frey": 37425, + "\u00e3\u0124\u00ba": 37426, + "dos": 37427, + "\u0120typh": 37428, + "\u0120cull": 37429, + "\u0120eq": 37430, + "\u0120decon": 37431, + "\u0120tossing": 37432, + "\u0120disparate": 37433, + "\u0120Brigham": 37434, + "printf": 37435, + "ledged": 37436, + "\u0120sund": 37437, + "\u0120cozy": 37438, + "\u0120hepatitis": 37439, + "performing": 37440, + "\u0120aval": 37441, + "\u0120GG": 37442, + "future": 37443, + "\u0120petertodd": 37444, + "\u0120Kosovo": 37445, + "\u0120magnets": 37446, + "Already": 37447, + "\u0120Edison": 37448, + "\u0120Ceres": 37449, + "\u0120RAID": 37450, + "\u0120brilliance": 37451, + "576": 37452, + "\u0120derives": 37453, + "\u0120hypertension": 37454, + "\u0120\u00ce\u0136": 37455, + "\u0120lambda": 37456, + "\u0120flair": 37457, + "\u0120missionaries": 37458, + "\u0120rapes": 37459, + "\u0120Starter": 37460, + "\u0120Months": 37461, + "\u0120defy": 37462, + "\u0120seismic": 37463, + "\u0120Raphael": 37464, + "\u0120eurozone": 37465, + "656": 37466, + "zsche": 37467, + "\u0120scratched": 37468, + "\u0120bows": 37469, + "\u0120Lennon": 37470, + "\u0120Gaia": 37471, + "\u0120dripping": 37472, + "facts": 37473, + "Ale": 37474, + "\u0120frogs": 37475, + "\u0120Breast": 37476, + "ogeneity": 37477, + "\u0120Prosecutor": 37478, + "\u0120amplified": 37479, + "\u0120Hodg": 37480, + "\u0120Fn": 37481, + "Thousands": 37482, + "\u0120NIH": 37483, + "\u0120Monitoring": 37484, + "FTWARE": 37485, + "\u0120Priebus": 37486, + "\u0120Growing": 37487, + "hunter": 37488, + "\u0120diagnose": 37489, + "\u0120Mald": 37490, + "\u0120LR": 37491, + "\u0120crowned": 37492, + "\u0120bursting": 37493, + "\u0120dissolution": 37494, + "javascript": 37495, + "\u0120usefulness": 37496, + "\u0120Execution": 37497, + ":(": 37498, + "\u0120Ivory": 37499, + "aah": 37500, + "\u0120persecuted": 37501, + "violence": 37502, + "istas": 37503, + "\u0120Crate": 37504, + "\u0120impulses": 37505, + "\u0120Spani": 37506, + "edes": 37507, + "Handle": 37508, + "\u0120Zerg": 37509, + "thinkable": 37510, + "Lastly": 37511, + "\u0120spontaneously": 37512, + "\u0120inconvenient": 37513, + "\u0120dismissing": 37514, + "\u0120plotted": 37515, + "\u0120eighty": 37516, + "\u0120737": 37517, + "rish": 37518, + "\u0120Thornton": 37519, + "atham": 37520, + "\u0120sitcom": 37521, + "Ven": 37522, + "Recipe": 37523, + "tel": 37524, + "lund": 37525, + "\u0120clears": 37526, + "\u0120Sasuke": 37527, + "\u0120258": 37528, + "\u0120opting": 37529, + "\u0120enraged": 37530, + "esthetic": 37531, + "\u0120Ae": 37532, + "uchs": 37533, + "Prep": 37534, + "Flow": 37535, + "\u0120runoff": 37536, + "\u0120Eating": 37537, + "\u0120Giles": 37538, + "\u0120Acting": 37539, + "resources": 37540, + "ibaba": 37541, + "\u0120rpm": 37542, + "\u0120skewed": 37543, + "\u0120Blanc": 37544, + "\u0120Sakuya": 37545, + "\u0120hotter": 37546, + "\u01201924": 37547, + "opian": 37548, + "cko": 37549, + "\u0120crumbling": 37550, + "\u0120captains": 37551, + "\u0120Appropriations": 37552, + "leaders": 37553, + "dropping": 37554, + "anuts": 37555, + "\u0120reversing": 37556, + "\u0120Pose": 37557, + "\u0120Sek": 37558, + "Scot": 37559, + "\u0120Idea": 37560, + "cise": 37561, + "\u0120Slovenia": 37562, + "\u0120317": 37563, + "Doctor": 37564, + "\u0120crocod": 37565, + "aldi": 37566, + "Sea": 37567, + "\u0120Farrell": 37568, + "\u0120mercenaries": 37569, + "\u0120RNC": 37570, + "\u0120Guess": 37571, + "\u0120pacing": 37572, + "Machine": 37573, + "StreamerBot": 37574, + "\u0120Charity": 37575, + "\u0120298": 37576, + "\u0120cannons": 37577, + "\u0120Toby": 37578, + "TPPStreamerBot": 37579, + "\u0120Passion": 37580, + "cfg": 37581, + "Thom": 37582, + "\u0120badges": 37583, + "\u0120Bernstein": 37584, + ".\u00e2\u0122\u0135": 37585, + "\u0120POP": 37586, + "\u0120Conj": 37587, + "\u0120initialization": 37588, + "\u0120biodiversity": 37589, + "Dub": 37590, + "\u0120feudal": 37591, + "\u0120disclaimer": 37592, + "\u0120crow": 37593, + "\u0120ignition": 37594, + "arf": 37595, + "SHA": 37596, + "\u0120kHz": 37597, + "hazard": 37598, + "\u0120Artists": 37599, + "oeuv": 37600, + "679": 37601, + "\u0120Rudy": 37602, + "Nine": 37603, + "\u0120Ramadan": 37604, + "\u00e5\u00bd": 37605, + "itto": 37606, + "\u0120adrenaline": 37607, + "Cert": 37608, + "\u0120smelled": 37609, + "\u0120impunity": 37610, + "\u0120agendas": 37611, + "\u0120Reborn": 37612, + "\u0120Concent": 37613, + "\u0120Seems": 37614, + "\u0120omega": 37615, + "\u0120Dustin": 37616, + "\u0120backer": 37617, + "\u0120Sauce": 37618, + "\u0120Boyle": 37619, + "WIN": 37620, + "\u0120spins": 37621, + "\u0120pauses": 37622, + "upt": 37623, + "\u0120shredded": 37624, + "\u0120strapped": 37625, + "\u0120Corruption": 37626, + "\u0120scratches": 37627, + "\u0120ni": 37628, + "\u0120attire": 37629, + "\u0120SAF": 37630, + "FactoryReloaded": 37631, + "\u0120IPS": 37632, + "\u0120(%": 37633, + "\u0120seminar": 37634, + "focus": 37635, + "civil": 37636, + "\u01201860": 37637, + "intosh": 37638, + "\u0120continual": 37639, + "\u0120abbrevi": 37640, + "\u0120Sok": 37641, + "ocobo": 37642, + "XM": 37643, + "\u0120frantic": 37644, + "\u0120unavoidable": 37645, + "\u0120artery": 37646, + "\u0120annotations": 37647, + "bath": 37648, + "Climate": 37649, + "\u0120dors": 37650, + "\u0120Slide": 37651, + "coord": 37652, + "\u0120Reload": 37653, + "\u0120LDL": 37654, + "\u0120Lovecraft": 37655, + "\u0120unimagin": 37656, + "\u0120resembled": 37657, + "\u0120barracks": 37658, + "np": 37659, + "\u0120surrogate": 37660, + "\u0120categorized": 37661, + "\u00e3\u0124\u00a9": 37662, + "\u0120vaccinated": 37663, + "\u0120drainage": 37664, + "\u0120indist": 37665, + "\u0120WhatsApp": 37666, + "\u01201870": 37667, + "olerance": 37668, + "invoke": 37669, + "amorph": 37670, + "\u0120reconnect": 37671, + "\u0120emanc": 37672, + "\u0120blindness": 37673, + "\u01201280": 37674, + "internet": 37675, + "collar": 37676, + "\u0120altru": 37677, + "\u0120abyss": 37678, + "\u0120TRI": 37679, + "657": 37680, + "\u0120infused": 37681, + "HEAD": 37682, + "\u0120forestry": 37683, + "\u0120Woody": 37684, + "\u0120Ci": 37685, + "wi": 37686, + "sam": 37687, + "784": 37688, + "holiday": 37689, + "\u0120mogul": 37690, + "\u0120Fees": 37691, + "\u0120DEN": 37692, + "Internal": 37693, + "urbed": 37694, + "fusc": 37695, + "atom": 37696, + "\u0120Illusion": 37697, + "\u0120polled": 37698, + "\u0120flap": 37699, + "\u0120coax": 37700, + "LGBT": 37701, + "Analy": 37702, + "\u0120Sections": 37703, + "\u0120Californ": 37704, + "emn": 37705, + "\u0120hither": 37706, + "\u0120NIGHT": 37707, + "\u0120nailed": 37708, + "\u0120Pipeline": 37709, + "391": 37710, + "oof": 37711, + "\u0120Primal": 37712, + "verend": 37713, + "\u0120slashing": 37714, + "\u0120retri": 37715, + "aviour": 37716, + "\u0120departing": 37717, + "gil": 37718, + "ISC": 37719, + "\u0120midway": 37720, + "\u0120ultrasound": 37721, + "\u0120behaving": 37722, + "\u0120Tara": 37723, + "classes": 37724, + "Virtual": 37725, + "\u0120Colonial": 37726, + "\u0120stripping": 37727, + "\u0120orchestrated": 37728, + "\u0120Graves": 37729, + "452": 37730, + "\u0120Ironically": 37731, + "\u0120Writers": 37732, + "\u0120lends": 37733, + "\u0120Manz": 37734, + "\u0120raven": 37735, + "\u0120oxidative": 37736, + "\u0120266": 37737, + "ELF": 37738, + "actually": 37739, + "ascar": 37740, + "Draft": 37741, + "\u0120favourable": 37742, + "\u0120humiliating": 37743, + "\u0120fidelity": 37744, + "\u0120Hof": 37745, + "\u0120Xuan": 37746, + "496": 37747, + "\u0120layered": 37748, + "atis": 37749, + "790": 37750, + "\u0120paycheck": 37751, + "iton": 37752, + "Kar": 37753, + "\u0120VMware": 37754, + "\u0120Farmer": 37755, + "\u0120servic": 37756, + "glomer": 37757, + "\u0120slump": 37758, + "\u0120Fabric": 37759, + "\u0120DOC": 37760, + "esting": 37761, + "\u0120reassure": 37762, + "\u0120phyl": 37763, + "volt": 37764, + "itory": 37765, + "Rules": 37766, + "\u0120oxidation": 37767, + "\u0120prized": 37768, + "\u0120mistress": 37769, + "\u0120Django": 37770, + "WARN": 37771, + "\u00e5\u0133": 37772, + "\u0120encode": 37773, + "\u0120Feedback": 37774, + "\u0120stupidity": 37775, + "Ian": 37776, + "\u0120Yugoslavia": 37777, + "\u00d7\u00a8": 37778, + "acl": 37779, + "UTE": 37780, + "1977": 37781, + "\u0120qualifies": 37782, + "\u0120pulses": 37783, + "pretty": 37784, + "\u0120froze": 37785, + "\u0120ss": 37786, + "Iterator": 37787, + "\u0120urgently": 37788, + "\u0120mailed": 37789, + "\u0120Cham": 37790, + "\u0120sustaining": 37791, + "\u0120basil": 37792, + "\u0120puppies": 37793, + "ilant": 37794, + "\u0120PLEASE": 37795, + "lap": 37796, + "aceous": 37797, + "Fear": 37798, + "\u0120Mastery": 37799, + "automatic": 37800, + "\u0120TAG": 37801, + "\u0120antim": 37802, + "agles": 37803, + "473": 37804, + "frames": 37805, + "\u0120whispers": 37806, + "\u0120Whoever": 37807, + "\u0120bravery": 37808, + "\u0120UKIP": 37809, + "ractions": 37810, + "\"\"\"": 37811, + "\u0120tame": 37812, + "\u0120parted": 37813, + "everything": 37814, + "CONT": 37815, + "\u0120indebted": 37816, + "\u0120addr": 37817, + "rek": 37818, + "IRED": 37819, + "\u0120eminent": 37820, + "clinton": 37821, + "\u0120ousted": 37822, + "\u0120reviewer": 37823, + "\u0120meltdown": 37824, + "\u0120rearr": 37825, + "\u0120Yao": 37826, + "thereal": 37827, + "abyte": 37828, + "\u0120stumbling": 37829, + "\u0120batches": 37830, + "\u0120259": 37831, + "\u0120contraceptive": 37832, + "\u0120prostitute": 37833, + "ensis": 37834, + "Decl": 37835, + "\u0120Strikes": 37836, + "Military": 37837, + "\u0120Oath": 37838, + "vacc": 37839, + "ppings": 37840, + "052": 37841, + "\u0120partName": 37842, + "amping": 37843, + "Reports": 37844, + "KI": 37845, + "CHR": 37846, + "\u0120subtly": 37847, + "swers": 37848, + "Blake": 37849, + "usual": 37850, + "\u0120contestants": 37851, + "\u0120cartridges": 37852, + "\u0120GREAT": 37853, + "\u0120blush": 37854, + "\u0120\u00e2\u0122\u00ba": 37855, + "472": 37856, + "\u0120reasoned": 37857, + "\u00e3\u0125\u00a4": 37858, + "paralleled": 37859, + "\u0120dyn": 37860, + "agate": 37861, + "\u0120nightly": 37862, + "\u00e5\u0128": 37863, + "556": 37864, + "\u0120semantic": 37865, + "\u0120Advoc": 37866, + "\u0120!!": 37867, + "\u0120disagrees": 37868, + "\u0120BW": 37869, + "Veh": 37870, + "\u0120harming": 37871, + "\u0120embraces": 37872, + "\u0120strives": 37873, + "\u0120inland": 37874, + "\u0120Kard": 37875, + "\u0120heats": 37876, + "\u0120Ginny": 37877, + "utan": 37878, + "ernaut": 37879, + "ylene": 37880, + "\u0120Elev": 37881, + "JD": 37882, + "\u0120hars": 37883, + "\u0120Starr": 37884, + "\u0120skysc": 37885, + "\u0120collaborators": 37886, + "Usually": 37887, + "\u0120revolutions": 37888, + "\u0120STATS": 37889, + "\u0120dismantle": 37890, + "\u0120confidently": 37891, + "\u0120kinetic": 37892, + "Ali": 37893, + "\u0120percentile": 37894, + "\u0120extracting": 37895, + "illian": 37896, + "estead": 37897, + "\u0120physicists": 37898, + "\u0120Marshal": 37899, + "\u0120fellowship": 37900, + "\u0120dashed": 37901, + "\u0120UR": 37902, + "\u0120Sioux": 37903, + "\u0120Compact": 37904, + "amide": 37905, + "Python": 37906, + "\u0120Leigh": 37907, + "\u0120Pharmac": 37908, + "istrates": 37909, + "herical": 37910, + "\u0120fue": 37911, + "\u0120Emin": 37912, + "\u0120({": 37913, + "\u0120Neighborhood": 37914, + "\u0120disrupting": 37915, + "\u0120Dup": 37916, + "\u0120gland": 37917, + "\u0120Sev": 37918, + "\u0120Marian": 37919, + "argon": 37920, + "\u0120Dund": 37921, + "\u0120": 46904, + "\u0120Philips": 46905, + "\u0120Kafka": 46906, + "\u0120upheaval": 46907, + "\u0120sentimental": 46908, + "\u0120sax": 46909, + "\u0120Akira": 46910, + "serial": 46911, + "Matrix": 46912, + "\u0120electing": 46913, + "\u0120commenter": 46914, + "\u0120Nebula": 46915, + "plets": 46916, + "\u0120Nadu": 46917, + "\u0120Adren": 46918, + "\u0120enshr": 46919, + "\u0120RAND": 46920, + "financial": 46921, + "\u0120Clyde": 46922, + "utherford": 46923, + "\u0120signage": 46924, + "\u0120deline": 46925, + "\u0120phosphate": 46926, + "roversial": 46927, + "fascist": 46928, + "\u0120Vall": 46929, + "\u0120Bethlehem": 46930, + "\u0120fors": 46931, + "\u0120english": 46932, + "Solid": 46933, + "Nature": 46934, + "\u0120va": 46935, + "\u0120Guests": 46936, + "\u0120tantal": 46937, + "\u0120autoimmune": 46938, + ";;;;;;;;;;;;": 46939, + "\u0120Totally": 46940, + "\u0120Ov": 46941, + "\u0120defences": 46942, + "\u0120Coconut": 46943, + "\u0120tranquil": 46944, + "\u0120ploy": 46945, + "\u0120flavours": 46946, + "\u0120Flask": 46947, + "\u00e3\u0124\u00a8\u00e3\u0125\u00ab": 46948, + "\u0120Weston": 46949, + "\u0120Volvo": 46950, + "870": 46951, + "\u0120microphones": 46952, + "verbal": 46953, + "RPG": 46954, + "\u0120iii": 46955, + ";}": 46956, + "028": 46957, + "\u0120headlined": 46958, + "\u0120primed": 46959, + "\u0120hoard": 46960, + "\u0120Shad": 46961, + "\u0120ENTER": 46962, + "\u0120triangular": 46963, + "\u0120capit": 46964, + "lik": 46965, + "\u0120Ancients": 46966, + "\u0120lash": 46967, + "\u0120convol": 46968, + "\u0120colonel": 46969, + "enemy": 46970, + "Gra": 46971, + "\u0120pubs": 46972, + "utters": 46973, + "\u0120assigns": 46974, + "\u0120Penet": 46975, + "\u0120Monstrous": 46976, + "\u0120Bowen": 46977, + "ilver": 46978, + "Haunted": 46979, + "\u0120Ding": 46980, + "started": 46981, + "plin": 46982, + "\u0120contaminants": 46983, + "\u0120DOE": 46984, + "ffen": 46985, + "\u0120Technician": 46986, + "Ry": 46987, + "\u0120robbers": 46988, + "\u0120hotline": 46989, + "\u0120Guardiola": 46990, + "\u0120Kaufman": 46991, + "rower": 46992, + "\u0120Dresden": 46993, + "\u0120Alpine": 46994, + "Elf": 46995, + "\u0120fmt": 46996, + "\u0120Sard": 46997, + "urses": 46998, + "gpu": 46999, + "Unix": 47000, + "\u0120unequivocally": 47001, + "\u0120Citizenship": 47002, + "quad": 47003, + "mire": 47004, + "\u0120Sweeney": 47005, + "Battery": 47006, + "615": 47007, + "\u0120pancakes": 47008, + "\u0120oats": 47009, + "Maps": 47010, + "\u0120Contrast": 47011, + "mbudsman": 47012, + "\u0120EPS": 47013, + "\u0120subcommittee": 47014, + "\u0120sourcing": 47015, + "\u0120sizing": 47016, + "\u0120Buffer": 47017, + "\u0120Mandatory": 47018, + "\u0120moderates": 47019, + "\u0120Patterns": 47020, + "\u0120Chocobo": 47021, + "\u0120Zan": 47022, + "\u0120STATES": 47023, + "\u0120Judging": 47024, + "\u0120Inher": 47025, + "*:": 47026, + "\u0120bil": 47027, + "\u0120Yen": 47028, + "\u0120exhilar": 47029, + "ollower": 47030, + "zers": 47031, + "\u0120snug": 47032, + "maximum": 47033, + "\u0120despicable": 47034, + "\u0120PACK": 47035, + "\u0120Annex": 47036, + "\u0120sarcastic": 47037, + "\u0120latex": 47038, + "\u0120tamp": 47039, + "\u0120Sao": 47040, + "bah": 47041, + "\u0120Reverend": 47042, + "\u0120Chinatown": 47043, + "\u0120AUT": 47044, + "documented": 47045, + "\u0120GABA": 47046, + "\u0120Canaan": 47047, + "\u0120\u00d9\u0127": 47048, + "\u0120governs": 47049, + "prev": 47050, + "Esc": 47051, + "\u0120Estimates": 47052, + "OSP": 47053, + "\u0120endeavour": 47054, + "\u0120Closing": 47055, + "ometime": 47056, + "everyone": 47057, + "\u0120worsen": 47058, + "\u0120scanners": 47059, + "\u0120deviations": 47060, + "\u0120Robotics": 47061, + "\u0120Compton": 47062, + "\u0120sorcerer": 47063, + "\u0120endogenous": 47064, + "\u0120emulation": 47065, + "\u0120Piercing": 47066, + "\u0120Aph": 47067, + "\u0120Socket": 47068, + "\u0120bould": 47069, + "\u0120OU": 47070, + "\u0120Borderlands": 47071, + "\u01201863": 47072, + "Gordon": 47073, + "\u0120WTO": 47074, + "\u0120restricts": 47075, + "\u0120mosaic": 47076, + "\u0120melodies": 47077, + "\u00e7\u0126": 47078, + "Tar": 47079, + "\u0120disson": 47080, + "\u0120Provides": 47081, + "\u0120......": 47082, + "bek": 47083, + "FIX": 47084, + "\u0120broom": 47085, + "anship": 47086, + "Doctors": 47087, + "\u0120nerds": 47088, + "\u0120Regions": 47089, + "naissance": 47090, + "\u0120mete": 47091, + "\u0120crept": 47092, + "plings": 47093, + "\u0120girlfriends": 47094, + "knit": 47095, + "igent": 47096, + "owe": 47097, + "\u0120ushered": 47098, + "\u0120Baz": 47099, + "Mobil": 47100, + "434": 47101, + "\u0120Presents": 47102, + "origin": 47103, + "\u0120insomnia": 47104, + "\u0120Aux": 47105, + "439": 47106, + "\u0120Chili": 47107, + "irsch": 47108, + "GAME": 47109, + "\u0120gestation": 47110, + "algia": 47111, + "romising": 47112, + "$,": 47113, + "crow": 47114, + "\u0120Inspection": 47115, + "atomic": 47116, + "Relations": 47117, + "JOHN": 47118, + "roman": 47119, + "\u0120Clockwork": 47120, + "\u0120Bakr": 47121, + "mone": 47122, + "MET": 47123, + "\u0120thirsty": 47124, + "\u0120bc": 47125, + "\u0120faculties": 47126, + "Rum": 47127, + "\u0120nuance": 47128, + "\u0120Darius": 47129, + "pleting": 47130, + "fters": 47131, + "etchup": 47132, + "Registration": 47133, + "\u0120KE": 47134, + "Rah": 47135, + "\u0120preferential": 47136, + "\u0120Lash": 47137, + "\u0120HH": 47138, + "Valid": 47139, + "\u0120NAV": 47140, + "\u0120starve": 47141, + "\u0120Gong": 47142, + "zynski": 47143, + "\u0120Actress": 47144, + "\u0120wik": 47145, + "\u0120unaccompanied": 47146, + "lvl": 47147, + "Bride": 47148, + "ADS": 47149, + "\u0120Commando": 47150, + "\u0120Vaughn": 47151, + "Wallet": 47152, + "\u0120hopping": 47153, + "\u0120Vie": 47154, + "\u0120caveats": 47155, + "\u0120alas": 47156, + "ifled": 47157, + "abuse": 47158, + "661": 47159, + "\u0120ibn": 47160, + "\u0120gul": 47161, + "\u0120robbing": 47162, + "til": 47163, + "ILA": 47164, + "\u0120mitigating": 47165, + "\u0120aptly": 47166, + "\u0120tyrant": 47167, + "\u0120midday": 47168, + "\u0120Gilmore": 47169, + "\u0120Decker": 47170, + "\u0120\u00c2\u00a7\u00c2\u00a7": 47171, + "partial": 47172, + "Exactly": 47173, + "\u0120phenotype": 47174, + "\u0120[+]": 47175, + "\u0120Plex": 47176, + "\u0120Ips": 47177, + "versions": 47178, + "\u0120ebook": 47179, + "\u0120chic": 47180, + "gross": 47181, + "\":\"\"},{\"": 47182, + "\u0120Surprisingly": 47183, + "Morgan": 47184, + "\u0120residues": 47185, + "\u0120Confederation": 47186, + "infeld": 47187, + "\u0120lyr": 47188, + "moderate": 47189, + "\u0120perpendicular": 47190, + "VK": 47191, + "\u0120synchronized": 47192, + "\u0120refreshed": 47193, + "\u0120adore": 47194, + "\u0120Torment": 47195, + "olina": 47196, + "\u01202600": 47197, + "ItemTracker": 47198, + "\u0120pies": 47199, + "\u0120FAT": 47200, + "\u0120RHP": 47201, + "048": 47202, + "\u0120RESP": 47203, + "\u0120BJ": 47204, + "allows": 47205, + "Pand": 47206, + "\u0120unwelcome": 47207, + "\u0120Voc": 47208, + "\u0120Bastard": 47209, + "\u0120OW": 47210, + "\u0120LAR": 47211, + "\u0120Healer": 47212, + "Environmental": 47213, + "\u0120Kenyan": 47214, + "\u0120Trance": 47215, + "\u0120Pats": 47216, + "\u0120aliases": 47217, + "\u0120Garfield": 47218, + "\u0120campaigner": 47219, + "\u0120advancements": 47220, + "\u0120Okinawa": 47221, + "\u0120Coh": 47222, + "owsky": 47223, + "\u0120starved": 47224, + "\u0120sizeable": 47225, + "\u0120:-)": 47226, + "\u0120mRNA": 47227, + "\u0120suspensions": 47228, + "istar": 47229, + "Scotland": 47230, + "Prin": 47231, + "------------------------------------------------": 47232, + "\u0120502": 47233, + "\u0120teaspoons": 47234, + "\u01201050": 47235, + "\u0120coercive": 47236, + "\u0120Masonic": 47237, + "edded": 47238, + "\u0120Passenger": 47239, + "\u0120latt": 47240, + "\u0120braces": 47241, + "\u0120Steal": 47242, + "\u0120NYT": 47243, + "\u0120Kats": 47244, + "\u0120Celest": 47245, + "aez": 47246, + "Tu": 47247, + "\u0120Coulter": 47248, + "\u00f0\u0141\u013a": 47249, + "Flickr": 47250, + "\u0120Wilmington": 47251, + "iths": 47252, + "++;": 47253, + "\u0120vending": 47254, + "\u0120negro": 47255, + "\u0120Phi": 47256, + "\u0120Yellowstone": 47257, + "Callback": 47258, + "\u0120shampoo": 47259, + "\u0120Shades": 47260, + "wat": 47261, + "\u0120superhuman": 47262, + "\u0120ridiculed": 47263, + "\u0120holiest": 47264, + "ombo": 47265, + "\u0120interns": 47266, + "\u0120hone": 47267, + "\u0120Paragu": 47268, + "URI": 47269, + "\u0120dangling": 47270, + "\u00e3\u0124\u00bb": 47271, + "sov": 47272, + "ictional": 47273, + "availability": 47274, + "\u0120revocation": 47275, + "\u0120dow": 47276, + "inic": 47277, + "\u0120THEIR": 47278, + "\u0120iso": 47279, + "\u0120outings": 47280, + "\u0120Lethal": 47281, + "\u0120)))": 47282, + "\u0120inaccur": 47283, + "\u0120outlandish": 47284, + "\u0120anus": 47285, + "letico": 47286, + "idon": 47287, + "lol": 47288, + "\u0120unregulated": 47289, + "\u0120succumbed": 47290, + "\u0120cuff": 47291, + "\u0120Wasteland": 47292, + "letal": 47293, + "\u0120substr": 47294, + "\u0120coffers": 47295, + "\u0120automakers": 47296, + "ovi": 47297, + "\u0120Xue": 47298, + "\u0120Daytona": 47299, + "\u0120jarring": 47300, + "\u0120fumes": 47301, + "\u0120disbanded": 47302, + "zik": 47303, + "itton": 47304, + "\u0120strikingly": 47305, + "\u0120spores": 47306, + "Adapter": 47307, + ".):": 47308, + "\u0120Lyndon": 47309, + "ivalry": 47310, + "\u0120orally": 47311, + "\u0120tumultuous": 47312, + "\u0120displeasure": 47313, + "\u0120cones": 47314, + "orrect": 47315, + "\u0120appease": 47316, + "\u0120derby": 47317, + "\u0120Tripoli": 47318, + "\u0120Aless": 47319, + "\u0120poked": 47320, + "\u0120Guilty": 47321, + "vP": 47322, + "Enough": 47323, + "\u0120originals": 47324, + "699": 47325, + "\u0120rabbi": 47326, + "\u0120proverbial": 47327, + "\u0120postpone": 47328, + "elope": 47329, + "\u0120Misty": 47330, + "\u0120staffed": 47331, + "\u0120Unemployment": 47332, + "reditary": 47333, + "\u0120diligent": 47334, + "recomm": 47335, + "measures": 47336, + "asin": 47337, + "825": 47338, + "\u0120ponds": 47339, + "\u0120mmol": 47340, + "\u0120SAR": 47341, + "\u0120CARE": 47342, + "\u0120371": 47343, + "\u0120clenched": 47344, + "\u0120Corsair": 47345, + "\u0120caricature": 47346, + "zn": 47347, + "attach": 47348, + "\u0120Schro": 47349, + "speak": 47350, + "painted": 47351, + "\u0120Suc": 47352, + "\u0120ENT": 47353, + "\u0120cellul": 47354, + "\u0120Paid": 47355, + "diagn": 47356, + "WHERE": 47357, + "\u0120texted": 47358, + "Barn": 47359, + "\u0120retracted": 47360, + "\u0120Referred": 47361, + "Sav": 47362, + "\u0120upkeep": 47363, + "\u0120workplaces": 47364, + "\u0120Tokens": 47365, + "\u0120amplify": 47366, + "clinical": 47367, + "\u0120multic": 47368, + "mberg": 47369, + "\u0120convoluted": 47370, + "Region": 47371, + "565": 47372, + "\u0120Topic": 47373, + "\u0120snail": 47374, + "\u0120saline": 47375, + "\u0120insurrection": 47376, + "\u0120Petr": 47377, + "forts": 47378, + "BAT": 47379, + "\u0120Navajo": 47380, + "\u0120rudimentary": 47381, + "\u0120Laksh": 47382, + "ONDON": 47383, + "Measure": 47384, + "\u0120transformer": 47385, + "\u0120Goddard": 47386, + "\u0120coincides": 47387, + "irin": 47388, + "Rex": 47389, + "\u0120Bok": 47390, + "quit": 47391, + "\u0120shotguns": 47392, + "\u0120proletarian": 47393, + "\u0120scorp": 47394, + "\u0120Ada": 47395, + "514": 47396, + "\u0120slander": 47397, + "recorded": 47398, + "\u0120embell": 47399, + "risome": 47400, + "\u0120apologizing": 47401, + "\u0120Mulcair": 47402, + "\u0120Gibraltar": 47403, + "Cla": 47404, + "\u0120allot": 47405, + "\u0120Attention": 47406, + "\u0120433": 47407, + "leave": 47408, + "\u0120whine": 47409, + "\u0120Issa": 47410, + "\u0120Faust": 47411, + "\u0120Barron": 47412, + "heny": 47413, + "\u0120victimized": 47414, + "Jews": 47415, + "\u0120nurturing": 47416, + "ettel": 47417, + "Winged": 47418, + "\u0120Subtle": 47419, + "\u0120flavorful": 47420, + "\u0120Reps": 47421, + "enged": 47422, + "callback": 47423, + "\u0120directional": 47424, + "\u0120clasp": 47425, + "\u0120Directions": 47426, + "planet": 47427, + "iculture": 47428, + "Helper": 47429, + "icion": 47430, + "acia": 47431, + "\u0120\u00e7\u00a5\u0140": 47432, + "\u0120surges": 47433, + "\u0120canoe": 47434, + "\u0120Premiership": 47435, + "been": 47436, + "\u0120defied": 47437, + "\u0120Trooper": 47438, + "\u0120tripod": 47439, + "\u0120gasp": 47440, + "\u0120Euph": 47441, + "\u0120Ads": 47442, + "vernight": 47443, + "highly": 47444, + "Role": 47445, + "\u0120entangled": 47446, + "\u0120Zeit": 47447, + "618": 47448, + "\u0120Rusty": 47449, + "\u0120havens": 47450, + "\u0120Vaughan": 47451, + "HAEL": 47452, + "\u0120SERVICE": 47453, + "/,": 47454, + "\u0120stricken": 47455, + "\u0120delusions": 47456, + "\u0120bis": 47457, + "\u0120Haf": 47458, + "\u0120gratification": 47459, + "\u0120enticing": 47460, + "UNCH": 47461, + "Adams": 47462, + "\u0120OLED": 47463, + "\u0120Beetle": 47464, + "\u01201899": 47465, + "\u0120SOFTWARE": 47466, + "ategor": 47467, + "VL": 47468, + "\u0120Totem": 47469, + "\u0120Gators": 47470, + "ATURES": 47471, + "\u0120impedance": 47472, + "Registered": 47473, + "\u0120Cary": 47474, + "\u0120Aerial": 47475, + "onne": 47476, + "enium": 47477, + "\u0120dred": 47478, + "\u0120Beg": 47479, + "\u0120concurrently": 47480, + "\u0120superpower": 47481, + "\u0120Xan": 47482, + "jew": 47483, + "imester": 47484, + "\u0120Dickinson": 47485, + "\u00e2\u0136\u0123": 47486, + "Fla": 47487, + "\u0120pree": 47488, + "\u0120Rollins": 47489, + "\u00a9\u00b6\u00e6": 47490, + "\u0120denomination": 47491, + "\u0120Lana": 47492, + "516": 47493, + "\u0120inciting": 47494, + "scribed": 47495, + "juries": 47496, + "\u0120Wonders": 47497, + "approximately": 47498, + "\u0120suspending": 47499, + "\u0120mountainous": 47500, + "\u0120Laugh": 47501, + "oidal": 47502, + "Ns": 47503, + "Detect": 47504, + ")=": 47505, + "\u0120Luthor": 47506, + "\u0120Schwarzenegger": 47507, + "\u0120Muller": 47508, + "\u0120Devi": 47509, + "ecycle": 47510, + "Jar": 47511, + "613": 47512, + "\u0120Longh": 47513, + "Bah": 47514, + "\u0120SPORTS": 47515, + "nw": 47516, + "\u0120refinement": 47517, + "\u0120waterways": 47518, + "\u0120diner": 47519, + "Blade": 47520, + "683": 47521, + "Fac": 47522, + "\u0120initials": 47523, + "\u0120rog": 47524, + "\u0120paranormal": 47525, + "BUT": 47526, + "\u0120[(": 47527, + "\u0120Swanson": 47528, + "\u0120Mesh": 47529, + "\u00e2\u0138\u00ac": 47530, + "Improve": 47531, + "\u0120Radiation": 47532, + "\u0120Esther": 47533, + "\u0120Esk": 47534, + "\u0120Aly": 47535, + "iky": 47536, + "\u0120irrad": 47537, + "\u0120Buckingham": 47538, + "\u0120refill": 47539, + "\u0120._": 47540, + "Repe": 47541, + "CONCLUS": 47542, + "\u0120differentiated": 47543, + "\u0120chirop": 47544, + "\u0120Atkins": 47545, + "Pattern": 47546, + "\u0120excise": 47547, + "\u0120cabal": 47548, + "NSA": 47549, + "\u0120STA": 47550, + "\u0120SIL": 47551, + "\u0120Paraly": 47552, + "\u0120rye": 47553, + "\u0120Howell": 47554, + "\u0120Countdown": 47555, + "nesses": 47556, + "alysed": 47557, + "\u0120resize": 47558, + "\u00e3\u0124\u00bd": 47559, + "\u0120budgetary": 47560, + "\u0120Stras": 47561, + "wang": 47562, + "\u0120apiece": 47563, + "\u0120precincts": 47564, + "\u0120peach": 47565, + "\u0120skyline": 47566, + "\u0120353": 47567, + "popular": 47568, + "Appearances": 47569, + "\u0120Mechanics": 47570, + "\u0120DevOnline": 47571, + "Sullivan": 47572, + "Zen": 47573, + "\u0120pu": 47574, + "opolis": 47575, + "544": 47576, + "\u0120deform": 47577, + "\u0120counteract": 47578, + "\u0120Lange": 47579, + "\u0120417": 47580, + "Console": 47581, + "774": 47582, + "\u0120nodding": 47583, + "\u0120populism": 47584, + "\u0120hep": 47585, + "\u0120counselling": 47586, + "compliance": 47587, + "UFF": 47588, + "\u0120undeniably": 47589, + "\u0120railing": 47590, + "\u0120Horowitz": 47591, + "\u0120Simone": 47592, + "\u0120Bungie": 47593, + "\u0120ak": 47594, + "\u0120Talks": 47595, + "xff": 47596, + "flake": 47597, + "Crash": 47598, + "\u0120sweaty": 47599, + "\u0120banquet": 47600, + "\u0120OFFIC": 47601, + "\u0120inventive": 47602, + "\u0120astronomer": 47603, + "\u0120Stamford": 47604, + "\u0120Scare": 47605, + "\u0120GREEN": 47606, + "olicited": 47607, + "\u0120rusher": 47608, + "\u0120centrist": 47609, + "ighting": 47610, + "\u0120subclass": 47611, + "\u0120disav": 47612, + "\u0120defund": 47613, + "\u0120Nanto": 47614, + "ociate": 47615, + "mast": 47616, + "\u0120pacif": 47617, + "\u0120mend": 47618, + "eers": 47619, + "immigration": 47620, + "ESSION": 47621, + "\u0120numbering": 47622, + "\u0120laughable": 47623, + "\u0120Ended": 47624, + "viation": 47625, + "emark": 47626, + "Pitt": 47627, + "\u0120meticulous": 47628, + "\u0120LF": 47629, + "\u0120congratulated": 47630, + "\u0120Birch": 47631, + "\u0120swayed": 47632, + "\u0120semifinals": 47633, + "\u0120humankind": 47634, + "matter": 47635, + "\u0120Equip": 47636, + "opausal": 47637, + "Said": 47638, + "\u0120Layout": 47639, + "\u0120voicing": 47640, + "\u0120thug": 47641, + "\u0120pornographic": 47642, + "IPS": 47643, + "\u0120moaning": 47644, + "\u0120grievance": 47645, + "\u0120confessions": 47646, + "escal": 47647, + "TEXTURE": 47648, + "Authent": 47649, + "osaurus": 47650, + "Purchase": 47651, + "\u0120relegation": 47652, + "alter": 47653, + "\u0120\u00c2\u0142\u00c2\u0142": 47654, + "\u0120riddled": 47655, + "\u0120ogre": 47656, + "\u0120Lowell": 47657, + "Occup": 47658, + "Eat": 47659, + "\u0120Hyder": 47660, + "\u0120Adviser": 47661, + "Commerce": 47662, + "Hunt": 47663, + "\u0120Orth": 47664, + "\u0120Competitive": 47665, + "\u0120CLA": 47666, + "CDC": 47667, + "\u0120salads": 47668, + "Fle": 47669, + "\u0120industrialized": 47670, + "`,": 47671, + "\u0120OWN": 47672, + "\u0120beck": 47673, + "\u0120Particularly": 47674, + "oubt": 47675, + "\u0120mM": 47676, + "\u0120Hussain": 47677, + "\u0120Chennai": 47678, + "\u0120920": 47679, + "\u0120appointing": 47680, + "\u0120Cullen": 47681, + ",,,,,,,,": 47682, + "\u0120pores": 47683, + "verified": 47684, + "\u0120biochemical": 47685, + "emate": 47686, + "\u0120cowardly": 47687, + "\u0120Helsinki": 47688, + "\u0120Ethiopian": 47689, + "SOURCE": 47690, + "ERC": 47691, + "estro": 47692, + "\u0120biotech": 47693, + "\u0120Sour": 47694, + "\u0120brewer": 47695, + "Bloomberg": 47696, + "\u0120intensify": 47697, + "Glass": 47698, + "anco": 47699, + "\u0120FDR": 47700, + "greSQL": 47701, + "\u0120Fires": 47702, + "\u00a9\u00b6\u00e6\u00a5\u00b5": 47703, + "eco": 47704, + "1001": 47705, + "\u0120Homeless": 47706, + "\u0120instantaneous": 47707, + "\u0120Haste": 47708, + "igel": 47709, + "Diamond": 47710, + "\u0120paving": 47711, + "\u0120landfill": 47712, + "\u0120dads": 47713, + "houn": 47714, + ":]": 47715, + "\u0120incendiary": 47716, + "\u0120Livingston": 47717, + "\u0120Hilbert": 47718, + "\u0120Checks": 47719, + "styles": 47720, + "inators": 47721, + "\u0120Clive": 47722, + "phrine": 47723, + "\u0120chimpanzees": 47724, + "\u0120pall": 47725, + "\u0120JM": 47726, + "\u0120Aadhaar": 47727, + "\u00f0\u013f": 47728, + "\u0120achievable": 47729, + "disabled": 47730, + "PET": 47731, + "OOOOOOOO": 47732, + "Mot": 47733, + "\u0120intangible": 47734, + "\u0120ballet": 47735, + "\u0120Webs": 47736, + "\u0120Estimated": 47737, + "Effects": 47738, + "\u0120bailed": 47739, + "Joshua": 47740, + "\u0120turbulence": 47741, + "\u0120occupant": 47742, + "\u0120Daylight": 47743, + "\u0120361": 47744, + "meet": 47745, + "\u0120statically": 47746, + "\u0120onlook": 47747, + "\u0120ki": 47748, + "illegal": 47749, + "\u0120velvet": 47750, + "\u0120dehydration": 47751, + "\u0120acquies": 47752, + "\u0120Rez": 47753, + "akura": 47754, + "\u0120Upton": 47755, + "atro": 47756, + "\u0120incomprehensible": 47757, + "\u0120backdoor": 47758, + "\u0120Rhino": 47759, + "727": 47760, + "\u0120maths": 47761, + ")+": 47762, + "\u0120heresy": 47763, + "\u0120df": 47764, + "\u0120Roche": 47765, + "\u0120Lydia": 47766, + "\u0120pancreat": 47767, + "reply": 47768, + "arrell": 47769, + "\u0120solicitation": 47770, + "\u0120circadian": 47771, + "BIP": 47772, + "\u0120foray": 47773, + "\u0120cryptic": 47774, + "izu": 47775, + "imeo": 47776, + "\u0120Tomato": 47777, + "\u0120Homs": 47778, + "examination": 47779, + "\u0120quarry": 47780, + "\u0120Valiant": 47781, + "\u0120Jericho": 47782, + "\u0120INCLUD": 47783, + "\u01201840": 47784, + "519": 47785, + "\u0120resists": 47786, + "\u0120snapshots": 47787, + "\u0120Spur": 47788, + "\u0120Antiqu": 47789, + "Login": 47790, + "\u0120bestselling": 47791, + "\u0120antic": 47792, + "\u0120Sutherland": 47793, + "\u00e3\u0124\u00a2\u00e3\u0125\u00ab": 47794, + "\u0120~/": 47795, + "\u0120Parm": 47796, + "\u00e8\u0125": 47797, + "Pages": 47798, + "intensity": 47799, + "\u0120immobil": 47800, + "\u01201865": 47801, + "zzo": 47802, + "\u0120nifty": 47803, + "\u0120fentanyl": 47804, + "\u0120Preservation": 47805, + "ophen": 47806, + "\u0120darts": 47807, + "\u0120Dinosaur": 47808, + "pointers": 47809, + "\u0120Rite": 47810, + "suggest": 47811, + "awareness": 47812, + "\u0120Sheridan": 47813, + "\u0120stances": 47814, + "\u0120sorcery": 47815, + "\u0120perjury": 47816, + "\u0120Nikola": 47817, + "iever": 47818, + "\u0120fiance": 47819, + "\u0120Jordanian": 47820, + "\u0120Balloon": 47821, + "\u0120nab": 47822, + "\u0120kb": 47823, + "\u0120humanities": 47824, + "\u0120Tanaka": 47825, + "hillary": 47826, + "\u0120consultancy": 47827, + "\u0120Zub": 47828, + "\u0120remission": 47829, + "\u0120confid": 47830, + "CHQ": 47831, + "\u0120Fug": 47832, + "\u0120improvis": 47833, + "Yep": 47834, + "/_": 47835, + "\u0120unwillingness": 47836, + "\u0120portfolios": 47837, + "055": 47838, + "\u0120Instructor": 47839, + "aiman": 47840, + "\u0120claimants": 47841, + "Mbps": 47842, + "\u0120Bye": 47843, + "received": 47844, + "Tweet": 47845, + "\u0120indemn": 47846, + "riz": 47847, + "amara": 47848, + "Nat": 47849, + "\u0120evaluates": 47850, + "\u0120Lur": 47851, + "epad": 47852, + "FOX": 47853, + "\u0120Thro": 47854, + "\u0120rusty": 47855, + "\u0120bedrock": 47856, + "\u0120Oprah": 47857, + "JB": 47858, + "\u0120manipulative": 47859, + "\u0120willful": 47860, + "\u0120relapse": 47861, + "\u0120extant": 47862, + "Theme": 47863, + "Sensor": 47864, + "\u0120Stability": 47865, + "govern": 47866, + "\u0120poppy": 47867, + "\u0120knack": 47868, + "\u0120insulated": 47869, + "\u0120Tile": 47870, + "\u0120Extrem": 47871, + "\u0120untold": 47872, + "\u0120converge": 47873, + "\u0120refuel": 47874, + "igroup": 47875, + "\u0120distortions": 47876, + "\u0120ravaged": 47877, + "\u0120mechanically": 47878, + "\u0120Reilly": 47879, + "\u0120Nose": 47880, + "\u0120Incarnation": 47881, + "\u0120Becky": 47882, + "abbling": 47883, + "\u0120taco": 47884, + "\u0120rake": 47885, + "\u0120melancholy": 47886, + "\u0120illustrious": 47887, + "\u0120Dartmouth": 47888, + "Guide": 47889, + "\u0120Razer": 47890, + "\u0120Benz": 47891, + "Ultimate": 47892, + "\u0120Surprise": 47893, + "\u0120pageant": 47894, + "offer": 47895, + "Whoever": 47896, + "\u0120wiser": 47897, + "\u0120chemist": 47898, + "\u0120HELL": 47899, + "\u0120Bulk": 47900, + "\u0120plutonium": 47901, + "\u0120COVER": 47902, + "\u00d6\u00bc": 47903, + "failed": 47904, + "\u0120tirelessly": 47905, + "\u0120infertility": 47906, + "\u0120Trident": 47907, + "\u0120Showtime": 47908, + "\u0120Civ": 47909, + "Vice": 47910, + "requires": 47911, + "ittance": 47912, + "\u0120uncontrolled": 47913, + "interesting": 47914, + "561": 47915, + "\u0120innovate": 47916, + "ategic": 47917, + "Lie": 47918, + "\u0120Selling": 47919, + "Ul": 47920, + "\u0120savior": 47921, + "\u0120Tosh": 47922, + "\u0120swast": 47923, + "PASS": 47924, + "\u0120rink": 47925, + "\u0120cardio": 47926, + "\u0120Iro": 47927, + "udi": 47928, + "\u0120vantage": 47929, + "\u0120vans": 47930, + "\u0120Ni\u00c3\u00b1o": 47931, + "+=": 47932, + "\u0120propagate": 47933, + "": 49029, + "\u0120leukemia": 49030, + "\u0120eluc": 49031, + "\u0120announcer": 49032, + "\u0120Lithuan": 49033, + "\u0120Armageddon": 49034, + "\u00e5\u0129": 49035, + "Lenin": 49036, + "\u0120Ruk": 49037, + "\u0120pepp": 49038, + "\u0120Romantic": 49039, + "\u0120PIT": 49040, + "\u0120Interstellar": 49041, + "\u0120Atkinson": 49042, + "Raid": 49043, + "Js": 49044, + "Goal": 49045, + "Course": 49046, + "\u0120vanishing": 49047, + "esley": 49048, + "\u0120Rounds": 49049, + "Elsa": 49050, + "593": 49051, + "\u0120redundancy": 49052, + "\u0120STAND": 49053, + "\u0120prophetic": 49054, + "\u0120habitable": 49055, + "ryu": 49056, + "\u0120faintly": 49057, + "MODE": 49058, + "\u0120flanked": 49059, + "IRC": 49060, + "Awesome": 49061, + "\u0120spurious": 49062, + "\u0120Zah": 49063, + "\u0120MSG": 49064, + "\u0120shading": 49065, + "\u0120motivational": 49066, + "\u0120Santana": 49067, + "\u0120SPR": 49068, + "\u0120excruciating": 49069, + "omial": 49070, + "\u0120Miko": 49071, + "\u0120Leopard": 49072, + "Abyss": 49073, + "\u0120[|": 49074, + "dirty": 49075, + "\u0120baths": 49076, + "\u0120demoral": 49077, + "andre": 49078, + "PB": 49079, + "\u0120unification": 49080, + "\u0120sacrament": 49081, + "\u0120[&": 49082, + "\u0120priceless": 49083, + "\u0120gelatin": 49084, + "\u0120emanating": 49085, + "\u0120Allaah": 49086, + "986": 49087, + "\u0120outburst": 49088, + "\u0120eras": 49089, + "\u0120XVI": 49090, + "\u0120SPI": 49091, + "Ott": 49092, + "\u0120Lazarus": 49093, + "PLIED": 49094, + "Flying": 49095, + "blogs": 49096, + "Wisconsin": 49097, + "Raven": 49098, + "\u0120rebate": 49099, + "\u0120creeps": 49100, + "\u0120Span": 49101, + "\u0120Painter": 49102, + "\u0120Kira": 49103, + "\u0120Amos": 49104, + "\u0120Corvette": 49105, + "Consumer": 49106, + "\u0120Recover": 49107, + "cki": 49108, + "\u0120pesky": 49109, + "\u0120Invention": 49110, + "Companies": 49111, + "\u0120challengers": 49112, + "ademic": 49113, + "\u0120Ukrainians": 49114, + "\u0120Neurolog": 49115, + "\u0120Forsaken": 49116, + "\u0120entrants": 49117, + "\u0120embattled": 49118, + "\u0120defunct": 49119, + "\u0120Glacier": 49120, + "\u0120poisons": 49121, + "\u0120Horses": 49122, + "makes": 49123, + "\u0120Dirt": 49124, + "\u0120423": 49125, + "hhh": 49126, + "\u0120Transformation": 49127, + "QUIRE": 49128, + "..................": 49129, + "\u0120traveller": 49130, + "\u0120Sexy": 49131, + "\u0120Kern": 49132, + "ipolar": 49133, + "\u0120ransomware": 49134, + "oooooooooooooooo": 49135, + "Ec": 49136, + "ruby": 49137, + "Professional": 49138, + "\u0120Outbreak": 49139, + "argument": 49140, + "Grey": 49141, + "\u0120Fifa": 49142, + "\u0120CHO": 49143, + "\u0120FORM": 49144, + "\u0120Amtrak": 49145, + "-[": 49146, + "\u0120cradle": 49147, + "\u0120antioxidants": 49148, + "\u00e3\u0123\u00ae\u00e5\u00ae": 49149, + "736": 49150, + "\u0120NASL": 49151, + "\u0120Contributions": 49152, + "Indiana": 49153, + "\u0120STEP": 49154, + "CSS": 49155, + "\u0120salient": 49156, + "\u0120allocations": 49157, + "yrights": 49158, + "\u0120mashed": 49159, + "\u0120Cutter": 49160, + "Sexual": 49161, + "\u0120pounded": 49162, + "\u0120fanbase": 49163, + "\u0120casc": 49164, + "\u0120Transparency": 49165, + "\u0120analytic": 49166, + "\u0120Summoner": 49167, + "\u00d7\u0140": 49168, + "\u0120ADC": 49169, + "detail": 49170, + "\u0120vanquished": 49171, + "\u0120crabs": 49172, + "arie": 49173, + "Destroy": 49174, + "\u0120Sack": 49175, + "\u0120transistor": 49176, + "Alabama": 49177, + "\u0120Koen": 49178, + "\u0120Fisheries": 49179, + "cone": 49180, + "\u0120annexed": 49181, + "\u0120MGM": 49182, + "esa": 49183, + "\u0120faked": 49184, + "\u0120Congratulations": 49185, + "\u0120hindered": 49186, + "\u0120correctional": 49187, + "\u0120ITV": 49188, + "leeve": 49189, + "\u0120inappropriately": 49190, + "licks": 49191, + "\u0120trespass": 49192, + "\u0120paws": 49193, + "\u0120negotiator": 49194, + "\u0120Christensen": 49195, + "limits": 49196, + "\u0120Dianne": 49197, + "\u0120elegance": 49198, + "\u0120Contracts": 49199, + "anke": 49200, + "Obj": 49201, + "\u0120vigilance": 49202, + "\u0120castles": 49203, + "\u0120NAD": 49204, + "\u0120Holo": 49205, + "\u0120emphatically": 49206, + "\u0120Titus": 49207, + "\u0120Serving": 49208, + "\u0120Richie": 49209, + "\u0120Pigs": 49210, + "568": 49211, + "\u0120animosity": 49212, + "\u0120Attributes": 49213, + "\u0120Uriel": 49214, + "MQ": 49215, + "myra": 49216, + "\u0120Applicant": 49217, + "\u0120psychiatrists": 49218, + "\u0120Vij": 49219, + "\u0120Abby": 49220, + "agree": 49221, + "Push": 49222, + "\u0120kWh": 49223, + "hiba": 49224, + "\u0120incite": 49225, + "\u0120Weasley": 49226, + "\u0120Taxi": 49227, + "ministic": 49228, + "hyper": 49229, + "\u0120Farn": 49230, + "\u0120601": 49231, + "\u0120Nationwide": 49232, + "Fake": 49233, + "952": 49234, + "\u0120maize": 49235, + "\u0120interacted": 49236, + "\u0120transitioned": 49237, + "\u0120parasitic": 49238, + "\u0120harmonic": 49239, + "\u0120decaying": 49240, + "\u0120baseless": 49241, + "nsics": 49242, + "\u0120transpired": 49243, + "\u0120abundantly": 49244, + "\u0120Forensic": 49245, + "\u0120treadmill": 49246, + "\u0120Jav": 49247, + "aband": 49248, + "\u0120sshd": 49249, + "\u0120frontman": 49250, + "\u0120Jakarta": 49251, + "oller": 49252, + "drops": 49253, + "\u0120SERVICES": 49254, + "romptu": 49255, + "ophical": 49256, + "hospital": 49257, + "bledon": 49258, + "645": 49259, + "\u0120midrange": 49260, + "\u0120EVENT": 49261, + "culated": 49262, + "rawled": 49263, + "\u0120perched": 49264, + "\u0120overboard": 49265, + "\u0120Peel": 49266, + "\u0120Pwr": 49267, + "\u0120Carth": 49268, + "\u0120COMPLE": 49269, + "coe": 49270, + "shall": 49271, + "\u0120deterrence": 49272, + "METHOD": 49273, + "\u0120Absent": 49274, + "MEN": 49275, + "\u0120sill": 49276, + "\u0120LEVEL": 49277, + "York": 49278, + "\u0120sinners": 49279, + "\u0120OPEC": 49280, + "\u0120Nur": 49281, + "\u0120Designs": 49282, + "selection": 49283, + "\u0120unworthy": 49284, + "CHA": 49285, + "\u0120strengthens": 49286, + "883": 49287, + "edly": 49288, + "\u0120slicing": 49289, + "\u0120malnutrition": 49290, + "\u0120filmmaking": 49291, + "\u0120Polk": 49292, + "urated": 49293, + "\u0120421": 49294, + "breakers": 49295, + "!'\"": 49296, + "\u0120wetlands": 49297, + "\u0120Discrimination": 49298, + "\u0120allowable": 49299, + "\u0120steered": 49300, + "\u0120Sicily": 49301, + "SAM": 49302, + "\u0120mustache": 49303, + "\u0120mids": 49304, + "\u0120clipped": 49305, + "\u0120circulate": 49306, + "\u0120brittle": 49307, + "\u0120Buildings": 49308, + "raised": 49309, + "\u0120Roundup": 49310, + "\u0120wealthier": 49311, + "\u0120overwrite": 49312, + "\u0120overpowered": 49313, + "\u0120Gerrard": 49314, + "sites": 49315, + "PDATED": 49316, + "\u0120acutely": 49317, + "\u0120Gamble": 49318, + "\u0120pim": 49319, + "\u0120Kus": 49320, + "Typically": 49321, + "Deploy": 49322, + "\u0120Moroccan": 49323, + "potion": 49324, + "combe": 49325, + "\u0120vigilante": 49326, + "\u0120363": 49327, + "Stew": 49328, + "\u0120Bagg": 49329, + "\u0120resided": 49330, + "\u0120Spo": 49331, + "\u0120remnant": 49332, + "\u0120emptiness": 49333, + "brainer": 49334, + "\u0120outpatient": 49335, + "priority": 49336, + "\u0120leptin": 49337, + "\u0120Payton": 49338, + "\u0120Gleaming": 49339, + "\u0120Shed": 49340, + "\u0120Polo": 49341, + "\u0120Mormonism": 49342, + "restricted": 49343, + "arlane": 49344, + "wx": 49345, + "\u0120creatine": 49346, + "\u0120Anon": 49347, + "\u0120STUD": 49348, + "\u0120JUL": 49349, + "\u0120Tee": 49350, + "528": 49351, + "089": 49352, + "\u0120hatched": 49353, + "Dispatch": 49354, + "\u0120Composite": 49355, + "\u0120451": 49356, + "puff": 49357, + "\u0120XCOM": 49358, + "\u0120Orn": 49359, + "\u0120THANK": 49360, + "ENDED": 49361, + "\u0120Asheville": 49362, + "\u0120\u00c3\u013e": 49363, + "\u0120mango": 49364, + "\u0120Slightly": 49365, + "worldly": 49366, + "\u0120Wander": 49367, + "\u0120Expand": 49368, + "\u0120Chr": 49369, + "Mist": 49370, + "\u0120orthodoxy": 49371, + "\u0120UNESCO": 49372, + "regate": 49373, + "Elsewhere": 49374, + "kie": 49375, + "irled": 49376, + "\u0120topple": 49377, + "\u0120adoptive": 49378, + "\u0120Legs": 49379, + "dress": 49380, + "\u0120Sagan": 49381, + "bare": 49382, + "\u0120Glou": 49383, + "Crunch": 49384, + "\u0120helpers": 49385, + "\u0120chronically": 49386, + "\u0120Huma": 49387, + "10000": 49388, + "\u0120accommodating": 49389, + "\u00e4\u00ba\u0136": 49390, + "\u0120wrinkles": 49391, + "\u0120dodged": 49392, + "fourth": 49393, + "\u0120precon": 49394, + "\u0120compressor": 49395, + "\u0120Kare": 49396, + "\u0120evict": 49397, + "\u0120Warwick": 49398, + "imar": 49399, + "\u0120modernization": 49400, + "\u0120bandwagon": 49401, + "\u0120refuted": 49402, + "\u0120netted": 49403, + "\u0120Naples": 49404, + "\u0120Genie": 49405, + "perors": 49406, + "\u0120fielded": 49407, + "\u0120dere": 49408, + "\u0120Parables": 49409, + "lees": 49410, + "\u0120trout": 49411, + "aspers": 49412, + "\u0120nihil": 49413, + "\u0120happiest": 49414, + "\u0120floppy": 49415, + "\u0120Loft": 49416, + "\u0120Heard": 49417, + "\u0120unison": 49418, + "\u0120lug": 49419, + "\u0120Redmond": 49420, + "classic": 49421, + "Supporters": 49422, + "SHIP": 49423, + "GMT": 49424, + "\u0120fuelled": 49425, + "\u00e7\u0132": 49426, + "\u0120dd": 49427, + "\u0120Eminem": 49428, + "\u01201897": 49429, + "NYSE": 49430, + "\u0120secretaries": 49431, + "\u0120FIA": 49432, + "\u0120Canaveral": 49433, + "Favorite": 49434, + "\u0120pomp": 49435, + "\u0120detainee": 49436, + "ership": 49437, + "aimon": 49438, + "iour": 49439, + "\u0120Apex": 49440, + "\u0120plantations": 49441, + "amia": 49442, + "acion": 49443, + "Rust": 49444, + "\u0120towed": 49445, + "\u0120Truly": 49446, + "577": 49447, + "\u0120sheltered": 49448, + "rider": 49449, + "Wo": 49450, + "\u0120lair": 49451, + "\u0120Intelligent": 49452, + "improve": 49453, + "matically": 49454, + "\u0120etiquette": 49455, + "adra": 49456, + "allo": 49457, + "\u0120Juno": 49458, + "anything": 49459, + "\u0120Struggle": 49460, + "\u0120Predict": 49461, + "\u0120Grimes": 49462, + "\u0120AMERICA": 49463, + "ctx": 49464, + "\u0120Situation": 49465, + "WOOD": 49466, + "\u0120soluble": 49467, + "meier": 49468, + "\u0120intolerable": 49469, + "angering": 49470, + "\u0120uninterrupted": 49471, + "\u0120tooltip": 49472, + "\u0120interrogated": 49473, + "\u0120gunned": 49474, + "\u0120Sneak": 49475, + "\u00e6\u0143\u00a6": 49476, + "\u0120tether": 49477, + "\u0120crumble": 49478, + "Lens": 49479, + "\u0120clustered": 49480, + "\u0120Syl": 49481, + "\u0120Hasan": 49482, + "\u0120dystopian": 49483, + "wana": 49484, + "\u0120joystick": 49485, + "\u0120Thib": 49486, + "ammu": 49487, + "Tomorrow": 49488, + "546": 49489, + "\u0120overcame": 49490, + "\u0120minimized": 49491, + "ceptor": 49492, + "Runner": 49493, + "ENGTH": 49494, + "\u0120Brenda": 49495, + "\u0120Achievements": 49496, + "\u0120torches": 49497, + "\u0120rapport": 49498, + "\u0120Investigator": 49499, + "\u0120Handling": 49500, + "relation": 49501, + "grey": 49502, + "815": 49503, + "\u0120kcal": 49504, + "\u0120Commands": 49505, + "dq": 49506, + "\u0120curls": 49507, + "\u0120bearer": 49508, + "\u0120cynicism": 49509, + "itri": 49510, + "\u0120Useful": 49511, + "Bee": 49512, + "DCS": 49513, + "\u0120abras": 49514, + "Pract": 49515, + "BILITIES": 49516, + "712": 49517, + "\u0120debugger": 49518, + "\u0120debtor": 49519, + "\u0120Lia": 49520, + "\u0120Kers": 49521, + "\u0120exacerbate": 49522, + "\u0120Stacy": 49523, + "\u0120Bland": 49524, + "\u0120Scenes": 49525, + "\u0120branching": 49526, + "\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a": 49527, + "apeake": 49528, + "\u0120salsa": 49529, + "\u0120mishand": 49530, + "\u0120Konami": 49531, + "\u0120Nib": 49532, + "\u0120anecdote": 49533, + "\u0120agreeable": 49534, + "\u00cf\u012b": 49535, + "\u0120Nathaniel": 49536, + "\u0120Heisman": 49537, + "\u0120Beware": 49538, + "\u01201886": 49539, + "spective": 49540, + "691": 49541, + "522": 49542, + "\u0120inhibits": 49543, + "\u0120hashing": 49544, + "\u01201889": 49545, + "\u00e5\u00b0\u0128": 49546, + "vich": 49547, + "Pure": 49548, + "\u0120solidly": 49549, + "\u0120aspirin": 49550, + "imaru": 49551, + "\u0120streetcar": 49552, + "\u0120UCS": 49553, + "\u0120Judd": 49554, + "\u0120flashbacks": 49555, + "pins": 49556, + "\u01201440": 49557, + "\u0120UNHCR": 49558, + "\u0120Symptoms": 49559, + "TIT": 49560, + "538": 49561, + "Fra": 49562, + "%);": 49563, + "\u0120ooz": 49564, + "\u0120curfew": 49565, + "\u0120calmed": 49566, + "\u0120participates": 49567, + "TeX": 49568, + "\u0120nonsensical": 49569, + "\u0120fullback": 49570, + "\u0120DeL": 49571, + "monkey": 49572, + "hari": 49573, + "\u0120metabolites": 49574, + "\u0120looted": 49575, + "\u0120ALWAYS": 49576, + "\u0120BCC": 49577, + "Lt": 49578, + "ochet": 49579, + "Bone": 49580, + "\u0120vetoed": 49581, + "\u0120gcc": 49582, + "\u0120CLICK": 49583, + "\u01201888": 49584, + "saf": 49585, + "\u0120stiffness": 49586, + "\u0120lowly": 49587, + "\u0120Geh": 49588, + "verson": 49589, + "orset": 49590, + "\u0120unforeseen": 49591, + "\u0120anesthesia": 49592, + "\u0120Optical": 49593, + "\u0120reconstructed": 49594, + "\u0120Tup": 49595, + "shows": 49596, + "NEWS": 49597, + "\u0120Newspaper": 49598, + "\u0120ASA": 49599, + "tera": 49600, + "Numbers": 49601, + "\u0120inexplicable": 49602, + "\u00d7\u0133": 49603, + "\u0120hardness": 49604, + "untarily": 49605, + "\u0120Acer": 49606, + "gradient": 49607, + "ARDIS": 49608, + "\u0120woodland": 49609, + "\u0120metaphors": 49610, + "\u0120Wembley": 49611, + "\u0120Pavel": 49612, + "philis": 49613, + "\u0120rewriting": 49614, + "\u0120perceptual": 49615, + "\u01201070": 49616, + "worms": 49617, + "\u0120Downs": 49618, + "\u0120unsurprisingly": 49619, + "\u0120tagging": 49620, + "flame": 49621, + "\u0120litres": 49622, + "\u0120bounces": 49623, + "\u0120Babe": 49624, + "shut": 49625, + "\u0120overdoses": 49626, + "\u0120Sheila": 49627, + "\u0120Chau": 49628, + "\u0120Bless": 49629, + "Capture": 49630, + "\u0120Significant": 49631, + "\u0120Scion": 49632, + "\u0120389": 49633, + "\u0120McH": 49634, + "\u0120Titanium": 49635, + "\u0120Meal": 49636, + "ameda": 49637, + "agents": 49638, + "aggressive": 49639, + "Billy": 49640, + "763": 49641, + "\u0120Saying": 49642, + "DERR": 49643, + "itone": 49644, + "Collins": 49645, + "Bound": 49646, + "\u0120bolted": 49647, + "\u0120DMCA": 49648, + "953": 49649, + "\u0120uniqueness": 49650, + "\u0120epigen": 49651, + "unci": 49652, + "antam": 49653, + "\u0120reckoning": 49654, + "chairs": 49655, + "OGR": 49656, + "\u0120Senegal": 49657, + "\u01201862": 49658, + "relevant": 49659, + "\u0120\u00c2\u00af": 49660, + "\u0120pharmacies": 49661, + "\u0120Geral": 49662, + "vier": 49663, + "Yan": 49664, + "ORPG": 49665, + "\u0120rabid": 49666, + "bending": 49667, + "\u0120UNITED": 49668, + "\u0120465": 49669, + "Assembly": 49670, + "\u0120weep": 49671, + "\u0120behest": 49672, + "\u0120Mothers": 49673, + "\u0120Jace": 49674, + "hid": 49675, + "\u0120whirlwind": 49676, + "\u0120UNIVERS": 49677, + "\u0120utopian": 49678, + "\u0120kidnap": 49679, + "Philipp": 49680, + "Kin": 49681, + "893": 49682, + "\u0120livestream": 49683, + "\u0120MISS": 49684, + "\u0120subversive": 49685, + "\u0120Techniques": 49686, + "\u0120JUSTICE": 49687, + "\u0120BASE": 49688, + "\u0120387": 49689, + "\u0120assailants": 49690, + "\u0120Hardcore": 49691, + "\u0120sprinkled": 49692, + "\u0120Pse": 49693, + "\u00e9\u013c": 49694, + "printed": 49695, + "\u0120Hau": 49696, + "ORGE": 49697, + "\u0120TOUR": 49698, + "\u0120laced": 49699, + "\u0120itch": 49700, + "Giving": 49701, + "\u0120ported": 49702, + "781": 49703, + "////////////////////////////////": 49704, + "breeding": 49705, + "\u0120logger": 49706, + "\u0120HOL": 49707, + "innie": 49708, + "Firstly": 49709, + "\u0120embryonic": 49710, + "\u0120delegated": 49711, + "pai": 49712, + "OIL": 49713, + "\u0120centrally": 49714, + "\u0120Rx": 49715, + "\u0120Scouting": 49716, + "Dutch": 49717, + "\u0120hereditary": 49718, + "\u0120Cruiser": 49719, + "sat": 49720, + "529": 49721, + "\u0120Marriott": 49722, + "othermal": 49723, + "\u0120prohibitions": 49724, + "Earn": 49725, + "\u0120Stab": 49726, + "\u0120Colleges": 49727, + "\u0120Belief": 49728, + "stretched": 49729, + "\u0120LH": 49730, + "\u0120EntityItem": 49731, + "CIA": 49732, + "\u0120unrem": 49733, + "\u0120laureate": 49734, + "\u0120denominations": 49735, + "summary": 49736, + "hler": 49737, + "Spect": 49738, + "\u0120Klaus": 49739, + "\u0120Beans": 49740, + "\u0120insur": 49741, + "\u0120PAX": 49742, + "\u0120fielder": 49743, + "\u0120Vet": 49744, + "\u0120Sparrow": 49745, + "zie": 49746, + "\u0120SQ": 49747, + "\u0120Mondays": 49748, + "\u0120Offline": 49749, + "\u0120Lerner": 49750, + "\u0120Extensions": 49751, + "Ireland": 49752, + "\u0120patronage": 49753, + "\u0120contrasted": 49754, + "\u0120Mania": 49755, + "hirt": 49756, + "Moscow": 49757, + "\u0120condemns": 49758, + "\u0120Ange": 49759, + "\u0120composing": 49760, + "\u0120Pepe": 49761, + "\u0120Paddock": 49762, + "\u0120heterogeneity": 49763, + "\u0120ideologically": 49764, + "\u0120fishes": 49765, + "\u0120cursing": 49766, + "\u0120Rutherford": 49767, + "\u0120Floating": 49768, + "\u0120Amelia": 49769, + "Tea": 49770, + "Synopsis": 49771, + "\u0120stunts": 49772, + "\u0120bead": 49773, + "\u0120stocking": 49774, + "\u0120MILL": 49775, + "obook": 49776, + "massive": 49777, + "\\<": 49778, + "\u0120hump": 49779, + "\u0120Preferences": 49780, + "EngineDebug": 49781, + "geist": 49782, + "\u0120Nieto": 49783, + "omever": 49784, + "ishy": 49785, + "evaluate": 49786, + "colonial": 49787, + "Alternative": 49788, + "\u0120GoPro": 49789, + "\u0120Vortex": 49790, + "\u0120NETWORK": 49791, + "ansky": 49792, + "Secure": 49793, + "\u0120Thrust": 49794, + "Snake": 49795, + "\u0120parcels": 49796, + "\u0120samurai": 49797, + "\u0120actresses": 49798, + "Nap": 49799, + "MF": 49800, + "iferation": 49801, + "Beer": 49802, + "523": 49803, + "\u0120Ily": 49804, + "ointment": 49805, + "Ping": 49806, + "\u0120striped": 49807, + "\u0120Mellon": 49808, + "ossession": 49809, + "\u0120neutron": 49810, + "endium": 49811, + "\u0120aph": 49812, + "\u0120Flavoring": 49813, + "\u0120383": 49814, + "\u0120responsiveness": 49815, + "\u0120Jindal": 49816, + "\u0120Hitchcock": 49817, + "Denver": 49818, + "\u0120DRAGON": 49819, + "smanship": 49820, + "\u0120Dupl": 49821, + "\u0120sly": 49822, + "\u0120webcam": 49823, + "\u0120Twain": 49824, + "\u0120Darling": 49825, + "iliate": 49826, + "consumer": 49827, + "DIT": 49828, + "\u0120namesake": 49829, + "\u0120unorthodox": 49830, + "\u0120funer": 49831, + "\u0120PLoS": 49832, + "\u0120CONTROL": 49833, + "ozyg": 49834, + "oglobin": 49835, + "FACE": 49836, + "ERG": 49837, + "\u0120Dia": 49838, + "\u0120Fiesta": 49839, + "cele": 49840, + "034": 49841, + "\u0120enclave": 49842, + "\u00e2\u0138\u00ac\u00e2\u0138\u00ac": 49843, + "onement": 49844, + "alist": 49845, + "Mand": 49846, + "\u0120homegrown": 49847, + "\u0120Fancy": 49848, + "\u0120conceptions": 49849, + "\u0120Contains": 49850, + "ureen": 49851, + "\u0120reiterate": 49852, + "\u0120meager": 49853, + "\u0120installments": 49854, + "Spawn": 49855, + "627": 49856, + "\u0120photoc": 49857, + "\u0120Cabrera": 49858, + "\u0120Rosenthal": 49859, + "\u0120Lansing": 49860, + "isner": 49861, + "\u0120invests": 49862, + "\u0120UFOs": 49863, + "EXP": 49864, + "Hardware": 49865, + "\u0120tragically": 49866, + "\u0120concedes": 49867, + "ieft": 49868, + "cham": 49869, + "borgh": 49870, + "\u0120Schr": 49871, + "\u0120Melanie": 49872, + "\u0120Hoy": 49873, + "\u0120visitation": 49874, + "\u0120idiosyncr": 49875, + "\u0120fractions": 49876, + "\u0120foreskin": 49877, + "obos": 49878, + "\u0120poaching": 49879, + "\u0120VIEW": 49880, + "\u0120stimulates": 49881, + "\u0120Gork": 49882, + "canon": 49883, + "MIC": 49884, + "\u0120Nemesis": 49885, + "\u0120Indra": 49886, + "\u0120DMV": 49887, + "\u0120529": 49888, + "\u0120inspecting": 49889, + "\u0120grandma": 49890, + "\u0120Whedon": 49891, + "\u0120Shant": 49892, + "\u0120Purg": 49893, + "ikan": 49894, + "\u0120Teg": 49895, + "\u0120CLR": 49896, + "zac": 49897, + "Victoria": 49898, + "\u0120Verify": 49899, + "ionics": 49900, + "\u0120partying": 49901, + "\u0120Mou": 49902, + "colour": 49903, + "\u0120testimonies": 49904, + "lations": 49905, + "\u0120pressuring": 49906, + "hiro": 49907, + "acers": 49908, + "\u0120fid": 49909, + "angler": 49910, + "\u0120CSI": 49911, + "\u0120hereafter": 49912, + "\u0120dissidents": 49913, + "reporting": 49914, + "iphany": 49915, + "chev": 49916, + "\u0120solitude": 49917, + "\u0120lobe": 49918, + "\u0120indis": 49919, + "\u0120credential": 49920, + "recent": 49921, + "adult": 49922, + "\u0120Nirvana": 49923, + "\u0120Franchise": 49924, + "Layer": 49925, + "Hyp": 49926, + "\u0120Berkshire": 49927, + "\u0120wills": 49928, + "tif": 49929, + "\u0120totem": 49930, + "\u0120Judah": 49931, + "repair": 49932, + "Instant": 49933, + "548": 49934, + "\u0120embassies": 49935, + "\u0120bottleneck": 49936, + "\u0120bount": 49937, + "\u0120typew": 49938, + "\u0120Alvin": 49939, + "jing": 49940, + "imilar": 49941, + "Rush": 49942, + "\u0120brim": 49943, + "\u0120HELP": 49944, + "Aim": 49945, + "]'": 49946, + "\u0120passively": 49947, + "\u0120bounded": 49948, + "\u0120Rated": 49949, + "\u0120criminality": 49950, + "\u0120biomark": 49951, + "\u0120dispatcher": 49952, + "\u0120Towards": 49953, + "\u0120+++": 49954, + "righteous": 49955, + "frog": 49956, + "\u0120Panc": 49957, + "Carter": 49958, + "032": 49959, + "\u00e6\u00a9\u0141": 49960, + "\u0120ultraviolet": 49961, + "\u0120Licensed": 49962, + "\u0120Tata": 49963, + "\u0120Blessing": 49964, + "\u0120GAM": 49965, + "\u0120chemically": 49966, + "\u0120Seaf": 49967, + "\u0120RELE": 49968, + "\u0120Mercenary": 49969, + "capitalist": 49970, + "\u0120formulations": 49971, + "\u0120annihilation": 49972, + "\u0120Verb": 49973, + "\u0120Argon": 49974, + "\u0120unloaded": 49975, + "\u0120morphed": 49976, + "\u0120conquering": 49977, + "backer": 49978, + "IELD": 49979, + "\u0120thefts": 49980, + "\u0120frontrunner": 49981, + "\u0120Royale": 49982, + "\u0120Fundamental": 49983, + "elight": 49984, + "Chip": 49985, + "necessary": 49986, + "ayn": 49987, + "\u0120Slip": 49988, + "\u0120448": 49989, + "cerned": 49990, + "Pause": 49991, + "\u0120shockingly": 49992, + "\u0120ABV": 49993, + "\u0120composure": 49994, + "733": 49995, + "\u0120Motorsport": 49996, + "ahime": 49997, + "Murray": 49998, + "Mach": 49999, + "\u0120grids": 50000, + "\u0120debian": 50001, + "\u0120furthermore": 50002, + "\u0120dexterity": 50003, + "\u0120Collections": 50004, + "oslov": 50005, + "ilage": 50006, + "bj": 50007, + "\u0120Monteneg": 50008, + "\u0120strutConnector": 50009, + "\u0120massacres": 50010, + "\u0120briefs": 50011, + "fetched": 50012, + "uvian": 50013, + "olition": 50014, + "Failure": 50015, + "emonic": 50016, + "\u0120flared": 50017, + "\u0120claimant": 50018, + "\u0120cures": 50019, + "\u0120giveaways": 50020, + "\u0120Substance": 50021, + "alions": 50022, + "\u0120cringe": 50023, + "\u0120Kul": 50024, + "\u0120aristocracy": 50025, + "\u0120Ulster": 50026, + "olated": 50027, + "housing": 50028, + "\u0120MIS": 50029, + "\u0120glared": 50030, + "\u0120Wilhelm": 50031, + "needs": 50032, + "lambda": 50033, + "builders": 50034, + "\u0120VIS": 50035, + "\u0120radiator": 50036, + "\u0120Ghostbusters": 50037, + "\u0120436": 50038, + "actual": 50039, + "\u0120herds": 50040, + "\u00c3\u00a7a": 50041, + "watching": 50042, + "\u0120countering": 50043, + "Charge": 50044, + "\u0120charred": 50045, + "\u0120warheads": 50046, + "\u0120iodine": 50047, + "\u0120Macy": 50048, + "041": 50049, + "\u0120departures": 50050, + "\u0120Sins": 50051, + "\u0120dyed": 50052, + "\u0120Concepts": 50053, + "gado": 50054, + "713": 50055, + "\u0120quotations": 50056, + "\u0120gist": 50057, + "\u0120Christy": 50058, + "\u0120antigen": 50059, + "\u0120Hemp": 50060, + "\u0120Drawn": 50061, + "\u0120Barg": 50062, + "ezvous": 50063, + "\u0120paternity": 50064, + "\u0120ardu": 50065, + "\u0120Anchorage": 50066, + "\u0120Rik": 50067, + "\u0120overloaded": 50068, + "\u0120Username": 50069, + "\u0120Tammy": 50070, + "\u0120Nau": 50071, + "\u0120Cellular": 50072, + "\u0120waning": 50073, + "\u0120rodent": 50074, + "\u0120Worcester": 50075, + "ilts": 50076, + "\u0120Tad": 50077, + "\u0120dwellings": 50078, + "\u0120bullish": 50079, + "431": 50080, + "\u0120retaliate": 50081, + "\u0120migraine": 50082, + "\u0120Chevron": 50083, + "CHECK": 50084, + "\u0120donkey": 50085, + "crim": 50086, + "SPA": 50087, + "\u0120Analog": 50088, + "\u0120marquee": 50089, + "\u0120Haas": 50090, + "Bir": 50091, + "\u0120GDDR": 50092, + "\u0120Downloads": 50093, + "\u0120willpower": 50094, + "\u0120Forth": 50095, + "\u0120Recorded": 50096, + "\u0120impossibility": 50097, + "\u0120Logged": 50098, + "\u0120Franks": 50099, + "\u0120Ratt": 50100, + "initions": 50101, + "\u0120cleaners": 50102, + "\u0120sorely": 50103, + "\u0120flickering": 50104, + "\u0120Examination": 50105, + "catching": 50106, + "alloween": 50107, + "Msg": 50108, + "\u0120dunno": 50109, + "Fa": 50110, + "\u0120dysph": 50111, + "crazy": 50112, + ".''.": 50113, + "\u0120mainline": 50114, + "\u0120cs": 50115, + "\u0120ptr": 50116, + "\u0120Wally": 50117, + "igun": 50118, + "951": 50119, + "\u0120Bigfoot": 50120, + "fights": 50121, + "\u0120retrieving": 50122, + "Jr": 50123, + "\u0120duplication": 50124, + "\u0120Explan": 50125, + "\u0120relational": 50126, + "\u0120quaint": 50127, + "\u0120biscuits": 50128, + "\u0120ado": 50129, + "\u0120shudder": 50130, + "\u0120antidote": 50131, + "blooded": 50132, + "ksh": 50133, + "\u0120sauces": 50134, + "\u0120reinvest": 50135, + "\u0120dispensary": 50136, + "\u0120Diver": 50137, + "\u01209000": 50138, + "student": 50139, + "\u0120insepar": 50140, + "escap": 50141, + "\u0120toddlers": 50142, + "\u0120GPIO": 50143, + "\u0120Assignment": 50144, + "headers": 50145, + "\u0120lackluster": 50146, + "\u0120aback": 50147, + "956": 50148, + "\u0120toolbar": 50149, + "745": 50150, + "\u0120oust": 50151, + "\u0120contemplation": 50152, + "\u0120PRESIDENT": 50153, + "\u0120458": 50154, + "======": 50155, + "\u0120guaranteeing": 50156, + "\u0120Heist": 50157, + "\u0120Cannes": 50158, + "\u013b\u00bd": 50159, + "\u0120collaborator": 50160, + "\u0120Amp": 50161, + "\u0120gou": 50162, + "\u0120SHALL": 50163, + "stories": 50164, + "783": 50165, + "\u0120mobilized": 50166, + "\u0120brood": 50167, + "\u0120LU": 50168, + "\u0120\u00f0\u0141\u0133": 50169, + "\u0120refin": 50170, + "\u0120Anthropology": 50171, + "vind": 50172, + "illi": 50173, + "\u0120warranties": 50174, + "\u0120Babel": 50175, + "\u0120swath": 50176, + "\u0120caches": 50177, + "\u0120antagonists": 50178, + "artifacts": 50179, + "\u0120hotly": 50180, + "\u0120Starts": 50181, + "\u0120G\u00c3\u00b6": 50182, + "zag": 50183, + "!!!!!": 50184, + "\u0120scourge": 50185, + "\u0120conspiring": 50186, + "ruits": 50187, + "reverse": 50188, + "\u0120Sheen": 50189, + "\u0120Jesuit": 50190, + "\u0120Giovanni": 50191, + "adies": 50192, + "\u0120buttocks": 50193, + "earcher": 50194, + "acan": 50195, + "\u0120volleyball": 50196, + "\u0120shrouded": 50197, + "\u0120scoreboard": 50198, + "bats": 50199, + "\u0120IPM": 50200, + "\u0120asses": 50201, + "\u0120deregulation": 50202, + "\u0120Telegram": 50203, + "\u0120Reboot": 50204, + "\u01207000": 50205, + "\u0120Canary": 50206, + "\u0120kernels": 50207, + "\u0120Fran\u00c3\u00a7ois": 50208, + "\u0120Duff": 50209, + "\u0120Pon": 50210, + "\u0120Leica": 50211, + "\u0120Garmin": 50212, + "\u0120orphans": 50213, + "\u0120Claudia": 50214, + "\u0120calendars": 50215, + "\u0120Leilan": 50216, + "ento": 50217, + "Rocket": 50218, + "\u0120brunch": 50219, + "\u0120Hawking": 50220, + "ainers": 50221, + "\u0120sensibilities": 50222, + "\u0120kW": 50223, + "\u0120Kand": 50224, + "\u0120reclaimed": 50225, + "\u0120interestingly": 50226, + "\u00d7\u00a9": 50227, + "romy": 50228, + "JM": 50229, + "\u0120Enhancement": 50230, + "bush": 50231, + "Skip": 50232, + "\u0120rappers": 50233, + "\u0120gazing": 50234, + "pedia": 50235, + "athlon": 50236, + "Revolution": 50237, + "\u0120snipers": 50238, + "\u0120reverted": 50239, + "\u0120conglomerate": 50240, + "Terry": 50241, + "794": 50242, + "\u0120harsher": 50243, + "\u0120desolate": 50244, + "\u0120Hitman": 50245, + "Commission": 50246, + "\u0120(/": 50247, + "\u00e2\u0122\u00a6.\"": 50248, + "Compar": 50249, + "\u0120amplification": 50250, + "ominated": 50251, + "\u0120regress": 50252, + "\u0120Collider": 50253, + "\u0120informants": 50254, + "\u0120gazed": 50255, + "<|endoftext|>": 50256 +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/vocab.bpe b/dotnet/samples/KernelSyntaxExamples/Resources/EnglishRoberta/vocab.bpe similarity index 100% rename from dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/vocab.bpe rename to dotnet/samples/KernelSyntaxExamples/Resources/EnglishRoberta/vocab.bpe diff --git a/dotnet/samples/KernelSyntaxExamples/Skills/EmailSkill.cs b/dotnet/samples/KernelSyntaxExamples/Skills/EmailSkill.cs deleted file mode 100644 index 0a029cdae571..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Skills/EmailSkill.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Skills; - -internal sealed class EmailSkill -{ - [SKFunction, Description("Given an e-mail and message body, send an email")] - public string SendEmail( - [Description("The body of the email message to send.")] string input, - [Description("The email address to send email to.")] string email_address) => - - $"Sent email to: {email_address}. Body: {input}"; - - [SKFunction, Description("Given a name, find email address")] - public string GetEmailAddress( - [Description("The name of the person whose email address needs to be found.")] string input, - ILogger? logger = null) - { - // Sensitive data, logging as trace, disabled by default - logger?.LogTrace("Returning hard coded email for {0}", input); - - return "johndoe1234@example.com"; - } -} diff --git a/dotnet/samples/KernelSyntaxExamples/Skills/StaticTextSkill.cs b/dotnet/samples/KernelSyntaxExamples/Skills/StaticTextSkill.cs deleted file mode 100644 index e89ae60c02b8..000000000000 --- a/dotnet/samples/KernelSyntaxExamples/Skills/StaticTextSkill.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Skills; - -public sealed class StaticTextSkill -{ - [SKFunction, Description("Change all string chars to uppercase")] - public static string Uppercase([Description("Text to uppercase")] string input) => - input.ToUpperInvariant(); - - [SKFunction, Description("Append the day variable")] - public static string AppendDay( - [Description("Text to append to")] string input, - [Description("Value of the day to append")] string day) => - input + day; -} diff --git a/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs b/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs index 1c2ff6f60078..d4f0c3b854e5 100644 --- a/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs +++ b/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs @@ -7,7 +7,7 @@ public sealed class TestConfiguration { - private IConfigurationRoot _configRoot; + private readonly IConfigurationRoot _configRoot; private static TestConfiguration? s_instance; private TestConfiguration(IConfigurationRoot configRoot) @@ -36,6 +36,7 @@ public static void Initialize(IConfigurationRoot configRoot) public static RedisConfig Redis => LoadSection(); public static JiraConfig Jira => LoadSection(); public static ChromaConfig Chroma => LoadSection(); + public static KustoConfig Kusto => LoadSection(); private static T LoadSection([CallerMemberName] string? caller = null) { @@ -82,6 +83,7 @@ public class ACSConfig { public string Endpoint { get; set; } public string ApiKey { get; set; } + public string IndexName { get; set; } } public class QdrantConfig @@ -154,5 +156,10 @@ public class ChromaConfig { public string Endpoint { get; set; } } + + public class KustoConfig + { + public string ConnectionString { get; set; } + } #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. } diff --git a/dotnet/samples/NCalcPlugins/LanguageCalculatorPlugin.cs b/dotnet/samples/NCalcPlugins/LanguageCalculatorPlugin.cs new file mode 100644 index 000000000000..7758d9f78b51 --- /dev/null +++ b/dotnet/samples/NCalcPlugins/LanguageCalculatorPlugin.cs @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Orchestration; +using NCalc; + +namespace NCalcPlugins; + +/// +/// Plugin that enables the comprehension of mathematical problems presented in English / natural-language text, followed by the execution of the necessary calculations to solve those problems. +/// +/// +/// usage : +/// var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); +/// var question = "what is the square root of 625"; +/// var calculatorPlugin = kernel.ImportFunctions(new LanguageCalculatorPlugin(kernel)); +/// var summary = await kernel.RunAsync(questions, calculatorPlugin["Calculate"]); +/// Console.WriteLine("Result :"); +/// Console.WriteLine(summary.Result); +/// +public class LanguageCalculatorPlugin +{ + private readonly ISKFunction _mathTranslator; + private const string MathTranslatorPrompt = + @"Translate a math problem into a expression that can be executed using .net NCalc library. Use the output of running this code to answer the question. +Available functions: Abs, Acos, Asin, Atan, Ceiling, Cos, Exp, Floor, IEEERemainder, Log, Log10, Max, Min, Pow, Round, Sign, Sin, Sqrt, Tan, and Truncate. in and if are also supported. + +Question: $((Question with math problem.)) +expression:``` $((single line mathematical expression that solves the problem))``` + +[Examples] +Question: What is 37593 * 67? +expression:```37593 * 67``` + +Question: what is 3 to the 2nd power? +expression:```Pow(3, 2)``` + +Question: what is sine of 0 radians? +expression:```Sin(0)``` + +Question: what is sine of 45 degrees? +expression:```Sin(45 * Pi /180 )``` + +Question: how many radians is 45 degrees? +expression:``` 45 * Pi / 180 ``` + +Question: what is the square root of 81? +expression:```Sqrt(81)``` + +Question: what is the angle whose sine is the number 1? +expression:```Asin(1)``` + +[End of Examples] + +Question: {{ $input }} +"; + + /// + /// Initializes a new instance of the class. + /// + /// The kernel to be used for creating the semantic function. + public LanguageCalculatorPlugin(IKernel kernel) + { + this._mathTranslator = kernel.CreateSemanticFunction( + MathTranslatorPrompt, + pluginName: nameof(LanguageCalculatorPlugin), + functionName: "TranslateMathProblem", + description: "Used by 'Calculator' function.", + requestSettings: new AIRequestSettings() + { + ExtensionData = new Dictionary() + { + { "MaxTokens", 256 }, + { "Temperature", 0.0 }, + { "TopP", 1 }, + } + }); + } + + /// + /// Calculates the result of a non-trivial math expression. + /// + /// A valid mathematical expression that could be executed by a calculator capable of more advanced math functions like sine/cosine/floor. + /// The context for the plugin execution. + /// A representing the result of the asynchronous operation. + [SKFunction, SKName("Calculator"), Description("Useful for getting the result of a non-trivial math expression.")] + public async Task CalculateAsync( + [Description("A valid mathematical expression that could be executed by a calculator capable of more advanced math functions like sin/cosine/floor.")] + string input, + SKContext context) + { + string answer; + + try + { + var result = await context.Runner.RunAsync(this._mathTranslator, new ContextVariables(input)).ConfigureAwait(false); + answer = result.GetValue() ?? string.Empty; + } + catch (Exception ex) + { + throw new InvalidOperationException($"Error in calculator for input {input} {ex.Message}", ex); + } + + string pattern = @"```\s*(.*?)\s*```"; + + Match match = Regex.Match(answer, pattern, RegexOptions.Singleline); + if (match.Success) + { + var result = EvaluateMathExpression(match); + return result; + } + + throw new InvalidOperationException($"Input value [{input}] could not be understood, received following {answer}"); + } + + private static string EvaluateMathExpression(Match match) + { + var textExpressions = match.Groups[1].Value; + var expr = new Expression(textExpressions, EvaluateOptions.IgnoreCase); + expr.EvaluateParameter += delegate (string name, ParameterArgs args) + { + args.Result = name.ToLower(System.Globalization.CultureInfo.CurrentCulture) switch + { + "pi" => Math.PI, + "e" => Math.E, + _ => args.Result + }; + }; + + try + { + if (expr.HasErrors()) + { + return "Error:" + expr.Error + " could not evaluate " + textExpressions; + } + + var result = expr.Evaluate(); + return "Answer:" + result.ToString(); + } + catch (Exception e) + { + throw new InvalidOperationException("could not evaluate " + textExpressions, e); + } + } +} diff --git a/dotnet/samples/NCalcPlugins/NCalcPlugins.csproj b/dotnet/samples/NCalcPlugins/NCalcPlugins.csproj new file mode 100644 index 000000000000..7aa023e93fd7 --- /dev/null +++ b/dotnet/samples/NCalcPlugins/NCalcPlugins.csproj @@ -0,0 +1,14 @@ + + + netstandard2.0 + 10 + + + + + + + + + + diff --git a/dotnet/samples/NCalcPlugins/SimpleCalculatorPlugin.cs b/dotnet/samples/NCalcPlugins/SimpleCalculatorPlugin.cs new file mode 100644 index 000000000000..a0ba143cbe0f --- /dev/null +++ b/dotnet/samples/NCalcPlugins/SimpleCalculatorPlugin.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; + +namespace NCalcPlugins; + +/// +/// Simple calculator plugin that evaluates a mathematical expression. +/// +public class SimpleCalculatorPlugin +{ + private readonly ISKFunction _mathTranslator; + + private static readonly string[] s_stopSequences = new[] { "Problem:", "Solution:" }; + + /// + /// Initializes a new instance of the class. + /// + /// The kernel used to create the semantic function. + public SimpleCalculatorPlugin(IKernel kernel) + { + this._mathTranslator = kernel.CreateSemanticFunction( + "Task: Give the final solution for the problem. Be as concise as possible.\nProblem:4+4\nSolution:8\nProblem:{{$input}}\nSolution:\n", + pluginName: nameof(SimpleCalculatorPlugin), + functionName: "Calculator", + description: "Evaluate a mathematical expression. Input is a valid mathematical expression that could be executed by a simple calculator i.e. add, subtract, multiply and divide. Cannot use variables.", + requestSettings: new AIRequestSettings() + { + ExtensionData = new Dictionary() + { + { "MaxTokens", 256 }, + { "Temperature", 0.0 }, + { "StopSequences", s_stopSequences }, + } + }); + } +} diff --git a/dotnet/samples/NCalcSkills/LanguageCalculatorSkill.cs b/dotnet/samples/NCalcSkills/LanguageCalculatorSkill.cs deleted file mode 100644 index c715346fc640..000000000000 --- a/dotnet/samples/NCalcSkills/LanguageCalculatorSkill.cs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ComponentModel; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using NCalc; - -namespace NCalcSkills; - -/// -/// Skill that enables the comprehension of mathematical problems presented in English / natural-language text, followed by the execution of the necessary calculations to solve those problems. -/// -/// -/// usage : -/// var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Logger).Build(); -/// var question = "what is the square root of 625"; -/// var calculatorSkill = kernel.ImportSkill(new LanguageCalculatorSkill(kernel)); -/// var summary = await kernel.RunAsync(questions, calculatorSkill["Calculate"]); -/// Console.WriteLine("Result :"); -/// Console.WriteLine(summary.Result); -/// -public class LanguageCalculatorSkill -{ - private readonly ISKFunction _mathTranslator; - - private const string MathTranslatorPrompt = - @"Translate a math problem into a expression that can be executed using .net NCalc library. Use the output of running this code to answer the question. -Available functions: Abs, Acos, Asin, Atan, Ceiling, Cos, Exp, Floor, IEEERemainder, Log, Log10, Max, Min, Pow, Round, Sign, Sin, Sqrt, Tan, and Truncate. in and if are also supported. - -Question: $((Question with math problem.)) -expression:``` $((single line mathematical expression that solves the problem))``` - -[Examples] -Question: What is 37593 * 67? -expression:```37593 * 67``` - -Question: what is 3 to the 2nd power? -expression:```Pow(3, 2)``` - -Question: what is sine of 0 radians? -expression:```Sin(0)``` - -Question: what is sine of 45 degrees? -expression:```Sin(45 * Pi /180 )``` - -Question: how many radians is 45 degrees? -expression:``` 45 * Pi / 180 ``` - -Question: what is the square root of 81? -expression:```Sqrt(81)``` - -Question: what is the angle whose sine is the number 1? -expression:```Asin(1)``` - -[End of Examples] - -Question: {{ $input }} -"; - - public LanguageCalculatorSkill(IKernel kernel) - { - this._mathTranslator = kernel.CreateSemanticFunction( - MathTranslatorPrompt, - skillName: nameof(LanguageCalculatorSkill), - functionName: "TranslateMathProblem", - description: "Used by 'Calculator' function.", - maxTokens: 256, - temperature: 0.0, - topP: 1); - } - - [SKFunction, SKName("Calculator"), Description("Useful for getting the result of a non-trivial math expression.")] - public async Task CalculateAsync( - [Description("A valid mathematical expression that could be executed by a calculator capable of more advanced math functions like sin/cosine/floor.")] - string input, - SKContext context) - { - var answer = await this._mathTranslator.InvokeAsync(input).ConfigureAwait(false); - - if (answer.ErrorOccurred) - { - throw new InvalidOperationException("error in calculator for input " + input + " " + answer.LastErrorDescription); - } - - string pattern = @"```\s*(.*?)\s*```"; - - Match match = Regex.Match(answer.Result, pattern, RegexOptions.Singleline); - if (match.Success) - { - var result = EvaluateMathExpression(match); - return result; - } - - throw new InvalidOperationException($"Input value [{input}] could not be understood, received following {answer.Result}"); - } - - private static string EvaluateMathExpression(Match match) - { - var textExpressions = match.Groups[1].Value; - var expr = new Expression(textExpressions, EvaluateOptions.IgnoreCase); - expr.EvaluateParameter += delegate (string name, ParameterArgs args) - { - args.Result = name.ToLower(System.Globalization.CultureInfo.CurrentCulture) switch - { - "pi" => Math.PI, - "e" => Math.E, - _ => args.Result - }; - }; - - try - { - if (expr.HasErrors()) - { - return "Error:" + expr.Error + " could not evaluate " + textExpressions; - } - - var result = expr.Evaluate(); - return "Answer:" + result.ToString(); - } - catch (Exception e) - { - throw new InvalidOperationException("could not evaluate " + textExpressions, e); - } - } -} diff --git a/dotnet/samples/NCalcSkills/NCalcSkills.csproj b/dotnet/samples/NCalcSkills/NCalcSkills.csproj deleted file mode 100644 index 88adf3e9190f..000000000000 --- a/dotnet/samples/NCalcSkills/NCalcSkills.csproj +++ /dev/null @@ -1,14 +0,0 @@ - - - netstandard2.0 - 10 - - - - - - - - - - diff --git a/dotnet/samples/NCalcSkills/SimpleCalculatorSkill.cs b/dotnet/samples/NCalcSkills/SimpleCalculatorSkill.cs deleted file mode 100644 index aa1842ea0ad3..000000000000 --- a/dotnet/samples/NCalcSkills/SimpleCalculatorSkill.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace NCalcSkills; - -/// -/// Simple calculator skill -/// -public class SimpleCalculatorSkill -{ - private readonly ISKFunction _mathTranslator; - - private static readonly string[] s_stopSequences = new[] { "Problem:", "Solution:" }; - - public SimpleCalculatorSkill(IKernel kernel) - { - this._mathTranslator = kernel.CreateSemanticFunction( - "Task: Give the final solution for the problem. Be as concise as possible.\nProblem:4+4\nSolution:8\nProblem:{{$input}}\nSolution:\n", - skillName: nameof(SimpleCalculatorSkill), - functionName: "Calculator", - description: "Evaluate a mathematical expression. Input is a valid mathematical expression that could be executed by a simple calculator i.e. add, subtract, multiply and divide. Cannot use variables.", - maxTokens: 256, - temperature: 0.0, - topP: 1, - stopSequences: s_stopSequences); - } -} diff --git a/dotnet/src/.editorconfig b/dotnet/src/.editorconfig index 307110cd27a4..b7d6bf9ace7e 100644 --- a/dotnet/src/.editorconfig +++ b/dotnet/src/.editorconfig @@ -2,3 +2,4 @@ [*.cs] dotnet_diagnostic.CA2007.severity = error # Do not directly await a Task dotnet_diagnostic.VSTHRD111.severity = error # Use .ConfigureAwait(bool) +dotnet_diagnostic.IDE1006.severity = error # Naming rule violations diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/Connectors.AI.HuggingFace.csproj b/dotnet/src/Connectors/Connectors.AI.HuggingFace/Connectors.AI.HuggingFace.csproj index 514f18aea4c5..be21cc3c7b13 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/Connectors.AI.HuggingFace.csproj +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/Connectors.AI.HuggingFace.csproj @@ -18,7 +18,7 @@ - + diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/HuggingFaceKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/HuggingFaceKernelBuilderExtensions.cs index 8867c71c0af8..82699068c568 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/HuggingFaceKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/HuggingFaceKernelBuilderExtensions.cs @@ -36,11 +36,11 @@ public static KernelBuilder WithHuggingFaceTextCompletionService(this KernelBuil bool setAsDefault = false, HttpClient? httpClient = null) { - builder.WithAIService(serviceId, (parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new HuggingFaceTextCompletion( model, apiKey, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), endpoint), setAsDefault); @@ -62,10 +62,10 @@ public static KernelBuilder WithHuggingFaceTextEmbeddingGenerationService(this K string? serviceId = null, bool setAsDefault = false) { - builder.WithAIService(serviceId, (parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new HuggingFaceTextEmbeddingGeneration( model, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient: null, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient: null, loggerFactory), endpoint), setAsDefault); @@ -89,10 +89,10 @@ public static KernelBuilder WithHuggingFaceTextEmbeddingGenerationService(this K string? serviceId = null, bool setAsDefault = false) { - builder.WithAIService(serviceId, (parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new HuggingFaceTextEmbeddingGeneration( model, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), endpoint), setAsDefault); diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/HuggingFaceModelResultExtension.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/HuggingFaceModelResultExtension.cs index f260ac8465e3..2591f411ea47 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/HuggingFaceModelResultExtension.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/HuggingFaceModelResultExtension.cs @@ -7,13 +7,16 @@ namespace Microsoft.SemanticKernel; +/// +/// Provides an extension method for working with Hugging Face model results. +/// public static class HuggingFaceModelResultExtension { /// - /// Retrieves a typed hugging face result from PromptResult/>. + /// Retrieves a typed hugging face result from . /// - /// Current context - /// Hugging face result + /// The instance to retrieve the hugging face result from. + /// A instance containing the hugging face result. public static TextCompletionResponse GetHuggingFaceResult(this ModelResult resultBase) { return resultBase.GetResult(); diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index fbfe4c330207..e57690f76f98 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.Net.Http; -using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -20,7 +19,6 @@ namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; public sealed class HuggingFaceTextCompletion : ITextCompletion #pragma warning restore CA1001 // Types that own disposable fields should be disposable. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. { - private const string HttpUserAgent = "Microsoft-Semantic-Kernel"; private const string HuggingFaceApiEndpoint = "https://api-inference.huggingface.co/models"; private readonly string _model; @@ -65,21 +63,19 @@ public HuggingFaceTextCompletion(string model, string? apiKey = null, HttpClient } /// - public async IAsyncEnumerable GetStreamingCompletionsAsync( + [Obsolete("Streaming capability is not supported, use GetCompletionsAsync instead")] + public IAsyncEnumerable GetStreamingCompletionsAsync( string text, - CompleteRequestSettings requestSettings, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) { - foreach (var completion in await this.ExecuteGetCompletionsAsync(text, cancellationToken).ConfigureAwait(false)) - { - yield return completion; - } + throw new NotSupportedException("Streaming capability is not supported"); } /// public async Task> GetCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { return await this.ExecuteGetCompletionsAsync(text, cancellationToken).ConfigureAwait(false); @@ -87,46 +83,36 @@ public async Task> GetCompletionsAsync( #region private ================================================================================ - private async Task> ExecuteGetCompletionsAsync(string text, CancellationToken cancellationToken = default) + private async Task> ExecuteGetCompletionsAsync(string text, CancellationToken cancellationToken = default) { - try + var completionRequest = new TextCompletionRequest { - var completionRequest = new TextCompletionRequest - { - Input = text - }; - - using var httpRequestMessage = HttpRequest.CreatePostRequest(this.GetRequestUri(), completionRequest); + Input = text + }; - httpRequestMessage.Headers.Add("User-Agent", HttpUserAgent); - if (!string.IsNullOrEmpty(this._apiKey)) - { - httpRequestMessage.Headers.Add("Authorization", $"Bearer {this._apiKey}"); - } + using var httpRequestMessage = HttpRequest.CreatePostRequest(this.GetRequestUri(), completionRequest); - using var response = await this._httpClient.SendAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + httpRequestMessage.Headers.Add("User-Agent", Telemetry.HttpUserAgent); + if (!string.IsNullOrEmpty(this._apiKey)) + { + httpRequestMessage.Headers.Add("Authorization", $"Bearer {this._apiKey}"); + } - var body = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + using var response = await this._httpClient.SendWithSuccessCheckAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); - List? completionResponse = JsonSerializer.Deserialize>(body); + var body = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); - if (completionResponse is null) - { - throw new AIException(AIException.ErrorCodes.InvalidResponseContent, "Unexpected response from model") - { - Data = { { "ResponseData", body } }, - }; - } + List? completionResponse = JsonSerializer.Deserialize>(body); - return completionResponse.ConvertAll(c => new TextCompletionStreamingResult(c)); - } - catch (Exception e) when (e is not AIException && !e.IsCriticalException()) + if (completionResponse is null) { - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Something went wrong: {e.Message}", e); + throw new SKException("Unexpected response from model") + { + Data = { { "ResponseData", body } }, + }; } + + return completionResponse.ConvertAll(c => new TextCompletionResult(c)); } /// diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/TextCompletionResult.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/TextCompletionResult.cs index 08bddc651164..e80ab08e2dd8 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/TextCompletionResult.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/TextCompletionResult.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI.TextCompletion; @@ -9,11 +7,11 @@ namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; -internal sealed class TextCompletionStreamingResult : ITextStreamingResult +internal sealed class TextCompletionResult : ITextResult { private readonly ModelResult _responseData; - public TextCompletionStreamingResult(TextCompletionResponse responseData) + public TextCompletionResult(TextCompletionResponse responseData) { this._responseData = new ModelResult(responseData); } @@ -24,9 +22,4 @@ public Task GetCompletionAsync(CancellationToken cancellationToken = def { return Task.FromResult(this._responseData.GetResult().Text ?? string.Empty); } - - public async IAsyncEnumerable GetCompletionStreamingAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) - { - yield return await this.GetCompletionAsync(cancellationToken).ConfigureAwait(false); - } } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/HuggingFaceTextEmbeddingGeneration.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/HuggingFaceTextEmbeddingGeneration.cs index 922d2d1802a3..103212317b7c 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/HuggingFaceTextEmbeddingGeneration.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/HuggingFaceTextEmbeddingGeneration.cs @@ -7,7 +7,6 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Diagnostics; @@ -20,8 +19,6 @@ namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextEmbedding; public sealed class HuggingFaceTextEmbeddingGeneration : ITextEmbeddingGeneration #pragma warning restore CA1001 // Types that own disposable fields should be disposable. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. { - private const string HttpUserAgent = "Microsoft-Semantic-Kernel"; - private readonly string _model; private readonly string? _endpoint; private readonly HttpClient _httpClient; @@ -76,14 +73,12 @@ public HuggingFaceTextEmbeddingGeneration(string model, HttpClient httpClient, s if (httpClient.BaseAddress == null && string.IsNullOrEmpty(endpoint)) { - throw new AIException( - AIException.ErrorCodes.InvalidConfiguration, - "The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); + throw new SKException("The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); } } /// - public async Task>> GenerateEmbeddingsAsync(IList data, CancellationToken cancellationToken = default) + public async Task>> GenerateEmbeddingsAsync(IList data, CancellationToken cancellationToken = default) { return await this.ExecuteEmbeddingRequestAsync(data, cancellationToken).ConfigureAwait(false); } @@ -96,33 +91,23 @@ public async Task>> GenerateEmbeddingsAsync(IList /// Data to embed. /// The to monitor for cancellation requests. The default is . /// List of generated embeddings. - /// Exception when backend didn't respond with generated embeddings. - private async Task>> ExecuteEmbeddingRequestAsync(IList data, CancellationToken cancellationToken) + private async Task>> ExecuteEmbeddingRequestAsync(IList data, CancellationToken cancellationToken) { - try + var embeddingRequest = new TextEmbeddingRequest { - var embeddingRequest = new TextEmbeddingRequest - { - Input = data - }; + Input = data + }; - using var httpRequestMessage = HttpRequest.CreatePostRequest(this.GetRequestUri(), embeddingRequest); + using var httpRequestMessage = HttpRequest.CreatePostRequest(this.GetRequestUri(), embeddingRequest); - httpRequestMessage.Headers.Add("User-Agent", HttpUserAgent); + httpRequestMessage.Headers.Add("User-Agent", Telemetry.HttpUserAgent); - var response = await this._httpClient.SendAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); - var body = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var response = await this._httpClient.SendWithSuccessCheckAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); + var body = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); - var embeddingResponse = JsonSerializer.Deserialize(body); + var embeddingResponse = JsonSerializer.Deserialize(body); - return embeddingResponse?.Embeddings?.Select(l => new Embedding(l.Embedding!, transferOwnership: true)).ToList()!; - } - catch (Exception e) when (e is not AIException && !e.IsCriticalException()) - { - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Something went wrong: {e.Message}", e); - } + return embeddingResponse?.Embeddings?.Select(l => l.Embedding).ToList()!; } /// @@ -145,7 +130,7 @@ private Uri GetRequestUri() } else { - throw new AIException(AIException.ErrorCodes.InvalidConfiguration, "No endpoint or HTTP client base address has been provided"); + throw new SKException("No endpoint or HTTP client base address has been provided"); } return new Uri($"{baseUrl!.TrimEnd('/')}/{this._model}"); diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/TextEmbeddingResponse.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/TextEmbeddingResponse.cs index 781959e59522..bdf722cce495 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/TextEmbeddingResponse.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/TextEmbeddingResponse.cs @@ -1,22 +1,28 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextEmbedding; /// -/// HTTP Schema for embedding response. +/// Represents the response from the Hugging Face text embedding API. /// public sealed class TextEmbeddingResponse { /// - /// Model containing embedding. + /// Represents the embedding vector for a given text. /// public sealed class EmbeddingVector { + /// + /// The embedding vector as a ReadOnlyMemory of float values. + /// [JsonPropertyName("embedding")] - public IList? Embedding { get; set; } + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Embedding { get; set; } } /// diff --git a/dotnet/src/Connectors/Connectors.AI.Oobabooga/Connectors.AI.Oobabooga.csproj b/dotnet/src/Connectors/Connectors.AI.Oobabooga/Connectors.AI.Oobabooga.csproj index 6daa5aaab4c1..f55b818e10d8 100644 --- a/dotnet/src/Connectors/Connectors.AI.Oobabooga/Connectors.AI.Oobabooga.csproj +++ b/dotnet/src/Connectors/Connectors.AI.Oobabooga/Connectors.AI.Oobabooga.csproj @@ -22,7 +22,7 @@ - + diff --git a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/OobaboogaInvalidResponseException.cs b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/OobaboogaInvalidResponseException.cs deleted file mode 100644 index a2e8e51d2a57..000000000000 --- a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/OobaboogaInvalidResponseException.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel.AI; - -namespace Microsoft.SemanticKernel.Connectors.AI.Oobabooga.TextCompletion; - -#pragma warning disable RCS1194 // Implement exception constructors. -internal sealed class OobaboogaInvalidResponseException : AIException -{ - public T? ResponseData { get; } - - public OobaboogaInvalidResponseException(T? responseData, string? message = null) : base(ErrorCodes.InvalidResponseContent, message) - { - this.ResponseData = responseData; - } -} diff --git a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/OobaboogaTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/OobaboogaTextCompletion.cs index e8d41d7b9411..6a0ee2174b46 100644 --- a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/OobaboogaTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/OobaboogaTextCompletion.cs @@ -23,9 +23,12 @@ namespace Microsoft.SemanticKernel.Connectors.AI.Oobabooga.TextCompletion; /// Oobabooga text completion service API. /// Adapted from /// +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] public sealed class OobaboogaTextCompletion : ITextCompletion { - public const string HttpUserAgent = "Microsoft-Semantic-Kernel"; + /// + /// The URI path for blocking API requests. + /// public const string BlockingUriPath = "/api/v1/generate"; private const string StreamingUriPath = "/api/v1/stream"; @@ -40,10 +43,11 @@ public sealed class OobaboogaTextCompletion : ITextCompletion private readonly ConcurrentBag _webSocketPool = new(); private readonly int _keepAliveWebSocketsDuration; private readonly ILogger? _logger; + private Task? _cleanupTask = null; private long _lastCallTicks = long.MaxValue; /// - /// Controls the size of the buffer used to received websocket packets + /// Controls the size of the buffer used to receive websocket packets. /// public int WebSocketBufferSize { get; set; } = 2048; @@ -131,7 +135,7 @@ public OobaboogaTextCompletion(Uri endpoint, /// public async IAsyncEnumerable GetStreamingCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { await this.StartConcurrentCallAsync(cancellationToken).ConfigureAwait(false); @@ -190,7 +194,7 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync /// public async Task> GetCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { try @@ -210,28 +214,21 @@ public async Task> GetCompletionsAsync( RequestUri = this._blockingUri.Uri, Content = stringContent }; - httpRequestMessage.Headers.Add("User-Agent", HttpUserAgent); + httpRequestMessage.Headers.Add("User-Agent", Telemetry.HttpUserAgent); - using var response = await this._httpClient.SendAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + using var response = await this._httpClient.SendWithSuccessCheckAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); - var body = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var body = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); TextCompletionResponse? completionResponse = JsonSerializer.Deserialize(body); if (completionResponse is null) { - throw new OobaboogaInvalidResponseException(body, "Unexpected response from Oobabooga API"); + throw new SKException($"Unexpected response from Oobabooga API: {body}"); } return completionResponse.Results.Select(completionText => new TextCompletionResult(completionText)).ToList(); } - catch (Exception e) when (e is not AIException && !e.IsCriticalException()) - { - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Something went wrong: {e.Message}", e); - } finally { this.FinishConcurrentCall(); @@ -241,28 +238,42 @@ public async Task> GetCompletionsAsync( #region private ================================================================================ /// - /// Creates an Oobabooga request, mapping CompleteRequestSettings fields to their Oobabooga API counter parts + /// Creates an Oobabooga request, mapping dynamic request setting fields to their Oobabooga API counter parts /// /// The text to complete. /// The request settings. /// An Oobabooga TextCompletionRequest object with the text and completion parameters. - private TextCompletionRequest CreateOobaboogaRequest(string text, CompleteRequestSettings requestSettings) + private TextCompletionRequest CreateOobaboogaRequest(string text, AIRequestSettings? requestSettings) { if (string.IsNullOrWhiteSpace(text)) { throw new ArgumentNullException(nameof(text)); } - // Prepare the request using the provided parameters. - return new TextCompletionRequest() + if (requestSettings is null) { - Prompt = text, - MaxNewTokens = requestSettings.MaxTokens, - Temperature = requestSettings.Temperature, - TopP = requestSettings.TopP, - RepetitionPenalty = GetRepetitionPenalty(requestSettings), - StoppingStrings = requestSettings.StopSequences.ToList() - }; + return new TextCompletionRequest() + { + Prompt = text + }; + } + + if (requestSettings is TextCompletionRequest requestSettingsTextCompletionRequest) + { + requestSettingsTextCompletionRequest.Prompt = text; + return requestSettingsTextCompletionRequest; + } + + var json = JsonSerializer.Serialize(requestSettings); + var textCompletionRequest = JsonSerializer.Deserialize(json); + + if (textCompletionRequest is not null) + { + textCompletionRequest.Prompt = text; + return textCompletionRequest; + } + + throw new ArgumentException("Invalid request settings, cannot convert to TextCompletionRequest", nameof(requestSettings)); } /// @@ -270,15 +281,7 @@ private TextCompletionRequest CreateOobaboogaRequest(string text, CompleteReques /// private void SetWebSocketOptions(ClientWebSocket clientWebSocket) { - clientWebSocket.Options.SetRequestHeader("User-Agent", HttpUserAgent); - } - - /// - /// Converts the semantic-kernel presence penalty, scaled -2:+2 with default 0 for no penalty to the Oobabooga repetition penalty, strictly positive with default 1 for no penalty. See and subsequent links for more details. - /// - private static double GetRepetitionPenalty(CompleteRequestSettings requestSettings) - { - return 1 + requestSettings.PresencePenalty / 2; + clientWebSocket.Options.SetRequestHeader("User-Agent", Telemetry.HttpUserAgent); } /// @@ -313,7 +316,7 @@ private async Task ProcessWebSocketMessagesAsync(ClientWebSocket clientWebSocket if (responseObject is null) { - throw new OobaboogaInvalidResponseException(messageText, "Unexpected response from Oobabooga API"); + throw new SKException($"Unexpected response from Oobabooga API: {messageText}"); } switch (responseObject.Event) @@ -396,7 +399,9 @@ private void FinishConcurrentCall() private void StartCleanupTask(CancellationToken cancellationToken) { - Task.Factory.StartNew( + if (this._cleanupTask == null || this._cleanupTask.IsCompleted) + { + this._cleanupTask = Task.Factory.StartNew( async () => { while (!cancellationToken.IsCancellationRequested) @@ -407,6 +412,7 @@ private void StartCleanupTask(CancellationToken cancellationToken) cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default); + } } /// diff --git a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionRequest.cs b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionRequest.cs index 8adcc088187a..0b64735a4683 100644 --- a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionRequest.cs +++ b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionRequest.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.AI; namespace Microsoft.SemanticKernel.Connectors.AI.Oobabooga.TextCompletion; @@ -11,7 +12,8 @@ namespace Microsoft.SemanticKernel.Connectors.AI.Oobabooga.TextCompletion; /// See and subsequent links for additional information. /// [Serializable] -public sealed class TextCompletionRequest +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] +public sealed class TextCompletionRequest : AIRequestSettings { /// /// The prompt text to complete. @@ -173,5 +175,5 @@ public sealed class TextCompletionRequest /// In addition to the defaults. Written between "" and separated by commas. For instance: "\nYour Assistant:", "\nThe assistant:" /// [JsonPropertyName("stopping_strings")] - public List StoppingStrings { get; set; } = new List(); + public IList StoppingStrings { get; set; } = new List(); } diff --git a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionResponse.cs b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionResponse.cs index e5058fe77cb2..ac3a60ad20a1 100644 --- a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionResponse.cs +++ b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionResponse.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; @@ -8,6 +9,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.Oobabooga.TextCompletion; /// /// HTTP Schema for Oobabooga completion response. Contains a list of results. Adapted from /// +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] public sealed class TextCompletionResponse { /// @@ -20,6 +22,7 @@ public sealed class TextCompletionResponse /// /// HTTP Schema for an single Oobabooga result as part of a completion response. /// +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] public sealed class TextCompletionResponseText { /// diff --git a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionResult.cs b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionResult.cs index 95097f9736ec..8961b758faf7 100644 --- a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionResult.cs +++ b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionResult.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI.TextCompletion; @@ -10,6 +11,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.Oobabooga.TextCompletion; /// /// Oobabooga implementation of . Actual response object is stored in a ModelResult instance, and completion text is simply passed forward. /// +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] internal sealed class TextCompletionResult : ITextResult { private readonly ModelResult _responseData; diff --git a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionStreamingResponse.cs b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionStreamingResponse.cs index 33d9abf68401..3951b638e962 100644 --- a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionStreamingResponse.cs +++ b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionStreamingResponse.cs @@ -1,15 +1,24 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Text.Json.Serialization; namespace Microsoft.SemanticKernel.Connectors.AI.Oobabooga.TextCompletion; /// -/// HTTP Schema for streaming completion response. Adapted from +/// Represents the HTTP schema for streaming completion response. Adapted from . /// +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] public sealed class TextCompletionStreamingResponse { + /// + /// Constant string representing the event that is fired when text is received from a websocket. + /// public const string ResponseObjectTextStreamEvent = "text_stream"; + + /// + /// Constant string representing the event that is fired when streaming from a websocket ends. + /// public const string ResponseObjectStreamEndEvent = "stream_end"; /// diff --git a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionStreamingResult.cs b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionStreamingResult.cs index 0575e6434cc2..e780d45cd62e 100644 --- a/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionStreamingResult.cs +++ b/dotnet/src/Connectors/Connectors.AI.Oobabooga/TextCompletion/TextCompletionStreamingResult.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Text; @@ -11,7 +12,8 @@ namespace Microsoft.SemanticKernel.Connectors.AI.Oobabooga.TextCompletion; -internal sealed class TextCompletionStreamingResult : ITextStreamingResult +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] +internal sealed class TextCompletionStreamingResult : ITextStreamingResult, ITextResult { private readonly List _modelResponses; private readonly Channel _responseChannel; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/AzureOpenAIClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/AzureOpenAIClientBase.cs index 4663f1bd8cf1..71f193e032c3 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/AzureOpenAIClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/AzureOpenAIClientBase.cs @@ -11,7 +11,9 @@ using Microsoft.SemanticKernel.Diagnostics; namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; - +/// +/// Base class for Azure OpenAI clients. +/// public abstract class AzureOpenAIClientBase : ClientBase { /// @@ -20,75 +22,68 @@ public abstract class AzureOpenAIClientBase : ClientBase private protected override OpenAIClient Client { get; } /// - /// Creates a new Azure OpenAI client instance using API Key auth + /// Initializes a new instance of the class using API Key authentication. /// /// Azure OpenAI model ID or deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. private protected AzureOpenAIClientBase( string modelId, string endpoint, string apiKey, HttpClient? httpClient = null, - ILogger? logger = null) : base(logger) + ILoggerFactory? loggerFactory = null) : base(loggerFactory) { Verify.NotNullOrWhiteSpace(modelId); Verify.NotNullOrWhiteSpace(endpoint); Verify.StartsWith(endpoint, "https://", "The Azure OpenAI endpoint must start with 'https://'"); Verify.NotNullOrWhiteSpace(apiKey); - var options = new OpenAIClientOptions(); - - if (httpClient != null) - { - options.Transport = new HttpClientTransport(httpClient); - } + var options = GetClientOptions(httpClient); this.ModelId = modelId; this.Client = new OpenAIClient(new Uri(endpoint), new AzureKeyCredential(apiKey), options); } /// - /// Creates a new Azure OpenAI client instance supporting AAD auth + /// Initializes a new instance of the class supporting AAD authentication. /// /// Azure OpenAI model ID or deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Token credential, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc. /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. private protected AzureOpenAIClientBase( string modelId, string endpoint, TokenCredential credential, HttpClient? httpClient = null, - ILogger? logger = null) : base(logger) + ILoggerFactory? loggerFactory = null) : base(loggerFactory) { Verify.NotNullOrWhiteSpace(modelId); Verify.NotNullOrWhiteSpace(endpoint); Verify.StartsWith(endpoint, "https://", "The Azure OpenAI endpoint must start with 'https://'"); - var options = new OpenAIClientOptions(); - if (httpClient != null) - { - options.Transport = new HttpClientTransport(httpClient); - } + var options = GetClientOptions(httpClient); this.ModelId = modelId; this.Client = new OpenAIClient(new Uri(endpoint), credential, options); } /// - /// Creates a new Azure OpenAI client instance using the specified OpenAIClient + /// Initializes a new instance of the class using the specified OpenAIClient. + /// Note: instances created this way might not have the default diagnostics settings, + /// it's up to the caller to configure the client. /// /// Azure OpenAI model ID or deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource /// Custom . - /// Application logger + /// The to use for logging. If null, no logging will be performed. private protected AzureOpenAIClientBase( string modelId, OpenAIClient openAIClient, - ILogger? logger = null) + ILoggerFactory? loggerFactory = null) : base(loggerFactory) { Verify.NotNullOrWhiteSpace(modelId); Verify.NotNull(openAIClient); @@ -97,6 +92,31 @@ private protected AzureOpenAIClientBase( this.Client = openAIClient; } + /// + /// Options used by the Azure OpenAI client, e.g. User Agent. + /// + /// Custom for HTTP requests. + /// An instance of . + private static OpenAIClientOptions GetClientOptions(HttpClient? httpClient) + { + var options = new OpenAIClientOptions + { + Diagnostics = + { + IsTelemetryEnabled = Telemetry.IsTelemetryEnabled, + ApplicationId = Telemetry.HttpUserAgent, + } + }; + + if (httpClient != null) + { + options.Transport = new HttpClientTransport(httpClient); + options.RetryPolicy = new RetryPolicy(maxRetries: 0); //Disabling Azure SDK retry policy to use the one provided by the custom HTTP client. + } + + return options; + } + /// /// Logs Azure OpenAI action details. /// diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatModelResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatModelResult.cs new file mode 100644 index 000000000000..8779bac9c9a6 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatModelResult.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Azure.AI.OpenAI; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// Represents a singular result of a chat completion. +public class ChatModelResult +{ + /// A unique identifier associated with this chat completion response. + public string Id { get; } + + /// + /// The first timestamp associated with generation activity for this completions response, + /// represented as seconds since the beginning of the Unix epoch of 00:00 on 1 Jan 1970. + /// + public DateTimeOffset Created { get; } + + /// + /// Content filtering results for zero or more prompts in the request. + /// + public IReadOnlyList PromptFilterResults { get; } + + /// + /// The completion choice associated with this completion result. + /// + public ChatChoice Choice { get; } + + /// Usage information for tokens processed and generated as part of this completions operation. + public CompletionsUsage Usage { get; } + + /// Initializes a new instance of TextModelResult. + /// A completions response object to populate the fields relative the response. + /// A choice object to populate the fields relative to the resulting choice. + internal ChatModelResult(ChatCompletions completionsData, ChatChoice choiceData) + { + this.Id = completionsData.Id; + this.Created = completionsData.Created; + this.PromptFilterResults = completionsData.PromptFilterResults; + this.Choice = choiceData; + this.Usage = completionsData.Usage; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatResult.cs index 67afa86457f7..19c1216ea482 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatResult.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatResult.cs @@ -18,7 +18,7 @@ public ChatResult(ChatCompletions resultData, ChatChoice choice) { Verify.NotNull(choice); this._choice = choice; - this.ModelResult = new ModelResult(resultData); + this.ModelResult = new(new ChatModelResult(resultData, choice)); } public ModelResult ModelResult { get; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatResultExtensions.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatResultExtensions.cs new file mode 100644 index 000000000000..fdd3475d28db --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatResultExtensions.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.AI.ChatCompletion; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Provides extension methods for the IChatResult interface. +/// +public static class ChatResultExtensions +{ + /// + /// Retrieve the resulting function from the chat result. + /// + /// + /// The , or null if no function was returned by the model. + public static OpenAIFunctionResponse? GetFunctionResponse(this IChatResult chatResult) + { + OpenAIFunctionResponse? functionResponse = null; + var functionCall = chatResult.ModelResult.GetResult().Choice.Message.FunctionCall; + if (functionCall is not null) + { + functionResponse = OpenAIFunctionResponse.FromFunctionCall(functionCall); + } + return functionResponse; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatStreamingResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatStreamingResult.cs index f9ae288381cd..011b779293b5 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatStreamingResult.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ChatStreamingResult.cs @@ -6,7 +6,6 @@ using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.ChatCompletion; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Diagnostics; @@ -14,7 +13,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; -internal sealed class ChatStreamingResult : IChatStreamingResult, ITextStreamingResult +internal sealed class ChatStreamingResult : IChatStreamingResult, ITextStreamingResult, IChatResult, ITextResult { private readonly ModelResult _modelResult; private readonly StreamingChatChoice _choice; @@ -37,7 +36,7 @@ public async Task GetChatMessageAsync(CancellationToken cancell if (chatMessage is null) { - throw new AIException(AIException.ErrorCodes.UnknownError, "Unable to get chat message from stream"); + throw new SKException("Unable to get chat message from stream"); } return new SKChatMessage(chatMessage); @@ -48,7 +47,10 @@ public async IAsyncEnumerable GetStreamingChatMessageAsync([Enu { await foreach (var message in this._choice.GetMessageStreaming(cancellationToken)) { - yield return new SKChatMessage(message); + if (message.Content is { Length: > 0 }) + { + yield return new SKChatMessage(message); + } } } @@ -63,7 +65,10 @@ public async IAsyncEnumerable GetCompletionStreamingAsync([EnumeratorCan { await foreach (var result in this.GetStreamingChatMessageAsync(cancellationToken).ConfigureAwait(false)) { - yield return result.Content; + if (result.Content is string content and { Length: > 0 }) + { + yield return content; + } } } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs index b9545640210a..c31ea4a316c0 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Metrics; using System.Linq; using System.Runtime.CompilerServices; using System.Threading; @@ -12,7 +13,6 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.ChatCompletion; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; using Microsoft.SemanticKernel.Diagnostics; @@ -22,14 +22,17 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; #pragma warning disable CA2208 // Instantiate argument exceptions correctly +/// +/// Base class for AI clients that provides common functionality for interacting with OpenAI services. +/// public abstract class ClientBase { private const int MaxResultsPerPrompt = 128; // Prevent external inheritors - private protected ClientBase(ILogger? logger = null) + private protected ClientBase(ILoggerFactory? loggerFactory = null) { - this.Logger = logger ?? NullLogger.Instance; + this.Logger = loggerFactory is not null ? loggerFactory.CreateLogger(this.GetType()) : NullLogger.Instance; } /// @@ -47,6 +50,35 @@ private protected ClientBase(ILogger? logger = null) /// private protected ILogger Logger { get; set; } + /// + /// Instance of for metrics. + /// + private static readonly Meter s_meter = new(typeof(ClientBase).Assembly.GetName().Name); + + /// + /// Instance of to keep track of the number of prompt tokens used. + /// + private static readonly Counter s_promptTokensCounter = + s_meter.CreateCounter( + name: "SK.Connectors.OpenAI.PromptTokens", + description: "Number of prompt tokens used"); + + /// + /// Instance of to keep track of the number of completion tokens used. + /// + private static readonly Counter s_completionTokensCounter = + s_meter.CreateCounter( + name: "SK.Connectors.OpenAI.CompletionTokens", + description: "Number of completion tokens used"); + + /// + /// Instance of to keep track of the total number of tokens used. + /// + private static readonly Counter s_totalTokensCounter = + s_meter.CreateCounter( + name: "SK.Connectors.OpenAI.TotalTokens", + description: "Total number of tokens used"); + /// /// Creates completions for the prompt and settings. /// @@ -56,29 +88,31 @@ private protected ClientBase(ILogger? logger = null) /// Completions generated by the remote model private protected async Task> InternalGetTextResultsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings, CancellationToken cancellationToken = default) { - Verify.NotNull(requestSettings); + OpenAIRequestSettings textRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings, OpenAIRequestSettings.DefaultTextMaxTokens); - ValidateMaxTokens(requestSettings.MaxTokens); - var options = CreateCompletionsOptions(text, requestSettings); + ValidateMaxTokens(textRequestSettings.MaxTokens); + var options = CreateCompletionsOptions(text, textRequestSettings); Response? response = await RunRequestAsync?>( () => this.Client.GetCompletionsAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false); - if (response == null) + if (response is null) { - throw new OpenAIInvalidResponseException(null, "Text completions null response"); + throw new SKException("Text completions null response"); } var responseData = response.Value; if (responseData.Choices.Count == 0) { - throw new OpenAIInvalidResponseException(responseData, "Text completions not found"); + throw new SKException("Text completions not found"); } + this.CaptureUsageDetails(responseData.Usage); + return responseData.Choices.Select(choice => new TextResult(responseData, choice)).ToList(); } @@ -91,13 +125,14 @@ private protected async Task> InternalGetTextResultsA /// Stream the completions generated by the remote model private protected async IAsyncEnumerable InternalGetTextStreamingResultsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - Verify.NotNull(requestSettings); + OpenAIRequestSettings textRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings, OpenAIRequestSettings.DefaultTextMaxTokens); + + ValidateMaxTokens(textRequestSettings.MaxTokens); - ValidateMaxTokens(requestSettings.MaxTokens); - var options = CreateCompletionsOptions(text, requestSettings); + var options = CreateCompletionsOptions(text, textRequestSettings); Response? response = await RunRequestAsync>( () => this.Client.GetCompletionsStreamingAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false); @@ -115,11 +150,11 @@ private protected async IAsyncEnumerable InternalGetTextStr /// List of strings to generate embeddings for /// The to monitor for cancellation requests. The default is . /// List of embeddings - private protected async Task>> InternalGetEmbeddingsAsync( + private protected async Task>> InternalGetEmbeddingsAsync( IList data, CancellationToken cancellationToken = default) { - var result = new List>(); + var result = new List>(data.Count); foreach (string text in data) { var options = new EmbeddingsOptions(text); @@ -127,19 +162,17 @@ private protected async Task>> InternalGetEmbeddingsAsync Response? response = await RunRequestAsync?>( () => this.Client.GetEmbeddingsAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false); - if (response == null) + if (response is null) { - throw new OpenAIInvalidResponseException(null, "Text embedding null response"); + throw new SKException("Text embedding null response"); } if (response.Value.Data.Count == 0) { - throw new OpenAIInvalidResponseException(response.Value, "Text embedding not found"); + throw new SKException("Text embedding not found"); } - EmbeddingItem x = response.Value.Data[0]; - - result.Add(new Embedding(x.Embedding, transferOwnership: true)); + result.Add(response.Value.Data[0].Embedding.ToArray()); } return result; @@ -149,34 +182,40 @@ private protected async Task>> InternalGetEmbeddingsAsync /// Generate a new chat message /// /// Chat history - /// AI request settings + /// AI request settings /// Async cancellation token /// Generated chat message in string format private protected async Task> InternalGetChatResultsAsync( ChatHistory chat, - ChatRequestSettings? chatSettings, + AIRequestSettings? requestSettings, CancellationToken cancellationToken = default) { Verify.NotNull(chat); - chatSettings ??= new(); - ValidateMaxTokens(chatSettings.MaxTokens); - var chatOptions = CreateChatCompletionsOptions(chatSettings, chat); + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + ValidateMaxTokens(chatRequestSettings.MaxTokens); + + var chatOptions = CreateChatCompletionsOptions(chatRequestSettings, chat); Response? response = await RunRequestAsync?>( () => this.Client.GetChatCompletionsAsync(this.ModelId, chatOptions, cancellationToken)).ConfigureAwait(false); - if (response == null) + if (response is null) { - throw new OpenAIInvalidResponseException(null, "Chat completions null response"); + throw new SKException("Chat completions null response"); } - if (response.Value.Choices.Count == 0) + var responseData = response.Value; + + if (responseData.Choices.Count == 0) { - throw new OpenAIInvalidResponseException(response.Value, "Chat completions not found"); + throw new SKException("Chat completions not found"); } - return response.Value.Choices.Select(chatChoice => new ChatResult(response.Value, chatChoice)).ToList(); + this.CaptureUsageDetails(responseData.Usage); + + return responseData.Choices.Select(chatChoice => new ChatResult(responseData, chatChoice)).ToList(); } /// @@ -188,22 +227,23 @@ private protected async Task> InternalGetChatResultsA /// Streaming of generated chat message in string format private protected async IAsyncEnumerable InternalGetChatStreamingResultsAsync( IEnumerable chat, - ChatRequestSettings? requestSettings, + AIRequestSettings? requestSettings, [EnumeratorCancellation] CancellationToken cancellationToken = default) { Verify.NotNull(chat); - requestSettings ??= new(); - ValidateMaxTokens(requestSettings.MaxTokens); + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + ValidateMaxTokens(chatRequestSettings.MaxTokens); - var options = CreateChatCompletionsOptions(requestSettings, chat); + var options = CreateChatCompletionsOptions(chatRequestSettings, chat); Response? response = await RunRequestAsync>( () => this.Client.GetChatCompletionsStreamingAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false); if (response is null) { - throw new OpenAIInvalidResponseException(null, "Chat completions null response"); + throw new SKException("Chat completions null response"); } using StreamingChatCompletions streamingChatCompletions = response.Value; @@ -225,11 +265,10 @@ private protected static OpenAIChatHistory InternalCreateNewChat(string? instruc private protected async Task> InternalGetChatResultsAsTextAsync( string text, - CompleteRequestSettings? textSettings, + AIRequestSettings? requestSettings, CancellationToken cancellationToken = default) { - textSettings ??= new(); - ChatHistory chat = PrepareChatHistory(text, textSettings, out ChatRequestSettings chatSettings); + ChatHistory chat = PrepareChatHistory(text, requestSettings, out OpenAIRequestSettings chatSettings); return (await this.InternalGetChatResultsAsync(chat, chatSettings, cancellationToken).ConfigureAwait(false)) .OfType() @@ -238,35 +277,27 @@ private protected async Task> InternalGetChatResultsA private protected async IAsyncEnumerable InternalGetChatStreamingResultsAsTextAsync( string text, - CompleteRequestSettings? textSettings, + AIRequestSettings? requestSettings, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - ChatHistory chat = PrepareChatHistory(text, textSettings, out ChatRequestSettings chatSettings); + ChatHistory chat = PrepareChatHistory(text, requestSettings, out OpenAIRequestSettings chatSettings); - await foreach (var chatCompletionStreamingResult in this.InternalGetChatStreamingResultsAsync(chat, chatSettings, cancellationToken)) + IAsyncEnumerable chatCompletionStreamingResults = this.InternalGetChatStreamingResultsAsync(chat, chatSettings, cancellationToken); + await foreach (var chatCompletionStreamingResult in chatCompletionStreamingResults) { yield return (ITextStreamingResult)chatCompletionStreamingResult; } } - private static OpenAIChatHistory PrepareChatHistory(string text, CompleteRequestSettings? requestSettings, out ChatRequestSettings settings) + private static OpenAIChatHistory PrepareChatHistory(string text, AIRequestSettings? requestSettings, out OpenAIRequestSettings settings) { - requestSettings ??= new(); - var chat = InternalCreateNewChat(requestSettings.ChatSystemPrompt); + settings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + var chat = InternalCreateNewChat(settings.ChatSystemPrompt); chat.AddUserMessage(text); - settings = new ChatRequestSettings - { - MaxTokens = requestSettings.MaxTokens, - Temperature = requestSettings.Temperature, - TopP = requestSettings.TopP, - PresencePenalty = requestSettings.PresencePenalty, - FrequencyPenalty = requestSettings.FrequencyPenalty, - StopSequences = requestSettings.StopSequences, - }; return chat; } - private static CompletionsOptions CreateCompletionsOptions(string text, CompleteRequestSettings requestSettings) + private static CompletionsOptions CreateCompletionsOptions(string text, OpenAIRequestSettings requestSettings) { if (requestSettings.ResultsPerPrompt is < 1 or > MaxResultsPerPrompt) { @@ -304,7 +335,7 @@ private static CompletionsOptions CreateCompletionsOptions(string text, Complete return options; } - private static ChatCompletionsOptions CreateChatCompletionsOptions(ChatRequestSettings requestSettings, IEnumerable chatHistory) + private static ChatCompletionsOptions CreateChatCompletionsOptions(OpenAIRequestSettings requestSettings, IEnumerable chatHistory) { if (requestSettings.ResultsPerPrompt is < 1 or > MaxResultsPerPrompt) { @@ -318,9 +349,32 @@ private static ChatCompletionsOptions CreateChatCompletionsOptions(ChatRequestSe NucleusSamplingFactor = (float?)requestSettings.TopP, FrequencyPenalty = (float?)requestSettings.FrequencyPenalty, PresencePenalty = (float?)requestSettings.PresencePenalty, - ChoiceCount = requestSettings.ResultsPerPrompt + ChoiceCount = requestSettings.ResultsPerPrompt, }; + if (requestSettings.Functions is not null) + { + if (requestSettings.FunctionCall == OpenAIRequestSettings.FunctionCallAuto) + { + options.FunctionCall = FunctionDefinition.Auto; + options.Functions = requestSettings.Functions.Select(f => f.ToFunctionDefinition()).ToList(); + } + else if (requestSettings.FunctionCall != OpenAIRequestSettings.FunctionCallNone + && !requestSettings.FunctionCall.IsNullOrEmpty()) + { + var filteredFunctions = requestSettings.Functions + .Where(f => f.FullyQualifiedName.Equals(requestSettings.FunctionCall, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + OpenAIFunction? function = filteredFunctions.FirstOrDefault(); + if (function is not null) + { + options.FunctionCall = function.ToFunctionDefinition(); + options.Functions = filteredFunctions.Select(f => f.ToFunctionDefinition()).ToList(); + } + } + } + foreach (var keyValue in requestSettings.TokenSelectionBiases) { options.TokenSelectionBiases.Add(keyValue.Key, keyValue.Value); @@ -361,9 +415,7 @@ private static void ValidateMaxTokens(int? maxTokens) { if (maxTokens.HasValue && maxTokens < 1) { - throw new AIException( - AIException.ErrorCodes.InvalidRequest, - $"MaxTokens {maxTokens} is not valid, the value must be greater than zero"); + throw new SKException($"MaxTokens {maxTokens} is not valid, the value must be greater than zero"); } } @@ -375,78 +427,22 @@ private static async Task RunRequestAsync(Func> request) } catch (RequestFailedException e) { - switch (e.Status) - { - case (int)HttpStatusCodeType.BadRequest: - case (int)HttpStatusCodeType.MethodNotAllowed: - case (int)HttpStatusCodeType.NotFound: - case (int)HttpStatusCodeType.NotAcceptable: - case (int)HttpStatusCodeType.Conflict: - case (int)HttpStatusCodeType.Gone: - case (int)HttpStatusCodeType.LengthRequired: - case (int)HttpStatusCodeType.PreconditionFailed: - case (int)HttpStatusCodeType.RequestEntityTooLarge: - case (int)HttpStatusCodeType.RequestUriTooLong: - case (int)HttpStatusCodeType.UnsupportedMediaType: - case (int)HttpStatusCodeType.RequestedRangeNotSatisfiable: - case (int)HttpStatusCodeType.ExpectationFailed: - case (int)HttpStatusCodeType.HttpVersionNotSupported: - case (int)HttpStatusCodeType.UpgradeRequired: - case (int)HttpStatusCodeType.MisdirectedRequest: - case (int)HttpStatusCodeType.UnprocessableEntity: - case (int)HttpStatusCodeType.Locked: - case (int)HttpStatusCodeType.FailedDependency: - case (int)HttpStatusCodeType.PreconditionRequired: - case (int)HttpStatusCodeType.RequestHeaderFieldsTooLarge: - throw new AIException( - AIException.ErrorCodes.InvalidRequest, - $"The request is not valid, HTTP status: {e.Status}", - e.Message, e); - - case (int)HttpStatusCodeType.Unauthorized: - case (int)HttpStatusCodeType.Forbidden: - case (int)HttpStatusCodeType.ProxyAuthenticationRequired: - case (int)HttpStatusCodeType.UnavailableForLegalReasons: - case (int)HttpStatusCodeType.NetworkAuthenticationRequired: - throw new AIException( - AIException.ErrorCodes.AccessDenied, - $"The request is not authorized, HTTP status: {e.Status}", - e.Message, e); - - case (int)HttpStatusCodeType.RequestTimeout: - throw new AIException( - AIException.ErrorCodes.RequestTimeout, - $"The request timed out, HTTP status: {e.Status}"); - - case (int)HttpStatusCodeType.TooManyRequests: - throw new AIException( - AIException.ErrorCodes.Throttling, - $"Too many requests, HTTP status: {e.Status}", - e.Message, e); - - case (int)HttpStatusCodeType.InternalServerError: - case (int)HttpStatusCodeType.NotImplemented: - case (int)HttpStatusCodeType.BadGateway: - case (int)HttpStatusCodeType.ServiceUnavailable: - case (int)HttpStatusCodeType.GatewayTimeout: - case (int)HttpStatusCodeType.InsufficientStorage: - throw new AIException( - AIException.ErrorCodes.ServiceError, - $"The service failed to process the request, HTTP status:{e.Status}", - e.Message, e); - - default: - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Unexpected HTTP response, status: {e.Status}", - e.Message, e); - } - } - catch (Exception e) when (e is not AIException) - { - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Something went wrong: {e.Message}", e); + throw e.ToHttpOperationException(); } } + + /// + /// Captures usage details, including token information. + /// + /// Instance of with usage details. + private void CaptureUsageDetails(CompletionsUsage usage) + { + this.Logger.LogInformation( + "Prompt tokens: {PromptTokens}. Completion tokens: {CompletionTokens}. Total tokens: {TotalTokens}.", + usage.PromptTokens, usage.CompletionTokens, usage.TotalTokens); + + s_promptTokensCounter.Add(usage.PromptTokens); + s_completionTokensCounter.Add(usage.CompletionTokens); + s_totalTokensCounter.Add(usage.TotalTokens); + } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/FunctionCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/FunctionCollectionExtensions.cs new file mode 100644 index 000000000000..45109a3296e1 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/FunctionCollectionExtensions.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Extension methods for . +/// +public static class FunctionCollectionExtensions +{ + /// + /// Given an object, tries to retrieve the corresponding and populate with its parameters. + /// + /// The SK function collection. + /// The object. + /// When this method returns, the function that was retrieved if one with the specified name was found; otherwise, + /// When this method returns, the context variables containing parameters for the function; otherwise, + /// if the function was found; otherwise, . + public static bool TryGetFunctionAndContext( + this IReadOnlyFunctionCollection functionCollection, + OpenAIFunctionResponse response, + [NotNullWhen(true)] out ISKFunction? availableFunction, + [NotNullWhen(true)] out ContextVariables? availableContext) + { + availableFunction = null; + availableContext = null; + + if (!functionCollection.TryGetFunction(response.PluginName, response.FunctionName, out availableFunction)) + { + if (!functionCollection.TryGetFunction(response.FunctionName, out availableFunction)) + { + // Function not found in collection + return false; + } + } + + // Add parameters to context variables + availableContext = new ContextVariables(); + foreach (var parameter in response.Parameters) + { + availableContext.Set(parameter.Key, parameter.Value.ToString()); + } + + return true; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/FunctionViewExtensions.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/FunctionViewExtensions.cs new file mode 100644 index 000000000000..70eab2739773 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/FunctionViewExtensions.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Extensions for specific to the OpenAI connector. +/// +public static class FunctionViewExtensions +{ + /// + /// Convert a to an . + /// + /// The object to convert. + /// An object. + public static OpenAIFunction ToOpenAIFunction(this FunctionView functionView) + { + var openAIParams = new List(); + foreach (ParameterView param in functionView.Parameters) + { + openAIParams.Add(new OpenAIFunctionParameter + { + Name = param.Name, + Description = (param.Description ?? string.Empty) + + (string.IsNullOrEmpty(param.DefaultValue) ? string.Empty : $" (default value: {param.DefaultValue})"), + Type = param.Type?.Name ?? "string", + IsRequired = param.IsRequired ?? false + }); + } + + return new OpenAIFunction + { + FunctionName = functionView.Name, + PluginName = functionView.PluginName, + Description = functionView.Description, + Parameters = openAIParams, + }; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIClientBase.cs index 25750230dbf7..ccd33cb90a4e 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIClientBase.cs @@ -10,6 +10,9 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +/// +/// Base class for OpenAI clients, providing common functionality and properties. +/// public abstract class OpenAIClientBase : ClientBase { /// @@ -18,30 +21,26 @@ public abstract class OpenAIClientBase : ClientBase private protected override OpenAIClient Client { get; } /// - /// Create an instance of the OpenAI connector + /// Initializes a new instance of the class. /// - /// Model name - /// OpenAI API Key - /// OpenAI Organization Id (usually optional) + /// Model name. + /// OpenAI API Key. + /// OpenAI Organization Id (usually optional). /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. private protected OpenAIClientBase( string modelId, string apiKey, string? organization = null, HttpClient? httpClient = null, - ILogger? logger = null) : base(logger) + ILoggerFactory? loggerFactory = null) : base(loggerFactory) { Verify.NotNullOrWhiteSpace(modelId); Verify.NotNullOrWhiteSpace(apiKey); this.ModelId = modelId; - var options = new OpenAIClientOptions(); - if (httpClient != null) - { - options.Transport = new HttpClientTransport(httpClient); - } + var options = GetClientOptions(httpClient); if (!string.IsNullOrWhiteSpace(organization)) { @@ -51,6 +50,26 @@ private protected OpenAIClientBase( this.Client = new OpenAIClient(apiKey, options); } + /// + /// Initializes a new instance of the class using the specified OpenAIClient. + /// Note: instances created this way might not have the default diagnostics settings, + /// it's up to the caller to configure the client. + /// + /// Azure OpenAI model ID or deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Custom . + /// The to use for logging. If null, no logging will be performed. + private protected OpenAIClientBase( + string modelId, + OpenAIClient openAIClient, + ILoggerFactory? loggerFactory = null) : base(loggerFactory) + { + Verify.NotNullOrWhiteSpace(modelId); + Verify.NotNull(openAIClient); + + this.ModelId = modelId; + this.Client = openAIClient; + } + /// /// Logs OpenAI action details. /// @@ -59,4 +78,29 @@ private protected void LogActionDetails([CallerMemberName] string? callerMemberN { this.Logger.LogInformation("Action: {Action}. OpenAI Model ID: {ModelId}.", callerMemberName, this.ModelId); } + + /// + /// Options used by the OpenAI client, e.g. User Agent. + /// + /// Custom for HTTP requests. + /// An instance of . + private static OpenAIClientOptions GetClientOptions(HttpClient? httpClient) + { + var options = new OpenAIClientOptions + { + Diagnostics = + { + IsTelemetryEnabled = Telemetry.IsTelemetryEnabled, + ApplicationId = Telemetry.HttpUserAgent, + } + }; + + if (httpClient != null) + { + options.Transport = new HttpClientTransport(httpClient); + options.RetryPolicy = new RetryPolicy(maxRetries: 0); //Disabling Azure SDK retry policy to use the one provided by the custom HTTP client. + } + + return options; + } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunction.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunction.cs new file mode 100644 index 000000000000..5bab87d6d3d7 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunction.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Azure.AI.OpenAI; +using Microsoft.SemanticKernel.Text; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Represents a function parameter that can be pass to the OpenAI API +/// +public class OpenAIFunctionParameter +{ + /// + /// Name of the parameter. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Description of the parameter. + /// + public string Description { get; set; } = string.Empty; + + /// + /// Type of the parameter. + /// + public string Type { get; set; } = string.Empty; + + /// + /// Whether the parameter is required or not. + /// + public bool IsRequired { get; set; } = false; +} + +/// +/// Represents a function that can be pass to the OpenAI API +/// +public class OpenAIFunction +{ + /// + /// Separator between the plugin name and the function name + /// + public const string NameSeparator = "-"; + + /// + /// Name of the function + /// + public string FunctionName { get; set; } = string.Empty; + + /// + /// Name of the function's associated plugin, if applicable + /// + public string PluginName { get; set; } = string.Empty; + + /// + /// Fully qualified name of the function. This is the concatenation of the plugin name and the function name, + /// separated by the value of . + /// If there is no plugin name, this is the same as the function name. + /// + public string FullyQualifiedName => + this.PluginName.IsNullOrEmpty() ? this.FunctionName : string.Join(NameSeparator, this.PluginName, this.FunctionName); + + /// + /// Description of the function + /// + public string Description { get; set; } = string.Empty; + + /// + /// List of parameters for the function + /// + public IList Parameters { get; set; } = new List(); + + /// + /// Converts the to OpenAI's . + /// + /// A containing all the function information. + public FunctionDefinition ToFunctionDefinition() + { + var requiredParams = new List(); + + var paramProperties = new Dictionary(); + foreach (var param in this.Parameters) + { + paramProperties.Add( + param.Name, + new + { + type = param.Type, + description = param.Description, + }); + + if (param.IsRequired) + { + requiredParams.Add(param.Name); + } + } + return new FunctionDefinition + { + Name = this.FullyQualifiedName, + Description = this.Description, + Parameters = BinaryData.FromObjectAsJson( + new + { + type = "object", + properties = paramProperties, + required = requiredParams, + }), + }; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunctionResponse.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunctionResponse.cs new file mode 100644 index 000000000000..fc77d866f85e --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunctionResponse.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Azure.AI.OpenAI; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Object containing function information and parameter values for a function call generated by the OpenAI model. +/// +public class OpenAIFunctionResponse +{ + /// + /// Name of the function chosen + /// + public string FunctionName { get; set; } = string.Empty; + + /// + /// Name of the function's associated plugin, if applicable + /// + public string PluginName { get; set; } = string.Empty; + + /// + /// Parameter values + /// + public Dictionary Parameters { get; set; } = new(); + + /// + /// Parses the function call and parameter information generated by the model. + /// + /// The OpenAI function call object generated by the model. + /// Instance of . + public static OpenAIFunctionResponse FromFunctionCall(FunctionCall functionCall) + { + OpenAIFunctionResponse response = new(); + if (functionCall.Name.Contains(OpenAIFunction.NameSeparator)) + { + var parts = functionCall.Name.Split(new string[] { OpenAIFunction.NameSeparator }, StringSplitOptions.RemoveEmptyEntries); + response.PluginName = parts[0]; + response.FunctionName = parts[1]; + } + else + { + response.FunctionName = functionCall.Name; + } + + var parameters = JsonSerializer.Deserialize>(functionCall.Arguments); + if (parameters is not null) + { + response.Parameters = parameters; + } + + return response; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIInvalidResponseException{T}.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIInvalidResponseException{T}.cs deleted file mode 100644 index 3b2c5042853a..000000000000 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIInvalidResponseException{T}.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel.AI; - -#pragma warning disable RCS1194 // Implement exception constructors. - -namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; - -internal sealed class OpenAIInvalidResponseException : AIException -{ - public T? ResponseData { get; } - - public OpenAIInvalidResponseException(T? responseData, string? message = null) : base(ErrorCodes.InvalidResponseContent, message) - { - this.ResponseData = responseData; - } -} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/RequestFailedExceptionExtensions.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/RequestFailedExceptionExtensions.cs new file mode 100644 index 000000000000..0c14eee46437 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/RequestFailedExceptionExtensions.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net; +using Azure; +using Microsoft.SemanticKernel.Diagnostics; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Provides extension methods for the class. +/// +public static class RequestFailedExceptionExtensions +{ + /// + /// Converts a to an . + /// + /// The original . + /// An instance. + public static HttpOperationException ToHttpOperationException(this RequestFailedException exception) + { + const int NoResponseReceived = 0; + + string? responseContent = null; + + try + { + responseContent = exception.GetRawResponse()?.Content?.ToString(); + } +#pragma warning disable CA1031 // Do not catch general exception types + catch { } // We want to suppress any exceptions that occur while reading the content, ensuring that an HttpOperationException is thrown instead. +#pragma warning restore CA1031 + + return new HttpOperationException( + exception.Status == NoResponseReceived ? null : (HttpStatusCode?)exception.Status, + responseContent, + exception.Message, + exception); + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/SKChatMessage.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/SKChatMessage.cs index 8ee6a8daf327..f8966e0aafb8 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/SKChatMessage.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/SKChatMessage.cs @@ -10,11 +10,21 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; public class SKChatMessage : ChatMessageBase { /// - /// Create a new instance of a chat message + /// Initializes a new instance of the class. /// /// OpenAI SDK chat message representation public SKChatMessage(Azure.AI.OpenAI.ChatMessage message) : base(new AuthorRole(message.Role.ToString()), message.Content) { } + + /// + /// Initializes a new instance of the class. + /// + /// Role of the author of the message. + /// Content of the message. + public SKChatMessage(string role, string content) + : base(new AuthorRole(role), content) + { + } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextModelResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextModelResult.cs new file mode 100644 index 000000000000..72a4ac0b66f8 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextModelResult.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Azure.AI.OpenAI; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// Represents a singular result of a text completion. +public sealed class TextModelResult +{ + /// A unique identifier associated with this text completion response. + public string Id { get; } + + /// + /// The first timestamp associated with generation activity for this completions response, + /// represented as seconds since the beginning of the Unix epoch of 00:00 on 1 Jan 1970. + /// + public DateTimeOffset Created { get; } + + /// + /// Content filtering results for zero or more prompts in the request. + /// + public IReadOnlyList PromptFilterResults { get; } + + /// + /// The completion choice associated with this completion result. + /// + public Choice Choice { get; } + + /// Usage information for tokens processed and generated as part of this completions operation. + public CompletionsUsage Usage { get; } + + /// Initializes a new instance of TextModelResult. + /// A completions response object to populate the fields relative the response. + /// A choice object to populate the fields relative to the resulting choice. + internal TextModelResult(Completions completionsData, Choice choiceData) + { + this.Id = completionsData.Id; + this.Created = completionsData.Created; + this.PromptFilterResults = completionsData.PromptFilterResults; + this.Choice = choiceData; + this.Usage = completionsData.Usage; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextResult.cs index 8448cba4a2a2..44893f9cd6b3 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextResult.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextResult.cs @@ -15,7 +15,7 @@ internal sealed class TextResult : ITextResult public TextResult(Completions resultData, Choice choice) { - this._modelResult = new ModelResult(resultData); + this._modelResult = new(new TextModelResult(resultData, choice)); this._choice = choice; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextStreamingResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextStreamingResult.cs index 553f4aac76b9..d2c5537cc0c0 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextStreamingResult.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/TextStreamingResult.cs @@ -10,7 +10,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; -internal sealed class TextStreamingResult : ITextStreamingResult +internal sealed class TextStreamingResult : ITextStreamingResult, ITextResult { private readonly StreamingChoice _choice; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureChatCompletion.cs index 27838f2a8f49..dc4d7ed845dd 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureChatCompletion.cs @@ -7,6 +7,7 @@ using Azure.AI.OpenAI; using Azure.Core; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.ChatCompletion; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; @@ -26,13 +27,13 @@ public sealed class AzureChatCompletion : AzureOpenAIClientBase, IChatCompletion /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public AzureChatCompletion( string modelId, string endpoint, string apiKey, HttpClient? httpClient = null, - ILogger? logger = null) : base(modelId, endpoint, apiKey, httpClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, endpoint, apiKey, httpClient, loggerFactory) { } @@ -43,13 +44,13 @@ public AzureChatCompletion( /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Token credentials, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc. /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public AzureChatCompletion( string modelId, string endpoint, TokenCredential credentials, HttpClient? httpClient = null, - ILogger? logger = null) : base(modelId, endpoint, credentials, httpClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, endpoint, credentials, httpClient, loggerFactory) { } @@ -58,18 +59,18 @@ public AzureChatCompletion( /// /// Azure OpenAI model ID or deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource /// Custom . - /// Application logger + /// The to use for logging. If null, no logging will be performed. public AzureChatCompletion( string modelId, OpenAIClient openAIClient, - ILogger? logger = null) : base(modelId, openAIClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, openAIClient, loggerFactory) { } /// public Task> GetChatCompletionsAsync( ChatHistory chat, - ChatRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.LogActionDetails(); @@ -79,7 +80,7 @@ public Task> GetChatCompletionsAsync( /// public IAsyncEnumerable GetStreamingChatCompletionsAsync( ChatHistory chat, - ChatRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.LogActionDetails(); @@ -95,7 +96,7 @@ public ChatHistory CreateNewChat(string? instructions = null) /// public IAsyncEnumerable GetStreamingCompletionsAsync( string text, - CompleteRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.LogActionDetails(); @@ -105,7 +106,7 @@ public IAsyncEnumerable GetStreamingCompletionsAsync( /// public Task> GetCompletionsAsync( string text, - CompleteRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.LogActionDetails(); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs index d444125bff98..a54acfd2fd89 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs @@ -4,7 +4,9 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Azure.AI.OpenAI; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.ChatCompletion; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; @@ -24,21 +26,33 @@ public sealed class OpenAIChatCompletion : OpenAIClientBase, IChatCompletion, IT /// OpenAI API Key /// OpenAI Organization Id (usually optional) /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public OpenAIChatCompletion( string modelId, string apiKey, string? organization = null, HttpClient? httpClient = null, - ILogger? logger = null - ) : base(modelId, apiKey, organization, httpClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, apiKey, organization, httpClient, loggerFactory) + { + } + + /// + /// Create an instance of the OpenAI chat completion connector + /// + /// Model name + /// Custom for HTTP requests. + /// The to use for logging. If null, no logging will be performed. + public OpenAIChatCompletion( + string modelId, + OpenAIClient openAIClient, + ILoggerFactory? loggerFactory = null) : base(modelId, openAIClient, loggerFactory) { } /// public Task> GetChatCompletionsAsync( ChatHistory chat, - ChatRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.LogActionDetails(); @@ -48,7 +62,7 @@ public Task> GetChatCompletionsAsync( /// public IAsyncEnumerable GetStreamingChatCompletionsAsync( ChatHistory chat, - ChatRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.LogActionDetails(); @@ -64,7 +78,7 @@ public ChatHistory CreateNewChat(string? instructions = null) /// public IAsyncEnumerable GetStreamingCompletionsAsync( string text, - CompleteRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.LogActionDetails(); @@ -74,7 +88,7 @@ public IAsyncEnumerable GetStreamingCompletionsAsync( /// public Task> GetCompletionsAsync( string text, - CompleteRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.LogActionDetails(); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureChatCompletionWithData.cs new file mode 100644 index 000000000000..6ad5b799016b --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureChatCompletionWithData.cs @@ -0,0 +1,317 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Text; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +/// +/// Azure OpenAI Chat Completion with data client. +/// More information: +/// +public sealed class AzureChatCompletionWithData : IChatCompletion, ITextCompletion +{ + /// + /// Initializes a new instance of the class. + /// + /// Instance of class with completion configuration. + /// Custom for HTTP requests. + /// Instance of to use for logging. + public AzureChatCompletionWithData( + AzureChatCompletionWithDataConfig config, + HttpClient? httpClient = null, + ILoggerFactory? loggerFactory = null) + { + this.ValidateConfig(config); + + this._config = config; + + this._httpClient = httpClient ?? new HttpClient(NonDisposableHttpClientHandler.Instance, disposeHandler: false); + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(this.GetType()) : NullLogger.Instance; + } + + /// + public ChatHistory CreateNewChat(string? instructions = null) + { + return new OpenAIChatHistory(instructions); + } + + /// + public async Task> GetChatCompletionsAsync( + ChatHistory chat, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + Verify.NotNull(chat); + + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + ValidateMaxTokens(chatRequestSettings.MaxTokens); + + return await this.ExecuteCompletionRequestAsync(chat, chatRequestSettings, cancellationToken).ConfigureAwait(false); + } + + /// + public IAsyncEnumerable GetStreamingChatCompletionsAsync( + ChatHistory chat, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + Verify.NotNull(chat); + + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + ValidateMaxTokens(chatRequestSettings.MaxTokens); + + return this.ExecuteCompletionStreamingRequestAsync(chat, chatRequestSettings, cancellationToken); + } + + /// + public async Task> GetCompletionsAsync( + string text, + AIRequestSettings? requestSettings, + CancellationToken cancellationToken = default) + { + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + var chat = this.PrepareChatHistory(text, chatRequestSettings); + + return (await this.GetChatCompletionsAsync(chat, chatRequestSettings, cancellationToken).ConfigureAwait(false)) + .OfType() + .ToList(); + } + + /// + public async IAsyncEnumerable GetStreamingCompletionsAsync( + string text, + AIRequestSettings? requestSettings, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + var chat = this.PrepareChatHistory(text, chatRequestSettings); + + IAsyncEnumerable results = this.GetStreamingChatCompletionsAsync(chat, chatRequestSettings, cancellationToken); + await foreach (var result in results) + { + yield return (ITextStreamingResult)result; + } + } + + #region private ================================================================================ + + private const string DefaultApiVersion = "2023-06-01-preview"; + + private readonly AzureChatCompletionWithDataConfig _config; + + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + private void ValidateConfig(AzureChatCompletionWithDataConfig config) + { + Verify.NotNull(config); + + Verify.NotNullOrWhiteSpace(config.CompletionModelId); + Verify.NotNullOrWhiteSpace(config.CompletionEndpoint); + Verify.NotNullOrWhiteSpace(config.CompletionApiKey); + Verify.NotNullOrWhiteSpace(config.DataSourceEndpoint); + Verify.NotNullOrWhiteSpace(config.DataSourceApiKey); + Verify.NotNullOrWhiteSpace(config.DataSourceIndex); + } + + private static void ValidateMaxTokens(int? maxTokens) + { + if (maxTokens.HasValue && maxTokens < 1) + { + throw new SKException($"MaxTokens {maxTokens} is not valid, the value must be greater than zero"); + } + } + + private async Task> ExecuteCompletionRequestAsync( + ChatHistory chat, + OpenAIRequestSettings requestSettings, + CancellationToken cancellationToken = default) + { + using var request = this.GetRequest(chat, requestSettings, isStreamEnabled: false); + using var response = await this.SendRequestAsync(request, cancellationToken).ConfigureAwait(false); + + var body = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); + + var chatWithDataResponse = this.DeserializeResponse(body); + + return chatWithDataResponse.Choices.Select(choice => new ChatWithDataResult(chatWithDataResponse, choice)).ToList(); + } + + private async IAsyncEnumerable ExecuteCompletionStreamingRequestAsync( + ChatHistory chat, + OpenAIRequestSettings requestSettings, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + using var request = this.GetRequest(chat, requestSettings, isStreamEnabled: true); + using var response = await this.SendRequestAsync(request, cancellationToken).ConfigureAwait(false); + + await foreach (var result in this.GetStreamingResultsAsync(response)) + { + yield return result; + } + } + + private async Task SendRequestAsync( + HttpRequestMessage request, + CancellationToken cancellationToken = default) + { + request.Headers.Add("User-Agent", Telemetry.HttpUserAgent); + request.Headers.Add("Api-Key", this._config.CompletionApiKey); + + try + { + return await this._httpClient.SendWithSuccessCheckAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HttpOperationException ex) + { + this._logger.LogError( + "Error occurred on chat completion with data request execution: {ExceptionMessage}", ex.Message); + + throw; + } + } + + private async IAsyncEnumerable GetStreamingResultsAsync(HttpResponseMessage response) + { + const string ServerEventPayloadPrefix = "data:"; + + using var stream = await response.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false); + using var reader = new StreamReader(stream); + + while (!reader.EndOfStream) + { + var body = await reader.ReadLineAsync().ConfigureAwait(false); + + if (string.IsNullOrWhiteSpace(body)) + { + continue; + } + + if (body.StartsWith(ServerEventPayloadPrefix, StringComparison.Ordinal)) + { + body = body.Substring(ServerEventPayloadPrefix.Length); + } + + var chatWithDataResponse = this.DeserializeResponse(body); + + foreach (var choice in chatWithDataResponse.Choices) + { + yield return new ChatWithDataStreamingResult(chatWithDataResponse, choice); + } + } + } + + private T DeserializeResponse(string body) + { + var response = Json.Deserialize(body); + + if (response is null) + { + const string ErrorMessage = "Error occurred on chat completion with data response deserialization"; + + this._logger.LogError(ErrorMessage); + + throw new SKException(ErrorMessage); + } + + return response; + } + + private HttpRequestMessage GetRequest( + ChatHistory chat, + OpenAIRequestSettings requestSettings, + bool isStreamEnabled) + { + var payload = new ChatWithDataRequest + { + Temperature = requestSettings.Temperature, + TopP = requestSettings.TopP, + IsStreamEnabled = isStreamEnabled, + StopSequences = requestSettings.StopSequences, + MaxTokens = requestSettings.MaxTokens, + PresencePenalty = requestSettings.PresencePenalty, + FrequencyPenalty = requestSettings.FrequencyPenalty, + TokenSelectionBiases = requestSettings.TokenSelectionBiases, + DataSources = this.GetDataSources(), + Messages = this.GetMessages(chat) + }; + + return HttpRequest.CreatePostRequest(this.GetRequestUri(), payload); + } + + private List GetDataSources() + { + return new List + { + new() { + Parameters = new ChatWithDataSourceParameters + { + Endpoint = this._config.DataSourceEndpoint, + ApiKey = this._config.DataSourceApiKey, + IndexName = this._config.DataSourceIndex + } + } + }; + } + + private List GetMessages(ChatHistory chat) + { + return chat + .Select(message => new ChatWithDataMessage + { + Role = message.Role.Label, + Content = message.Content + }) + .ToList(); + } + + private ChatHistory PrepareChatHistory(string text, OpenAIRequestSettings requestSettings) + { + var chat = this.CreateNewChat(requestSettings.ChatSystemPrompt); + + chat.AddUserMessage(text); + + return chat; + } + + private string GetRequestUri() + { + const string EndpointUriFormat = "{0}/openai/deployments/{1}/extensions/chat/completions?api-version={2}"; + + var apiVersion = this._config.CompletionApiVersion; + + if (string.IsNullOrWhiteSpace(apiVersion)) + { + apiVersion = DefaultApiVersion; + } + + return string.Format( + CultureInfo.InvariantCulture, + EndpointUriFormat, + this._config.CompletionEndpoint.TrimEnd('/'), + this._config.CompletionModelId, + apiVersion); + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureChatCompletionWithDataConfig.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureChatCompletionWithDataConfig.cs new file mode 100644 index 000000000000..dc4d58bb503c --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureChatCompletionWithDataConfig.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +/// +/// Required configuration for Azure OpenAI chat completion with data. +/// More information: +/// +public class AzureChatCompletionWithDataConfig +{ + /// + /// Azure OpenAI model ID or deployment name, see + /// + public string CompletionModelId { get; set; } = string.Empty; + + /// + /// Azure OpenAI deployment URL, see + /// + public string CompletionEndpoint { get; set; } = string.Empty; + + /// + /// Azure OpenAI API key, see + /// + public string CompletionApiKey { get; set; } = string.Empty; + + /// + /// Azure OpenAI Completion API version (e.g. 2023-06-01-preview) + /// + public string CompletionApiVersion { get; set; } = string.Empty; + + /// + /// Data source endpoint URL. + /// For Azure Cognitive Search, see + /// + public string DataSourceEndpoint { get; set; } = string.Empty; + + /// + /// Data source API key. + /// For Azure Cognitive Search keys, see + /// + public string DataSourceApiKey { get; set; } = string.Empty; + + /// + /// Data source index name. + /// For Azure Cognitive Search indexes, see + /// + public string DataSourceIndex { get; set; } = string.Empty; +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataChoice.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataChoice.cs new file mode 100644 index 000000000000..842d236d0586 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataChoice.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +[SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Used for JSON deserialization")] +internal sealed class ChatWithDataChoice +{ + [JsonPropertyName("messages")] + public IList Messages { get; set; } = Array.Empty(); +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataMessage.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataMessage.cs new file mode 100644 index 000000000000..533d87d30cee --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataMessage.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +internal sealed class ChatWithDataMessage +{ + [JsonPropertyName("role")] + public string Role { get; set; } = string.Empty; + + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataModelResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataModelResult.cs new file mode 100644 index 000000000000..8c18c24e9545 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataModelResult.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +/// +/// Represents result of a chat completion with data. +/// +public class ChatWithDataModelResult +{ + /// + /// A unique identifier associated with chat completion with data response. + /// + public string Id { get; } + + /// + /// The first timestamp associated with generation activity for chat completion with data response, + /// represented as seconds since the beginning of the Unix epoch of 00:00 on 1 Jan 1970. + /// + public DateTimeOffset Created { get; } + + /// + /// Content from data source, including citations. + /// For more information see . + /// + public string? ToolContent { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// A unique identifier associated with chat completion with data response. + /// The first timestamp associated with generation activity for chat completion with data response. + public ChatWithDataModelResult(string id, DateTimeOffset created) + { + this.Id = id; + this.Created = created; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataRequest.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataRequest.cs new file mode 100644 index 000000000000..ab57062b3c3b --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataRequest.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +internal sealed class ChatWithDataRequest +{ + [JsonPropertyName("temperature")] + public double Temperature { get; set; } = 0; + + [JsonPropertyName("top_p")] + public double TopP { get; set; } = 0; + + [JsonPropertyName("stream")] + public bool IsStreamEnabled { get; set; } + + [JsonPropertyName("stop")] + public IList StopSequences { get; set; } = Array.Empty(); + + [JsonPropertyName("max_tokens")] + public int? MaxTokens { get; set; } + + [JsonPropertyName("presence_penalty")] + public double PresencePenalty { get; set; } = 0; + + [JsonPropertyName("frequency_penalty")] + public double FrequencyPenalty { get; set; } = 0; + + [JsonPropertyName("logit_bias")] + public IDictionary TokenSelectionBiases { get; set; } = new Dictionary(); + + [JsonPropertyName("dataSources")] + public IList DataSources { get; set; } = Array.Empty(); + + [JsonPropertyName("messages")] + public IList Messages { get; set; } = Array.Empty(); +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataResponse.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataResponse.cs new file mode 100644 index 000000000000..cb39255c76c9 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataResponse.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +#pragma warning disable CA1812 // Avoid uninstantiated internal classes + +[Serializable] +internal sealed class ChatWithDataResponse +{ + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + + [JsonPropertyName("created")] + public int Created { get; set; } = default; + + [JsonPropertyName("choices")] + public IList Choices { get; set; } = Array.Empty(); +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataResult.cs new file mode 100644 index 000000000000..451edf5c8aba --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataResult.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +internal sealed class ChatWithDataResult : IChatResult, ITextResult +{ + public ModelResult ModelResult { get; } + + public ChatWithDataResult(ChatWithDataResponse response, ChatWithDataChoice choice) + { + Verify.NotNull(response); + Verify.NotNull(choice); + + this.ModelResult = new(new ChatWithDataModelResult(response.Id, DateTimeOffset.FromUnixTimeSeconds(response.Created)) + { + ToolContent = this.GetToolContent(choice) + }); + + this._choice = choice; + } + + public Task GetChatMessageAsync(CancellationToken cancellationToken = default) + { + var message = this._choice.Messages + .FirstOrDefault(message => message.Role.Equals(AuthorRole.Assistant.Label, StringComparison.Ordinal)); + + return Task.FromResult(new SKChatMessage(message.Role, message.Content)); + } + + public async Task GetCompletionAsync(CancellationToken cancellationToken = default) + { + var message = await this.GetChatMessageAsync(cancellationToken).ConfigureAwait(false); + + return message.Content; + } + + #region private ================================================================================ + + private readonly ChatWithDataChoice _choice; + + private string? GetToolContent(ChatWithDataChoice choice) + { + var message = choice.Messages + .FirstOrDefault(message => message.Role.Equals(AuthorRole.Tool.Label, StringComparison.Ordinal)); + + return message?.Content; + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSource.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSource.cs new file mode 100644 index 000000000000..3877e5a21ad1 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSource.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +internal sealed class ChatWithDataSource +{ + [JsonPropertyName("type")] + public string Type { get; set; } = ChatWithDataSourceType.AzureCognitiveSearch.ToString(); + + [JsonPropertyName("parameters")] + public ChatWithDataSourceParameters Parameters { get; set; } = new ChatWithDataSourceParameters(); +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSourceParameters.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSourceParameters.cs new file mode 100644 index 000000000000..e0e5cb0de81d --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSourceParameters.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +internal sealed class ChatWithDataSourceParameters +{ + [JsonPropertyName("endpoint")] + public string Endpoint { get; set; } = string.Empty; + + [JsonPropertyName("key")] + public string ApiKey { get; set; } = string.Empty; + + [JsonPropertyName("indexName")] + public string IndexName { get; set; } = string.Empty; +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSourceType.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSourceType.cs new file mode 100644 index 000000000000..4aadf06e149f --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataSourceType.cs @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +internal enum ChatWithDataSourceType +{ + AzureCognitiveSearch +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingChoice.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingChoice.cs new file mode 100644 index 000000000000..1718e386279a --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingChoice.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +[SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Used for JSON deserialization")] +internal sealed class ChatWithDataStreamingChoice +{ + [JsonPropertyName("messages")] + public IList Messages { get; set; } = Array.Empty(); +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingDelta.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingDelta.cs new file mode 100644 index 000000000000..1096ed22c4a5 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingDelta.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +internal sealed class ChatWithDataStreamingDelta +{ + [JsonPropertyName("role")] + public string? Role { get; set; } + + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingMessage.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingMessage.cs new file mode 100644 index 000000000000..80c1f258e2ca --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingMessage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +[SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Used for JSON deserialization")] +internal sealed class ChatWithDataStreamingMessage +{ + [JsonPropertyName("delta")] + public ChatWithDataStreamingDelta Delta { get; set; } = new(); + + [JsonPropertyName("end_turn")] + public bool EndTurn { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResponse.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResponse.cs new file mode 100644 index 000000000000..ce0c8d0e637c --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResponse.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +[Serializable] +[SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Used for JSON deserialization")] +internal sealed class ChatWithDataStreamingResponse +{ + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + + [JsonPropertyName("created")] + public int Created { get; set; } = default; + + [JsonPropertyName("choices")] + public IList Choices { get; set; } = Array.Empty(); +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResult.cs new file mode 100644 index 000000000000..92ec4e463544 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResult.cs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +internal sealed class ChatWithDataStreamingResult : IChatStreamingResult, ITextStreamingResult, IChatResult, ITextResult +{ + public ModelResult ModelResult { get; } + + public ChatWithDataStreamingResult(ChatWithDataStreamingResponse response, ChatWithDataStreamingChoice choice) + { + Verify.NotNull(response); + Verify.NotNull(choice); + + this.ModelResult = new(new ChatWithDataModelResult(response.Id, DateTimeOffset.FromUnixTimeSeconds(response.Created)) + { + ToolContent = this.GetToolContent(choice) + }); + + this._choice = choice; + } + + public async Task GetChatMessageAsync(CancellationToken cancellationToken = default) + { + var message = this._choice.Messages.FirstOrDefault(this.IsValidMessage); + + var result = new SKChatMessage(AuthorRole.Assistant.Label, message?.Delta?.Content ?? string.Empty); + + return await Task.FromResult(result).ConfigureAwait(false); + } + + public async IAsyncEnumerable GetStreamingChatMessageAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var message = await this.GetChatMessageAsync(cancellationToken).ConfigureAwait(false); + + if (message.Content is { Length: > 0 }) + { + yield return message; + } + } + + public async IAsyncEnumerable GetCompletionStreamingAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var result in this.GetStreamingChatMessageAsync(cancellationToken)) + { + if (result.Content is string content and { Length: > 0 }) + { + yield return content; + } + } + } + + public async Task GetCompletionAsync(CancellationToken cancellationToken = default) + { + var message = await this.GetChatMessageAsync(cancellationToken).ConfigureAwait(false); + + return message.Content; + } + + #region private ================================================================================ + + private readonly ChatWithDataStreamingChoice _choice; + + private bool IsValidMessage(ChatWithDataStreamingMessage message) + { + return !message.EndTurn && + (message.Delta.Role is null || !message.Delta.Role.Equals(AuthorRole.Tool.Label, StringComparison.Ordinal)); + } + + private string? GetToolContent(ChatWithDataStreamingChoice choice) + { + var message = choice.Messages + .FirstOrDefault(message => message.Delta.Role is not null && message.Delta.Role.Equals(AuthorRole.Tool.Label, StringComparison.Ordinal)); + + return message?.Delta?.Content; + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/Connectors.AI.OpenAI.csproj b/dotnet/src/Connectors/Connectors.AI.OpenAI/Connectors.AI.OpenAI.csproj index 5f1852a86808..bbf1fd8fc532 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/Connectors.AI.OpenAI.csproj +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/Connectors.AI.OpenAI.csproj @@ -20,24 +20,11 @@ - + + - - - - - PreserveNewest - true - - - - PreserveNewest - true - - - diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/CustomClient/OpenAIClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/CustomClient/OpenAIClientBase.cs index 3fb7a3783140..1a098ceac4e3 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/CustomClient/OpenAIClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/CustomClient/OpenAIClientBase.cs @@ -5,14 +5,10 @@ using System.Linq; using System.Net.Http; using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ImageGeneration; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; using Microsoft.SemanticKernel.Diagnostics; @@ -27,17 +23,17 @@ public abstract class OpenAIClientBase /// Initializes a new instance of the class. /// /// The HttpClient used for making HTTP requests. - /// The ILogger used for logging. If null, a NullLogger instance will be used. - private protected OpenAIClientBase(HttpClient? httpClient, ILogger? logger = null) + /// The ILoggerFactory used to create a logger for logging. If null, no logging will be performed. + private protected OpenAIClientBase(HttpClient? httpClient, ILoggerFactory? loggerFactory = null) { this._httpClient = httpClient ?? new HttpClient(NonDisposableHttpClientHandler.Instance, disposeHandler: false); - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(this.GetType()) : NullLogger.Instance; } /// Adds headers to use for OpenAI HTTP requests. private protected virtual void AddRequestHeaders(HttpRequestMessage request) { - request.Headers.Add("User-Agent", HttpUserAgent); + request.Headers.Add("User-Agent", Telemetry.HttpUserAgent); } /// @@ -47,8 +43,7 @@ private protected virtual void AddRequestHeaders(HttpRequestMessage request) /// Request payload /// The to monitor for cancellation requests. The default is . /// List of text embeddings - /// AIException thrown during the request. - private protected async Task>> ExecuteTextEmbeddingRequestAsync( + private protected async Task>> ExecuteTextEmbeddingRequestAsync( string url, string requestBody, CancellationToken cancellationToken = default) @@ -56,12 +51,10 @@ private protected async Task>> ExecuteTextEmbeddingReques var result = await this.ExecutePostRequestAsync(url, requestBody, cancellationToken).ConfigureAwait(false); if (result.Embeddings is not { Count: >= 1 }) { - throw new AIException( - AIException.ErrorCodes.InvalidResponseContent, - "Embeddings not found"); + throw new SKException("Embeddings not found"); } - return result.Embeddings.Select(e => new Embedding(e.Values, transferOwnership: true)).ToList(); + return result.Embeddings.Select(e => e.Values).ToList(); } /// @@ -72,7 +65,6 @@ private protected async Task>> ExecuteTextEmbeddingReques /// Function to invoke to extract the desired portion of the image generation response. /// The to monitor for cancellation requests. The default is . /// List of image URLs - /// AIException thrown during the request. private protected async Task> ExecuteImageGenerationRequestAsync( string url, string requestBody, @@ -83,27 +75,8 @@ private protected async Task> ExecuteImageGenerationRequestAsync( return result.Images.Select(extractResponseFunc).ToList(); } - private protected virtual string? GetErrorMessageFromResponse(string jsonResponsePayload) - { - try - { - JsonNode? root = JsonSerializer.Deserialize(jsonResponsePayload); - - return root?["error"]?["message"]?.GetValue(); - } - catch (Exception ex) when (ex is NotSupportedException or JsonException) - { - this._logger.LogError(ex, "Unable to extract error from response body content. Exception: {0}:{1}", ex.GetType(), ex.Message); - } - - return null; - } - #region private ================================================================================ - // HTTP user agent sent to remote endpoints - private const string HttpUserAgent = "Microsoft-Semantic-Kernel"; - /// /// Logger /// @@ -116,20 +89,11 @@ private protected async Task> ExecuteImageGenerationRequestAsync( private protected async Task ExecutePostRequestAsync(string url, string requestBody, CancellationToken cancellationToken = default) { - try - { - using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); - using var response = await this.ExecuteRequestAsync(url, HttpMethod.Post, content, cancellationToken).ConfigureAwait(false); - string responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - T result = this.JsonDeserialize(responseJson); - return result; - } - catch (Exception e) when (e is not AIException) - { - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Something went wrong: {e.Message}", e); - } + using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); + using var response = await this.ExecuteRequestAsync(url, HttpMethod.Post, content, cancellationToken).ConfigureAwait(false); + string responseJson = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); + T result = this.JsonDeserialize(responseJson); + return result; } private protected T JsonDeserialize(string responseJson) @@ -137,7 +101,7 @@ private protected T JsonDeserialize(string responseJson) var result = Json.Deserialize(responseJson); if (result is null) { - throw new AIException(AIException.ErrorCodes.InvalidResponseContent, "Response JSON parse error"); + throw new SKException("Response JSON parse error"); } return result; @@ -145,102 +109,20 @@ private protected T JsonDeserialize(string responseJson) private protected async Task ExecuteRequestAsync(string url, HttpMethod method, HttpContent? content, CancellationToken cancellationToken = default) { - try - { - HttpResponseMessage? response = null; - using (var request = new HttpRequestMessage(method, url)) - { - this.AddRequestHeaders(request); - if (content != null) - { - request.Content = content; - } - - response = await this._httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); - } - - this._logger.LogDebug("HTTP response: {0} {1}", (int)response.StatusCode, response.StatusCode.ToString("G")); - - if (response.IsSuccessStatusCode) - { - return response; - } - - string responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - string? errorDetail = this.GetErrorMessageFromResponse(responseJson); - switch ((HttpStatusCodeType)response.StatusCode) - { - case HttpStatusCodeType.BadRequest: - case HttpStatusCodeType.MethodNotAllowed: - case HttpStatusCodeType.NotFound: - case HttpStatusCodeType.NotAcceptable: - case HttpStatusCodeType.Conflict: - case HttpStatusCodeType.Gone: - case HttpStatusCodeType.LengthRequired: - case HttpStatusCodeType.PreconditionFailed: - case HttpStatusCodeType.RequestEntityTooLarge: - case HttpStatusCodeType.RequestUriTooLong: - case HttpStatusCodeType.UnsupportedMediaType: - case HttpStatusCodeType.RequestedRangeNotSatisfiable: - case HttpStatusCodeType.ExpectationFailed: - case HttpStatusCodeType.HttpVersionNotSupported: - case HttpStatusCodeType.UpgradeRequired: - case HttpStatusCodeType.MisdirectedRequest: - case HttpStatusCodeType.UnprocessableEntity: - case HttpStatusCodeType.Locked: - case HttpStatusCodeType.FailedDependency: - case HttpStatusCodeType.PreconditionRequired: - case HttpStatusCodeType.RequestHeaderFieldsTooLarge: - throw new AIException( - AIException.ErrorCodes.InvalidRequest, - $"The request is not valid, HTTP status: {response.StatusCode:G}", - errorDetail); - - case HttpStatusCodeType.Unauthorized: - case HttpStatusCodeType.Forbidden: - case HttpStatusCodeType.ProxyAuthenticationRequired: - case HttpStatusCodeType.UnavailableForLegalReasons: - case HttpStatusCodeType.NetworkAuthenticationRequired: - throw new AIException( - AIException.ErrorCodes.AccessDenied, - $"The request is not authorized, HTTP status: {response.StatusCode:G}", - errorDetail); - - case HttpStatusCodeType.RequestTimeout: - throw new AIException( - AIException.ErrorCodes.RequestTimeout, - $"The request timed out, HTTP status: {response.StatusCode:G}"); - - case HttpStatusCodeType.TooManyRequests: - throw new AIException( - AIException.ErrorCodes.Throttling, - $"Too many requests, HTTP status: {response.StatusCode:G}", - errorDetail); - - case HttpStatusCodeType.InternalServerError: - case HttpStatusCodeType.NotImplemented: - case HttpStatusCodeType.BadGateway: - case HttpStatusCodeType.ServiceUnavailable: - case HttpStatusCodeType.GatewayTimeout: - case HttpStatusCodeType.InsufficientStorage: - throw new AIException( - AIException.ErrorCodes.ServiceError, - $"The service failed to process the request, HTTP status: {response.StatusCode:G}", - errorDetail); - - default: - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Unexpected HTTP response, status: {response.StatusCode:G}", - errorDetail); - } - } - catch (Exception e) when (e is not AIException) + using var request = new HttpRequestMessage(method, url); + + this.AddRequestHeaders(request); + + if (content != null) { - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Something went wrong: {e.Message}", e); + request.Content = content; } + + var response = await this._httpClient.SendWithSuccessCheckAsync(request, cancellationToken).ConfigureAwait(false); + + this._logger.LogDebug("HTTP response: {0} {1}", (int)response.StatusCode, response.StatusCode.ToString("G")); + + return response; } #endregion diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ImageGeneration/AzureOpenAIImageGeneration.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ImageGeneration/AzureOpenAIImageGeneration.cs index 910ae36dbb99..b0f110c65714 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ImageGeneration/AzureOpenAIImageGeneration.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ImageGeneration/AzureOpenAIImageGeneration.cs @@ -6,7 +6,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.ImageGeneration; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.CustomClient; using Microsoft.SemanticKernel.Diagnostics; @@ -56,10 +55,10 @@ public class AzureOpenAIImageGeneration : OpenAIClientBase, IImageGeneration /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Custom for HTTP requests. - /// Application logger + /// The ILoggerFactory used to create a logger for logging. If null, no logging will be performed. /// Maximum number of attempts to retrieve the image generation operation result. /// Azure OpenAI Endpoint ApiVersion - public AzureOpenAIImageGeneration(string endpoint, string apiKey, HttpClient? httpClient = null, ILogger? logger = null, int maxRetryCount = 5, string apiVersion = "2023-06-01-preview") : base(httpClient, logger) + public AzureOpenAIImageGeneration(string endpoint, string apiKey, HttpClient? httpClient = null, ILoggerFactory? loggerFactory = null, int maxRetryCount = 5, string apiVersion = "2023-06-01-preview") : base(httpClient, loggerFactory) { Verify.NotNullOrWhiteSpace(endpoint); Verify.NotNullOrWhiteSpace(apiKey); @@ -77,19 +76,17 @@ public AzureOpenAIImageGeneration(string endpoint, string apiKey, HttpClient? ht /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Custom for HTTP requests. /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart - /// Application logger + /// The ILoggerFactory used to create a logger for logging. If null, no logging will be performed. /// Maximum number of attempts to retrieve the image generation operation result. /// Azure OpenAI Endpoint ApiVersion - public AzureOpenAIImageGeneration(string apiKey, HttpClient httpClient, string? endpoint = null, ILogger? logger = null, int maxRetryCount = 5, string apiVersion = "2023-06-01-preview") : base(httpClient, logger) + public AzureOpenAIImageGeneration(string apiKey, HttpClient httpClient, string? endpoint = null, ILoggerFactory? loggerFactory = null, int maxRetryCount = 5, string apiVersion = "2023-06-01-preview") : base(httpClient, loggerFactory) { Verify.NotNull(httpClient); Verify.NotNullOrWhiteSpace(apiKey); if (httpClient.BaseAddress == null && string.IsNullOrEmpty(endpoint)) { - throw new AIException( - AIException.ErrorCodes.InvalidConfiguration, - "The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); + throw new SKException("The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); } endpoint = !string.IsNullOrEmpty(endpoint) ? endpoint! : httpClient.BaseAddress!.AbsoluteUri; @@ -107,14 +104,14 @@ public async Task GenerateImageAsync(string description, int width, int var operationId = await this.StartImageGenerationAsync(description, width, height, cancellationToken).ConfigureAwait(false); var result = await this.GetImageGenerationResultAsync(operationId, cancellationToken).ConfigureAwait(false); - if (result.Result == null) + if (result.Result is null) { - throw new AzureSdk.OpenAIInvalidResponseException(null, "Azure Image Generation null response"); + throw new SKException("Azure Image Generation null response"); } if (result.Result.Images.Count == 0) { - throw new AzureSdk.OpenAIInvalidResponseException(result, "Azure Image Generation result not found"); + throw new SKException("Azure Image Generation result not found"); } return result.Result.Images.First().Url; @@ -148,7 +145,7 @@ private async Task StartImageGenerationAsync(string description, int wid if (result == null || string.IsNullOrWhiteSpace(result.Id)) { - throw new AIException(AIException.ErrorCodes.InvalidResponseContent, "Response not contains result"); + throw new SKException("Response not contains result"); } return result.Id; @@ -165,42 +162,34 @@ private async Task GetImageGenerationResultAsync(s var operationLocation = this.GetUri(GetImageOperation, operationId); var retryCount = 0; - try + + while (true) { - while (true) + if (this._maxRetryCount == retryCount) { - if (this._maxRetryCount == retryCount) - { - throw new AIException(AIException.ErrorCodes.RequestTimeout, "Reached maximum retry attempts"); - } - - using var response = await this.ExecuteRequestAsync(operationLocation, HttpMethod.Get, null, cancellationToken).ConfigureAwait(false); - var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - var result = this.JsonDeserialize(responseJson); - - if (result.Status.Equals(AzureImageOperationStatus.Succeeded, StringComparison.OrdinalIgnoreCase)) - { - return result; - } - else if (this.IsFailedOrCancelled(result.Status)) - { - throw new AzureSdk.OpenAIInvalidResponseException(result, $"Azure OpenAI image generation {result.Status}"); - } - - if (response.Headers.TryGetValues("retry-after", out var afterValues) && long.TryParse(afterValues.FirstOrDefault(), out var after)) - { - await Task.Delay(TimeSpan.FromSeconds(after), cancellationToken).ConfigureAwait(false); - } - - // increase retry count - retryCount++; + throw new SKException("Reached maximum retry attempts"); } - } - catch (Exception e) when (e is not AIException) - { - throw new AIException( - AIException.ErrorCodes.UnknownError, - $"Something went wrong: {e.Message}", e); + + using var response = await this.ExecuteRequestAsync(operationLocation, HttpMethod.Get, null, cancellationToken).ConfigureAwait(false); + var responseJson = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); + var result = this.JsonDeserialize(responseJson); + + if (result.Status.Equals(AzureImageOperationStatus.Succeeded, StringComparison.OrdinalIgnoreCase)) + { + return result; + } + else if (this.IsFailedOrCancelled(result.Status)) + { + throw new SKException($"Azure OpenAI image generation {result.Status}"); + } + + if (response.Headers.TryGetValues("retry-after", out var afterValues) && long.TryParse(afterValues.FirstOrDefault(), out var after)) + { + await Task.Delay(TimeSpan.FromSeconds(after), cancellationToken).ConfigureAwait(false); + } + + // increase retry count + retryCount++; } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ImageGeneration/OpenAIImageGeneration.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ImageGeneration/OpenAIImageGeneration.cs index 2be866699aaf..7bd0c1bc2199 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ImageGeneration/OpenAIImageGeneration.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ImageGeneration/OpenAIImageGeneration.cs @@ -12,7 +12,9 @@ using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.ImageGeneration; - +/// +/// A class for generating images using OpenAI's API. +/// public class OpenAIImageGeneration : OpenAIClientBase, IImageGeneration { /// @@ -31,18 +33,18 @@ public class OpenAIImageGeneration : OpenAIClientBase, IImageGeneration private readonly string _authorizationHeaderValue; /// - /// Create a new instance of OpenAI image generation service + /// Initializes a new instance of the class. /// /// OpenAI API key, see https://platform.openai.com/account/api-keys /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public OpenAIImageGeneration( string apiKey, string? organization = null, HttpClient? httpClient = null, - ILogger? logger = null - ) : base(httpClient, logger) + ILoggerFactory? loggerFactory = null + ) : base(httpClient, loggerFactory) { Verify.NotNullOrWhiteSpace(apiKey); this._authorizationHeaderValue = $"Bearer {apiKey}"; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIKernelBuilderExtensions.cs index 591373ef5261..8f39afe34bd6 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIKernelBuilderExtensions.cs @@ -1,17 +1,22 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Net.Http; +using Azure; using Azure.AI.OpenAI; using Azure.Core; +using Azure.Core.Pipeline; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.ChatCompletion; using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.AI.ImageGeneration; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ImageGeneration; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; +using Microsoft.SemanticKernel.Http; #pragma warning disable IDE0130 // ReSharper disable once CheckNamespace - Using NS of KernelConfig @@ -45,14 +50,11 @@ public static KernelBuilder WithAzureTextCompletionService(this KernelBuilder bu bool setAsDefault = false, HttpClient? httpClient = null) { - builder.WithAIService(serviceId, (parameters) => - new AzureTextCompletion( - deploymentName, - endpoint, - apiKey, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger), - setAsDefault); + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => + { + var client = CreateAzureOpenAIClient(loggerFactory, httpHandlerFactory, deploymentName, endpoint, new AzureKeyCredential(apiKey), httpClient); + return new AzureTextCompletion(deploymentName, client, loggerFactory); + }, setAsDefault); return builder; } @@ -77,14 +79,11 @@ public static KernelBuilder WithAzureTextCompletionService(this KernelBuilder bu bool setAsDefault = false, HttpClient? httpClient = null) { - builder.WithAIService(serviceId, (parameters) => - new AzureTextCompletion( - deploymentName, - endpoint, - credentials, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger), - setAsDefault); + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => + { + var client = CreateAzureOpenAIClient(loggerFactory, httpHandlerFactory, deploymentName, endpoint, credentials, httpClient); + return new AzureTextCompletion(deploymentName, client, loggerFactory); + }, setAsDefault); return builder; } @@ -105,11 +104,11 @@ public static KernelBuilder WithAzureTextCompletionService(this KernelBuilder bu string? serviceId = null, bool setAsDefault = false) { - builder.WithAIService(serviceId, (parameters) => + builder.WithAIService(serviceId, (loggerFactory) => new AzureTextCompletion( deploymentName, openAIClient, - parameters.Logger), + loggerFactory), setAsDefault); return builder; @@ -135,13 +134,13 @@ public static KernelBuilder WithOpenAITextCompletionService(this KernelBuilder b bool setAsDefault = false, HttpClient? httpClient = null) { - builder.WithAIService(serviceId, (parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new OpenAITextCompletion( modelId, apiKey, orgId, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory), setAsDefault); return builder; } @@ -170,13 +169,13 @@ public static KernelBuilder WithAzureTextEmbeddingGenerationService(this KernelB bool setAsDefault = false, HttpClient? httpClient = null) { - builder.WithAIService(serviceId, (parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new AzureTextEmbeddingGeneration( deploymentName, endpoint, apiKey, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory), setAsDefault); return builder; } @@ -201,13 +200,13 @@ public static KernelBuilder WithAzureTextEmbeddingGenerationService(this KernelB bool setAsDefault = false, HttpClient? httpClient = null) { - builder.WithAIService(serviceId, (parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new AzureTextEmbeddingGeneration( deploymentName, endpoint, credential, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory), setAsDefault); return builder; } @@ -232,13 +231,13 @@ public static KernelBuilder WithOpenAITextEmbeddingGenerationService(this Kernel bool setAsDefault = false, HttpClient? httpClient = null) { - builder.WithAIService(serviceId, (parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new OpenAITextEmbeddingGeneration( modelId, apiKey, orgId, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory), setAsDefault); return builder; } @@ -269,12 +268,12 @@ public static KernelBuilder WithAzureChatCompletionService(this KernelBuilder bu bool setAsDefault = false, HttpClient? httpClient = null) { - AzureChatCompletion Factory((ILogger Logger, KernelConfig Config) parameters) => new( - deploymentName, - endpoint, - apiKey, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger); + AzureChatCompletion Factory(ILoggerFactory loggerFactory, IDelegatingHandlerFactory httpHandlerFactory) + { + OpenAIClient client = CreateAzureOpenAIClient(loggerFactory, httpHandlerFactory, deploymentName, endpoint, new AzureKeyCredential(apiKey), httpClient); + + return new(deploymentName, client, loggerFactory); + }; builder.WithAIService(serviceId, Factory, setAsDefault); @@ -309,12 +308,12 @@ public static KernelBuilder WithAzureChatCompletionService(this KernelBuilder bu bool setAsDefault = false, HttpClient? httpClient = null) { - AzureChatCompletion Factory((ILogger Logger, KernelConfig Config) parameters) => new( - deploymentName, - endpoint, - credentials, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger); + AzureChatCompletion Factory(ILoggerFactory loggerFactory, IDelegatingHandlerFactory httpHandlerFactory) + { + OpenAIClient client = CreateAzureOpenAIClient(loggerFactory, httpHandlerFactory, deploymentName, endpoint, credentials, httpClient); + + return new(deploymentName, client, loggerFactory); + }; builder.WithAIService(serviceId, Factory, setAsDefault); @@ -327,6 +326,39 @@ public static KernelBuilder WithAzureChatCompletionService(this KernelBuilder bu return builder; } + /// + /// Adds the Azure OpenAI chat completion with data service to the list. + /// More information: + /// + /// The instance. + /// Required configuration for Azure OpenAI chat completion with data. + /// Whether to use the service also for text completion, if supported. + /// A local identifier for the given AI service. + /// Whether the service should be the default for its type. + /// Custom for HTTP requests. + /// Self instance + public static KernelBuilder WithAzureChatCompletionService(this KernelBuilder builder, + AzureChatCompletionWithDataConfig config, + bool alsoAsTextCompletion = true, + string? serviceId = null, + bool setAsDefault = false, + HttpClient? httpClient = null) + { + AzureChatCompletionWithData Factory(ILoggerFactory loggerFactory, IDelegatingHandlerFactory httpHandlerFactory) => new( + config, + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory); + + builder.WithAIService(serviceId, Factory, setAsDefault); + + if (alsoAsTextCompletion && typeof(ITextCompletion).IsAssignableFrom(typeof(AzureChatCompletionWithData))) + { + builder.WithAIService(serviceId, Factory, setAsDefault); + } + + return builder; + } + /// /// Adds the OpenAI ChatGPT completion service to the list. /// See https://platform.openai.com/docs for service details. @@ -349,12 +381,12 @@ public static KernelBuilder WithOpenAIChatCompletionService(this KernelBuilder b bool setAsDefault = false, HttpClient? httpClient = null) { - OpenAIChatCompletion Factory((ILogger Logger, KernelConfig Config) parameters) => new( + OpenAIChatCompletion Factory(ILoggerFactory loggerFactory, IDelegatingHandlerFactory httpHandlerFactory) => new( modelId, apiKey, orgId, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger); + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory); builder.WithAIService(serviceId, Factory, setAsDefault); @@ -367,6 +399,74 @@ public static KernelBuilder WithOpenAIChatCompletionService(this KernelBuilder b return builder; } + /// + /// Adds the Azure OpenAI ChatGPT completion service to the list. + /// See https://platform.openai.com/docs for service details. + /// + /// The instance + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Custom for HTTP requests. + /// Whether to use the service also for text completion, if supported + /// A local identifier for the given AI service + /// Whether the service should be the default for its type. + /// Self instance + public static KernelBuilder WithAzureChatCompletionService(this KernelBuilder builder, + string deploymentName, + OpenAIClient openAIClient, + bool alsoAsTextCompletion = true, + string? serviceId = null, + bool setAsDefault = false) + { + AzureChatCompletion Factory(ILoggerFactory loggerFactory) + { + return new(deploymentName, openAIClient, loggerFactory); + }; + + builder.WithAIService(serviceId, Factory, setAsDefault); + + // If the class implements the text completion interface, allow to use it also for semantic functions + if (alsoAsTextCompletion && typeof(ITextCompletion).IsAssignableFrom(typeof(AzureChatCompletion))) + { + builder.WithAIService(serviceId, Factory, setAsDefault); + } + + return builder; + } + + /// + /// Adds the OpenAI ChatGPT completion service to the list. + /// See https://platform.openai.com/docs for service details. + /// + /// The instance + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Custom for HTTP requests. + /// Whether to use the service also for text completion, if supported + /// A local identifier for the given AI service + /// Whether the service should be the default for its type. + /// Self instance + public static KernelBuilder WithOpenAIChatCompletionService(this KernelBuilder builder, + string deploymentName, + OpenAIClient openAIClient, + bool alsoAsTextCompletion = true, + string? serviceId = null, + bool setAsDefault = false) + { + OpenAIChatCompletion Factory(ILoggerFactory loggerFactory) + { + return new(deploymentName, openAIClient, loggerFactory); + }; + + builder.WithAIService(serviceId, Factory, setAsDefault); + + // If the class implements the text completion interface, allow to use it also for semantic functions + if (alsoAsTextCompletion && typeof(ITextCompletion).IsAssignableFrom(typeof(AzureChatCompletion))) + { + builder.WithAIService(serviceId, Factory, setAsDefault); + } + + return builder; + } + #endregion #region Images @@ -388,12 +488,12 @@ public static KernelBuilder WithOpenAIImageGenerationService(this KernelBuilder bool setAsDefault = false, HttpClient? httpClient = null) { - builder.WithAIService(serviceId, ((ILogger Logger, KernelConfig Config) parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new OpenAIImageGeneration( apiKey, orgId, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory), setAsDefault); return builder; @@ -418,12 +518,12 @@ public static KernelBuilder WithAzureOpenAIImageGenerationService(this KernelBui HttpClient? httpClient = null, int maxRetryCount = 5) { - builder.WithAIService(serviceId, ((ILogger Logger, KernelConfig Config) parameters) => + builder.WithAIService(serviceId, (loggerFactory, httpHandlerFactory) => new AzureOpenAIImageGeneration( endpoint, apiKey, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), - parameters.Logger, + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory, maxRetryCount), setAsDefault); @@ -431,4 +531,29 @@ public static KernelBuilder WithAzureOpenAIImageGenerationService(this KernelBui } #endregion + + private static OpenAIClient CreateAzureOpenAIClient(ILoggerFactory loggerFactory, IDelegatingHandlerFactory httpHandlerFactory, string deploymentName, string endpoint, AzureKeyCredential credentials, HttpClient? httpClient) + { + OpenAIClientOptions options = CreateOpenAIClientOptions(loggerFactory, httpHandlerFactory, httpClient); + + return new(new Uri(endpoint), credentials, options); + } + + private static OpenAIClient CreateAzureOpenAIClient(ILoggerFactory loggerFactory, IDelegatingHandlerFactory httpHandlerFactory, string deploymentName, string endpoint, TokenCredential credentials, HttpClient? httpClient) + { + OpenAIClientOptions options = CreateOpenAIClientOptions(loggerFactory, httpHandlerFactory, httpClient); + + return new(new Uri(endpoint), credentials, options); + } + + private static OpenAIClientOptions CreateOpenAIClientOptions(ILoggerFactory loggerFactory, IDelegatingHandlerFactory httpHandlerFactory, HttpClient? httpClient) + { + OpenAIClientOptions options = new(); +#pragma warning disable CA2000 // Dispose objects before losing scope + options.Transport = new HttpClientTransport(HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory)); + options.RetryPolicy = new RetryPolicy(maxRetries: 0); //Disabling Azure SDK retry policy to use the one provided by the delegating handler factory or the HTTP client. +#pragma warning restore CA2000 // Dispose objects before losing scope + + return options; + } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIKernelExtensions.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIKernelExtensions.cs new file mode 100644 index 000000000000..d263b6ca53f3 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIKernelExtensions.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Orchestration; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the namespace of IKernel +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Class for extension methods for using OpenAI request settings. +/// +public static class OpenAIKernelExtensions +{ + /// + /// Define a string-to-string semantic function, with no direct support for input context. + /// The function can be referenced in templates and will receive the context, but when invoked programmatically you + /// can only pass in a string in input and receive a string in output. + /// + /// Semantic Kernel instance + /// Plain language definition of the semantic function, using SK template language + /// OpenAI LLM request settings + /// A name for the given function. The name can be referenced in templates and used by the pipeline planner. + /// Optional plugin name, for namespacing and avoid collisions + /// Optional description, useful for the planner + /// A function ready to use + public static ISKFunction CreateSemanticFunction( + this IKernel kernel, + string promptTemplate, + OpenAIRequestSettings requestSettings, + string? functionName = null, + string? pluginName = null, + string? description = null) + { + return kernel.CreateSemanticFunction( + promptTemplate, + functionName, + pluginName, + description, + requestSettings); + } + + /// + /// Invoke a semantic function using the provided prompt template. + /// + /// Semantic Kernel instance + /// Plain language definition of the semantic function, using SK template language + /// OpenAI LLM request settings + /// Options name for the given function. The name can be referenced in templates and used by the pipeline planner. + /// Optional plugin name, for namespacing and avoid collisions + /// Optional description, useful for the planner + /// A function ready to use + public static Task InvokeSemanticFunctionAsync( + this IKernel kernel, + string promptTemplate, + OpenAIRequestSettings requestSettings, + string? functionName = null, + string? pluginName = null, + string? description = null) + { + return kernel.InvokeSemanticFunctionAsync( + promptTemplate, + functionName, + pluginName, + description, + requestSettings); + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIMemoryBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIMemoryBuilderExtensions.cs new file mode 100644 index 000000000000..4d351c5d774a --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIMemoryBuilderExtensions.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Azure.Core; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; +using Microsoft.SemanticKernel.Plugins.Memory; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI; + +/// +/// Provides extension methods for the class to configure OpenAI and AzureOpenAI connectors. +/// +public static class OpenAIMemoryBuilderExtensions +{ + /// + /// Adds an Azure OpenAI text embeddings service. + /// See https://learn.microsoft.com/azure/cognitive-services/openai for service details. + /// + /// The instance + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// A local identifier for the given AI service + /// Whether the service should be the default for its type. + /// Custom for HTTP requests. + /// Self instance + public static MemoryBuilder WithAzureTextEmbeddingGenerationService( + this MemoryBuilder builder, + string deploymentName, + string endpoint, + string apiKey, + string? serviceId = null, + bool setAsDefault = false, + HttpClient? httpClient = null) + { + builder.WithTextEmbeddingGeneration((loggerFactory, httpHandlerFactory) => + new AzureTextEmbeddingGeneration( + deploymentName, + endpoint, + apiKey, + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory)); + + return builder; + } + + /// + /// Adds an Azure OpenAI text embeddings service. + /// See https://learn.microsoft.com/azure/cognitive-services/openai for service details. + /// + /// The instance + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Token credentials, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc. + /// A local identifier for the given AI service + /// Whether the service should be the default for its type. + /// Custom for HTTP requests. + /// Self instance + public static MemoryBuilder WithAzureTextEmbeddingGenerationService( + this MemoryBuilder builder, + string deploymentName, + string endpoint, + TokenCredential credential, + string? serviceId = null, + bool setAsDefault = false, + HttpClient? httpClient = null) + { + builder.WithTextEmbeddingGeneration((loggerFactory, httpHandlerFactory) => + new AzureTextEmbeddingGeneration( + deploymentName, + endpoint, + credential, + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory)); + + return builder; + } + + /// + /// Adds the OpenAI text embeddings service. + /// See https://platform.openai.com/docs for service details. + /// + /// The instance + /// OpenAI model name, see https://platform.openai.com/docs/models + /// OpenAI API key, see https://platform.openai.com/account/api-keys + /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. + /// A local identifier for the given AI service + /// Whether the service should be the default for its type. + /// Custom for HTTP requests. + /// Self instance + public static MemoryBuilder WithOpenAITextEmbeddingGenerationService( + this MemoryBuilder builder, + string modelId, + string apiKey, + string? orgId = null, + string? serviceId = null, + bool setAsDefault = false, + HttpClient? httpClient = null) + { + builder.WithTextEmbeddingGeneration((loggerFactory, httpHandlerFactory) => + new OpenAITextEmbeddingGeneration( + modelId, + apiKey, + orgId, + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + loggerFactory)); + + return builder; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIModelResultExtensions.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIModelResultExtensions.cs index 0db6222f56d2..aa5f06e511e4 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIModelResultExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIModelResultExtensions.cs @@ -1,12 +1,16 @@ // Copyright (c) Microsoft. All rights reserved. using Azure.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; using Microsoft.SemanticKernel.Orchestration; #pragma warning disable IDE0130 namespace Microsoft.SemanticKernel; +/// +/// Provides extension methods for working with OpenAI model results. +/// public static class OpenAIModelResultExtension { /// @@ -14,9 +18,9 @@ public static class OpenAIModelResultExtension /// /// Current context /// OpenAI / AzureOpenAI result - public static Completions GetOpenAITextResult(this ModelResult resultBase) + public static TextModelResult GetOpenAITextResult(this ModelResult resultBase) { - return resultBase.GetResult(); + return resultBase.GetResult(); } /// @@ -24,8 +28,8 @@ public static Completions GetOpenAITextResult(this ModelResult resultBase) /// /// Current context /// OpenAI / AzureOpenAI result - public static ChatCompletions GetOpenAIChatResult(this ModelResult resultBase) + public static ChatModelResult GetOpenAIChatResult(this ModelResult resultBase) { - return resultBase.GetResult(); + return resultBase.GetResult(); } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIRequestSettings.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIRequestSettings.cs new file mode 100644 index 000000000000..307ab324d79a --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIRequestSettings.cs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI; + +/// +/// Request settings for an OpenAI completion request. +/// +public class OpenAIRequestSettings : AIRequestSettings +{ + /// + /// Value for to indicate that the model + /// can optionally generate a function call from . + /// + public const string FunctionCallAuto = "auto"; + + /// + /// Value for to indicate that no + /// function call should be generated. + /// + public const string FunctionCallNone = "none"; + + /// + /// Temperature controls the randomness of the completion. + /// The higher the temperature, the more random the completion. + /// + [JsonPropertyName("temperature")] + public double Temperature { get; set; } = 0; + + /// + /// TopP controls the diversity of the completion. + /// The higher the TopP, the more diverse the completion. + /// + [JsonPropertyName("top_p")] + public double TopP { get; set; } = 0; + + /// + /// Number between -2.0 and 2.0. Positive values penalize new tokens + /// based on whether they appear in the text so far, increasing the + /// model's likelihood to talk about new topics. + /// + [JsonPropertyName("presence_penalty")] + public double PresencePenalty { get; set; } = 0; + + /// + /// Number between -2.0 and 2.0. Positive values penalize new tokens + /// based on their existing frequency in the text so far, decreasing + /// the model's likelihood to repeat the same line verbatim. + /// + [JsonPropertyName("frequency_penalty")] + public double FrequencyPenalty { get; set; } = 0; + + /// + /// The maximum number of tokens to generate in the completion. + /// + [JsonPropertyName("max_tokens")] + public int? MaxTokens { get; set; } + + /// + /// Sequences where the completion will stop generating further tokens. + /// + [JsonPropertyName("stop_sequences")] + public IList StopSequences { get; set; } = Array.Empty(); + + /// + /// How many completions to generate for each prompt. Default is 1. + /// Note: Because this parameter generates many completions, it can quickly consume your token quota. + /// Use carefully and ensure that you have reasonable settings for max_tokens and stop. + /// + [JsonPropertyName("results_per_prompt")] + public int ResultsPerPrompt { get; set; } = 1; + + /// + /// The system prompt to use when generating text completions using a chat model. + /// Defaults to "Assistant is a large language model." + /// + [JsonPropertyName("chat_system_prompt")] + public string ChatSystemPrompt + { + get => this._chatSystemPrompt; + set + { + if (string.IsNullOrEmpty(value)) + { + value = OpenAIRequestSettings.DefaultChatSystemPrompt; + } + this._chatSystemPrompt = value; + } + } + + /// + /// Modify the likelihood of specified tokens appearing in the completion. + /// + [JsonPropertyName("token_selection_biases")] + public IDictionary TokenSelectionBiases { get; set; } = new Dictionary(); + + /// + /// Possible values are , , + /// or the name of a specific function that OpenAI should use to respond to the chat + /// request. If the latter, this function must exist in . + /// + public string? FunctionCall { get; set; } = null; + + /// + /// The set of functions to choose from if function calling is enabled by the model. + /// + public IList? Functions { get; set; } = null; + + /// + /// Default value for chat system property. + /// + internal static string DefaultChatSystemPrompt { get; } = "Assistant is a large language model."; + + /// + /// Default max tokens for a text completion + /// + internal static int DefaultTextMaxTokens { get; } = 256; + + /// + /// Create a new settings object with the values from another settings object. + /// + /// Template configuration + /// Default max tokens + /// An instance of OpenAIRequestSettings + public static OpenAIRequestSettings FromRequestSettings(AIRequestSettings? requestSettings, int? defaultMaxTokens = null) + { + if (requestSettings is null) + { + return new OpenAIRequestSettings() + { + MaxTokens = defaultMaxTokens + }; + } + + if (requestSettings is OpenAIRequestSettings requestSettingsOpenAIRequestSettings) + { + return requestSettingsOpenAIRequestSettings; + } + + var json = JsonSerializer.Serialize(requestSettings); + var openAIRequestSettings = JsonSerializer.Deserialize(json, s_options); + + if (openAIRequestSettings is not null) + { + return openAIRequestSettings; + } + + throw new ArgumentException($"Invalid request settings, cannot convert to {nameof(OpenAIRequestSettings)}", nameof(requestSettings)); + } + + #region private ================================================================================ + + private string _chatSystemPrompt = OpenAIRequestSettings.DefaultChatSystemPrompt; + + private static readonly JsonSerializerOptions s_options = CreateOptions(); + + private static JsonSerializerOptions CreateOptions() + { + JsonSerializerOptions options = new() + { + WriteIndented = true, + MaxDepth = 20, + AllowTrailingCommas = true, + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + Converters = { new OpenAIRequestSettingsConverter() } + }; + + return options; + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIRequestSettingsConverter.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIRequestSettingsConverter.cs new file mode 100644 index 000000000000..a61f8400ebdb --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/OpenAIRequestSettingsConverter.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI; + +/// +/// JSON converter for +/// +public class OpenAIRequestSettingsConverter : JsonConverter +{ + /// + public override OpenAIRequestSettings? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var requestSettings = new OpenAIRequestSettings(); + + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? propertyName = reader.GetString(); + + if (propertyName is not null) + { + // normalise property name to uppercase + propertyName = propertyName.ToUpperInvariant(); + } + + reader.Read(); + + switch (propertyName) + { + case "TEMPERATURE": + requestSettings.Temperature = reader.GetDouble(); + break; + case "TOPP": + case "TOP_P": + requestSettings.TopP = reader.GetDouble(); + break; + case "FREQUENCYPENALTY": + case "FREQUENCY_PENALTY": + requestSettings.FrequencyPenalty = reader.GetDouble(); + break; + case "PRESENCEPENALTY": + case "PRESENCE_PENALTY": + requestSettings.PresencePenalty = reader.GetDouble(); + break; + case "MAXTOKENS": + case "MAX_TOKENS": + requestSettings.MaxTokens = reader.GetInt32(); + break; + case "STOPSEQUENCES": + case "STOP_SEQUENCES": + requestSettings.StopSequences = JsonSerializer.Deserialize>(ref reader, options) ?? Array.Empty(); + break; + case "RESULTSPERPROMPT": + case "RESULTS_PER_PROMPT": + requestSettings.ResultsPerPrompt = reader.GetInt32(); + break; + case "CHATSYSTEMPROMPT": + case "CHAT_SYSTEM_PROMPT": + requestSettings.ChatSystemPrompt = reader.GetString() ?? OpenAIRequestSettings.DefaultChatSystemPrompt; + break; + case "TOKENSELECTIONBIASES": + case "TOKEN_SELECTION_BIASES": + requestSettings.TokenSelectionBiases = JsonSerializer.Deserialize>(ref reader, options) ?? new Dictionary(); + break; + case "SERVICEID": + case "SERVICE_ID": + requestSettings.ServiceId = reader.GetString(); + break; + default: + reader.Skip(); + break; + } + } + } + + return requestSettings; + } + + /// + public override void Write(Utf8JsonWriter writer, OpenAIRequestSettings value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + writer.WriteNumber("temperature", value.Temperature); + writer.WriteNumber("top_p", value.TopP); + writer.WriteNumber("frequency_penalty", value.FrequencyPenalty); + writer.WriteNumber("presence_penalty", value.PresencePenalty); + if (value.MaxTokens is null) + { + writer.WriteNull("max_tokens"); + } + else + { + writer.WriteNumber("max_tokens", (decimal)value.MaxTokens); + } + writer.WritePropertyName("stop_sequences"); + JsonSerializer.Serialize(writer, value.StopSequences, options); + writer.WriteNumber("results_per_prompt", value.ResultsPerPrompt); + writer.WriteString("chat_system_prompt", value.ChatSystemPrompt); + writer.WritePropertyName("token_selection_biases"); + JsonSerializer.Serialize(writer, value.TokenSelectionBiases, options); + writer.WriteString("service_id", value.ServiceId); + + writer.WriteEndObject(); + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs index 26350604742c..2a549835a04a 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs @@ -7,6 +7,7 @@ using Azure.AI.OpenAI; using Azure.Core; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; @@ -25,13 +26,13 @@ public sealed class AzureTextCompletion : AzureOpenAIClientBase, ITextCompletion /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public AzureTextCompletion( string modelId, string endpoint, string apiKey, HttpClient? httpClient = null, - ILogger? logger = null) : base(modelId, endpoint, apiKey, httpClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, endpoint, apiKey, httpClient, loggerFactory) { } @@ -42,13 +43,13 @@ public AzureTextCompletion( /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Token credentials, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc. /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public AzureTextCompletion( string modelId, string endpoint, TokenCredential credential, HttpClient? httpClient = null, - ILogger? logger = null) : base(modelId, endpoint, credential, httpClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, endpoint, credential, httpClient, loggerFactory) { } @@ -57,18 +58,18 @@ public AzureTextCompletion( /// /// Azure OpenAI model ID or deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource /// Custom . - /// Application logger + /// The to use for logging. If null, no logging will be performed. public AzureTextCompletion( string modelId, OpenAIClient openAIClient, - ILogger? logger = null) : base(modelId, openAIClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, openAIClient, loggerFactory) { } /// public IAsyncEnumerable GetStreamingCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings, CancellationToken cancellationToken = default) { this.LogActionDetails(); @@ -78,7 +79,7 @@ public IAsyncEnumerable GetStreamingCompletionsAsync( /// public Task> GetCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings, CancellationToken cancellationToken = default) { this.LogActionDetails(); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs index 0c9bcc0f28e4..9394be351f4d 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; @@ -23,21 +24,21 @@ public sealed class OpenAITextCompletion : OpenAIClientBase, ITextCompletion /// OpenAI API Key /// OpenAI Organization Id (usually optional) /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public OpenAITextCompletion( string modelId, string apiKey, string? organization = null, HttpClient? httpClient = null, - ILogger? logger = null - ) : base(modelId, apiKey, organization, httpClient, logger) + ILoggerFactory? loggerFactory = null + ) : base(modelId, apiKey, organization, httpClient, loggerFactory) { } /// public IAsyncEnumerable GetStreamingCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings, CancellationToken cancellationToken = default) { this.LogActionDetails(); @@ -47,7 +48,7 @@ public IAsyncEnumerable GetStreamingCompletionsAsync( /// public Task> GetCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings, CancellationToken cancellationToken = default) { this.LogActionDetails(); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/AzureTextEmbeddingGeneration.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/AzureTextEmbeddingGeneration.cs index 3f7b50959610..ceed977b10a0 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/AzureTextEmbeddingGeneration.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/AzureTextEmbeddingGeneration.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Net.Http; using System.Threading; @@ -23,13 +24,13 @@ public sealed class AzureTextEmbeddingGeneration : AzureOpenAIClientBase, ITextE /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public AzureTextEmbeddingGeneration( string modelId, string endpoint, string apiKey, HttpClient? httpClient = null, - ILogger? logger = null) : base(modelId, endpoint, apiKey, httpClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, endpoint, apiKey, httpClient, loggerFactory) { } @@ -40,13 +41,13 @@ public AzureTextEmbeddingGeneration( /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Token credentials, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc. /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public AzureTextEmbeddingGeneration( string modelId, string endpoint, TokenCredential credential, HttpClient? httpClient = null, - ILogger? logger = null) : base(modelId, endpoint, credential, httpClient, logger) + ILoggerFactory? loggerFactory = null) : base(modelId, endpoint, credential, httpClient, loggerFactory) { } @@ -56,7 +57,7 @@ public AzureTextEmbeddingGeneration( /// List of strings to generate embeddings for /// The to monitor for cancellation requests. The default is . /// List of embeddings - public Task>> GenerateEmbeddingsAsync( + public Task>> GenerateEmbeddingsAsync( IList data, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/OpenAITextEmbeddingGeneration.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/OpenAITextEmbeddingGeneration.cs index 4f41aae08b9f..1d19b0b546d5 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/OpenAITextEmbeddingGeneration.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/OpenAITextEmbeddingGeneration.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Net.Http; using System.Threading; @@ -22,14 +23,14 @@ public sealed class OpenAITextEmbeddingGeneration : OpenAIClientBase, ITextEmbed /// OpenAI API Key /// OpenAI Organization Id (usually optional) /// Custom for HTTP requests. - /// Application logger + /// The to use for logging. If null, no logging will be performed. public OpenAITextEmbeddingGeneration( string modelId, string apiKey, string? organization = null, HttpClient? httpClient = null, - ILogger? logger = null - ) : base(modelId, apiKey, organization, httpClient, logger) + ILoggerFactory? loggerFactory = null + ) : base(modelId, apiKey, organization, httpClient, loggerFactory) { } @@ -39,7 +40,7 @@ public OpenAITextEmbeddingGeneration( /// List of strings to generate embeddings for /// The to monitor for cancellation requests. The default is . /// List of embeddings - public Task>> GenerateEmbeddingsAsync( + public Task>> GenerateEmbeddingsAsync( IList data, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/TextEmbeddingResponse.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/TextEmbeddingResponse.cs index 0e6db886925f..1dcdca9841e0 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/TextEmbeddingResponse.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/TextEmbeddingResponse.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; @@ -19,7 +21,8 @@ public sealed class EmbeddingResponseIndex /// The embedding vector /// [JsonPropertyName("embedding")] - public IList Values { get; set; } = new List(); + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Values { get; set; } /// /// Index of the embedding vector diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/GPT3Tokenizer.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/GPT3Tokenizer.cs deleted file mode 100644 index a0f64a558a9c..000000000000 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/GPT3Tokenizer.cs +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Buffers; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Runtime.InteropServices; -using System.Text; -using System.Text.RegularExpressions; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI.Tokenizers.Settings; - -namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.Tokenizers; - -/// -/// Port of GPT3 Javascript tokenizer recommended by OpenAI. -/// See https://platform.openai.com/tokenizer and -/// https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them -/// -public static class GPT3Tokenizer -{ - private static readonly ConcurrentDictionary> s_bpeCache = new(); - - /// Lookup table from byte (index) to associated char. - /// Computed result of function at https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L9-L28. - private static readonly char[] s_bytesToUnicode = new char[] - { - (char)0x0100, (char)0x0101, (char)0x0102, (char)0x0103, (char)0x0104, (char)0x0105, (char)0x0106, (char)0x0107, - (char)0x0108, (char)0x0109, (char)0x010A, (char)0x010B, (char)0x010C, (char)0x010D, (char)0x010E, (char)0x010F, - (char)0x0110, (char)0x0111, (char)0x0112, (char)0x0113, (char)0x0114, (char)0x0115, (char)0x0116, (char)0x0117, - (char)0x0118, (char)0x0119, (char)0x011A, (char)0x011B, (char)0x011C, (char)0x011D, (char)0x011E, (char)0x011F, - (char)0x0120, (char)0x0021, (char)0x0022, (char)0x0023, (char)0x0024, (char)0x0025, (char)0x0026, (char)0x0027, - (char)0x0028, (char)0x0029, (char)0x002A, (char)0x002B, (char)0x002C, (char)0x002D, (char)0x002E, (char)0x002F, - (char)0x0030, (char)0x0031, (char)0x0032, (char)0x0033, (char)0x0034, (char)0x0035, (char)0x0036, (char)0x0037, - (char)0x0038, (char)0x0039, (char)0x003A, (char)0x003B, (char)0x003C, (char)0x003D, (char)0x003E, (char)0x003F, - (char)0x0040, (char)0x0041, (char)0x0042, (char)0x0043, (char)0x0044, (char)0x0045, (char)0x0046, (char)0x0047, - (char)0x0048, (char)0x0049, (char)0x004A, (char)0x004B, (char)0x004C, (char)0x004D, (char)0x004E, (char)0x004F, - (char)0x0050, (char)0x0051, (char)0x0052, (char)0x0053, (char)0x0054, (char)0x0055, (char)0x0056, (char)0x0057, - (char)0x0058, (char)0x0059, (char)0x005A, (char)0x005B, (char)0x005C, (char)0x005D, (char)0x005E, (char)0x005F, - (char)0x0060, (char)0x0061, (char)0x0062, (char)0x0063, (char)0x0064, (char)0x0065, (char)0x0066, (char)0x0067, - (char)0x0068, (char)0x0069, (char)0x006A, (char)0x006B, (char)0x006C, (char)0x006D, (char)0x006E, (char)0x006F, - (char)0x0070, (char)0x0071, (char)0x0072, (char)0x0073, (char)0x0074, (char)0x0075, (char)0x0076, (char)0x0077, - (char)0x0078, (char)0x0079, (char)0x007A, (char)0x007B, (char)0x007C, (char)0x007D, (char)0x007E, (char)0x0121, - (char)0x0122, (char)0x0123, (char)0x0124, (char)0x0125, (char)0x0126, (char)0x0127, (char)0x0128, (char)0x0129, - (char)0x012A, (char)0x012B, (char)0x012C, (char)0x012D, (char)0x012E, (char)0x012F, (char)0x0130, (char)0x0131, - (char)0x0132, (char)0x0133, (char)0x0134, (char)0x0135, (char)0x0136, (char)0x0137, (char)0x0138, (char)0x0139, - (char)0x013A, (char)0x013B, (char)0x013C, (char)0x013D, (char)0x013E, (char)0x013F, (char)0x0140, (char)0x0141, - (char)0x0142, (char)0x00A1, (char)0x00A2, (char)0x00A3, (char)0x00A4, (char)0x00A5, (char)0x00A6, (char)0x00A7, - (char)0x00A8, (char)0x00A9, (char)0x00AA, (char)0x00AB, (char)0x00AC, (char)0x0143, (char)0x00AE, (char)0x00AF, - (char)0x00B0, (char)0x00B1, (char)0x00B2, (char)0x00B3, (char)0x00B4, (char)0x00B5, (char)0x00B6, (char)0x00B7, - (char)0x00B8, (char)0x00B9, (char)0x00BA, (char)0x00BB, (char)0x00BC, (char)0x00BD, (char)0x00BE, (char)0x00BF, - (char)0x00C0, (char)0x00C1, (char)0x00C2, (char)0x00C3, (char)0x00C4, (char)0x00C5, (char)0x00C6, (char)0x00C7, - (char)0x00C8, (char)0x00C9, (char)0x00CA, (char)0x00CB, (char)0x00CC, (char)0x00CD, (char)0x00CE, (char)0x00CF, - (char)0x00D0, (char)0x00D1, (char)0x00D2, (char)0x00D3, (char)0x00D4, (char)0x00D5, (char)0x00D6, (char)0x00D7, - (char)0x00D8, (char)0x00D9, (char)0x00DA, (char)0x00DB, (char)0x00DC, (char)0x00DD, (char)0x00DE, (char)0x00DF, - (char)0x00E0, (char)0x00E1, (char)0x00E2, (char)0x00E3, (char)0x00E4, (char)0x00E5, (char)0x00E6, (char)0x00E7, - (char)0x00E8, (char)0x00E9, (char)0x00EA, (char)0x00EB, (char)0x00EC, (char)0x00ED, (char)0x00EE, (char)0x00EF, - (char)0x00F0, (char)0x00F1, (char)0x00F2, (char)0x00F3, (char)0x00F4, (char)0x00F5, (char)0x00F6, (char)0x00F7, - (char)0x00F8, (char)0x00F9, (char)0x00FA, (char)0x00FB, (char)0x00FC, (char)0x00FD, (char)0x00FE, (char)0x00FF - }; - - /// - /// Regex for English contractions, e.g. "he's", "we'll", "I'm" etc. - /// - private static readonly Regex s_encodingRegex = new( - @"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+", - RegexOptions.Compiled, - TimeSpan.FromSeconds(5)); - - /// - /// The tokenizer uses a byte-pair encoding (BPE) algorithm to split words into - /// sub-words based on frequency and merges rules. It can handle out-of-vocabulary - /// words, punctuation, and special tokens. - /// - /// Text to tokenize - /// List of token IDs - public static List Encode(string text) - { - var bpeTokens = new List(); - - if (!string.IsNullOrEmpty(text)) - { - // Find all the matches. - MatchCollection matches = s_encodingRegex.Matches(text); - - // Determine the maximum number of UTF8 bytes that any match value could require. - int maxUtf8Length = 0; - for (int i = 0; i < matches.Count; i++) - { - Match m = matches[i]; - int utf8Length = EncodingUtf8GetByteCount(text.AsSpan(m.Index, m.Length)); - if (utf8Length > maxUtf8Length) - { - maxUtf8Length = utf8Length; - } - } - - // Ensure we have a sufficient Span buffer to accommodate maxUtf8Length chars. - // The byte-to-char mapping scheme employed is 1:1, so we'll end up needing 1 char - // for every 1 UTF8 byte. If we can reasonably stack-allocate the space, we do, otherwise - // we temporarily rent a pooled array. - char[]? arrayPoolArray = null; - Span chars = maxUtf8Length <= 256 - ? stackalloc char[maxUtf8Length] - : (arrayPoolArray = ArrayPool.Shared.Rent(maxUtf8Length)); - - // Rather than using separate space for the UTF8 bytes, we just reinterpret the Span - // as a Span. Since our mapping is 1:1, the space required for the bytes will always - // be half of the space required for the chars. We can UTF8-encode into the first half, and - // then walk backwards through the bytes, using the byte-to-char mapping scheme to populate - // the chars from the back to the front. By going in reverse, we guarantee we won't overwrite - // any bytes we haven't yet seen. - Span bytes = MemoryMarshal.AsBytes(chars); - - // Now that our space is created, do the actual encoding. - for (int matchIndex = 0; matchIndex < matches.Count; matchIndex++) - { - // Get the matched text as a span. - Match m = matches[matchIndex]; - ReadOnlySpan matchValue = text.AsSpan(m.Index, m.Length); - - // Encode the UTF8 bytes. - int bytesWritten = EncodingUtf8GetBytes(matchValue, bytes); - - // Translate those bytes into chars. - char[] bytesToUnicode = s_bytesToUnicode; - for (int i = bytesWritten - 1; i >= 0; i--) - { - chars[i] = bytesToUnicode[bytes[i]]; - } - - // Evaluate the BPE for the encoded chars. - foreach (string encoding in BytePairEncoding(chars.Slice(0, bytesWritten).ToString())) - { - bpeTokens.Add(GPT3Settings.Encoder[encoding]); - } - } - - // Return the rented array to the pool if we rented one. - if (arrayPoolArray is not null) - { - ArrayPool.Shared.Return(arrayPoolArray); - } - } - - return bpeTokens; - - static unsafe int EncodingUtf8GetByteCount(ReadOnlySpan chars) - { - fixed (char* charsPtr = chars) - { - return Encoding.UTF8.GetByteCount(charsPtr, chars.Length); - } - } - - static unsafe int EncodingUtf8GetBytes(ReadOnlySpan chars, Span bytes) - { - fixed (char* charPtr = chars) - { - fixed (byte* bytesPtr = bytes) - { - return Encoding.UTF8.GetBytes(charPtr, chars.Length, bytesPtr, bytes.Length); - } - } - } - } - - public static List Encode(StringBuilder? stringBuilder) => - stringBuilder is not null - ? Encode(stringBuilder.ToString()) - : new List(); - - public static List Encode(char[]? chars) => - chars is not null - ? Encode(new string(chars)) - : new List(); - - public static List Encode(IEnumerable? chars) => - chars is not null - ? Encode(string.Concat(chars)) - : new List(); - - private static List BytePairEncoding(string token) - { - if (s_bpeCache.TryGetValue(token, out List? value)) - { - return value; - } - - if (token.Length <= 1) - { - var list = new List(1) { token }; - s_bpeCache.TryAdd(token, list); - return list; - } - - List word = new(token.Length); - foreach (char c in token) - { - word.Add(c.ToString()); - } - - long smallestRank = long.MaxValue; - (string, string) smallestPair = ("", ""); - List? newWord = null; - - while (word.Count >= 2) - { - for (int pairIndex = 0; pairIndex < word.Count - 1; pairIndex++) - { - (string, string) pair = (word[pairIndex], word[pairIndex + 1]); - - long pairRank = GPT3Settings.BpeRanks.TryGetValue(pair, out int rank) ? rank : 100_000_000_000; - - if (pairRank <= smallestRank) - { - smallestRank = pairRank; - smallestPair = pair; - } - } - - if (!GPT3Settings.BpeRanks.ContainsKey(smallestPair)) - { - break; - } - - string first = smallestPair.Item1; - string second = smallestPair.Item2; - - newWord ??= new List(word.Count); - for (int i = 0; i < word.Count; i++) - { - int j = word.IndexOf(first, i); - - int limit = j < 0 ? word.Count : j; - for (int copy = i; copy < limit; copy++) - { - newWord.Add(word[copy]); - } - - if (j < 0) - { - break; - } - - i = j; - - if (i < (word.Count - 1) && - word[i] == first && - word[i + 1] == second) - { - newWord.Add($"{first}{second}"); - i++; - } - else - { - newWord.Add(word[i]); - } - } - - // Swap the new word in for the old - (word, newWord) = (newWord, word); - - // And reset state for the next go-around - newWord.Clear(); - smallestRank = long.MaxValue; - } - - s_bpeCache.TryAdd(token, word); - return word; - } -} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/EmbeddedResource.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/EmbeddedResource.cs deleted file mode 100644 index 6b9510d2f51c..000000000000 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/EmbeddedResource.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.IO; -using System.Reflection; - -namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.Tokenizers.Settings; - -internal static class EmbeddedResource -{ - // This is usually the assembly name, if the project follows the naming conventions about namespaces and assembly names - private const string PrefixToIgnore = "Microsoft.SemanticKernel.Connectors.AI.OpenAI"; - - private static readonly string s_namespace = typeof(EmbeddedResource).Namespace; - - /// - /// Return content of BPE file. - /// - /// BPE file content - internal static string ReadBytePairEncodingTable() - { - return ReadFile("vocab.bpe"); - } - - /// - /// Return content of encoding table file. - /// - /// Encoding table string - internal static string ReadEncodingTable() - { - return ReadFile("encoder.json"); - } - - /// - /// Read a content file embedded in the project. Files are read from disk, - /// not from the assembly, to avoid inflating the assembly size. - /// - /// Filename to read - /// File content - /// Error in case the file doesn't exist - private static string ReadFile(string fileName) - { - // Assume the class namespace matches the directory structure - var currentClassDir = s_namespace - .Replace(PrefixToIgnore, string.Empty) - .Trim('.') - .Replace('.', Path.DirectorySeparatorChar); - - // Check the execution assembly directory first - var assembly1 = Assembly.GetExecutingAssembly(); - var assembly1Dir = Path.GetDirectoryName(Path.GetFullPath(assembly1.Location)); - - // Concatenate assembly location with class namespace with file name - var filePath1 = Path.Combine(assembly1Dir, currentClassDir, fileName); - if (File.Exists(filePath1)) - { - return File.ReadAllText(filePath1); - } - - // Check the current assembly, in case that's a different file on a different directory - Assembly? assembly2 = Assembly.GetAssembly(typeof(EmbeddedResource)); - if (assembly2 == null) - { - throw new FileNotFoundException($"{fileName} not found, path: '{filePath1}'"); - } - - // Path where the assembly is - var assembly2Dir = Path.GetDirectoryName(Path.GetFullPath(assembly2.Location)); - - // No need to continue if the path is the same - if (assembly2Dir == assembly1Dir) - { - throw new FileNotFoundException($"{fileName} not found, path: '{filePath1}'"); - } - - // Concatenate assembly location with class namespace with file name - var filePath2 = Path.Combine(assembly2Dir, currentClassDir, fileName); - if (File.Exists(filePath2)) - { - return File.ReadAllText(filePath2); - } - - throw new FileNotFoundException($"{fileName} not found, paths: '{filePath1}', '{filePath2}'"); - } -} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/GPT3Settings.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/GPT3Settings.cs deleted file mode 100644 index 6419d731155c..000000000000 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/GPT3Settings.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Text.Json; -using Microsoft.SemanticKernel.AI; - -namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.Tokenizers.Settings; - -internal static class GPT3Settings -{ - /// Gets the cached encoding table (encoder.json). - internal static Dictionary Encoder => s_encoder.Value; - - /// Gets the cached byte pair encoding table (vocab.bpe). - internal static Dictionary<(string, string), int> BpeRanks => s_bpeRanks.Value; - - /// Lazy load the cached encoding table (encoder.json). - private static readonly Lazy> s_encoder = new(() => - JsonSerializer.Deserialize>( - EmbeddedResource.ReadEncodingTable()) ?? throw new AIException( - AIException.ErrorCodes.InvalidConfiguration, - "Encoding table deserialization returned NULL")); - - /// Lazy load the cached byte pair encoding table (vocab.bpe). - private static readonly Lazy> s_bpeRanks = new(() => - { - string table = EmbeddedResource.ReadBytePairEncodingTable(); - Debug.Assert(table.StartsWith("#version: 0.2", StringComparison.Ordinal)); - - // Skip past the header line - int pos = table.IndexOf('\n') + 1; - Debug.Assert(pos > 0); - - // For each line, split on the first space and add the pair to the dictionary as a key with the value being the entry number. - var result = new Dictionary<(string, string), int>(); - int nextPos; - while ((nextPos = table.IndexOf('\n', pos)) >= 0) - { - ReadOnlySpan span = table.AsSpan(pos, nextPos - pos).Trim(); - pos = span.IndexOf(' '); - if (pos >= 0) - { - result.Add((span.Slice(0, pos).ToString(), span.Slice(pos + 1).ToString()), result.Count); - } - - pos = nextPos + 1; - } - - return result; - }); -} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/encoder.json b/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/encoder.json deleted file mode 100644 index 1f1d9aaca301..000000000000 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/Tokenizers/Settings/encoder.json +++ /dev/null @@ -1 +0,0 @@ -{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "\u00a1": 94, "\u00a2": 95, "\u00a3": 96, "\u00a4": 97, "\u00a5": 98, "\u00a6": 99, "\u00a7": 100, "\u00a8": 101, "\u00a9": 102, "\u00aa": 103, "\u00ab": 104, "\u00ac": 105, "\u00ae": 106, "\u00af": 107, "\u00b0": 108, "\u00b1": 109, "\u00b2": 110, "\u00b3": 111, "\u00b4": 112, "\u00b5": 113, "\u00b6": 114, "\u00b7": 115, "\u00b8": 116, "\u00b9": 117, "\u00ba": 118, "\u00bb": 119, "\u00bc": 120, "\u00bd": 121, "\u00be": 122, "\u00bf": 123, "\u00c0": 124, "\u00c1": 125, "\u00c2": 126, "\u00c3": 127, "\u00c4": 128, "\u00c5": 129, "\u00c6": 130, "\u00c7": 131, "\u00c8": 132, "\u00c9": 133, "\u00ca": 134, "\u00cb": 135, "\u00cc": 136, "\u00cd": 137, "\u00ce": 138, "\u00cf": 139, "\u00d0": 140, "\u00d1": 141, "\u00d2": 142, "\u00d3": 143, "\u00d4": 144, "\u00d5": 145, "\u00d6": 146, "\u00d7": 147, "\u00d8": 148, "\u00d9": 149, "\u00da": 150, "\u00db": 151, "\u00dc": 152, "\u00dd": 153, "\u00de": 154, "\u00df": 155, "\u00e0": 156, "\u00e1": 157, "\u00e2": 158, "\u00e3": 159, "\u00e4": 160, "\u00e5": 161, "\u00e6": 162, "\u00e7": 163, "\u00e8": 164, "\u00e9": 165, "\u00ea": 166, "\u00eb": 167, "\u00ec": 168, "\u00ed": 169, "\u00ee": 170, "\u00ef": 171, "\u00f0": 172, "\u00f1": 173, "\u00f2": 174, "\u00f3": 175, "\u00f4": 176, "\u00f5": 177, "\u00f6": 178, "\u00f7": 179, "\u00f8": 180, "\u00f9": 181, "\u00fa": 182, "\u00fb": 183, "\u00fc": 184, "\u00fd": 185, "\u00fe": 186, "\u00ff": 187, "\u0100": 188, "\u0101": 189, "\u0102": 190, "\u0103": 191, "\u0104": 192, "\u0105": 193, "\u0106": 194, "\u0107": 195, "\u0108": 196, "\u0109": 197, "\u010a": 198, "\u010b": 199, "\u010c": 200, "\u010d": 201, "\u010e": 202, "\u010f": 203, "\u0110": 204, "\u0111": 205, "\u0112": 206, "\u0113": 207, "\u0114": 208, "\u0115": 209, "\u0116": 210, "\u0117": 211, "\u0118": 212, "\u0119": 213, "\u011a": 214, "\u011b": 215, "\u011c": 216, "\u011d": 217, "\u011e": 218, "\u011f": 219, "\u0120": 220, "\u0121": 221, "\u0122": 222, "\u0123": 223, "\u0124": 224, "\u0125": 225, "\u0126": 226, "\u0127": 227, "\u0128": 228, "\u0129": 229, "\u012a": 230, "\u012b": 231, "\u012c": 232, "\u012d": 233, "\u012e": 234, "\u012f": 235, "\u0130": 236, "\u0131": 237, "\u0132": 238, "\u0133": 239, "\u0134": 240, "\u0135": 241, "\u0136": 242, "\u0137": 243, "\u0138": 244, "\u0139": 245, "\u013a": 246, "\u013b": 247, "\u013c": 248, "\u013d": 249, "\u013e": 250, "\u013f": 251, "\u0140": 252, "\u0141": 253, "\u0142": 254, "\u0143": 255, "\u0120t": 256, "\u0120a": 257, "he": 258, "in": 259, "re": 260, "on": 261, "\u0120the": 262, "er": 263, "\u0120s": 264, "at": 265, "\u0120w": 266, "\u0120o": 267, "en": 268, "\u0120c": 269, "it": 270, "is": 271, "an": 272, "or": 273, "es": 274, "\u0120b": 275, "ed": 276, "\u0120f": 277, "ing": 278, "\u0120p": 279, "ou": 280, "\u0120an": 281, "al": 282, "ar": 283, "\u0120to": 284, "\u0120m": 285, "\u0120of": 286, "\u0120in": 287, "\u0120d": 288, "\u0120h": 289, "\u0120and": 290, "ic": 291, "as": 292, "le": 293, "\u0120th": 294, "ion": 295, "om": 296, "ll": 297, "ent": 298, "\u0120n": 299, "\u0120l": 300, "st": 301, "\u0120re": 302, "ve": 303, "\u0120e": 304, "ro": 305, "ly": 306, "\u0120be": 307, "\u0120g": 308, "\u0120T": 309, "ct": 310, "\u0120S": 311, "id": 312, "ot": 313, "\u0120I": 314, "ut": 315, "et": 316, "\u0120A": 317, "\u0120is": 318, "\u0120on": 319, "im": 320, "am": 321, "ow": 322, "ay": 323, "ad": 324, "se": 325, "\u0120that": 326, "\u0120C": 327, "ig": 328, "\u0120for": 329, "ac": 330, "\u0120y": 331, "ver": 332, "ur": 333, "\u0120u": 334, "ld": 335, "\u0120st": 336, "\u0120M": 337, "'s": 338, "\u0120he": 339, "\u0120it": 340, "ation": 341, "ith": 342, "ir": 343, "ce": 344, "\u0120you": 345, "il": 346, "\u0120B": 347, "\u0120wh": 348, "ol": 349, "\u0120P": 350, "\u0120with": 351, "\u01201": 352, "ter": 353, "ch": 354, "\u0120as": 355, "\u0120we": 356, "\u0120(": 357, "nd": 358, "ill": 359, "\u0120D": 360, "if": 361, "\u01202": 362, "ag": 363, "ers": 364, "ke": 365, "\u0120\"": 366, "\u0120H": 367, "em": 368, "\u0120con": 369, "\u0120W": 370, "\u0120R": 371, "her": 372, "\u0120was": 373, "\u0120r": 374, "od": 375, "\u0120F": 376, "ul": 377, "ate": 378, "\u0120at": 379, "ri": 380, "pp": 381, "ore": 382, "\u0120The": 383, "\u0120se": 384, "us": 385, "\u0120pro": 386, "\u0120ha": 387, "um": 388, "\u0120are": 389, "\u0120de": 390, "ain": 391, "and": 392, "\u0120or": 393, "igh": 394, "est": 395, "ist": 396, "ab": 397, "rom": 398, "\u0120N": 399, "th": 400, "\u0120com": 401, "\u0120G": 402, "un": 403, "op": 404, "00": 405, "\u0120L": 406, "\u0120not": 407, "ess": 408, "\u0120ex": 409, "\u0120v": 410, "res": 411, "\u0120E": 412, "ew": 413, "ity": 414, "ant": 415, "\u0120by": 416, "el": 417, "os": 418, "ort": 419, "oc": 420, "qu": 421, "\u0120from": 422, "\u0120have": 423, "\u0120su": 424, "ive": 425, "ould": 426, "\u0120sh": 427, "\u0120this": 428, "nt": 429, "ra": 430, "pe": 431, "ight": 432, "art": 433, "ment": 434, "\u0120al": 435, "ust": 436, "end": 437, "--": 438, "all": 439, "\u0120O": 440, "ack": 441, "\u0120ch": 442, "\u0120le": 443, "ies": 444, "red": 445, "ard": 446, "\u00e2\u0122": 447, "out": 448, "\u0120J": 449, "\u0120ab": 450, "ear": 451, "iv": 452, "ally": 453, "our": 454, "ost": 455, "gh": 456, "pt": 457, "\u0120pl": 458, "ast": 459, "\u0120can": 460, "ak": 461, "ome": 462, "ud": 463, "The": 464, "\u0120his": 465, "\u0120do": 466, "\u0120go": 467, "\u0120has": 468, "ge": 469, "'t": 470, "\u0120U": 471, "rou": 472, "\u0120sa": 473, "\u0120j": 474, "\u0120but": 475, "\u0120wor": 476, "\u0120all": 477, "ect": 478, "\u0120k": 479, "ame": 480, "\u0120will": 481, "ok": 482, "\u0120whe": 483, "\u0120they": 484, "ide": 485, "01": 486, "ff": 487, "ich": 488, "pl": 489, "ther": 490, "\u0120tr": 491, "..": 492, "\u0120int": 493, "ie": 494, "ure": 495, "age": 496, "\u0120ne": 497, "ial": 498, "ap": 499, "ine": 500, "ice": 501, "\u0120me": 502, "\u0120out": 503, "ans": 504, "one": 505, "ong": 506, "ions": 507, "\u0120who": 508, "\u0120K": 509, "\u0120up": 510, "\u0120their": 511, "\u0120ad": 512, "\u01203": 513, "\u0120us": 514, "ated": 515, "ous": 516, "\u0120more": 517, "ue": 518, "og": 519, "\u0120St": 520, "ind": 521, "ike": 522, "\u0120so": 523, "ime": 524, "per": 525, ".\"": 526, "ber": 527, "iz": 528, "act": 529, "\u0120one": 530, "\u0120said": 531, "\u0120-": 532, "are": 533, "\u0120your": 534, "cc": 535, "\u0120Th": 536, "\u0120cl": 537, "ep": 538, "ake": 539, "able": 540, "ip": 541, "\u0120cont": 542, "\u0120which": 543, "ia": 544, "\u0120im": 545, "\u0120about": 546, "\u0120were": 547, "very": 548, "ub": 549, "\u0120had": 550, "\u0120en": 551, "\u0120comp": 552, ",\"": 553, "\u0120In": 554, "\u0120un": 555, "\u0120ag": 556, "ire": 557, "ace": 558, "au": 559, "ary": 560, "\u0120would": 561, "ass": 562, "ry": 563, "\u0120\u00e2\u0122": 564, "cl": 565, "ook": 566, "ere": 567, "so": 568, "\u0120V": 569, "ign": 570, "ib": 571, "\u0120off": 572, "\u0120te": 573, "ven": 574, "\u0120Y": 575, "ile": 576, "ose": 577, "ite": 578, "orm": 579, "\u0120201": 580, "\u0120res": 581, "\u0120man": 582, "\u0120per": 583, "\u0120other": 584, "ord": 585, "ult": 586, "\u0120been": 587, "\u0120like": 588, "ase": 589, "ance": 590, "ks": 591, "ays": 592, "own": 593, "ence": 594, "\u0120dis": 595, "ction": 596, "\u0120any": 597, "\u0120app": 598, "\u0120sp": 599, "int": 600, "ress": 601, "ations": 602, "ail": 603, "\u01204": 604, "ical": 605, "\u0120them": 606, "\u0120her": 607, "ount": 608, "\u0120Ch": 609, "\u0120ar": 610, "\u0120if": 611, "\u0120there": 612, "\u0120pe": 613, "\u0120year": 614, "av": 615, "\u0120my": 616, "\u0120some": 617, "\u0120when": 618, "ough": 619, "ach": 620, "\u0120than": 621, "ru": 622, "ond": 623, "ick": 624, "\u0120over": 625, "vel": 626, "\u0120qu": 627, "\u010a\u010a": 628, "\u0120sc": 629, "reat": 630, "ree": 631, "\u0120It": 632, "ound": 633, "port": 634, "\u0120also": 635, "\u0120part": 636, "fter": 637, "\u0120kn": 638, "\u0120bec": 639, "\u0120time": 640, "ens": 641, "\u01205": 642, "ople": 643, "\u0120what": 644, "\u0120no": 645, "du": 646, "mer": 647, "ang": 648, "\u0120new": 649, "----": 650, "\u0120get": 651, "ory": 652, "ition": 653, "ings": 654, "\u0120just": 655, "\u0120into": 656, "\u01200": 657, "ents": 658, "ove": 659, "te": 660, "\u0120people": 661, "\u0120pre": 662, "\u0120its": 663, "\u0120rec": 664, "\u0120tw": 665, "ian": 666, "irst": 667, "ark": 668, "ors": 669, "\u0120work": 670, "ade": 671, "ob": 672, "\u0120she": 673, "\u0120our": 674, "wn": 675, "ink": 676, "lic": 677, "\u012019": 678, "\u0120He": 679, "ish": 680, "nder": 681, "ause": 682, "\u0120him": 683, "ons": 684, "\u0120[": 685, "\u0120ro": 686, "form": 687, "ild": 688, "ates": 689, "vers": 690, "\u0120only": 691, "oll": 692, "\u0120spe": 693, "ck": 694, "ell": 695, "amp": 696, "\u0120acc": 697, "\u0120bl": 698, "ious": 699, "urn": 700, "ft": 701, "ood": 702, "\u0120how": 703, "hed": 704, "\u0120'": 705, "\u0120after": 706, "aw": 707, "\u0120att": 708, "ov": 709, "ne": 710, "\u0120play": 711, "erv": 712, "ict": 713, "\u0120could": 714, "itt": 715, "\u0120am": 716, "\u0120first": 717, "\u01206": 718, "\u0120act": 719, "\u0120$": 720, "ec": 721, "hing": 722, "ual": 723, "ull": 724, "\u0120comm": 725, "oy": 726, "old": 727, "ces": 728, "ater": 729, "\u0120fe": 730, "\u0120bet": 731, "we": 732, "iff": 733, "\u0120two": 734, "ock": 735, "\u0120back": 736, ").": 737, "ident": 738, "\u0120under": 739, "rough": 740, "sel": 741, "xt": 742, "\u0120may": 743, "round": 744, "\u0120po": 745, "ph": 746, "iss": 747, "\u0120des": 748, "\u0120most": 749, "\u0120did": 750, "\u0120add": 751, "ject": 752, "\u0120inc": 753, "fore": 754, "\u0120pol": 755, "ont": 756, "\u0120again": 757, "clud": 758, "tern": 759, "\u0120know": 760, "\u0120need": 761, "\u0120cons": 762, "\u0120co": 763, "\u0120.": 764, "\u0120want": 765, "\u0120see": 766, "\u01207": 767, "ning": 768, "iew": 769, "\u0120This": 770, "ced": 771, "\u0120even": 772, "\u0120ind": 773, "ty": 774, "\u0120We": 775, "ath": 776, "\u0120these": 777, "\u0120pr": 778, "\u0120use": 779, "\u0120because": 780, "\u0120fl": 781, "ng": 782, "\u0120now": 783, "\u0120\u00e2\u0122\u0135": 784, "com": 785, "ise": 786, "\u0120make": 787, "\u0120then": 788, "ower": 789, "\u0120every": 790, "\u0120Un": 791, "\u0120sec": 792, "oss": 793, "uch": 794, "\u0120em": 795, "\u0120=": 796, "\u0120Re": 797, "ied": 798, "rit": 799, "\u0120inv": 800, "lect": 801, "\u0120supp": 802, "ating": 803, "\u0120look": 804, "man": 805, "pect": 806, "\u01208": 807, "row": 808, "\u0120bu": 809, "\u0120where": 810, "ific": 811, "\u0120years": 812, "ily": 813, "\u0120diff": 814, "\u0120should": 815, "\u0120rem": 816, "Th": 817, "In": 818, "\u0120ev": 819, "day": 820, "'re": 821, "rib": 822, "\u0120rel": 823, "ss": 824, "\u0120def": 825, "\u0120right": 826, "\u0120sy": 827, "),": 828, "les": 829, "000": 830, "hen": 831, "\u0120through": 832, "\u0120Tr": 833, "__": 834, "\u0120way": 835, "\u0120don": 836, "\u0120,": 837, "\u012010": 838, "ased": 839, "\u0120ass": 840, "ublic": 841, "\u0120reg": 842, "\u0120And": 843, "ix": 844, "\u0120very": 845, "\u0120includ": 846, "other": 847, "\u0120imp": 848, "oth": 849, "\u0120sub": 850, "\u0120\u00e2\u0122\u0136": 851, "\u0120being": 852, "arg": 853, "\u0120Wh": 854, "==": 855, "ible": 856, "\u0120does": 857, "ange": 858, "ram": 859, "\u01209": 860, "ert": 861, "ps": 862, "ited": 863, "ational": 864, "\u0120br": 865, "\u0120down": 866, "\u0120many": 867, "aking": 868, "\u0120call": 869, "uring": 870, "ities": 871, "\u0120ph": 872, "ics": 873, "als": 874, "\u0120dec": 875, "ative": 876, "ener": 877, "\u0120before": 878, "ility": 879, "\u0120well": 880, "\u0120much": 881, "erson": 882, "\u0120those": 883, "\u0120such": 884, "\u0120ke": 885, "\u0120end": 886, "\u0120But": 887, "ason": 888, "ting": 889, "\u0120long": 890, "ef": 891, "\u0120think": 892, "ys": 893, "\u0120bel": 894, "\u0120sm": 895, "its": 896, "ax": 897, "\u0120own": 898, "\u0120prov": 899, "\u0120set": 900, "ife": 901, "ments": 902, "ble": 903, "ward": 904, "\u0120show": 905, "\u0120pres": 906, "ms": 907, "omet": 908, "\u0120ob": 909, "\u0120say": 910, "\u0120Sh": 911, "ts": 912, "ful": 913, "\u0120eff": 914, "\u0120gu": 915, "\u0120inst": 916, "und": 917, "ren": 918, "cess": 919, "\u0120ent": 920, "\u0120You": 921, "\u0120good": 922, "\u0120start": 923, "ince": 924, "\u0120made": 925, "tt": 926, "stem": 927, "olog": 928, "up": 929, "\u0120|": 930, "ump": 931, "\u0120hel": 932, "vern": 933, "ular": 934, "ually": 935, "\u0120ac": 936, "\u0120mon": 937, "\u0120last": 938, "\u0120200": 939, "10": 940, "\u0120stud": 941, "ures": 942, "\u0120Ar": 943, "self": 944, "ars": 945, "meric": 946, "ues": 947, "cy": 948, "\u0120min": 949, "ollow": 950, "\u0120col": 951, "io": 952, "\u0120mod": 953, "\u0120count": 954, "\u0120Com": 955, "hes": 956, "\u0120fin": 957, "air": 958, "ier": 959, "\u00e2\u0122\u0136": 960, "read": 961, "ank": 962, "atch": 963, "ever": 964, "\u0120str": 965, "\u0120point": 966, "ork": 967, "\u0120New": 968, "\u0120sur": 969, "ool": 970, "alk": 971, "ement": 972, "\u0120used": 973, "ract": 974, "ween": 975, "\u0120same": 976, "oun": 977, "\u0120Al": 978, "ci": 979, "\u0120differe": 980, "\u0120while": 981, "--------": 982, "\u0120game": 983, "cept": 984, "\u0120sim": 985, "...": 986, "\u0120inter": 987, "ek": 988, "\u0120report": 989, "\u0120produ": 990, "\u0120still": 991, "led": 992, "ah": 993, "\u0120here": 994, "\u0120world": 995, "\u0120though": 996, "\u0120num": 997, "arch": 998, "imes": 999, "ale": 1000, "\u0120Se": 1001, "\u0120If": 1002, "//": 1003, "\u0120Le": 1004, "\u0120ret": 1005, "\u0120ref": 1006, "\u0120trans": 1007, "ner": 1008, "ution": 1009, "ters": 1010, "\u0120take": 1011, "\u0120Cl": 1012, "\u0120conf": 1013, "way": 1014, "ave": 1015, "\u0120going": 1016, "\u0120sl": 1017, "ug": 1018, "\u0120Americ": 1019, "\u0120spec": 1020, "\u0120hand": 1021, "\u0120between": 1022, "ists": 1023, "\u0120De": 1024, "oot": 1025, "It": 1026, "\u0120ear": 1027, "\u0120against": 1028, "\u0120high": 1029, "gan": 1030, "az": 1031, "ather": 1032, "\u0120exp": 1033, "\u0120op": 1034, "\u0120ins": 1035, "\u0120gr": 1036, "\u0120help": 1037, "\u0120requ": 1038, "ets": 1039, "ins": 1040, "\u0120Pro": 1041, "ism": 1042, "\u0120found": 1043, "land": 1044, "ata": 1045, "uss": 1046, "ames": 1047, "\u0120person": 1048, "\u0120great": 1049, "pr": 1050, "\u0120sign": 1051, "\u0120An": 1052, "'ve": 1053, "\u0120somet": 1054, "\u0120ser": 1055, "hip": 1056, "\u0120run": 1057, "\u0120:": 1058, "\u0120ter": 1059, "irect": 1060, "\u0120follow": 1061, "\u0120det": 1062, "ices": 1063, "\u0120find": 1064, "12": 1065, "\u0120mem": 1066, "\u0120cr": 1067, "ered": 1068, "ex": 1069, "\u0120ext": 1070, "uth": 1071, "ense": 1072, "co": 1073, "\u0120team": 1074, "ving": 1075, "ouse": 1076, "ash": 1077, "att": 1078, "ved": 1079, "\u0120system": 1080, "\u0120As": 1081, "der": 1082, "ives": 1083, "min": 1084, "\u0120lead": 1085, "\u0120Bl": 1086, "cent": 1087, "\u0120around": 1088, "\u0120govern": 1089, "\u0120cur": 1090, "velop": 1091, "any": 1092, "\u0120cour": 1093, "alth": 1094, "ages": 1095, "ize": 1096, "\u0120car": 1097, "ode": 1098, "\u0120law": 1099, "\u0120read": 1100, "'m": 1101, "con": 1102, "\u0120real": 1103, "\u0120support": 1104, "\u012012": 1105, "....": 1106, "\u0120really": 1107, "ness": 1108, "\u0120fact": 1109, "\u0120day": 1110, "\u0120both": 1111, "ying": 1112, "\u0120serv": 1113, "\u0120For": 1114, "\u0120three": 1115, "\u0120wom": 1116, "\u0120med": 1117, "ody": 1118, "\u0120They": 1119, "50": 1120, "\u0120exper": 1121, "ton": 1122, "\u0120each": 1123, "akes": 1124, "\u0120che": 1125, "\u0120cre": 1126, "ines": 1127, "\u0120rep": 1128, "19": 1129, "gg": 1130, "illion": 1131, "\u0120grou": 1132, "ute": 1133, "ik": 1134, "We": 1135, "get": 1136, "ER": 1137, "\u0120met": 1138, "\u0120says": 1139, "ox": 1140, "\u0120during": 1141, "ern": 1142, "ized": 1143, "ared": 1144, "\u0120fam": 1145, "ically": 1146, "\u0120happ": 1147, "\u0120Is": 1148, "\u0120char": 1149, "med": 1150, "vent": 1151, "\u0120gener": 1152, "ient": 1153, "ple": 1154, "iet": 1155, "rent": 1156, "11": 1157, "ves": 1158, "ption": 1159, "\u012020": 1160, "formation": 1161, "\u0120cor": 1162, "\u0120offic": 1163, "ield": 1164, "\u0120too": 1165, "ision": 1166, "\u0120inf": 1167, "\u0120Z": 1168, "the": 1169, "oad": 1170, "\u0120public": 1171, "\u0120prog": 1172, "ric": 1173, "**": 1174, "\u0120war": 1175, "\u0120power": 1176, "view": 1177, "\u0120few": 1178, "\u0120loc": 1179, "\u0120different": 1180, "\u0120state": 1181, "\u0120head": 1182, "'ll": 1183, "\u0120poss": 1184, "\u0120stat": 1185, "ret": 1186, "ants": 1187, "\u0120val": 1188, "\u0120iss": 1189, "\u0120cle": 1190, "ivers": 1191, "anc": 1192, "\u0120expl": 1193, "\u0120another": 1194, "\u0120Q": 1195, "\u0120av": 1196, "thing": 1197, "nce": 1198, "Wh": 1199, "\u0120child": 1200, "\u0120since": 1201, "ired": 1202, "less": 1203, "\u0120life": 1204, "\u0120develop": 1205, "ittle": 1206, "\u0120dep": 1207, "\u0120pass": 1208, "\u00e3\u0125": 1209, "\u0120turn": 1210, "orn": 1211, "This": 1212, "bers": 1213, "ross": 1214, "\u0120Ad": 1215, "\u0120fr": 1216, "\u0120resp": 1217, "\u0120second": 1218, "oh": 1219, "\u0120/": 1220, "\u0120disc": 1221, "\u0120&": 1222, "\u0120something": 1223, "\u0120comple": 1224, "\u0120ed": 1225, "\u0120fil": 1226, "\u0120month": 1227, "aj": 1228, "uc": 1229, "\u0120government": 1230, "\u0120without": 1231, "\u0120leg": 1232, "\u0120dist": 1233, "\u0120put": 1234, "\u0120quest": 1235, "ann": 1236, "\u0120prot": 1237, "20": 1238, "\u0120never": 1239, "ience": 1240, "\u0120level": 1241, "\u0120art": 1242, "\u0120things": 1243, "\u0120might": 1244, "\u0120effect": 1245, "\u0120contro": 1246, "\u0120cent": 1247, "\u012018": 1248, "\u0120allow": 1249, "\u0120belie": 1250, "chool": 1251, "ott": 1252, "\u0120incre": 1253, "\u0120feel": 1254, "\u0120result": 1255, "\u0120lot": 1256, "\u0120fun": 1257, "ote": 1258, "\u0120ty": 1259, "erest": 1260, "\u0120contin": 1261, "\u0120using": 1262, "\u0120big": 1263, "201": 1264, "\u0120ask": 1265, "\u0120best": 1266, "\u0120)": 1267, "IN": 1268, "\u0120opp": 1269, "30": 1270, "\u0120number": 1271, "iness": 1272, "St": 1273, "lease": 1274, "\u0120ca": 1275, "\u0120must": 1276, "\u0120direct": 1277, "\u0120gl": 1278, "\u0120<": 1279, "\u0120open": 1280, "\u0120post": 1281, "\u0120come": 1282, "\u0120seem": 1283, "ording": 1284, "\u0120week": 1285, "ately": 1286, "ital": 1287, "\u0120el": 1288, "riend": 1289, "\u0120far": 1290, "\u0120tra": 1291, "inal": 1292, "\u0120pri": 1293, "\u0120US": 1294, "\u0120place": 1295, "\u0120form": 1296, "\u0120told": 1297, "\":": 1298, "ains": 1299, "ature": 1300, "\u0120Trump": 1301, "\u0120stand": 1302, "\u0120#": 1303, "ider": 1304, "\u0120Fr": 1305, "\u0120next": 1306, "\u0120soc": 1307, "\u0120pur": 1308, "\u0120let": 1309, "\u0120little": 1310, "\u0120hum": 1311, "\u0120i": 1312, "ron": 1313, "15": 1314, "\u012015": 1315, "\u0120commun": 1316, "\u0120mark": 1317, "\u0120There": 1318, "\u0120wr": 1319, "\u0120That": 1320, "\u0120information": 1321, "ways": 1322, "\u0120bus": 1323, "app": 1324, "\u0120invest": 1325, "me": 1326, "\u0120hard": 1327, "ained": 1328, "ead": 1329, "\u0120import": 1330, "\u0120appro": 1331, "\u0120test": 1332, "\u0120tri": 1333, "\u0120rest": 1334, "osed": 1335, "\u0120full": 1336, "\u0120care": 1337, "\u0120Sp": 1338, "\u0120case": 1339, "ON": 1340, "\u0120sk": 1341, "\u0120less": 1342, "\u0120+": 1343, "\u0120partic": 1344, "\u0120Pl": 1345, "ably": 1346, "uck": 1347, "ished": 1348, "chn": 1349, "be": 1350, "\u0120list": 1351, "ator": 1352, "\u0120top": 1353, "\u0120adv": 1354, "\u0120Be": 1355, "ruct": 1356, "\u0120dem": 1357, "ration": 1358, "ling": 1359, "gy": 1360, "reen": 1361, "ger": 1362, "\u0120home": 1363, "\u0120left": 1364, "\u0120better": 1365, "\u0120data": 1366, "\u012011": 1367, "\u0120attack": 1368, "\u0120proble": 1369, "line": 1370, "ards": 1371, "\u0120beh": 1372, "ral": 1373, "\u0120How": 1374, "\u0120She": 1375, "arge": 1376, "\u0120--": 1377, "://": 1378, "\u0120bro": 1379, "\u0120Ph": 1380, "ats": 1381, "\u0120build": 1382, "ww": 1383, "ided": 1384, "aim": 1385, "ases": 1386, "ency": 1387, "\u0120main": 1388, "ined": 1389, "\u0120including": 1390, "\u0120{": 1391, "\u0120got": 1392, "\u0120interest": 1393, "\u0120keep": 1394, "\u0120X": 1395, "\u0120eas": 1396, "aining": 1397, "\u0120class": 1398, "\u00e2\u0122\u00a6": 1399, "\u0120No": 1400, "\u0120var": 1401, "\u0120small": 1402, "ample": 1403, "AT": 1404, "\u0120ide": 1405, "\u0120So": 1406, "\u0120rece": 1407, "\u0120polit": 1408, "\u0120mov": 1409, "\u0120plan": 1410, "\u0120percent": 1411, "iving": 1412, "\u0120camp": 1413, "\u0120pay": 1414, "14": 1415, "sc": 1416, "ised": 1417, "\u0120unt": 1418, "oney": 1419, "ploy": 1420, "====": 1421, "\u0120didn": 1422, "\u0120Ind": 1423, "els": 1424, "ertain": 1425, "\u0120pos": 1426, "____": 1427, "iver": 1428, "\u0120process": 1429, "\u0120program": 1430, "ified": 1431, "\u0120Rep": 1432, "16": 1433, "uro": 1434, "ology": 1435, "atter": 1436, "ina": 1437, "\u0120name": 1438, "\u0120All": 1439, "\u0120four": 1440, "\u0120return": 1441, "vious": 1442, "bs": 1443, "\u0120called": 1444, "\u0120move": 1445, "\u0120Sc": 1446, "ird": 1447, "\u0120group": 1448, "\u0120bre": 1449, "\u0120men": 1450, "\u0120cap": 1451, "ten": 1452, "ee": 1453, "\u0120dri": 1454, "leg": 1455, "here": 1456, "uthor": 1457, "\u0120pat": 1458, "\u0120current": 1459, "ides": 1460, "\u0120pop": 1461, "to": 1462, "ention": 1463, "\u0120always": 1464, "\u0120mil": 1465, "\u0120women": 1466, "\u012016": 1467, "\u0120old": 1468, "iven": 1469, "raph": 1470, "\u0120Or": 1471, "ror": 1472, "ently": 1473, "\u0120near": 1474, "\u0120Ex": 1475, "ream": 1476, "sh": 1477, "\u012014": 1478, "\u0120free": 1479, "ission": 1480, "stand": 1481, "\u0120Con": 1482, "ality": 1483, "used": 1484, "13": 1485, "\u0120design": 1486, "\u0120change": 1487, "\u0120chang": 1488, "\u0120bo": 1489, "\u0120vis": 1490, "ember": 1491, "\u0120book": 1492, "ready": 1493, "\u0120kill": 1494, "25": 1495, "pped": 1496, "\u0120away": 1497, "\u0120able": 1498, "\u0120country": 1499, "\u0120const": 1500, "arn": 1501, "\u0120order": 1502, "AR": 1503, "ior": 1504, "ium": 1505, "orth": 1506, "18": 1507, "ailable": 1508, "\u0120sw": 1509, "\u0120million": 1510, "\u012013": 1511, "atic": 1512, "ted": 1513, "\u0120Go": 1514, "\u0120oper": 1515, "eng": 1516, "\u0120thing": 1517, "ajor": 1518, "conom": 1519, "\u0120Comm": 1520, "\u0120why": 1521, "ured": 1522, "ural": 1523, "\u0120school": 1524, "by": 1525, "\u0120Mar": 1526, "\u0120aff": 1527, "\u0120days": 1528, "\u0120ann": 1529, "ush": 1530, "ane": 1531, "If": 1532, "eg": 1533, "\u0120prof": 1534, "\u0120health": 1535, "outh": 1536, "But": 1537, "ional": 1538, ".,": 1539, "\u0120sol": 1540, "\u0120already": 1541, "\u012030": 1542, "\u0120charact": 1543, "He": 1544, "\u0120friend": 1545, "ES": 1546, "ians": 1547, "icle": 1548, "'d": 1549, "\u0120On": 1550, "\u0120least": 1551, "\u0120prom": 1552, "\u0120dr": 1553, "\u0120hist": 1554, "ither": 1555, "\u0120est": 1556, "iqu": 1557, "17": 1558, "son": 1559, "\u0120tell": 1560, "\u0120talk": 1561, "ohn": 1562, "oint": 1563, "lection": 1564, "AN": 1565, "\u0120until": 1566, "augh": 1567, "\u0120later": 1568, "\u0120ve": 1569, "\u0120view": 1570, "ending": 1571, "ived": 1572, "\u0120word": 1573, "ware": 1574, "\u0120cost": 1575, "\u0120enough": 1576, "\u0120give": 1577, "\u0120United": 1578, "\u0120techn": 1579, "arent": 1580, "OR": 1581, "\u0120par": 1582, "\u0120Dr": 1583, "\u01202016": 1584, "rist": 1585, "ering": 1586, "\u0120\u00c2": 1587, "\u0120large": 1588, "side": 1589, "acy": 1590, "ccess": 1591, "\u0120win": 1592, "\u0120important": 1593, "\u0120199": 1594, "\u0120doesn": 1595, "\u012017": 1596, "\u0120business": 1597, "\u0120clear": 1598, "\u0120rese": 1599, "\",": 1600, "ury": 1601, "\u0120equ": 1602, "aster": 1603, "alf": 1604, "\u0120American": 1605, "nect": 1606, "\u0120expect": 1607, "iversity": 1608, "\u0120occ": 1609, "\u0120Fl": 1610, "\u0120kind": 1611, "\u0120mean": 1612, "\u0120past": 1613, "\u0120dev": 1614, "\u0120bas": 1615, "let": 1616, "raft": 1617, "\u0120organ": 1618, "\u0120del": 1619, "\u0120perform": 1620, "\u0120story": 1621, "\u0120season": 1622, "\u0120Col": 1623, "\u0120claim": 1624, "\u0120came": 1625, "\u0120within": 1626, "\u0120line": 1627, "\u0120project": 1628, "\u0120At": 1629, "\u0120control": 1630, "ended": 1631, "\u0120Sy": 1632, "\u0120air": 1633, "ization": 1634, "\u0120*": 1635, "ley": 1636, "\u0120money": 1637, "idd": 1638, "You": 1639, "for": 1640, "\u0120family": 1641, "\u0120making": 1642, "\u0120bit": 1643, "\u0120police": 1644, "\u0120happen": 1645, "\u0120vers": 1646, "ony": 1647, "uff": 1648, "\u0120When": 1649, "\u0120sit": 1650, "ideo": 1651, "lf": 1652, "ison": 1653, "\u0120sure": 1654, "gin": 1655, "\u0120appear": 1656, "\u0120light": 1657, "\u0120es": 1658, "of": 1659, "\u0120water": 1660, "\u0120times": 1661, "not": 1662, "\u0120grow": 1663, "\u0120company": 1664, "\u0120Te": 1665, "ows": 1666, "\u0120mar": 1667, "ource": 1668, "iol": 1669, "arm": 1670, "br": 1671, "\u0120example": 1672, "\u0120conc": 1673, "\u0120fore": 1674, "\u0120To": 1675, "pro": 1676, "EN": 1677, "ries": 1678, "\u012025": 1679, "\u0120Can": 1680, "ney": 1681, "\u0120actually": 1682, "\u0120ever": 1683, "urity": 1684, "aken": 1685, "aps": 1686, "\u0120tax": 1687, "\u0120major": 1688, "ama": 1689, "\u0120often": 1690, "eral": 1691, "\u0120human": 1692, "\u0120job": 1693, "ister": 1694, "\u0120available": 1695, "ocr": 1696, "enn": 1697, "aid": 1698, "ivid": 1699, "\u0120record": 1700, "?\"": 1701, "\u0120sing": 1702, "\u0120Am": 1703, "idence": 1704, "\u0120news": 1705, "ster": 1706, "\u0120econom": 1707, "\u0120following": 1708, "\u0120Br": 1709, "ising": 1710, "\u0120hour": 1711, "most": 1712, "ument": 1713, "\u0120sex": 1714, "\u0120desc": 1715, "\u0120become": 1716, "\u0120Ed": 1717, "\u0120took": 1718, "\u0120having": 1719, "\u0120product": 1720, "ault": 1721, "As": 1722, "aring": 1723, "\u0120means": 1724, "\u0120hop": 1725, "une": 1726, "\u0120cho": 1727, "\u0120certain": 1728, "\u0120non": 1729, "\u0120deal": 1730, "24": 1731, "lement": 1732, "oci": 1733, "ene": 1734, "\u0120side": 1735, "\u0120Pr": 1736, "\u0120May": 1737, "\u0120reason": 1738, "ued": 1739, "ched": 1740, "ulation": 1741, "\u0120elect": 1742, "\u0120official": 1743, "\u0120possible": 1744, "\u0120hold": 1745, "ands": 1746, "ots": 1747, "\u0120city": 1748, "ories": 1749, "\u0120sever": 1750, "\u0120children": 1751, "\u0120once": 1752, "\u0120activ": 1753, "ler": 1754, "\u0120night": 1755, "itions": 1756, "\u0120John": 1757, "ape": 1758, "play": 1759, "\u0120done": 1760, "\u0120lim": 1761, "\u0120working": 1762, "\u0120Pres": 1763, "orld": 1764, "eb": 1765, "\u0120Co": 1766, "\u0120body": 1767, "ails": 1768, "utes": 1769, "\u0120Mr": 1770, "\u0120whether": 1771, "\u0120author": 1772, "rop": 1773, "\u0120proper": 1774, "\u0120seen": 1775, ");": 1776, "\u0120fac": 1777, "\u0120Su": 1778, "\u0120cond": 1779, "iting": 1780, "\u0120course": 1781, "\u0120}": 1782, "----------------": 1783, "aign": 1784, "\u0120event": 1785, "\u0120eng": 1786, "\u0120pot": 1787, "\u0120intern": 1788, "iam": 1789, "\u0120short": 1790, "empt": 1791, "\u00e3\u0124": 1792, "\u0120God": 1793, "ilar": 1794, "80": 1795, "\u0120orig": 1796, "IS": 1797, "ourn": 1798, "ability": 1799, "itive": 1800, "\u0120dam": 1801, "\u0120100": 1802, "\u0120press": 1803, "\u0120doing": 1804, "\u0120protect": 1805, "ring": 1806, "\u0120thought": 1807, "\u0120question": 1808, "rew": 1809, "\u0120War": 1810, "\u0120several": 1811, "\u0120State": 1812, "\u0120given": 1813, "\u0120fund": 1814, "\u0120Tw": 1815, "\u0120went": 1816, "ances": 1817, "work": 1818, "por": 1819, "my": 1820, "40": 1821, "\u0120arg": 1822, "artment": 1823, "ustom": 1824, "\u0120polic": 1825, "\u0120meet": 1826, "\u0120creat": 1827, "22": 1828, "\u0120States": 1829, "\u0120games": 1830, "raw": 1831, "uture": 1832, "\u0120understand": 1833, "urs": 1834, "\u0120Ob": 1835, "lish": 1836, "sy": 1837, "\u0120makes": 1838, "\u0120won": 1839, "agon": 1840, "\u0120htt": 1841, "\u0120love": 1842, "ential": 1843, "\u0120complete": 1844, "par": 1845, "\u0120Im": 1846, "AL": 1847, "\u0120account": 1848, "\u00c2\u0142": 1849, "ored": 1850, "vert": 1851, "\u0120ident": 1852, "\u01202015": 1853, "\u0120others": 1854, "\u0120Min": 1855, "iber": 1856, "verage": 1857, "There": 1858, "itional": 1859, "dd": 1860, "\u0120prob": 1861, "\u0120young": 1862, "\u0120along": 1863, "\u0120according": 1864, "\u0120yet": 1865, "\u0120members": 1866, "\u0120What": 1867, "oid": 1868, "\u0120Man": 1869, "And": 1870, "\u0120among": 1871, "ai": 1872, "\u0120employ": 1873, "\u0120Res": 1874, "\u0120>": 1875, "\u0120invol": 1876, "\u0120low": 1877, "af": 1878, "\u0120Car": 1879, "\u0120hig": 1880, "\u0120One": 1881, "\u0120Sec": 1882, "ination": 1883, "\u0120likely": 1884, "\u0120ant": 1885, "aged": 1886, "\u0120Russ": 1887, "\u0120ben": 1888, "\u0120rele": 1889, "For": 1890, "back": 1891, "\u0120Not": 1892, "\u0120president": 1893, "ball": 1894, "\u0120access": 1895, "ividual": 1896, "\u0120Dem": 1897, "\u0120Euro": 1898, "60": 1899, "\u0120known": 1900, "irl": 1901, "\u0120Gr": 1902, "\u0120early": 1903, "use": 1904, "iety": 1905, "\u00e2\u0122\u0135": 1906, "\u0120fight": 1907, "\u0120sent": 1908, "\u0120today": 1909, "\u0120market": 1910, "\".": 1911, "\u0120based": 1912, "\u0120strong": 1913, "urther": 1914, "\u0120deb": 1915, "mber": 1916, "\u0120problem": 1917, "\u0120death": 1918, "\u0120social": 1919, "imate": 1920, "AS": 1921, "ortun": 1922, "\u0120campaign": 1923, "ery": 1924, "Ch": 1925, "\u0120ey": 1926, "ially": 1927, "\u0120mus": 1928, "wh": 1929, "pos": 1930, "\u0120er": 1931, "\u0120saf": 1932, "\u0120months": 1933, "iron": 1934, "\u0120viol": 1935, "\u0120five": 1936, "\u0120stre": 1937, "\u0120players": 1938, "inc": 1939, "ald": 1940, "year": 1941, "aun": 1942, "\u0120success": 1943, "\u0120present": 1944, "erence": 1945, "\u01202014": 1946, "\u0120sugg": 1947, "\u0120particular": 1948, "\u0120try": 1949, "\u0120suggest": 1950, "\u0120Christ": 1951, "ones": 1952, "\u0120priv": 1953, "23": 1954, "\u0120crit": 1955, "\u0120land": 1956, "\u0120local": 1957, "ify": 1958, "29": 1959, "\u0120aut": 1960, "ED": 1961, "\u0120Gu": 1962, "\u0120mult": 1963, "\u0120political": 1964, "\u0120asked": 1965, "\u0120former": 1966, "itter": 1967, "ript": 1968, "\u0120close": 1969, "\u0120pract": 1970, "\u0120York": 1971, "\u0120getting": 1972, "\u0120across": 1973, "\u0120comb": 1974, "\u0120believe": 1975, "\u0120z": 1976, "\u0120toget": 1977, "\u0120together": 1978, "\u0120Cent": 1979, "irc": 1980, "\u0120individual": 1981, "\u0120Mc": 1982, "27": 1983, "isk": 1984, "\u0120Eng": 1985, "\u0120face": 1986, "\u012024": 1987, "\u0120value": 1988, "\u0120area": 1989, "ev": 1990, "\u0120writ": 1991, "\u0120President": 1992, "\u0120vot": 1993, "\u0120key": 1994, "\u0120mom": 1995, "put": 1996, "\u0120anything": 1997, "\u0120experience": 1998, "attle": 1999, "\u0120mind": 2000, "aff": 2001, "omm": 2002, "\u0120future": 2003, "ged": 2004, "\u0120cut": 2005, "\u0120tot": 2006, "itch": 2007, "\u0120video": 2008, "\u0120investig": 2009, "\u0120net": 2010, "\u0120My": 2011, "rict": 2012, "ien": 2013, ".)": 2014, "\u0120impro": 2015, "though": 2016, "wards": 2017, "\u0120connect": 2018, "\u0120Med": 2019, "selves": 2020, "ensive": 2021, "mb": 2022, "ober": 2023, "ators": 2024, "An": 2025, "\u012050": 2026, "\u0120redu": 2027, "resent": 2028, "\u0120above": 2029, "\u0120fre": 2030, "\u0120Europe": 2031, "sw": 2032, "\u0120amount": 2033, "\u0120App": 2034, "\u0120either": 2035, "\u0120milit": 2036, "\u0120anal": 2037, "\u0120fail": 2038, "\u0120En": 2039, "ales": 2040, "\u0120special": 2041, "\u0120black": 2042, "IT": 2043, "cher": 2044, "\u0120looking": 2045, "\u0120fire": 2046, "yn": 2047, "\u0120almost": 2048, "oon": 2049, "\u0120study": 2050, "\u0120miss": 2051, "ches": 2052, "rown": 2053, "\u0120tre": 2054, "\u0120community": 2055, "\u0120media": 2056, "\u0120food": 2057, "\u0120comes": 2058, "\u0120University": 2059, "\u0120single": 2060, "What": 2061, "uly": 2062, "\u0120half": 2063, "ague": 2064, "hod": 2065, "\u0120Republic": 2066, "\u0120started": 2067, "\u0120quick": 2068, "oto": 2069, "book": 2070, "\u0120issue": 2071, "itor": 2072, "\u0120else": 2073, "\u0120consider": 2074, "26": 2075, "rodu": 2076, "\u0120taken": 2077, "28": 2078, "99": 2079, "\u0120With": 2080, "\u0120true": 2081, "\u0120wa": 2082, "\u0120trad": 2083, "\u0120ago": 2084, "\u0120mess": 2085, "ief": 2086, "\u0120added": 2087, "oke": 2088, "\u0120bad": 2089, "\u0120fav": 2090, "33": 2091, "\u0120similar": 2092, "ask": 2093, "\u0120Don": 2094, "\u0120character": 2095, "orts": 2096, "\u0120House": 2097, "\u0120reported": 2098, "\u0120type": 2099, "val": 2100, "iod": 2101, "\u0120However": 2102, "\u0120targ": 2103, "\u0120entire": 2104, "pping": 2105, "\u0120history": 2106, "\u0120live": 2107, "ffic": 2108, "........": 2109, "ederal": 2110, "\u0120trying": 2111, "\u0120discuss": 2112, "\u0120Har": 2113, "aces": 2114, "lished": 2115, "\u0120self": 2116, "osp": 2117, "rest": 2118, "\u0120room": 2119, "elt": 2120, "\u0120fall": 2121, "olution": 2122, "\u0120et": 2123, "\u0120x": 2124, "\u0120isn": 2125, "\u0120idea": 2126, "bo": 2127, "\u0120sound": 2128, "\u0120Dep": 2129, "\u0120someone": 2130, "cially": 2131, "ully": 2132, "\u0120foc": 2133, "\u0120object": 2134, "ift": 2135, "aper": 2136, "\u0120player": 2137, "\u0120rather": 2138, "\u0120service": 2139, "ashing": 2140, "\u0120Do": 2141, "\u0120Part": 2142, "rug": 2143, "mon": 2144, "ply": 2145, "\u0120mor": 2146, "\u0120nothing": 2147, "\u0120provide": 2148, "IC": 2149, "ung": 2150, "\u0120party": 2151, "\u0120exist": 2152, "\u0120mag": 2153, "70": 2154, "\u0120rul": 2155, "\u0120house": 2156, "\u0120behind": 2157, "\u0120however": 2158, "\u0120World": 2159, "\u0120sum": 2160, "\u0120applic": 2161, "\u0120;": 2162, "\u0120function": 2163, "gr": 2164, "\u0120Pol": 2165, "\u0120front": 2166, "200": 2167, "\u0120series": 2168, "\u0120tem": 2169, "\u0120typ": 2170, "ills": 2171, "\u0120opt": 2172, "\u0120points": 2173, "\u0120below": 2174, "itted": 2175, "\u0120specific": 2176, "\u01202017": 2177, "umb": 2178, "\u0120ra": 2179, "\u0120previous": 2180, "\u0120pret": 2181, "reme": 2182, "\u0120custom": 2183, "\u0120court": 2184, "\u0120Me": 2185, "\u0120repl": 2186, "\u0120whole": 2187, "go": 2188, "cer": 2189, "\u0120treat": 2190, "\u0120Act": 2191, "\u0120probably": 2192, "\u0120learn": 2193, "ender": 2194, "\u0120Ass": 2195, "\u0120version": 2196, "now": 2197, "\u0120check": 2198, "\u0120Cal": 2199, "RE": 2200, "minist": 2201, "On": 2202, "ources": 2203, "\u0120benef": 2204, "\u0120doc": 2205, "\u0120deter": 2206, "\u0120enc": 2207, "\u0120super": 2208, "\u0120address": 2209, "\u0120vict": 2210, "\u01202013": 2211, "\u0120meas": 2212, "tr": 2213, "\u0120field": 2214, "When": 2215, "\u0120signific": 2216, "uge": 2217, "\u0120feat": 2218, "\u0120common": 2219, "load": 2220, "\u0120begin": 2221, "\u0120bring": 2222, "\u0120action": 2223, "erman": 2224, "\u0120describ": 2225, "\u0120indust": 2226, "\u0120wanted": 2227, "ried": 2228, "ming": 2229, "\u0120attempt": 2230, "45": 2231, "fer": 2232, "\u0120due": 2233, "ression": 2234, "##": 2235, "\u0120shall": 2236, "\u0120six": 2237, "oo": 2238, "\u0120step": 2239, "\u0120pub": 2240, "\u0120himself": 2241, "\u012023": 2242, "\u0120cop": 2243, "\u0120dest": 2244, "\u0120stop": 2245, "AC": 2246, "ibility": 2247, "\u0120lab": 2248, "icult": 2249, "\u0120hours": 2250, "\u0120create": 2251, "\u0120further": 2252, "\u0120America": 2253, "\u0120City": 2254, "\u0120dou": 2255, "head": 2256, "ST": 2257, "\u0120North": 2258, "cing": 2259, "\u0120national": 2260, "ule": 2261, "\u0120Inst": 2262, "\u0120taking": 2263, "\u0120Qu": 2264, "irt": 2265, "\u0120red": 2266, "\u0120research": 2267, "viron": 2268, "\u0120Ge": 2269, "\u0120break": 2270, "ana": 2271, "\u0120space": 2272, "aterial": 2273, "\u0120recent": 2274, "\u0120Ab": 2275, "\u0120general": 2276, "\u0120hit": 2277, "\u0120period": 2278, "\u0120everything": 2279, "ively": 2280, "\u0120phys": 2281, "\u0120saying": 2282, "anks": 2283, "\u0120cou": 2284, "\u0120cult": 2285, "aced": 2286, "eal": 2287, "uation": 2288, "\u0120coun": 2289, "lu": 2290, "\u0120include": 2291, "\u0120position": 2292, "\u0120After": 2293, "\u0120Canad": 2294, "\u0120Em": 2295, "\u0120imm": 2296, "\u0120Red": 2297, "\u0120pick": 2298, "\u0120compl": 2299, "\u0120matter": 2300, "reg": 2301, "ext": 2302, "angu": 2303, "isc": 2304, "ole": 2305, "aut": 2306, "\u0120compet": 2307, "eed": 2308, "fect": 2309, "\u012021": 2310, "\u0120Sen": 2311, "\u0120These": 2312, "asing": 2313, "\u0120cannot": 2314, "\u0120init": 2315, "\u0120relations": 2316, "ached": 2317, "\u0120bar": 2318, "\u012040": 2319, "\u0120TH": 2320, "\u01202012": 2321, "\u0120vol": 2322, "\u0120ground": 2323, "\u0120security": 2324, "\u0120upd": 2325, "ilt": 2326, "35": 2327, "\u0120concern": 2328, "\u0120Just": 2329, "\u0120white": 2330, "\u0120seems": 2331, "\u0120Her": 2332, "pecially": 2333, "ients": 2334, "\u0120announ": 2335, "\u0120fig": 2336, "ights": 2337, "\u0120stri": 2338, "like": 2339, "ids": 2340, "\u0120sus": 2341, "\u0120watch": 2342, "\u0120\u00e2": 2343, "\u0120wind": 2344, "\u0120Cont": 2345, "\u0120itself": 2346, "\u0120mass": 2347, "Al": 2348, "yle": 2349, "ique": 2350, "\u0120National": 2351, "\u0120abs": 2352, "\u0120pack": 2353, "\u0120outside": 2354, "\u0120anim": 2355, "\u0120pain": 2356, "eter": 2357, "\u0120manag": 2358, "duct": 2359, "ogn": 2360, "\u0120]": 2361, "\u0120Sept": 2362, "sec": 2363, "off": 2364, "\u0120Jan": 2365, "\u0120foot": 2366, "ades": 2367, "\u0120third": 2368, "\u0120mot": 2369, "\u0120evidence": 2370, "inton": 2371, "\u0120threat": 2372, "apt": 2373, "ples": 2374, "cle": 2375, "\u0120lo": 2376, "\u0120decl": 2377, "\u0120item": 2378, "medi": 2379, "\u0120represent": 2380, "omb": 2381, "amer": 2382, "\u0120significant": 2383, "ograph": 2384, "su": 2385, "\u0120cal": 2386, "ires": 2387, "0000": 2388, "ID": 2389, "AM": 2390, "\u0120simply": 2391, "\u0120longer": 2392, "\u0120file": 2393, "OT": 2394, "che": 2395, "So": 2396, "ateg": 2397, "org": 2398, "\u0120His": 2399, "\u0120ener": 2400, "\u0120dom": 2401, "\u0120upon": 2402, "ili": 2403, "\":\"": 2404, "\u0120themselves": 2405, "\u0120coming": 2406, "\u0120quite": 2407, "\u0120difficult": 2408, "\u0120Bar": 2409, "ilities": 2410, "rel": 2411, "ends": 2412, "cial": 2413, "64": 2414, "\u0120woman": 2415, "rap": 2416, "yr": 2417, "\u0120necess": 2418, "ips": 2419, "\u0120text": 2420, "\u0120require": 2421, "\u0120military": 2422, "\u0120review": 2423, "\u0120respons": 2424, "75": 2425, "\u0120subject": 2426, "\u0120instead": 2427, "\u0120issues": 2428, "\u0120gen": 2429, "\",\"": 2430, "\u0120minutes": 2431, "\u0120weap": 2432, "ray": 2433, "amed": 2434, "time": 2435, "bl": 2436, "How": 2437, "\u0120code": 2438, "\u0120Sm": 2439, "\u0120higher": 2440, "\u0120Ste": 2441, "ris": 2442, "\u0120page": 2443, "\u0120students": 2444, "\u0120Intern": 2445, "\u0120method": 2446, "\u0120Aug": 2447, "\u0120Per": 2448, "\u0120Ag": 2449, "\u0120policy": 2450, "\u0120Sw": 2451, "\u0120exec": 2452, "\u0120accept": 2453, "ume": 2454, "ribut": 2455, "\u0120words": 2456, "\u0120final": 2457, "\u0120changes": 2458, "\u0120Democr": 2459, "\u0120friends": 2460, "\u0120respect": 2461, "\u0120ep": 2462, "\u0120compan": 2463, "ivil": 2464, "\u0120damage": 2465, "****": 2466, "ogle": 2467, "vironment": 2468, "\u0120neg": 2469, "ental": 2470, "\u0120ap": 2471, "\u0120total": 2472, "ival": 2473, "!\"": 2474, "lim": 2475, "\u0120needs": 2476, "\u0120agre": 2477, "\u0120development": 2478, "\u0120age": 2479, "iple": 2480, "21": 2481, "\u0120results": 2482, "\u0120Af": 2483, "Sh": 2484, "\u0120gun": 2485, "\u0120Obama": 2486, "roll": 2487, "\u0120@": 2488, "\u0120rights": 2489, "\u0120Brit": 2490, "\u0120running": 2491, "\u0120wasn": 2492, "\u0120port": 2493, "\u0120rate": 2494, "\u0120pretty": 2495, "\u0120target": 2496, "\u0120saw": 2497, "\u0120circ": 2498, "\u0120works": 2499, "icro": 2500, "alt": 2501, "over": 2502, "www": 2503, "That": 2504, "lier": 2505, "\u0120everyone": 2506, "ude": 2507, "\u0120pie": 2508, "iddle": 2509, "rael": 2510, "\u0120rad": 2511, "\u0120block": 2512, "\u0120walk": 2513, "To": 2514, "\u00e3\u0123": 2515, "nes": 2516, "\u0120Aust": 2517, "aul": 2518, "rote": 2519, "\u0120South": 2520, "ession": 2521, "oph": 2522, "\u0120shows": 2523, "\u0120site": 2524, "\u0120jo": 2525, "\u0120risk": 2526, "clus": 2527, "lt": 2528, "\u0120inj": 2529, "iding": 2530, "\u0120Spe": 2531, "\u0120chall": 2532, "irm": 2533, "\u012022": 2534, "itting": 2535, "str": 2536, "\u0120hy": 2537, "LE": 2538, "key": 2539, "\u0120began": 2540, "atur": 2541, "ashington": 2542, "lam": 2543, "\u0120Dav": 2544, "bit": 2545, "\u0120size": 2546, "\u0120Par": 2547, "38": 2548, "ournal": 2549, "face": 2550, "\u0120decision": 2551, "\u0120larg": 2552, "\u0120jud": 2553, "rect": 2554, "\u0120continue": 2555, "\u0120Oct": 2556, "overed": 2557, "\u0120Int": 2558, "========": 2559, "\u0120parent": 2560, "\u0120Will": 2561, "\u0120easy": 2562, "\u0120drug": 2563, "anger": 2564, "\u0120sense": 2565, "\u0120di": 2566, "iday": 2567, "\u0120energy": 2568, "istic": 2569, "\u0120associ": 2570, "arter": 2571, "obal": 2572, "eks": 2573, "\u0120El": 2574, "urch": 2575, "\u0120girl": 2576, "oe": 2577, "itle": 2578, "\u012028": 2579, "\u0120Che": 2580, "\u0120request": 2581, "\u0120soon": 2582, "\u0120host": 2583, "ky": 2584, "\u0120states": 2585, "omes": 2586, "\u0120material": 2587, "lex": 2588, "\u0120moment": 2589, "\u0120answ": 2590, "onse": 2591, "\u0120especially": 2592, "\u0120norm": 2593, "\u0120services": 2594, "pite": 2595, "ran": 2596, "\u0120role": 2597, "44": 2598, "):": 2599, "\u0120cred": 2600, "Cl": 2601, "________": 2602, "\u0120mat": 2603, "\u0120log": 2604, "\u0120Clinton": 2605, "OU": 2606, "\u0120office": 2607, "\u012026": 2608, "\u0120charg": 2609, "\u0120track": 2610, "ma": 2611, "\u0120heart": 2612, "\u0120ball": 2613, "\u0120personal": 2614, "\u0120building": 2615, "na": 2616, "set": 2617, "body": 2618, "\u0120Black": 2619, "\u0120increase": 2620, "itten": 2621, "\u0120needed": 2622, "36": 2623, "32": 2624, "=\"": 2625, "\u0120lost": 2626, "\u0120became": 2627, "\u0120groups": 2628, "\u0120Mus": 2629, "\u0120wrote": 2630, "\u0120Pe": 2631, "\u0120prop": 2632, "joy": 2633, "\u00c3\u00a9": 2634, "\u0120White": 2635, "\u0120dead": 2636, ".'": 2637, "\u0120http": 2638, "\u0120webs": 2639, "OS": 2640, "\u0120inside": 2641, "\u0120wrong": 2642, "\u0120statement": 2643, "\u0120...": 2644, "yl": 2645, "\u0120film": 2646, "\u0120music": 2647, "\u0120share": 2648, "ification": 2649, "\u0120release": 2650, "\u0120forward": 2651, "\u0120stay": 2652, "\u0120comput": 2653, "itte": 2654, "ser": 2655, "\u0120original": 2656, "\u0120card": 2657, "\u0120cand": 2658, "\u0120div": 2659, "atural": 2660, "\u0120favor": 2661, "OM": 2662, "\u0120cases": 2663, "uses": 2664, "\u0120section": 2665, "\u0120leave": 2666, "ging": 2667, "oved": 2668, "\u0120Washington": 2669, "39": 2670, "\u0120Gl": 2671, "\u0120required": 2672, "action": 2673, "apan": 2674, "oor": 2675, "iter": 2676, "\u0120King": 2677, "\u0120countries": 2678, "\u0120German": 2679, "lling": 2680, "\u012027": 2681, "34": 2682, "\u0120questions": 2683, "\u0120prim": 2684, "\u0120cell": 2685, "\u0120shoot": 2686, "\u0120anyone": 2687, "\u0120West": 2688, "\u0120affect": 2689, "epend": 2690, "\u0120online": 2691, "\u0120Israel": 2692, "\u0120September": 2693, "\u0120ability": 2694, "\u0120content": 2695, "ises": 2696, "\u0120reve": 2697, "\u0120laun": 2698, "\u0120indic": 2699, "\u0120force": 2700, "cast": 2701, "\u0120sold": 2702, "aving": 2703, "fl": 2704, "\u0120soft": 2705, "\u0120companies": 2706, "ceed": 2707, "\u0120article": 2708, "\u0120aud": 2709, "\u0120rev": 2710, "\u0120educ": 2711, "\u0120playing": 2712, "05": 2713, "\u0120held": 2714, "ctor": 2715, "\u0120released": 2716, "\u0120federal": 2717, "37": 2718, "\u0120administ": 2719, "\u0120interview": 2720, "\u0120install": 2721, "\u0120received": 2722, "\u0120source": 2723, "uk": 2724, "Ph": 2725, "\u0120serious": 2726, "\u0120created": 2727, "\u0120cause": 2728, "\u0120immedi": 2729, "\u0120defin": 2730, "uel": 2731, "\u0120Department": 2732, "ctions": 2733, "\u0120Cour": 2734, "\u0120Now": 2735, "ze": 2736, "ites": 2737, "itution": 2738, "\u0120late": 2739, "\u0120speak": 2740, "ners": 2741, "\u0120legal": 2742, "ari": 2743, "\u0120Cor": 2744, "\u0120weeks": 2745, "\u0120model": 2746, "\u0120pred": 2747, "\u0120exact": 2748, "BC": 2749, "\u0120By": 2750, "ING": 2751, "osing": 2752, "\u0120takes": 2753, "\u0120regard": 2754, "\u0120opportun": 2755, "\u0120price": 2756, "\u0120198": 2757, "\u0120Apr": 2758, "fully": 2759, "\u0120ord": 2760, "\u0120problems": 2761, "ruction": 2762, "ham": 2763, "\u0120Count": 2764, "lege": 2765, "\u0120leaders": 2766, "ET": 2767, "lev": 2768, "\u0120deep": 2769, "ological": 2770, "ese": 2771, "haps": 2772, "\u0120Some": 2773, "\u0120pers": 2774, "\u0120contract": 2775, "\u0120relationship": 2776, "sp": 2777, "oud": 2778, "\u0120base": 2779, "48": 2780, "mit": 2781, "Ad": 2782, "ancial": 2783, "\u0120consum": 2784, "\u0120potential": 2785, "\u0120langu": 2786, "rem": 2787, "eth": 2788, "\u0120relig": 2789, "ressed": 2790, "66": 2791, "\u0120link": 2792, "\u0120lower": 2793, "ayer": 2794, "\u0120June": 2795, "\u0120fem": 2796, "unt": 2797, "erc": 2798, "urd": 2799, "\u0120contact": 2800, "\u0120ill": 2801, "\u0120mother": 2802, "\u0120estab": 2803, "htt": 2804, "\u0120March": 2805, "\u0120Bro": 2806, "\u0120China": 2807, "\u012029": 2808, "\u0120squ": 2809, "\u0120provided": 2810, "\u0120average": 2811, "asons": 2812, "\u01202011": 2813, "\u0120exam": 2814, "lin": 2815, "55": 2816, "ned": 2817, "\u0120perfect": 2818, "\u0120tou": 2819, "alse": 2820, "ux": 2821, "\u0120buy": 2822, "\u0120shot": 2823, "\u0120collect": 2824, "\u0120phot": 2825, "\u0120played": 2826, "\u0120surpr": 2827, "\u0120officials": 2828, "\u0120simple": 2829, "avy": 2830, "\u0120industry": 2831, "\u0120hands": 2832, "ground": 2833, "\u0120pull": 2834, "\u0120round": 2835, "\u0120user": 2836, "\u0120range": 2837, "uary": 2838, "\u0120private": 2839, "ops": 2840, "ees": 2841, "\u0120ways": 2842, "\u0120Mich": 2843, "\u0120veh": 2844, "\u0120except": 2845, "\u0120terms": 2846, "imum": 2847, "pper": 2848, "ION": 2849, "ores": 2850, "\u0120Dragon": 2851, "oul": 2852, "\u0120den": 2853, "\u0120performance": 2854, "\u0120bill": 2855, "cil": 2856, "47": 2857, "\u0120environment": 2858, "\u0120exc": 2859, "add": 2860, "\u0120worth": 2861, "\u0120pict": 2862, "\u0120chance": 2863, "\u01202018": 2864, "bor": 2865, "\u0120speed": 2866, "iction": 2867, "\u0120alleg": 2868, "\u0120Japan": 2869, "atory": 2870, "reet": 2871, "\u0120match": 2872, "\u0120II": 2873, "\u0120stru": 2874, "order": 2875, "\u0120ste": 2876, "\u0120living": 2877, "\u0120struct": 2878, "ino": 2879, "\u0120separ": 2880, "hern": 2881, "\u0120response": 2882, "\u0120enjoy": 2883, "\u0120via": 2884, "AD": 2885, "uments": 2886, "acebook": 2887, "\u0120member": 2888, "ibr": 2889, "izing": 2890, "\u0120tool": 2891, "\u0120Mon": 2892, "\u0120While": 2893, "hood": 2894, "\u0120Ang": 2895, "\u0120Def": 2896, "\u0120offer": 2897, "Tr": 2898, "aur": 2899, "\u0120turned": 2900, "\u0120July": 2901, "down": 2902, "anced": 2903, "\u0120recently": 2904, "\u0120Ear": 2905, "\u0120ce": 2906, "\u0120Star": 2907, "\u0120Cong": 2908, "rought": 2909, "\u0120blood": 2910, "\u0120hope": 2911, "\u0120comment": 2912, "aint": 2913, "\u0120arri": 2914, "iles": 2915, "\u0120particip": 2916, "ought": 2917, "ription": 2918, "08": 2919, "49": 2920, "\u0120gave": 2921, "\u0120select": 2922, "\u0120killed": 2923, "sych": 2924, "\u0120goes": 2925, "ij": 2926, "\u0120coll": 2927, "\u0120impact": 2928, "atives": 2929, "\u0120Ser": 2930, "09": 2931, "\u0120August": 2932, "\u0120boy": 2933, "de": 2934, "\u0120Des": 2935, "\u0120felt": 2936, "US": 2937, "\u0120expected": 2938, "\u0120image": 2939, "\u0120Mark": 2940, "ccording": 2941, "oice": 2942, "EC": 2943, "\u0120Mag": 2944, "ened": 2945, "hold": 2946, "\u0120Post": 2947, "\u0120prevent": 2948, "No": 2949, "\u0120involved": 2950, "\u0120eyes": 2951, "\u0120quickly": 2952, "At": 2953, "unk": 2954, "\u0120behav": 2955, "\u0120ur": 2956, "\u0120led": 2957, "come": 2958, "ey": 2959, "\u0120candid": 2960, "\u0120earlier": 2961, "\u0120focus": 2962, "ety": 2963, "Pro": 2964, "ledge": 2965, "ixed": 2966, "illed": 2967, "\u0120popular": 2968, "AP": 2969, "\u0120sett": 2970, "light": 2971, "\u0120various": 2972, "inks": 2973, "\u0120levels": 2974, "\u0120road": 2975, "ellig": 2976, "ables": 2977, "hel": 2978, "ittee": 2979, "\u0120Gener": 2980, "ype": 2981, "\u0120heard": 2982, "icles": 2983, "\u0120mis": 2984, "\u0120users": 2985, "\u0120San": 2986, "\u0120improve": 2987, "\u0120father": 2988, "\u0120search": 2989, "They": 2990, "vil": 2991, "\u0120profess": 2992, "\u0120knew": 2993, "\u0120loss": 2994, "\u0120events": 2995, "65": 2996, "\u0120billion": 2997, "07": 2998, "02": 2999, "\u0120News": 3000, "\u0120AM": 3001, "\u0120cover": 3002, "where": 3003, "ension": 3004, "\u0120bott": 3005, "\u0120areas": 3006, "ences": 3007, "ope": 3008, "\u0120Twitter": 3009, "ael": 3010, "\u0120gets": 3011, "\u0120Google": 3012, "\u0120sn": 3013, "iant": 3014, "\u0120vote": 3015, "\u0120nearly": 3016, "\u0120included": 3017, "\u0120recogn": 3018, "zz": 3019, "mm": 3020, "aled": 3021, "\u0120happened": 3022, "04": 3023, "\u0120hot": 3024, "\u0120whose": 3025, "\u0120civil": 3026, "\u0120suff": 3027, "oes": 3028, "itiz": 3029, "\u0120Syri": 3030, "\u0120respond": 3031, "\u0120hon": 3032, "\u0120features": 3033, "\u0120economic": 3034, "\u0120April": 3035, "rim": 3036, "\u0120technology": 3037, "\u0120option": 3038, "aging": 3039, "\u0120purch": 3040, "Re": 3041, "\u0120lat": 3042, "chie": 3043, "isl": 3044, "\u0120recomm": 3045, "uf": 3046, "\u0120training": 3047, "\u0120effects": 3048, "\u0120fast": 3049, "\u01202010": 3050, "\u0120occur": 3051, "\u0120website": 3052, "\u0120email": 3053, "\u0120sens": 3054, "ech": 3055, "\u0120oil": 3056, "\u0120influ": 3057, "\u0120currently": 3058, "\u0120Sch": 3059, "\u0120Add": 3060, "\u0120goal": 3061, "\u0120scient": 3062, "\u0120conv": 3063, "100": 3064, "emy": 3065, "\u0120decided": 3066, "\u0120travel": 3067, "\u0120mention": 3068, "LL": 3069, "03": 3070, "\u0120election": 3071, "\u0120phone": 3072, "\u0120looks": 3073, "\u0120situation": 3074, "\u0120cy": 3075, "\u0120hor": 3076, "bed": 3077, "\u0120Court": 3078, "aily": 3079, "aves": 3080, "\u0120quality": 3081, "\u0120Comp": 3082, "wise": 3083, "\u0120table": 3084, "\u0120staff": 3085, "\u0120Wind": 3086, "ett": 3087, "\u0120tried": 3088, "idered": 3089, "\u0120addition": 3090, "\u0120box": 3091, "\u0120lack": 3092, "arily": 3093, "\u0120wide": 3094, "\u0120mid": 3095, "\u0120board": 3096, "ysis": 3097, "\u0120anti": 3098, "ha": 3099, "\u0120dig": 3100, "ening": 3101, "\u0120dro": 3102, "Con": 3103, "68": 3104, "\u0120slow": 3105, "based": 3106, "sequ": 3107, "\u0120path": 3108, "Ex": 3109, "aker": 3110, "\u0120worked": 3111, "\u0120pen": 3112, "\u0120engine": 3113, "\u0120looked": 3114, "\u0120Super": 3115, "\u0120Serv": 3116, "\u0120victim": 3117, "Un": 3118, "\u0120property": 3119, "\u0120introdu": 3120, "\u0120execut": 3121, "\u0120PM": 3122, "Le": 3123, "\u0120color": 3124, "\u0120More": 3125, "\u012060": 3126, "\u0120network": 3127, "\u0120date": 3128, "cul": 3129, "idge": 3130, "\u0120extra": 3131, "31": 3132, "\u0120sle": 3133, "67": 3134, "\u0120wond": 3135, "\u0120reports": 3136, "just": 3137, "\u0120Austral": 3138, "\u0120capital": 3139, "\u0120ens": 3140, "\u0120command": 3141, "\u0120allowed": 3142, "\u0120prep": 3143, "\u0120capt": 3144, "hib": 3145, "\u0120numbers": 3146, "chan": 3147, "\u0120fair": 3148, "mp": 3149, "oms": 3150, "\u0120reach": 3151, "With": 3152, "tain": 3153, "\u0120broad": 3154, "\u0120couple": 3155, "ecause": 3156, "lying": 3157, "\u0120Feb": 3158, "\u0120screen": 3159, "\u0120lives": 3160, "\u0120prior": 3161, "\u0120Congress": 3162, "Ar": 3163, "\u0120approach": 3164, "\u0120emer": 3165, "aries": 3166, "\u0120Dis": 3167, "serv": 3168, "\u0120Ne": 3169, "\u0120built": 3170, "cies": 3171, "\u0120repe": 3172, "\u0120rules": 3173, "force": 3174, "\u0120Pal": 3175, "\u0120financial": 3176, "\u0120considered": 3177, "\u0120Char": 3178, "nces": 3179, "\u0120IS": 3180, "\u0120brought": 3181, "\u0120bi": 3182, "iers": 3183, "\u0120Sim": 3184, "OP": 3185, "\u0120products": 3186, "\u0120visit": 3187, "\u0120document": 3188, "\u0120conduct": 3189, "\u0120completely": 3190, "ining": 3191, "\u0120Calif": 3192, "ibly": 3193, "\u0120written": 3194, "\u0120TV": 3195, "ements": 3196, "\u0120draw": 3197, "One": 3198, "\u0120published": 3199, "\u0120secret": 3200, "rain": 3201, "het": 3202, "\u0120Facebook": 3203, "onday": 3204, "\u0120Up": 3205, "\u0120sexual": 3206, "\u0120thous": 3207, "\u0120Pat": 3208, "\u0120ess": 3209, "\u0120standard": 3210, "\u0120arm": 3211, "ges": 3212, "ection": 3213, "\u0120fell": 3214, "\u0120foreign": 3215, "ani": 3216, "\u0120Friday": 3217, "\u0120regular": 3218, "inary": 3219, "\u0120increased": 3220, "\u0120usually": 3221, "\u0120demon": 3222, "\u0120dark": 3223, "\u0120additional": 3224, "rol": 3225, "\u0120Of": 3226, "\u0120production": 3227, "!!": 3228, "undred": 3229, "\u0120international": 3230, "idents": 3231, "\u0120Free": 3232, "roup": 3233, "\u0120race": 3234, "\u0120mach": 3235, "\u0120huge": 3236, "All": 3237, "lear": 3238, "ovember": 3239, "\u0120town": 3240, "\u0120attention": 3241, "\u0120Off": 3242, "yond": 3243, "\u0120Then": 3244, "field": 3245, "\u0120terror": 3246, "raz": 3247, "\u0120Bo": 3248, "\u0120meeting": 3249, "\u0120Park": 3250, "\u0120arrest": 3251, "\u0120fear": 3252, "\u0120aw": 3253, "\u0120Val": 3254, "oring": 3255, "',": 3256, "\u0120extreme": 3257, "arr": 3258, "\u0120workers": 3259, "After": 3260, "\u012031": 3261, "net": 3262, "ament": 3263, "\u0120directly": 3264, "\u0120population": 3265, "ube": 3266, "\u0120October": 3267, "\u0120IN": 3268, "\u0120January": 3269, "59": 3270, "\u0120David": 3271, "\u0120cross": 3272, "cember": 3273, "\u0120First": 3274, "\u0120message": 3275, "irit": 3276, "\u0120nation": 3277, "\u0120poll": 3278, "isions": 3279, "\u0120answer": 3280, "ny": 3281, "isode": 3282, "\u0120carry": 3283, "\u0120Russia": 3284, "\u0120hear": 3285, "ength": 3286, "roy": 3287, "\u0120natural": 3288, "inally": 3289, "\u0120dog": 3290, "mitted": 3291, "\u0120trade": 3292, "\u0120subst": 3293, "\u0120multiple": 3294, "\u0120Afric": 3295, "\u0120fans": 3296, "\u0120sort": 3297, "\u0120global": 3298, "ication": 3299, "\u0120Wed": 3300, "ara": 3301, "\u0120achie": 3302, "\u0120language": 3303, "vey": 3304, "\u0120tal": 3305, "\u0120necessary": 3306, "\u0120details": 3307, "\u0120sen": 3308, "\u0120Sund": 3309, "\u0120Reg": 3310, "\u0120Rec": 3311, "06": 3312, "\u0120sil": 3313, "ressive": 3314, "\u0120medical": 3315, "unch": 3316, "ornia": 3317, "\u0120und": 3318, "fort": 3319, "ocks": 3320, "\u0120Monday": 3321, "uesday": 3322, "craft": 3323, "77": 3324, "urt": 3325, "\u0120ver": 3326, "\u0120Hill": 3327, "\u0120receive": 3328, "\u0120morning": 3329, "estern": 3330, "\u0120bank": 3331, "\u0120sat": 3332, "irth": 3333, "\u0120High": 3334, "\u0120device": 3335, "\u0120THE": 3336, "\u0120Center": 3337, "\u0120safe": 3338, "\u0120ple": 3339, "\u0120Canada": 3340, "\u0120systems": 3341, "\u0120assist": 3342, "\u0120surv": 3343, "\u0120battle": 3344, "\u0120Soc": 3345, "vertis": 3346, "She": 3347, "\u0120paper": 3348, "\u0120growth": 3349, "\u0120cast": 3350, "Sc": 3351, "\u0120plans": 3352, "lled": 3353, "\u0120parts": 3354, "\u0120wall": 3355, "\u0120movement": 3356, "\u0120practice": 3357, "imately": 3358, "\u0120display": 3359, "\u0120sometimes": 3360, "omp": 3361, "\u0120Paul": 3362, "\u0120Yes": 3363, "king": 3364, "58": 3365, "oly": 3366, "\u0120son": 3367, "\u0120avoid": 3368, "okes": 3369, "\u0120Jew": 3370, "\u0120towards": 3371, "asc": 3372, "\u0120//": 3373, "\u0120Kore": 3374, "\u0120talking": 3375, "\u0120correct": 3376, "\u0120spent": 3377, "icks": 3378, "iable": 3379, "eared": 3380, "\u0120term": 3381, "\u0120wants": 3382, "oming": 3383, "\u0120ut": 3384, "\u0120doub": 3385, "\u0120forces": 3386, "\u0120please": 3387, "69": 3388, "\u0120November": 3389, "atform": 3390, "ondon": 3391, "\u0120ones": 3392, "\u0120immediately": 3393, "\u0120Russian": 3394, "\u0120Met": 3395, "\u0120deg": 3396, "\u0120parents": 3397, "CH": 3398, "\u0120Americans": 3399, "aly": 3400, "\u0120Mod": 3401, "\u0120shown": 3402, "\u0120conditions": 3403, "\u0120stuff": 3404, "\u0120reb": 3405, "\u0120Your": 3406, "\u0120includes": 3407, "nown": 3408, "\u0120Sam": 3409, "\u0120experien": 3410, "mission": 3411, "\u0120Even": 3412, "aught": 3413, "\u0120announced": 3414, "\u0120Republican": 3415, "\u0120determin": 3416, "\u0120described": 3417, "\u0120County": 3418, "()": 3419, "\u0120door": 3420, "\u0120changed": 3421, "\u0120neigh": 3422, "\u0120Here": 3423, "\u0120clean": 3424, "\u0120pan": 3425, "\u0120December": 3426, "\u0120European": 3427, "iring": 3428, "apter": 3429, "\u0120club": 3430, "\u0120Tuesday": 3431, "\u0120paid": 3432, "\u0120Net": 3433, "\u0120attacks": 3434, "\u0120characters": 3435, "\u0120alone": 3436, "\u0120director": 3437, "dom": 3438, "\u012035": 3439, "\u0120load": 3440, "\u0120rout": 3441, "\u0120California": 3442, "\u0120finally": 3443, "\u0120rac": 3444, "\u0120contr": 3445, "\u0120exactly": 3446, "resh": 3447, "pri": 3448, "\u0120Islam": 3449, "\u0120nature": 3450, "\u0120career": 3451, "\u0120latest": 3452, "\u0120convers": 3453, "\u0120Sl": 3454, "pose": 3455, "cient": 3456, "\u0120Inc": 3457, "ivity": 3458, "88": 3459, "\u0120Att": 3460, "\u0120Mor": 3461, "nesday": 3462, "\u0120weight": 3463, "ken": 3464, "\u0120note": 3465, "\u0120teams": 3466, "\u0120\\": 3467, "airs": 3468, "\u0120Green": 3469, "\u0120hundred": 3470, "onent": 3471, "\u0120streng": 3472, "\u0120consist": 3473, "icated": 3474, "\u0120regul": 3475, "\u0120lic": 3476, "astic": 3477, "\u0120ten": 3478, "ursday": 3479, "elligence": 3480, "ously": 3481, "\u0120UK": 3482, "BI": 3483, "\u0120costs": 3484, "\u0120independ": 3485, "\u0120AP": 3486, "\u0120normal": 3487, "\u0120hom": 3488, "\u0120obvious": 3489, "\u0120swe": 3490, "\u0120star": 3491, "\u0120ready": 3492, "acher": 3493, "\u0120implement": 3494, "gest": 3495, "\u0120song": 3496, "\u0120Get": 3497, "\u0120Lab": 3498, "\u0120interesting": 3499, "using": 3500, "\u0120giving": 3501, "\u0120Sunday": 3502, "\u0120etc": 3503, "\u0120middle": 3504, "\u0120remember": 3505, "right": 3506, "osition": 3507, "utions": 3508, "\u0120max": 3509, "46": 3510, "\u0120yourself": 3511, "\u0120demand": 3512, "\u0120treatment": 3513, "\u0120danger": 3514, "\u0120Cons": 3515, "\u0120guy": 3516, "\u0120British": 3517, "\u0120physical": 3518, "\u0120related": 3519, "\u0120remain": 3520, "\u0120couldn": 3521, "\u0120refer": 3522, "\u0120citiz": 3523, "box": 3524, "ENT": 3525, "board": 3526, "\u0120inn": 3527, "IG": 3528, "ero": 3529, "\u0120Street": 3530, "ospital": 3531, "rench": 3532, "chers": 3533, "\u0120stra": 3534, "OL": 3535, "ager": 3536, "\u0120AN": 3537, "\u0120easily": 3538, "IA": 3539, "enge": 3540, "iny": 3541, "\u0120clos": 3542, "ocked": 3543, "\u0120uses": 3544, "\u0120Coun": 3545, "Im": 3546, "uild": 3547, "??": 3548, "more": 3549, "\u0120ang": 3550, "\u0120write": 3551, "olute": 3552, "57": 3553, "\u0120leader": 3554, "\u0120reading": 3555, "": 3784, "\u0120figure": 3785, "\u0120disapp": 3786, "enty": 3787, "\u0120software": 3788, "\u0120ult": 3789, "\u0120officers": 3790, "New": 3791, "Is": 3792, "\u0120remains": 3793, "\u0120India": 3794, "\u0120psych": 3795, "rief": 3796, "\u0120cat": 3797, "esc": 3798, "\u0120observ": 3799, "\u0120stage": 3800, "\u0120Dark": 3801, "\u0120enter": 3802, "change": 3803, "\u0120passed": 3804, "\u0120despite": 3805, "\u0120Out": 3806, "\u0120movie": 3807, "rs": 3808, "\u0120voice": 3809, "mine": 3810, "\u0120Play": 3811, "\u0120toward": 3812, "\u0120Ter": 3813, "\u0120region": 3814, "\u0120values": 3815, "orters": 3816, "\u0120mount": 3817, "\u0120officer": 3818, "\u0120Other": 3819, "ban": 3820, "\u0120hous": 3821, "wood": 3822, "room": 3823, "IV": 3824, "\u0120Sun": 3825, "see": 3826, "\u0120Over": 3827, "rog": 3828, "90": 3829, "\u0120lay": 3830, "\u0120Tur": 3831, "awn": 3832, "\u0120pressure": 3833, "\u0120Sub": 3834, "\u0120books": 3835, "edom": 3836, "\u0120Sand": 3837, "AA": 3838, "ago": 3839, "\u0120reasons": 3840, "ford": 3841, "\u0120activity": 3842, "UT": 3843, "Now": 3844, "\u0120Senate": 3845, "cell": 3846, "night": 3847, "\u0120calls": 3848, "inter": 3849, "\u0120letter": 3850, "\u0120Rob": 3851, "\u0120Je": 3852, "\u0120choose": 3853, "\u0120Law": 3854, "Get": 3855, "Be": 3856, "\u0120rob": 3857, "\u0120types": 3858, "\u0120platform": 3859, "\u0120quarter": 3860, "RA": 3861, "\u0120Time": 3862, "\u0120maybe": 3863, "\u0120Cr": 3864, "95": 3865, "pre": 3866, "\u0120moving": 3867, "\u0120lif": 3868, "\u0120gold": 3869, "\u0120som": 3870, "\u0120patients": 3871, "\u0120truth": 3872, "\u0120Ke": 3873, "urance": 3874, "antly": 3875, "mar": 3876, "\u0120charge": 3877, "\u0120Great": 3878, "\u0120cele": 3879, "--------------------------------": 3880, "\u0120rock": 3881, "roid": 3882, "ancy": 3883, "\u0120credit": 3884, "aud": 3885, "By": 3886, "\u0120Every": 3887, "\u0120moved": 3888, "inger": 3889, "ribution": 3890, "\u0120names": 3891, "\u0120straight": 3892, "\u0120Health": 3893, "\u0120Well": 3894, "\u0120feature": 3895, "\u0120rule": 3896, "\u0120sche": 3897, "inated": 3898, "\u0120Michael": 3899, "berg": 3900, "41": 3901, "iled": 3902, "band": 3903, "\u0120click": 3904, "\u0120Angel": 3905, "onents": 3906, "\u00c2\u0143": 3907, "\u0120Iraq": 3908, "\u0120Saturday": 3909, "\u0120aware": 3910, "part": 3911, "\u0120pattern": 3912, "OW": 3913, "\u0120Let": 3914, "\u0120grad": 3915, "igned": 3916, "\u0120associated": 3917, "\u0120style": 3918, "no": 3919, "iation": 3920, "aith": 3921, "ilies": 3922, "\u0120stories": 3923, "uration": 3924, "\u0120individuals": 3925, "\u0120\u00e2\u0122\u00a6": 3926, "miss": 3927, "\u0120Associ": 3928, "ishing": 3929, "aby": 3930, "\u0120summer": 3931, "\u0120Ben": 3932, "\u012032": 3933, "\u0120arch": 3934, "uty": 3935, "\u0120Texas": 3936, "hol": 3937, "\u0120fully": 3938, "\u0120mill": 3939, "\u0120followed": 3940, "\u0120Bill": 3941, "\u0120Indian": 3942, "\u0120Secret": 3943, "\u0120Bel": 3944, "\u0120February": 3945, "\u0120jobs": 3946, "\u0120seemed": 3947, "\u0120Govern": 3948, "ipped": 3949, "\u0120reality": 3950, "\u0120lines": 3951, "\u0120park": 3952, "\u0120measure": 3953, "\u0120Our": 3954, "IM": 3955, "\u0120brother": 3956, "\u0120growing": 3957, "\u0120ban": 3958, "\u0120estim": 3959, "\u0120cry": 3960, "\u0120School": 3961, "\u0120mechan": 3962, "\u0120OF": 3963, "\u0120Windows": 3964, "\u0120rates": 3965, "\u0120Oh": 3966, "\u0120positive": 3967, "\u0120culture": 3968, "istics": 3969, "ica": 3970, "\u0120har": 3971, "ya": 3972, "itely": 3973, "ipp": 3974, "\u0120map": 3975, "encies": 3976, "\u0120William": 3977, "II": 3978, "akers": 3979, "56": 3980, "\u0120Mart": 3981, "\u0120Rem": 3982, "\u0120altern": 3983, "itude": 3984, "\u0120coach": 3985, "rowd": 3986, "Don": 3987, "\u0120kids": 3988, "\u0120journal": 3989, "\u0120corpor": 3990, "\u0120false": 3991, "\u0120web": 3992, "\u0120sleep": 3993, "\u0120contain": 3994, "\u0120sto": 3995, "\u0120bed": 3996, "iverse": 3997, "\u0120Rich": 3998, "\u0120Chinese": 3999, "\u0120pun": 4000, "\u0120meant": 4001, "known": 4002, "\u0120notice": 4003, "\u0120favorite": 4004, "aven": 4005, "\u0120condition": 4006, "\u0120purpose": 4007, "))": 4008, "\u0120organization": 4009, "\u0120challeng": 4010, "\u0120manufact": 4011, "\u0120susp": 4012, "\u0120Ac": 4013, "\u0120critic": 4014, "unes": 4015, "uclear": 4016, "\u0120mer": 4017, "vention": 4018, "\u012080": 4019, "\u0120mist": 4020, "\u0120Us": 4021, "\u0120Tor": 4022, "http": 4023, "olf": 4024, "\u0120larger": 4025, "\u0120advant": 4026, "\u0120resear": 4027, "\u0120actions": 4028, "ml": 4029, "\u0120kept": 4030, "\u0120aim": 4031, ",'": 4032, "col": 4033, "\u0120benefits": 4034, "ifying": 4035, "\u0120actual": 4036, "\u0120International": 4037, "\u0120vehicle": 4038, "\u0120chief": 4039, "\u0120efforts": 4040, "\u0120League": 4041, "\u0120Most": 4042, "\u0120wait": 4043, "\u0120adult": 4044, "\u0120overall": 4045, "\u0120speech": 4046, "\u0120highly": 4047, "\u0120female": 4048, "\u0120error": 4049, "\u0120effective": 4050, "54": 4051, "\u0120encour": 4052, "well": 4053, "\u0120failed": 4054, "\u0120conserv": 4055, "\u0120programs": 4056, "\u0120trou": 4057, "\u0120ahead": 4058, "500": 4059, "vertisement": 4060, "IP": 4061, "\u0120Found": 4062, "pir": 4063, "\u0120%": 4064, "\u0120crime": 4065, "ander": 4066, "\u0120location": 4067, "\u0120Iran": 4068, "\u0120behavior": 4069, "azing": 4070, "\u0120rare": 4071, "\u0120emb": 4072, "\u0120caused": 4073, "\u0120ship": 4074, "\u0120active": 4075, "\u0120contribut": 4076, "\u0120green": 4077, "\u0120acqu": 4078, "\u0120reflect": 4079, "venue": 4080, "\u0120firm": 4081, "\u0120birth": 4082, "].": 4083, "\u0120clearly": 4084, "\u0120emot": 4085, "\u0120agency": 4086, "riage": 4087, "\u0120memory": 4088, "98": 4089, "SA": 4090, "\u0120See": 4091, "acing": 4092, "CC": 4093, "\u0120biggest": 4094, "\u0120rap": 4095, "\u0120basic": 4096, "\u0120band": 4097, "eat": 4098, "\u0120suspect": 4099, "\u0120Mac": 4100, "\u012090": 4101, "mark": 4102, "istan": 4103, "\u0120spread": 4104, "ams": 4105, "ki": 4106, "asy": 4107, "rav": 4108, "\u0120Rober": 4109, "\u0120demonstr": 4110, "rated": 4111, "\u0120absolute": 4112, "\u0120places": 4113, "\u0120impl": 4114, "ibrary": 4115, "\u0120cards": 4116, "\u0120destroy": 4117, "\u0120virt": 4118, "vere": 4119, "\u0120appeared": 4120, "yan": 4121, "point": 4122, "\u0120beg": 4123, "\u0120temper": 4124, "spe": 4125, "anted": 4126, "ears": 4127, "\u0120Direct": 4128, "\u0120length": 4129, "\u0120blog": 4130, "amb": 4131, "\u0120integ": 4132, "\u0120resources": 4133, "acc": 4134, "iful": 4135, "\u0120spot": 4136, "\u0120forced": 4137, "\u0120thousands": 4138, "\u0120Minister": 4139, "\u0120qual": 4140, "\u0120French": 4141, "atically": 4142, "\u0120generally": 4143, "\u0120drink": 4144, "\u0120thus": 4145, "IL": 4146, "odes": 4147, "\u0120appropri": 4148, "\u0120Read": 4149, "\u0120whom": 4150, "\u0120eye": 4151, "\u0120college": 4152, "\u012045": 4153, "irection": 4154, "\u0120ensure": 4155, "\u0120apparent": 4156, "iders": 4157, "\u0120religious": 4158, "\u0120minor": 4159, "olic": 4160, "\u0120tro": 4161, "\u0120Why": 4162, "ribute": 4163, "met": 4164, "\u0120primary": 4165, "\u0120developed": 4166, "\u0120peace": 4167, "\u0120skin": 4168, "ste": 4169, "ava": 4170, "\u0120blue": 4171, "\u0120families": 4172, "\u0120ir": 4173, "\u0120apply": 4174, "\u0120inform": 4175, "\u0120Smith": 4176, "CT": 4177, "ii": 4178, "\u0120limit": 4179, "\u0120resist": 4180, "................": 4181, "umn": 4182, "\u0120conflic": 4183, "\u0120twe": 4184, "udd": 4185, "\u0120Tom": 4186, "\u0120liter": 4187, "que": 4188, "bon": 4189, "\u0120hair": 4190, "\u0120eventually": 4191, "\u0120pus": 4192, "\u0120helped": 4193, "\u0120agg": 4194, "orney": 4195, "\u0120Apple": 4196, "\u0120fit": 4197, "\u0120Sur": 4198, "\u0120prem": 4199, "\u0120sales": 4200, "\u0120seconds": 4201, "\u0120strength": 4202, "\u0120feeling": 4203, "\u00bf\u00bd": 4204, "\u0120tour": 4205, "\u0120knows": 4206, "oom": 4207, "\u0120exerc": 4208, "\u0120somew": 4209, "\u00ef\u00bf\u00bd": 4210, ">>": 4211, "\u0120spokes": 4212, "\u0120ideas": 4213, "\u0120regist": 4214, "soft": 4215, "\u0120Del": 4216, "\u0120PC": 4217, "\u0120propos": 4218, "\u0120launch": 4219, "\u0120bottom": 4220, "TH": 4221, "\u0120Please": 4222, "vest": 4223, "itz": 4224, "\u0120Inter": 4225, "\u0120script": 4226, "\u0120rat": 4227, "arning": 4228, "\u0120il": 4229, "\u0120Jer": 4230, "\u0120Are": 4231, "\u0120whatever": 4232, "oken": 4233, "cience": 4234, "\u0120mode": 4235, "\u0120agree": 4236, "\u0120sources": 4237, "\u0120initial": 4238, "\u0120restrict": 4239, "\u0120wonder": 4240, "usion": 4241, "####": 4242, "\u0120Sil": 4243, "ville": 4244, "\u0120burn": 4245, "tw": 4246, "asion": 4247, "\u0120\u00c2\u00a3": 4248, "\u0120nor": 4249, "uing": 4250, "\u0120reached": 4251, "\u0120sun": 4252, "\u0120categ": 4253, "igration": 4254, "\u0120cook": 4255, "\u0120promot": 4256, "\u0120male": 4257, "\u0120climate": 4258, "\u0120fix": 4259, "\u0120alleged": 4260, "UR": 4261, "alled": 4262, "\u0120images": 4263, "Cont": 4264, "ota": 4265, "\u0120schools": 4266, "ios": 4267, "\u0120drop": 4268, "\u0120stream": 4269, "\u0120Mo": 4270, "\u0120previously": 4271, "aling": 4272, "\u0120pet": 4273, "\u0120double": 4274, "\u0120(@": 4275, "annel": 4276, "\u0120default": 4277, "ties": 4278, "\u0120rank": 4279, "\u0120Dec": 4280, "\u0120Council": 4281, "\u0120weapon": 4282, "\u0120stock": 4283, "\u0120analy": 4284, "\u0120Str": 4285, "\u0120picture": 4286, "\u0120Police": 4287, "ference": 4288, "\u0120century": 4289, "\u0120citizens": 4290, "\u0120onto": 4291, "\u0120expand": 4292, "\u0120hero": 4293, "\u0120Sol": 4294, "\u0120wild": 4295, "\u0120update": 4296, "\u0120customers": 4297, "ront": 4298, "def": 4299, "\u0120lik": 4300, "\u0120criminal": 4301, "\u0120Christian": 4302, "SP": 4303, "76": 4304, "\u0120leaving": 4305, "\u0120otherwise": 4306, "\u0120Dist": 4307, "\u0120basis": 4308, "52": 4309, "53": 4310, "icip": 4311, "\u0120Ber": 4312, "\u0120recommend": 4313, "\u0120floor": 4314, "\u0120crowd": 4315, "oles": 4316, "\u012070": 4317, "\u0120central": 4318, "\u0120Ev": 4319, "\u0120dream": 4320, "\u0120download": 4321, "\u0120confir": 4322, "\u0120Thom": 4323, "\u0120window": 4324, "\u0120happens": 4325, "\u0120unit": 4326, "\u0120tend": 4327, "\u0120spl": 4328, "\u0120becomes": 4329, "\u0120fighting": 4330, "\u0120predict": 4331, "\u0120Press": 4332, "\u0120Power": 4333, "\u0120heavy": 4334, "aked": 4335, "\u0120fan": 4336, "orter": 4337, "ategy": 4338, "BA": 4339, "izes": 4340, "\u0120spend": 4341, "Here": 4342, "\u01202007": 4343, "\u0120adop": 4344, "\u0120Ham": 4345, "\u0120football": 4346, "\u0120Port": 4347, "oday": 4348, "51": 4349, "ampions": 4350, "\u0120transfer": 4351, "ht": 4352, "\u012038": 4353, "term": 4354, "acity": 4355, "\u0120bur": 4356, "],": 4357, "ternal": 4358, "rig": 4359, "but": 4360, "\u0120therefore": 4361, "\u0120Because": 4362, "resp": 4363, "rey": 4364, "\u0120mission": 4365, "Some": 4366, "\u0120noted": 4367, "\u0120assum": 4368, "\u0120disease": 4369, "\u0120edit": 4370, "\u0120progress": 4371, "rd": 4372, "\u0120Brown": 4373, "ocal": 4374, "\u0120adding": 4375, "\u0120raised": 4376, "\u0120Any": 4377, "\u0120tick": 4378, "\u0120seeing": 4379, "\u0120People": 4380, "\u0120agreement": 4381, "\u0120server": 4382, "\u0120wat": 4383, "\u0120debate": 4384, "\u0120supposed": 4385, "iling": 4386, "\u0120largest": 4387, "\u0120successful": 4388, "\u0120Pri": 4389, "\u0120Democratic": 4390, "\u0120jump": 4391, "\u0120Syria": 4392, "\u0120owners": 4393, "\u0120offers": 4394, "\u0120shooting": 4395, "\u0120effic": 4396, "sey": 4397, "\u0120haven": 4398, "verse": 4399, "tered": 4400, "\u0120Light": 4401, "imal": 4402, "\u0120Big": 4403, "\u0120defend": 4404, "\u0120beat": 4405, "\u0120records": 4406, "%)": 4407, "\u0120scen": 4408, "\u0120employees": 4409, "\u0120devices": 4410, "hem": 4411, "\u0120commer": 4412, "\u0120Mex": 4413, "\u0120benefit": 4414, "\u0120Prof": 4415, "\u0120illeg": 4416, "\u0120surface": 4417, "\u0120Also": 4418, "\u0120harm": 4419, "ingly": 4420, "wide": 4421, "\u0120Alex": 4422, "\u0120shut": 4423, "\u0120Cur": 4424, "\u0120lose": 4425, "pm": 4426, "\u0120challenge": 4427, "semb": 4428, "\u0120station": 4429, "\u0120intelligence": 4430, "\u0120accur": 4431, "\u0120Flor": 4432, "\u0120requires": 4433, "\u0120Mal": 4434, "bum": 4435, "\u0120hospital": 4436, "\u0120spirit": 4437, "\u0120offered": 4438, "\u0120produce": 4439, "\u0120Commun": 4440, "\u0120creating": 4441, "\u0120cris": 4442, "spect": 4443, "\u0120ended": 4444, "\u0120daily": 4445, "\u0120voters": 4446, "lands": 4447, "ias": 4448, "ih": 4449, "ona": 4450, "\u0120smart": 4451, "\u0120Office": 4452, "\u0120Lord": 4453, "rial": 4454, "\u0120Internet": 4455, "\u0120circum": 4456, "\u0120extremely": 4457, "'.": 4458, "\u0120opinion": 4459, "\u0120Mil": 4460, "\u0120gain": 4461, "BS": 4462, "\u0120Fin": 4463, "yp": 4464, "\u0120useful": 4465, "\u0120budget": 4466, "\u0120comfort": 4467, "isf": 4468, "\u0120background": 4469, "eline": 4470, "\u0120episode": 4471, "\u0120enemy": 4472, "\u0120trial": 4473, "\u0120establish": 4474, "date": 4475, "\u0120Cap": 4476, "\u0120continues": 4477, "\u0120showing": 4478, "\u0120Union": 4479, "with": 4480, "\u0120posted": 4481, "\u0120System": 4482, "\u0120eat": 4483, "rian": 4484, "\u0120rise": 4485, "\u0120Germany": 4486, "ils": 4487, "\u0120signed": 4488, "\u0120vill": 4489, "\u0120grand": 4490, "mor": 4491, "\u0120England": 4492, "\u0120projects": 4493, "umber": 4494, "\u0120conference": 4495, "za": 4496, "\u0120responsible": 4497, "\u0120Arab": 4498, "\u0120learned": 4499, "\u00e2\u0122\u0136\u00e2\u0122\u0136": 4500, "ipping": 4501, "\u0120George": 4502, "OC": 4503, "\u0120returned": 4504, "\u0120Australia": 4505, "\u0120brief": 4506, "Qu": 4507, "\u0120brand": 4508, "illing": 4509, "abled": 4510, "\u0120highest": 4511, "\u0120train": 4512, "\u0120Commission": 4513, "while": 4514, "\u0120nom": 4515, "ception": 4516, "\u0120mut": 4517, "\u0120Blue": 4518, "\u0120incident": 4519, "vant": 4520, "86": 4521, "\u0120ID": 4522, "\u0120nuclear": 4523, "74": 4524, "\u0120Like": 4525, "\u0120RE": 4526, "\u0120Micro": 4527, "li": 4528, "mail": 4529, "\u0120charges": 4530, "89": 4531, "\u0120adjust": 4532, "ado": 4533, "\u0120earth": 4534, "NA": 4535, "\u0120prices": 4536, "PA": 4537, "\u0120draft": 4538, "\u0120runs": 4539, "\u0120candidate": 4540, "enses": 4541, "\u0120management": 4542, "\u0120Phil": 4543, "\u0120Miss": 4544, "\u0120teach": 4545, "gram": 4546, "\u0120understanding": 4547, "ait": 4548, "icago": 4549, "Add": 4550, "\u0120Ep": 4551, "secut": 4552, "\u0120separate": 4553, "\u0120instance": 4554, "\u0120eth": 4555, "\u0120unless": 4556, "********": 4557, "\u0120Fore": 4558, "inate": 4559, "\u0120operations": 4560, "Sp": 4561, "\u0120faith": 4562, "gar": 4563, "\u0120Church": 4564, "ronic": 4565, "\u0120config": 4566, "osure": 4567, "\u0120activities": 4568, "\u0120traditional": 4569, "\u012036": 4570, "\u0120direction": 4571, "\u0120machine": 4572, "\u0120surround": 4573, "\u0120push": 4574, "unction": 4575, "\u0120EU": 4576, "\u0120easier": 4577, "\u0120argument": 4578, "GB": 4579, "\u0120micro": 4580, "\u0120spending": 4581, "izations": 4582, "\u0120theory": 4583, "adow": 4584, "\u0120calling": 4585, "\u0120Last": 4586, "\u0120der": 4587, "\u0120influence": 4588, "\u0120commit": 4589, "\u0120photo": 4590, "\u0120unc": 4591, "istry": 4592, "gn": 4593, "aste": 4594, "acks": 4595, "\u0120disp": 4596, "ady": 4597, "do": 4598, "\u0120Good": 4599, "\u0120`": 4600, "\u0120wish": 4601, "\u0120revealed": 4602, "\u00c2\u0142\u00c2\u0142": 4603, "lig": 4604, "\u0120enforce": 4605, "\u0120Committee": 4606, "\u0120chem": 4607, "\u0120miles": 4608, "\u0120interested": 4609, "\u0120solution": 4610, "icy": 4611, "inct": 4612, "\u0120->": 4613, "\u0120Det": 4614, "\u0120removed": 4615, "\u0120compar": 4616, "eah": 4617, "\u0120plant": 4618, "\u0120Since": 4619, "\u0120achieve": 4620, "\u0120advantage": 4621, "\u0120slightly": 4622, "bing": 4623, "\u0120placed": 4624, "under": 4625, "2015": 4626, "\u0120Mad": 4627, "\u0120tim": 4628, "oses": 4629, "\u0120cru": 4630, "\u0120Rock": 4631, "\u0120mostly": 4632, "\u0120negative": 4633, "\u0120setting": 4634, "\u0120produced": 4635, "\u0120mur": 4636, "\u0120connection": 4637, "\u0120Mer": 4638, "\u0120driver": 4639, "\u0120executive": 4640, "\u0120assault": 4641, "\u0120born": 4642, "\u0120Ver": 4643, "tained": 4644, "\u0120structure": 4645, "\u0120reduce": 4646, "\u0120decades": 4647, "\u0120ded": 4648, "uke": 4649, "\u0120Many": 4650, "idden": 4651, "\u0120league": 4652, "Se": 4653, "\u0120join": 4654, "\u0120disco": 4655, "\u0120die": 4656, "cks": 4657, "actions": 4658, "\u0120assess": 4659, "agn": 4660, "\u0120goals": 4661, "ours": 4662, "IR": 4663, "\u0120senior": 4664, "iller": 4665, "mod": 4666, "ipment": 4667, "ocol": 4668, "uy": 4669, "\u0120Que": 4670, "\u0120parties": 4671, "irgin": 4672, "\u0120learning": 4673, "itable": 4674, "\u0120street": 4675, "\u0120camera": 4676, "App": 4677, "\u0120skills": 4678, "bre": 4679, "cious": 4680, "\u0120celebr": 4681, "\u0120Franc": 4682, "\u0120existing": 4683, "\u0120willing": 4684, "lor": 4685, "\u0120id": 4686, "\u0120Space": 4687, "\u0120critical": 4688, "\u0120La": 4689, "ortunately": 4690, "\u0120serve": 4691, "\u0120cold": 4692, "\u0120species": 4693, "TS": 4694, "\u0120animals": 4695, "\u0120Bay": 4696, "\u0120older": 4697, "\u0120Under": 4698, "estic": 4699, "\u0120Tre": 4700, "\u0120teacher": 4701, "\u0120prefer": 4702, "vis": 4703, "\u0120thread": 4704, "\u0120Matt": 4705, "\u0120manager": 4706, "\u00e3\u0125\u00bb": 4707, "\u0120professional": 4708, "\u0120Vol": 4709, "\u0120notes": 4710, "These": 4711, "ula": 4712, "\u0120fresh": 4713, "ented": 4714, "uzz": 4715, "edy": 4716, "clusion": 4717, "\u0120Rel": 4718, "\u0120doubt": 4719, "EO": 4720, "\u0120opened": 4721, "\u0120Bit": 4722, "Advertisement": 4723, "\u0120guess": 4724, "\u0120UN": 4725, "\u0120sequ": 4726, "\u0120explain": 4727, "otten": 4728, "\u0120attract": 4729, "aks": 4730, "\u0120string": 4731, "\u0120context": 4732, "ossible": 4733, "\u0120Republicans": 4734, "\u0120solid": 4735, "\u0120cities": 4736, "\u0120asking": 4737, "\u0120random": 4738, "ups": 4739, "uries": 4740, "arant": 4741, "dden": 4742, "gl": 4743, "\u0120Florida": 4744, "\u0120depend": 4745, "\u0120Scott": 4746, "\u012033": 4747, "\u0120iT": 4748, "icon": 4749, "\u0120mentioned": 4750, "\u01202000": 4751, "\u0120claimed": 4752, "\u0120definitely": 4753, "ulf": 4754, "\u0120core": 4755, "\u0120opening": 4756, "\u0120Const": 4757, "which": 4758, "\u0120Tra": 4759, "AG": 4760, "72": 4761, "\u0120believed": 4762, "ada": 4763, "\u012048": 4764, "\u0120Security": 4765, "yright": 4766, "\u0120Pet": 4767, "\u0120Lou": 4768, "\u0120holding": 4769, "================": 4770, "\u0120ice": 4771, "\u0120brow": 4772, "\u0120authorities": 4773, "host": 4774, "word": 4775, "\u0120score": 4776, "\u0120Div": 4777, "\u0120cells": 4778, "\u0120transl": 4779, "\u0120neighbor": 4780, "\u0120remove": 4781, "uct": 4782, "\u0120district": 4783, "\u0120According": 4784, "\u0120worse": 4785, "\u0120concerns": 4786, "\u0120presidential": 4787, "\u0120policies": 4788, "\u0120Hall": 4789, "73": 4790, "\u0120hus": 4791, "AY": 4792, "\u01202006": 4793, "\u0120Jud": 4794, "\u0120independent": 4795, "\u0120Justice": 4796, "iliar": 4797, "print": 4798, "ighter": 4799, "\u0120protection": 4800, "zen": 4801, "\u0120sudden": 4802, "house": 4803, "\u0120Jes": 4804, "PR": 4805, "\u0120Inf": 4806, "\u0120bul": 4807, "\u0120_": 4808, "\u0120Service": 4809, "\u0120PR": 4810, "\u0120strategy": 4811, "ffect": 4812, "\u0120girls": 4813, "\u0120missing": 4814, "oyal": 4815, "\u0120Team": 4816, "ulated": 4817, "\u0120dat": 4818, "\u0120politics": 4819, "abor": 4820, "According": 4821, "\u0120spell": 4822, "\u0120graph": 4823, "orthern": 4824, "TC": 4825, "Ab": 4826, "\u0120labor": 4827, "isher": 4828, "\u0120kick": 4829, "\u0120iTunes": 4830, "\u0120steps": 4831, "poses": 4832, "\u0120smaller": 4833, "En": 4834, "bert": 4835, "\u0120roll": 4836, "\u0120researchers": 4837, "\u0120closed": 4838, "\u0120transport": 4839, "\u0120lawy": 4840, "________________": 4841, "\u0120Chicago": 4842, "\u0120aspect": 4843, "\u0120none": 4844, "\u0120marriage": 4845, "96": 4846, "\u0120elements": 4847, "\u0120Fre": 4848, "\u0120Sal": 4849, "\u0120dram": 4850, "FC": 4851, "top": 4852, "equ": 4853, "\u0120hearing": 4854, "\u0120supported": 4855, "\u0120testing": 4856, "cohol": 4857, "\u0120massive": 4858, "\u0120stick": 4859, "\u0120guard": 4860, "isco": 4861, "phone": 4862, "From": 4863, "However": 4864, "\u0120border": 4865, "\u0120copy": 4866, "ography": 4867, "list": 4868, "71": 4869, "\u0120owner": 4870, "class": 4871, "ruit": 4872, "rate": 4873, "\u0120Once": 4874, "\u0120digital": 4875, "\u0120task": 4876, "ERS": 4877, "\u0120incred": 4878, "tes": 4879, "++": 4880, "\u0120France": 4881, "\u0120breat": 4882, "owl": 4883, "\u0120issued": 4884, "\u0120Western": 4885, "\u0120detect": 4886, "\u0120partners": 4887, "\u0120shared": 4888, "\u0120Call": 4889, "\u0120cancer": 4890, "ache": 4891, "ribe": 4892, "\u0120explained": 4893, "\u0120heat": 4894, "{\"": 4895, "\u0120investment": 4896, "\u0120Book": 4897, "\u0120wood": 4898, "\u0120tools": 4899, "\u0120Although": 4900, "\u0120belief": 4901, "\u0120crisis": 4902, "\u0120ge": 4903, "\u0120MP": 4904, "\u0120operation": 4905, "type": 4906, "~~": 4907, "ga": 4908, "\u0120contains": 4909, "anta": 4910, "\u0120express": 4911, "\u0120Group": 4912, "\u0120Journal": 4913, "ka": 4914, "\u0120amb": 4915, "\u0120USA": 4916, "\u0120finding": 4917, "\u0120funding": 4918, "how": 4919, "\u0120established": 4920, "ideos": 4921, "\u0120degree": 4922, "\u0120dangerous": 4923, "anging": 4924, "\u0120freedom": 4925, "pport": 4926, "outhern": 4927, "\u0120church": 4928, "\u0120catch": 4929, "\u0120Two": 4930, "\u0120presence": 4931, "\u0120Guard": 4932, "Up": 4933, "\u0120authority": 4934, "\u0120Project": 4935, "\u0120button": 4936, "\u0120consequ": 4937, "\u0120valid": 4938, "\u0120weak": 4939, "\u0120starts": 4940, "\u0120reference": 4941, "\u0120Mem": 4942, "\")": 4943, "UN": 4944, "orage": 4945, "\u0120Open": 4946, "\u0120collection": 4947, "ym": 4948, "gency": 4949, "\u0120beautiful": 4950, "ros": 4951, "\u0120tells": 4952, "\u0120waiting": 4953, "nel": 4954, "\u0120providing": 4955, "\u0120Democrats": 4956, "\u0120daughter": 4957, "\u0120master": 4958, "\u0120purposes": 4959, "\u0120Japanese": 4960, "\u0120equal": 4961, "\u0120turns": 4962, "\u0120documents": 4963, "\u0120watching": 4964, "Res": 4965, "\u0120ran": 4966, "2014": 4967, "\u0120reject": 4968, "\u0120Korea": 4969, "\u0120victims": 4970, "Level": 4971, "erences": 4972, "\u0120witness": 4973, "\u012034": 4974, "\u0120reform": 4975, "coming": 4976, "\u0120occup": 4977, "\u0120caught": 4978, "\u0120traffic": 4979, "ading": 4980, "\u0120models": 4981, "ario": 4982, "\u0120served": 4983, "\u0120batter": 4984, "uate": 4985, "\u0120Secretary": 4986, "\u0120agreed": 4987, "\u0120truly": 4988, "ynam": 4989, "\u0120Ret": 4990, "\u0120units": 4991, "\u0120Research": 4992, "hand": 4993, "azine": 4994, "\u0120Mike": 4995, "\u0120variety": 4996, "otal": 4997, "\u0120amazing": 4998, "\u0120confirmed": 4999, "\u0120entirely": 5000, "\u0120purchase": 5001, "\u0120element": 5002, "\u0120cash": 5003, "\u0120determine": 5004, "De": 5005, "\u0120cars": 5006, "\u0120Wall": 5007, "\u00e2\u0138": 5008, "\u0120views": 5009, "\u0120drugs": 5010, "\u0120department": 5011, "\u0120Step": 5012, "uit": 5013, "\u012039": 5014, "asure": 5015, "\u0120Class": 5016, "\u0120covered": 5017, "\u0120Bank": 5018, "\u0120mere": 5019, "uana": 5020, "\u0120multi": 5021, "\u0120mix": 5022, "\u0120unlike": 5023, "levision": 5024, "\u0120stopped": 5025, "\u0120sem": 5026, "\u0120Gal": 5027, "ules": 5028, "\u0120wel": 5029, "\u0120Johnson": 5030, "la": 5031, "\u0120skill": 5032, "\u0120becoming": 5033, "rie": 5034, "\u0120appropriate": 5035, "fe": 5036, "ellow": 5037, "\u0120Prot": 5038, "ulate": 5039, "ocation": 5040, "\u0120weekend": 5041, "odies": 5042, "\u0120sites": 5043, "\u0120animal": 5044, "\u0120Tim": 5045, "\u0120scale": 5046, "\u0120charged": 5047, "\u0120instruct": 5048, "illa": 5049, "\u0120methods": 5050, "\u0120cert": 5051, "\u0120judge": 5052, "\u0120Hel": 5053, "\u0120dollars": 5054, "\u0120standing": 5055, "\u0120Squ": 5056, "\u0120debt": 5057, "liam": 5058, "\u0120driving": 5059, "\u0120Sum": 5060, "\u0120Edition": 5061, "\u0120album": 5062, "andon": 5063, "IF": 5064, "\u0120Uk": 5065, "63": 5066, "ader": 5067, "\u0120commercial": 5068, "esh": 5069, "\u0120Government": 5070, "\u0120discovered": 5071, "\u0120output": 5072, "\u0120Hillary": 5073, "\u0120Carol": 5074, "\u01202005": 5075, "\u0120abuse": 5076, "ancing": 5077, "\u0120switch": 5078, "\u0120annual": 5079, "Tw": 5080, "\u0120stated": 5081, "agement": 5082, "inner": 5083, "\u0120democr": 5084, "\u0120residents": 5085, "\u0120allowing": 5086, "\u0120factors": 5087, "odd": 5088, "\u0120fuck": 5089, "emies": 5090, "\u0120occurred": 5091, "oti": 5092, "\u0120north": 5093, "\u0120Public": 5094, "\u0120injury": 5095, "\u0120insurance": 5096, "CL": 5097, "olly": 5098, "\u00e3\u0122": 5099, "\u0120repeated": 5100, "\u0120arms": 5101, "anged": 5102, "\u0120construction": 5103, "\u0120fle": 5104, "PU": 5105, "icians": 5106, "\u0120forms": 5107, "\u0120McC": 5108, "antic": 5109, "\u0120mental": 5110, "pire": 5111, "\u0120equipment": 5112, "\u0120fant": 5113, "\u0120discussion": 5114, "\u0120regarding": 5115, "kin": 5116, "arp": 5117, "\u0120chair": 5118, "ogue": 5119, "\u0120proceed": 5120, "\u0120Id": 5121, "Our": 5122, "\u0120murder": 5123, "Man": 5124, "\u012049": 5125, "asp": 5126, "\u0120supply": 5127, "\u0120input": 5128, "\u0120wealth": 5129, "liament": 5130, "\u0120proced": 5131, "orial": 5132, "\u0120Stat": 5133, "\u0120NFL": 5134, "hens": 5135, "\u0120Institute": 5136, "\u0120putting": 5137, "ournament": 5138, "etic": 5139, "\u0120located": 5140, "\u0120kid": 5141, "eria": 5142, "run": 5143, "\u0120princ": 5144, "\u0120!": 5145, "going": 5146, "\u0120Bet": 5147, "\u0120clot": 5148, "\u0120telling": 5149, "\u0120proposed": 5150, "iot": 5151, "orry": 5152, "\u0120funds": 5153, "gment": 5154, "\u0120Life": 5155, "\u0120baby": 5156, "\u0120Back": 5157, "\u0120spoke": 5158, "Image": 5159, "\u0120earn": 5160, "\u0120AT": 5161, "gu": 5162, "\u0120exchange": 5163, "\u0120Lin": 5164, "oving": 5165, "\u0120pair": 5166, "More": 5167, "azon": 5168, "\u0120arrested": 5169, "\u0120killing": 5170, "can": 5171, "\u0120Card": 5172, "yd": 5173, "\u0120identified": 5174, "\u0120mobile": 5175, "\u0120thanks": 5176, "onym": 5177, "\u0120Form": 5178, "\u0120hundreds": 5179, "\u0120Chris": 5180, "\u0120Cat": 5181, "\u0120trend": 5182, "hat": 5183, "\u0120Av": 5184, "oman": 5185, "\u0120electric": 5186, "\u0120Wil": 5187, "SE": 5188, "Of": 5189, "\u0120restaur": 5190, "oted": 5191, "\u0120trig": 5192, "\u0120nine": 5193, "\u0120bomb": 5194, "Why": 5195, "\u00c2\u00af": 5196, "\u0120coverage": 5197, "\u0120appeal": 5198, "\u0120Robert": 5199, "\u0120Sup": 5200, "\u0120finished": 5201, "\u0120flow": 5202, "\u0120deliver": 5203, "\u0120calcul": 5204, "\u0120photos": 5205, "\u0120phil": 5206, "\u0120pieces": 5207, "\u0120appre": 5208, "kes": 5209, "\u0120rough": 5210, "Do": 5211, "\u0120partner": 5212, "\u0120concerned": 5213, "\u012037": 5214, "\u0120Gen": 5215, "Col": 5216, "ctors": 5217, "\u0120=>": 5218, "state": 5219, "\u0120suggested": 5220, "\u0120Force": 5221, "CE": 5222, "\u0120herself": 5223, "\u0120Plan": 5224, "works": 5225, "ooth": 5226, "rency": 5227, "\u0120corner": 5228, "\u0120husband": 5229, "\u0120internet": 5230, "\u0120Aut": 5231, "ems": 5232, "osen": 5233, "\u0120Atl": 5234, "gen": 5235, "\u0120balance": 5236, "62": 5237, "\u0120sounds": 5238, "text": 5239, "\u0120arr": 5240, "oves": 5241, "\u0120millions": 5242, "\u0120radio": 5243, "\u0120satisf": 5244, "\u0120Dam": 5245, "Mr": 5246, "Go": 5247, "Spe": 5248, "\u0120combat": 5249, "rant": 5250, "\u0120Gree": 5251, "\u0120fuel": 5252, "\u0120distance": 5253, "\u0120tests": 5254, "\u0120decre": 5255, "\u0120Er": 5256, "\u0120managed": 5257, "DS": 5258, "\u0120tit": 5259, "\u0120measures": 5260, "\u0120Liber": 5261, "\u0120attend": 5262, "ashed": 5263, "\u0120Jose": 5264, "\u0120Night": 5265, "dit": 5266, "\u0120Nov": 5267, "\u0120End": 5268, "outs": 5269, "\u0120generation": 5270, "\u0120advoc": 5271, "yth": 5272, "\u0120conversation": 5273, "\u0120Sky": 5274, "active": 5275, "cel": 5276, "rier": 5277, "\u0120Frank": 5278, "\u0120gender": 5279, "\u0120concent": 5280, "\u0120carried": 5281, "anda": 5282, "\u0120Virgin": 5283, "\u0120arrived": 5284, "icide": 5285, "aded": 5286, "\u0120failure": 5287, "\u0120minimum": 5288, "lets": 5289, "\u0120worst": 5290, "\u0120keeping": 5291, "\u0120intended": 5292, "\u0120illegal": 5293, "\u0120subsc": 5294, "\u0120determined": 5295, "\u0120trip": 5296, "Yes": 5297, "\u0120raise": 5298, "\u0120~": 5299, "\u0120feels": 5300, "\u0120package": 5301, "\u0120Jo": 5302, "hi": 5303, "2016": 5304, "real": 5305, "\u0120fra": 5306, "\u0120symb": 5307, "Me": 5308, "ucky": 5309, "pret": 5310, "\u0120Kh": 5311, "\u0120Edit": 5312, "\u0120Web": 5313, "emic": 5314, "\u0120Color": 5315, "\u0120justice": 5316, "Int": 5317, "\u0120farm": 5318, "cknow": 5319, "\">": 5320, "eless": 5321, "\u0120reduced": 5322, "\u0120500": 5323, "xx": 5324, "\u0120Rad": 5325, "\u0120Wood": 5326, "\u0120clin": 5327, "\u0120hyp": 5328, "iler": 5329, "ura": 5330, "kins": 5331, "85": 5332, "61": 5333, "\u0120Their": 5334, "\u0120Mary": 5335, "\u0120san": 5336, "\u0120novel": 5337, "\u0120Who": 5338, "\u0120capacity": 5339, "\u0120impossible": 5340, "\u0120plays": 5341, "\u0120minister": 5342, "ijuana": 5343, "icate": 5344, "\u0120Set": 5345, "\u0120fram": 5346, "\u0120ing": 5347, "\u0120communities": 5348, "\u0120FBI": 5349, "ita": 5350, "\u0120bon": 5351, "\u0120strateg": 5352, "\u0120interests": 5353, "lock": 5354, "gers": 5355, "mas": 5356, "\u0120AND": 5357, "\u0120conflict": 5358, "\u0120requirements": 5359, "\u0120sac": 5360, "\u0120operating": 5361, "ini": 5362, "related": 5363, "\u0120committed": 5364, "\u0120relatively": 5365, "\u0120south": 5366, "\u00c2\u00af\u00c2\u00af": 5367, "\u0120afford": 5368, "\u0120identity": 5369, "\u0120decisions": 5370, "\u0120accused": 5371, "place": 5372, "\u0120victory": 5373, "och": 5374, "iat": 5375, "Name": 5376, "Com": 5377, "tion": 5378, "eds": 5379, "\u0120seek": 5380, "\u0120tight": 5381, "\u0120Images": 5382, "\u0120initi": 5383, "\u0120humans": 5384, "\u0120familiar": 5385, "\u0120audience": 5386, "\u0120internal": 5387, "venture": 5388, "\u0120sides": 5389, "\u0120TO": 5390, "\u0120dim": 5391, "\u0120conclud": 5392, "\u0120appoint": 5393, "\u0120enforcement": 5394, "\u0120Jim": 5395, "\u0120Association": 5396, "\u0120circumst": 5397, "\u0120Canadian": 5398, "\u0120joined": 5399, "\u0120differences": 5400, "\u0120Los": 5401, "\u0120protest": 5402, "\u0120twice": 5403, "win": 5404, "\u0120glass": 5405, "arsh": 5406, "\u0120Army": 5407, "\u0120expression": 5408, "\u0120decide": 5409, "\u0120planning": 5410, "ania": 5411, "\u0120handle": 5412, "\u0120Microsoft": 5413, "\u0120Nor": 5414, "\u0120maximum": 5415, "\u0120Rev": 5416, "\u0120sea": 5417, "\u0120eval": 5418, "\u0120helps": 5419, "ref": 5420, "\u0120bound": 5421, "\u0120mouth": 5422, "\u0120standards": 5423, "\u0120clim": 5424, "\u0120Camp": 5425, "\u0120Fox": 5426, "cles": 5427, "\u0120army": 5428, "\u0120Techn": 5429, "acking": 5430, "xy": 5431, "SS": 5432, "\u012042": 5433, "\u0120bug": 5434, "\u0120Ukrain": 5435, "\u0120Max": 5436, "\u0120Jones": 5437, "\u0120Show": 5438, "lo": 5439, "\u0120planet": 5440, "\u012075": 5441, "\u0120winning": 5442, "\u0120faster": 5443, "\u0120spect": 5444, "\u0120broken": 5445, "TR": 5446, "\u0120defined": 5447, "\u0120healthy": 5448, "\u0120competition": 5449, "https": 5450, "\u0120Island": 5451, "\u0120Fe": 5452, "\u0120announce": 5453, "\u0120Cup": 5454, "\u0120Instead": 5455, "\u0120client": 5456, "\u0120possibly": 5457, "section": 5458, "ocket": 5459, "look": 5460, "\u0120finish": 5461, "\u0120crew": 5462, "\u0120reserv": 5463, "\u0120editor": 5464, "\u0120hate": 5465, "\u0120sale": 5466, "\u0120controvers": 5467, "\u0120pages": 5468, "wing": 5469, "\u0120numer": 5470, "\u0120opposition": 5471, "\u01202004": 5472, "\u0120refuge": 5473, "\u0120flight": 5474, "\u0120apart": 5475, "\u0120Lat": 5476, "Americ": 5477, "\u0120Africa": 5478, "\u0120applications": 5479, "\u0120Palest": 5480, "\u0120Bur": 5481, "\u0120gar": 5482, "\u0120Social": 5483, "\u0120upgr": 5484, "\u0120shape": 5485, "\u0120speaking": 5486, "ansion": 5487, "ao": 5488, "\u0120Sn": 5489, "\u0120worry": 5490, "\u0120Britain": 5491, "Please": 5492, "roud": 5493, "\u0120hun": 5494, "\u0120introduced": 5495, "\u0120diet": 5496, "Ind": 5497, "\u0120Second": 5498, "\u0120functions": 5499, "uts": 5500, "\u0120Each": 5501, "\u0120Jeff": 5502, "\u0120stress": 5503, "\u0120accounts": 5504, "\u0120guarant": 5505, "\u0120Ann": 5506, "edia": 5507, "\u0120honest": 5508, "\u0120tree": 5509, "\u0120African": 5510, "\u0120Bush": 5511, "},": 5512, "\u0120sch": 5513, "\u0120Only": 5514, "\u0120fif": 5515, "igan": 5516, "\u0120exercise": 5517, "\u0120Exp": 5518, "\u0120scientists": 5519, "\u0120legislation": 5520, "\u0120Work": 5521, "\u0120Spr": 5522, "\u00c3\u0124": 5523, "\u0120Human": 5524, "\u0120\u00e8": 5525, "\u0120survey": 5526, "\u0120rich": 5527, "rip": 5528, "\u0120maintain": 5529, "\u0120flo": 5530, "\u0120leadership": 5531, "stream": 5532, "\u0120Islamic": 5533, "\u012001": 5534, "\u0120College": 5535, "\u0120magic": 5536, "\u0120Prime": 5537, "\u0120figures": 5538, "2017": 5539, "inder": 5540, "xual": 5541, "\u0120Dead": 5542, "\u0120absolutely": 5543, "\u0120fourth": 5544, "\u0120presented": 5545, "respond": 5546, "rible": 5547, "\u0120alcohol": 5548, "ato": 5549, "\u0120DE": 5550, "porary": 5551, "\u0120grab": 5552, "\u0120vari": 5553, "\u0120quant": 5554, "\u0120Photo": 5555, "\u0120plus": 5556, "rick": 5557, "arks": 5558, "\u0120alternative": 5559, "\u0120pil": 5560, "\u0120approx": 5561, "that": 5562, "\u0120objects": 5563, "\u0120Ro": 5564, "\u0120Android": 5565, "\u0120significantly": 5566, "\u0120Road": 5567, "kay": 5568, "Read": 5569, "avor": 5570, "\u0120acknow": 5571, "\u0120HD": 5572, "\u0120Sing": 5573, "Or": 5574, "\u0120Mont": 5575, "\u0120uns": 5576, "prof": 5577, "\u0120negoti": 5578, "\u0120Arch": 5579, "iki": 5580, "\u0120television": 5581, "\u0120Jewish": 5582, "\u0120committee": 5583, "\u0120motor": 5584, "\u0120appearance": 5585, "\u0120sitting": 5586, "\u0120strike": 5587, "\u0120Down": 5588, "comp": 5589, "\u0120Hist": 5590, "\u0120fold": 5591, "acement": 5592, "\u0120Louis": 5593, "\u0120belong": 5594, "\u0120\u00e2\u0122\u00a2": 5595, "\u0120mort": 5596, "\u0120prepared": 5597, "\u012064": 5598, "\u0120Master": 5599, "\u0120indeed": 5600, "\u0120Den": 5601, "\u0120rent": 5602, "TA": 5603, "ourney": 5604, "arc": 5605, "Su": 5606, "97": 5607, "\u0120advice": 5608, "\u0120changing": 5609, "\u0120listed": 5610, "\u0120launched": 5611, "isation": 5612, "\u0120Peter": 5613, "ishes": 5614, "\u0120lived": 5615, "\u0120Mel": 5616, "\u0120Supreme": 5617, "\u0120Federal": 5618, "\u0120);": 5619, "ructure": 5620, "\u0120sets": 5621, "\u0120philos": 5622, "uous": 5623, "\u0120\u00c2\u0142": 5624, "\u0120applied": 5625, "\u0120NOT": 5626, "\u0120housing": 5627, "\u0120Mount": 5628, "\u0120odd": 5629, "\u0120sust": 5630, "DA": 5631, "fficient": 5632, "\u0120?": 5633, "olved": 5634, "\u0120powers": 5635, "\u0120thr": 5636, "\u0120remaining": 5637, "\u0120Water": 5638, "LC": 5639, "\u0120causes": 5640, "\u00e3\u0123\u00ae": 5641, "\u0120manner": 5642, "ads": 5643, "\u0120suggests": 5644, "\u0120ends": 5645, "standing": 5646, "fig": 5647, "\u0120Dun": 5648, "idth": 5649, "\u0120gay": 5650, "\u0120termin": 5651, "\u0120Angeles": 5652, "MS": 5653, "\u0120scientific": 5654, "\u0120coal": 5655, "apers": 5656, "bar": 5657, "\u0120Thomas": 5658, "\u0120sym": 5659, "\u0120Run": 5660, "this": 5661, "PC": 5662, "igrants": 5663, "\u0120minute": 5664, "\u0120District": 5665, "cellent": 5666, "\u0120leaves": 5667, "\u0120completed": 5668, "amin": 5669, "\u0120focused": 5670, "\u0120monitor": 5671, "\u0120vehicles": 5672, "MA": 5673, "\u0120Mass": 5674, "\u0120Grand": 5675, "\u0120affected": 5676, "itutional": 5677, "\u0120construct": 5678, "\u0120follows": 5679, "\u0120ton": 5680, "reens": 5681, "\u0120homes": 5682, "\u0120Ext": 5683, "\u0120Level": 5684, "rast": 5685, "\u0120Ir": 5686, "\u0120elim": 5687, "\u0120largely": 5688, "\u0120Joe": 5689, "\u0120votes": 5690, "alls": 5691, "\u0120businesses": 5692, "\u0120Foundation": 5693, "\u0120Central": 5694, "\u0120yards": 5695, "\u0120materials": 5696, "ulner": 5697, "\u0120guide": 5698, "\u0120closer": 5699, "ums": 5700, "\u0120sports": 5701, "eder": 5702, "Just": 5703, "\u0120taxes": 5704, "84": 5705, "\u0120Old": 5706, "\u0120decade": 5707, "ola": 5708, "\u0120vir": 5709, "\u0120dropped": 5710, "\u0120delay": 5711, "itect": 5712, "\u0120secure": 5713, "stein": 5714, "level": 5715, "\u0120treated": 5716, "\u0120filed": 5717, "aine": 5718, "\u0120van": 5719, "\u0120mir": 5720, "\u0120column": 5721, "icted": 5722, "eper": 5723, "\u0120rot": 5724, "\u0120consult": 5725, "\u0120entry": 5726, "\u0120marijuana": 5727, "\u0120Dou": 5728, "\u0120apparently": 5729, "oking": 5730, "clusive": 5731, "\u0120increases": 5732, "ano": 5733, "\u0120specifically": 5734, "\u0120tele": 5735, "ensions": 5736, "\u0120religion": 5737, "abilities": 5738, "\u0120frame": 5739, "\u0120Note": 5740, "\u0120Lee": 5741, "\u0120helping": 5742, "\u0120edge": 5743, "oston": 5744, "\u0120organizations": 5745, "\u00c3\u0125": 5746, "\u0120Both": 5747, "hips": 5748, "\u0120bigger": 5749, "\u0120boost": 5750, "\u0120Stand": 5751, "\u0120row": 5752, "uls": 5753, "abase": 5754, "\u0120rid": 5755, "Let": 5756, "aren": 5757, "rave": 5758, "\u0120stret": 5759, "PD": 5760, "\u0120vision": 5761, "\u0120wearing": 5762, "\u0120appreci": 5763, "\u0120award": 5764, "\u0120Use": 5765, "\u0120factor": 5766, "war": 5767, "ulations": 5768, ")(": 5769, "\u0120god": 5770, "\u0120territ": 5771, "\u0120param": 5772, "asts": 5773, "87": 5774, "\u0120enemies": 5775, "\u0120Games": 5776, "FF": 5777, "\u0120accident": 5778, "Well": 5779, "\u0120Martin": 5780, "TER": 5781, "\u0120ath": 5782, "\u0120Hell": 5783, "\u0120forg": 5784, "\u0120veter": 5785, "\u0120Medic": 5786, "free": 5787, "\u0120stars": 5788, "\u0120expensive": 5789, "\u0120acad": 5790, "rawn": 5791, "\u0120Whe": 5792, "\u0120lock": 5793, "\u0120format": 5794, "\u0120soldiers": 5795, "sm": 5796, "\u0120agent": 5797, "\u0120responsibility": 5798, "ora": 5799, "\u0120Science": 5800, "\u0120rapid": 5801, "\u0120tough": 5802, "\u0120Jesus": 5803, "\u0120believes": 5804, "ML": 5805, "\u0120wear": 5806, "lete": 5807, "\u00c3\u0125\u00c3\u0124": 5808, "\u0120Dri": 5809, "\u0120commission": 5810, "\u0120Bob": 5811, "Oh": 5812, "aped": 5813, "\u0120warm": 5814, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 5815, "\u01202003": 5816, "ortion": 5817, "\u0120hasn": 5818, "uster": 5819, "\u0120univers": 5820, "\u0120Ill": 5821, "\u0120king": 5822, "ologies": 5823, "94": 5824, "\u0120Tem": 5825, "\u0120Mos": 5826, "\u0120patient": 5827, "\u0120Mexico": 5828, "cean": 5829, "\u0120Death": 5830, "\u0120Sanders": 5831, "you": 5832, "\u0120Cast": 5833, "\u0120Company": 5834, "pty": 5835, "\u0120happening": 5836, "FP": 5837, "\u0120Battle": 5838, "\u0120bought": 5839, "Am": 5840, "Mod": 5841, "Us": 5842, "uters": 5843, "\u0120Cre": 5844, "\u0120Those": 5845, "\u012044": 5846, "iser": 5847, "\u0120soul": 5848, "\u0120Top": 5849, "\u0120Harry": 5850, "\u0120Aw": 5851, "\u0120seat": 5852, "ffee": 5853, "\u0120revolution": 5854, "\u0120(\"": 5855, "\u0120During": 5856, "ette": 5857, "\u0120ring": 5858, "\u0120offensive": 5859, "\u0120returns": 5860, "\u0120videos": 5861, "\u0120discl": 5862, "\u0120famous": 5863, "enced": 5864, "\u0120Sign": 5865, "\u0120River": 5866, "\u0120300": 5867, "PM": 5868, "\u0120Bus": 5869, "\u0120CH": 5870, "\u0120candidates": 5871, "arden": 5872, "\u0120percentage": 5873, "\u0120visual": 5874, "\u0120thank": 5875, "\u0120trouble": 5876, "nergy": 5877, "\u01202001": 5878, "\u0120prove": 5879, "ashion": 5880, "\u0120enh": 5881, "\u0120Long": 5882, "UM": 5883, "\u0120connected": 5884, "\u0120possibility": 5885, "Over": 5886, "\u0120expert": 5887, "\u0120library": 5888, "arts": 5889, "\u0120Director": 5890, "\u0120fellow": 5891, "92": 5892, "irty": 5893, "\u0120dry": 5894, "\u0120signs": 5895, "\u0120Love": 5896, "\u0120quiet": 5897, "foot": 5898, "\u0120pure": 5899, "\u0120Hun": 5900, "\u0120filled": 5901, "phas": 5902, "\u0120Elect": 5903, "endment": 5904, "\u0120Expl": 5905, "\u0120unable": 5906, "ns": 5907, "mo": 5908, "\u0120vast": 5909, "obe": 5910, "\u0120identify": 5911, "apping": 5912, "\u0120Carolina": 5913, "gress": 5914, "\u0120prote": 5915, "\u0120fish": 5916, "\u0120circumstances": 5917, "razy": 5918, "\u0120Phot": 5919, "\u0120bodies": 5920, "\u0120Mur": 5921, "\u0120developing": 5922, "\u0120AR": 5923, "\u0120experienced": 5924, "\u0120substant": 5925, "\u0120Board": 5926, "esome": 5927, "\u0120domestic": 5928, "\u0120combined": 5929, "\u0120Put": 5930, "\u0120chemical": 5931, "\u0120Child": 5932, "\u0120pool": 5933, "\u0120Cy": 5934, "\u0120egg": 5935, "cons": 5936, "sters": 5937, "\u0120hurt": 5938, "\u0120markets": 5939, "\u0120conservative": 5940, "\u0120supporters": 5941, "\u0120agencies": 5942, "idel": 5943, "Ob": 5944, "urb": 5945, "\u012043": 5946, "\u0120Defense": 5947, "ye": 5948, "\u0120Ap": 5949, "dule": 5950, "\u0120temperature": 5951, "\u0120conducted": 5952, "\u0120Chief": 5953, "\u0120pulled": 5954, "\u0120fol": 5955, "Last": 5956, "onto": 5957, "osis": 5958, "VER": 5959, "Des": 5960, "\u0120Pan": 5961, "First": 5962, "\u0120advance": 5963, "\u0120license": 5964, "rors": 5965, "\u0120Jon": 5966, "\u0120imagine": 5967, "\u0120hell": 5968, "\u0120fixed": 5969, "\u0120incor": 5970, "osite": 5971, "\u0120Log": 5972, "icken": 5973, "]:": 5974, "\u0120surprise": 5975, "hab": 5976, "\u0120craft": 5977, "olt": 5978, "\u0120Jul": 5979, "\u0120dial": 5980, "\u0120relevant": 5981, "\u0120entered": 5982, "\u0120leads": 5983, "\u0120AD": 5984, "\u0120Clean": 5985, "\u0120pictures": 5986, "essor": 5987, "\u0120alt": 5988, "\u0120paying": 5989, "Per": 5990, "\u0120Market": 5991, "\u0120updates": 5992, "amily": 5993, "\u0120Type": 5994, "\u0120Home": 5995, "\u012055": 5996, "sembly": 5997, "rome": 5998, "83": 5999, "\u0120greatest": 6000, "\u0120height": 6001, "\u0120heav": 6002, "aints": 6003, "\u0120listen": 6004, "aser": 6005, "\u0120SH": 6006, "\u0120capable": 6007, "acle": 6008, "\u0120perspect": 6009, "inating": 6010, "\u0120offering": 6011, "rypt": 6012, "\u0120Develop": 6013, "abin": 6014, "rc": 6015, "\u0120bright": 6016, "alty": 6017, "arrow": 6018, "\u0120suppl": 6019, "inding": 6020, "acked": 6021, "gypt": 6022, "\u0120Another": 6023, "pg": 6024, "\u0120Virginia": 6025, "\u0120Lu": 6026, "\u0120planned": 6027, "\u0120pit": 6028, "\u0120sweet": 6029, "Type": 6030, "\u0120Di": 6031, "\u0120typically": 6032, "\u0120Francisco": 6033, "\u0120prospect": 6034, "\u0120Dan": 6035, "\u0120teen": 6036, "rees": 6037, "\u0120sched": 6038, "\u0120hol": 6039, "\u0120scr": 6040, "\u0120lots": 6041, "life": 6042, "\u0120newsp": 6043, "\u0120forget": 6044, "\u0120None": 6045, "\u0120Middle": 6046, "\u0120Ryan": 6047, "edd": 6048, "\u0120severe": 6049, "\u0120suit": 6050, "ller": 6051, "93": 6052, "\u0120correspond": 6053, "\u0120explos": 6054, "uations": 6055, "\u0120flag": 6056, "game": 6057, "rid": 6058, "\u0120prin": 6059, "\u0120Data": 6060, "\u0120deploy": 6061, "\u0120Enter": 6062, "suit": 6063, "ghan": 6064, "\u0120Men": 6065, "\u0120thoughts": 6066, "\u0120matters": 6067, "\u0120adapt": 6068, "\u0120Ari": 6069, "\u0120fill": 6070, "\u0120forth": 6071, "\u0120sam": 6072, "\u012041": 6073, "\u0120payment": 6074, "\u0120Hor": 6075, "\u0120spring": 6076, "duc": 6077, "\u0120losing": 6078, "\u0120bringing": 6079, "FO": 6080, "ala": 6081, "\u0120distribution": 6082, "hered": 6083, "bour": 6084, "\u0120Israeli": 6085, "oma": 6086, "\u0120combination": 6087, "\u0120plenty": 6088, "VE": 6089, "Can": 6090, "\u0120Haw": 6091, "\u0120perman": 6092, "\u0120Special": 6093, "\u0120tow": 6094, "\u0120seeking": 6095, "\u0120examples": 6096, "\u0120classes": 6097, "cr": 6098, "\u0120beer": 6099, "\u0120moves": 6100, "\u0120IP": 6101, "\u0120Kn": 6102, "\u0120panel": 6103, "Even": 6104, "\u0120properly": 6105, "\u0120ris": 6106, "\u0120plug": 6107, "\u0120estimated": 6108, "Every": 6109, "\u0120defensive": 6110, "agraph": 6111, "\u0120pregn": 6112, "\u0120instit": 6113, "\u0120Vict": 6114, "\u0120volume": 6115, "\u0120positions": 6116, "\u0120links": 6117, "\u0120Program": 6118, "\u0120Week": 6119, "agues": 6120, "\u0120transform": 6121, "ker": 6122, "\u0120CEO": 6123, "\u0120cas": 6124, "\u0120opponent": 6125, "\u0120tweet": 6126, "\u0120Code": 6127, "\u0120shop": 6128, "\u0120fly": 6129, "\u0120talks": 6130, "\u0120bag": 6131, "Phone": 6132, "\u0120aid": 6133, "\u0120plants": 6134, "\u012065": 6135, "\u0120attorney": 6136, "arters": 6137, "quest": 6138, "\u0120Magic": 6139, "\u0120begins": 6140, "\u0120myster": 6141, "\u0120environmental": 6142, "\u0120storage": 6143, "NN": 6144, "\u0120marg": 6145, "\u0120ske": 6146, "\u0120metal": 6147, "elly": 6148, "\u0120ordered": 6149, "\u0120remained": 6150, "\u0120loved": 6151, "\u0120prompt": 6152, "\u0120updated": 6153, "\u0120experts": 6154, "\u0120walking": 6155, "\u0120ancient": 6156, "\u0120performed": 6157, "ATE": 6158, "\u0120neither": 6159, "iency": 6160, "\u0120manufacture": 6161, "\u0120Pak": 6162, "\u0120selected": 6163, "\u0120mine": 6164, "\u0120ultimately": 6165, "\u0120explan": 6166, "\u0120label": 6167, "\u0120Services": 6168, "ributed": 6169, "Trump": 6170, "\u0120syn": 6171, "\u0120Ult": 6172, "SC": 6173, "\u0120meat": 6174, "\u0120giant": 6175, "\u0120Wars": 6176, "\u0120ON": 6177, "\u0120adm": 6178, "\u0120interpret": 6179, "\u0120evening": 6180, "\u0120evil": 6181, "\u0120Boston": 6182, "\u0120Wild": 6183, "\u0120\u00c3": 6184, "\u0120Bitcoin": 6185, "\u0120Amazon": 6186, "Dr": 6187, "\u0120Information": 6188, "\u0120obviously": 6189, "\u0120advanced": 6190, "Photo": 6191, "olar": 6192, "\u0120weather": 6193, "\u0120symbol": 6194, "\u0120sole": 6195, "\u0120potentially": 6196, "oster": 6197, "\u0120originally": 6198, "mun": 6199, "300": 6200, "aze": 6201, "essions": 6202, "\u0120deck": 6203, "\u0120stood": 6204, "\u0120youth": 6205, "\u0120Bern": 6206, "Rep": 6207, "\u0120Test": 6208, "\u0120basically": 6209, "otic": 6210, "\u0120involve": 6211, "olit": 6212, "lyn": 6213, "See": 6214, "\u0120aircraft": 6215, "\u0120confirm": 6216, "EW": 6217, "\u0120messages": 6218, "\u0120Richard": 6219, "\u0120kit": 6220, "\u0120prohib": 6221, "\u0120vulner": 6222, "isters": 6223, "\u0120existence": 6224, "\u0120turning": 6225, "\u0120SP": 6226, "\u0120desire": 6227, "\u0120flat": 6228, "\u0120ment": 6229, "season": 6230, "anges": 6231, "\u0120neighborhood": 6232, "\u0120Lake": 6233, "ATION": 6234, "\u0120pointed": 6235, "bur": 6236, "\u0120innov": 6237, "ucks": 6238, "UL": 6239, "\u0120professor": 6240, "\u0120expressed": 6241, "AB": 6242, "icious": 6243, "\u01202002": 6244, "\u0120Dev": 6245, "\u0120session": 6246, "\u0120bare": 6247, "sen": 6248, "\u0120diss": 6249, "\u0120Cath": 6250, "\u0120Pass": 6251, "\u0120Point": 6252, "\u0120doctor": 6253, "orrow": 6254, "ailed": 6255, "\u0120Rub": 6256, "\u0120DC": 6257, "\u0120Charl": 6258, "person": 6259, "\u0120writer": 6260, "ighters": 6261, "ureau": 6262, "\u0120oblig": 6263, "\u0120recorded": 6264, "\u0120broke": 6265, "\u0120orders": 6266, "ilty": 6267, "\u0120motion": 6268, "inity": 6269, "law": 6270, "adium": 6271, "\u0120immigration": 6272, "\u0120contrast": 6273, "\u0120batt": 6274, "\u0120excellent": 6275, "\u0120technical": 6276, "ami": 6277, "\u0120tun": 6278, "\u0120cloud": 6279, "\u0120Year": 6280, "geon": 6281, "\u0120creation": 6282, "\u0120strange": 6283, "\u0120auth": 6284, "\u0120fort": 6285, "born": 6286, "\u0120extent": 6287, "\u0120Today": 6288, "\u0120Club": 6289, "\u0120rain": 6290, "\u0120sample": 6291, "\u0120accepted": 6292, "\u0120tact": 6293, "\u0120fired": 6294, "\u0120Son": 6295, "\u0120stands": 6296, "\u0120boot": 6297, "\u012047": 6298, "\u0120statements": 6299, "\u0120versions": 6300, "\u0120selling": 6301, "ounded": 6302, "\u01201990": 6303, "\u0120weren": 6304, "\u0120Watch": 6305, "\u0120experiment": 6306, "Post": 6307, "\u0120retail": 6308, "uled": 6309, "Inst": 6310, "unte": 6311, "\u00e3\u0125\u00bc": 6312, "\u0120depart": 6313, "\u0120bond": 6314, "ivery": 6315, "ompl": 6316, "\u0120reaction": 6317, "\u0120Syrian": 6318, "\u0120Pac": 6319, "apped": 6320, "aniel": 6321, "DP": 6322, "\u0120resolution": 6323, "\u0120react": 6324, "\u0120approved": 6325, "onom": 6326, "mond": 6327, "\u0120Offic": 6328, "---": 6329, "\u0120replace": 6330, "\u0120tack": 6331, "\u0120sport": 6332, "\u0120chain": 6333, "\u0120emergency": 6334, "rad": 6335, "\u0120Palestin": 6336, "\u012046": 6337, "\u0120automatically": 6338, "\u0120route": 6339, "\u0120pal": 6340, "\u0120banks": 6341, "\u0120Paris": 6342, "\u0120Media": 6343, "road": 6344, "icing": 6345, "ixt": 6346, "isted": 6347, "\u0120grew": 6348, "\u0120coord": 6349, "\u0120Where": 6350, "omin": 6351, "\u0120subs": 6352, "\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 6353, "\u0120\u00c2\u00b1": 6354, "\u0120corporate": 6355, "\u0120selection": 6356, "noon": 6357, "\u0120Report": 6358, "cs": 6359, "cluding": 6360, "orders": 6361, "anche": 6362, "\u0120Its": 6363, "\u0120slowly": 6364, "\u0120Egypt": 6365, "\u0120Acc": 6366, "\u0120colle": 6367, "iques": 6368, "EX": 6369, "\u0120attempts": 6370, "url": 6371, "\u0120Cross": 6372, "\u0120findings": 6373, "\u0120SC": 6374, "\u0120OR": 6375, "\u0120index": 6376, "ensity": 6377, "\u0120Way": 6378, "\u0120Land": 6379, "\u0120shock": 6380, "dis": 6381, "\u0120dynam": 6382, "\u0120cart": 6383, "mosp": 6384, "Since": 6385, "iest": 6386, "\u0120Boy": 6387, "\u0120storm": 6388, "\u0120Contin": 6389, "2013": 6390, "hew": 6391, "ilit": 6392, "\u0120essential": 6393, "iquid": 6394, "Other": 6395, "ivered": 6396, "\u0120reasonable": 6397, "Act": 6398, "\u0120subsequ": 6399, "\u0120Pack": 6400, "\u0120Fort": 6401, "\u0120considering": 6402, "\u0120university": 6403, "log": 6404, "\u0120married": 6405, "\u0120illust": 6406, "\u0120True": 6407, "\u00a3\u0131": 6408, "\u0120numerous": 6409, "rastructure": 6410, "\u0120seriously": 6411, "\u0120referred": 6412, "ua": 6413, "\u0120consistent": 6414, "onna": 6415, "\u0120Real": 6416, "ruption": 6417, "ciples": 6418, "\u0120facts": 6419, "91": 6420, "otes": 6421, "erg": 6422, "Then": 6423, "\u0120accompl": 6424, "Note": 6425, "\u0120revenue": 6426, "\u0120passing": 6427, "\u0120mal": 6428, "een": 6429, "\u0120Yet": 6430, "\u0120gather": 6431, "terday": 6432, "ework": 6433, "\u0120Author": 6434, "Pe": 6435, "\u0120optim": 6436, "\u0120rub": 6437, "\u0120\u00e8\u00a3\u0131": 6438, "\u0120unknown": 6439, "stone": 6440, "\u0120union": 6441, "olve": 6442, "\u0120opportunities": 6443, "\u0120browser": 6444, "\u0120Wal": 6445, "\u0120Cost": 6446, "\u0120reporting": 6447, "sts": 6448, "pet": 6449, "\u0120sand": 6450, "\u0120suddenly": 6451, "\u0120surprising": 6452, "\u0120VR": 6453, "\u0120somewhat": 6454, "\u0120Bas": 6455, "ulture": 6456, "izz": 6457, "\u0120CD": 6458, "\u0120challenges": 6459, "\u0120settings": 6460, "\u0120experiences": 6461, "\u0120Full": 6462, "\u0120cann": 6463, "\u0120receiving": 6464, "EST": 6465, "\u0120joint": 6466, "\u0120cultural": 6467, "\u0120ast": 6468, "82": 6469, "astern": 6470, "ceived": 6471, "\u0120Cru": 6472, "\u0120bull": 6473, "pired": 6474, "amm": 6475, "\u0120facing": 6476, "power": 6477, "\u0120boss": 6478, "\u0120Hol": 6479, "\u0120instr": 6480, "\u0120increasingly": 6481, "\u0120shift": 6482, "\u0120streets": 6483, "\u0120Williams": 6484, "abb": 6485, "\u0120lie": 6486, "\u0120laugh": 6487, "\u0120Ca": 6488, "PL": 6489, "\u0120adults": 6490, "\u0120customer": 6491, "\u0120obtained": 6492, "\u0120supporting": 6493, "html": 6494, "fire": 6495, "\u0120detailed": 6496, "\u0120picked": 6497, "\u0120Right": 6498, "lder": 6499, "EE": 6500, "stood": 6501, "\u0120Kim": 6502, "\u0120wire": 6503, "\u0120sight": 6504, "\u0120developers": 6505, "\u0120persons": 6506, "\u0120sad": 6507, "\u0120cup": 6508, "\u0120warning": 6509, "\u0120boys": 6510, "long": 6511, "\u0120bird": 6512, "fo": 6513, "\u0120wal": 6514, "\u0120observed": 6515, "\u0120zone": 6516, "iveness": 6517, "\u0120channel": 6518, "cript": 6519, "\u0120refused": 6520, "\u0120Again": 6521, "\u0120suc": 6522, "\u0120spokesman": 6523, "\u0120Ref": 6524, "rite": 6525, "ouston": 6526, "\u00e3\u0125\u00b3": 6527, "\u0120Sher": 6528, "\u0120acts": 6529, "\u0120Name": 6530, "\u0120struggle": 6531, "arry": 6532, "ometimes": 6533, "\u0120discrim": 6534, "HT": 6535, "\u0120category": 6536, "\u0120realize": 6537, "\u0120employee": 6538, "\u0120Afghan": 6539, "enger": 6540, "\u0120guns": 6541, "\u0120Steve": 6542, "\u0120Mot": 6543, "\u0120Ol": 6544, "oked": 6545, "\u0120thick": 6546, "\u0120fairly": 6547, "illy": 6548, "\u0120surve": 6549, "\u0120Mat": 6550, "weight": 6551, "\u00e2\u0136": 6552, "\u0120troops": 6553, "\u0120agents": 6554, "\u0120battery": 6555, "\u0120motiv": 6556, "\u00c3\u00a1": 6557, "Sec": 6558, "den": 6559, "overy": 6560, "LS": 6561, "\u0120flu": 6562, "\u0120confident": 6563, "\u0120Oper": 6564, "\u0120empty": 6565, "\u0120phen": 6566, "\u0120sector": 6567, "\u0120excited": 6568, "\u0120remote": 6569, "aph": 6570, "oen": 6571, "\u0120destroyed": 6572, "\u0120moral": 6573, "\u0120HP": 6574, "\u0120Ron": 6575, "\u0120dress": 6576, "\u0120Bat": 6577, "\u0120lit": 6578, "\u0120MS": 6579, "\u0120af": 6580, "HL": 6581, "rum": 6582, "isms": 6583, "\u0120shouldn": 6584, "\u0120sympt": 6585, "\u0120Toronto": 6586, "hetic": 6587, "\u0120carbon": 6588, "\u0120installed": 6589, "\u0120violent": 6590, "\u0120solar": 6591, "ja": 6592, "\u0120practices": 6593, "\u0120ride": 6594, "\u0120Penn": 6595, "\u0120improved": 6596, "\u0120audio": 6597, "\u0120behavi": 6598, "\u0120PS": 6599, "\u0120eating": 6600, "Data": 6601, "\u0120Review": 6602, "pass": 6603, "claim": 6604, "uated": 6605, "angers": 6606, "chen": 6607, "\u0120properties": 6608, "\u0120anywhere": 6609, "Another": 6610, "\u0120blow": 6611, "\u0120Jackson": 6612, "\u0120proud": 6613, "\u0120plane": 6614, "lines": 6615, "\u0120square": 6616, "\u0120proof": 6617, "ansas": 6618, "\u0120talked": 6619, "makers": 6620, "\u0120sister": 6621, "\u0120holds": 6622, "\u0120resident": 6623, "\u0120==": 6624, "\u0120resistance": 6625, "\u0120split": 6626, "\u0120prosecut": 6627, "\u0120confidence": 6628, "resents": 6629, "\u0120cuts": 6630, "\u0120exception": 6631, "\u0120zero": 6632, "Getty": 6633, "\u0120copyright": 6634, "\u0120totally": 6635, "ormal": 6636, "ifications": 6637, "\u0120Australian": 6638, "\u0120sick": 6639, "\u0120150": 6640, "\u0120household": 6641, "\u0120fees": 6642, "\u0120drivers": 6643, "ogen": 6644, "\u0120NY": 6645, "\u0120necessarily": 6646, "\u0120regulations": 6647, "earing": 6648, "sl": 6649, "\u0120perspective": 6650, "care": 6651, "icial": 6652, "His": 6653, "\u0120escape": 6654, "\u0120surprised": 6655, "\u0120Van": 6656, "urrent": 6657, "\u0120vac": 6658, "81": 6659, "\u0120Thus": 6660, "\u0120emphas": 6661, "\u0120Champions": 6662, "\u0120Ice": 6663, "\u0120narr": 6664, "\u0120heads": 6665, "\u0120causing": 6666, "bel": 6667, "fortunately": 6668, "\u0120Ma": 6669, "\u0120targets": 6670, "cipl": 6671, "\u0120afternoon": 6672, "\u0120adds": 6673, "\u0120Maybe": 6674, "\u0120Four": 6675, "essed": 6676, "plete": 6677, "\u0120usual": 6678, "cho": 6679, "ingu": 6680, "\u0120withd": 6681, "\u0120Energy": 6682, "\u0120Econom": 6683, "OO": 6684, "\u0120articles": 6685, "\u0120injured": 6686, "\u0120manage": 6687, "\u0120explains": 6688, "\u0120diagn": 6689, "Rec": 6690, "atures": 6691, "\u0120linked": 6692, "\u0120discussed": 6693, "\u0120explo": 6694, "\u0120occasion": 6695, "athan": 6696, "\u0120opposite": 6697, "\u0120faces": 6698, "\u0120denied": 6699, "\u0120Knight": 6700, "\u0120nut": 6701, "\u0120approximately": 6702, "\u0120disappoint": 6703, "onymous": 6704, "\u0120Best": 6705, "\u0120Lo": 6706, "\u0120Hy": 6707, "\u0120Aff": 6708, "\u0120voting": 6709, "anwhile": 6710, "\u0120III": 6711, "\u0120institutions": 6712, "agram": 6713, "\u0120Daily": 6714, "\u0120drag": 6715, "\u0120nearby": 6716, "\u0120guilty": 6717, "\u0120conver": 6718, "Pre": 6719, "ship": 6720, "\u0120reward": 6721, "\u0120philosoph": 6722, "\u0120SS": 6723, "ugh": 6724, "\u0120apps": 6725, "friend": 6726, "\u0120upper": 6727, "\u0120advert": 6728, "\u0120snow": 6729, "\u0120frust": 6730, "\u0120ourselves": 6731, "Fr": 6732, "\u0120Die": 6733, "ampion": 6734, "\u0120dismiss": 6735, "\u0120cere": 6736, "\u0120signal": 6737, "from": 6738, "\u0120).": 6739, "\u012052": 6740, "\u0120crimes": 6741, "itors": 6742, "estival": 6743, "useum": 6744, "\u0120council": 6745, "\u0120Saud": 6746, "May": 6747, "\u0120Gun": 6748, "ician": 6749, "ether": 6750, "\u0120sufficient": 6751, "\u0120Hen": 6752, "sole": 6753, "\u0120historical": 6754, "\u0120Far": 6755, "\u0120Turn": 6756, "\u0120pin": 6757, "\u0120succeed": 6758, "mat": 6759, "lymp": 6760, "\u0120tradition": 6761, "\u0120Ok": 6762, "\u0120cro": 6763, "\u0120description": 6764, "alle": 6765, "\u0120sky": 6766, "Te": 6767, "\u0120widely": 6768, "\u0120wave": 6769, "\u0120definition": 6770, "\u0120Jews": 6771, "\u0120cycle": 6772, "\u0120refere": 6773, "\u0120brings": 6774, "usal": 6775, "\u0120alive": 6776, "\u0120frequently": 6777, "\u0120intention": 6778, "\u0120Control": 6779, "lv": 6780, "ystem": 6781, "\u0120privacy": 6782, "gent": 6783, "rence": 6784, "\u0120Quest": 6785, "\u0120Christmas": 6786, "\u0120rail": 6787, "\u0120cooper": 6788, "\u0120tested": 6789, "\u0120Capt": 6790, "asks": 6791, "\u0120comfortable": 6792, "\u0120delivered": 6793, "scape": 6794, "\u0120depth": 6795, "\u0120GOP": 6796, "\u0120writes": 6797, "\u0120assets": 6798, "\u0120sav": 6799, "iments": 6800, "\u0120transition": 6801, "\u0120artist": 6802, "\u0120Look": 6803, "\u0120lob": 6804, "\u0120components": 6805, "arity": 6806, "\u0120walked": 6807, "\u0120root": 6808, "\u0120participants": 6809, "\u0120noticed": 6810, "\u0120resc": 6811, "\u0120nav": 6812, "\u0120Administ": 6813, "da": 6814, "utral": 6815, "plate": 6816, "\u0120importance": 6817, "\u0120assert": 6818, "iously": 6819, "cription": 6820, "\u0120injuries": 6821, "\u0120Check": 6822, "\u0120registered": 6823, "\u0120intent": 6824, "\u0120missed": 6825, "ographic": 6826, "\u0120sentence": 6827, "ounter": 6828, "\u0120assistance": 6829, "evin": 6830, "\u0120database": 6831, "\u0120buildings": 6832, "\u0120classic": 6833, "\u0120thinks": 6834, "\u0120Ohio": 6835, "Pr": 6836, "ugg": 6837, "\u0120fee": 6838, "pan": 6839, "\u0120effectively": 6840, "\u0120facility": 6841, "\u0120bear": 6842, "\u0120chapter": 6843, "\u0120dogs": 6844, "\u0120Columb": 6845, "\u0120latter": 6846, "itial": 6847, "\u0120admitted": 6848, "TV": 6849, "\u0120Georg": 6850, "\u0120posts": 6851, "\\\\": 6852, "\u0120lawyer": 6853, "\u0120equival": 6854, "\u0120mand": 6855, "\u0120controlled": 6856, "\u0120Walk": 6857, "\u0120Andrew": 6858, "\u0120menu": 6859, "amental": 6860, "\u0120protected": 6861, "va": 6862, "\u0120administr": 6863, "oral": 6864, "\u0120rein": 6865, "\u0120Sar": 6866, "\u0120amounts": 6867, "\u0120native": 6868, "\u0120Moon": 6869, "\u0120represents": 6870, "\u0120abandon": 6871, "\u0120carrying": 6872, "\u0120tank": 6873, "mary": 6874, "\u0120declared": 6875, "Tube": 6876, "\u0120hat": 6877, "\u0120punish": 6878, "ellect": 6879, "mes": 6880, "\u0120universe": 6881, "\u0120Rod": 6882, "phy": 6883, "\u0120infrastructure": 6884, "\u012051": 6885, "\u0120opposed": 6886, "ownt": 6887, "ca": 6888, "\u0120Make": 6889, "\u0120hardware": 6890, "\u0120coffee": 6891, "Rel": 6892, "bal": 6893, "world": 6894, "\u0120Saf": 6895, "\u0120Sea": 6896, "inals": 6897, "\u0120owned": 6898, "\u0120hall": 6899, "ersion": 6900, "\u0120describe": 6901, "\u0120Pot": 6902, "\u0120portion": 6903, "\u0120atmosp": 6904, "\u0120governments": 6905, "\u0120depending": 6906, "\u0120offense": 6907, "\u0120trick": 6908, "awa": 6909, "\u0120Line": 6910, "\u0120Vis": 6911, "\u0120Hard": 6912, "\u0120Orig": 6913, "\u0120Click": 6914, "\u0120desk": 6915, "\u0120Valley": 6916, "\u0120Sov": 6917, "\u0120movies": 6918, "\u0120remark": 6919, "\u0120mail": 6920, "\u0120conscious": 6921, "\u0120ruling": 6922, "\u0120Rights": 6923, "\u0120medic": 6924, "hent": 6925, "\u0120Women": 6926, "><": 6927, "\u0120replaced": 6928, "\u0120Prem": 6929, "\u0120Thanks": 6930, "\u0120renew": 6931, "\u0120Ball": 6932, "iform": 6933, "\u0120shots": 6934, "Comm": 6935, "\u0120armed": 6936, "\u0120constant": 6937, "\u0120taste": 6938, "\u0120realized": 6939, "\u0120buff": 6940, "\u0120mo": 6941, "\u0120efficient": 6942, "Most": 6943, "oration": 6944, "ifies": 6945, "\u0120communication": 6946, "\u0120flood": 6947, "\u0120consequences": 6948, "\u0120anyway": 6949, "igg": 6950, "\u0120GM": 6951, "\u0120Thank": 6952, "\u0120iron": 6953, "\u0120evolution": 6954, "\u0120Cop": 6955, "twitter": 6956, "\u012095": 6957, "\u0120relationships": 6958, "adel": 6959, "\u0120Young": 6960, "\u0120proposal": 6961, "ayers": 6962, "uilding": 6963, "\u0120Hot": 6964, "ORE": 6965, "cos": 6966, "\u0120collabor": 6967, "PG": 6968, "axy": 6969, "\u0120knowing": 6970, "\u0120supports": 6971, "owed": 6972, "\u0120controls": 6973, "\u0120merely": 6974, "umer": 6975, "\u0120athlet": 6976, "\u0120fashion": 6977, "path": 6978, "\u0120gift": 6979, "\u0120era": 6980, "AND": 6981, "\u0120kinds": 6982, "\u0120Korean": 6983, "\u0120legit": 6984, "ulous": 6985, "\u0120essentially": 6986, "\u0120therap": 6987, "nic": 6988, "\u0120suffered": 6989, "\u0120hur": 6990, "\u0120promise": 6991, "\u0120excess": 6992, "\u0120overw": 6993, "\u0120prime": 6994, "\u0120Houston": 6995, "erry": 6996, "\u0120Ms": 6997, "RS": 6998, "2012": 6999, "\u0120stores": 7000, "\u0120Olymp": 7001, "\u0120journey": 7002, "Although": 7003, "Sub": 7004, "\u0120Educ": 7005, "\u0120Chapter": 7006, "\u0120requests": 7007, "\u0120consumers": 7008, "\u0120tiny": 7009, "\u0120isol": 7010, "\u0120Fair": 7011, "ba": 7012, "\u0120YOU": 7013, "\u0120crash": 7014, "celer": 7015, "\u0120emotional": 7016, "\u0120goods": 7017, "\u0120elected": 7018, "\u0120moder": 7019, "\u0120Linux": 7020, "\u0120blocks": 7021, "\u0120island": 7022, "\u0120Society": 7023, "\u0120elections": 7024, "\u0120broadcast": 7025, "\u0120cheap": 7026, "\u0120nations": 7027, "\u0120seasons": 7028, "400": 7029, "\u0120waste": 7030, "\u0120Sat": 7031, "\u0120fields": 7032, "employ": 7033, "\u0120profile": 7034, "\u0120authors": 7035, "ALL": 7036, "\u0120Gra": 7037, "west": 7038, "\u0120Ty": 7039, "\u0120deaths": 7040, "\u0120vacc": 7041, "\u0120formed": 7042, "\u0120du": 7043, "\u0120ongoing": 7044, "\u0120Muslims": 7045, "elf": 7046, "igure": 7047, "\u0120assume": 7048, "\u0120Ukraine": 7049, "water": 7050, "\u0120coast": 7051, "\u0120voted": 7052, "gor": 7053, "\u0120AS": 7054, "\u0120Michigan": 7055, "aza": 7056, "\u0120Arm": 7057, "iro": 7058, "\u0120flex": 7059, "asters": 7060, "''": 7061, "\u0120welcome": 7062, "arl": 7063, "\u0120locations": 7064, "igation": 7065, "\u0120Fil": 7066, "\u0120buying": 7067, "\u0120architect": 7068, "\u0120harder": 7069, "\u0120Cub": 7070, "\u0120interface": 7071, "\u0120restaurant": 7072, "\u0120discover": 7073, "\u0120exceed": 7074, "\u0120favour": 7075, "gery": 7076, "\u0120duty": 7077, "\u0120pitch": 7078, "ador": 7079, "\u0120Mach": 7080, "boy": 7081, "\u0120responded": 7082, "\u0120extended": 7083, "hers": 7084, "Many": 7085, "raid": 7086, "ifer": 7087, "\u0120Ins": 7088, "Ser": 7089, "\u0120medium": 7090, "she": 7091, "\u0120Sports": 7092, "\u0120magazine": 7093, "utation": 7094, "\u0120limits": 7095, "\u0120Gall": 7096, "\u0120external": 7097, "razil": 7098, "\u0120younger": 7099, "tle": 7100, "\u0120remind": 7101, "\u0120CON": 7102, "\u0120immediate": 7103, "\u0120hidden": 7104, "\u0120volunte": 7105, "\u0120simpl": 7106, "odcast": 7107, "\u0120phase": 7108, "dr": 7109, "\u0120plot": 7110, "\u0120exposure": 7111, "RI": 7112, "ograp": 7113, "vin": 7114, "anish": 7115, "\u0120Acad": 7116, "\u0120Engine": 7117, "\u0120expansion": 7118, "\u0120Pay": 7119, "Your": 7120, "\u0120pushed": 7121, "\u0120Ell": 7122, "\u0120Head": 7123, "\u0120marketing": 7124, "\u0120AC": 7125, "ket": 7126, "\u0120hits": 7127, "\u0120gro": 7128, "\u0120Age": 7129, "\u0120Scot": 7130, "][": 7131, "\u0120stim": 7132, "\u0120iPhone": 7133, "\u012a\u0134": 7134, "\u0120narrow": 7135, "\u0120Getty": 7136, "\u0120Turkey": 7137, "\u0120perfectly": 7138, "\u0120enable": 7139, "utch": 7140, "\u0120precise": 7141, "\u0120regime": 7142, "\u0120shif": 7143, "\u0120compens": 7144, "gun": 7145, "div": 7146, "\u0120chosen": 7147, "\u0120Ken": 7148, "Any": 7149, "\u0120trees": 7150, "\u0120recommended": 7151, "\u0120Ren": 7152, "uable": 7153, "\u0120HT": 7154, "Follow": 7155, "EG": 7156, "\u0120Hand": 7157, "\u0120Kenn": 7158, "\u0120arguments": 7159, "\u0120exists": 7160, "\u0120bike": 7161, "\u0120Conserv": 7162, "\u0120breaking": 7163, "\u0120Gar": 7164, "\u0120crazy": 7165, "\u0120virtual": 7166, "aylor": 7167, "ixel": 7168, "\u01201980": 7169, "\u0120permission": 7170, "\u0120Series": 7171, "\u0120consumer": 7172, "\u0120closely": 7173, "called": 7174, "\u012054": 7175, "\u0120hopes": 7176, "\u0120array": 7177, "\u0120Win": 7178, "\u0120Labour": 7179, "\u0120spons": 7180, "\u0120Ire": 7181, "\u0120pow": 7182, "\u0120readers": 7183, "\u0120employment": 7184, "\u0120creature": 7185, "\u0120resulting": 7186, "\u0120accurate": 7187, "\u0120moments": 7188, "\u0120argued": 7189, "\u0120ped": 7190, "During": 7191, "\u012053": 7192, "\u0120Tal": 7193, "\u0120sought": 7194, "\u0120suffering": 7195, "\u0120icon": 7196, "lee": 7197, "\u0120($": 7198, "alian": 7199, "\u00c2\u00b0": 7200, "\u0120pra": 7201, "\u0120bonus": 7202, "(\"": 7203, "ko": 7204, "\u0120acting": 7205, "DE": 7206, "fall": 7207, "\u0120comparison": 7208, "\u0120smooth": 7209, "\u0120NAS": 7210, "upp": 7211, "\u0120Joseph": 7212, "eping": 7213, "\u0120Take": 7214, "\u0120Mid": 7215, "\u0120sending": 7216, "fast": 7217, "\u0120Fall": 7218, "\u0120dealing": 7219, "user": 7220, "\u0120Organ": 7221, "Co": 7222, "\u0120attached": 7223, "\u0120sees": 7224, "%.": 7225, "\u0120typical": 7226, "ART": 7227, "\u0120finds": 7228, "\u0120Asia": 7229, "umin": 7230, "\u0120Core": 7231, "\u0120Ent": 7232, "inent": 7233, "uce": 7234, "\u0120Blood": 7235, "\u0120Never": 7236, "\u0120emails": 7237, "\u0120highlight": 7238, "\u0120confront": 7239, "atus": 7240, "uted": 7241, "\u0120unus": 7242, "\u0120topic": 7243, "\u0120Adam": 7244, "\u0120ble": 7245, "ati": 7246, "\u0120understood": 7247, "Set": 7248, "struct": 7249, "TP": 7250, "\u0120mob": 7251, "aa": 7252, "\u0120Start": 7253, "pected": 7254, "sell": 7255, "\u0120dedicated": 7256, "\u0120CA": 7257, "uan": 7258, "\u0120songs": 7259, "escription": 7260, "\u0120tech": 7261, "\u0120rape": 7262, "\u0120aside": 7263, "\u0120grant": 7264, "\u012056": 7265, "sub": 7266, "\u0120argue": 7267, "\u0120containing": 7268, "\u0120schedule": 7269, "\u0120liberal": 7270, "\u0120publicly": 7271, "\u0120heavily": 7272, "\u0120Ut": 7273, "iner": 7274, "\u0120Section": 7275, "\u0120Care": 7276, "weet": 7277, "ls": 7278, "Dis": 7279, "\u00e2\u0136\u0122": 7280, "\u0120Follow": 7281, "Back": 7282, "\u0120IT": 7283, "\u0120bes": 7284, "ji": 7285, "\u0120Hit": 7286, "ested": 7287, "\u0120everybody": 7288, "\u0120Swed": 7289, "\u0120femin": 7290, "\u0120facilities": 7291, "\u0120conven": 7292, "Comp": 7293, "\u0120OS": 7294, "core": 7295, "\u0120anx": 7296, "\u0120division": 7297, "\u0120Cam": 7298, "\u0120Stan": 7299, "mates": 7300, "\u0120explore": 7301, "plom": 7302, "\u0120shares": 7303, "pload": 7304, "anes": 7305, "\u0120ideal": 7306, "eters": 7307, "\u0120Base": 7308, "\u0120plastic": 7309, "\u0120distinct": 7310, "\u0120Network": 7311, "\u0120Seattle": 7312, "\u0120trading": 7313, "ensus": 7314, "intend": 7315, "\u0120exhib": 7316, "\u0120initially": 7317, "\u0120Food": 7318, "\u0120thousand": 7319, "\u0120Business": 7320, "acter": 7321, "\u0120paragraph": 7322, "\u0120roughly": 7323, "\u0120www": 7324, "\u0120creative": 7325, "\u0120Conf": 7326, "\u0120consumption": 7327, "\u0120films": 7328, "agan": 7329, "\u0120obtain": 7330, "\u0120tall": 7331, "\u0120tor": 7332, "\u0120acknowled": 7333, "\u0120grown": 7334, "alo": 7335, "KE": 7336, "\u0120400": 7337, "enders": 7338, "taining": 7339, "UG": 7340, "\u0120suicide": 7341, "\u0120watched": 7342, "\u0120List": 7343, "ali": 7344, "rehens": 7345, "\u0120surrounding": 7346, "\u0120pip": 7347, "\u0120flying": 7348, "\u0120Java": 7349, "ordan": 7350, "\u0120serving": 7351, "inations": 7352, "post": 7353, "\u0120sho": 7354, "Av": 7355, "\u0120jail": 7356, "zy": 7357, "\u01201999": 7358, "\u0120>": 9609, "orous": 9610, "\u0120firms": 9611, "screen": 9612, "una": 9613, "\u0120embarrass": 9614, "ulse": 9615, "\u0120letting": 9616, "\u0120threw": 9617, "iley": 9618, "\u0120channels": 9619, "lan": 9620, "\u0120Vegas": 9621, "\u0120sear": 9622, "\u0120fantastic": 9623, "arre": 9624, "uzzle": 9625, "\u0120Der": 9626, "Those": 9627, "\u0120swing": 9628, "\u0120sheet": 9629, "index": 9630, "cover": 9631, "ogan": 9632, "\u0120variables": 9633, "\u0120Tech": 9634, "\u0120spoken": 9635, "achel": 9636, "\u0120Da": 9637, "\u0120Mountain": 9638, "\u0120loaded": 9639, "\u0120footage": 9640, "version": 9641, "\u0120unl": 9642, "\u0120Phoenix": 9643, "\u0120throwing": 9644, "\u0120firing": 9645, "\u0120tracking": 9646, "\u0120width": 9647, "\u0120struggling": 9648, "rooms": 9649, "otion": 9650, "\u0120monthly": 9651, "\u0120Server": 9652, "\u0120eggs": 9653, "open": 9654, "MC": 9655, "\u01201993": 9656, "\u0120hired": 9657, "\u0120stayed": 9658, "\u0120Allen": 9659, "\u0120stro": 9660, "\u012098": 9661, "step": 9662, "\u0120Turkish": 9663, "\u0120fabric": 9664, "isting": 9665, "\u0120Dom": 9666, "\u0120dates": 9667, "\u0120pron": 9668, "\u0120basketball": 9669, "\u0120lucky": 9670, "\u0120Arabia": 9671, "\u0120assumed": 9672, "esty": 9673, "\u0120affairs": 9674, "\u0120glad": 9675, "\u0120Indeed": 9676, "\u0120FA": 9677, "\u0120Word": 9678, "\u0120joining": 9679, "ifice": 9680, "pread": 9681, "irts": 9682, "\u0120Select": 9683, "\u0120populations": 9684, "aware": 9685, "\u0120nose": 9686, "\u0120complaints": 9687, "start": 9688, "\u0120scoring": 9689, "Thanks": 9690, "\u0120mining": 9691, "\u0120visitors": 9692, "SH": 9693, "\u0120damaged": 9694, "\u0120characteristics": 9695, "\u0120Pent": 9696, "DC": 9697, "\u012083": 9698, "\u0120Six": 9699, "rates": 9700, "\u0120flags": 9701, "\u0120Brew": 9702, "dog": 9703, "Mark": 9704, "////": 9705, "\u0120execution": 9706, "\u0120joke": 9707, "phones": 9708, "\u0120testimony": 9709, "\u0120obst": 9710, "QL": 9711, "\u0120Cut": 9712, "\u0120studied": 9713, "\u0120Nintendo": 9714, "icket": 9715, "\u0120NBC": 9716, "\u0120lad": 9717, "\u0120Bra": 9718, "\u0120Moh": 9719, "\u0120kernel": 9720, "\u0120overwhelming": 9721, "\u0120aged": 9722, "\u0120applicable": 9723, "\u0120Cond": 9724, "\u0120roads": 9725, "\u0120Block": 9726, "made": 9727, "odge": 9728, "\u0120commands": 9729, "\u0120offices": 9730, "veland": 9731, "\u0120tut": 9732, "\u0120receiver": 9733, "\u0120Fro": 9734, "\u0120shopping": 9735, "\u0120iP": 9736, "\u0120Stre": 9737, "\u0120ABC": 9738, "\u0120entertainment": 9739, "\u0120Bow": 9740, "orted": 9741, "Mc": 9742, "\u0120reads": 9743, "grad": 9744, "\u0120Collect": 9745, "\u0120\u00e2\u012a\u0134": 9746, "\u0120Capital": 9747, "ederation": 9748, "\u0120employer": 9749, "\u0120involvement": 9750, "\u0120anxiety": 9751, "alia": 9752, "\u0120roof": 9753, "\u0120Among": 9754, "\u0120Democrat": 9755, "\u0120stats": 9756, "\u0120Vill": 9757, "\u0120constitutional": 9758, "\u0120referring": 9759, "itty": 9760, "\u0120tackle": 9761, "outube": 9762, "\u0120backed": 9763, "\u0120Hong": 9764, "\u0120Broad": 9765, "\u0120ele": 9766, "\u0120Ott": 9767, "\u01201992": 9768, "hour": 9769, "achusetts": 9770, "Cal": 9771, "\u0120defeated": 9772, "\u012081": 9773, "esp": 9774, "\u0120seemingly": 9775, "was": 9776, "\u0120Jenn": 9777, "\u0120Kurd": 9778, "\u0120gene": 9779, "\u0120discount": 9780, "Ret": 9781, "ECT": 9782, "();": 9783, "\u0120clubs": 9784, "\u0120sid": 9785, "\u0120Marsh": 9786, "Check": 9787, "\u0120pp": 9788, "\u0120Eag": 9789, "idespread": 9790, "\u0120beings": 9791, "FT": 9792, "\u0120introduction": 9793, "\u0120Change": 9794, "ARD": 9795, "\u0120110": 9796, "adows": 9797, "ierce": 9798, "\u0120meal": 9799, "author": 9800, "\u0120Bang": 9801, "lahoma": 9802, "\u0120ranks": 9803, "2011": 9804, "????": 9805, "max": 9806, "\u0120collapse": 9807, "\u0120opens": 9808, "\u0120echo": 9809, "\u0120soph": 9810, "\u0120racist": 9811, "\u0120enormous": 9812, "\u0120waves": 9813, "\u0120tap": 9814, "\u0120comprehensive": 9815, ".--": 9816, "\u0120Roy": 9817, "\u0120farmers": 9818, "Related": 9819, "aired": 9820, "rones": 9821, "\u0120Crim": 9822, "\u0120proportion": 9823, "\u0120designs": 9824, "\u0120negotiations": 9825, "\u0120virtually": 9826, "\u0120Batman": 9827, "\u0120warn": 9828, "\u0120legitimate": 9829, "mate": 9830, "\u0120convention": 9831, ",,": 9832, "netic": 9833, "\u0120SD": 9834, "\u0120consistently": 9835, "\u0120compensation": 9836, "\u0120punishment": 9837, "\u0120ye": 9838, "\u0120tie": 9839, "\u0120Bureau": 9840, "irlf": 9841, "\u0120Bu": 9842, "\u0120Aren": 9843, "\u0120Philipp": 9844, "\u0120knife": 9845, "\u0120memories": 9846, "\u0120Ross": 9847, "\u0120angle": 9848, "\u012086": 9849, "\u0120Thunder": 9850, "\u0120rend": 9851, "\u0120Tour": 9852, "\u0120counts": 9853, "sung": 9854, "\u0120Imp": 9855, "\u0120educational": 9856, "\u0120accessible": 9857, "COM": 9858, "\u0120drew": 9859, "yer": 9860, "Gl": 9861, "amine": 9862, "ORT": 9863, "OB": 9864, "IB": 9865, "master": 9866, "\u0120trials": 9867, "ogy": 9868, "har": 9869, "\u0120Trust": 9870, "\u0120preferred": 9871, "irlfriend": 9872, "\u0120Nev": 9873, "\u0120bin": 9874, "\u0120cow": 9875, "Page": 9876, "\u0120signature": 9877, "\u0120BL": 9878, "700": 9879, "\u0120retired": 9880, "\u0120bytes": 9881, "\u0120neighb": 9882, "\u0120Legend": 9883, "\u0120devast": 9884, "\u0120suspected": 9885, "isons": 9886, "\u0120Pok\u00c3\u00a9mon": 9887, "scale": 9888, "\u0120capabilities": 9889, "\u0120revel": 9890, "\u0120cheese": 9891, "dy": 9892, "igrant": 9893, "\u0120failing": 9894, "bits": 9895, "\u0120Heroes": 9896, "\u0120Ghost": 9897, "\u0120Scient": 9898, "\u0120appointed": 9899, "uri": 9900, "\u0120institution": 9901, "\u0120expanded": 9902, "greg": 9903, "\u0120monitoring": 9904, "\u0120podcast": 9905, "\u0120coalition": 9906, "\u012096": 9907, "Jo": 9908, "\u0120stolen": 9909, "\u0120Sab": 9910, "\u0120stops": 9911, "\u0120holiday": 9912, "\u0120intr": 9913, "Car": 9914, "Black": 9915, "\u0120LGBT": 9916, "\u0120warming": 9917, "\u0120Anderson": 9918, "\u012089": 9919, "\u0120producer": 9920, "Med": 9921, "\u0120accuracy": 9922, "\u0120Marvel": 9923, "izabeth": 9924, "\u0120Patrick": 9925, "mony": 9926, "\u0120mini": 9927, "acles": 9928, "\u0120overt": 9929, "they": 9930, "\u0120membership": 9931, "\u0120Ven": 9932, "\u0120exch": 9933, "\u0120removal": 9934, "\u0120Dave": 9935, "TY": 9936, "mad": 9937, "\u0120Find": 9938, "\u0120adequ": 9939, "\u0120ec": 9940, "\u0120teeth": 9941, "\u0120emotion": 9942, "\u0120perm": 9943, "\u0120solely": 9944, "db": 9945, "\u0120extraord": 9946, "IGHT": 9947, "cal": 9948, "\u0120guidelines": 9949, "\u0120dying": 9950, "\u0120suspended": 9951, "\u0120Premier": 9952, "\u0120Anthony": 9953, "elve": 9954, "\u0120dad": 9955, "\u0120Eth": 9956, "\u0120Football": 9957, "\u0120abandoned": 9958, "\u0120<<": 9959, "\u0120march": 9960, "\u0120horror": 9961, "\u00e2\u0122\u00a6\"": 9962, "\u0120childhood": 9963, "\u0120campaigns": 9964, "\u0120lunch": 9965, "\u0120Albert": 9966, "block": 9967, "\u00e2\u0138\u012a\u00e2\u0138\u012a": 9968, "ounding": 9969, "\u0120bone": 9970, "organ": 9971, "aders": 9972, "\u0120Flash": 9973, "\u0120Drive": 9974, "\u0120tonight": 9975, "\u0120wars": 9976, "\u0120FL": 9977, "\u0120formation": 9978, "const": 9979, "News": 9980, "\u0120compe": 9981, "orious": 9982, "\u0120Staff": 9983, "\u0120discussions": 9984, "\u0120Protection": 9985, "\u0120Jam": 9986, "\u0120criteria": 9987, "\u0120installation": 9988, "\u0120accomplish": 9989, "izza": 9990, "\u0120publisher": 9991, "\u0120rescue": 9992, "\u0120Try": 9993, "ULL": 9994, "\u0120Som": 9995, "\u0120Hop": 9996, "oret": 9997, "ths": 9998, "ordon": 9999, "\u0120pocket": 10000, "\u0120Inv": 10001, "Download": 10002, "\u0120Crime": 10003, "\u0120bene": 10004, "\u0120Guide": 10005, "\u0120Assembly": 10006, "\u0120parameters": 10007, "IE": 10008, "\u0120Alexander": 10009, "\u0120concert": 10010, "\u0120Sche": 10011, "\u0120shoes": 10012, "\u0120visiting": 10013, "\u0120recall": 10014, "\u0120bub": 10015, "\u0120rural": 10016, "\u0120concrete": 10017, "\u0120Ros": 10018, "Next": 10019, "Russ": 10020, "\u0120loans": 10021, "\u0120Shield": 10022, "\u0120trem": 10023, "hemat": 10024, "kg": 10025, "\u0120Harris": 10026, "isition": 10027, "\u0120Move": 10028, "\u0120FC": 10029, "\u0120fate": 10030, "\u0120Cho": 10031, "\u0120tired": 10032, "\u0120principal": 10033, "hist": 10034, "iences": 10035, "athy": 10036, "\u0120sevent": 10037, "\u0120mood": 10038, "\u0120strategic": 10039, "\u0120diseases": 10040, "\u0120forum": 10041, "\u0120tempor": 10042, "\u0120headquarters": 10043, "Par": 10044, "ige": 10045, "flix": 10046, "\u0120guitar": 10047, "\u012094": 10048, "Only": 10049, "\u0120releases": 10050, "roph": 10051, "================================": 10052, "\u0120600": 10053, "\u0120Continue": 10054, "igate": 10055, "\u0120Crit": 10056, "system": 10057, "\u0120disabled": 10058, "\u0120unexpected": 10059, "ithub": 10060, "\u0120unclear": 10061, "\u0120Est": 10062, "\u0120contrad": 10063, "\u0120strategies": 10064, "ventures": 10065, "\u0120passage": 10066, "AME": 10067, "\u0120improving": 10068, "\u0120reveals": 10069, "\u0120decrease": 10070, "ova": 10071, "\u0120annoy": 10072, "\u0120Short": 10073, "\u0120Library": 10074, "\u0120cyber": 10075, "nell": 10076, "\u0120Hur": 10077, "\u0120CB": 10078, "\u0120photograp": 10079, "UI": 10080, "\u0120sed": 10081, "Ge": 10082, "\u012087": 10083, "\u0120diverse": 10084, "\u0120encouraged": 10085, "\u0120conspiracy": 10086, "\u0120birds": 10087, "\u0120operator": 10088, "\u0120handful": 10089, "\u0120classified": 10090, "?)": 10091, "\u0120dramatic": 10092, "\u0120investigators": 10093, "ito": 10094, "\u0120widespread": 10095, "\u0120Room": 10096, "----------------------------------------------------------------": 10097, "\u0120collective": 10098, "\u0120journalist": 10099, "String": 10100, "\u0120temperatures": 10101, "ila": 10102, "\u0120guid": 10103, "\u0120inspect": 10104, "\u0120missile": 10105, "\u0120Mayor": 10106, "\u0120manual": 10107, "\u0120simultane": 10108, "\u0120ratings": 10109, "\u0120suck": 10110, "\u012097": 10111, "\u0120universal": 10112, "\u0120pharm": 10113, "\u0120disrupt": 10114, "iano": 10115, "AV": 10116, "\u0120ft": 10117, "\u0120statist": 10118, "olds": 10119, "\u0120Walker": 10120, "php": 10121, "\u0120undert": 10122, "\u0120Las": 10123, "ishop": 10124, "ntil": 10125, "reshold": 10126, "\u0120Whether": 10127, "Ms": 10128, "\u0120deny": 10129, "\u0120Cloud": 10130, "\u0120provider": 10131, "\u0120surviv": 10132, "\u0120Update": 10133, "has": 10134, "\u0120mistakes": 10135, "charge": 10136, "pled": 10137, "rity": 10138, "\u0120node": 10139, "\u0120Massachusetts": 10140, "ools": 10141, "lication": 10142, "\u0120fails": 10143, "emale": 10144, "ori": 10145, "backs": 10146, "\u0120shirt": 10147, "\u0120''": 10148, "\u0120NAT": 10149, "\u0120waters": 10150, "elson": 10151, "\u0120ease": 10152, "\u0120scar": 10153, "\u0120contents": 10154, "mind": 10155, "\u0120contribution": 10156, "\u0120shr": 10157, "\u0120handed": 10158, "\u0120stability": 10159, "\u0120trave": 10160, "Em": 10161, "\u0120mirror": 10162, "123": 10163, "\u0120weigh": 10164, "\u0120fiction": 10165, "ouver": 10166, "istant": 10167, "rition": 10168, "\u0120Fed": 10169, "\u0120physically": 10170, "\u0120stake": 10171, "\u0120Article": 10172, "\u0120Arc": 10173, "\u0120Lewis": 10174, "\u0120Mind": 10175, "\u0120demonstrate": 10176, "\u0120profits": 10177, "vision": 10178, "omic": 10179, "olid": 10180, "\u0120battles": 10181, "\u0120drives": 10182, "\u0120eastern": 10183, "\u0120Sony": 10184, "!!!": 10185, "aration": 10186, "vard": 10187, "\u0120GL": 10188, "portation": 10189, "\u012092": 10190, "\u0120lawmakers": 10191, "\u0120protecting": 10192, "\u0120EPA": 10193, "\u0120yeah": 10194, "\u0120shame": 10195, "olph": 10196, "even": 10197, "xit": 10198, "\u0120attach": 10199, "\u0120representing": 10200, "\u0120obs": 10201, "\u0120Utah": 10202, "iffs": 10203, "\u0120Freedom": 10204, "\u00c3\u00b3": 10205, "AK": 10206, "\u0120incidents": 10207, "itage": 10208, "\u0120viewers": 10209, "cd": 10210, "\u0120mouse": 10211, "\u0120clar": 10212, "\u0120accordance": 10213, "\u0120bot": 10214, "cor": 10215, "\u0120Summer": 10216, "held": 10217, "\u0120innocent": 10218, "\u0120initiative": 10219, "ols": 10220, "________________________________": 10221, "\u0120spots": 10222, "pace": 10223, "\u0120conventional": 10224, "\u0120corporations": 10225, "\u0120blocked": 10226, "HD": 10227, "attered": 10228, "\u0120refers": 10229, "\u0120buck": 10230, "\u0120Digital": 10231, "120": 10232, "\u0120topics": 10233, "TF": 10234, "\u00c4\u0123": 10235, "brid": 10236, "reement": 10237, "\u0120underlying": 10238, "\u0120Member": 10239, "\u0120investigating": 10240, "\u0120pregnancy": 10241, "\u0120touchdown": 10242, "\u0120Band": 10243, "\u0120Caller": 10244, "\u0120instances": 10245, "PP": 10246, "wa": 10247, "Good": 10248, "\u01201991": 10249, "\u0120Cold": 10250, "\u0120fears": 10251, "\u0120remarks": 10252, "\u0128\u0134": 10253, "atal": 10254, "\u0120mit": 10255, "\u0120experiments": 10256, "ipt": 10257, "Color": 10258, "indu": 10259, "Update": 10260, "\u012093": 10261, "Ag": 10262, "\u0120\u00e5": 10263, "ancouver": 10264, "Both": 10265, "\u0120judges": 10266, "Object": 10267, "\u0120stere": 10268, "umbn": 10269, "\u0120participation": 10270, "\u0120Stars": 10271, "\u0120Jere": 10272, "\u0120weekly": 10273, "\u0120Ban": 10274, "\u0120conversations": 10275, "\u0120Pitt": 10276, "uz": 10277, "\u0120Indiana": 10278, "\u0120Kick": 10279, "\u0120infection": 10280, "\u0120heroes": 10281, "\u0120settled": 10282, "\u0120strip": 10283, "\u0120hal": 10284, "\u0120dump": 10285, "\u0120Sci": 10286, "\u0120les": 10287, "\u0120references": 10288, "\u0120URL": 10289, "\u0120Bridge": 10290, "\u0120wanting": 10291, "Force": 10292, "\u0120exclus": 10293, "Meanwhile": 10294, "mn": 10295, "\u0120gentle": 10296, "maker": 10297, "senal": 10298, "\u0120Gro": 10299, "ouri": 10300, "\u0120Rain": 10301, "\u0120Alliance": 10302, "\u0120lift": 10303, "ela": 10304, "SD": 10305, "\u0120Cleveland": 10306, "\u0120ranked": 10307, "\u0120stadium": 10308, "\u0120deadly": 10309, "\u00e4\u00b8": 10310, "\u0120riding": 10311, "aria": 10312, "\u0120Armor": 10313, "\u0120documentation": 10314, "\u0120Greece": 10315, "reek": 10316, "\u0120lens": 10317, "\u0120Sa": 10318, "\u0120gross": 10319, "\u0120Emer": 10320, "agers": 10321, "\u0120Dub": 10322, "\u0120Rh": 10323, "\u0120AMD": 10324, "\u0120arrival": 10325, "\u0120desert": 10326, "\u0120supplement": 10327, "\u0120Resp": 10328, "\u0120knee": 10329, "\u0120margin": 10330, "font": 10331, "ogg": 10332, "2010": 10333, "\u0120Pir": 10334, "\u0120Prom": 10335, "ivals": 10336, "\u0120intake": 10337, "\u0120differently": 10338, "ugs": 10339, "\u0120bits": 10340, "cluded": 10341, "\u0120searching": 10342, "\u0120Du": 10343, "umble": 10344, "\u0120functional": 10345, "\u0120Baltimore": 10346, "\u0120Could": 10347, "\u0120desired": 10348, "\u0120circuit": 10349, "\u0120Lyn": 10350, "\u0120GO": 10351, "\u0120False": 10352, "repre": 10353, "':": 10354, "alties": 10355, "\u0120minim": 10356, "\u0120drove": 10357, "\u0120Should": 10358, "\u0120hip": 10359, "\u0120pros": 10360, "\u0120utility": 10361, "\u0120Nature": 10362, "\u0120Mode": 10363, "President": 10364, "opp": 10365, "rat": 10366, "formance": 10367, "\u0120concentration": 10368, "\u0120font": 10369, "\u0120Bud": 10370, "\u0120amid": 10371, "\u0120revers": 10372, "\u0120ML": 10373, "Bar": 10374, "\u0120interaction": 10375, "\u0120jurisd": 10376, "\u0120spells": 10377, "dep": 10378, "fil": 10379, "\u0120civilians": 10380, "utter": 10381, "\u0120Cooper": 10382, "\u0120Below": 10383, "\u0120entrance": 10384, "\u0120convert": 10385, "\u0120controversy": 10386, "owered": 10387, "\u0120contrary": 10388, "\u0120arc": 10389, "\u0120Executive": 10390, "\u0120Officer": 10391, "\u0120packages": 10392, "\u0120progressive": 10393, "width": 10394, "\u0120reserved": 10395, "vol": 10396, "\u0120Samsung": 10397, "\u0120printed": 10398, "\u0120centers": 10399, "\u0120introduce": 10400, "\u0120Kennedy": 10401, "\u0120odds": 10402, "\u0120surely": 10403, "\u0120independence": 10404, "\u0120passengers": 10405, "reprene": 10406, "\u0120Beh": 10407, "\u0120loves": 10408, "\u0120ESPN": 10409, "\u0120facilit": 10410, "\u0120identical": 10411, "\u0120doct": 10412, "\u0120partnership": 10413, "conf": 10414, "\u0120Hide": 10415, "\u0120confused": 10416, "\u0120Cow": 10417, "Men": 10418, "\u0120wrest": 10419, "\u0120Iraqi": 10420, "\u0120holes": 10421, "\u0120Studies": 10422, "\u0120pregnant": 10423, "hard": 10424, "\u0120signals": 10425, "IX": 10426, "\u0120pulling": 10427, "\u0120graduate": 10428, "\u0120nominee": 10429, "Date": 10430, "\u0120permitted": 10431, "\u0120\u00e2\u0124\u00ac": 10432, "\u0120Oklahoma": 10433, "Start": 10434, "\u0120authorized": 10435, "\u0120alarm": 10436, "\u0120Cos": 10437, "van": 10438, "\u0120generations": 10439, "cular": 10440, "\u0120dragon": 10441, "\u0120Software": 10442, "\u0120Edward": 10443, "\u0120controller": 10444, "Sen": 10445, "gered": 10446, "\u0120Vik": 10447, "\u0120approached": 10448, "Thank": 10449, "\u0120cance": 10450, "\u0120formula": 10451, "\u0120Small": 10452, "\u0120weakness": 10453, "\u0120ramp": 10454, "itudes": 10455, "jud": 10456, "\u0120brilliant": 10457, "\u0120accus": 10458, "source": 10459, "\u0120800": 10460, "\u0120Evil": 10461, "Sw": 10462, "\u0120homeless": 10463, "week": 10464, "iens": 10465, "rics": 10466, "\u0120Third": 10467, "TO": 10468, "\u0120organic": 10469, "\u0120presentation": 10470, "agh": 10471, "\u0120Download": 10472, "vation": 10473, "\u0120assembly": 10474, "orable": 10475, "holders": 10476, "\u0120Bernie": 10477, "\u0120Help": 10478, "\u0120tong": 10479, "\u0120Fight": 10480, "\u0120beach": 10481, "Book": 10482, "\u0120Lic": 10483, "\u0120rush": 10484, "\u0120Round": 10485, "oup": 10486, "\u0120Marx": 10487, "\u0120calculated": 10488, "\u0120Devil": 10489, "\u0120Sarah": 10490, "\u0120occasionally": 10491, "\u0120bullet": 10492, "Available": 10493, "gate": 10494, "\u012091": 10495, "\u0120hosp": 10496, "\u0120promises": 10497, "\u0120HIV": 10498, "\u0120Stadium": 10499, "\u0120Stock": 10500, "\u0120Corporation": 10501, "gage": 10502, "NG": 10503, "\u0120Credit": 10504, "\u0120sne": 10505, "ibl": 10506, "\u0120accum": 10507, "such": 10508, "\u0120terrorists": 10509, "\u0120consciousness": 10510, "\u0120Zh": 10511, "\u0120drama": 10512, "oola": 10513, "piration": 10514, "\u0120labour": 10515, "\u0120Nin": 10516, "\u0120utter": 10517, "\u0120democratic": 10518, "\u0120assass": 10519, "ilation": 10520, "\u0120gest": 10521, "\u0120abroad": 10522, "\u0120metab": 10523, "\u0120sorts": 10524, "\u0120flav": 10525, "UB": 10526, "\u0120mg": 10527, "\u0120Nothing": 10528, "\u0120Od": 10529, "\u0120musical": 10530, "2009": 10531, "\u0120drops": 10532, "ocated": 10533, "ateral": 10534, "000000": 10535, "\u0120gre": 10536, "\u0120equality": 10537, "\u0120burden": 10538, "\u0120vig": 10539, "\u0120Leader": 10540, "------------": 10541, "\u0120ceremony": 10542, "\u0120fighter": 10543, "\u0120actors": 10544, "\u0120\u00e6": 10545, "aman": 10546, "Fi": 10547, "\u0120align": 10548, "puter": 10549, "\u0120elder": 10550, "\u0120NSA": 10551, "\u0120representation": 10552, "\u0120Ontario": 10553, "ITH": 10554, "usalem": 10555, "\u0120harassment": 10556, "itzer": 10557, "\u0120symp": 10558, "\u0120boxes": 10559, "\u0120DR": 10560, "\u0120manifest": 10561, "atre": 10562, "\u0120^": 10563, "\u0120dies": 10564, "leton": 10565, "\u0120missions": 10566, "ethe": 10567, "\u0120resolve": 10568, "\u0120followers": 10569, "\u0120asc": 10570, "\u0120km": 10571, "lord": 10572, "ammed": 10573, "\u0120silent": 10574, "\u0120Associated": 10575, "\u0120timing": 10576, "\u0120prisoners": 10577, "\u0120Kings": 10578, "\u0120Five": 10579, "\u0120tower": 10580, "\u0120approaches": 10581, "\u0120precisely": 10582, "\u0120bureau": 10583, "\u0120Mother": 10584, "\u0120Iss": 10585, "\u0120keyboard": 10586, "itual": 10587, "\u0120funded": 10588, "\u0120staying": 10589, "\u0120psychological": 10590, "\u0120mile": 10591, "\u0120Leon": 10592, "\u0120Barb": 10593, "will": 10594, "\u0120wider": 10595, "\u0120Atlantic": 10596, "\u0120till": 10597, "\u0120Rome": 10598, "rot": 10599, "\u0120accompan": 10600, "\u0120flour": 10601, "aco": 10602, "World": 10603, "\u0120Express": 10604, "\u0120Yu": 10605, "Cor": 10606, "\u0120pleased": 10607, "party": 10608, "\u0120pointing": 10609, "\u0120inflation": 10610, "\u0120roy": 10611, "\u0120),": 10612, "ainer": 10613, "\u0120wedding": 10614, "ormon": 10615, "\u0120requiring": 10616, "\u0120qualified": 10617, "\u0120segment": 10618, "END": 10619, "\u0120sizes": 10620, "eals": 10621, "\u0120corrupt": 10622, "assador": 10623, "\u0120celeb": 10624, "\u0120dreams": 10625, "\u0120Mess": 10626, "\u0120checking": 10627, "\u0120Version": 10628, "\u0120preparing": 10629, "\u0120actively": 10630, "\u0120Diff": 10631, "\u0120lux": 10632, "\u0120Winter": 10633, "acteria": 10634, "\u0120NE": 10635, "\u0120deputy": 10636, "\u0120transgender": 10637, "\u0120summary": 10638, "\u0120inher": 10639, "eries": 10640, "char": 10641, "\u0120Yan": 10642, "\u0120knock": 10643, "\u0120Path": 10644, "\u0120lip": 10645, "roller": 10646, "\u0120impression": 10647, "\u0120celebrate": 10648, "\u0120slide": 10649, "\u0120guests": 10650, "\u0120clip": 10651, "FS": 10652, "\u0120savings": 10653, "\u0120captain": 10654, "\u0120legacy": 10655, "\u0120Denver": 10656, "\u0120wounded": 10657, "taboola": 10658, "ACT": 10659, "\u0120pursue": 10660, "\u0120oxy": 10661, "\u0120q": 10662, "\u0120semi": 10663, "\u0120Need": 10664, "\u0120Affairs": 10665, "\u0120obsc": 10666, "\u0120checked": 10667, "\u0120dual": 10668, "Code": 10669, "\u0120MD": 10670, "lem": 10671, "ulty": 10672, "\u0120\u00c2\u00a9": 10673, "\u0120Elizabeth": 10674, "\u0120centuries": 10675, "arded": 10676, "src": 10677, "\u0120evident": 10678, "ennis": 10679, "atin": 10680, "\u0120unemployment": 10681, "\u0120Mario": 10682, "\u0120intim": 10683, "Christ": 10684, "\u0120biological": 10685, "\u0120soldier": 10686, "\u0120Added": 10687, "\u0120math": 10688, "\u0120Gil": 10689, "\u0120bias": 10690, "\u0120dating": 10691, "\u0120Ocean": 10692, "\u0120mice": 10693, "Mus": 10694, "hire": 10695, "\u0120Tes": 10696, "Server": 10697, "limited": 10698, "Size": 10699, "\u0120meters": 10700, "\u0120rocket": 10701, "essee": 10702, "\u0120certificate": 10703, "\u0120Iranian": 10704, "ASS": 10705, "\u0120grid": 10706, "Dec": 10707, "\u0120rolling": 10708, "commun": 10709, "\u0120Sweden": 10710, "bury": 10711, "\u0120tissue": 10712, "\u0120racism": 10713, "\u0120Local": 10714, "\u0120mystery": 10715, "\u0120examine": 10716, "\u0120stem": 10717, "\u0120sits": 10718, "\u0120hoped": 10719, "oting": 10720, "\u0120dialogue": 10721, "\u0120persu": 10722, "Watch": 10723, "lay": 10724, "MAN": 10725, "\u0120chronic": 10726, "\u0120Portland": 10727, "market": 10728, "\u0120SEC": 10729, "\u0120parallel": 10730, "\u0120scandal": 10731, "\u0120carries": 10732, "\u0120phenomenon": 10733, "human": 10734, "acker": 10735, "\u0120Ox": 10736, "\u0120retirement": 10737, "tainment": 10738, "ovie": 10739, "\u0120Gear": 10740, "\u0120duties": 10741, "\u0120dose": 10742, "\u0120scroll": 10743, "MB": 10744, "inf": 10745, "\u0120sauce": 10746, "\u0120landscape": 10747, "reddit": 10748, "\u0120Championship": 10749, "\u0120Reddit": 10750, "alid": 10751, "\u0120coin": 10752, "\u0120overs": 10753, "\u0120posting": 10754, "about": 10755, "\u0120fel": 10756, "andy": 10757, "\u0120bold": 10758, "\u0120focusing": 10759, "effect": 10760, "GR": 10761, "\u0120deemed": 10762, "\u0120recommendations": 10763, "\u0120stepped": 10764, "\u0120voter": 10765, "\u0120Deep": 10766, "\u0120Instagram": 10767, "\u0120moderate": 10768, "\u0120Maryland": 10769, "\u0120restricted": 10770, "\u0120MB": 10771, "\u0120Chall": 10772, "\u0120tob": 10773, "\u0120cir": 10774, "\u0120Occ": 10775, "\u0120Ever": 10776, "\u0120collaps": 10777, "INFO": 10778, "=-": 10779, "\u0120Pict": 10780, "\u0120Account": 10781, "nc": 10782, "\u0120ought": 10783, "\u0120export": 10784, "\u0120drunk": 10785, "('": 10786, "\u0120wise": 10787, "\u0120Mort": 10788, "necess": 10789, "\u0120ancest": 10790, "\u0120Incre": 10791, "\u0120frequent": 10792, "mir": 10793, "\u0120interpretation": 10794, "\u0120dependent": 10795, "\u0120coins": 10796, "\u0120Bol": 10797, "Video": 10798, "\u0120Justin": 10799, "\u0120fatal": 10800, "\u0120cooking": 10801, "\u0120confusion": 10802, "ipher": 10803, "\u0120custody": 10804, "\u0120Morgan": 10805, "omach": 10806, "\u0120Governor": 10807, "\u0120restaurants": 10808, "eling": 10809, "\u0120acknowledged": 10810, "\u0120ther": 10811, "\u0120genes": 10812, "ching": 10813, "Hey": 10814, "\u0120tactics": 10815, "\u0120Mexican": 10816, "\u0120vend": 10817, "\u0120hes": 10818, "quer": 10819, "\u0120noting": 10820, "\u0120Cameron": 10821, "\u0120targeting": 10822, "rock": 10823, "\u0120credits": 10824, "\u0120emotions": 10825, "\u0120representatives": 10826, "news": 10827, "\u0120legislative": 10828, "\u0120removing": 10829, "\u0120tweeted": 10830, "\u0120Carter": 10831, "\u0120Fixed": 10832, "\u0120forcing": 10833, "\u0120speaker": 10834, "\u0120males": 10835, "\u0120Vietnam": 10836, "lined": 10837, "\u0120concepts": 10838, "\u0120voices": 10839, "oir": 10840, "\u0120Trib": 10841, "Whe": 10842, "\u0120Jerusalem": 10843, "\u0120Sant": 10844, "\u0120cul": 10845, "\u0120lady": 10846, "\u0120Hawai": 10847, "\u0120arts": 10848, "\u0120Inn": 10849, "\u0120Machine": 10850, "\u0120Emperor": 10851, "\u0120slot": 10852, "gly": 10853, "\u0120Process": 10854, "III": 10855, "\u0120athletes": 10856, "\u0120Temple": 10857, "\u0120Represent": 10858, "\u0120presc": 10859, "\u0120tons": 10860, "\u0120golden": 10861, "\u0120punch": 10862, "\u0120GR": 10863, "iverpool": 10864, "\u0120enact": 10865, "\u0120lobby": 10866, "\u0120mos": 10867, "\u0120picking": 10868, "\u0120lifetime": 10869, "\u0120cognitive": 10870, "Each": 10871, "zo": 10872, "\u0120dub": 10873, "\u0120consists": 10874, "oln": 10875, "\u0120festival": 10876, "amous": 10877, "\u0120intellig": 10878, "words": 10879, "\u0120Smart": 10880, "\u0120dele": 10881, "\u0120lapt": 10882, "\u0120magical": 10883, "\u0120Sin": 10884, "bus": 10885, "urities": 10886, "ighth": 10887, "\u0120Ruby": 10888, "\u0120Sure": 10889, "olving": 10890, "\u0120jun": 10891, "OST": 10892, "\u0120imposed": 10893, "\u0120astron": 10894, "\u0120correl": 10895, "\u0120NS": 10896, "\u0120Kit": 10897, "\u0120Future": 10898, "burn": 10899, "\u0120immune": 10900, "ocus": 10901, "\u0120courses": 10902, "\u0120String": 10903, "\u0120lean": 10904, "\u0120ghost": 10905, "\u0120outcomes": 10906, "\u0120expense": 10907, "\u0120everyday": 10908, "\u0120acceptable": 10909, "Ah": 10910, "\u0120equipped": 10911, "\u0120orange": 10912, "FR": 10913, "\u0120Dutch": 10914, "Though": 10915, "\u0120Rank": 10916, "QU": 10917, "\u0120Roberts": 10918, "what": 10919, "rend": 10920, "\u0120disappear": 10921, "\u0120spawn": 10922, "\u0120Lam": 10923, "ois": 10924, "\u0120deserve": 10925, "\u0120minimal": 10926, "\u0120nervous": 10927, "\u0120Would": 10928, "\u0120rook": 10929, "\u0120Vancouver": 10930, "\u0120resign": 10931, "shire": 10932, "\u0120Works": 10933, "\u0120Build": 10934, "\u0120affordable": 10935, "\u0120Gary": 10936, "\u0120Arena": 10937, "\u0120hanging": 10938, "\u0120implications": 10939, "\u0120Song": 10940, "\u0120maintaining": 10941, "\u0120guards": 10942, "CON": 10943, "\u0120derived": 10944, "\u0120executed": 10945, "\u0120theories": 10946, "\u0120quoted": 10947, "\u0120Andre": 10948, "oga": 10949, "seless": 10950, "info": 10951, "\u0120Belg": 10952, "\u0120tears": 10953, "\u0120Surv": 10954, "\u0120birthday": 10955, "igious": 10956, "immer": 10957, "\u0120spectrum": 10958, "\u0120architecture": 10959, "\u0120recruit": 10960, "arma": 10961, "Table": 10962, "\u0120monsters": 10963, "\u0120Gov": 10964, "\u0120destination": 10965, "\u0120attractive": 10966, "\u0120foss": 10967, "\u0120Moreover": 10968, "\u0120presents": 10969, "THE": 10970, "\u0120reply": 10971, "pton": 10972, "\u0120cum": 10973, "\u0120delight": 10974, "\u0120affects": 10975, "\u0120donations": 10976, "\u0120Toy": 10977, "\u0120Him": 10978, "MENT": 10979, "\u0120overcome": 10980, "itched": 10981, "\u0120Fantasy": 10982, "\u0120Hat": 10983, "\u0120Beast": 10984, "bott": 10985, "\u0120investigations": 10986, "Run": 10987, "\u0120hunting": 10988, "di": 10989, "fund": 10990, "\u0120sessions": 10991, "estyle": 10992, "\u0120portray": 10993, "oids": 10994, "Yeah": 10995, "\u0120communicate": 10996, "\u0120comedy": 10997, "\u0120Yang": 10998, "\u0120belt": 10999, "\u0120Marine": 11000, "\u0120predicted": 11001, "Play": 11002, "\u0120importantly": 11003, "\u0120remarkable": 11004, "\u0120eliminate": 11005, "David": 11006, "\u0120bind": 11007, "VID": 11008, "\u0120advocates": 11009, "\u0120Gaza": 11010, "imp": 11011, "DB": 11012, "\u0120Na": 11013, "\u0120Similar": 11014, "IES": 11015, "\u0120charity": 11016, "vas": 11017, "math": 11018, "\u0120\u00e2\u0138": 11019, "oker": 11020, "ndum": 11021, "\u0120caps": 11022, "\u0120Hal": 11023, "2000": 11024, "ean": 11025, "\u0120fleet": 11026, "\u0120recre": 11027, "Right": 11028, "\u0120sleeping": 11029, "ijing": 11030, "kind": 11031, "\u0120designated": 11032, "\u00c3\u00a4": 11033, "\u0120animation": 11034, "kee": 11035, "\u0120Introdu": 11036, "\u0120/>": 11037, "\u0120delayed": 11038, "\u0120tremend": 11039, "\u0120curious": 11040, "Use": 11041, "\u0120lect": 11042, "dam": 11043, "\u0120innovation": 11044, "\u0120Points": 11045, "\u0120loading": 11046, "\u0120dispute": 11047, "ctic": 11048, "irds": 11049, "\u0120BY": 11050, "\u0120nurs": 11051, "\u0120Value": 11052, "IONS": 11053, "\u0120Hum": 11054, "\u0120template": 11055, "mers": 11056, "\u0120appearances": 11057, "\u0120Entertainment": 11058, "\u0120translation": 11059, "\u0120sake": 11060, "\u0120beneath": 11061, "\u0120inhib": 11062, "\u0120euro": 11063, "abetes": 11064, "\u0120studying": 11065, "\u0120Mas": 11066, "\u0120perceived": 11067, "\u0120examined": 11068, "\u0120eager": 11069, "\u0120coaches": 11070, "\u0120imper": 11071, "chi": 11072, "\u0120produces": 11073, "\").": 11074, "\u0120Everyone": 11075, "\u0120municip": 11076, "\u0120girlfriend": 11077, "\u0120hire": 11078, "\u0120Vice": 11079, "\u0120suitable": 11080, "opy": 11081, "\u0120inequ": 11082, "\u0120Duke": 11083, "fish": 11084, "first": 11085, "\u0120Obs": 11086, "\u0120interior": 11087, "\u0120Bruce": 11088, "\u0120Ry": 11089, "\u0120analys": 11090, "\u0120considerable": 11091, "\u0120forecast": 11092, "\u0120fert": 11093, "orship": 11094, "\u0120Drug": 11095, "\u0120ALL": 11096, ":\"": 11097, "thur": 11098, "\u0120Mail": 11099, "\u0120ballot": 11100, "\u0120instantly": 11101, "\u0120Channel": 11102, "\u0120picks": 11103, "\u01201989": 11104, "\u0120tent": 11105, "oli": 11106, "\u0120civilian": 11107, "bling": 11108, "ello": 11109, "bu": 11110, "\u0120inch": 11111, "\u0120logo": 11112, "\u0120cooperation": 11113, "\u0120walks": 11114, "\u0120investments": 11115, "\u0120imprison": 11116, "\u0120Festival": 11117, "\u0120Ky": 11118, "\u0120legally": 11119, "\u0120gri": 11120, "charg": 11121, "Sl": 11122, "\u0120threatening": 11123, "duction": 11124, "flow": 11125, "\u0120dismissed": 11126, "ibraries": 11127, "cap": 11128, "ele": 11129, "\u0120McG": 11130, "\u0120Harvard": 11131, "\u0120Conservative": 11132, "\u0120CBS": 11133, "png": 11134, "\u0120roots": 11135, "\u0120Having": 11136, "umbled": 11137, "\u0120Fun": 11138, "\\/": 11139, "\u0120Search": 11140, "plex": 11141, "\u0120discussing": 11142, "\u0120continu": 11143, "\u0120Tai": 11144, "\u0120Wik": 11145, "Free": 11146, "fit": 11147, "\u0120refuse": 11148, "\u0120managing": 11149, "\u0120synd": 11150, "ipedia": 11151, "walk": 11152, "\u0120professionals": 11153, "\u0120guidance": 11154, "\u0120universities": 11155, "\u0120assemb": 11156, "untu": 11157, "Finally": 11158, "ASE": 11159, "\u0120Auto": 11160, "\u0120Had": 11161, "\u0120anniversary": 11162, "LD": 11163, "\u0120Dur": 11164, "\u0120Ultimate": 11165, "ihad": 11166, "product": 11167, "\u0120transit": 11168, "\u0120restore": 11169, "\u0120explaining": 11170, "\u0120asset": 11171, "\u0120transferred": 11172, "\u0120burst": 11173, "apolis": 11174, "\u0120Magazine": 11175, "\u0120Cra": 11176, "\u0120BR": 11177, "gged": 11178, "\u0120HE": 11179, "Mich": 11180, "bet": 11181, "\u0120Lady": 11182, "ylum": 11183, "erves": 11184, "\u0120meets": 11185, "white": 11186, "Log": 11187, "\u0120corresponding": 11188, "\u0120insisted": 11189, "GG": 11190, "\u0120surrounded": 11191, "\u0120tens": 11192, "\u0120lane": 11193, "\u0120coinc": 11194, "home": 11195, "\u0120existed": 11196, "ected": 11197, "\u0120Double": 11198, "lamm": 11199, "\u0120skept": 11200, "exp": 11201, "\u0120perception": 11202, "iev": 11203, "\u0120Being": 11204, "oft": 11205, "\u0120adopt": 11206, ".:": 11207, "];": 11208, "Windows": 11209, "\u0120satellite": 11210, "ASH": 11211, "\u0120infant": 11212, "description": 11213, "\u0120Meanwhile": 11214, "cm": 11215, "oca": 11216, "\u0120Treat": 11217, "actor": 11218, "\u0120tobacco": 11219, "\u0120Norm": 11220, "emption": 11221, "\u0120flesh": 11222, "\u0120je": 11223, "oop": 11224, "\u0120Heaven": 11225, "\u0120beating": 11226, "anim": 11227, "\u0120gathering": 11228, "\u0120cultiv": 11229, "GO": 11230, "abe": 11231, "\u0120Jonathan": 11232, "\u0120Safety": 11233, "\u0120badly": 11234, "prot": 11235, "\u0120choosing": 11236, "\u0120contacted": 11237, "\u0120quit": 11238, "\u0120distur": 11239, "\u0120stir": 11240, "\u0120token": 11241, "Det": 11242, "\u0120Pa": 11243, "\u0120functionality": 11244, "003": 11245, "some": 11246, "\u0120limitations": 11247, "\u0120meth": 11248, "build": 11249, "config": 11250, "NT": 11251, "rell": 11252, "blem": 11253, "\u0120Mom": 11254, "\u0120veterans": 11255, "\u0120Hu": 11256, "\u0120trends": 11257, "arer": 11258, "\u0120Given": 11259, "\u0120Caption": 11260, "may": 11261, "AST": 11262, "\u0120wondering": 11263, "\u0120Clark": 11264, "normal": 11265, "\u0120separated": 11266, "\u0120desp": 11267, "stic": 11268, "brew": 11269, "\u0120relating": 11270, "\u0120Nik": 11271, "\u0120Farm": 11272, "\u0120enthusi": 11273, "good": 11274, "deb": 11275, "\u0120activist": 11276, "\u0120mart": 11277, "\u0120explosion": 11278, "\u0120Economic": 11279, "Link": 11280, "\u0120insight": 11281, "\u0120convenient": 11282, "\u0120counterpart": 11283, "support": 11284, "\u0120Virt": 11285, "agen": 11286, "\u0120Tennessee": 11287, "\u0120Simon": 11288, "\u0120Award": 11289, "OCK": 11290, "\u0120Figure": 11291, "\u0120overseas": 11292, "\u0120pride": 11293, "\u0120Cas": 11294, "note": 11295, "mg": 11296, "Current": 11297, "\u0120displays": 11298, "content": 11299, "\u0120traveling": 11300, "\u0120hospitals": 11301, "\u0120Financial": 11302, "\u0120Past": 11303, "\u0120defendant": 11304, "\u0120streaming": 11305, "mble": 11306, "\u0120Berlin": 11307, "uki": 11308, "\u0120distribut": 11309, "\u0120antib": 11310, "\u0120chocolate": 11311, "\u0120Castle": 11312, "\u0120interrupt": 11313, "\u0120Row": 11314, "\u0120conversion": 11315, "\u0120bugs": 11316, "\u0120Rather": 11317, "liest": 11318, "LY": 11319, "\u0120Jean": 11320, "common": 11321, "akh": 11322, "\u0120130": 11323, "otton": 11324, "\u0120Dean": 11325, "\u0120amendment": 11326, "\u0120gameplay": 11327, "\u0120Warren": 11328, "oda": 11329, "\u0120highlights": 11330, "\u0120irre": 11331, "\u0120NATO": 11332, "\u0120balls": 11333, "\u0120demanding": 11334, "URE": 11335, "\u0120Luke": 11336, "Figure": 11337, "stop": 11338, "onia": 11339, "zone": 11340, "izers": 11341, "\u0120WR": 11342, "\u0120awarded": 11343, "\u0120regulatory": 11344, "\u0120Hart": 11345, "\u0120SN": 11346, "pling": 11347, "\u0120sour": 11348, "\u0120Pixel": 11349, "usive": 11350, "\u0120fet": 11351, "\u0120Sent": 11352, "\u0120automatic": 11353, "\u0120fer": 11354, "vernment": 11355, "\u0120Khan": 11356, "TON": 11357, "father": 11358, "\u0120extraordinary": 11359, "throp": 11360, "\u0120Python": 11361, "\u0120GPU": 11362, "\u0120sexually": 11363, "\u0120desktop": 11364, "itivity": 11365, "\u0120Antonio": 11366, "\u0120orient": 11367, "\u0120ears": 11368, "obby": 11369, "ouses": 11370, "vertisements": 11371, "\u0120manufacturers": 11372, "icient": 11373, "minute": 11374, "\u0120conviction": 11375, "\u0120garden": 11376, "public": 11377, "\u0120satisfied": 11378, "fold": 11379, "OK": 11380, "\u0120inhab": 11381, "\u0120Think": 11382, "\u0120programme": 11383, "\u0120stomach": 11384, "\u0120coordin": 11385, "\u0120holy": 11386, "\u0120threshold": 11387, "\u0120rhet": 11388, "\u0120serial": 11389, "\u0120employers": 11390, "\u0120Everything": 11391, "rah": 11392, "\u0120bother": 11393, "\u0120brands": 11394, "Value": 11395, "\u0120Ted": 11396, "\u0120Planet": 11397, "\u0120pink": 11398, "\u0120Furthermore": 11399, "sa": 11400, "PE": 11401, "reck": 11402, "\u0120USD": 11403, "otte": 11404, "\u0120&&": 11405, "\u0120landed": 11406, "gets": 11407, "\u0120producers": 11408, "\u0120healthcare": 11409, "\u0120dominant": 11410, "\u0120destro": 11411, "\u0120amended": 11412, "chron": 11413, "\u0120fits": 11414, "\u0120Syd": 11415, "\u0120Authority": 11416, "ATCH": 11417, "\u0120fights": 11418, "\u0120LLC": 11419, "\u0120---": 11420, "\u0120Corp": 11421, "\u0120toxic": 11422, "specific": 11423, "\u0120Corn": 11424, "\u0120Chel": 11425, "\u0120telephone": 11426, "\u0120Pant": 11427, "\u0120mysterious": 11428, "aunch": 11429, "odox": 11430, "media": 11431, "\u0120witnesses": 11432, "agu": 11433, "\u0120questioned": 11434, "\u0120Brexit": 11435, "\u0120Remember": 11436, "enez": 11437, "\u0120endorse": 11438, "iatric": 11439, "\u0120Ident": 11440, "\u0120ridiculous": 11441, "110": 11442, "\u0120prayer": 11443, "\u0120scientist": 11444, "\u01201950": 11445, "\u0120Aqu": 11446, "\u0120underground": 11447, "\u0120UFC": 11448, "mare": 11449, "\u0120Later": 11450, "wich": 11451, "\u0120subscrib": 11452, "\u0120hosts": 11453, "\u0120err": 11454, "\u0120grants": 11455, "antom": 11456, "\u0120summon": 11457, "early": 11458, "\u0120Clear": 11459, "\u0120Prim": 11460, "\u0120suspension": 11461, "\u0120guaranteed": 11462, "apper": 11463, "\u0120rice": 11464, "\u0120Sean": 11465, "\u0120Shin": 11466, "\u0120referendum": 11467, "\u0120fled": 11468, "rust": 11469, "\u0120360": 11470, "tery": 11471, "\u0120shocked": 11472, "BR": 11473, "\u0120Oil": 11474, "\u0120Allah": 11475, "\u0120partly": 11476, "\u0120ignor": 11477, "\u0120transmission": 11478, "\u0120homosexual": 11479, "iversal": 11480, "\u0120hopefully": 11481, "\u00e3\u0124\u00a4": 11482, "\u0120lesson": 11483, "Leg": 11484, "\u0120..": 11485, "Yet": 11486, "table": 11487, "appropri": 11488, "rett": 11489, "\u0120boards": 11490, "\u0120incorrect": 11491, "\u0120bacteria": 11492, "aru": 11493, "amac": 11494, "\u0120snap": 11495, ".'\"": 11496, "\u0120parad": 11497, "tem": 11498, "heart": 11499, "\u0120availability": 11500, "\u0120wisdom": 11501, "\u0120(+": 11502, "\u0120priest": 11503, "\u0120\u00c2\u0142\u0120\u00c2\u0142": 11504, "Open": 11505, "\u0120span": 11506, "\u0120parameter": 11507, "\u0120convince": 11508, "\u0120(%)": 11509, "rac": 11510, "\u0120fo": 11511, "\u0120safely": 11512, "\u0120converted": 11513, "\u0120Olympic": 11514, "\u0120reserve": 11515, "\u0120healing": 11516, "\u0120Mine": 11517, "Max": 11518, "\u0120inherent": 11519, "\u0120Graham": 11520, "\u0120integrated": 11521, "Dem": 11522, "\u0120pipeline": 11523, "\u0120applying": 11524, "\u0120embed": 11525, "\u0120Charlie": 11526, "\u0120cave": 11527, "2008": 11528, "\u0120consensus": 11529, "\u0120rewards": 11530, "Pal": 11531, "\u0120HTML": 11532, "\u0120popularity": 11533, "looking": 11534, "\u0120Sword": 11535, "\u0120Arts": 11536, "')": 11537, "\u0120electron": 11538, "clusions": 11539, "\u0120integrity": 11540, "\u0120exclusively": 11541, "\u0120grace": 11542, "\u0120torture": 11543, "\u0120burned": 11544, "two": 11545, "\u0120180": 11546, "Produ": 11547, "\u0120entreprene": 11548, "raphics": 11549, "\u0120gym": 11550, "ricane": 11551, "\u0120Tam": 11552, "\u0120administrative": 11553, "\u0120manufacturer": 11554, "\u0120vel": 11555, "\u0120Ni": 11556, "\u0120isolated": 11557, "\u0120Medicine": 11558, "\u0120backup": 11559, "\u0120promoting": 11560, "\u0120commander": 11561, "\u0120flee": 11562, "\u0120Russell": 11563, "\u0120forgotten": 11564, "\u0120Missouri": 11565, "\u0120residence": 11566, "mons": 11567, "\u0120resemb": 11568, "\u0120wand": 11569, "\u0120meaningful": 11570, "PT": 11571, "\u0120bol": 11572, "\u0120helic": 11573, "\u0120wealthy": 11574, "\u0120rifle": 11575, "strong": 11576, "rowing": 11577, "plan": 11578, "asury": 11579, "\u00e2\u0122\u00a6.": 11580, "\u0120expanding": 11581, "\u0120Hamilton": 11582, "\u0120receives": 11583, "SI": 11584, "eatures": 11585, "\u0120Anim": 11586, "REE": 11587, "Put": 11588, "\u0120briefly": 11589, "rive": 11590, "\u0120stimul": 11591, "\u0120``(": 11592, "\u0120__": 11593, "\u0120chip": 11594, "\u0120haz": 11595, "\u0120prize": 11596, "\u0120Things": 11597, "ACE": 11598, "ulin": 11599, "dict": 11600, "oku": 11601, "\u0120associate": 11602, "ockets": 11603, "youtube": 11604, "Story": 11605, "ategory": 11606, "\u0120mild": 11607, "ailing": 11608, "\u0120Ye": 11609, "Orig": 11610, "\u0120Ka": 11611, "orig": 11612, "\u0120propaganda": 11613, "\u0120anonymous": 11614, "\u0120struggled": 11615, "\u0120outrage": 11616, "ATED": 11617, "\u0120Beijing": 11618, "rary": 11619, "\u0120leather": 11620, "\u0120worlds": 11621, "\u0120broader": 11622, "125": 11623, "idal": 11624, "\u0120Better": 11625, "\u0120tear": 11626, "Ext": 11627, "\u0120proposals": 11628, "\u0120iter": 11629, "\u0120Squad": 11630, "\u0120volunt": 11631, "mi": 11632, "Did": 11633, "\u0120Pu": 11634, "pin": 11635, "\u0120speakers": 11636, "\u0120borders": 11637, "\u0120figured": 11638, "='": 11639, "\u0120simultaneously": 11640, "aeda": 11641, "\u0120charging": 11642, "\u0120urged": 11643, "\u0120conj": 11644, "256": 11645, "\u0120Gordon": 11646, "merce": 11647, "\u0120documentary": 11648, "Share": 11649, "itol": 11650, "ONE": 11651, "\u0120Garden": 11652, "hatt": 11653, "\u0120Thompson": 11654, "aneous": 11655, "apore": 11656, "\u0120tanks": 11657, "\u0120lessons": 11658, "track": 11659, "\u0120outstanding": 11660, "\u0120volunteers": 11661, "\u0120spray": 11662, "\u0120managers": 11663, "large": 11664, "\u0120camps": 11665, "\u0120artificial": 11666, "\u0120Ru": 11667, "\u0120bags": 11668, "thal": 11669, "\u0120compatible": 11670, "\u0120Blade": 11671, "\u0120fed": 11672, "\u0120argues": 11673, "FI": 11674, "\u0120unfair": 11675, "\u0120corn": 11676, "\u0120offset": 11677, "\u0120directions": 11678, "\u0120disappointed": 11679, "\u0120Convention": 11680, "\u0120viewing": 11681, "ME": 11682, "ocity": 11683, "\u0120towns": 11684, "\u0120layers": 11685, "\u0120rolled": 11686, "\u0120jumped": 11687, "\u0120attribute": 11688, "\u0120unnecess": 11689, "incoln": 11690, "\u0120suppose": 11691, "\u0120Nether": 11692, "cha": 11693, "\u0120buried": 11694, "\u0120sixth": 11695, "Ben": 11696, "ressing": 11697, "OUR": 11698, "\u0120wound": 11699, "\u0120cycl": 11700, "\u0120mechanisms": 11701, "\u0120congressional": 11702, "\u0120Element": 11703, "\u0120agreements": 11704, "\u0120decor": 11705, "\u0120closest": 11706, "\u0120Mit": 11707, "Google": 11708, "}}": 11709, "\u0120mixture": 11710, "\u0120fluid": 11711, "Sign": 11712, "\u0120Scholar": 11713, "\u0120pist": 11714, "asket": 11715, "abling": 11716, "\u0120racing": 11717, "hero": 11718, "riel": 11719, "assy": 11720, "\u0120cheaper": 11721, "ben": 11722, "\u0120vertical": 11723, "amacare": 11724, "\u0120Reading": 11725, "gments": 11726, "\u0120helicop": 11727, "\u0120sacrifice": 11728, "aya": 11729, "paren": 11730, "VA": 11731, "\u0120Les": 11732, "\u0120Studio": 11733, "\u0120violations": 11734, "\u0120Anna": 11735, "acer": 11736, "\u00e9\u00be": 11737, "\u0120Rat": 11738, "\u0120Beck": 11739, "\u0120Dick": 11740, "\u0120ACT": 11741, "\u0120composition": 11742, "\u0120texture": 11743, "\u0120Own": 11744, "\u0120smartphone": 11745, "\u0120NA": 11746, "\u0120forb": 11747, "import": 11748, "\u0120defending": 11749, "ilst": 11750, "rer": 11751, "\u0120oh": 11752, "\u0120Jeremy": 11753, "\u0120banking": 11754, "ceptions": 11755, "\u0120respective": 11756, "/.": 11757, "\u0120drinks": 11758, "\u0120Wi": 11759, "\u0120bands": 11760, "\u0120Liverpool": 11761, "\u0120grip": 11762, "\u0120Buy": 11763, "\u0120openly": 11764, "\u0120reviewed": 11765, "pert": 11766, "\u0120verify": 11767, "\u0120Cole": 11768, "\u0120Wales": 11769, "MO": 11770, "\u0120unpre": 11771, "\u0120shelter": 11772, "\u0120Imperial": 11773, "\u0120gui": 11774, "\u0120Dak": 11775, "\u0120suggestions": 11776, "\u0120explicitly": 11777, "\u0120slave": 11778, "\u0120blockchain": 11779, "\u0120competing": 11780, "\u0120promising": 11781, "SON": 11782, "\u0120soccer": 11783, "\u0120constitution": 11784, "429": 11785, "\u0120distract": 11786, "\u0120User": 11787, "esides": 11788, "\u0120Method": 11789, "\u0120Tokyo": 11790, "\u0120accompanied": 11791, "Client": 11792, "sur": 11793, "alog": 11794, "\u0120identification": 11795, "\u0120invasion": 11796, "asma": 11797, "\u0120industries": 11798, "ppers": 11799, "\u0120subtle": 11800, "\u0120Unit": 11801, "natural": 11802, "\u0120survived": 11803, "\u0120flaw": 11804, "\u013a\u0127": 11805, "\u0120Holl": 11806, "\u0120deficit": 11807, "\u0120tutorial": 11808, "\u0120Chance": 11809, "\u0120arguing": 11810, "\u0120contemporary": 11811, "\u0120integration": 11812, "forward": 11813, "\u0120tum": 11814, "itis": 11815, "\u0120hiding": 11816, "\u0120Domin": 11817, "\u0120Tan": 11818, "\u0120Building": 11819, "\u0120Vin": 11820, "\u0120spokesperson": 11821, "\u0120Notes": 11822, "\u0120emerging": 11823, "\u0120preparation": 11824, "\u0120prost": 11825, "\u0120suspects": 11826, "\u0120autonom": 11827, "Description": 11828, "\u0120dealt": 11829, "\u0120Pear": 11830, "\u0120steady": 11831, "\u0120decreased": 11832, "\u0120sovere": 11833, "\u0120Clin": 11834, "\u0120gradually": 11835, "orses": 11836, "\u0120WAR": 11837, "Serv": 11838, "\u00e3\u0124\u00a2": 11839, "hr": 11840, "\u0120dirty": 11841, "\u0120Barn": 11842, "\u0120BC": 11843, "\u0120dil": 11844, "\u0120calendar": 11845, "\u0120compliance": 11846, "\u0120chamber": 11847, "bb": 11848, "\u0120passenger": 11849, "ateful": 11850, "\u0120Title": 11851, "\u0120Sydney": 11852, "\u0120Got": 11853, "\u0120darkness": 11854, "\u0120defect": 11855, "\u0120packed": 11856, "assion": 11857, "\u0120gods": 11858, "\u0120harsh": 11859, "ICK": 11860, "leans": 11861, "\u0120algorithm": 11862, "\u0120oxygen": 11863, "\u0120visits": 11864, "\u0120blade": 11865, "\u0120kilomet": 11866, "\u0120Kentucky": 11867, "\u0120killer": 11868, "Pack": 11869, "enny": 11870, "\u0120divine": 11871, "\u0120nomination": 11872, "being": 11873, "\u0120engines": 11874, "\u0120cats": 11875, "\u0120buffer": 11876, "\u0120Phill": 11877, "\u0120traff": 11878, "AGE": 11879, "\u0120tongue": 11880, "\u0120radiation": 11881, "erer": 11882, "mem": 11883, "\u0120Explicit": 11884, "\u00e9\u00be\u012f": 11885, "\u0120couples": 11886, "\u0120physics": 11887, "\u0120McK": 11888, "\u0120politically": 11889, "awks": 11890, "\u0120Bloom": 11891, "\u0120worship": 11892, "eger": 11893, "uter": 11894, "\u0120FO": 11895, "\u0120mathemat": 11896, "\u0120sentenced": 11897, "\u0120disk": 11898, "\u0120Marg": 11899, "\u0120/*": 11900, "PI": 11901, "\u0120optional": 11902, "\u0120babies": 11903, "\u0120seeds": 11904, "\u0120Scottish": 11905, "\u0120thy": 11906, "]]": 11907, "\u0120Hitler": 11908, "PH": 11909, "ngth": 11910, "\u0120recovered": 11911, "inge": 11912, "\u0120powder": 11913, "\u0120lips": 11914, "\u0120designer": 11915, "\u0120disorders": 11916, "\u0120courage": 11917, "\u0120chaos": 11918, "\"},{\"": 11919, "\u0120carrier": 11920, "bably": 11921, "High": 11922, "\u0120RT": 11923, "esity": 11924, "len": 11925, "\u0120routes": 11926, "uating": 11927, "Fil": 11928, "NOT": 11929, "wall": 11930, "sburgh": 11931, "\u0120engaging": 11932, "\u0120JavaScript": 11933, "orer": 11934, "lihood": 11935, "\u0120unions": 11936, "\u0120Federation": 11937, "\u0120Tesla": 11938, "\u0120completion": 11939, "\u0120Ta": 11940, "\u0120privilege": 11941, "\u0120Orange": 11942, "\u0120neur": 11943, "parency": 11944, "\u0120bones": 11945, "\u0120titled": 11946, "\u0120prosecutors": 11947, "\u0120ME": 11948, "\u0120engineer": 11949, "\u0120Universe": 11950, "\u0120Hig": 11951, "nie": 11952, "oard": 11953, "\u0120hearts": 11954, "\u0120Gre": 11955, "ussion": 11956, "\u0120ministry": 11957, "\u0120penet": 11958, "\u0120Nut": 11959, "\u0120Ow": 11960, "\u0120XP": 11961, "instein": 11962, "\u0120bulk": 11963, "System": 11964, "icism": 11965, "\u0120Marketable": 11966, "\u0120preval": 11967, "\u0120poster": 11968, "\u0120attending": 11969, "urable": 11970, "\u0120licensed": 11971, "\u0120Gh": 11972, "etry": 11973, "\u0120Tradable": 11974, "\u0120blast": 11975, "\u00e0\u00a4": 11976, "\u0120Titan": 11977, "elled": 11978, "die": 11979, "Have": 11980, "\u0120Flame": 11981, "\u0120profound": 11982, "\u0120participating": 11983, "\u0120anime": 11984, "\u0120Ess": 11985, "\u0120specify": 11986, "\u0120regarded": 11987, "\u0120Spell": 11988, "\u0120sons": 11989, "owned": 11990, "\u0120merc": 11991, "\u0120experimental": 11992, "lando": 11993, "hs": 11994, "\u0120Dungeon": 11995, "inos": 11996, "\u0120comply": 11997, "\u0120Systems": 11998, "arth": 11999, "\u0120seized": 12000, "local": 12001, "\u0120Girls": 12002, "udo": 12003, "oned": 12004, "\u0120Fle": 12005, "\u0120constructed": 12006, "\u0120hosted": 12007, "\u0120scared": 12008, "actic": 12009, "\u0120Islands": 12010, "\u0120MORE": 12011, "\u0120bless": 12012, "\u0120blocking": 12013, "\u0120chips": 12014, "\u0120evac": 12015, "Ps": 12016, "\u0120corporation": 12017, "\u0120ox": 12018, "\u0120lighting": 12019, "\u0120neighbors": 12020, "\u0120Ub": 12021, "aro": 12022, "\u0120beef": 12023, "\u0120Uber": 12024, "Facebook": 12025, "armed": 12026, "itate": 12027, "\u0120Rating": 12028, "\u0120Quick": 12029, "\u0120occupied": 12030, "\u0120aims": 12031, "\u0120Additionally": 12032, "\u0120Interest": 12033, "\u0120dramatically": 12034, "\u0120heal": 12035, "\u0120painting": 12036, "\u0120engineers": 12037, "MM": 12038, "\u0120Must": 12039, "\u0120quantity": 12040, "Paul": 12041, "\u0120earnings": 12042, "\u0120Posts": 12043, "stra": 12044, "\u00e3\u0125\u00bc\u00e3\u0125": 12045, "\u0120stance": 12046, "\u0120dropping": 12047, "script": 12048, "\u0120dressed": 12049, "Make": 12050, "\u0120justify": 12051, "\u0120Ltd": 12052, "\u0120prompted": 12053, "\u0120scrut": 12054, "\u0120speeds": 12055, "\u0120Giants": 12056, "omer": 12057, "\u0120Editor": 12058, "\u0120describing": 12059, "\u0120Lie": 12060, "mented": 12061, "\u0120nowhere": 12062, "ocaly": 12063, "\u0120instruction": 12064, "fortable": 12065, "\u0120entities": 12066, "\u0120cm": 12067, "\u0120Natural": 12068, "\u0120inquiry": 12069, "\u0120pressed": 12070, "izont": 12071, "forced": 12072, "\u0120raises": 12073, "\u0120Netflix": 12074, "\u0120Side": 12075, "\u0120outer": 12076, "\u0120amongst": 12077, "ims": 12078, "owski": 12079, "\u0120climb": 12080, "never": 12081, "\u0120combine": 12082, "ding": 12083, "\u0120compr": 12084, "\u0120significance": 12085, "\u0120remembered": 12086, "\u0120Nevada": 12087, "\u0120Tel": 12088, "\u0120Scar": 12089, "\u0120Warriors": 12090, "\u0120Jane": 12091, "\u0120coup": 12092, "bas": 12093, "\u0120terminal": 12094, ",-": 12095, "OH": 12096, "\u0120tension": 12097, "\u0120wings": 12098, "\u0120Myster": 12099, "\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 12100, "\u0120Unlike": 12101, "valid": 12102, "vironments": 12103, "\u0120Ali": 12104, "\u0120naked": 12105, "books": 12106, "\u0120Mun": 12107, "\u0120Gulf": 12108, "\u0120density": 12109, "\u0120dimin": 12110, "\u0120desperate": 12111, "\u0120presidency": 12112, "\u01201986": 12113, "hy": 12114, "IND": 12115, "\u0120unlock": 12116, "imens": 12117, "\u0120handled": 12118, "\u0120Eb": 12119, "\u0120disappeared": 12120, "\u0120genre": 12121, "\u01201988": 12122, "\u0120determination": 12123, "Stream": 12124, "iko": 12125, "apters": 12126, "\u0120acknowledge": 12127, "Jan": 12128, "\u0120capitalism": 12129, "Pat": 12130, "\u01202020": 12131, "\u0120painful": 12132, "\u0120curve": 12133, "\u0120bombs": 12134, "storm": 12135, "\u0120Metal": 12136, "encer": 12137, "\u0120Fig": 12138, "\u0120Aaron": 12139, "anches": 12140, "\u0120inspiration": 12141, "\u0120exhaust": 12142, "tains": 12143, "ashi": 12144, "\u0120descript": 12145, "\u0120ritual": 12146, "\u0120Chelsea": 12147, "\u0120promotion": 12148, "\u0120Hung": 12149, "\u0120Ward": 12150, "iva": 12151, "\u0120ET": 12152, "\u0120toss": 12153, "allow": 12154, "\u0120Francis": 12155, "Dep": 12156, "\u0120happiness": 12157, "\u0120Glass": 12158, "\u0120beta": 12159, "\u0120strengthen": 12160, "NE": 12161, "oa": 12162, "\u0120buttons": 12163, "\u0120Murray": 12164, "\u0120kicked": 12165, "Quest": 12166, "\u0120Talk": 12167, "\u0120Several": 12168, "\u0120Zero": 12169, "\u0120drone": 12170, "ulk": 12171, "\u0120cam": 12172, "\u0120Mobile": 12173, "\u0120preventing": 12174, "\u0120retro": 12175, "\u0120Ax": 12176, "\u0120cruel": 12177, "\u0120float": 12178, ".),": 12179, "\u0120filing": 12180, "\u0120Grant": 12181, "\u0120Bor": 12182, "\u0120rib": 12183, "\u0120championship": 12184, "\u0120Merc": 12185, "\u0120styles": 12186, "\u0120cake": 12187, "\u0120builds": 12188, "\u0120Self": 12189, "iox": 12190, "\u0120epic": 12191, "oyd": 12192, "Bel": 12193, "\u0120Stew": 12194, ".(": 12195, "ahu": 12196, "\u0120Beyond": 12197, "\u0120outs": 12198, "\u0120solo": 12199, "\u0120Tree": 12200, "\u0120preserve": 12201, "\u0120tub": 12202, "ARE": 12203, "roc": 12204, "\u0120Impro": 12205, "\u0120Wright": 12206, "\u0120bund": 12207, "\u0120traged": 12208, "\u0120occasional": 12209, "bian": 12210, "Second": 12211, "rons": 12212, "\u0120interactions": 12213, "formed": 12214, "sing": 12215, "\u0120owns": 12216, "\u0120hockey": 12217, "General": 12218, "\u0120logical": 12219, "\u0120expend": 12220, "\u0120escal": 12221, "\u0120Griff": 12222, "\u0120Crown": 12223, "\u0120Reserve": 12224, "\u0120stopping": 12225, "\u0120excuse": 12226, "second": 12227, "\u0120operated": 12228, "\u0120reaches": 12229, "\u0120Malays": 12230, "\u0120pollution": 12231, "\u0120Brooklyn": 12232, "\u0120delete": 12233, "\u0120hash": 12234, "Block": 12235, "aha": 12236, "\u00e2\u0122\u00b3": 12237, "\u0120shorter": 12238, "piece": 12239, ">>>": 13163, "\u0120Mormon": 13164, "tor": 13165, "\u0120particles": 13166, "\u0120Bart": 13167, "ryption": 13168, "\u0120admin": 13169, "\u0120squee": 13170, "VIDIA": 13171, "\u0120creator": 13172, "iameter": 13173, "icular": 13174, "NBC": 13175, "\u0120grabbed": 13176, "\u0120nodd": 13177, "\u0120rated": 13178, "\u0120rotation": 13179, "\u0120grasp": 13180, "\u0120excessive": 13181, "\u0120EC": 13182, "\u0120Whit": 13183, "\u0120inventory": 13184, "aults": 13185, "\u0120FB": 13186, "\u0120ecosystem": 13187, "\u0120billions": 13188, "\u0120venture": 13189, "named": 13190, "\u0120defender": 13191, "oute": 13192, "Instead": 13193, "irable": 13194, "War": 13195, "\u0120assumption": 13196, "\u0120bite": 13197, "\u0120earthqu": 13198, "tail": 13199, "space": 13200, "\u0120gifts": 13201, "boys": 13202, "\u0120inevitable": 13203, "\u0120structural": 13204, "\u0120beneficial": 13205, "\u0120compelling": 13206, "hole": 13207, "ervation": 13208, "\u0120coat": 13209, "oj": 13210, "incarn": 13211, "\u0120Years": 13212, "\u0120determining": 13213, "\u0120rhetoric": 13214, "\u0120boundaries": 13215, "\u0120whites": 13216, "Ant": 13217, "addy": 13218, ")-": 13219, "raham": 13220, "etermin": 13221, "\u0120harvest": 13222, "\u0120Conc": 13223, "\u0120laptop": 13224, "\u0120Match": 13225, "\u0120enjoying": 13226, "cca": 13227, "ollar": 13228, "\u0120trips": 13229, "\u0120addiction": 13230, "\u0120Sak": 13231, "\u0120powered": 13232, "\u0120cous": 13233, "\u0120Russians": 13234, "iere": 13235, "\u0120retrie": 13236, "quality": 13237, "\u0120differ": 13238, "\u0120kingdom": 13239, "\u0120Laur": 13240, "\u0120Capitol": 13241, "\u0120conclusions": 13242, "\u0120Altern": 13243, "\u0120Nav": 13244, "\u0120transparent": 13245, "BER": 13246, "Group": 13247, "\u0120Complete": 13248, "\u0120infer": 13249, "\u0120intrig": 13250, "\u0120insane": 13251, "RO": 13252, "ophob": 13253, "isen": 13254, "qual": 13255, "Michael": 13256, "\u0120museum": 13257, "\u0120Pope": 13258, "\u0120reset": 13259, "rative": 13260, "five": 13261, "\u0120aggreg": 13262, "ittees": 13263, "ository": 13264, "\u0120carb": 13265, "\u0120Record": 13266, "\u0120decides": 13267, "\u0120Fix": 13268, "\u0120exceptions": 13269, "\u0120Commissioner": 13270, "uns": 13271, "\u0120Environmental": 13272, "\u0120legendary": 13273, "istence": 13274, "\u0120tunnel": 13275, "km": 13276, "\u0120insult": 13277, "\u0120troll": 13278, "\u0120shake": 13279, "\u0120detention": 13280, "ques": 13281, "\u0120Chrome": 13282, "\u0120Files": 13283, "\u0120subt": 13284, "\u0120prospects": 13285, "\u0120prol": 13286, "render": 13287, "proof": 13288, "\u0120performances": 13289, "Str": 13290, "\u0120href": 13291, "ername": 13292, "\u0120achievement": 13293, "\u0120fut": 13294, "Full": 13295, "\u0120Leban": 13296, "google": 13297, "\u00e3\u0125\u012a": 13298, "ampa": 13299, "Maybe": 13300, "\u0120projected": 13301, "\u0120Emb": 13302, "\u0120colleg": 13303, "\u0120awards": 13304, "\u0120\u00e2\u0136": 13305, "Gold": 13306, "\u0120Blake": 13307, "\u0120Raj": 13308, "ifting": 13309, "\u0120pending": 13310, "\u0120instinct": 13311, "\u0120developments": 13312, "Connect": 13313, "\u0120Mand": 13314, "\u0120WITH": 13315, "\u0120Philippines": 13316, "profile": 13317, "\u0120altogether": 13318, "\u0120Bund": 13319, "\u0120TD": 13320, "oooo": 13321, "amped": 13322, "iph": 13323, "\u0120steam": 13324, "\u0120oldest": 13325, "\u0120detection": 13326, "ulpt": 13327, "\u0120\u00e7": 13328, "\u0120Wayne": 13329, "2006": 13330, "fa": 13331, "\u0120circles": 13332, "\u0120Fu": 13333, "\u0120donors": 13334, "appropriate": 13335, "\u0120Dakota": 13336, "jamin": 13337, "\u0120motivated": 13338, "\u0120purchases": 13339, "\u0120Louisiana": 13340, "\u0120Spl": 13341, "\u0120globe": 13342, "\u0120105": 13343, "zip": 13344, "call": 13345, "\u0120departments": 13346, "\u0120sustainable": 13347, "105": 13348, "\u0120OP": 13349, "ifiers": 13350, "\u0120prevented": 13351, "\u0120incomp": 13352, "\u0120Commander": 13353, "\u0120dominated": 13354, "\u0120\u00c2\u00bb": 13355, "\u0120invested": 13356, "\u0120complexity": 13357, "\u0120incl": 13358, "\u0120ensuring": 13359, "\u0120realm": 13360, "ync": 13361, "\u0120Independent": 13362, "rained": 13363, "\u0120Jen": 13364, "\u0120Flight": 13365, "\u0120athe": 13366, "\u0120speculation": 13367, "\u0120TE": 13368, "ocate": 13369, "tic": 13370, "\u0120plaint": 13371, "herry": 13372, "\u0120toy": 13373, "\u0120111": 13374, "\u0120plates": 13375, "status": 13376, "\u0120Isa": 13377, "\u0120devoted": 13378, "Cop": 13379, "\u0120ES": 13380, "255": 13381, "urrency": 13382, "Main": 13383, "\u0120slaves": 13384, "\u0120pepper": 13385, "\u0120quotes": 13386, "\u0120ceiling": 13387, "\u0120Fish": 13388, "\u0120transformation": 13389, "\u0120fraction": 13390, "\u0120advantages": 13391, "\u0120toile": 13392, "\u0120stunning": 13393, "\u0120moist": 13394, "breaking": 13395, "si": 13396, "\u0120Location": 13397, "\u0120Medium": 13398, "\u0120texts": 13399, "\u0120ugly": 13400, "\u0120bio": 13401, ".\u00e2\u0122\u0136": 13402, "\u0120Based": 13403, "\u0120trains": 13404, "\u0120Wing": 13405, "\u0120Ancient": 13406, "\u0120Records": 13407, "\u0120Hope": 13408, "Special": 13409, "adesh": 13410, "obi": 13411, "[/": 13412, "\u0120temporarily": 13413, "Ver": 13414, "hu": 13415, "oser": 13416, "\u0120overnight": 13417, "\u0120mamm": 13418, "\u0120Treasury": 13419, "\u0120Venezuel": 13420, "\u0120Mega": 13421, "\u0120tar": 13422, "\u0120expects": 13423, "black": 13424, "orph": 13425, "\\\\\\\\": 13426, "\u0120acceptance": 13427, "\u0120radar": 13428, "sis": 13429, "\u0120junior": 13430, "\u0120frames": 13431, "\u0120observation": 13432, "acies": 13433, "Power": 13434, "\u0120Advanced": 13435, "Mag": 13436, "ologically": 13437, "\u0120Mechan": 13438, "\u0120sentences": 13439, "\u0120analysts": 13440, "aughters": 13441, "forcement": 13442, "\u0120vague": 13443, "\u0120clause": 13444, "\u0120directors": 13445, "\u0120evaluate": 13446, "\u0120cabinet": 13447, "Matt": 13448, "\u0120Classic": 13449, "Ang": 13450, "\u0120cler": 13451, "\u0120Buck": 13452, "\u0120researcher": 13453, "\u0120160": 13454, "\u0120poorly": 13455, "\u0120experiencing": 13456, "\u0120Ped": 13457, "\u0120Manhattan": 13458, "\u0120freed": 13459, "\u0120themes": 13460, "advant": 13461, "\u0120nin": 13462, "\u0120praise": 13463, "104": 13464, "\u0120Libya": 13465, "best": 13466, "\u0120trusted": 13467, "\u0120cease": 13468, "\u0120dign": 13469, "Direct": 13470, "\u0120bombing": 13471, "\u0120migration": 13472, "\u0120Sciences": 13473, "\u0120municipal": 13474, "\u0120Average": 13475, "\u0120glory": 13476, "\u0120revealing": 13477, "\u0120arena": 13478, "\u0120uncertainty": 13479, "\u0120battlefield": 13480, "iao": 13481, "God": 13482, "\u0120cinem": 13483, "rape": 13484, "elle": 13485, "apons": 13486, "\u0120listing": 13487, "\u0120waited": 13488, "\u0120spotted": 13489, "keley": 13490, "\u0120Audio": 13491, "eor": 13492, "arding": 13493, "idding": 13494, "igma": 13495, "\u0120Neg": 13496, "\u0120lone": 13497, "\u0120----": 13498, "exe": 13499, "deg": 13500, "\u0120transf": 13501, "\u0120wash": 13502, "\u0120slavery": 13503, "\u0120exploring": 13504, "\u0120WW": 13505, "atson": 13506, "\u0120encl": 13507, "lies": 13508, "\u0120Creek": 13509, "\u0120wooden": 13510, "Manager": 13511, "\u0120Brand": 13512, "ummy": 13513, "\u0120Arthur": 13514, "\u0120bureaucr": 13515, "\u0120blend": 13516, "arians": 13517, "Further": 13518, "\u0120supposedly": 13519, "\u0120winds": 13520, "\u01201979": 13521, "\u0120gravity": 13522, "\u0120analyses": 13523, "\u0120Travel": 13524, "\u0120Veter": 13525, "\u0120dumb": 13526, "\u0120alternate": 13527, "gal": 13528, "\u0120consumed": 13529, "\u0120effectiveness": 13530, ".''": 13531, "\u0120paths": 13532, "onda": 13533, "LA": 13534, "\u0120Strong": 13535, "\u0120enables": 13536, "\u0120escaped": 13537, "\u0120\"\"": 13538, "\u0120112": 13539, "\u01201983": 13540, "\u0120smiled": 13541, "\u0120tendency": 13542, "Fire": 13543, "\u0120pars": 13544, "\u0120Roc": 13545, "\u0120lake": 13546, "\u0120fitness": 13547, "\u0120Ath": 13548, "\u0120Horn": 13549, "\u0120hier": 13550, "\u0120impose": 13551, "mother": 13552, "\u0120pension": 13553, "icut": 13554, "borne": 13555, "iciary": 13556, "._": 13557, "\u0120SU": 13558, "\u0120polar": 13559, "isy": 13560, "engu": 13561, "itialized": 13562, "ATA": 13563, "write": 13564, "\u0120exercises": 13565, "\u0120Diamond": 13566, "otypes": 13567, "\u0120harmful": 13568, "onz": 13569, "\u0120printing": 13570, "story": 13571, "\u0120expertise": 13572, "\u0120Ger": 13573, "\u0120tragedy": 13574, "\u0120Fly": 13575, "\u0120divid": 13576, "ampire": 13577, "stock": 13578, "Mem": 13579, "\u0120reign": 13580, "\u0120unve": 13581, "\u0120amend": 13582, "\u0120Prophet": 13583, "\u0120mutual": 13584, "\u0120Fac": 13585, "\u0120replacing": 13586, "Har": 13587, "\u0120Circuit": 13588, "\u0120throat": 13589, "\u0120Shot": 13590, "\u0120batteries": 13591, "\u0120toll": 13592, "\u0120addressing": 13593, "\u0120Medicaid": 13594, "\u0120pupp": 13595, "\u0120Nar": 13596, "olk": 13597, "\u0120equity": 13598, "MR": 13599, "\u0120Hispan": 13600, "\u0120Large": 13601, "mid": 13602, "Dev": 13603, "\u0120exped": 13604, "\u0120demo": 13605, "\u0120Marshall": 13606, "ergus": 13607, "\u0120fiber": 13608, "\u0120divorce": 13609, "\u0120Create": 13610, "\u0120slower": 13611, "\u0120Parker": 13612, "\u0120Student": 13613, "\u0120Training": 13614, "Return": 13615, "\u0120Tru": 13616, "\u0120cub": 13617, "\u0120Reached": 13618, "\u0120panic": 13619, "\u0120quarters": 13620, "\u0120rect": 13621, "\u0120treating": 13622, "\u0120rats": 13623, "\u0120Christianity": 13624, "oler": 13625, "\u0120sacred": 13626, "\u0120declare": 13627, "ulative": 13628, "eting": 13629, "\u0120delivering": 13630, "estone": 13631, "\u0120tel": 13632, "\u0120Larry": 13633, "\u0120meta": 13634, "accept": 13635, "artz": 13636, "\u0120Roger": 13637, "handed": 13638, "\u0120header": 13639, "\u0120trapped": 13640, "\u0120Century": 13641, "\u0120knocked": 13642, "\u0120Oxford": 13643, "\u0120survivors": 13644, "bot": 13645, "\u0120demonstration": 13646, "\u0120dirt": 13647, "\u0120assists": 13648, "OME": 13649, "\u0120Draft": 13650, "ortunate": 13651, "folio": 13652, "pered": 13653, "usters": 13654, "gt": 13655, "\u0120Lock": 13656, "\u0120judicial": 13657, "verted": 13658, "\u0120secured": 13659, "outing": 13660, "\u0120Books": 13661, "\u0120hosting": 13662, "\u0120lifted": 13663, "length": 13664, "\u0120jer": 13665, "\u0120wheels": 13666, "\u0120Range": 13667, "umbnails": 13668, "\u0120diagnosis": 13669, "tech": 13670, "\u0120Stewart": 13671, "\u0120Pract": 13672, "\u0120nationwide": 13673, "\u0120dear": 13674, "\u0120obligations": 13675, "\u0120grows": 13676, "\u0120mandatory": 13677, "\u0120suspicious": 13678, "!'": 13679, "Apr": 13680, "Great": 13681, "\u0120mortgage": 13682, "\u0120prosecutor": 13683, "\u0120editorial": 13684, "\u0120Kr": 13685, "\u0120processed": 13686, "ungle": 13687, "\u0120flexibility": 13688, "Earlier": 13689, "\u0120Cart": 13690, "\u0120Sug": 13691, "\u0120focuses": 13692, "\u0120startup": 13693, "\u0120breach": 13694, "\u0120Tob": 13695, "cycle": 13696, "\u00e3\u0122\u012e": 13697, "rose": 13698, "\u0120bizarre": 13699, "\u00e3\u0122\u012f": 13700, "\u0120vegetables": 13701, "$$": 13702, "\u0120retreat": 13703, "oshi": 13704, "\u0120Shop": 13705, "\u0120Ground": 13706, "\u0120Stop": 13707, "\u0120Hawaii": 13708, "\u0120Ay": 13709, "Perhaps": 13710, "\u0120Beaut": 13711, "uffer": 13712, "enna": 13713, "\u0120productivity": 13714, "Fixed": 13715, "control": 13716, "\u0120absent": 13717, "\u0120Campaign": 13718, "Green": 13719, "\u0120identifying": 13720, "\u0120regret": 13721, "\u0120promoted": 13722, "\u0120Seven": 13723, "\u0120eru": 13724, "neath": 13725, "aughed": 13726, "\u0120Pin": 13727, "\u0120Living": 13728, "Cost": 13729, "omatic": 13730, "mega": 13731, "\u0120Nig": 13732, "ocy": 13733, "\u0120inbox": 13734, "\u0120empire": 13735, "\u0120horizont": 13736, "\u0120branches": 13737, "\u0120metaph": 13738, "Active": 13739, "edi": 13740, "\u0120Film": 13741, "\u0120Something": 13742, "\u0120mods": 13743, "incial": 13744, "\u0120Original": 13745, "Gen": 13746, "\u0120spirits": 13747, "\u0120earning": 13748, "Hist": 13749, "\u0120riders": 13750, "\u0120sacrific": 13751, "MT": 13752, "\u0120VA": 13753, "\u0120Salt": 13754, "\u0120occupation": 13755, "\u0120Mi": 13756, "\u0120disg": 13757, "lict": 13758, "\u0120nit": 13759, "\u0120nodes": 13760, "eem": 13761, "\u0120Pier": 13762, "\u0120hatred": 13763, "psy": 13764, "\u00e3\u0125\u012b": 13765, "\u0120theater": 13766, "\u0120sophisticated": 13767, "\u0120defended": 13768, "\u0120besides": 13769, "\u0120thoroughly": 13770, "\u0120Medicare": 13771, "\u0120blamed": 13772, "arently": 13773, "\u0120crying": 13774, "FOR": 13775, "priv": 13776, "\u0120singing": 13777, "\u0120Il": 13778, "\u0120cute": 13779, "oided": 13780, "olitical": 13781, "\u0120Neuro": 13782, "\u00e5\u00a4": 13783, "\u0120donation": 13784, "\u0120Eagles": 13785, "\u0120Give": 13786, "Tom": 13787, "\u0120substantially": 13788, "\u0120License": 13789, "\u0120Ja": 13790, "\u0120grey": 13791, "\u0120Animal": 13792, "\u0120ER": 13793, "\u0120Und": 13794, "\u0120keen": 13795, "\u0120conclude": 13796, "\u0120Mississippi": 13797, "Engine": 13798, "\u0120Studios": 13799, "Press": 13800, "overs": 13801, "llers": 13802, "\u0120350": 13803, "\u0120Rangers": 13804, "\u0120rou": 13805, "erto": 13806, "Ep": 13807, "issa": 13808, "ivan": 13809, "\u0120seal": 13810, "\u0120Regist": 13811, "display": 13812, "\u0120weaken": 13813, "uum": 13814, "\u0120Commons": 13815, "\u0120Say": 13816, "\u0120cultures": 13817, "\u0120laughed": 13818, "\u0120slip": 13819, "\u0120treatments": 13820, "izable": 13821, "mart": 13822, "\u0120Rice": 13823, "\u0120beast": 13824, "\u0120obesity": 13825, "\u0120Laure": 13826, "iga": 13827, "Which": 13828, "holder": 13829, "\u0120elderly": 13830, "\u0120pays": 13831, "\u0120complained": 13832, "\u0120crop": 13833, "\u0120proc": 13834, "\u0120explosive": 13835, "\u0120Fan": 13836, "\u0120Arsenal": 13837, "Author": 13838, "eful": 13839, "\u0120meals": 13840, "\u0120(-": 13841, "idays": 13842, "\u0120imagination": 13843, "\u0120annually": 13844, "\u0120ms": 13845, "asures": 13846, "Head": 13847, "ikh": 13848, "matic": 13849, "\u0120boyfriend": 13850, "\u0120Computer": 13851, "\u0120bump": 13852, "\u0120surge": 13853, "\u0120Craig": 13854, "\u0120Kirk": 13855, "Del": 13856, "mediate": 13857, "\u0120scenarios": 13858, "\u0120Mut": 13859, "\u0120Stream": 13860, "\u0120competitors": 13861, "\u00d9\u0126": 13862, "\u0120Stanford": 13863, "\u0120Resources": 13864, "azed": 13865, "bage": 13866, "\u0120organis": 13867, "\u0120Release": 13868, "\u0120separately": 13869, "\u0120habits": 13870, "\u0120measurements": 13871, "\u0120Close": 13872, "\u0120accompany": 13873, "\u0120gly": 13874, "\u0120tang": 13875, "\u0120Rou": 13876, "\u0120plugin": 13877, "\u0120convey": 13878, "\u0120Challenge": 13879, "oots": 13880, "jan": 13881, "\u0120curs": 13882, "\u0120Relations": 13883, "keeper": 13884, "\u0120approaching": 13885, "ping": 13886, "Speaking": 13887, "\u0120arrangement": 13888, "\u0120VI": 13889, "arettes": 13890, "\u0120affecting": 13891, "\u0120permits": 13892, "because": 13893, "\u0120useless": 13894, "\u0120Hus": 13895, "!!!!": 13896, "\u0120destroying": 13897, "Unfortunately": 13898, "\u0120fascinating": 13899, "Sem": 13900, "\u0120electoral": 13901, "\u0120transparency": 13902, "\u0120Chaos": 13903, "\u0120volunteer": 13904, "\u0120statistical": 13905, "\u0120activated": 13906, "rox": 13907, "Web": 13908, "HE": 13909, "\u0120Hampshire": 13910, "isive": 13911, "Map": 13912, "\u0120trash": 13913, "\u0120Lawrence": 13914, "stick": 13915, "Cr": 13916, "\u0120rings": 13917, "EXT": 13918, "\u0120operational": 13919, "opes": 13920, "Does": 13921, "\u0120Evans": 13922, "\u0120witnessed": 13923, "Port": 13924, "\u0120launching": 13925, "econom": 13926, "wear": 13927, "\u0120Particip": 13928, "umm": 13929, "cules": 13930, "\u0120RAM": 13931, "\u0120Tun": 13932, "\u0120assured": 13933, "\u0120binary": 13934, "\u0120betray": 13935, "\u0120exploration": 13936, "\u0120Fel": 13937, "\u0120admission": 13938, "itated": 13939, "Sy": 13940, "\u0120avoided": 13941, "\u0120Simulator": 13942, "\u0120celebrated": 13943, "\u0120Electric": 13944, "\u00a5\u0140": 13945, "\u0120cluster": 13946, "itzerland": 13947, "health": 13948, "Line": 13949, "\u0120Nash": 13950, "aton": 13951, "\u0120spare": 13952, "\u0120enterprise": 13953, "\u0120DIS": 13954, "cludes": 13955, "\u0120flights": 13956, "\u0120regards": 13957, "\u0120\u00c3\u0139": 13958, "half": 13959, "\u0120trucks": 13960, "\u0120contacts": 13961, "\u0120uncons": 13962, "\u0120Climate": 13963, "\u0120immense": 13964, "NEW": 13965, "occ": 13966, "ective": 13967, "\u0120embod": 13968, "\u0120patrol": 13969, "\u0120beside": 13970, "\u0120viable": 13971, "\u0120creep": 13972, "\u0120triggered": 13973, "verning": 13974, "\u0120comparable": 13975, "ql": 13976, "\u0120gaining": 13977, "asses": 13978, "\u0120();": 13979, "\u0120Grey": 13980, "\u0120MLS": 13981, "sized": 13982, "\u0120prosper": 13983, "\"?": 13984, "\u0120polling": 13985, "\u0120shar": 13986, "\u0120RC": 13987, "\u0120firearm": 13988, "orient": 13989, "\u0120fence": 13990, "\u0120variations": 13991, "giving": 13992, "\u0120Pi": 13993, "ospel": 13994, "\u0120pledge": 13995, "\u0120cure": 13996, "\u0120spy": 13997, "\u0120violated": 13998, "\u0120rushed": 13999, "\u0120stroke": 14000, "\u0120Blog": 14001, "sels": 14002, "\u0120Ec": 14003, ",''": 14004, "\u0120pale": 14005, "\u0120Collins": 14006, "terror": 14007, "\u0120Canadians": 14008, "\u0120tune": 14009, "\u0120laboratory": 14010, "\u0120nons": 14011, "tarian": 14012, "\u0120disability": 14013, "\u0120Gam": 14014, "\u0120singer": 14015, "alg": 14016, "\u0120Senior": 14017, "\u0120traded": 14018, "\u0120Warrior": 14019, "\u0120infring": 14020, "\u0120Franklin": 14021, "\u0120strain": 14022, "\u0120Swedish": 14023, "\u0120seventh": 14024, "\u0120Benn": 14025, "\u0120Tell": 14026, "\u0120syndrome": 14027, "\u0120wondered": 14028, "iden": 14029, "++++": 14030, "igo": 14031, "\u0120purple": 14032, "\u0120journalism": 14033, "\u0120rebel": 14034, "\u0120fu": 14035, "blog": 14036, "\u0120invite": 14037, "rencies": 14038, "\u0120Contact": 14039, "Israel": 14040, "\u0120Content": 14041, "\u0120cheer": 14042, "\u0120bedroom": 14043, "\u0120Engineering": 14044, "\u0120Queens": 14045, "\u0120dwell": 14046, "\u0120PlayStation": 14047, "\u0120Dim": 14048, "\u0120Colon": 14049, "lr": 14050, "\u0120operates": 14051, "\u0120motivation": 14052, "USA": 14053, "astered": 14054, "Core": 14055, "\u0120Truth": 14056, "olo": 14057, "OSE": 14058, "\u0120Memory": 14059, "\u0120predec": 14060, "\u0120anarch": 14061, "\u01201920": 14062, "\u0120Yam": 14063, "\u00c3\u00a8": 14064, "bid": 14065, "\u0120grateful": 14066, "\u0120excitement": 14067, "\u0120treasure": 14068, "\u0120longest": 14069, "ctive": 14070, "\u0120deserves": 14071, "\u0120reserves": 14072, "\u0120cops": 14073, "\u0120Ottawa": 14074, "\u0120Egyptian": 14075, "anked": 14076, "\u0120artif": 14077, "\u0120hypothesis": 14078, ":/": 14079, "\u0120purchasing": 14080, "\u0120lovely": 14081, "HP": 14082, "\u0120divide": 14083, "\u0120strictly": 14084, "\u0120questioning": 14085, "\u0120taxpayers": 14086, "\u0120Joy": 14087, "\u0120rolls": 14088, "\u0120Heavy": 14089, "\u0120ports": 14090, "\u0120magnetic": 14091, "\u0120inflamm": 14092, "\u0120brush": 14093, "tics": 14094, "\u00e2\u012a\u0134": 14095, "\u0120bottles": 14096, "ppy": 14097, "\u0120padd": 14098, "\u00e3\u0124\u00af": 14099, "million": 14100, "\u0120devastating": 14101, "\u0120compiled": 14102, "\u0120medication": 14103, "\u0120twelve": 14104, "\u0120Perry": 14105, "Space": 14106, "imb": 14107, "your": 14108, "\u0120leaked": 14109, "\u0120Tar": 14110, "\u0120unity": 14111, "\u0120infected": 14112, "\u0120traveled": 14113, "IDE": 14114, "\u0120McDonald": 14115, "txt": 14116, "\u0120Princ": 14117, "\u0120interven": 14118, "\u0120Taiwan": 14119, "\u0120Pow": 14120, "\u0120bearing": 14121, "\u0120Thread": 14122, "\u0120zones": 14123, "izards": 14124, "unks": 14125, "Chapter": 14126, "llor": 14127, "\u0120\u00c2\u00b7": 14128, "\u0120wounds": 14129, "\u0120discretion": 14130, "\u0120succeeded": 14131, "iking": 14132, "\u0120iconic": 14133, "Call": 14134, "\u0120screening": 14135, "\u0120Mis": 14136, "icts": 14137, "\u0120ministers": 14138, "\u0120separation": 14139, "Player": 14140, "\u0120bip": 14141, "\u0120beloved": 14142, "\u0120counting": 14143, "\u0120Eye": 14144, "around": 14145, "inging": 14146, "\u0120tablet": 14147, "\u0120offence": 14148, "inance": 14149, "have": 14150, "\u0120Info": 14151, "\u0120Ninja": 14152, "\u0120protective": 14153, "\u0120Cass": 14154, "Mac": 14155, "\u0120Quality": 14156, "North": 14157, "\u0120ic": 14158, "\u0120Cuba": 14159, "\u0120Chronicle": 14160, "\u0120Property": 14161, "\u0120fastest": 14162, "otos": 14163, "\u0120Germ": 14164, "OWN": 14165, "\u0120boom": 14166, "\u0120Stanley": 14167, "erguson": 14168, "\u0120clever": 14169, "\u0120enters": 14170, "mode": 14171, "terior": 14172, "\u0120Sens": 14173, "\u0120linear": 14174, "ARK": 14175, "\u0120comparing": 14176, "\u0120purely": 14177, "\u0120safer": 14178, "\u0120Potter": 14179, "\u0120cups": 14180, "RT": 14181, "\u0120gluc": 14182, "\u0120attributed": 14183, "\u0120dupl": 14184, "\u0120Pap": 14185, "\u0120precious": 14186, "\u0120pa": 14187, "ictionary": 14188, "\u0120Tig": 14189, "\u0120Too": 14190, "olutions": 14191, "stan": 14192, "\u0120robots": 14193, "\u0120lobb": 14194, "\u0120statute": 14195, "\u0120prevention": 14196, "western": 14197, "160": 14198, "\u0120Active": 14199, "\u0120Maria": 14200, "hal": 14201, "None": 14202, "ellar": 14203, "\u0120KB": 14204, "\u0120Partners": 14205, "\u0120Single": 14206, "\u0120Following": 14207, "ango": 14208, "acious": 14209, "\u0120thou": 14210, "\u0120kg": 14211, "\u0120influential": 14212, "\u0120Friends": 14213, "Sur": 14214, "ainted": 14215, "\u0120forums": 14216, "\u0120starter": 14217, "\u0120citizenship": 14218, "\u0120Election": 14219, "onge": 14220, "otation": 14221, "osph": 14222, ";;;;": 14223, "utical": 14224, "pur": 14225, "eren": 14226, "\u0120accusations": 14227, "bitious": 14228, "abbit": 14229, "\u0120Ord": 14230, "Posted": 14231, "irk": 14232, "\u0120sensitivity": 14233, "iche": 14234, "\u0120Amy": 14235, "\u0120Fab": 14236, "\u0120summit": 14237, "\u0120pedest": 14238, "\u0120rubber": 14239, "\u0120agricultural": 14240, "\u0120cancel": 14241, "AE": 14242, "\u0120inaug": 14243, "\u0120contam": 14244, "\u0120firmly": 14245, "iw": 14246, "stage": 14247, "\u0120Kan": 14248, "\u0120tier": 14249, "\u0120invention": 14250, "\u0120translated": 14251, "\u0120Rules": 14252, "Box": 14253, "Twitter": 14254, "IDS": 14255, "\u0120pizza": 14256, "\u0120debug": 14257, "\u0120Drop": 14258, "vs": 14259, "\u0120horses": 14260, "big": 14261, "\u0120boring": 14262, "\u0120hood": 14263, "\u0120McCain": 14264, "atched": 14265, "\u0120Bros": 14266, "\u0120skip": 14267, "\u0120essay": 14268, "stat": 14269, "\u0120Legends": 14270, "\u0120ammunition": 14271, "auc": 14272, "\u0120shooter": 14273, "\u0120unh": 14274, "\u0120supplied": 14275, "\u0120generic": 14276, "\u0120SK": 14277, "iban": 14278, "yrics": 14279, "\u0120255": 14280, "\u0120climbing": 14281, "Former": 14282, "\u0120flip": 14283, "\u0120jumping": 14284, "\u0120frustration": 14285, "\u0120Terry": 14286, "\u0120neighborhoods": 14287, "\u0120median": 14288, "bean": 14289, "\u0120brains": 14290, "Following": 14291, "\u0120shaped": 14292, "\u0120draws": 14293, "\u0120altered": 14294, "Jack": 14295, "\u0120recipes": 14296, "\u0120skilled": 14297, "wealth": 14298, "achi": 14299, "election": 14300, "\u0120behaviors": 14301, "deals": 14302, "\u0120Until": 14303, "Fe": 14304, "\u0120declaration": 14305, "marks": 14306, "\u0120Between": 14307, "celona": 14308, "\u0120reson": 14309, "\u0120bubble": 14310, "Among": 14311, "\u0120imperial": 14312, "GS": 14313, "\u0120feminist": 14314, "2005": 14315, "\u0120Kyle": 14316, "\u0120accounting": 14317, "\u0120Tele": 14318, "\u0120Tyr": 14319, "\u0120connecting": 14320, "\u0120rehab": 14321, "\u0120Pred": 14322, "sim": 14323, "\u0120meantime": 14324, "\u0120physician": 14325, "MW": 14326, "\u0120Campbell": 14327, "\u0120Brandon": 14328, "\u0120contributing": 14329, "\u0120Rule": 14330, "\u0120Weight": 14331, "\u0120Nap": 14332, "\u0120interactive": 14333, "\u0120vag": 14334, "\u0120helmet": 14335, "\u0120Comb": 14336, "four": 14337, "\u0120shipped": 14338, "\u0120completing": 14339, "\u0120PD": 14340, "PDATE": 14341, "\u0120spreading": 14342, "\u0120scary": 14343, "erving": 14344, "\u0120Gas": 14345, "\u0120frank": 14346, "school": 14347, "\u0120romantic": 14348, "\u0120stabil": 14349, "Rob": 14350, "\u0120accurately": 14351, "\u0120acute": 14352, "\u0120Hann": 14353, "\u0120symbols": 14354, "\u0120civilization": 14355, "\u0120AW": 14356, "\u0120lightning": 14357, "\u0120considers": 14358, "\u0120venue": 14359, "\u0120\u00d7": 14360, "\u0120oven": 14361, "\u0120SF": 14362, "his": 14363, "\u0120nu": 14364, "\u0120Learn": 14365, "\u0120peoples": 14366, "\u0120std": 14367, "\u0120slee": 14368, "\u0120slic": 14369, "\u0120Statistics": 14370, "\u0120corners": 14371, "\u0120Baker": 14372, "\u0120:)": 14373, "mentation": 14374, "olver": 14375, "\u0120laughing": 14376, "\u0120Todd": 14377, "onde": 14378, "\u0120Hills": 14379, "\u0120nuts": 14380, "\u0120Woman": 14381, "plane": 14382, "\u0120liver": 14383, "\u0120Inside": 14384, "Sorry": 14385, "\u0120agrees": 14386, "\u0120fundament": 14387, "\u0120Fisher": 14388, "\u0120auction": 14389, "\u0120threads": 14390, "glas": 14391, "\u0120Basic": 14392, "\u0120Nat": 14393, "\u0120lacking": 14394, "\u0120celebration": 14395, "ju": 14396, "\u0120silly": 14397, "Euro": 14398, "\u0120tatt": 14399, "ighty": 14400, "controlled": 14401, "Test": 14402, "\u0120Singh": 14403, "\u0120rage": 14404, "\u0120rhyth": 14405, "offic": 14406, "\u0120Phantom": 14407, "\u0120headlines": 14408, "\u0120responding": 14409, "\u0120Morning": 14410, "\u0120vitamin": 14411, "\u0120boots": 14412, "\u0120Site": 14413, "alin": 14414, "pi": 14415, "\u0120viral": 14416, "\u0120UC": 14417, "DER": 14418, "\u0120Sex": 14419, "\u0120stocks": 14420, "current": 14421, "\u0120churches": 14422, "\u0120Rare": 14423, "\u0120Murphy": 14424, "\u0120denial": 14425, "\u0120Gaming": 14426, "\u0120toug": 14427, "\u0120nick": 14428, "\u0120makers": 14429, "\u0120Ronald": 14430, "\u0120generous": 14431, "\u0120Doc": 14432, "\u0120Morris": 14433, "\u0120transformed": 14434, "\u0120Normal": 14435, "\u0120104": 14436, "\u0120Kickstarter": 14437, "\u0120Upon": 14438, "Online": 14439, "\u0120IRS": 14440, "\u0120wrap": 14441, "\u0120loving": 14442, "\u0120arrives": 14443, "\u0120Due": 14444, "\u0120heter": 14445, "\u0120Made": 14446, "\u0120rental": 14447, "\u0120belongs": 14448, "\u0120attorneys": 14449, "\u0120crops": 14450, "\u0120matched": 14451, "ulum": 14452, "oline": 14453, "109": 14454, "\u0120dispar": 14455, "\u0120buyers": 14456, "\u0120Cambridge": 14457, "\u0120ethics": 14458, "roups": 14459, "\u0120justified": 14460, "\u0120marginal": 14461, "\u0120respected": 14462, "winning": 14463, "\u0120nodded": 14464, "\u0120Serge": 14465, "\u0120Former": 14466, "Craft": 14467, "################": 14468, "\u0120Warner": 14469, "\u0120dash": 14470, "ete": 14471, "\u0120entert": 14472, "\u0120Escape": 14473, "outheast": 14474, "\u0120knees": 14475, "\u0120Bomb": 14476, "\u0120rug": 14477, "Pass": 14478, "\u0120attitudes": 14479, "government": 14480, "\u0120Prior": 14481, "\u0120qualities": 14482, "\u0120notification": 14483, "\u0120Phone": 14484, "lie": 14485, "\u0120anticipated": 14486, "\u0120Combat": 14487, "\u0120Barry": 14488, "\u01201982": 14489, "Users": 14490, "oner": 14491, "\u0120computing": 14492, "\u0120Connecticut": 14493, "\u0120lesser": 14494, "\u0120peers": 14495, "\u0120Cu": 14496, "\u0120technically": 14497, "\u0120submission": 14498, "\u0120Universal": 14499, "\u0120manually": 14500, "ourge": 14501, "\u0120respondents": 14502, "\u0120BTC": 14503, "\u0120Host": 14504, "\u0120fare": 14505, "\u0120Bird": 14506, "\u0120receipt": 14507, "also": 14508, "\u0120jack": 14509, "\u0120agriculture": 14510, "\u0120skull": 14511, "\u0120!=": 14512, "\u0120passive": 14513, "\u0120CI": 14514, "\u0120societies": 14515, "\u0120reminded": 14516, "\u0120interference": 14517, "Buy": 14518, "\u0120\u00e2\u013e": 14519, "gon": 14520, "\u0120scrutiny": 14521, "\u0120Witch": 14522, "\u0120conducting": 14523, "\u0120\u00e3\u0125": 14524, "\u0120exchanges": 14525, "\u0120Mitchell": 14526, "\u0120inhabit": 14527, "\u0120twist": 14528, "BD": 14529, "\u0120wherever": 14530, "groupon": 14531, "\u0120jokes": 14532, "\u0120Benjamin": 14533, "\u0120Random": 14534, "frame": 14535, "\u0120Lions": 14536, "\u0120highlighted": 14537, "\u0120Arkansas": 14538, "Ent": 14539, "\u0120pile": 14540, "\u0120prelim": 14541, "gs": 14542, "minded": 14543, "\u0120felony": 14544, "\u0120GA": 14545, "\u0120Luck": 14546, "\u0120practically": 14547, "\u0120Bos": 14548, "\u0120actress": 14549, "Dam": 14550, "\u0120Bou": 14551, "\u0120visa": 14552, "\u0120embedded": 14553, "\u0120hybrid": 14554, "\u0120earliest": 14555, "\u0120sooner": 14556, "social": 14557, "\u0120HA": 14558, "\u0120steep": 14559, "\u0120disadvant": 14560, "\u0120exploit": 14561, "\u0120Egg": 14562, "\u0120Ultra": 14563, "\u0120necessity": 14564, "Local": 14565, "iege": 14566, "\u0120dated": 14567, "\u0120masses": 14568, "\u0120subscription": 14569, "pless": 14570, "\u0120anonym": 14571, "\u0120presumably": 14572, "Blue": 14573, "Their": 14574, "asketball": 14575, "\u0120Philip": 14576, "\u0120comed": 14577, "loaded": 14578, "rane": 14579, "\u0120reflection": 14580, "China": 14581, "\u0120extends": 14582, "\u0120forming": 14583, "\u0120unders": 14584, "2001": 14585, "\u0120grat": 14586, "\u0120concentrations": 14587, "\u0120insulin": 14588, "\u0120secular": 14589, "\u0120whilst": 14590, "\u0120winners": 14591, "Advertisements": 14592, "\u0120deliberately": 14593, "\u0120Working": 14594, "\u0120sink": 14595, "etics": 14596, "dale": 14597, "\u0120mandate": 14598, "\u0120gram": 14599, "\u0120vacation": 14600, "\u0120warnings": 14601, "ripp": 14602, "\u0120THAT": 14603, "\u0120commentary": 14604, "\u0120intu": 14605, "\u0120aest": 14606, "\u0120reasoning": 14607, "\u0120breakdown": 14608, "\u0120Zombie": 14609, "\u0120-->": 14610, "\u0120Political": 14611, "cott": 14612, "\u0120thrust": 14613, "\u0120technological": 14614, "\u0120deciding": 14615, "\u0120trafficking": 14616, "Long": 14617, "Welcome": 14618, "prising": 14619, "\u0120Communications": 14620, "\u0120endors": 14621, "\u0120swift": 14622, "\u0120metabol": 14623, "coins": 14624, "resa": 14625, "\u0120HTTP": 14626, "\u0120enroll": 14627, "\u0120Happy": 14628, "usr": 14629, "intage": 14630, "\u0120[\"": 14631, "uably": 14632, "\u0120Material": 14633, "\u0120repeal": 14634, "Sept": 14635, "kh": 14636, "\u0120Modi": 14637, "\u0120underneath": 14638, "\u0120IL": 14639, "shore": 14640, "\u0120diagnosed": 14641, "aceutical": 14642, "\u0120shower": 14643, "aux": 14644, "\u0120Switch": 14645, "\u0120Strength": 14646, "\u0120jihad": 14647, "national": 14648, "\u0120trauma": 14649, "ussy": 14650, "oni": 14651, "\u0120consolid": 14652, "\u0120calories": 14653, "\u0120Flynn": 14654, "agged": 14655, "168": 14656, "\u0120Pink": 14657, "\u0120fulfill": 14658, "\u0120chains": 14659, "\u0120notably": 14660, "\u0120AV": 14661, "Life": 14662, "\u0120Chuck": 14663, "mus": 14664, "\u0120Urban": 14665, "\u0120Hend": 14666, "\u0120deposit": 14667, "\u0120Sad": 14668, "\u0120affair": 14669, "ORK": 14670, "ieval": 14671, "\u0120FDA": 14672, "\u0120trop": 14673, "\u0120Overall": 14674, "\u0120virtue": 14675, "\u0120satisfaction": 14676, "aund": 14677, "\u0120lun": 14678, "\u0120Switzerland": 14679, "\u0120Operation": 14680, "process": 14681, "\u0120shook": 14682, "\u0120counties": 14683, "leased": 14684, "\u0120Charlotte": 14685, "112": 14686, "\u0120transcript": 14687, "\u0120redd": 14688, "push": 14689, "\u0120Hey": 14690, "\u0120Analysis": 14691, "[\"": 14692, "\u0120alternatives": 14693, "ardless": 14694, "\u0120eleph": 14695, "\u0120prejud": 14696, "\u0120Leaf": 14697, "Having": 14698, "\u0120Hub": 14699, "\u0120expressions": 14700, "\u0120Volume": 14701, "\u0120shocking": 14702, "\u0120Reds": 14703, "\u0120readily": 14704, "\u0120planets": 14705, "adata": 14706, "\u0120collapsed": 14707, "\u0120Madrid": 14708, "\u0120irrit": 14709, "ipper": 14710, "\u0120Enc": 14711, "\u0120Wire": 14712, "\u0120buzz": 14713, "\u0120GP": 14714, "asha": 14715, "\u0120accidentally": 14716, "uru": 14717, "\u0120frustrated": 14718, "\u0120SA": 14719, "\u0120hungry": 14720, "\u0120Huff": 14721, "\u0120labels": 14722, "anto": 14723, "\u0120EP": 14724, "\u0120barriers": 14725, ")|": 14726, "\u0120Berkeley": 14727, "\u0120Jets": 14728, "\u0120pairs": 14729, "\u0120Lan": 14730, "James": 14731, "\u0120Bear": 14732, "\u0120humor": 14733, "\u0120Liberty": 14734, "\u0120magnitude": 14735, "\u0120aging": 14736, "\u0120Mason": 14737, "\u0120friendship": 14738, "umbling": 14739, "\u0120emerge": 14740, "\u0120newspapers": 14741, "\u0120ambitious": 14742, "\u0120Richards": 14743, "aternal": 14744, "\u01201981": 14745, "\u0120cookies": 14746, "\u0120sculpt": 14747, "\u0120pursuit": 14748, "Location": 14749, "\u0120scripts": 14750, "pc": 14751, "\u0120arrangements": 14752, "\u0120diameter": 14753, "\u0120loses": 14754, "amation": 14755, "\u0120liqu": 14756, "\u0120Jake": 14757, "arette": 14758, "\u0120understands": 14759, "\u0120Zen": 14760, "vm": 14761, "\u0120approve": 14762, "\u0120wip": 14763, "\u0120ultra": 14764, "\u0120intend": 14765, "\u0120DI": 14766, "ascular": 14767, "\u0120stays": 14768, "\u0120Kor": 14769, "\u0120Kl": 14770, "\u0120investing": 14771, "La": 14772, "\u0120believing": 14773, "bad": 14774, "mouth": 14775, "\u0120taxpayer": 14776, "\u00e3\u0125\u0125": 14777, "\u0120Quebec": 14778, "\u0120lap": 14779, "\u0120Swiss": 14780, "drop": 14781, "\u0120drain": 14782, "iri": 14783, "etc": 14784, "ften": 14785, "\u0120Nex": 14786, "\u0120straw": 14787, "\u0120screaming": 14788, "\u0120counted": 14789, "\u0120damaging": 14790, "\u0120ambassador": 14791, "century": 14792, "\u0120prox": 14793, "\u0120arrests": 14794, "uv": 14795, "ilateral": 14796, "\u0120Charg": 14797, "\u0120prescribed": 14798, "\u0120independently": 14799, "\u0120fierce": 14800, "\u0120Baby": 14801, "\u0120brave": 14802, "\u0120suits": 14803, "=>": 14804, "\u0120baseline": 14805, "\u0120Rate": 14806, "\u0120islands": 14807, "\u0120((": 14808, "green": 14809, "ixels": 14810, "\u0120namely": 14811, "\u0120Village": 14812, "than": 14813, "amy": 14814, "Version": 14815, "gmail": 14816, "entials": 14817, "\u0120Sud": 14818, "\u0120Melbourne": 14819, "\u0120arriving": 14820, "\u0120quantum": 14821, "eff": 14822, "ropolitan": 14823, "Tri": 14824, "\u0120funeral": 14825, "\u0120IR": 14826, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 14827, "\u0120Cob": 14828, "itably": 14829, "\u0120turb": 14830, "\u0120combo": 14831, "Review": 14832, "\u0120deployment": 14833, "uity": 14834, "\u0120Bott": 14835, "\u0120invisible": 14836, "\u0120rendering": 14837, "\u0120unlocked": 14838, "\u0120aqu": 14839, "\u0120Vladimir": 14840, "\u0120pad": 14841, "\u0120Brain": 14842, "\u0120Legacy": 14843, "dragon": 14844, "\u0120Kurdish": 14845, "\u0120sounded": 14846, "\u0120detained": 14847, "\u0120DM": 14848, "gary": 14849, "\u0120daughters": 14850, "\u0120disturbing": 14851, "uka": 14852, "\u0120Parad": 14853, "\u0120tast": 14854, "\u0120unfortunate": 14855, "\u0120ul": 14856, "emin": 14857, "\u0120attendance": 14858, "trl": 14859, "\u0120parks": 14860, "\u0120Memorial": 14861, "\u0120Alice": 14862, "othy": 14863, "guard": 14864, "\u0120Dise": 14865, "\u0120Shan": 14866, "\u0120Forum": 14867, "Rich": 14868, "\u0120shifted": 14869, "uez": 14870, "\u0120lighter": 14871, "\u0120Magn": 14872, "\u0120cod": 14873, "Sch": 14874, "hammad": 14875, "Pub": 14876, "350": 14877, "\u0120Pokemon": 14878, "\u0120prototype": 14879, "\u0120unre": 14880, "Base": 14881, "\u0120Students": 14882, "\u0120Reply": 14883, "\u0120Communist": 14884, "\u0120gau": 14885, "\u0120Tyler": 14886, "IZ": 14887, "\u0120participated": 14888, "\u0120suprem": 14889, "\u0120Details": 14890, "\u0120vessels": 14891, "rod": 14892, "\u0120tribe": 14893, "keep": 14894, "\u0120assumptions": 14895, "\u0120pound": 14896, "\u0120crude": 14897, "\u0120Available": 14898, "\u0120swimming": 14899, "\u0120inclusion": 14900, "\u0120advances": 14901, "culation": 14902, "\u0120conservation": 14903, "\u0120overd": 14904, "\u0120Buffalo": 14905, "Article": 14906, "edge": 14907, "\u0120awa": 14908, "\u0120Madison": 14909, "\u0120sidew": 14910, "\u0120catast": 14911, "\u0120Krist": 14912, "ucle": 14913, "\u0120Highway": 14914, "\u0120Terror": 14915, "\u0120activation": 14916, "\u0120unconscious": 14917, "\u0120Satan": 14918, "\u0120Susan": 14919, "illery": 14920, "\u0120arranged": 14921, "iop": 14922, "\u0120rumors": 14923, "urring": 14924, "think": 14925, "\u0120Keith": 14926, "\u0120Kind": 14927, "\u0120avoiding": 14928, "byn": 14929, "nut": 14930, "\u0120Speaker": 14931, "rus": 14932, "names": 14933, "\u0120guilt": 14934, "\u0120Olympics": 14935, "\u0120sail": 14936, "\u0120Mes": 14937, "levant": 14938, "\u0120Columbus": 14939, "aft": 14940, "City": 14941, "South": 14942, "\u0120Harvey": 14943, "\u0120Pun": 14944, "Several": 14945, "\u0120mentally": 14946, "\u0120impress": 14947, "mount": 14948, "\u0120Ubuntu": 14949, "\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136": 14950, "\u0120Superman": 14951, "\u0120MPs": 14952, "\u0120intentions": 14953, "\u0120Racing": 14954, "\u0120likelihood": 14955, "\u0120240": 14956, "Total": 14957, "\u0120toys": 14958, "\u0120Watson": 14959, "\u0120urge": 14960, "Lear": 14961, "\u0120Paper": 14962, "\u0120occurring": 14963, "\u0120Beng": 14964, "\u0120Cert": 14965, "\u0120stones": 14966, "Tim": 14967, "\u0120Twin": 14968, "zb": 14969, "\u0120Dynam": 14970, "\u0120politician": 14971, "kens": 14972, "\u0120Enterprise": 14973, "UTERS": 14974, "\u0120abol": 14975, "\u0120refresh": 14976, "\u0120arbitrary": 14977, "pection": 14978, "\u0120troubles": 14979, "\u0120});": 14980, "tv": 14981, "\u0120pilots": 14982, "\u0120distribute": 14983, "\u0120audit": 14984, "\u0120pause": 14985, "original": 14986, "\u0120rivals": 14987, "\u00c2\u00a3": 14988, "Fig": 14989, "TL": 14990, "abil": 14991, "rying": 14992, "Lin": 14993, "ioned": 14994, "lon": 14995, "\u0120fancy": 14996, "\u0120crashed": 14997, "\u0120tract": 14998, "\u0120shed": 14999, "\u0120consume": 15000, "Based": 15001, "download": 15002, "init": 15003, "\u0120voltage": 15004, "Introdu": 15005, "\u0120condemned": 15006, "\u0120Finance": 15007, "respect": 15008, "\u0120excluded": 15009, "\u0120establishing": 15010, "heric": 15011, "\u0120heritage": 15012, "\u0120spectacular": 15013, "\u0120unst": 15014, "\u0120Snowden": 15015, "\u0120Lane": 15016, "San": 15017, "\u0120protections": 15018, "struction": 15019, "incinn": 15020, "\u0120macro": 15021, "Custom": 15022, "iosity": 15023, "\u0120esp": 15024, "\u0120functioning": 15025, "\u0120mush": 15026, "\u0120puzzle": 15027, "\u0120ethical": 15028, "Mal": 15029, "\u0120governing": 15030, "\u0120Ferguson": 15031, "\u0120restored": 15032, "\u0120stressed": 15033, "\u0120Counter": 15034, "\u0120Kas": 15035, "clip": 15036, "ANS": 15037, "\u0120seiz": 15038, "UK": 15039, "byss": 15040, "oldown": 15041, "api": 15042, "\u0120permanently": 15043, "ounters": 15044, "West": 15045, "Through": 15046, "Light": 15047, "atoes": 15048, "\u0120neat": 15049, "\u0120cord": 15050, "urer": 15051, "\u0120severely": 15052, "\u0120Aven": 15053, "\u0120interrog": 15054, "\u0120triple": 15055, "Given": 15056, "Number": 15057, "\u0120arise": 15058, "\u0120sher": 15059, "plant": 15060, "\u0120flower": 15061, "\u0120Cou": 15062, "\u0120ate": 15063, "\u0120newer": 15064, "bul": 15065, "\u0120meanwhile": 15066, "\u0120Lair": 15067, "\u0120adjustment": 15068, "\u0120Copyright": 15069, "\u0120divers": 15070, "iological": 15071, "\u0120gamers": 15072, "oat": 15073, "\u0120historically": 15074, "\u0120analog": 15075, "\u0120longtime": 15076, "\u0120prescription": 15077, "\u0120Mist": 15078, "\u0120Hyper": 15079, "\u0120Maine": 15080, "\u0120Deity": 15081, "\u0120multipl": 15082, "\u0120Reincarn": 15083, "\u0120Hyd": 15084, "\u0120Pic": 15085, "Sil": 15086, "rants": 15087, "\u0120Cris": 15088, ".;": 15089, "({": 15090, "ependence": 15091, "\u0120recy": 15092, "ateur": 15093, "\u0120quad": 15094, "\u0120glob": 15095, "\u0120conced": 15096, "team": 15097, "\u0120capitalist": 15098, "\u0120Lot": 15099, "\u0120royal": 15100, "\u0120Cyber": 15101, "\u0120blacks": 15102, "metic": 15103, "riv": 15104, "\u0120Danny": 15105, "\u0120spo": 15106, "\u0120RO": 15107, "\u0120animated": 15108, "rypted": 15109, "\u0120Deputy": 15110, "\u0120rendered": 15111, "FE": 15112, "\u0120streak": 15113, "\u0120clouds": 15114, "\u0120Doug": 15115, "~~~~~~~~": 15116, "\u0120discour": 15117, "\u0120Veh": 15118, "\u0120psychology": 15119, "\u0120Journey": 15120, "\u0120crystal": 15121, "\u0120Frost": 15122, "\u0120suspicion": 15123, "\u0120relate": 15124, "orus": 15125, "\u0120Crypt": 15126, "\u0120NVIDIA": 15127, "comed": 15128, "uting": 15129, "incinnati": 15130, "\u0120vulnerability": 15131, "ostic": 15132, "\u0120isolation": 15133, "\u0120cooling": 15134, "\u0120Coalition": 15135, "\u0120119": 15136, "Four": 15137, "\u0120Deal": 15138, "\u0120\u00e2\u012b": 15139, "semble": 15140, "rament": 15141, "\u0120Barcelona": 15142, "\u0120102": 15143, "\u0120cocaine": 15144, "ocalypse": 15145, "Feb": 15146, "ogenic": 15147, "\u0120mutation": 15148, "\u0120cryptoc": 15149, "\u0120Kel": 15150, "\u0120Git": 15151, "ais": 15152, "\u0120sisters": 15153, "ANK": 15154, "\u0120activate": 15155, "Ter": 15156, "\u0120dread": 15157, "ylon": 15158, "\u0120propri": 15159, "Aust": 15160, "\u0120Default": 15161, "\u0120outdoor": 15162, "\u0120sheer": 15163, "ceive": 15164, "\u0120gently": 15165, "\u00d0\u00be": 15166, "Program": 15167, "\u0120\u00e2\u0128\u0134": 15168, "\u0120vegan": 15169, "\u0120Crus": 15170, "\u0120responsibilities": 15171, "\u0120HR": 15172, "OLD": 15173, "\u0120prevents": 15174, "\u0120stiff": 15175, "\u0120Were": 15176, "\u0120athletic": 15177, "\u0120Score": 15178, "\u0120):": 15179, "\u0120columns": 15180, "\u0120Loc": 15181, "available": 15182, "\u0120Fram": 15183, "\u0120Sessions": 15184, "\u0120companion": 15185, "\u0120packs": 15186, "140": 15187, "\u0120Knights": 15188, "\u0120fart": 15189, "\u0120streams": 15190, "\u0120shore": 15191, "\u0120appeals": 15192, "\u0120Performance": 15193, "haul": 15194, "\u0120Stra": 15195, "\u0120Nag": 15196, "103": 15197, "\u0120Transportation": 15198, "BB": 15199, "Ev": 15200, "zan": 15201, "Public": 15202, "\u0120twin": 15203, "ulsion": 15204, "Mult": 15205, "\u0120electro": 15206, "\u0120statue": 15207, "ationally": 15208, "\u0120Nort": 15209, "\u0120inspection": 15210, "/*": 15211, "igue": 15212, "\u0120compassion": 15213, "\u0120Tales": 15214, "\u0120Stein": 15215, "\u0120Screen": 15216, "\u0120Bug": 15217, "\u0120Lion": 15218, "girl": 15219, "\u0120withdrawal": 15220, "\u0120objectives": 15221, "\u0120bloody": 15222, "\u0120preliminary": 15223, "\u0120jacket": 15224, "\u0120dimensions": 15225, "\u0120Cool": 15226, "\u0120Occup": 15227, "\u0120wreck": 15228, "\u0120doubled": 15229, "anking": 15230, "\u01201975": 15231, "\u0120glasses": 15232, "\u0120Wang": 15233, "prov": 15234, "Path": 15235, "connected": 15236, "\u0120Multi": 15237, "\u0120Norway": 15238, "agonist": 15239, "\u0120feared": 15240, "\u0120touching": 15241, "\u0120arguably": 15242, "\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af": 15243, "\u0120NCAA": 15244, "chem": 15245, "\u0120spat": 15246, "\u0120WWE": 15247, "\u0120Cel": 15248, "igger": 15249, "\u0120attacker": 15250, "\u0120Join": 15251, "object": 15252, "etta": 15253, "\u0120eliminated": 15254, "det": 15255, "\u0120destruct": 15256, "\u0120Lucas": 15257, "ctuary": 15258, "180": 15259, "\u0120Brady": 15260, "\u0120Blues": 15261, "Bay": 15262, "aukee": 15263, "\u0120timeline": 15264, "\u0120delegates": 15265, "written": 15266, "ufficient": 15267, "\u0120shapes": 15268, "Copyright": 15269, "ouble": 15270, "service": 15271, "\u0120pione": 15272, "\u0120colleges": 15273, "\u0120rows": 15274, "\u0120spite": 15275, "\u0120assessed": 15276, "360": 15277, "\u0120lease": 15278, "\u0120confidential": 15279, "cker": 15280, "\u0120Manning": 15281, "\u0120Voice": 15282, "\u0120sealed": 15283, "\u0120calculate": 15284, "NO": 15285, "\u0120Assistant": 15286, "\u0120teenager": 15287, "ulent": 15288, "atherine": 15289, "\u0120mock": 15290, "\u0120diamond": 15291, "\u0120fest": 15292, "\u0120switched": 15293, "\u0120resume": 15294, "\u0120Puerto": 15295, "\u0120lanes": 15296, "iration": 15297, "\u0120Similarly": 15298, "\u0120rod": 15299, "\u0120Sel": 15300, "\u0120Palace": 15301, "\u0120Limited": 15302, "eous": 15303, "\u0120variant": 15304, "\u0120ward": 15305, "\u0120))": 15306, "Show": 15307, "OOK": 15308, "Alex": 15309, "\u0120Nep": 15310, "bris": 15311, "\u0120Wikipedia": 15312, "\u0120exceptional": 15313, "\u0120manages": 15314, "\u0120Draw": 15315, "Again": 15316, "\u0120copper": 15317, "utt": 15318, "\u0120exports": 15319, "\u0120portfolio": 15320, "\u0120elevated": 15321, "Rated": 15322, "\u0120Otherwise": 15323, "\u0120Tact": 15324, "\u0120Shel": 15325, "\u0120TX": 15326, "\"\u00e2\u0122\u0136": 15327, "\u0120resur": 15328, "\u0120Wa": 15329, "venant": 15330, "\u0120monetary": 15331, "people": 15332, "Email": 15333, "\u0120fifty": 15334, "\u0120Sweet": 15335, "\u0120Malaysia": 15336, "\u0120confusing": 15337, "\u0120Rio": 15338, "uda": 15339, "utenant": 15340, "\");": 15341, "\u0120praised": 15342, "\u0120volumes": 15343, "turn": 15344, "\u0120mature": 15345, "\u0120nonprofit": 15346, "\u0120passionate": 15347, "\u0120Private": 15348, "\u0120103": 15349, "\u0120descend": 15350, "\u00e7\u00a5\u0140": 15351, "uffy": 15352, "headed": 15353, "Whether": 15354, "rien": 15355, "zech": 15356, "beit": 15357, "\u0120chrom": 15358, "\u0120McM": 15359, "\u0120dancing": 15360, "\u0120eleg": 15361, "\u0120Noticed": 15362, "115": 15363, "\u0120advocacy": 15364, "ENTS": 15365, "ambling": 15366, "\u0120Minor": 15367, "\u0120Finn": 15368, "\u0120priorities": 15369, "\u0120thereof": 15370, "\u0120Stage": 15371, "\u0120Rogers": 15372, "\u0120substitute": 15373, "\u0120Jar": 15374, "\u0120Jefferson": 15375, "\u0120lightly": 15376, "102": 15377, "\u0120Lisa": 15378, "uits": 15379, "ysical": 15380, "\u0120shifts": 15381, "\u0120drones": 15382, "\u0120workplace": 15383, "\u0120resid": 15384, "ensed": 15385, "ahn": 15386, "\u0120preferences": 15387, "server": 15388, "\u0120debates": 15389, "doc": 15390, "\u0120Gods": 15391, "\u0120helicopter": 15392, "\u0120honour": 15393, "\u0120considerably": 15394, "eded": 15395, "\u0120Female": 15396, "\u0120Anne": 15397, "\u0120reun": 15398, "\u0120Face": 15399, "\u0120Hallow": 15400, "\u0120Budget": 15401, "\u0120condemn": 15402, "\u0120tender": 15403, "Prof": 15404, "ocratic": 15405, "\u0120Turner": 15406, "\u0120Agric": 15407, "\u01201976": 15408, "\u0120apt": 15409, "disc": 15410, "\u0120Fighter": 15411, "\u0120Aur": 15412, "\u0120garbage": 15413, "input": 15414, "\u0120Karl": 15415, "\u0120Oliver": 15416, "\u0120Language": 15417, "kn": 15418, "Non": 15419, "\u0120Clar": 15420, "\u0120traditions": 15421, "\u0120advertisement": 15422, "\u0120Sor": 15423, "\u0120archive": 15424, "\u0120villages": 15425, "750": 15426, "\u0120implementing": 15427, "waukee": 15428, "\u0120dietary": 15429, "\u0120switching": 15430, "Republic": 15431, "\u0120velocity": 15432, "\u0120cit": 15433, "\u0120Awards": 15434, "\u0120financing": 15435, "\u0120lasted": 15436, ")]": 15437, "\u0120reminder": 15438, "Person": 15439, "\u0120precision": 15440, "\u0120designers": 15441, "\u0120Fried": 15442, "\u0120Border": 15443, "\u0120tragic": 15444, "\u0120wield": 15445, "\u0120initiatives": 15446, "\u0120Tank": 15447, "wer": 15448, "\u0120joins": 15449, "Ro": 15450, "inery": 15451, "\u0120arrow": 15452, "\u0120generating": 15453, "founder": 15454, "\u0120searches": 15455, "\u0120randomly": 15456, "Access": 15457, "\u0120batch": 15458, "\u0120posed": 15459, "lat": 15460, "\u0120pursuing": 15461, "asa": 15462, "\u0120testified": 15463, "forming": 15464, "\u0120Shar": 15465, "wiki": 15466, "\u0120Either": 15467, "Sometimes": 15468, "\u0120senators": 15469, "\u0120Johnny": 15470, "\u0120Taliban": 15471, "\u0120GPS": 15472, "\":\"/": 15473, "\u00e3\u0123\u00ae\u00e5": 15474, "\u0120analyzed": 15475, "\u0120Rubio": 15476, "\u0120Movement": 15477, "opard": 15478, "iii": 15479, "Stand": 15480, "fight": 15481, "\u0120ignoring": 15482, "iang": 15483, "\u0120GN": 15484, "soever": 15485, "\u0120STAT": 15486, "\u0120refusing": 15487, "\u0120sweat": 15488, "\u0120bay": 15489, "PORT": 15490, "irmed": 15491, "aky": 15492, "\u0120dispro": 15493, "\u0120labeled": 15494, "\u0120108": 15495, "Hello": 15496, "\u0120pleasant": 15497, "aba": 15498, "\u0120triumph": 15499, "\u0120aboard": 15500, "\u0120incom": 15501, "\u0120Crow": 15502, "lett": 15503, "\u0120folk": 15504, "\u0120chase": 15505, "``": 15506, "\u0120Brus": 15507, "\u0120teens": 15508, "cue": 15509, "\u0120terrain": 15510, "hyd": 15511, "ilight": 15512, "ORY": 15513, "Support": 15514, "ews": 15515, "lli": 15516, "raints": 15517, "\u0120Cand": 15518, "\u0120abused": 15519, "achment": 15520, "larg": 15521, "Bas": 15522, "\u0120Cancer": 15523, "\u01201978": 15524, "\u0120supporter": 15525, "access": 15526, "\u0120Termin": 15527, "\u0120Tampa": 15528, "\u0120ANY": 15529, "\u0120newest": 15530, "\u0120Criminal": 15531, "edu": 15532, "\u01201930": 15533, "\u0120admits": 15534, "\u0120ende": 15535, "\u0120failures": 15536, "urate": 15537, "fulness": 15538, "cycl": 15539, "\u0120Subject": 15540, "\u0120infinite": 15541, "three": 15542, "WA": 15543, "pit": 15544, "\u0120Install": 15545, "Rad": 15546, "iliation": 15547, "GM": 15548, "\u0120continent": 15549, "\u0120accommodate": 15550, "\u0120Clay": 15551, "\u0120pup": 15552, "\u0120Function": 15553, "\u0120hammer": 15554, "\u0120Alberta": 15555, "\u0120revised": 15556, "\u0120minorities": 15557, "\u0120measurement": 15558, "Connell": 15559, "\u0120disable": 15560, "\u0120Mix": 15561, "Incre": 15562, "\u0120fork": 15563, "\u0120Rosen": 15564, "\u0120implies": 15565, "umblr": 15566, "ANG": 15567, "\u0120proteins": 15568, "\u0120aggression": 15569, "\u0120facilitate": 15570, "SN": 15571, "\u0120illegally": 15572, "uer": 15573, "\u0120academ": 15574, "\u0120puzz": 15575, "\u0120Shift": 15576, "pay": 15577, "ollo": 15578, "\u0120audiences": 15579, "Build": 15580, "\u0120noble": 15581, "\u0120syntax": 15582, "\u00e2\u013a\u0127": 15583, "\u0120beam": 15584, "\u0120Bed": 15585, "\u0120Ald": 15586, "\u0120origins": 15587, "video": 15588, "\u01201977": 15589, "\u0120Assault": 15590, "\u0120garage": 15591, "Team": 15592, "\u0120verdict": 15593, "\u0120dwar": 15594, "\u0120Virtual": 15595, "event": 15596, "Keep": 15597, "\u0120sentiment": 15598, "\u0120wildlife": 15599, "shirt": 15600, "\u0120burg": 15601, "\u0120recommendation": 15602, "represent": 15603, "\u0120gallery": 15604, "owners": 15605, "\u0120scholar": 15606, "\u0120convenience": 15607, "\u0120Swift": 15608, "\u0120convinc": 15609, "Cap": 15610, "\u0120warfare": 15611, "\u0120Visual": 15612, "\u0120constitute": 15613, "\u0120abort": 15614, "\u0120Weather": 15615, "\u0120Looking": 15616, "\u0120Hem": 15617, "\u0120martial": 15618, "\u0120incoming": 15619, "etition": 15620, "\u0120tolerance": 15621, "\u0120Created": 15622, "\u0120flows": 15623, "\u0120Elder": 15624, "\u0120souls": 15625, "\u0120foul": 15626, "\u0120Pain": 15627, "\u0120CAN": 15628, "\u0120220": 15629, "bc": 15630, "hend": 15631, "\u0120genius": 15632, "Real": 15633, "\u0120Wr": 15634, "ometer": 15635, "pad": 15636, "\u0120limiting": 15637, "\u0120Si": 15638, "\u0120Lore": 15639, "\u0120Adventures": 15640, "\u0120varied": 15641, "Disc": 15642, "fin": 15643, "\u0120Personal": 15644, "Chris": 15645, "\u0120invented": 15646, "\u0120dive": 15647, "\u0120Rise": 15648, "\u0120oz": 15649, "\u0120Comics": 15650, "\u0120expose": 15651, "\u0120Reb": 15652, "letters": 15653, "site": 15654, "imated": 15655, "\u0120hacking": 15656, "\u0120educated": 15657, "\u0120Nobody": 15658, "\u0120depri": 15659, "\u0120incentive": 15660, "\u00e3\u0124\u00b7": 15661, "\u0120oversight": 15662, "\u0120tribes": 15663, "\u0120Belgium": 15664, "\u0120licensing": 15665, "ourt": 15666, "Product": 15667, "ahl": 15668, "\u0120Gem": 15669, "\u0120specialist": 15670, "\u0120cra": 15671, "anners": 15672, "\u0120Corbyn": 15673, "\u01201973": 15674, "READ": 15675, "\u0120summar": 15676, "\u0120overlook": 15677, "\u0120Application": 15678, "\u0120inappropriate": 15679, "\u0120downloaded": 15680, "Que": 15681, "\u0120Bears": 15682, "\u0120thumb": 15683, "\u0120Character": 15684, "\u0120Reincarnated": 15685, "\u0120Sid": 15686, "\u0120demonstrates": 15687, "sky": 15688, "\u0120Bloomberg": 15689, "\u0120Array": 15690, "\u0120Results": 15691, "\u0120Fourth": 15692, "\u0120EDT": 15693, "\u0120Oscar": 15694, "cend": 15695, "\u0120106": 15696, "\u0120NULL": 15697, "\u0120HERE": 15698, "match": 15699, "\u0120Brun": 15700, "\u0120glucose": 15701, "ieg": 15702, "egu": 15703, "\u0120certified": 15704, "\u0120relie": 15705, "\u0120humanitarian": 15706, "\u0120prayers": 15707, "King": 15708, "\u0120nan": 15709, "hou": 15710, "108": 15711, "ulu": 15712, "\u0120renewable": 15713, "\u0120distinguish": 15714, "\u0120dense": 15715, "\u0120Vent": 15716, "\u0120Package": 15717, "\u0120Boss": 15718, "\u0120editors": 15719, "\u0120migr": 15720, "Tra": 15721, "\u0120Peters": 15722, "\u0120Arctic": 15723, "2004": 15724, "\u0120Cape": 15725, "\u0120locally": 15726, "\u0120lasting": 15727, "\u0120handy": 15728, ".).": 15729, "Pan": 15730, "\u0120RES": 15731, "Index": 15732, "\u0120tensions": 15733, "\u0120formerly": 15734, "\u0120ideological": 15735, "\u0120sensors": 15736, "\u0120dealers": 15737, "\u0120defines": 15738, "Sk": 15739, "\u0120proceeds": 15740, "\u0120proxy": 15741, "azines": 15742, "\u0120Bash": 15743, "\u0120Pad": 15744, "\u0120Craft": 15745, "ealous": 15746, "\u0120sheets": 15747, "ometry": 15748, "June": 15749, "clock": 15750, "TT": 15751, "\u0120Theatre": 15752, "\u0120Buzz": 15753, "\u0120chapters": 15754, "\u0120millenn": 15755, "\u0120dough": 15756, "\u0120Congressional": 15757, "\u0120imagined": 15758, "avior": 15759, "\u0120clinic": 15760, "\u01201945": 15761, "\u0120holder": 15762, "root": 15763, "olester": 15764, "\u0120restart": 15765, "BN": 15766, "\u0120Hamas": 15767, "\u0120Job": 15768, "\u0120orb": 15769, "\u0120ram": 15770, "\u0120disclose": 15771, "\u0120translate": 15772, "\u0120immigrant": 15773, "\u0120annoying": 15774, "\u0120treaty": 15775, "anium": 15776, "\u0120Tea": 15777, "\u0120Legion": 15778, "\u0120crowds": 15779, "\u0120Bec": 15780, "\u0120Aer": 15781, "ohyd": 15782, "Bro": 15783, "Looking": 15784, "\u0120lbs": 15785, "\u0120aggress": 15786, "\u0120seam": 15787, "\u0120intercept": 15788, "\u0120MI": 15789, "mercial": 15790, "activ": 15791, "\u0120Cit": 15792, "\u0120dimension": 15793, "\u0120consistency": 15794, "\u0120rushing": 15795, "\u0120Douglas": 15796, "\u0120trim": 15797, "Install": 15798, "icker": 15799, "\u0120shy": 15800, "106": 15801, "\u0120mentions": 15802, "pelled": 15803, "\u0120Tak": 15804, "cost": 15805, "\u0120classroom": 15806, "\u0120fortune": 15807, "driven": 15808, "\u0120unle": 15809, "\u0120Wheel": 15810, "\u0120investor": 15811, "\u0120Masters": 15812, "kit": 15813, "\u0120associations": 15814, "\u0120Evolution": 15815, "oping": 15816, "uscript": 15817, "\u0120provincial": 15818, "\u0120Walter": 15819, "avi": 15820, "SO": 15821, "\u0120unlimited": 15822, "English": 15823, "\u0120Cards": 15824, "\u0120Ebola": 15825, "nered": 15826, "\u0120revenge": 15827, "\u0120outright": 15828, "umper": 15829, "\u0120fitting": 15830, "\u0120Solid": 15831, "\u0120formally": 15832, "\u0120problematic": 15833, "\u0120hazard": 15834, "\u0120encryption": 15835, "\u0120straightforward": 15836, "\u0120AK": 15837, "\u0120pse": 15838, "\u0120Orb": 15839, "\u0120Chamber": 15840, "\u0120Mak": 15841, "Contents": 15842, "\u0120loyalty": 15843, "\u0120lyrics": 15844, "\u0120Sym": 15845, "\u0120welcomed": 15846, "\u0120cooked": 15847, "\u0120monop": 15848, "\u0120nurse": 15849, "\u0120misleading": 15850, "\u0120eternal": 15851, "\u0120shifting": 15852, "\u0120+=": 15853, "Vis": 15854, "\u0120institutional": 15855, "illary": 15856, "\u0120pant": 15857, "VERT": 15858, "\u0120ACC": 15859, "\u0120Enh": 15860, "\u0120incon": 15861, "\u0120REUTERS": 15862, "\u0120donated": 15863, "\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6": 15864, "Intern": 15865, "\u0120exhibit": 15866, "\u0120tire": 15867, "\u0120Ric": 15868, "\u0120Champion": 15869, "\u0120Muhammad": 15870, "NING": 15871, "\u0120Soccer": 15872, "\u0120mobility": 15873, "\u0120varying": 15874, "\u0120Movie": 15875, "\u0120lord": 15876, "oak": 15877, "Field": 15878, "\u0120vector": 15879, "usions": 15880, "\u0120scrap": 15881, "\u0120enabling": 15882, "make": 15883, "Tor": 15884, ".*": 15885, "||": 15886, "\u0120Website": 15887, "\u0120NPC": 15888, "\u0120socialist": 15889, "\u0120Billy": 15890, "\u0120Additional": 15891, "\u0120cargo": 15892, "\u0120farms": 15893, "\u0120Soon": 15894, "\u0120Prize": 15895, "\u0120midnight": 15896, "\u0120900": 15897, "seen": 15898, "\u0120Spot": 15899, "\u0120sheep": 15900, "\u0120sponsored": 15901, "\u0120Hi": 15902, "\u0120Jump": 15903, "\u01201967": 15904, "Microsoft": 15905, "\u0120Agent": 15906, "\u0120charts": 15907, "dir": 15908, "\u0120adjacent": 15909, "\u0120tricks": 15910, "\u0120manga": 15911, "\u0120exagger": 15912, "/>": 15913, "football": 15914, "\u0120FCC": 15915, "GC": 15916, "\u0120Tier": 15917, "andra": 15918, "OUND": 15919, "%),": 15920, "\u0120fruits": 15921, "VC": 15922, "\u0120AA": 15923, "Rober": 15924, "\u0120midst": 15925, "\u00e2\u0139": 15926, "anka": 15927, "\u0120legislature": 15928, "\u0120Neil": 15929, "\u0120tourists": 15930, "\"\"": 15931, "\u0120Warning": 15932, "\u0120Nevertheless": 15933, "\u0120Official": 15934, "\u0120Whatever": 15935, "\u0120mold": 15936, "\u0120drafted": 15937, "\u0120substances": 15938, "\u0120breed": 15939, "\u0120tags": 15940, "\u0120Task": 15941, "\u0120verb": 15942, "\u0120manufactured": 15943, "comments": 15944, "\u0120Polish": 15945, "Prov": 15946, "\u0120determines": 15947, "Obama": 15948, "kers": 15949, "\u0120utterly": 15950, "\u0120sect": 15951, "sche": 15952, "\u0120Gates": 15953, "\u0120Chap": 15954, "\u0120aluminum": 15955, "\u0120zombie": 15956, "\u0120Touch": 15957, "\u0120UP": 15958, "\u0120satisfy": 15959, "\u0120predomin": 15960, "ascript": 15961, "\u0120elaborate": 15962, "\u01201968": 15963, "\u0120measuring": 15964, "\u0120Vari": 15965, "anyahu": 15966, "\u0120sir": 15967, "ulates": 15968, "idges": 15969, "ickets": 15970, "\u0120Spencer": 15971, "TM": 15972, "oubted": 15973, "\u0120prey": 15974, "\u0120installing": 15975, "\u0120Cab": 15976, "reed": 15977, "reated": 15978, "Supp": 15979, "\u0120wrist": 15980, "\u0120Kerry": 15981, "107": 15982, "\u0120Kle": 15983, "\u0120Rachel": 15984, "\u0120cotton": 15985, "\u0120ARE": 15986, "\u0120Ele": 15987, "Control": 15988, "\u0120loads": 15989, "\u0120Dod": 15990, "anas": 15991, "bone": 15992, "\u0120classical": 15993, "\u0120Regional": 15994, "\u0120Integ": 15995, "VM": 15996, "\u0120desires": 15997, "\u0120autism": 15998, "supported": 15999, "\u0120Message": 16000, "\u0120compact": 16001, "writer": 16002, "\u0120109": 16003, "\u0120Hurricane": 16004, "cision": 16005, "\u0120cycles": 16006, "\u0120drill": 16007, "\u0120colleague": 16008, "\u0120maker": 16009, "German": 16010, "\u0120mistaken": 16011, "Sun": 16012, "\u0120Gay": 16013, "\u0120whatsoever": 16014, "\u0120sells": 16015, "\u0120Airl": 16016, "liv": 16017, "\u0120Option": 16018, "\u0120solved": 16019, "\u0120sectors": 16020, "\u0120horizontal": 16021, "\u0120equation": 16022, "\u0120Skill": 16023, "\u0120Bio": 16024, "gement": 16025, "\u0120Snap": 16026, "\u0120Legal": 16027, "\u0120trademark": 16028, "\u0120makeup": 16029, "\u0120assembled": 16030, "\u0120saves": 16031, "\u0120Halloween": 16032, "\u0120Vermont": 16033, "\u0120FROM": 16034, "\u0120farming": 16035, "\u0120Podcast": 16036, "acceptable": 16037, "\u0120Higher": 16038, "\u0120asleep": 16039, "ullivan": 16040, "\u0120referen": 16041, "\u0120Lev": 16042, "\u0120bullets": 16043, "oko": 16044, "HC": 16045, "\u0120stairs": 16046, "\u0120maintains": 16047, "\u0120Lower": 16048, "\u0120Vi": 16049, "\u0120marine": 16050, "\u0120acres": 16051, "\u0120coordinator": 16052, "\u0120Joh": 16053, "\u0120counterparts": 16054, "\u0120Brothers": 16055, "\u0120indict": 16056, "bra": 16057, "\u0120chunk": 16058, "\u0120cents": 16059, "Home": 16060, "\u0120Month": 16061, "\u0120accordingly": 16062, "ifles": 16063, "\u0120Germans": 16064, "\u0120Syn": 16065, "Hub": 16066, "\u0120eyeb": 16067, "\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122": 16068, "\u0120ranges": 16069, "\u0120Holland": 16070, "\u0120Robot": 16071, "fc": 16072, "Mike": 16073, "\u0120plasma": 16074, "\u0120swap": 16075, "\u0120athlete": 16076, "\u0120Rams": 16077, ",'\"": 16078, "\u0120infections": 16079, "\u0120corrid": 16080, "\u0120vib": 16081, "\u0120patches": 16082, "\u0120traditionally": 16083, "\u0120revelation": 16084, "\u0120sweep": 16085, "\u0120glance": 16086, "\u0120inex": 16087, "2003": 16088, "\u0120Raw": 16089, "working": 16090, "osures": 16091, "\u0120Dat": 16092, "\u0120Lynch": 16093, "\u0120leverage": 16094, "\u0120Reid": 16095, "\u0120correlation": 16096, "iances": 16097, "avascript": 16098, "\u0120repository": 16099, "retty": 16100, "\u01201972": 16101, "240": 16102, "\u0120oun": 16103, "pol": 16104, "\u0120Reed": 16105, "\u0120tactical": 16106, "isite": 16107, "Apple": 16108, "\u0120Quinn": 16109, "\u0120raped": 16110, "illo": 16111, "Europe": 16112, "\u0120algorithms": 16113, "\u0120Rodrig": 16114, "iu": 16115, "\u0120illum": 16116, "\u0120fame": 16117, "\u0120introducing": 16118, "\u0120delays": 16119, "\u0120Raiders": 16120, "\u0120whistle": 16121, "\u0120novels": 16122, "\u0120Really": 16123, "\u0120deriv": 16124, "\u0120publications": 16125, "\u0120Neither": 16126, "\u0120Commerce": 16127, "\u0120aston": 16128, "language": 16129, "Notes": 16130, "\u0120Roth": 16131, "\u0120Fear": 16132, "\u0120mate": 16133, "\u0120parade": 16134, "\u0120QB": 16135, "\u0120maneu": 16136, "\u0120Cincinnati": 16137, "mitting": 16138, "\u0120waist": 16139, "\u0120Rew": 16140, "\u0120discont": 16141, "\u00d0\u00b0": 16142, "\u0120staring": 16143, "\u0120alias": 16144, "\u0120securities": 16145, "\u0120toilet": 16146, "\u0120Jedi": 16147, "\u0120unlaw": 16148, "vised": 16149, "////////": 16150, "](": 16151, "\u0120Weiss": 16152, "\u0120prest": 16153, "\u0120Compan": 16154, "\u0120memo": 16155, "\u0120Grace": 16156, "July": 16157, "\u0120Elite": 16158, "center": 16159, "\u0120Stay": 16160, "\u0120galaxy": 16161, "\u0120tooth": 16162, "\u0120Settings": 16163, "\u0120subjected": 16164, "\u00e3\u0124\u00a6": 16165, "\u0120lineback": 16166, "\u0120retailers": 16167, "\u0120Want": 16168, "\u0120dangers": 16169, "Air": 16170, "\u0120voluntary": 16171, "eway": 16172, "\u0120interpreted": 16173, "otine": 16174, "\u00c3\u00a7": 16175, "\u0120pel": 16176, "Service": 16177, "\u0120Eventually": 16178, "\u0120careers": 16179, "\u0120threaten": 16180, "\u0120memor": 16181, "\u0120Bradley": 16182, "ancies": 16183, "sn": 16184, "\u0120Unknown": 16185, "National": 16186, "\u0120shadows": 16187, "ailand": 16188, "\u0120Dash": 16189, "Everyone": 16190, "izzard": 16191, "March": 16192, "=(": 16193, "\u0120pulls": 16194, "\u0120stranger": 16195, "\u0120backwards": 16196, "\u0120Bernard": 16197, "imensional": 16198, "\u0120chron": 16199, "\u0120theoretical": 16200, "ktop": 16201, "\u0120ware": 16202, "\u0120Investig": 16203, "\u0120Initi": 16204, "\u0120Operations": 16205, "oven": 16206, "ocide": 16207, "*/": 16208, "\u0120flames": 16209, "\u0120Cash": 16210, "shit": 16211, "\u0120cab": 16212, "\u0120Analy": 16213, "\u0120Seah": 16214, "\u0120defining": 16215, "\u0120ordering": 16216, "\u0120immun": 16217, "\u0120persistent": 16218, "ACH": 16219, "Russian": 16220, "mans": 16221, "\u0120hind": 16222, "\u0120photography": 16223, "\u00c2\u00a9": 16224, "\u0120hug": 16225, "\u0120107": 16226, "\u0120Hence": 16227, "iots": 16228, "udeau": 16229, "\u0120subsidies": 16230, "\u0120routinely": 16231, "\u0120Device": 16232, "itic": 16233, "\u0120disgust": 16234, "lander": 16235, "\u01201940": 16236, "\u0120assignment": 16237, "\u0120Besides": 16238, "wick": 16239, "\u0120Dust": 16240, "usc": 16241, "structed": 16242, "111": 16243, "develop": 16244, "\u0120fond": 16245, "\u0120intersection": 16246, "\u0120dignity": 16247, "\u0120commissioner": 16248, "Without": 16249, "reach": 16250, "\u0120cartoon": 16251, "\u0120scales": 16252, "\u00e3\u0125\u0143": 16253, "FIG": 16254, "\u0120surveys": 16255, "\u0120Indonesia": 16256, "\u0120artwork": 16257, "\u0120unch": 16258, "\u0120cycling": 16259, "unct": 16260, "auer": 16261, "orate": 16262, "\u0120Obviously": 16263, "\u0120characterized": 16264, "feld": 16265, "\u0120affirm": 16266, "\u0120innings": 16267, "\u0120\u00e9": 16268, "\u0120aliens": 16269, "\u0120cloth": 16270, "etooth": 16271, "\u0120Certain": 16272, "\u00c2\u00a7": 16273, "\u0120digest": 16274, "know": 16275, "\u0120XL": 16276, "\u0120predictions": 16277, "\u0120din": 16278, "WAR": 16279, "\u0120aftermath": 16280, "Example": 16281, "\u0120Success": 16282, "\u0120Thr": 16283, "IGN": 16284, "\u0120miner": 16285, "Bus": 16286, "\u0120clarity": 16287, "heimer": 16288, "\u0120OUT": 16289, "\u0120Send": 16290, "\u0120Circle": 16291, "\u0120Diet": 16292, "\u0120pronounced": 16293, "\u0120creators": 16294, "\u0120earthquake": 16295, "attery": 16296, "geons": 16297, "\u0120od": 16298, "\u0120laying": 16299, "orp": 16300, "Ult": 16301, "project": 16302, "\u0120undermin": 16303, "\u0120sequel": 16304, "Sam": 16305, "\u0120Darkness": 16306, "\u0120reception": 16307, "bull": 16308, "YS": 16309, "\u0120Vir": 16310, "\u0120sequences": 16311, "\u0120Coin": 16312, "\u0120outfit": 16313, "\u0120Wait": 16314, "119": 16315, "\u0120delivers": 16316, "......": 16317, "\u0120blown": 16318, "\u0120Esc": 16319, "\u0120Math": 16320, "perm": 16321, "\u0120Ul": 16322, "\u0120glim": 16323, "\u0120facial": 16324, "\u0120greenhouse": 16325, "\u0120tokens": 16326, "/-": 16327, "\u0120Annual": 16328, "\u0120ONE": 16329, "\u0120teenage": 16330, "\u0120Physical": 16331, "\u0120Lang": 16332, "\u0120Celt": 16333, "\u0120sued": 16334, "ividually": 16335, "\u0120patience": 16336, "chair": 16337, "regular": 16338, "\u0120aug": 16339, "inv": 16340, "except": 16341, "\u0120Lil": 16342, "\u0120nest": 16343, "fd": 16344, "sum": 16345, "\u0120Chase": 16346, "Russia": 16347, "\u0120Jennifer": 16348, "\u0120offseason": 16349, "Overall": 16350, "Fore": 16351, "\u0120riot": 16352, "Aud": 16353, "former": 16354, "\u0120defenders": 16355, "\u0120CT": 16356, "iotic": 16357, "ribly": 16358, "\u0120automated": 16359, "\u0120penis": 16360, "\u0120insist": 16361, "\u0120diagram": 16362, "\u0120SQL": 16363, "\u0120Garc": 16364, "\u0120witch": 16365, "client": 16366, "ierra": 16367, "ambers": 16368, "\u0120recount": 16369, "far": 16370, "Very": 16371, "osterone": 16372, "\u0120appreciated": 16373, "\u0120Perfect": 16374, "Section": 16375, "\u0120doses": 16376, "ocaust": 16377, "\u0120costly": 16378, "\u0120grams": 16379, "\u0120Shi": 16380, "\u0120wrestling": 16381, "\u01201971": 16382, "\u0120trophy": 16383, "\u0120nerve": 16384, "\u0120Kaz": 16385, "\u0120Experience": 16386, "\u0120pledged": 16387, "\u0120playback": 16388, "\u0120creativity": 16389, "bye": 16390, "\u0120attackers": 16391, "\u0120holders": 16392, "\u0120Coach": 16393, "\u0120PhD": 16394, "\u0120transfers": 16395, "\u0120colored": 16396, "\u0120Hindu": 16397, "\u0120drown": 16398, "\u0120listened": 16399, "\u0120WA": 16400, "iasm": 16401, "PO": 16402, "\u0120appealing": 16403, "\u0120disclosed": 16404, "\u0120Chicken": 16405, "agging": 16406, "\u0120pleaded": 16407, "\u0120navigation": 16408, "\u0120Returns": 16409, "\u0120[[": 16410, "ROR": 16411, "EA": 16412, "\u0120photographer": 16413, "\u0120Rider": 16414, "ippers": 16415, "\u0120slice": 16416, "\u0120erect": 16417, "\u0120hed": 16418, "issance": 16419, "\u0120Vikings": 16420, "urious": 16421, "\u0120appet": 16422, "oubtedly": 16423, "Child": 16424, "\u0120authentic": 16425, "oos": 16426, "\u0120Making": 16427, "\u0120announcing": 16428, "\u0120bod": 16429, "\u0120meter": 16430, "\u0120Nine": 16431, "\u0120Rogue": 16432, "\u0120workforce": 16433, "\u0120renewed": 16434, "\u0120organisations": 16435, "acs": 16436, "PLE": 16437, "Short": 16438, "\u0120compounds": 16439, "\u0120Visit": 16440, "\u0120envelop": 16441, "earth": 16442, "\u0120supportive": 16443, "ggle": 16444, "\u0120Brussels": 16445, "\u0120Guild": 16446, "Create": 16447, "REL": 16448, "\u0120averaged": 16449, "\u01201969": 16450, "riages": 16451, "\u0120lengthy": 16452, "\u0120forgot": 16453, "Okay": 16454, "\u0120Erd": 16455, "\u0120dealer": 16456, "\u0120recession": 16457, "DD": 16458, "\u0120desperately": 16459, "\u0120hunger": 16460, "\u0120sticks": 16461, "\u0120mph": 16462, "\u0120Faith": 16463, "\u0120intentionally": 16464, "\u0120demol": 16465, "ueller": 16466, "\u0120Sale": 16467, "\u0120debris": 16468, "spring": 16469, "\u0120leap": 16470, ">>>>": 16471, "\u0120containers": 16472, "selling": 16473, "ranean": 16474, "attering": 16475, "\u0120commented": 16476, "\u0120CM": 16477, "onut": 16478, "\u0120woods": 16479, "especially": 16480, "\u0120organize": 16481, "ivic": 16482, "\u0120Woods": 16483, "anga": 16484, "squ": 16485, "\u0120maj": 16486, "amon": 16487, "\u0120axis": 16488, "\u01201974": 16489, "\u0120Denmark": 16490, "\u0120warrior": 16491, "\u0120Pand": 16492, "\u0120outlined": 16493, "\u0120BO": 16494, "insula": 16495, "zilla": 16496, "ebook": 16497, "\u0120dare": 16498, "\u0120searched": 16499, "\u0120navigate": 16500, "Sn": 16501, "writing": 16502, "\u0120united": 16503, "Japan": 16504, "\u0120Hebrew": 16505, "\u0120flame": 16506, "\u0120relies": 16507, "\u0120catching": 16508, "\u0120Sho": 16509, "\u0120imprisonment": 16510, "\u0120pockets": 16511, "\u0120closure": 16512, "\u0120Fam": 16513, "tim": 16514, "adequ": 16515, "Activity": 16516, "\u0120recruiting": 16517, "\u0120WATCH": 16518, "\u0120Argentina": 16519, "dest": 16520, "\u0120apologize": 16521, "oro": 16522, "\u0120lacks": 16523, "\u0120tuned": 16524, "\u0120Griffin": 16525, "\u0120infamous": 16526, "\u0120celebrity": 16527, "sson": 16528, "\u0120----------------------------------------------------------------": 16529, "\u0120Isis": 16530, "\u0120Display": 16531, "\u0120credibility": 16532, "\u0120economies": 16533, "\u0120headline": 16534, "\u0120Cowboys": 16535, "\u0120indef": 16536, "\u0120lately": 16537, "\u0120incentives": 16538, "button": 16539, "\u0120Mob": 16540, "Aut": 16541, "\u0120resigned": 16542, "\u0120Om": 16543, "camp": 16544, "\u0120profiles": 16545, "\u0120schemes": 16546, "olphins": 16547, "ayed": 16548, "Clinton": 16549, "enh": 16550, "\u0120Yahoo": 16551, "\u0120abst": 16552, "\u0120ank": 16553, "suits": 16554, "\u0120wished": 16555, "\u0120Marco": 16556, "udden": 16557, "\u0120sphere": 16558, "\u0120Bishop": 16559, "\u0120incorporated": 16560, "\u0120Plant": 16561, "114": 16562, "\u0120hated": 16563, "pic": 16564, "\u0120donate": 16565, "\u0120lined": 16566, "\u0120beans": 16567, "\u0120stealing": 16568, "\u0120costume": 16569, "\u0120sheriff": 16570, "\u0120forty": 16571, "\u0120intact": 16572, "\u0120adapted": 16573, "\u0120travelling": 16574, "bart": 16575, "\u0120nicely": 16576, "\u0120dried": 16577, "\u0120scal": 16578, "osity": 16579, "NOTE": 16580, "\u0120Bh": 16581, "\u0120Broncos": 16582, "\u0120Ign": 16583, "\u0120intimate": 16584, "\u0120chemistry": 16585, "\u0120optimal": 16586, "Deb": 16587, "\u0120Generation": 16588, "\u0120],": 16589, "ichi": 16590, "\u0120Wii": 16591, "\u0120YOUR": 16592, "ventions": 16593, "Write": 16594, "\u0120popul": 16595, "unning": 16596, "\u0120Wor": 16597, "Vol": 16598, "\u0120queen": 16599, "heads": 16600, "KK": 16601, "\u0120analyze": 16602, "opic": 16603, "earchers": 16604, "\u0120dot": 16605, "legraph": 16606, "astically": 16607, "\u0120upgrades": 16608, "\u0120cares": 16609, "\u0120extending": 16610, "\u0120freeze": 16611, "\u0120inability": 16612, "\u0120organs": 16613, "\u0120pretend": 16614, "\u0120outlet": 16615, "113": 16616, "olan": 16617, "\u0120Mall": 16618, "uling": 16619, "talk": 16620, "\u0120expressing": 16621, "\u0120Always": 16622, "\u0120Begin": 16623, "files": 16624, "\u0120licenses": 16625, "%%": 16626, "\u0120Mitt": 16627, "\u0120filters": 16628, "\u0120Milwaukee": 16629, "GN": 16630, "\u0120unfold": 16631, "Mo": 16632, "\u0120nutrition": 16633, "ppo": 16634, "Bo": 16635, "\u0120founding": 16636, "\u0120undermine": 16637, "\u0120easiest": 16638, "\u0120Czech": 16639, "\u0120Mack": 16640, "\u0120sexuality": 16641, "\u0120Nixon": 16642, "Win": 16643, "\u0120Arn": 16644, "\u0120Kin": 16645, "\u00e3\u0124\u00a3": 16646, "icer": 16647, "\u0120fortun": 16648, "\u0120surfaces": 16649, "aghd": 16650, "\u0120carriers": 16651, "\u0120PART": 16652, "\u0120Tib": 16653, "\u0120interval": 16654, "\u0120frustrating": 16655, "\u0120Ship": 16656, "\u0120Armed": 16657, "ffe": 16658, "\u0120boats": 16659, "\u0120Abraham": 16660, "inis": 16661, "\u0120suited": 16662, "thread": 16663, "iov": 16664, "abul": 16665, "\u0120Venezuela": 16666, "\u0120tom": 16667, "super": 16668, "\u0120castle": 16669, "although": 16670, "ioxide": 16671, "eches": 16672, "\u0120evolutionary": 16673, "\u0120negotiate": 16674, "\u0120confronted": 16675, "Remember": 16676, "\u0120170": 16677, "Such": 16678, "\u0120911": 16679, "mult": 16680, "\u0120Abyss": 16681, "urry": 16682, "kees": 16683, "spec": 16684, "\u0120Barbara": 16685, "\u0120belonging": 16686, "\u0120villain": 16687, "istani": 16688, "\u0120accountable": 16689, "\u0120portions": 16690, "\u0120Decl": 16691, "Ur": 16692, "\u0120Kate": 16693, "gre": 16694, "\u0120magazines": 16695, "UCK": 16696, "\u0120regulate": 16697, "omon": 16698, "\u0120Almost": 16699, "\u0120overview": 16700, "\u0120scram": 16701, "\u0120loot": 16702, "\u0120Fitz": 16703, "\u0120characteristic": 16704, "\u0120Snake": 16705, "say": 16706, "\u0120Rico": 16707, "\u0120trait": 16708, "\u0120Joined": 16709, "aucus": 16710, "\u0120adaptation": 16711, "\u0120Airlines": 16712, "\u0120archae": 16713, "\u0120Ide": 16714, "\u0120bikes": 16715, "\u0120literary": 16716, "\u0120influences": 16717, "\u0120Used": 16718, "Creat": 16719, "\u0120plea": 16720, "\u0120Defence": 16721, "\u0120Assass": 16722, "\u0120pond": 16723, "ULT": 16724, ")\"": 16725, "\u0120evaluated": 16726, "\u0120obtaining": 16727, "\u0120demographic": 16728, "\u0120vigil": 16729, "aley": 16730, "\u0120spouse": 16731, "\u0120Seahawks": 16732, "respons": 16733, "\u0120Belt": 16734, "umatic": 16735, "\u0120rises": 16736, "runner": 16737, "\u0120Michelle": 16738, "\u0120potent": 16739, "race": 16740, "\u0120PAC": 16741, "Find": 16742, "olesterol": 16743, "ISS": 16744, "\u0120Introduced": 16745, "resses": 16746, "ignment": 16747, "Os": 16748, "\u0120Tu": 16749, "\u0120Dex": 16750, "icides": 16751, "\u0120sparked": 16752, "\u0120Laura": 16753, "\u0120Bryant": 16754, "\u0120smiling": 16755, "\u0120Nexus": 16756, "\u0120defendants": 16757, "\u0120Catal": 16758, "\u0120dishes": 16759, "shaped": 16760, "\u0120prolong": 16761, "mt": 16762, "($": 16763, "\u00e3\u0122\u0124": 16764, "\u0120calculations": 16765, "\u0120Same": 16766, "\u0120piv": 16767, "HH": 16768, "\u0120cancelled": 16769, "\u0120grin": 16770, "\u0120territories": 16771, "istically": 16772, "Come": 16773, "\u0120Parent": 16774, "Project": 16775, "\u0120neglig": 16776, "\u0120Privacy": 16777, "\u0120ammo": 16778, "LECT": 16779, "olutely": 16780, "\u0120Epic": 16781, "\u0120misunder": 16782, "wal": 16783, "April": 16784, "mos": 16785, "pathy": 16786, "\u0120Carson": 16787, "\u0120albums": 16788, "\u0120Easy": 16789, "\u0120pistol": 16790, "<<": 16791, "\u0120\\(": 16792, "target": 16793, "help": 16794, "\u0120interpre": 16795, "conscious": 16796, "\u0120Housing": 16797, "\u0120Joint": 16798, "127": 16799, "\u0120beers": 16800, "science": 16801, "\u0120Firefox": 16802, "effective": 16803, "\u0120Cabin": 16804, "\u0120Okay": 16805, "\u0120Applic": 16806, "\u0120spacecraft": 16807, "\u0120SR": 16808, "vet": 16809, "\u0120Strange": 16810, "SB": 16811, "\u0120corps": 16812, "iberal": 16813, "efficient": 16814, "\u0120prevalence": 16815, "\u0120economists": 16816, "118": 16817, "Thread": 16818, "ordable": 16819, "ODE": 16820, "\u0120Cant": 16821, "=-=-": 16822, "ifiable": 16823, "\u0120Around": 16824, "\u0120pole": 16825, "\u0120willingness": 16826, "CLA": 16827, "\u0120Kid": 16828, "\u0120complement": 16829, "\u0120scattered": 16830, "\u0120inmates": 16831, "\u0120bleeding": 16832, "every": 16833, "\u0120queue": 16834, "\u0120Train": 16835, "\u0120hij": 16836, "\u0120melee": 16837, "pleted": 16838, "\u0120digit": 16839, "\u0120gem": 16840, "official": 16841, "\u0120lifting": 16842, "\u00d0\u00b5": 16843, "Requ": 16844, "itutes": 16845, "\u0120packaging": 16846, "\u0120Workers": 16847, "hran": 16848, "\u0120Lebanon": 16849, "olesc": 16850, "\u0120punished": 16851, "\u0120Juan": 16852, "\u0120jam": 16853, "\u0120Document": 16854, "\u0120mapping": 16855, "icates": 16856, "\u0120inevitably": 16857, "\u0120vanilla": 16858, "\u0120Ton": 16859, "\u0120watches": 16860, "\u0120leagues": 16861, "\u0120initiated": 16862, "degree": 16863, "portion": 16864, "\u0120recalls": 16865, "\u0120ruin": 16866, "\u0120melt": 16867, "IAN": 16868, "\u0120hem": 16869, "Exp": 16870, "\u0120baking": 16871, "\u0120Colomb": 16872, "atible": 16873, "\u0120radius": 16874, "plug": 16875, "\u0120IF": 16876, "etically": 16877, "\u0120fict": 16878, "HER": 16879, "\u0120Tap": 16880, "atinum": 16881, "\u0120ink": 16882, "\u0120coh": 16883, "\u0120Wizard": 16884, "both": 16885, "tex": 16886, "\u0120spends": 16887, "\u0120Currently": 16888, "\u0120Pit": 16889, "\u0120neurons": 16890, "ignt": 16891, "\u0120rall": 16892, "\u0120buses": 16893, "building": 16894, "\u0120adjustments": 16895, "\u0120cried": 16896, "iblical": 16897, "atted": 16898, "\u0120Zion": 16899, "\u0120Matter": 16900, "\u0120meditation": 16901, "\u0120Dennis": 16902, "\u0120ours": 16903, "\u0120Tab": 16904, "\u0120rankings": 16905, "ortal": 16906, "\u0120advers": 16907, "\u0120surrender": 16908, "\u0120Gob": 16909, "cium": 16910, "omas": 16911, "imeter": 16912, "\u0120multiplayer": 16913, "\u0120heroin": 16914, "\u0120optimistic": 16915, "\u0120indicator": 16916, "\u0120Brig": 16917, "\u0120grocery": 16918, "\u0120applicant": 16919, "\u0120Rocket": 16920, "vid": 16921, "Exception": 16922, "pent": 16923, "\u0120organizing": 16924, "\u0120encounters": 16925, "\u0120TOD": 16926, "\u0120jewel": 16927, "Save": 16928, "\u0120Christie": 16929, "\u0120heating": 16930, "\u0120lazy": 16931, "\u0120CP": 16932, "\u0120cousin": 16933, "Config": 16934, "\u0120regener": 16935, "\u0120nearest": 16936, "\u0120achieving": 16937, "ENS": 16938, "throw": 16939, "\u0120Richmond": 16940, "antle": 16941, "2002": 16942, "\u0120anten": 16943, "bird": 16944, "133": 16945, "\u0120narc": 16946, "raint": 16947, "unny": 16948, "\u0120Hispanic": 16949, "ournaments": 16950, "\u0120prophe": 16951, "\u0120Thailand": 16952, "\u0120Ti": 16953, "\u0120injection": 16954, "\u0120inherit": 16955, "ravis": 16956, "\u0120medi": 16957, "\u0120whoever": 16958, "\u0120DEBUG": 16959, "GP": 16960, "\u0120Hud": 16961, "Card": 16962, "prom": 16963, "\u0120por": 16964, "\u0120overhead": 16965, "Law": 16966, "\u0120violate": 16967, "\u0120heated": 16968, "\u0120descriptions": 16969, "\u0120achievements": 16970, "\u0120Beer": 16971, "\u0120Quant": 16972, "Was": 16973, "\u0120eighth": 16974, "\u0120Iv": 16975, "\u0120specialized": 16976, "UPDATE": 16977, "\u0120Delta": 16978, "Pop": 16979, "Jul": 16980, "\u0120Ask": 16981, "ophy": 16982, "\u0120newsletters": 16983, "\u0120Tool": 16984, "\u0120gard": 16985, "\u0120Confeder": 16986, "\u0120GMT": 16987, "\u0120Abbott": 16988, "\u0120immunity": 16989, "\u0120VM": 16990, "Islam": 16991, "\u0120implicit": 16992, "wd": 16993, "\u01201944": 16994, "ravity": 16995, "ometric": 16996, "\u0120surviving": 16997, "urai": 16998, "\u0120Prison": 16999, "\u0120rust": 17000, "\u0120Sketch": 17001, "\u0120bees": 17002, "\u0120Theory": 17003, "\u0120merit": 17004, "Tex": 17005, "chat": 17006, "\u0120mim": 17007, "\u0120paste": 17008, "\u0120Koch": 17009, "\u0120ignorance": 17010, "\u0120Shoot": 17011, "\u0120basement": 17012, "United": 17013, "\u0120Advis": 17014, "height": 17015, "\u0120foster": 17016, "\u0120detain": 17017, "information": 17018, "\u0120neural": 17019, "';": 17020, "\u0120proves": 17021, "allery": 17022, "\u0120invitation": 17023, "umbers": 17024, "\u0120cattle": 17025, "\u0120bicycle": 17026, "zi": 17027, "\u0120consultant": 17028, "\u0120apology": 17029, "\u0120Tiger": 17030, "\u0120123": 17031, "999": 17032, "\u0120individually": 17033, "rt": 17034, "igion": 17035, "\u0120Brazilian": 17036, "\u0120disturb": 17037, "\u0120entrepreneurs": 17038, "\u0120forests": 17039, "cerpt": 17040, "plates": 17041, "pher": 17042, "clipse": 17043, "\u0120twitter": 17044, "\u0120acids": 17045, "ographical": 17046, "hum": 17047, "\u0120Bald": 17048, "ifully": 17049, "\u0120compiler": 17050, "\u0120DA": 17051, "\u0120donor": 17052, "asi": 17053, "\u0120tribal": 17054, "lash": 17055, "\u0120Config": 17056, "\u0120applicants": 17057, "\u0120salaries": 17058, "135": 17059, "Putin": 17060, "\u0120Focus": 17061, "irs": 17062, "\u0120misconduct": 17063, "\u0120Haz": 17064, "\u0120eaten": 17065, "Mobile": 17066, "Muslim": 17067, "\u0120Marcus": 17068, "viol": 17069, "\u0120favorable": 17070, "\u0120stub": 17071, "adin": 17072, "\u0120Hob": 17073, "\u0120faithful": 17074, "\u0120electronics": 17075, "\u0120vacuum": 17076, "wait": 17077, "backed": 17078, "economic": 17079, "dist": 17080, "\u0120tenure": 17081, "\u0120sincere": 17082, "\u0120Together": 17083, "\u0120Wave": 17084, "\u0120progression": 17085, "\u0120denying": 17086, "\u0120distress": 17087, "braska": 17088, "third": 17089, "\u0120mixing": 17090, "\u0120colonial": 17091, "\u0120privately": 17092, "\u0120unrest": 17093, "aternity": 17094, "\u0120premises": 17095, "anti": 17096, "gregation": 17097, "\u0120licence": 17098, "\u0120Hind": 17099, "\u0120Samuel": 17100, "\u0120convincing": 17101, "\u0120Ace": 17102, "\u0120Rust": 17103, "\u0120Netanyahu": 17104, "\u0120handles": 17105, "\u0120Patch": 17106, "oriented": 17107, "aho": 17108, "\u0120Gonz": 17109, "\u0120hackers": 17110, "claimer": 17111, "\u0120customs": 17112, "\u0120Gran": 17113, "fighters": 17114, "\u0120luc": 17115, "\u0120manuscript": 17116, "arenthood": 17117, "\u0120devil": 17118, "\u0120warriors": 17119, "\u0120offenders": 17120, "William": 17121, "\u0120holidays": 17122, "\u0120nightmare": 17123, "\u0120lever": 17124, "ifferent": 17125, "Stat": 17126, "\u0120exhibition": 17127, "puted": 17128, "\u0120Pure": 17129, "\u0120alpha": 17130, "\u0120enthusiasm": 17131, "\u0120Representatives": 17132, "EAR": 17133, "\u0120Typ": 17134, "\u0120wheat": 17135, "\u0120Alf": 17136, "\u0120correction": 17137, "\u0120evangel": 17138, "ATT": 17139, "Miss": 17140, "\u0120soup": 17141, "\u0120implied": 17142, "param": 17143, "\u0120sexy": 17144, "\u0120Lux": 17145, "\u0120republic": 17146, "patch": 17147, "ablish": 17148, "\u0120icons": 17149, "\u0120fathers": 17150, "\u0120GET": 17151, "\u0120Carib": 17152, "\u0120regulated": 17153, "\u0120Cohen": 17154, "\u0120Bobby": 17155, "\u0120ner": 17156, "\u0120bent": 17157, "ventory": 17158, "\u0120Along": 17159, "\u0120EST": 17160, "\u0120Wallace": 17161, "\u0120murders": 17162, "rise": 17163, "kell": 17164, "\u0120Commonwealth": 17165, "\u0120nasty": 17166, "eta": 17167, "\u0120MIT": 17168, "\u0120administered": 17169, "\u0120genuinely": 17170, "Editor": 17171, "nick": 17172, "\u0120hydro": 17173, "********************************": 17174, "\u0120Ble": 17175, "\u0120fines": 17176, "\u0120gorge": 17177, "ausible": 17178, "rh": 17179, "\u0120apple": 17180, "mentioned": 17181, "\u0120rope": 17182, "otyp": 17183, "HR": 17184, "\u0120disappointing": 17185, "\u0120cage": 17186, "nik": 17187, "\u0120doubts": 17188, "\u0120FREE": 17189, "prints": 17190, "\u0120MUST": 17191, "\u0120vendors": 17192, "\u0120Inqu": 17193, "\u0120liberals": 17194, "\u0120contractor": 17195, "\u0120upside": 17196, "children": 17197, "\u0120tricky": 17198, "\u0120regulators": 17199, "charged": 17200, "liter": 17201, "\u0120***": 17202, "\u0120rebell": 17203, "lang": 17204, "\u0120locals": 17205, "\u0120physicians": 17206, "\u0120hey": 17207, "arse": 17208, "tm": 17209, "\u0120Lex": 17210, "\u0120behavioral": 17211, "successful": 17212, "FX": 17213, "\u0120brick": 17214, "ovic": 17215, "\u0120conform": 17216, "\u0120reviewing": 17217, "\u0120insights": 17218, "\u0120biology": 17219, "\u0120Remove": 17220, "\u0120Extra": 17221, "\u0120committing": 17222, "induced": 17223, "ignty": 17224, "igm": 17225, "\u0120atomic": 17226, "Common": 17227, "\u0120EM": 17228, "\u0120Pere": 17229, "\u0120Items": 17230, "eh": 17231, "\u0120preserved": 17232, "\u0120Hood": 17233, "\u0120prisoner": 17234, "\u0120bankruptcy": 17235, "\u0120gren": 17236, "ushes": 17237, "\u0120exploitation": 17238, "\u0120signatures": 17239, "\u0120finan": 17240, "],\"": 17241, "\u0120MR": 17242, "\u0120meg": 17243, "remlin": 17244, "\u0120musicians": 17245, "\u0120selecting": 17246, "\u0120examining": 17247, "INK": 17248, "lated": 17249, "Hi": 17250, "\u0120artic": 17251, "\u0120pets": 17252, "\u0120impair": 17253, "\u0120MAN": 17254, "\u0120tablets": 17255, "include": 17256, "Range": 17257, "\u0120caut": 17258, "\u0120logs": 17259, "\u0120mounting": 17260, "\u0120unaware": 17261, "\u0120dynamics": 17262, "\u0120Palestine": 17263, "\u0120Quarter": 17264, "\u0120Purple": 17265, "\u0120ma": 17266, "\u0120Import": 17267, "\u0120collections": 17268, "ciation": 17269, "\u0120successor": 17270, "\u0120clone": 17271, "\u0120aiming": 17272, "\u0120possessed": 17273, "\u0120sticking": 17274, "\u0120shaking": 17275, "\u0120locate": 17276, "\u0120Hockey": 17277, "Turn": 17278, "170": 17279, "\u0120fifteen": 17280, "\u0120Harrison": 17281, "\u0120continuously": 17282, "\u0120TC": 17283, "\u0120Valent": 17284, "\u0120Rescue": 17285, "\u0120bypass": 17286, "amount": 17287, "\u0120mast": 17288, "\u0120protects": 17289, "\u0120artistic": 17290, "\u0120sometime": 17291, "\u0120shoe": 17292, "\u0120shouted": 17293, "ificant": 17294, "etitive": 17295, "\u0120Register": 17296, "\u0120Jin": 17297, "\u0120concentrated": 17298, "lington": 17299, "onies": 17300, "\u0120generator": 17301, "yrim": 17302, "\u0120Armen": 17303, "\u0120clearing": 17304, "ido": 17305, "\u0120TW": 17306, "alph": 17307, "\u0120ladies": 17308, "Hard": 17309, "\u0120dialog": 17310, "\u0120inputs": 17311, "\u00e6\u013e": 17312, "\u0120poses": 17313, "\u0120slots": 17314, "\u0120Premium": 17315, "\u0120leaks": 17316, "\u0120bosses": 17317, "\u0120113": 17318, "course": 17319, "Acc": 17320, "\u0120Newton": 17321, "\u0120Austria": 17322, "\u0120Mage": 17323, "\u0120teaches": 17324, "abad": 17325, "\u0120wears": 17326, "\u0120cyl": 17327, "\u0120curse": 17328, "\u0120Sales": 17329, "\u0120Wings": 17330, "\u0120psy": 17331, "\u0120gaps": 17332, "\u0120Iceland": 17333, "\u0120Pinterest": 17334, "\u0120landlord": 17335, "\u0120definitions": 17336, "\u0120Ker": 17337, "\u0120sufficiently": 17338, "\u0120Pence": 17339, "\u0120Architect": 17340, "\u0120surpass": 17341, "\u0120114": 17342, "\u0120superhero": 17343, "\u0120Disease": 17344, "\u0120priests": 17345, "\u0120Culture": 17346, "\u0120definitive": 17347, "\u0120secretly": 17348, "\u0120Dance": 17349, "install": 17350, "chief": 17351, "\u0120Jessica": 17352, "Would": 17353, "Updated": 17354, "\u0120locker": 17355, "\u0120Kay": 17356, "\u0120memorial": 17357, "\u00e8\u00a6": 17358, "fat": 17359, "\u0120disgu": 17360, "\u0120flavors": 17361, "\u0120Baseball": 17362, "\u0120Resistance": 17363, "\u0120kicks": 17364, "\u0120env": 17365, "\u0120teenagers": 17366, "Dark": 17367, "\u0120CAR": 17368, "\u0120halt": 17369, "\u0120LG": 17370, "\u0120Gabriel": 17371, "\u0120fever": 17372, "\u0120satur": 17373, "\u0120mall": 17374, "\u0120affiliate": 17375, "\u0120Sleep": 17376, "\u0120Specific": 17377, "\u0120Vel": 17378, "\u0120jar": 17379, "\u0120Sacred": 17380, "\u0120Edwards": 17381, "\u0120ACL": 17382, "\u0120retained": 17383, "\u0120Giant": 17384, "\u0120limitation": 17385, "inces": 17386, "\u0120refusal": 17387, "\u0120Tale": 17388, "\u0120Butler": 17389, "\u0120accidents": 17390, "\u0120CSS": 17391, "\u0120imported": 17392, "\u0120Copy": 17393, "\u00ce\u00b1": 17394, "ERT": 17395, "zel": 17396, "\u0120divisions": 17397, "hots": 17398, "\u0120Alb": 17399, "\u0120DS": 17400, "Loader": 17401, "Washington": 17402, "atisf": 17403, "\u0120Creative": 17404, "\\.": 17405, "\u0120Autom": 17406, "redict": 17407, "\u0120receptor": 17408, "\u0120Carlos": 17409, "Method": 17410, "oka": 17411, "\u0120malicious": 17412, "\u0120stepping": 17413, ",[": 17414, "\u0120Dad": 17415, "\u0120attraction": 17416, "\u0120Effects": 17417, "\u0120Pirate": 17418, "\u0120Cer": 17419, "\u0120Industry": 17420, "\u0120Rud": 17421, "\u0120charter": 17422, "\u0120dining": 17423, "\u0120insists": 17424, "\u0120configure": 17425, "\u0120(#": 17426, "\u0120Simple": 17427, "\u0120Scroll": 17428, "UTC": 17429, "175": 17430, "\u0120Kon": 17431, "\u0120marketplace": 17432, "\u0120\u00e3\u0124": 17433, "\u0120refres": 17434, "\u0120gates": 17435, "erred": 17436, "\u0120Pod": 17437, "\u0120behave": 17438, "Frank": 17439, "node": 17440, "\u0120endorsed": 17441, "hett": 17442, "asive": 17443, "\u0120Homeland": 17444, "\u0120rides": 17445, "\u0120Leave": 17446, "erness": 17447, "\u0120flooding": 17448, "AFP": 17449, "\u0120risen": 17450, "\u0120continually": 17451, "\u0120unanim": 17452, "\u0120Contract": 17453, "\u0120Pas": 17454, "\u0120guided": 17455, "\u0120Chile": 17456, "bd": 17457, "\u0120succ": 17458, "ptic": 17459, "\u0120committees": 17460, "\u0120Luther": 17461, "\u0120Anyone": 17462, "\u0120sab": 17463, "124": 17464, "\u0120pixel": 17465, "\u0120Bak": 17466, "\u0120Tag": 17467, "\u0120Bennett": 17468, "Enter": 17469, "small": 17470, "\u0120Presidential": 17471, "\u0120pul": 17472, "\u0120contrace": 17473, "archive": 17474, "\u0120coastal": 17475, "\u0120Kids": 17476, "192": 17477, "\u00e2\u0122\u00b2": 17478, "icky": 17479, "INGTON": 17480, "\u0120wolf": 17481, "\u0120Stalin": 17482, "Tur": 17483, "idget": 17484, "amas": 17485, "\u0120Unless": 17486, "\u0120sponsor": 17487, "\u0120morph": 17488, "\u0120Choose": 17489, "\u0120runner": 17490, "\u0120unbel": 17491, "\u0120mud": 17492, "\u0120Mana": 17493, "\u0120dubbed": 17494, "\u0120godd": 17495, "urers": 17496, "window": 17497, "\u0120relied": 17498, "\u0120celebrating": 17499, "osc": 17500, "\u0120135": 17501, "\u0120lobbying": 17502, "\u0120incomplete": 17503, "\u0120restriction": 17504, "\u0120incap": 17505, "itus": 17506, "\u0120expectation": 17507, "\u0120Apollo": 17508, "\u0120intens": 17509, "\u0120sync": 17510, "GH": 17511, "\u0120manipulation": 17512, "BY": 17513, "\u0120spear": 17514, "\u0120breasts": 17515, "\u0120volcan": 17516, "ilia": 17517, "Material": 17518, "\u0120formats": 17519, "\u0120Bast": 17520, "\u0120parliamentary": 17521, "\u0120snake": 17522, "\u0120servants": 17523, "\u0120Trudeau": 17524, "\u0120Grim": 17525, "\u0120Arabic": 17526, "\u0120SCP": 17527, "\u0120Boys": 17528, "station": 17529, "\u0120prospective": 17530, "orde": 17531, "initialized": 17532, "\u0120bored": 17533, "ABLE": 17534, "\u0120accessed": 17535, "\u0120taxi": 17536, "\u0120Shell": 17537, "aiden": 17538, "ursed": 17539, "inates": 17540, "\u0120Insurance": 17541, "\u0120Pete": 17542, "September": 17543, "650": 17544, "\u0120adventures": 17545, "\u0120Cover": 17546, "\u0120tribute": 17547, "\u0120sketch": 17548, "\u0120empower": 17549, "\u0120\u00d8": 17550, "\u0120Glenn": 17551, "\u0120Daw": 17552, "=\\\"": 17553, "\u0120Politics": 17554, "\u0120guides": 17555, "\u0120dioxide": 17556, "\u0120Gore": 17557, "\u0120Bright": 17558, "\u0120Sierra": 17559, "\u0120valued": 17560, "cond": 17561, "\u0120pointer": 17562, "Select": 17563, "\u0120risky": 17564, "\u0120absorb": 17565, "images": 17566, "\u0120refuses": 17567, "\u0120bonuses": 17568, "___": 17569, "\u0120hilar": 17570, "\u0120Features": 17571, "220": 17572, "\u0120Collector": 17573, "Foot": 17574, "\u01201964": 17575, "culus": 17576, "\u0120dawn": 17577, "\u0120workout": 17578, "\u0120LO": 17579, "\u0120philosophical": 17580, "\u0120Sandy": 17581, "\u0120Youth": 17582, "\u0120liable": 17583, "Af": 17584, "blue": 17585, "\u0120overturn": 17586, "lessness": 17587, "\u0120Tribune": 17588, "\u0120Ing": 17589, "\u0120factories": 17590, "\u0120catches": 17591, "\u0120prone": 17592, "\u0120matrix": 17593, "\u0120login": 17594, "\u0120inacc": 17595, "\u0120exert": 17596, "sys": 17597, "\u0120needle": 17598, "\u0120Qur": 17599, "\u0120notified": 17600, "oulder": 17601, "tx": 17602, "\u0120reminds": 17603, "\u0120publishers": 17604, "\u0120nort": 17605, "\u0120git": 17606, "\u0120flies": 17607, "\u0120Emily": 17608, "\u0120flowing": 17609, "\u0120Alien": 17610, "\u0120Strateg": 17611, "\u0120hardest": 17612, "\u0120modification": 17613, "API": 17614, "\u0120MY": 17615, "\u0120crashes": 17616, "stairs": 17617, "number": 17618, "\u0120urging": 17619, "channel": 17620, "\u0120Falcon": 17621, "\u0120inhabitants": 17622, "\u0120terrifying": 17623, "\u0120utilize": 17624, "\u0120banner": 17625, "\u0120cigarettes": 17626, "\u0120senses": 17627, "\u0120Holmes": 17628, "\u0120practition": 17629, "\u0120Phillips": 17630, "otto": 17631, "\u0120compile": 17632, "Model": 17633, "\u0120Ko": 17634, "\u0120[]": 17635, "Americans": 17636, "\u0120Terms": 17637, "\u0120medications": 17638, "\u0120Ana": 17639, "\u0120fundamentally": 17640, "\u0120Notice": 17641, "\u0120weaker": 17642, "\u01200000": 17643, "\u0120garlic": 17644, "\u0120outbreak": 17645, "\u0120economist": 17646, "\u0120Birth": 17647, "\u0120obstacles": 17648, "arcer": 17649, "\u0120Orthodox": 17650, "\u0120placebo": 17651, "\u0120Crew": 17652, "aspberry": 17653, "\u0120Angels": 17654, "\u0120discharge": 17655, "\u0120destructive": 17656, "117": 17657, "\u0120Rising": 17658, "\u0120dairy": 17659, "late": 17660, "\u0120collision": 17661, "\u0120Tigers": 17662, "eanor": 17663, "ocumented": 17664, "\u0120Invalid": 17665, "\u0120dont": 17666, "\u0120Liter": 17667, "\u0120Va": 17668, "\u0120hydrogen": 17669, "\u0120variants": 17670, "\u0120Browns": 17671, "\u01201965": 17672, "\u0120indigenous": 17673, "\u0120trades": 17674, "\u0120remainder": 17675, "\u0120swept": 17676, "\u0120Impact": 17677, "\u0120redist": 17678, "\u0120unint": 17679, "graduate": 17680, "\u00e3\u0125\u0137": 17681, "\u0120WILL": 17682, "\u00e3\u0123\u00ae\u00e7": 17683, "\u0120Critical": 17684, "\u0120fisher": 17685, "\u0120vicious": 17686, "\u0120reversed": 17687, "Year": 17688, "\u0120Sox": 17689, "\u0120shootings": 17690, "\u0120filming": 17691, "\u0120touchdowns": 17692, "aires": 17693, "mel": 17694, "\u0120grandfather": 17695, "\u0120affection": 17696, "ingle": 17697, "\u0120overly": 17698, "Additional": 17699, "\u0120supreme": 17700, "\u0120Grad": 17701, "\u0120sporting": 17702, "\u0120mercy": 17703, "\u0120Brooks": 17704, "ounty": 17705, "\u0120performs": 17706, "\u0120tightly": 17707, "\u0120demons": 17708, "\u0120killings": 17709, "\u0120faction": 17710, "\u0120Nova": 17711, "auts": 17712, "\u0120undoubtedly": 17713, "arin": 17714, "\u0120underway": 17715, "rak": 17716, "\u0120liv": 17717, "\u0120Region": 17718, "\u0120briefing": 17719, "sers": 17720, "cloud": 17721, "\u0120Mik": 17722, "usp": 17723, "\u0120prediction": 17724, "azor": 17725, "\u0120portable": 17726, "\u0120Gand": 17727, "\u0120presenting": 17728, "\u01201080": 17729, "\u00c2\u00bb": 17730, "ushi": 17731, "\u0120Spark": 17732, "thereum": 17733, "\u0120justification": 17734, "\u0120Ny": 17735, "\u0120contractors": 17736, "mingham": 17737, "\u0120Style": 17738, "\u00e5\u0127": 17739, "\u0120Chronicles": 17740, "\u0120Picture": 17741, "\u0120proving": 17742, "\u0120wives": 17743, "sett": 17744, "\u0120molecules": 17745, "\u0120Fairy": 17746, "\u0120consisting": 17747, "\u0120pier": 17748, "alone": 17749, "inition": 17750, "\u0120nucle": 17751, "json": 17752, "\u0120gotta": 17753, "\u0120mobil": 17754, "\u0120verbal": 17755, "arium": 17756, "\u0120monument": 17757, "ucked": 17758, "\u0120256": 17759, "Tech": 17760, "minecraft": 17761, "\u0120Track": 17762, "\u0120tile": 17763, "\u0120compatibility": 17764, "asis": 17765, "\u0120sadd": 17766, "\u0120instructed": 17767, "\u0120Mueller": 17768, "\u0120lethal": 17769, "\u0120hormone": 17770, "\u0120orche": 17771, "else": 17772, "\u0120skelet": 17773, "\u0120entertaining": 17774, "\u0120minimize": 17775, "again": 17776, "\u0120undergo": 17777, "\u0120constraints": 17778, "\u0120cigarette": 17779, "\u0120Islamist": 17780, "\u0120travels": 17781, "\u0120Panthers": 17782, "lings": 17783, "Care": 17784, "\u0120lawsuits": 17785, "uras": 17786, "\u0120cryst": 17787, "\u0120lowered": 17788, "\u0120aerial": 17789, "\u0120combinations": 17790, "\u0120haun": 17791, "\u0120cha": 17792, "\u0120vine": 17793, "\u0120quantities": 17794, "\u0120linking": 17795, "bank": 17796, "\u0120soy": 17797, "Bill": 17798, "\u0120Angela": 17799, "\u0120recipient": 17800, "\u0120Protest": 17801, "\u0120socket": 17802, "\u0120solidarity": 17803, "\u0120\u00e2\u0128": 17804, "mill": 17805, "\u0120varies": 17806, "\u0120Pakistani": 17807, "Dragon": 17808, "\u0120une": 17809, "\u0120horizon": 17810, "\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142": 17811, "\u0120provinces": 17812, "\u0120frankly": 17813, "\u0120enacted": 17814, "notes": 17815, "['": 17816, "\u0120192": 17817, "ocracy": 17818, "\u0120endorsement": 17819, "\u0120overtime": 17820, "True": 17821, "Lab": 17822, "licted": 17823, "\u0120DNC": 17824, "\u0120beats": 17825, "\u0120Jamie": 17826, "152": 17827, "\u0120INT": 17828, "Contact": 17829, "\u0120accounted": 17830, "hash": 17831, "\u0120Packers": 17832, "pires": 17833, "\u0120lesbian": 17834, "\u0120amendments": 17835, "\u0120hopeful": 17836, "\u0120Finland": 17837, "\u0120spotlight": 17838, "\u0120configured": 17839, "\u0120troubled": 17840, "\u0120gaze": 17841, "\u0120Calgary": 17842, "\u0120reliability": 17843, "\u0120insurg": 17844, "swer": 17845, "buy": 17846, "\u0120Skin": 17847, "\u0120pixels": 17848, "\u0120handgun": 17849, "\u0120paras": 17850, "\u0120categor": 17851, "\u0120EL": 17852, "\u0120Rex": 17853, "Indeed": 17854, "\u0120kinda": 17855, "\u0120conjunction": 17856, "\u0120Bryan": 17857, "\u0120Manufact": 17858, "yang": 17859, "Plus": 17860, "SQL": 17861, "ishment": 17862, "\u0120dominate": 17863, "\u0120nail": 17864, "\u0120oath": 17865, "\u0120erupt": 17866, "\u0120Fine": 17867, "itbart": 17868, "\u0120Chip": 17869, "\u0120Abd": 17870, "\u0120Nam": 17871, "\u0120buyer": 17872, "\u0120dissent": 17873, "Leaks": 17874, "Contin": 17875, "\u0120rider": 17876, "\u0120Someone": 17877, "\u0120illusion": 17878, "cin": 17879, "\u0120Boeing": 17880, "\u0120inadequ": 17881, "ovation": 17882, "iants": 17883, "\u0120rebuild": 17884, "450": 17885, "\u0120Destiny": 17886, "SW": 17887, "\u0120Till": 17888, "Hit": 17889, "iaz": 17890, "\u0120Bangl": 17891, "achers": 17892, "\u0120Reform": 17893, "\u0120segments": 17894, "\u0120systematic": 17895, "dc": 17896, "\u0120Conservatives": 17897, "\u0120portal": 17898, "hor": 17899, "\u0120Dragonbound": 17900, "\u0120dragged": 17901, "omo": 17902, "\u0120thee": 17903, "advert": 17904, "\u0120Reports": 17905, "\u0120Et": 17906, "\u0120barrels": 17907, "August": 17908, "\u0120comparisons": 17909, "\u0120hex": 17910, "\u0120anthrop": 17911, "\"[": 17912, "borough": 17913, "abi": 17914, "\u0120pictured": 17915, "playing": 17916, "\u0120Address": 17917, "\u0120Mirror": 17918, "Smith": 17919, "\u0120tires": 17920, "\u0120NPR": 17921, "AAAA": 17922, "\u0120classification": 17923, "\u0120Than": 17924, "\u0120Harm": 17925, "\u0120RA": 17926, "\u0120rejection": 17927, "mination": 17928, "\u0120ranged": 17929, "\u0120Falls": 17930, "DI": 17931, "Host": 17932, "\u00e3\u0124\u00b4": 17933, "\u0120Example": 17934, "listed": 17935, "thirds": 17936, "\u0120safegu": 17937, "brand": 17938, "\u0120probable": 17939, "Canada": 17940, "ITION": 17941, "\u0120Qaeda": 17942, "\u0120chick": 17943, "\u0120imports": 17944, "hit": 17945, "loc": 17946, "WW": 17947, "\u0120blew": 17948, "\u0120anytime": 17949, "\u0120wholes": 17950, "iked": 17951, "\u0120calculation": 17952, "create": 17953, "\u0120Ori": 17954, "\u0120upgraded": 17955, "\u0120appar": 17956, "utory": 17957, "\u0120Mol": 17958, "Brit": 17959, "\u0120Jong": 17960, "INAL": 17961, "\u0120Starting": 17962, "\u0120dice": 17963, "urtle": 17964, "\u0120relying": 17965, "closure": 17966, "\u0120profitable": 17967, "\u0120slaughter": 17968, "\u0120Manual": 17969, "caster": 17970, "\u0120\"$": 17971, "\u0120feather": 17972, "\u0120Simply": 17973, "ieves": 17974, "\u0120deterior": 17975, "\u0120PCI": 17976, "\u0120stamp": 17977, "\u0120flaws": 17978, "\u0120shade": 17979, "hammer": 17980, "\u0120passport": 17981, "\u0120conting": 17982, "amel": 17983, "\u0120observers": 17984, "\u0120neglect": 17985, "\u0120RB": 17986, "\u0120Brotherhood": 17987, "\u0120skeptical": 17988, "family": 17989, "usk": 17990, "\u0120emotionally": 17991, "\u00e2\u013b": 17992, "\u0120Beta": 17993, "asonable": 17994, "idity": 17995, "\u0120Mul": 17996, "\u0120kicking": 17997, "\u0120Carm": 17998, "ollah": 17999, "VERTIS": 18000, "\u0120Athen": 18001, "\u0120ladder": 18002, "\u0120Bullet": 18003, "\u00e5\u00a3": 18004, "0001": 18005, "\u0120Wildlife": 18006, "\u0120Mask": 18007, "\u0120Nan": 18008, "Rev": 18009, "\u0120unacceptable": 18010, "legal": 18011, "\u0120crowded": 18012, "agi": 18013, "\u0120Cox": 18014, "je": 18015, "\u0120morality": 18016, "\u0120fuels": 18017, "\u0120cables": 18018, "\u0120mankind": 18019, "\u0120Caribbean": 18020, "\u0120anchor": 18021, "\u0120byte": 18022, "\u0120Often": 18023, "\u0120Oz": 18024, "\u0120crafted": 18025, "\u0120historian": 18026, "\u0120Wu": 18027, "\u0120towers": 18028, "\u0120Citizens": 18029, "\u0120helm": 18030, "\u0120credentials": 18031, "\u0120singular": 18032, "\u0120Jesse": 18033, "\u0120tackles": 18034, "\u0120contempt": 18035, "\u0120afore": 18036, "\u0120Shadows": 18037, "\u0120nil": 18038, "\u0120urgent": 18039, "apple": 18040, "blood": 18041, "\u0120von": 18042, "\u0120offline": 18043, "\u0120breathe": 18044, "\u0120jumps": 18045, "\u0120irrelevant": 18046, "oxic": 18047, "omal": 18048, "important": 18049, "Jim": 18050, "\u0120gloves": 18051, "arming": 18052, "depth": 18053, "\u0120talents": 18054, "ookie": 18055, "\u0120SB": 18056, "\u0120palm": 18057, "uffs": 18058, "esta": 18059, "IGH": 18060, "\u0120canon": 18061, "\u0120Verizon": 18062, "\u0120Ple": 18063, "\u0120coupled": 18064, "velt": 18065, "\u0120fundraising": 18066, "\u0120Getting": 18067, "\u0120DLC": 18068, "\u0120mathematical": 18069, "\u0120HS": 18070, "\u0120Cardinals": 18071, "telling": 18072, "\u0120sponsors": 18073, "\u0120\u00cf": 18074, "\u0120Bulls": 18075, "option": 18076, "\u0120propose": 18077, "\u0120memorable": 18078, "\u0120embraced": 18079, "\u0120declining": 18080, "Health": 18081, "eda": 18082, "\u0120};": 18083, "\u0120spam": 18084, "mile": 18085, "\u0120pitcher": 18086, "\u0120Eight": 18087, "\u0120caring": 18088, "utic": 18089, "role": 18090, "\u0120airline": 18091, "ernandez": 18092, "\u0120Athlet": 18093, "\u0120certification": 18094, "uxe": 18095, "riger": 18096, "\u0120empir": 18097, "\u0120sensation": 18098, "\u0120dism": 18099, "\u0120bolt": 18100, "\u0120evolve": 18101, "House": 18102, "\u0120consultation": 18103, "\u0120Duty": 18104, "\u0120touches": 18105, "\u0120Nathan": 18106, "\u0120faint": 18107, "had": 18108, "\"(": 18109, "\u0120Consumer": 18110, "\u0120Extreme": 18111, "\u0120127": 18112, "\u0120Herm": 18113, "\u0120Sacrament": 18114, "izoph": 18115, "\u0120anxious": 18116, "ulously": 18117, "\u0120socially": 18118, "\u0120UTC": 18119, "\u0120solving": 18120, "\u0120Letter": 18121, "History": 18122, "educ": 18123, "Price": 18124, "));": 18125, "\u0120reload": 18126, "amic": 18127, "\u0120pork": 18128, "\u0120discourse": 18129, "\u0120tournaments": 18130, "airo": 18131, "\u0120Kur": 18132, "\u0120Costa": 18133, "\u0120violating": 18134, "\u0120interfere": 18135, "\u0120recreational": 18136, "uffle": 18137, "\u0120speeches": 18138, "\u0120needing": 18139, "\u0120remembers": 18140, "\u0120credited": 18141, "nia": 18142, "focused": 18143, "amera": 18144, "\u0120bru": 18145, "umbs": 18146, "\u0120Cuban": 18147, "\u0120preceding": 18148, "\u0120nonsense": 18149, "acial": 18150, "\u0120smartphones": 18151, "\u0120Stories": 18152, "Sports": 18153, "\u0120Emergency": 18154, "ouncing": 18155, "efined": 18156, "\u0120ber": 18157, "\u0120consulting": 18158, "\u0120masters": 18159, "heastern": 18160, ".\"[": 18161, "\u0120Running": 18162, "\u0120suscept": 18163, "\u0120Feng": 18164, "America": 18165, "prises": 18166, "stitial": 18167, "\u0120Weekly": 18168, "\u0120Greater": 18169, "modules": 18170, "ifter": 18171, "Graphics": 18172, "uler": 18173, "\u0120wholly": 18174, "\u0120suppress": 18175, "\u0120concealed": 18176, "\u0120happily": 18177, "\u0120accepts": 18178, "\u0120Enjoy": 18179, "\u0120rivers": 18180, "\u0120Except": 18181, "225": 18182, "\u0120NHS": 18183, "\u0120McConnell": 18184, "\u0120pussy": 18185, "ferred": 18186, "utable": 18187, "\u0120attain": 18188, "\u0120>=": 18189, "\u0120deposits": 18190, "rophic": 18191, "\u0120notorious": 18192, "\u0120Shaw": 18193, "ilitation": 18194, "\u0120epidemic": 18195, "allic": 18196, "\u0120smallest": 18197, "ovich": 18198, "\u0120accessories": 18199, "perties": 18200, "\u0120surplus": 18201, "\u0120Mech": 18202, "\u0120ambig": 18203, "\u0120Immigration": 18204, "\u0120chim": 18205, "eval": 18206, "\u0120practicing": 18207, "\u0120Mystery": 18208, "\u0120domains": 18209, "\u0120Silicon": 18210, "apps": 18211, "\u0120kilometers": 18212, "ea": 18213, "\u0120Smash": 18214, "\u0120warranty": 18215, "\u0120nost": 18216, "sil": 18217, "rev": 18218, "Jon": 18219, "\u0120Dublin": 18220, "\u0120tastes": 18221, "\u0120bout": 18222, "great": 18223, "error": 18224, "\u0120switches": 18225, "\u0120Bapt": 18226, "DO": 18227, "oki": 18228, "\u0120sourced": 18229, "produ": 18230, "\u0120attachment": 18231, "\u0120Issue": 18232, "\u0120Question": 18233, "Join": 18234, "\u0120fitted": 18235, "\u0120unlawful": 18236, "^^": 18237, "erek": 18238, "\u0120authentication": 18239, "\u0120stole": 18240, "\u0120accountability": 18241, "label": 18242, "Search": 18243, "\u0120albeit": 18244, "atican": 18245, "funded": 18246, "\u0120Adding": 18247, "\u0120IQ": 18248, "\u0120submar": 18249, "lit": 18250, "aque": 18251, "\u0120Learning": 18252, "\u0120integer": 18253, "Master": 18254, "\u0120Chrom": 18255, "\u0120premier": 18256, "Op": 18257, "\u0120Liu": 18258, "\u0120blessed": 18259, "\u0120Globe": 18260, "\u0120Response": 18261, "\u0120legitim": 18262, "\u0120Merkel": 18263, "\u0120disposal": 18264, "\u00c2\u00b4": 18265, "\u0120gauge": 18266, "peat": 18267, "\u0120induced": 18268, "\u0120questionable": 18269, "arthy": 18270, "\u0120Vit": 18271, "\u0120Feed": 18272, "Until": 18273, "Ut": 18274, "worthy": 18275, "RY": 18276, "\u0120Herald": 18277, "\u0120Hammer": 18278, "\u0120medal": 18279, "\u0120Rivers": 18280, "\u0120Hack": 18281, "\u0120clarify": 18282, "\u0120tracked": 18283, "\u0120autonomous": 18284, "\u0120tenant": 18285, "\u0120Qatar": 18286, "erie": 18287, "\u0120grim": 18288, "\u0120Monitor": 18289, "\u0120resistant": 18290, "\u0120Spec": 18291, "\u0120Wells": 18292, "NAS": 18293, "148": 18294, "\u0120miners": 18295, "iotics": 18296, "\u0120misses": 18297, "116": 18298, "gian": 18299, "git": 18300, "\u0120Eyes": 18301, "pres": 18302, "\u0120graduated": 18303, "\u0120angel": 18304, "\u0120synchron": 18305, "\u0120efficiently": 18306, "\u0120transmitted": 18307, "Harry": 18308, "\u0120globally": 18309, "ENCE": 18310, "\u0120Montana": 18311, "raged": 18312, "\u0120Prevention": 18313, "\u0120piss": 18314, "\u0120Ll": 18315, "\u0120shelf": 18316, "\u0120BJP": 18317, "\u0120Testament": 18318, "\u0120Late": 18319, "iker": 18320, "\u0120Happ": 18321, "\u0120Julian": 18322, "hall": 18323, "\u0120spont": 18324, "\u0120shutdown": 18325, "\u0120inconsistent": 18326, "\u0120subscribers": 18327, "\u0120skeleton": 18328, "\u0120Nebraska": 18329, "\u0120inspire": 18330, "\u0120Void": 18331, "Feed": 18332, "\u0120angles": 18333, "\u0120Springs": 18334, "\u0120benchmark": 18335, "\u0120vaccines": 18336, "izophren": 18337, "sexual": 18338, "uffed": 18339, "\u0120shine": 18340, "\u0120Kath": 18341, "\u0120gesture": 18342, "inea": 18343, "\u0120rip": 18344, "\u0120oppression": 18345, "\u0120conscience": 18346, "bt": 18347, "\u0120Lum": 18348, "\u0120incidence": 18349, "\u0120Fa": 18350, "wr": 18351, "\u0120mineral": 18352, "\u0120Spurs": 18353, "alky": 18354, "\u0120thunder": 18355, "\u0120opio": 18356, "Being": 18357, "\u0120Palm": 18358, "\u0120wasted": 18359, "\u0120lb": 18360, "iaries": 18361, "\u0120Initiative": 18362, "\u0120curric": 18363, "\u0120marker": 18364, "\u0120McL": 18365, "\u0120extensions": 18366, "\u0120Pv": 18367, "\u0120Arms": 18368, "\u0120offerings": 18369, "\u0120defenses": 18370, "\u0120vendor": 18371, "\u0120contradict": 18372, "\u0120Colin": 18373, "\u0120reddit": 18374, "\u0120peripher": 18375, "122": 18376, "\u0120sins": 18377, "Edit": 18378, "ICT": 18379, "Soft": 18380, "\u0120Shah": 18381, "\u0120administrator": 18382, "\u0120Trip": 18383, "\u0120pornography": 18384, "\u0120tuition": 18385, "inence": 18386, "\u0120Progress": 18387, "\u0120catalog": 18388, "\u0120suite": 18389, "\u0120hike": 18390, "\u0120reproductive": 18391, "engine": 18392, "\u0120drought": 18393, "\u0120Noah": 18394, "\u0120230": 18395, "\u0120dude": 18396, "\u0120relaxed": 18397, "\u0120partition": 18398, "\u0120participant": 18399, "\u0120telesc": 18400, "\u0120feas": 18401, "\u0120FF": 18402, "owner": 18403, "\u0120sweeping": 18404, "\u0120lenses": 18405, "\u0120matchup": 18406, "\u0120Repl": 18407, "ournals": 18408, "\u0120credible": 18409, "\u0120grandmother": 18410, "\u0120thermal": 18411, "\u0120subscribing": 18412, "\u0120identities": 18413, "colm": 18414, "UCT": 18415, "\u0120reluctant": 18416, "users": 18417, "\u0120Cort": 18418, "\u0120assisted": 18419, "OSS": 18420, "ATIONS": 18421, "ISH": 18422, "\u0120pharmaceutical": 18423, "icable": 18424, "adian": 18425, "\u0120Sonic": 18426, "\u0120Fury": 18427, "\u0120Mong": 18428, "AH": 18429, "\u0120Psychology": 18430, "\u0120phosph": 18431, "\u0120treats": 18432, "\u0143\u0136": 18433, "\u0120steadily": 18434, "\u0120Hello": 18435, "\u0120relates": 18436, "\u0120clue": 18437, "Expl": 18438, "auth": 18439, "\u0120revision": 18440, "\u0120eld": 18441, "osion": 18442, "\u0120bron": 18443, "144": 18444, "rikes": 18445, "\u0120mines": 18446, "\u0120blanket": 18447, "\u0120Fail": 18448, "eled": 18449, "\u0120Imagine": 18450, "\u0120Planned": 18451, "aic": 18452, "Request": 18453, "Mad": 18454, "\u0120Horse": 18455, "\u0120Eagle": 18456, "\u0120capac": 18457, "157": 18458, "\u0120ling": 18459, "\u0120Nice": 18460, "\u0120Parenthood": 18461, "minster": 18462, "ogs": 18463, "ensitive": 18464, "Nothing": 18465, "\u0120carn": 18466, "Fin": 18467, "\u0120PE": 18468, "\u0120rifles": 18469, "\u0120LP": 18470, "Sand": 18471, "\u0120guiActive": 18472, "\u0120tourist": 18473, "CNN": 18474, "\u0120unveiled": 18475, "\u0120predecessor": 18476, "}{": 18477, "uber": 18478, "\u0120offshore": 18479, "\u0120optical": 18480, "\u0120Rot": 18481, "\u0120Pearl": 18482, "eton": 18483, "\u0120stared": 18484, "\u0120farther": 18485, "atility": 18486, "contin": 18487, "\u0120Gy": 18488, "\u0120Foster": 18489, "\u0120Coc": 18490, "rients": 18491, "\u0120designing": 18492, "\u0120Economy": 18493, "ONG": 18494, "Women": 18495, "\u0120Nancy": 18496, "erver": 18497, "\u0120mascul": 18498, "\u0120casualties": 18499, "\u0120225": 18500, "\u0120Sullivan": 18501, "\u0120Choice": 18502, "\u0120aster": 18503, "ws": 18504, "\u0120hotels": 18505, "\u0120considerations": 18506, "\u0120couch": 18507, "\u0120Strip": 18508, "\u0120Gn": 18509, "\u0120manipulate": 18510, "lied": 18511, "\u0120synthetic": 18512, "\u0120assaulted": 18513, "\u0120offenses": 18514, "\u0120Drake": 18515, "\u0120impe": 18516, "October": 18517, "\u0120Heritage": 18518, "hl": 18519, "\u0120Blair": 18520, "Unlike": 18521, "\u0120grief": 18522, "\u0120450": 18523, "\u0120opted": 18524, "\u0120resignation": 18525, "ilo": 18526, "\u0120verse": 18527, "\u0120Tomb": 18528, "\u0120upt": 18529, "\u0120aired": 18530, "\u0120Hook": 18531, "\u0120MLB": 18532, "\u0120assumes": 18533, "outed": 18534, "\u0120Vers": 18535, "\u0120inferior": 18536, "\u0120bundle": 18537, "\u0120DNS": 18538, "ographer": 18539, "\u0120multip": 18540, "\u0120Souls": 18541, "\u0120illustrated": 18542, "\u0120tactic": 18543, "\u0120dressing": 18544, "\u0120duo": 18545, "Conf": 18546, "\u0120relent": 18547, "\u0120cant": 18548, "\u0120scarce": 18549, "\u0120candy": 18550, "\u0120CF": 18551, "\u0120affiliated": 18552, "\u0120sprint": 18553, "ylan": 18554, "\u0120Garcia": 18555, "\u0120junk": 18556, "Print": 18557, "exec": 18558, "Crit": 18559, "\u0120portrait": 18560, "iries": 18561, "\u0120OFF": 18562, "\u0120disputes": 18563, "WR": 18564, "Love": 18565, "\u00e3\u0123\u0126": 18566, "\u0120Reyn": 18567, "\u0120hipp": 18568, "opath": 18569, "\u0120floors": 18570, "\u0120Feel": 18571, "\u0120worries": 18572, "\u0120settlements": 18573, "\u0120Pos": 18574, "\u0120mosque": 18575, "\u0120finals": 18576, "\u0120crushed": 18577, "\u0120Probably": 18578, "\u0120Bot": 18579, "\u0120Mans": 18580, "\u0120Period": 18581, "\u0120sovereignty": 18582, "\u0120seller": 18583, "\u0120apost": 18584, "\u0120amateur": 18585, "\u0120dorm": 18586, "\u0120consuming": 18587, "\u0120armour": 18588, "\u0120Roose": 18589, "\u0120intensive": 18590, "\u0120eliminating": 18591, "\u0120Sunni": 18592, "\u0120Aleppo": 18593, "jin": 18594, "\u0120advise": 18595, "pal": 18596, "\u0120Halo": 18597, "\u0120descent": 18598, "\u0120simpler": 18599, "\u0120booth": 18600, "STR": 18601, "Later": 18602, "\u0120Cave": 18603, "===": 18604, "\u0120mol": 18605, "\u0120fist": 18606, "\u0120shotgun": 18607, "supp": 18608, "\u0120robbery": 18609, "Effect": 18610, "\u0120obscure": 18611, "\u0120Professional": 18612, "\u0120embassy": 18613, "\u0120militant": 18614, "\u0120incarcer": 18615, "\u0120generates": 18616, "\u0120launches": 18617, "\u0120administrators": 18618, "\u0120shaft": 18619, "\u0120circular": 18620, "\u0120freshman": 18621, "\u0120Wes": 18622, "\u0120Joel": 18623, "\u0120Drew": 18624, "\u0120Duncan": 18625, "\u0120Apparently": 18626, "sight": 18627, "\u0120Internal": 18628, "\u0120Individual": 18629, "\u0120FE": 18630, "\u0120bore": 18631, "\u0120Mt": 18632, "\u0120broadly": 18633, "\u0120Options": 18634, "ountain": 18635, "ipes": 18636, "\u0120Videos": 18637, "204": 18638, "\u0120hills": 18639, "\u0120simulation": 18640, "\u0120disappointment": 18641, "itan": 18642, "\u0120Laboratory": 18643, "\u0120upward": 18644, "\u0120boundary": 18645, "\u0120darker": 18646, "hart": 18647, "\u0120dominance": 18648, "Cong": 18649, "\u0120Oracle": 18650, "\u0120Lords": 18651, "\u0120scholarship": 18652, "\u0120Vincent": 18653, "ede": 18654, "\u0120Rah": 18655, "\u0120encourages": 18656, "rov": 18657, "\u0120quo": 18658, "\u0120premise": 18659, "\u0120Crisis": 18660, "\u0120Holocaust": 18661, "\u0120rhythm": 18662, "\u0120metric": 18663, "club": 18664, "\u0120transported": 18665, "\u0120nod": 18666, "\u0120Pist": 18667, "\u0120ancestors": 18668, "\u0120Freder": 18669, "thumbnails": 18670, "\u0120CE": 18671, "OND": 18672, "Phil": 18673, "venge": 18674, "\u0120Products": 18675, "castle": 18676, "\u0120qualifying": 18677, "\u0120Karen": 18678, "VERTISEMENT": 18679, "\u0120mighty": 18680, "\u0120explanations": 18681, "\u0120fixing": 18682, "Di": 18683, "\u0120declaring": 18684, "\u0120anonymity": 18685, "\u0120juven": 18686, "\u0120Nord": 18687, "\u0120Doom": 18688, "\u0120Actually": 18689, "Ok": 18690, "phis": 18691, "\u0120Desert": 18692, "\u0120116": 18693, "IK": 18694, "\u0120FM": 18695, "\u0120incomes": 18696, "VEL": 18697, "okers": 18698, "\u0120pecul": 18699, "\u0120lightweight": 18700, "gue": 18701, "\u0120accent": 18702, "\u0120increment": 18703, "\u0120Chan": 18704, "\u0120complaining": 18705, "\u0120Baghd": 18706, "\u0120midfielder": 18707, "\u0120overhaul": 18708, "Process": 18709, "\u0120Hollow": 18710, "\u0120Titans": 18711, "Small": 18712, "manuel": 18713, "\u0120Unity": 18714, "\u0120Events": 18715, "Sty": 18716, "\u0120disproportion": 18717, "nesty": 18718, "enes": 18719, "\u0120Cod": 18720, "\u0120demonstrations": 18721, "\u0120Crimson": 18722, "\u0120OH": 18723, "\u0120enrolled": 18724, "\u0120cel": 18725, "\u0120Brett": 18726, "\u0120aide": 18727, "\u0120heels": 18728, "\u0120broadband": 18729, "\u0120marking": 18730, "\u0120wizard": 18731, "\u0120NJ": 18732, "\u0120Chiefs": 18733, "\u0120ingredient": 18734, "\u0120dug": 18735, "\u0120Shut": 18736, "urchase": 18737, "endor": 18738, "\u0120farmer": 18739, "\u0120Goldman": 18740, "129": 18741, "155": 18742, "Order": 18743, "\u0120lion": 18744, "iably": 18745, "\u0120stain": 18746, "array": 18747, "ilitary": 18748, "\u0120FAQ": 18749, "\u0120exploded": 18750, "\u0120McCarthy": 18751, "\u0120Tweet": 18752, "\u0120Greens": 18753, "eking": 18754, "ln": 18755, "ensen": 18756, "\u0120motorcycle": 18757, "\u0120particle": 18758, "\u0120cholesterol": 18759, "Bron": 18760, "\u0120stair": 18761, "\u0120oxid": 18762, "\u0120desirable": 18763, "ibles": 18764, "\u0120theor": 18765, "forcing": 18766, "\u0120promotional": 18767, "ovo": 18768, "boot": 18769, "\u0120Bonus": 18770, "rawling": 18771, "\u0120shortage": 18772, "\u0120Psy": 18773, "\u0120recruited": 18774, "\u0120infants": 18775, "\u0120testosterone": 18776, "\u0120deduct": 18777, "\u0120distinctive": 18778, "\u0120firmware": 18779, "built": 18780, "145": 18781, "\u0120explored": 18782, "\u0120factions": 18783, "\u0120vide": 18784, "\u0120tattoo": 18785, "\u0120financially": 18786, "\u0120fatigue": 18787, "\u0120proceeding": 18788, "constitutional": 18789, "\u0120miser": 18790, "\u0120chairs": 18791, "gging": 18792, "ipple": 18793, "\u0120dent": 18794, "\u0120disreg": 18795, "\u00e7\u0136": 18796, "stant": 18797, "llo": 18798, "bps": 18799, "akening": 18800, "\u0120abnormal": 18801, "\u0120ERA": 18802, "\u00e5\u00a3\u00ab": 18803, "\u0120HBO": 18804, "\u0120MAR": 18805, "\u0120concess": 18806, "\u0120servant": 18807, "\u0120aspir": 18808, "lav": 18809, "\u0120Panel": 18810, "amo": 18811, "\u0120precip": 18812, "\u0120recordings": 18813, "\u0120proceeded": 18814, "\u0120colony": 18815, "\u0120Tang": 18816, "ablo": 18817, "\u0120stripped": 18818, "Left": 18819, "too": 18820, "\u0120potatoes": 18821, "\u0120finest": 18822, "%).": 18823, "\u0120crap": 18824, "\u0120Zach": 18825, "abases": 18826, "\u0120Goth": 18827, "\u0120billionaire": 18828, "wolf": 18829, "\u0120sanction": 18830, "SK": 18831, "\u0120logged": 18832, "Po": 18833, "eyed": 18834, "unal": 18835, "\u0120cricket": 18836, "\u0120armies": 18837, "\u0120uncovered": 18838, "Cloud": 18839, "\u00c3\u00b3n": 18840, "\u0120rebounds": 18841, "\u0120mes": 18842, "Oper": 18843, "Pac": 18844, "\u0120nationally": 18845, "\u0120inserted": 18846, "pict": 18847, "\u0120governance": 18848, "\u00d0\u00b8": 18849, "\u0120privileges": 18850, "GET": 18851, "\u0120favorites": 18852, "imity": 18853, "\u0120lover": 18854, "them": 18855, "empl": 18856, "\u0120gorgeous": 18857, "Ann": 18858, "\u0120slipped": 18859, "\u0120veto": 18860, "Bob": 18861, "\u0120slim": 18862, "ucc": 18863, "\u0120Fame": 18864, "uddenly": 18865, "\u0120denies": 18866, "\u0120Maur": 18867, "\u0120distances": 18868, "\u0120wanna": 18869, "tar": 18870, "\u0120SER": 18871, "\u0120\u00e2\u012a": 18872, "\u0120lemon": 18873, "athetic": 18874, "\u0120literal": 18875, "\u0120distinguished": 18876, "\u0120answering": 18877, "GI": 18878, "\u0120religions": 18879, "\u0120Philos": 18880, "\u0120Lay": 18881, "\u0120compos": 18882, "irements": 18883, "\u0120Kos": 18884, "inez": 18885, "rolling": 18886, "\u0120youngest": 18887, "andise": 18888, "\u0120Born": 18889, "\u0120altar": 18890, "amina": 18891, "\u0120Boot": 18892, "voc": 18893, "\u0120digging": 18894, "\u0120pressures": 18895, "\u0120len": 18896, "264": 18897, "\u0120assassination": 18898, "\u0120Birmingham": 18899, "\u0120Myth": 18900, "\u0120sovereign": 18901, "\u0120Artist": 18902, "\u0120Photograph": 18903, "\u0120depicted": 18904, "\u0120dispens": 18905, "orthy": 18906, "\u0120ambul": 18907, "integ": 18908, "\u0120Cele": 18909, "\u0120Tibet": 18910, "\u0120hierarchy": 18911, "\u0120cu": 18912, "\u0120preseason": 18913, "\u0120Peterson": 18914, "\u0120colours": 18915, "\u0120worrying": 18916, "\u0120backers": 18917, "\u0120Palmer": 18918, "\u0120\u00ce\u00bc": 18919, "\u0120contributor": 18920, "\u0120hearings": 18921, "\u0120urine": 18922, "\u0120\u00d9": 18923, "ourgeois": 18924, "Similar": 18925, "\u0120Zimmer": 18926, "something": 18927, "\u0120USC": 18928, "\u0120strengths": 18929, "\u0120FI": 18930, "\u0120logging": 18931, "Asked": 18932, "\u0120Thai": 18933, "inqu": 18934, "\u0120Walt": 18935, "\u0120crews": 18936, "itism": 18937, "301": 18938, "\u0120sharply": 18939, "umed": 18940, "\u0120redirect": 18941, "rators": 18942, "Inf": 18943, "\u0120Weapons": 18944, "\u0120teasp": 18945, "1999": 18946, "Live": 18947, "\u0120Especially": 18948, "\u0120Ster": 18949, "\u0120Veterans": 18950, "\u0120intro": 18951, "otherapy": 18952, "\u0120malware": 18953, "\u0120breeding": 18954, "\u0120molecular": 18955, "\u0120Route": 18956, "\u0120Comment": 18957, "ochem": 18958, "\u0120ain": 18959, "Season": 18960, "\u0120linebacker": 18961, "\u00c4\u00ab": 18962, "\u0120Economics": 18963, "esar": 18964, "\u0120Lives": 18965, "\u0120Emma": 18966, "\u0120kin": 18967, "\u0120Territ": 18968, "\u0120planted": 18969, "oton": 18970, "\u0120Butter": 18971, "\u0120Spons": 18972, "PER": 18973, "\u0120dungeon": 18974, "\u0120symbolic": 18975, "\u0120filmed": 18976, "\u0120diets": 18977, "\u0120concludes": 18978, "\u0120certainty": 18979, "\u0120Format": 18980, "\u0120strangers": 18981, "format": 18982, "\u0120Phase": 18983, "\u0120copied": 18984, "\u0120metres": 18985, "lda": 18986, "\u0120Users": 18987, "\u0120deliberate": 18988, "\u0120washed": 18989, "\u0120Lance": 18990, "imation": 18991, "\u0120improper": 18992, "\u0120Genesis": 18993, "ickr": 18994, "\u0120Kush": 18995, "\u0120realise": 18996, "\u0120embarrassing": 18997, "alking": 18998, "bucks": 18999, "\u0120verified": 19000, "\u0120outline": 19001, "years": 19002, "\u0120Income": 19003, "202": 19004, "\u0120zombies": 19005, "Final": 19006, "\u0120Millenn": 19007, "\u0120modifications": 19008, "\u0120Vision": 19009, "\u0120Moses": 19010, "verb": 19011, "iterranean": 19012, "\u0120Jet": 19013, "\u0120naval": 19014, "\u0120Agg": 19015, "\u0120url": 19016, "\u0120victories": 19017, "\u0120nonetheless": 19018, "\u0120injust": 19019, "\u0120Fact": 19020, "\u00e7\u013c": 19021, "\u0120insufficient": 19022, "review": 19023, "facebook": 19024, "\u0120negotiating": 19025, "\u0120guarantees": 19026, "imen": 19027, "utenberg": 19028, "\u0120gambling": 19029, "\u0120congr": 19030, "Loading": 19031, "\u0120nevertheless": 19032, "\u0120presidents": 19033, "\u0120Industrial": 19034, "\u0120118": 19035, "\u0120poured": 19036, "\u0120Tory": 19037, "\u0120175": 19038, "\u0120:=": 19039, "Scott": 19040, "angered": 19041, "Tok": 19042, "\u0120organizers": 19043, "Mat": 19044, "\u0120Growth": 19045, "\u0120adul": 19046, "\u0120ensures": 19047, "\u0120117": 19048, "\u00e9\u00be\u012f\u00e5": 19049, "\u0120massacre": 19050, "\u0120grades": 19051, "before": 19052, "ADVERTISEMENT": 19053, "\u0120Slow": 19054, "\u0120MMA": 19055, "\u00e2\u0122\u0136\"": 19056, "\u0120Vatican": 19057, "Qaeda": 19058, "\u0120owe": 19059, "6666": 19060, "\u0120Sorry": 19061, "\u0120Grass": 19062, "\u0120backgrounds": 19063, "\u0120exhausted": 19064, "\u0120clan": 19065, "\u0120compromised": 19066, "\u0120Elf": 19067, "\u0120Isaac": 19068, "enson": 19069, "Invest": 19070, "IFA": 19071, "\u0120interrupted": 19072, "\u00e3\u0125\u012b\u00e3\u0125\u00a9": 19073, "\u0120twisted": 19074, "\u0120Dragons": 19075, "Mode": 19076, "\u0120Kremlin": 19077, "\u0120fertil": 19078, "heres": 19079, "phan": 19080, "\u0120Node": 19081, "fed": 19082, "\u0120Orc": 19083, "\u0120unwilling": 19084, "Cent": 19085, "\u0120priorit": 19086, "\u0120graduates": 19087, "\u0120subjective": 19088, "\u0120issuing": 19089, "\u0120Lt": 19090, "\u0120viewer": 19091, "\u0120woke": 19092, "Thus": 19093, "brook": 19094, "\u0120depressed": 19095, "\u0120bracket": 19096, "\u0120Gor": 19097, "\u0120Fighting": 19098, "\u0120striker": 19099, "Report": 19100, "\u0120Portugal": 19101, "\u0120neo": 19102, "wed": 19103, "199": 19104, "\u0120fleeing": 19105, "shadow": 19106, "identified": 19107, "USE": 19108, "Steam": 19109, "\u0120stretched": 19110, "\u0120revelations": 19111, "arted": 19112, "\u0120Dw": 19113, "\u0120alignment": 19114, "eston": 19115, "\u0120Jared": 19116, "Sep": 19117, "\u0120blogs": 19118, "update": 19119, "gom": 19120, "risk": 19121, "\u0120clash": 19122, "\u0120Hour": 19123, "\u0120runtime": 19124, "\u0120unwanted": 19125, "\u0120scam": 19126, "\u0120rack": 19127, "\u0120enlight": 19128, "onest": 19129, "\u0120Ferr": 19130, "\u0120convictions": 19131, "\u0120piano": 19132, "\u0120circulation": 19133, "\u0120Welcome": 19134, "\u0120backlash": 19135, "\u0120Wade": 19136, "\u0120receivers": 19137, "otive": 19138, "Jeff": 19139, "\u0120networking": 19140, "\u0120Prep": 19141, "\u0120Explorer": 19142, "\u0120lecture": 19143, "\u0120uploaded": 19144, "\u0120Meat": 19145, "BLE": 19146, "\u0120Nazis": 19147, "\u0120Synd": 19148, "stud": 19149, "roots": 19150, "rians": 19151, "\u0120portrayed": 19152, "\u0120??": 19153, "\u0120Buddha": 19154, "sun": 19155, "Robert": 19156, "\u0120Complex": 19157, "\u0120oversee": 19158, "\u0120stealth": 19159, "Title": 19160, "\u0120Jobs": 19161, "\u0120Kum": 19162, "\u0120appreciation": 19163, "\u0120MOD": 19164, "\u0120basics": 19165, "\u0120clips": 19166, "\u0120nursing": 19167, "\u0120proposition": 19168, "\u0120realised": 19169, "\u0120NYC": 19170, "\u0120allocated": 19171, "rium": 19172, "aran": 19173, "\u0120Production": 19174, "\u0120Vote": 19175, "\u0120smugg": 19176, "\u0120hunter": 19177, "azer": 19178, "\u0120Changes": 19179, "\u0120fluct": 19180, "yon": 19181, "Array": 19182, "\u0120kits": 19183, "Water": 19184, "\u0120uncommon": 19185, "\u0120resting": 19186, "ells": 19187, "would": 19188, "\u0120pursued": 19189, "\u0120assertion": 19190, "ometown": 19191, "\u0120Mosul": 19192, "\u0120Platform": 19193, "iolet": 19194, "\u0120shareholders": 19195, "\u0120trails": 19196, "Pay": 19197, "\u0120Enforcement": 19198, "types": 19199, "\u0120Anonymous": 19200, "\u0120satisfying": 19201, "ilogy": 19202, "\u0120('": 19203, "wave": 19204, "city": 19205, "Steve": 19206, "\u0120confrontation": 19207, "\u0120Eld": 19208, "Capt": 19209, "ahan": 19210, "htm": 19211, "\u0120Ctrl": 19212, "ONS": 19213, "230": 19214, "ifa": 19215, "holding": 19216, "\u0120delicate": 19217, "\u0120jaw": 19218, "\u0120Going": 19219, "orum": 19220, "Sal": 19221, "\u0120dull": 19222, "\u0120Beth": 19223, "\u0120prisons": 19224, "\u0120ego": 19225, "\u0120Elsa": 19226, "avorite": 19227, "\u0120Gang": 19228, "\u0120Nuclear": 19229, "\u0120spider": 19230, "atsu": 19231, "\u0120sampling": 19232, "\u0120absorbed": 19233, "\u0120Pharm": 19234, "ieth": 19235, "\u0120bucket": 19236, "\u0120Recomm": 19237, "OF": 19238, "\u0120Factory": 19239, "ANCE": 19240, "\u0120bacter": 19241, "Has": 19242, "\u0120Observ": 19243, "121": 19244, "\u0120premiere": 19245, "Develop": 19246, "\u0120currencies": 19247, "Cast": 19248, "\u0120accompanying": 19249, "\u0120Nashville": 19250, "\u0120fatty": 19251, "\u0120Brend": 19252, "\u0120locks": 19253, "\u0120centered": 19254, "\u0120UT": 19255, "aughs": 19256, "orie": 19257, "\u0120Affordable": 19258, "vance": 19259, "DL": 19260, "emet": 19261, "\u0120throne": 19262, "\u0120Bluetooth": 19263, "\u0120naming": 19264, "ifts": 19265, "ADE": 19266, "\u0120corrected": 19267, "\u0120promptly": 19268, "\u0120STR": 19269, "\u0120genome": 19270, "\u0120cope": 19271, "\u0120valley": 19272, "\u0120rounded": 19273, "\u0120Kend": 19274, "alion": 19275, "pers": 19276, "\u0120tourism": 19277, "\u0120stark": 19278, "vl": 19279, "\u0120blowing": 19280, "\u0120Schedule": 19281, "std": 19282, "\u0120unhappy": 19283, "\u0120litigation": 19284, "cedes": 19285, "\u0120android": 19286, "\u0120integral": 19287, "erers": 19288, "uded": 19289, "tax": 19290, "\u0120reiter": 19291, "\u0120Motors": 19292, "ociated": 19293, "\u0120wonders": 19294, "\u0120Apost": 19295, "ucking": 19296, "\u0120Roosevelt": 19297, "fram": 19298, "\u0120yields": 19299, "\u0120constitutes": 19300, "awk": 19301, "Interest": 19302, "\u0120interim": 19303, "\u0120breakthrough": 19304, "\u0120Cher": 19305, "\u0120prosec": 19306, "\u0120Dj": 19307, "\u0120MT": 19308, "Resp": 19309, "\u0120PT": 19310, "\u0120sperm": 19311, "edit": 19312, "BT": 19313, "Linux": 19314, "country": 19315, "league": 19316, "\u0120dick": 19317, "\u0120oct": 19318, "\u0120inserting": 19319, "\u0120scra": 19320, "\u0120Brewing": 19321, "\u01201966": 19322, "\u0120runners": 19323, "\u0120plun": 19324, "idy": 19325, "\u0120Dian": 19326, "\u0120dysfunction": 19327, "\u0120exclusion": 19328, "\u0120disgr": 19329, "\u0120incorporate": 19330, "\u0120reconc": 19331, "\u0120nominated": 19332, "\u0120Archer": 19333, "draw": 19334, "achelor": 19335, "\u0120writings": 19336, "\u0120shallow": 19337, "\u0120hast": 19338, "\u0120BMW": 19339, "\u0120RS": 19340, "\u0120thigh": 19341, "\u01201963": 19342, "\u0120lamb": 19343, "\u0120favored": 19344, "agle": 19345, "\u0120cooler": 19346, "\u0120Hours": 19347, "\u0120GU": 19348, "\u0120Origin": 19349, "\u0120glimpse": 19350, "--------------------": 19351, "Lim": 19352, "\u0120cheek": 19353, "\u0120jealous": 19354, "-'": 19355, "\u0120harness": 19356, "\u0120Poison": 19357, "\u0120disabilities": 19358, "neapolis": 19359, "\u0120outlook": 19360, "\u0120notify": 19361, "\u0120Indianapolis": 19362, "\u0120abrupt": 19363, "nsic": 19364, "\u0120encrypted": 19365, "\u0120forfe": 19366, "reath": 19367, "\u0120rabb": 19368, "\u0120foundations": 19369, "\u0120compliment": 19370, "\u0120Interview": 19371, "\u0120Swe": 19372, "\u0120adolesc": 19373, "\u0120monitors": 19374, "\u0120Sacramento": 19375, "\u0120timely": 19376, "\u0120contempl": 19377, "\u0120positioned": 19378, "\u0120posters": 19379, "phies": 19380, "iovascular": 19381, "void": 19382, "\u0120Fifth": 19383, "\u0120investigative": 19384, "OUN": 19385, "\u0120integrate": 19386, "\u0120INC": 19387, "isha": 19388, "iblings": 19389, "\u0120Request": 19390, "\u0120Rodriguez": 19391, "\u0120slides": 19392, "\u0120DX": 19393, "\u0120feminism": 19394, "\u0120datas": 19395, "\u0120bend": 19396, "irus": 19397, "\u0120Nigeria": 19398, "Fox": 19399, "Change": 19400, "\u0120airplane": 19401, "\u0120Laden": 19402, "\u0120publicity": 19403, "ixty": 19404, "\u0120commitments": 19405, "\u0120aggregate": 19406, "\u0120displaying": 19407, "\u0120Arrow": 19408, "\u0120122": 19409, "\u0120respects": 19410, "android": 19411, "six": 19412, "\u0120Sha": 19413, "\u0120restoration": 19414, ")\\": 19415, "WS": 19416, "oys": 19417, "\u0120illustrate": 19418, "without": 19419, "126": 19420, "\u0120\u00e2\u0136\u0124": 19421, "\u0120pickup": 19422, "nels": 19423, "\u0120....": 19424, "food": 19425, "\u0120Fen": 19426, ")?": 19427, "\u0120phenomena": 19428, "\u0120companions": 19429, "\u0120Write": 19430, "\u0120spill": 19431, "\u0120bridges": 19432, "\u0120Updated": 19433, "\u0120Fo": 19434, "\u0120insects": 19435, "ASHINGTON": 19436, "\u0120scare": 19437, "iltr": 19438, "\u0120Zhang": 19439, "\u0120severity": 19440, "\u0120indul": 19441, "149": 19442, "\u0120Coffee": 19443, "\u0120norms": 19444, "\u0120pulse": 19445, "\u0120FT": 19446, "\u0120horrific": 19447, "\u0120Destroy": 19448, "\u0120JSON": 19449, "\u0120olive": 19450, "\u0120discusses": 19451, "Rest": 19452, "Elect": 19453, "\u0120Winn": 19454, "\u0120Surviv": 19455, "\u0120Hait": 19456, "Sure": 19457, "oped": 19458, "\u0120rooted": 19459, "\u0120Ske": 19460, "\u0120Bronze": 19461, "\u0120lol": 19462, "Default": 19463, "\u0120commodity": 19464, "redited": 19465, "\u0120libertarian": 19466, "\u0120forbidden": 19467, "\u0120gran": 19468, "\u00e0\u00a8": 19469, "\u0120lag": 19470, "enz": 19471, "drive": 19472, "\u0120mathematics": 19473, "\u0120wires": 19474, "\u0120critically": 19475, "\u0120carbohyd": 19476, "\u0120Chancellor": 19477, "\u0120Eddie": 19478, "\u0120banning": 19479, "\u0120Fri": 19480, "\u0120complications": 19481, "etric": 19482, "\u0120Bangladesh": 19483, "\u0120bandwidth": 19484, "Stop": 19485, "\u0120Originally": 19486, "\u0120halfway": 19487, "ynasty": 19488, "shine": 19489, "\u0120tales": 19490, "rities": 19491, "avier": 19492, "\u0120spinning": 19493, "\u0120WHO": 19494, "\u0120neighbourhood": 19495, "bach": 19496, "\u0120commerce": 19497, "\u0120Sle": 19498, "BU": 19499, "\u0120entrepreneur": 19500, "\u0120peculiar": 19501, "\u0120Comments": 19502, "fre": 19503, "320": 19504, "ICS": 19505, "\u0120imagery": 19506, "\u0120Canon": 19507, "\u0120Electronic": 19508, "short": 19509, "((": 19510, "Dig": 19511, "\u0120commem": 19512, "uced": 19513, "\u0120inclined": 19514, "\u0120Summon": 19515, "\u0120cliff": 19516, "\u0120Mediterranean": 19517, "\u0120poetry": 19518, "\u0120prosperity": 19519, "\u0120Rece": 19520, "\u0120pills": 19521, "member": 19522, "\u0120finale": 19523, "unc": 19524, "\u0120Gig": 19525, "\u00e4\u00bd": 19526, "\u0120lod": 19527, "\u0120backward": 19528, "-+": 19529, "\u0120Forward": 19530, "\u0120thri": 19531, "sure": 19532, "\u0120soap": 19533, "\u0120FX": 19534, "RES": 19535, "\u0120Sexual": 19536, "oulos": 19537, "\u0120foolish": 19538, "\u0120righteous": 19539, "\u0120coff": 19540, "terrorism": 19541, "ustain": 19542, "oter": 19543, "\u0120abuses": 19544, "next": 19545, "\u0120abusive": 19546, "\u0120thereafter": 19547, "\u0120prohibition": 19548, "\u0120SUP": 19549, "\u0120dip": 19550, "\u0120ripped": 19551, "\u0120inherited": 19552, "\u0120bats": 19553, "stru": 19554, "GT": 19555, "\u0120flawed": 19556, "phabet": 19557, "\u0120fog": 19558, "doors": 19559, "\u0120imaging": 19560, "\u0120digits": 19561, "\u0120Hungary": 19562, "\u0120arrog": 19563, "\u0120teachings": 19564, "\u0120protocols": 19565, "\u0120Banks": 19566, "\u00e0\u00b8": 19567, "pound": 19568, "\u0120Curt": 19569, ".\")": 19570, "./": 19571, "\u0120exemption": 19572, "endix": 19573, "\u0120Mull": 19574, "\u0120improves": 19575, "\u0120Gamer": 19576, "dimensional": 19577, "Icon": 19578, "\u0120Margaret": 19579, "Status": 19580, "dates": 19581, "\u0120intends": 19582, "\u0120depict": 19583, "\u0120parked": 19584, "Joe": 19585, "\u0120Marines": 19586, "chnology": 19587, "!).": 19588, "\u0120judged": 19589, "\u0120weights": 19590, "Ray": 19591, "\u0120apartments": 19592, "hester": 19593, "\u0120reinforce": 19594, "\u0120offender": 19595, "occup": 19596, "\u0120sore": 19597, "ept": 19598, "\u0120PHP": 19599, "\u0120Brow": 19600, "\u0120authorization": 19601, "\u0120Risk": 19602, "\u0120Delaware": 19603, "\u0120QU": 19604, "\u0120notifications": 19605, "\u0120sunlight": 19606, "\u0120exclude": 19607, "dat": 19608, "\u0120mesh": 19609, "\u0120Sudan": 19610, "\u0120belonged": 19611, "\u0120subway": 19612, "\u0120noon": 19613, "\u0120Interior": 19614, "olics": 19615, "\u0120Lakers": 19616, "\u0120coding": 19617, "Disclaimer": 19618, "Calif": 19619, "Old": 19620, "\u0120disl": 19621, "?????": 19622, "\u0120confirms": 19623, "\u0120recruitment": 19624, "\u0120homicide": 19625, "Consider": 19626, "\u0120Jeffrey": 19627, "fty": 19628, "};": 19629, "\u0120objection": 19630, "doing": 19631, "\u0120Leo": 19632, "Want": 19633, "\u0120glow": 19634, "\u0120Clarke": 19635, "\u0120Norman": 19636, "\u0120verification": 19637, "\u0120packet": 19638, "\u0120Formula": 19639, "\u0120plag": 19640, "esville": 19641, "\u0120shouting": 19642, "\u0120ov": 19643, "\u0120REC": 19644, "\u0120Bub": 19645, "\u0120ninth": 19646, "\u0120energ": 19647, "\u0120validity": 19648, "\u0120ups": 19649, "jack": 19650, "\u0120neighboring": 19651, "\u0120Nec": 19652, "eworks": 19653, "\u0120Hab": 19654, "arez": 19655, "\u0120spine": 19656, "\u0120eventual": 19657, "\u0120Leaders": 19658, "\u0120Carn": 19659, "\u0120probation": 19660, "\u0120romance": 19661, "msg": 19662, "\u0120Mechanical": 19663, "ERY": 19664, "Rock": 19665, "\u0120partisan": 19666, "Node": 19667, "assets": 19668, "minent": 19669, "\u0120foreigners": 19670, "\u0120testify": 19671, "\u0120Usually": 19672, "lords": 19673, "\u0120Gren": 19674, "\u0120Powell": 19675, "BIL": 19676, "\u0120sr": 19677, "\u0120addict": 19678, "\u0120shells": 19679, "\u0120sigh": 19680, "\u0120Yale": 19681, "ternity": 19682, "\u0120750": 19683, "EU": 19684, "\u0120Rifle": 19685, "\u0120patron": 19686, "ema": 19687, "\u0120Bannon": 19688, "anity": 19689, "\u0120tropical": 19690, "\u0120VII": 19691, "cross": 19692, "Everything": 19693, "\u0120ISO": 19694, "\u0120humble": 19695, "assing": 19696, "\u0120FIG": 19697, "\u0120updating": 19698, "yson": 19699, "\u0120calcium": 19700, "\u0120competent": 19701, "\u0120steering": 19702, "Prot": 19703, "\u0120SY": 19704, "\u0120Finals": 19705, "\u0120Rug": 19706, "159": 19707, "137": 19708, "\u0120Golf": 19709, "\u0120126": 19710, "\u0120accommodation": 19711, "\u0120Hughes": 19712, "\u0120aesthetic": 19713, "artisan": 19714, "\u0120Twilight": 19715, "\u0120prince": 19716, "\u0120Agriculture": 19717, "\u0120Disco": 19718, "\u0120precedent": 19719, "\u0120typing": 19720, "authorized": 19721, "Option": 19722, "\u0120Aub": 19723, "lishes": 19724, "acht": 19725, "mag": 19726, "Peter": 19727, "\u0120UFO": 19728, "monton": 19729, "\u0120Lith": 19730, "\u0120arom": 19731, "\u0120securing": 19732, "\u0120confined": 19733, "private": 19734, "\u0120swords": 19735, "\u0120markers": 19736, "\u0120metabolic": 19737, "select": 19738, "\u0120Curse": 19739, "\u0120Ot": 19740, "gressive": 19741, "\u0120incumb": 19742, "\u0120Saga": 19743, "\u0120priced": 19744, "\u0120clearance": 19745, "Content": 19746, "\u0120drilling": 19747, "\u0120notices": 19748, "\u0120bourgeois": 19749, "\u0120vest": 19750, "\u0120cookie": 19751, "\u0120Guardians": 19752, "rys": 19753, "inyl": 19754, "\u0120124": 19755, "\u0120plausible": 19756, "ongh": 19757, "\u0120Odin": 19758, "\u0120conception": 19759, "\u0120Yuk": 19760, "\u0120Baghdad": 19761, "\u0120Flag": 19762, "Austral": 19763, "\u0120IBM": 19764, "\u0120internationally": 19765, "\u0120WikiLeaks": 19766, "IED": 19767, "\u0120cyn": 19768, "\u0120chooses": 19769, "\u0120Pill": 19770, "\u0120combining": 19771, "\u0120radi": 19772, "\u0120Mohammed": 19773, "defense": 19774, "atching": 19775, "Subject": 19776, "iciency": 19777, "Frame": 19778, "\u0120{\"": 19779, "\u0120chess": 19780, "\u0120timer": 19781, "190": 19782, "\u0120tin": 19783, "\u0120ordinance": 19784, "emetery": 19785, "\u0120accusing": 19786, "\u0120noticeable": 19787, "\u0120centres": 19788, "\u0120lid": 19789, "\u0120Mills": 19790, "imgur": 19791, "\u0120zoom": 19792, "ergic": 19793, "\u0120compression": 19794, "prim": 19795, "find": 19796, "\u0120surg": 19797, "\u0120pand": 19798, "\u0120Kee": 19799, "\u0120Chad": 19800, "cellence": 19801, "oyle": 19802, "\u0120socialism": 19803, "\u0120Travis": 19804, "\u0120MHz": 19805, "\u0120guild": 19806, "ALLY": 19807, "\u0120Subscribe": 19808, "\u0120Related": 19809, "\u0120occurrence": 19810, "itching": 19811, "\u0120fictional": 19812, "\u0120crush": 19813, "\u0120EA": 19814, "cod": 19815, "mix": 19816, "\u0120Triple": 19817, "\u0120retrieve": 19818, "\u0120stimulus": 19819, "\u0120psychiat": 19820, "\u0120Door": 19821, "\u0120homosexuality": 19822, "\u0120elementary": 19823, "\u0120cellular": 19824, "idian": 19825, "\u0120Laun": 19826, "\u0120intriguing": 19827, "\u0120foam": 19828, "\u0120Bass": 19829, "idi": 19830, "itsu": 19831, "\u0120assure": 19832, "\u0120congrat": 19833, "\u0120businessman": 19834, "\u0120Boost": 19835, "close": 19836, "\u0120lied": 19837, "\u0120sciences": 19838, "\u0120Omega": 19839, "\u0120Graphics": 19840, "\u0120<=": 19841, "spoken": 19842, "\u0120connectivity": 19843, "Saturday": 19844, "\u0120Avengers": 19845, "\u0120toggle": 19846, "\u0120ankle": 19847, "\u0120nationalist": 19848, "model": 19849, "\u0120Pool": 19850, "ophobia": 19851, "Var": 19852, "\u0120Mons": 19853, "atories": 19854, "\u0120aggressively": 19855, "Clear": 19856, "Forge": 19857, "acters": 19858, "\u0120hedge": 19859, "\u0120pipes": 19860, "\u0120blunt": 19861, "\u0120sq": 19862, "\u0120remotely": 19863, "Wed": 19864, "asers": 19865, "\u0120refriger": 19866, "\u0120tiles": 19867, "\u0120rescued": 19868, "\u0120comprised": 19869, "insky": 19870, "\u0120manif": 19871, "avanaugh": 19872, "\u0120prolifer": 19873, "\u0120aligned": 19874, "xml": 19875, "\u0120triv": 19876, "\u0120coordination": 19877, "\u0120PER": 19878, "\u0120Quote": 19879, "134": 19880, "bf": 19881, "\u0120Saw": 19882, "\u0120termination": 19883, "\u0120190": 19884, "\u0120additions": 19885, "\u0120trio": 19886, "\u0120projections": 19887, "\u0120positively": 19888, "\u0120inclusive": 19889, "\u0120membr": 19890, "1990": 19891, "older": 19892, "\u0120practiced": 19893, "inkle": 19894, "Arch": 19895, "\u0120starters": 19896, "arius": 19897, "\u0120intermediate": 19898, "\u0120Benef": 19899, "\u0120Killer": 19900, "\u0120interventions": 19901, "\u0120Kil": 19902, "\u0120Flying": 19903, "Inv": 19904, "\u0120premature": 19905, "\u0120psychiatric": 19906, "\u0120indie": 19907, "\u0120collar": 19908, "\u0120Rainbow": 19909, "afi": 19910, "\u0120disruption": 19911, "\u0120FOX": 19912, "casting": 19913, "\u0120misdem": 19914, "cro": 19915, "\u0120wipe": 19916, "ardon": 19917, "\u0120bast": 19918, "\u0120Tommy": 19919, "\u0120Representative": 19920, "\u0120belly": 19921, "\u0120PO": 19922, "\u0120Breitbart": 19923, "132": 19924, "\u0120messaging": 19925, "Should": 19926, "References": 19927, "\u0120GRE": 19928, "istical": 19929, "LP": 19930, "\u0120Cav": 19931, "\u0120Crazy": 19932, "\u0120intuitive": 19933, "keeping": 19934, "\u0120Moss": 19935, "\u0120discontin": 19936, "\u0120Module": 19937, "\u0120unrelated": 19938, "\u0120Practice": 19939, "\u0120Transport": 19940, "\u0120statistically": 19941, "orns": 19942, "\u0120sized": 19943, "pu": 19944, "\u0120caf": 19945, "\u0120Worlds": 19946, "\u0120Rodgers": 19947, "\u0120Lun": 19948, "\u0120Comic": 19949, "living": 19950, "\u0120cared": 19951, "\u0120climbed": 19952, "){": 19953, "\u0120consisted": 19954, "\u0120medieval": 19955, "folk": 19956, "\u0120hacked": 19957, "\u0120dire": 19958, "\u0120Hermione": 19959, "\u0120tended": 19960, "ceans": 19961, "Daniel": 19962, "went": 19963, "\u0120legislators": 19964, "\u0120redes": 19965, "games": 19966, "\u0120gn": 19967, "amiliar": 19968, "\u0120++": 19969, "ggy": 19970, "threat": 19971, "\u0120magnet": 19972, "\u0120perceive": 19973, "\u0120zip": 19974, "\u0120indictment": 19975, "\u0120critique": 19976, "gard": 19977, "\u0120Safe": 19978, "\u0120Cream": 19979, "\u0120advent": 19980, "oba": 19981, "\u0120vowed": 19982, "ousands": 19983, "\u0120ski": 19984, "\u0120abortions": 19985, "uart": 19986, "\u0120stunned": 19987, "\u0120advancing": 19988, "\u0120lacked": 19989, "\u0120\\\"": 19990, "\u0120schizophren": 19991, "\u0120elegant": 19992, "\u0120conferences": 19993, "\u0120canceled": 19994, "\u0120Hudson": 19995, "\u0120Hopefully": 19996, "\u0120trump": 19997, "\u0120frequencies": 19998, "\u0120meteor": 19999, "\u0120Junior": 20000, "\u0120Fleet": 20001, "\u0120Malcolm": 20002, "\u0120Tools": 20003, "\u0120........": 20004, "\u0120hobby": 20005, "\u0120Europeans": 20006, "\u01201500": 20007, "\u0120Into": 20008, "\u0120sway": 20009, "\u0120Appro": 20010, "\u0120Compl": 20011, "Community": 20012, "\u0120tide": 20013, "\u0120Summit": 20014, "\u00e4\u00bb": 20015, "\u0120intervals": 20016, "\u0120Ether": 20017, "\u0120habitat": 20018, "\u0120Stevens": 20019, "lishing": 20020, "\u0120Domain": 20021, "\u0120triggers": 20022, "\u0120chasing": 20023, "\u0120charm": 20024, "\u0120Flower": 20025, "itored": 20026, "\u0120blessing": 20027, "\u0120textures": 20028, "Five": 20029, "\u0120liquor": 20030, "RP": 20031, "FIN": 20032, "\u01201962": 20033, "CAR": 20034, "Unknown": 20035, "\u0120resil": 20036, "\u0120Lily": 20037, "\u0120abundance": 20038, "\u0120predictable": 20039, "rar": 20040, "\u0120bullshit": 20041, "leen": 20042, "chet": 20043, "Mor": 20044, "Much": 20045, "\u00e4\u00b9": 20046, "\u0120emphasized": 20047, "\u0120crust": 20048, "\u0120primitive": 20049, "\u0120enjoyable": 20050, "\u0120Pictures": 20051, "\u0120teammate": 20052, "pler": 20053, "\u0120Tol": 20054, "\u0120Kane": 20055, "\u0120summoned": 20056, "thy": 20057, "rama": 20058, "\u0120Honda": 20059, "\u0120realizing": 20060, "\u0120quicker": 20061, "\u0120concentrate": 20062, "clear": 20063, "\u0120210": 20064, "\u0120Erdogan": 20065, "aris": 20066, "\u0120responds": 20067, "\u0120BI": 20068, "\u0120eligibility": 20069, "\u0120pushes": 20070, "\u0120Idaho": 20071, "\u0120aggrav": 20072, "\u0120ruins": 20073, "urations": 20074, "\u0120bans": 20075, "\u0120anat": 20076, "share": 20077, "\u0120grind": 20078, "hin": 20079, "umen": 20080, "\u0120utilities": 20081, "\u0120Yankees": 20082, "\u0120databases": 20083, "\u0120DD": 20084, "\u0120displaced": 20085, "\u0120dependencies": 20086, "\u0120stimulation": 20087, "hun": 20088, "houses": 20089, "\u0120Pretty": 20090, "\u0120Ravens": 20091, "\u0120TODAY": 20092, "\u0120associates": 20093, "\u0120therape": 20094, "cled": 20095, "\u0120deer": 20096, "\u0120repairs": 20097, "rentice": 20098, "\u0120receptors": 20099, "\u0120remed": 20100, "\u0120Ce": 20101, "\u0120marriages": 20102, "\u0120ballots": 20103, "\u0120Soldier": 20104, "\u0120hilarious": 20105, "opl": 20106, "138": 20107, "\u0120inherently": 20108, "\u0120ignorant": 20109, "\u0120bounce": 20110, "\u0120Easter": 20111, "RELATED": 20112, "\u0120Currency": 20113, "EV": 20114, "\u00e3\u0125\u0140": 20115, "\u0120Lead": 20116, "\u0120deceased": 20117, "Brien": 20118, "\u0120Musk": 20119, "JS": 20120, "\u0120merge": 20121, "hearted": 20122, "creat": 20123, "mitt": 20124, "mund": 20125, "\u0120\u00e2\u0122\u012d": 20126, "\u0120Bag": 20127, "\u0120projection": 20128, "\u0120java": 20129, "\u0120Standards": 20130, "\u0120Leonard": 20131, "\u0120coconut": 20132, "\u0120Population": 20133, "\u0120traject": 20134, "\u0120imply": 20135, "\u0120curiosity": 20136, "\u0120DB": 20137, "\u0120Fresh": 20138, "\u0120Por": 20139, "\u0120heavier": 20140, "neys": 20141, "gomery": 20142, "\u0120deserved": 20143, "\u0120phrases": 20144, "\u0120GC": 20145, "\u0120yeast": 20146, "desc": 20147, "Death": 20148, "\u0120reboot": 20149, "\u0120metadata": 20150, "ICAL": 20151, "\u0120repay": 20152, "\u0120Independence": 20153, "\u0120suburban": 20154, "icals": 20155, "\u0120atop": 20156, "\u0120allocation": 20157, "generation": 20158, "\u0120Gram": 20159, "\u0120moisture": 20160, "\u0120pine": 20161, "\u0120Liberals": 20162, "\u0120aides": 20163, "\u0120underest": 20164, "\u0120Berry": 20165, "\u0120ceremon": 20166, "370": 20167, "astrous": 20168, "\u0120Pirates": 20169, "\u0120tense": 20170, "\u0120Industries": 20171, "\u0120Appeals": 20172, "\u0120Near": 20173, "\u0120\u00e8\u00a3\u0131\u00e7": 20174, "\u0120lovers": 20175, "\u0120CAP": 20176, "\u0120Craw": 20177, "\u0120giants": 20178, "\u0120efficacy": 20179, "Element": 20180, "\u0120Behavior": 20181, "\u0120Toyota": 20182, "\u0120intest": 20183, "Priv": 20184, "AI": 20185, "\u0120maneuver": 20186, "\u0120perfection": 20187, "\u0120bang": 20188, "paper": 20189, "rill": 20190, "George": 20191, "border": 20192, "inters": 20193, "\u0120Seth": 20194, "\u0120clues": 20195, "\u0120Levi": 20196, "\u0120Revenue": 20197, "147": 20198, "\u0120vapor": 20199, "\u0120fortunate": 20200, "\u0120threatens": 20201, "\u0120vet": 20202, "\u0120dependency": 20203, "ersed": 20204, "article": 20205, "\u0120Blizzard": 20206, "\u0120chlor": 20207, "\u0120minus": 20208, "\u0120Bills": 20209, "\u0120cryptocurrency": 20210, "\u0120metabolism": 20211, "tering": 20212, "\u0120pestic": 20213, "steps": 20214, "\u0120Treasure": 20215, "racted": 20216, "\u0120Constant": 20217, "\u0120temp": 20218, "139": 20219, "\u0120Detective": 20220, "urally": 20221, "\u0120recovering": 20222, "\u0120cortex": 20223, "\u0120144": 20224, "closed": 20225, "\u0120prejudice": 20226, "aunted": 20227, "\u0120storms": 20228, "\u0120NOW": 20229, "\u0120machinery": 20230, "Address": 20231, "\u0120compelled": 20232, "270": 20233, "\u0120despair": 20234, "bane": 20235, "\u0120vegetable": 20236, "\u0120beds": 20237, "Learn": 20238, "\u0120colorful": 20239, "\u0120spike": 20240, "\u0120margins": 20241, "\u0120sympathy": 20242, "\u0120workshop": 20243, "\u0120CBC": 20244, "Sat": 20245, "\u0120burns": 20246, "\u0120Gender": 20247, "\u0120129": 20248, "\u0120Cable": 20249, "\u0120debts": 20250, "\u0120Theresa": 20251, "\u0120reflecting": 20252, "\u0120airst": 20253, "\u0120rim": 20254, "ramid": 20255, "\u0120weaknesses": 20256, "Writ": 20257, "oggle": 20258, "ti": 20259, "\u0120Charge": 20260, "\u0120weighed": 20261, "\u0120(.": 20262, "\u0120laughter": 20263, "\u0120router": 20264, "\u0120Democracy": 20265, "Dear": 20266, "\u0120hasht": 20267, "\u0120dy": 20268, "\u0120hints": 20269, "running": 20270, "\u0120finishes": 20271, "arus": 20272, "Mass": 20273, "result": 20274, "ascus": 20275, "\u0120vintage": 20276, "\u0120conqu": 20277, "\u0120wildly": 20278, "acist": 20279, "\u0120lingu": 20280, "\u0120protagonist": 20281, "strom": 20282, "teenth": 20283, "\u0120Solo": 20284, "mac": 20285, "filled": 20286, "\u0120renown": 20287, "itives": 20288, "\u0120motive": 20289, "\u0120Antar": 20290, "\u0120Mann": 20291, "\u0120Adjust": 20292, "\u0120rockets": 20293, "\u0120troubling": 20294, "ei": 20295, "\u0120organisms": 20296, "assis": 20297, "Christian": 20298, "\u0120145": 20299, "\u0120Hass": 20300, "\u0120swall": 20301, "\u0120wax": 20302, "\u0120Survival": 20303, "VS": 20304, "\u0120Murd": 20305, "vd": 20306, "standard": 20307, "\u0120dragons": 20308, "\u0120acceleration": 20309, "rational": 20310, "final": 20311, "\u0120paired": 20312, "\u0120Ethereum": 20313, "\u0120interfaces": 20314, "\u0120resent": 20315, "\u0120artifacts": 20316, "\u00c5\u00ab": 20317, "arel": 20318, "\u0120competitor": 20319, "\u0120Nicholas": 20320, "\u0120Surface": 20321, "cpp": 20322, "\u0120Tot": 20323, "\u0120economically": 20324, "\u0120organised": 20325, "\u0120enforced": 20326, "inho": 20327, "\u0120varieties": 20328, "\u0120abdom": 20329, "\u0120Bailey": 20330, "idav": 20331, "\u0120Salv": 20332, "paid": 20333, "\u0120altitude": 20334, "essert": 20335, "\u0120Gutenberg": 20336, "area": 20337, "opoulos": 20338, "\u0120professors": 20339, "iggs": 20340, "\u0120Fate": 20341, "hey": 20342, "\u01203000": 20343, "Dist": 20344, "\u0120twins": 20345, "cill": 20346, "\u0120Maps": 20347, "\u0120traps": 20348, "\u0120weed": 20349, "\u0120Kiss": 20350, "\u0120yoga": 20351, "\u0120recipients": 20352, "\u0120Westminster": 20353, "\u0120pools": 20354, "\u0120Walmart": 20355, "188": 20356, "\u0120Schools": 20357, "attack": 20358, "\u0120ARM": 20359, "paragraph": 20360, "Warning": 20361, "jl": 20362, "\u0120selfish": 20363, "anchez": 20364, "\u0120Heights": 20365, "Fre": 20366, "\u0120Soph": 20367, "\u0120--------------------------------": 20368, "tml": 20369, "333": 20370, "\u0120raids": 20371, "\u0120satellites": 20372, "KEY": 20373, "\u0120lasts": 20374, "\u00d1\u0124": 20375, "Ins": 20376, "\u0120Dame": 20377, "\u0120unpredict": 20378, "///": 20379, "ghai": 20380, "\u0120artillery": 20381, "\u0120cruise": 20382, "\u0120gel": 20383, "\u0120Cabinet": 20384, "\u0120blows": 20385, "\u0120Esp": 20386, "\u0120proximity": 20387, "othe": 20388, "\u0120Skills": 20389, "\u0120Upper": 20390, "obo": 20391, "\u0120NDP": 20392, "\u0120enjoys": 20393, "\u0120repeating": 20394, "\u0120Construction": 20395, "\u0120Questions": 20396, "Hillary": 20397, "\u0120uint": 20398, "\u0120processors": 20399, "\u0120Gibson": 20400, "\u0120Multiple": 20401, "qa": 20402, "\u0120Bom": 20403, "\u0120Miles": 20404, "ventional": 20405, "\u0120hurts": 20406, "skin": 20407, "\u0120AIDS": 20408, "\u0120advisers": 20409, "\u0120Root": 20410, "\u0120methodology": 20411, "\u0120Dale": 20412, "\u0120deton": 20413, "\u0120Knowledge": 20414, "sequently": 20415, "\u0120121": 20416, "\u0120connects": 20417, "Cy": 20418, "\u0120Danger": 20419, "\u0120contributors": 20420, "\u0120Bent": 20421, "\u0120brass": 20422, "\u0120Guns": 20423, "into": 20424, "\u0120Fortune": 20425, "\u0120broker": 20426, "balance": 20427, "\u0120lengths": 20428, "\u0120vic": 20429, "\u0120averaging": 20430, "\u0120appropriately": 20431, "\u0120Camera": 20432, "\u0120sandwich": 20433, "\u0120CDC": 20434, "\u0120coordinate": 20435, "\u0120navig": 20436, "\u0120goodness": 20437, "laim": 20438, "\u0120brake": 20439, "\u0120extremist": 20440, "\u0120Wake": 20441, "\u0120Mend": 20442, "\u0120Tiny": 20443, "\u0120COL": 20444, "\u0120RF": 20445, "\u0120Dual": 20446, "\u0120Wine": 20447, "Case": 20448, "\u0120refined": 20449, "\u0120lamp": 20450, "Lead": 20451, "\u0120bapt": 20452, "\u0120Carb": 20453, "\u0120Sadd": 20454, "\u0120Minneapolis": 20455, "PDF": 20456, "Early": 20457, "\u0120Hidden": 20458, "Its": 20459, "\u0120TIME": 20460, "\u0120pap": 20461, "\u0120commissioned": 20462, "\u0120Few": 20463, "\u0120Colts": 20464, "\u0120Bren": 20465, "\u0120bothered": 20466, "\u0120likewise": 20467, "Exper": 20468, "\u0120Schw": 20469, "cry": 20470, "nn": 20471, "\u0120Mitch": 20472, "imon": 20473, "MG": 20474, "bm": 20475, "UMP": 20476, "rays": 20477, "\u0120registry": 20478, "\u0120270": 20479, "achine": 20480, "rella": 20481, "anting": 20482, "00000": 20483, "\u0120ruined": 20484, "spot": 20485, "\u0120ta": 20486, "\u0120maximize": 20487, "\u0120inconven": 20488, "Dead": 20489, "Human": 20490, "Enabled": 20491, "\u0120Marie": 20492, "\u0120chill": 20493, "\u0120Paradise": 20494, "\u0120starring": 20495, "\u0120Latino": 20496, "\u0120Protocol": 20497, "\u0120EVER": 20498, "\u0120suppliers": 20499, "message": 20500, "\u0120Brock": 20501, "\u0120serum": 20502, "\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a": 20503, "\u0120encomp": 20504, "\u0120ambition": 20505, "uese": 20506, "\u0120arrows": 20507, "Andrew": 20508, "\u0120antenna": 20509, "\u01201961": 20510, "\u0120Bark": 20511, "\u0120bool": 20512, "\u00e3\u0124\u00aa": 20513, "\u0120Storage": 20514, "\u0120railway": 20515, "\u0120tougher": 20516, "\u0120Cad": 20517, "\u0120washing": 20518, "Py": 20519, "']": 20520, "embed": 20521, "\u0120Memphis": 20522, "ackle": 20523, "\u0120famously": 20524, "\u0120Fortunately": 20525, "ovies": 20526, "\u0120mindset": 20527, "\u0120sneak": 20528, "\u0120Dh": 20529, "RAW": 20530, "\u0120Simpson": 20531, "\u0120livest": 20532, "\u0120landmark": 20533, "\u0120cement": 20534, "Low": 20535, "\u0120thrilled": 20536, "\u0120Course": 20537, "inel": 20538, "\u0120chuck": 20539, "idate": 20540, "global": 20541, "\u0120whit": 20542, "\u0120\u00ef\u00bf\u00bd": 20543, "adays": 20544, "ski": 20545, "\u0120SV": 20546, "\u0120viruses": 20547, "306": 20548, "\u0120Respons": 20549, "\u0120theaters": 20550, "\u0120Branch": 20551, "\u0120Geneva": 20552, "\u0120MK": 20553, "\u0120unbeliev": 20554, "\u0120communist": 20555, "Original": 20556, "\u0120Received": 20557, "\u0120Transfer": 20558, "\u0120Arg": 20559, "Input": 20560, "\u0120Strategy": 20561, "\u0120palace": 20562, "thening": 20563, "Dri": 20564, "\u0120sentencing": 20565, "umbnail": 20566, "\u0120pins": 20567, "recy": 20568, "\u0120siblings": 20569, "Getting": 20570, "\u0120BU": 20571, "\u0120Northwest": 20572, "\u0120prolonged": 20573, "\u0120Sakura": 20574, "Comb": 20575, "\u0120Bour": 20576, "\u0120inadequate": 20577, "\u0120Kash": 20578, "\u0120username": 20579, "\u0120Improve": 20580, "\u0120battling": 20581, "\u0120MAC": 20582, "\u0120curriculum": 20583, "\u0120soda": 20584, "\u0120Cannon": 20585, "\u0120sensible": 20586, "spons": 20587, "December": 20588, "\u0120wicked": 20589, "\u0120Pengu": 20590, "\u0120dictators": 20591, "\u0120Hearts": 20592, "ogyn": 20593, "\u0120similarities": 20594, "\u0120Stats": 20595, "\u0120hollow": 20596, "itations": 20597, "\":[": 20598, "\u0120hover": 20599, "\u0120Listen": 20600, "sch": 20601, "Sund": 20602, "\u0120cad": 20603, "\u0120Parks": 20604, "\u0120lur": 20605, "\u0120hype": 20606, "\u0120Lem": 20607, "NAME": 20608, "isure": 20609, "Friday": 20610, "\u0120shoots": 20611, "\u0120closes": 20612, "\u0120db": 20613, "\u0120Ridge": 20614, "\u0120Different": 20615, "\u0120replies": 20616, "\u0120Broadway": 20617, "opers": 20618, "\u0120intoler": 20619, "\u0120Zeus": 20620, "akespe": 20621, "\u0120proprietary": 20622, "\u0120requesting": 20623, "\u0120controllers": 20624, "\u0120MIN": 20625, "imedia": 20626, "becca": 20627, "\u0120expans": 20628, "\u0120oils": 20629, "Bot": 20630, "\u0120Chand": 20631, "\u0120printer": 20632, "\u0120topped": 20633, "\u0120POL": 20634, "\u0120Earlier": 20635, "Social": 20636, "avin": 20637, "\u0120decreases": 20638, "\u0120Seb": 20639, "\u0120specifications": 20640, "\u0120Blast": 20641, "\u0120Kurt": 20642, "\u0120freel": 20643, "Brown": 20644, "\u0120dilig": 20645, "roe": 20646, "\u0120Problem": 20647, "\u0120Quad": 20648, "\u0120decentral": 20649, "\u0120Vector": 20650, "anut": 20651, "\u0120plugins": 20652, "\u0120Gregory": 20653, "\u0120fucked": 20654, "elines": 20655, "\u0120Ambassador": 20656, "take": 20657, "\u0120cleans": 20658, "ongyang": 20659, "Anonymous": 20660, "stro": 20661, "\"}": 20662, "aline": 20663, "\u0120Odd": 20664, "\u0120Eug": 20665, "216": 20666, "\u0120boil": 20667, "\u0120Powers": 20668, "\u0120nurses": 20669, "Obviously": 20670, "\u0120Technical": 20671, "\u0120exceeded": 20672, "ORS": 20673, "\u0120extremists": 20674, "\u0120traces": 20675, "expl": 20676, "\u0120comr": 20677, "\u0120Sach": 20678, ")/": 20679, "\u0120masks": 20680, "\u0120sci": 20681, "Bon": 20682, "\u0120regression": 20683, "wegian": 20684, "\u0120advisor": 20685, "itures": 20686, "\u0120Vo": 20687, "example": 20688, "\u0120Instruct": 20689, "\u0120siege": 20690, "\u0120reductions": 20691, "ptr": 20692, "\u0120statutory": 20693, "\u0120removes": 20694, "\u0120puck": 20695, "redits": 20696, "\u0120bee": 20697, "\u0120salad": 20698, "\u0120promotions": 20699, "\u0120Joshua": 20700, "withstanding": 20701, "ETH": 20702, "\u0120Cha": 20703, "imus": 20704, "\u0120expenditure": 20705, "aunting": 20706, "\u0120delighted": 20707, "\u0120155": 20708, "beh": 20709, "\u0120carpet": 20710, "\u0120Spart": 20711, "\u0120jungle": 20712, "lists": 20713, "\u0120bullying": 20714, "\u0120Nobel": 20715, "\u0120Glen": 20716, "\u0120referenced": 20717, "\u0120introduces": 20718, "sein": 20719, "\u0120chopped": 20720, "glass": 20721, "\u0120Wrest": 20722, "\u0120neutrality": 20723, "\u0120\u00e2\u013b": 20724, "\u0120investigator": 20725, "\u0120shelves": 20726, "\u0120unconstitutional": 20727, "\u0120reproduction": 20728, "\u0120merchant": 20729, "mia": 20730, "\u0120metrics": 20731, "\u0120explosives": 20732, "\u0120Sonia": 20733, "\u0120bodily": 20734, "\u0120thickness": 20735, "\u0120predominantly": 20736, "\u0120Ability": 20737, "\u0120monitored": 20738, "ICH": 20739, "\u0120].": 20740, "\u0120Martinez": 20741, "\u0120visibility": 20742, "\u0120queries": 20743, "\u0120genocide": 20744, "\u0120Warfare": 20745, "Query": 20746, "\u0120studios": 20747, "\u0120embry": 20748, "\u0120corridor": 20749, "\u0120cleaned": 20750, "complete": 20751, "\u0120MH": 20752, "\u0120enrollment": 20753, "INGS": 20754, "\u0120impacted": 20755, "\u0120disastrous": 20756, "\u0120Yun": 20757, "\u0120Claire": 20758, "\u0120Basically": 20759, "yt": 20760, "usterity": 20761, "\u0120indirectly": 20762, "wik": 20763, "\u0120dod": 20764, "\u0120Carr": 20765, "\u0120amp": 20766, "\u0120prohibit": 20767, "\u0120Initial": 20768, "\u0120Rd": 20769, "iji": 20770, "\u0120educate": 20771, "corn": 20772, "iott": 20773, "\u0120Beauty": 20774, "\u0120detective": 20775, "\u0120Conn": 20776, "since": 20777, "\u0120stagger": 20778, "\u0120obese": 20779, "\u0120bree": 20780, "ologic": 20781, "isse": 20782, "walker": 20783, "\u0120blades": 20784, "\u0120lawful": 20785, "func": 20786, "\u0120Behind": 20787, "\u0120appetite": 20788, "\u0120(*": 20789, "\u0120tennis": 20790, "\u0120offspring": 20791, "\u0120jets": 20792, "\u0120structured": 20793, "\u0120aforementioned": 20794, "Nov": 20795, "\u0120scaling": 20796, "fill": 20797, "\u0120stew": 20798, "\u0120curb": 20799, "\u0120Stephan": 20800, "edIn": 20801, "SF": 20802, "obic": 20803, "\u00e9\u0143\u0136": 20804, "oug": 20805, "\u0120MM": 20806, "\u0120genetically": 20807, "opez": 20808, "136": 20809, "\u0120umb": 20810, "ancers": 20811, "\u0120cohort": 20812, "\u0120merchandise": 20813, "\u0120imposing": 20814, "\u0120Legislature": 20815, "\u0120Archive": 20816, "ivia": 20817, "\u0120Naval": 20818, "\u0120offences": 20819, "\u0120miracle": 20820, "\u0120snapped": 20821, "\u0120foes": 20822, "\u0120extensively": 20823, "\u0120Raf": 20824, "\u0120cater": 20825, "edience": 20826, "Kit": 20827, "\u0120Bin": 20828, "\u0120recommends": 20829, "\u0120Cities": 20830, "\u0120rigid": 20831, "\u0120READ": 20832, "\u0120Noble": 20833, "\u0120Tian": 20834, "\u0120certificates": 20835, "antis": 20836, "oiler": 20837, "\u0120Buddhist": 20838, "did": 20839, "\u0120surveyed": 20840, "\u0120downward": 20841, "\u0120prints": 20842, "\u0120Motion": 20843, "ronics": 20844, "\u0120Sans": 20845, "ossibly": 20846, "uctions": 20847, "\u0120colonies": 20848, "\u0120Danish": 20849, "unit": 20850, "\u0120spoil": 20851, "\u0120advisory": 20852, "berries": 20853, "Plan": 20854, "\u0120specification": 20855, "ophers": 20856, "\u0120Resource": 20857, "\u0120shirts": 20858, "prisingly": 20859, "communications": 20860, "\u0120trivial": 20861, "\u0120mentioning": 20862, "isexual": 20863, "\u0120supplements": 20864, "\u0120supervision": 20865, "BP": 20866, "vor": 20867, "\u0120wit": 20868, "\u0120cooldown": 20869, "\u0120plaintiff": 20870, "\u0120Reviews": 20871, "\u0120Sri": 20872, "\u0120Mint": 20873, "\u0120Sugar": 20874, "\u0120afterward": 20875, "\u0120Priest": 20876, "\u0120Investment": 20877, "ogene": 20878, "\u0120Taking": 20879, "\u0120stretching": 20880, "\u0120inflammation": 20881, "\u0120Tehran": 20882, "\u0120lining": 20883, "\u0120freezing": 20884, "\u0120Entity": 20885, "\u0120inspiring": 20886, "special": 20887, "price": 20888, "\u0120sue": 20889, "\u0120Porter": 20890, "ounge": 20891, "ETA": 20892, "\u0120Derek": 20893, "\u0120Luis": 20894, "uo": 20895, "ymph": 20896, "\u0120exterior": 20897, "ihil": 20898, "\u0120Ashley": 20899, "inator": 20900, "\u0120nutrients": 20901, "\u0120Thrones": 20902, "\u0120finances": 20903, "\u0120Inspect": 20904, "\u0120specially": 20905, "\u0120Required": 20906, "\u0120PTS": 20907, "\u0120Violence": 20908, "ointed": 20909, "shots": 20910, "\u0120excerpt": 20911, "coon": 20912, "INS": 20913, "\u0120Gri": 20914, "\u0120recognised": 20915, "Week": 20916, "Young": 20917, "\u0120vom": 20918, "isle": 20919, "\u0120Curry": 20920, "\u0120Buddh": 20921, "\u0120notebook": 20922, "\u0120durable": 20923, "/?": 20924, "\u0120Gad": 20925, "\u0120Pupp": 20926, "\u0120forgive": 20927, "park": 20928, "\u0120personalities": 20929, "analysis": 20930, "clamation": 20931, "\u0120elevator": 20932, "\u0120warehouse": 20933, "\u0120Role": 20934, "unn": 20935, "\u0120illustration": 20936, "\u0120Scan": 20937, "\u0120atmospheric": 20938, "Import": 20939, "ANC": 20940, "ricted": 20941, "fu": 20942, "010": 20943, "\u0120arche": 20944, "\u0120rewarded": 20945, "akespeare": 20946, "\u0120internally": 20947, "\u0120RBI": 20948, "alker": 20949, "\u0120elephant": 20950, "owitz": 20951, "\u0120Pizza": 20952, "\u0120bipartisan": 20953, "\u00c3\u00a9s": 20954, "\u0120slowed": 20955, "\u0120Stark": 20956, "\u0120override": 20957, "OUS": 20958, "\u0120320": 20959, "undreds": 20960, "\u0120Deck": 20961, "\u0120Census": 20962, "bee": 20963, "146": 20964, "otor": 20965, "\u0120ip": 20966, "\u0120ub": 20967, "ocations": 20968, "\u0120Button": 20969, "rice": 20970, "\u0120cripp": 20971, "fff": 20972, "\u0120originated": 20973, "\u0120overwhelmed": 20974, "appa": 20975, "\u0120foremost": 20976, "\u00e2\u0122\u0133": 20977, "\u0120LEG": 20978, "release": 20979, "eatured": 20980, "atches": 20981, "\u0120reps": 20982, "\u0120lending": 20983, "\u0120Reference": 20984, "\u0120Client": 20985, "165": 20986, "venth": 20987, "Complete": 20988, "\u0120Patrol": 20989, "\u0120sworn": 20990, "cam": 20991, "\u0120shuttle": 20992, "\u0120Ralph": 20993, "\u0120hometown": 20994, "-,": 20995, "onal": 20996, "\u0120BP": 20997, "\u00e5\u0131": 20998, "\u0120persuade": 20999, "\u0120Alexand": 21000, "\u0120combines": 21001, "\u0120vivid": 21002, "\u0120Lag": 21003, "\u0120encoding": 21004, "\u0120salvation": 21005, "wen": 21006, "\u0120Recovery": 21007, "iya": 21008, "University": 21009, "\u0120Biden": 21010, "\u0120budgets": 21011, "\u0120Texans": 21012, "fits": 21013, "\u0120honored": 21014, "\u0120python": 21015, "TD": 21016, "###": 21017, "clone": 21018, "\u0120blink": 21019, "\u0120Liquid": 21020, "\u0120unemployed": 21021, "\u0120clashes": 21022, "\u0120Counsel": 21023, "\u0120directing": 21024, "\u0120punct": 21025, "\u0120Falcons": 21026, "\u0120shark": 21027, "\u0120Damascus": 21028, "\u0120jeans": 21029, "\u0120embark": 21030, "\u0120seize": 21031, "\u0120upwards": 21032, "280": 21033, "\u0120Ez": 21034, "\u0120Anything": 21035, "\u0120exotic": 21036, "lower": 21037, "\u0120Creator": 21038, "\u0120Um": 21039, "\u0120suburbs": 21040, "berger": 21041, "\u0120Wend": 21042, "\u0120mint": 21043, "\u0120XX": 21044, "\u0120Dro": 21045, "\u0120suffers": 21046, "\u0120herb": 21047, "tree": 21048, "\u0120fragile": 21049, "\u0120flooded": 21050, "\u0120Alcohol": 21051, "olean": 21052, "nyder": 21053, "\u0120KO": 21054, "Fram": 21055, "\u0120136": 21056, "\u0120owed": 21057, "\u0120Melee": 21058, "\u0120Hash": 21059, "\u0120whisk": 21060, "\u0120sudo": 21061, "rr": 21062, "Quick": 21063, "appro": 21064, "\u0120ii": 21065, "\u0120Examples": 21066, "hee": 21067, "\u0120promotes": 21068, "perature": 21069, "kar": 21070, "\u0120Honor": 21071, "\u0120sodium": 21072, "\u0120Lif": 21073, "rosso": 21074, "intendent": 21075, "\u0120correspondent": 21076, "Found": 21077, "secret": 21078, "\u0120identifies": 21079, "agne": 21080, "\u0120lou": 21081, "\u0120PP": 21082, "\u0120coincidence": 21083, "move": 21084, "\u0120militia": 21085, "\u0120infiltr": 21086, "\u0120Primary": 21087, "\u0120pitching": 21088, "\u0120Ib": 21089, "\u0120GOOD": 21090, "\u00e3\u0124\u00b8": 21091, "\u0120Wizards": 21092, "iral": 21093, "\u0120Venus": 21094, "RR": 21095, "\u0120\u00e2\u0122\u0137": 21096, "\u0120Casey": 21097, "\u0120sadly": 21098, "\u0120admire": 21099, "\u0120embarrassed": 21100, "cb": 21101, "Mel": 21102, "\u0120tubes": 21103, "\u0120beautifully": 21104, "\u0120Queensland": 21105, "Below": 21106, "rez": 21107, "quet": 21108, "pleasant": 21109, "\u0120\u00c2\u00ab": 21110, "Camp": 21111, "\u0120decisive": 21112, "1998": 21113, "\u0120Lamb": 21114, "utton": 21115, "hn": 21116, "\u0120Jagu": 21117, "aunder": 21118, "\u0120Cord": 21119, "\u0120clerk": 21120, "\u0120caffe": 21121, "\u0120wiped": 21122, "\u0120reim": 21123, "\u0120Mountains": 21124, "\u0120imprisoned": 21125, "\u0120develops": 21126, "\u0120Pra": 21127, "\u0120modeling": 21128, "Anyone": 21129, "ancel": 21130, "\u0120Sit": 21131, "\u0120shields": 21132, "\u0120lawn": 21133, "\u0120cardiovascular": 21134, "\u0120demonstrating": 21135, "\u0120parse": 21136, "\u0120Israelis": 21137, "\u0120euros": 21138, "143": 21139, "\u0120glorious": 21140, "inski": 21141, "ecd": 21142, "\u0120conditioning": 21143, "\u0120helpless": 21144, "\u0120microsc": 21145, "\u0120Harbor": 21146, "\u0120stakes": 21147, "\u0120260": 21148, "\u0120unequ": 21149, "\u0120Floyd": 21150, "\u0120damp": 21151, "\u0120apparatus": 21152, "\u0120Laws": 21153, "\u0120counters": 21154, "\u0120induce": 21155, "atable": 21156, "\u0120Ahmed": 21157, "\u0120slam": 21158, "November": 21159, "\u0120persist": 21160, "\u0120imminent": 21161, "\u00c3\u00a1n": 21162, "\u0120shred": 21163, "\u0120phases": 21164, "\u0120Edmonton": 21165, "\u0120Armstrong": 21166, "\u0120Meet": 21167, "\u0120Kitty": 21168, "\u00d1\u0122": 21169, "circ": 21170, "\u0120Adult": 21171, "\u0120arose": 21172, "\u0120Xen": 21173, "Dan": 21174, "gow": 21175, "\u0120superf": 21176, "\u0120Admir": 21177, "\u0120endure": 21178, "\u0120keyword": 21179, "yrus": 21180, "\u0120yarn": 21181, "\u0120pathway": 21182, "\u0120Hopkins": 21183, "midt": 21184, "\u0120censorship": 21185, "dependent": 21186, "\u0120instructor": 21187, "Sources": 21188, "\u0120toe": 21189, "\u0120balloon": 21190, "Nob": 21191, "\u0120swear": 21192, "\u0120Castro": 21193, "\u0120gloss": 21194, "\u0120Kavanaugh": 21195, "\u0120remarkably": 21196, "Photos": 21197, "\u0120Nom": 21198, "\u0120Southeast": 21199, "yers": 21200, "\u0120validation": 21201, "\u0120cannon": 21202, "\u0120Victory": 21203, "\u0120Pierre": 21204, "\u0120cautious": 21205, "Audio": 21206, "\u0120fetch": 21207, "\u0120Gift": 21208, "\u0120Hyp": 21209, "\u0120remedy": 21210, "ZE": 21211, "\u0120scent": 21212, "\u0120beard": 21213, "\u0120Rut": 21214, "-\"": 21215, "\u0120patents": 21216, "Hy": 21217, "\u0120unjust": 21218, "\u0120potato": 21219, "\u0120forthcoming": 21220, "\u0120chef": 21221, "\u0120Rift": 21222, "affe": 21223, "\u0120ROM": 21224, "\u0120Launch": 21225, "\u0120pads": 21226, "\u0120Neo": 21227, "\u0120onset": 21228, "\u0120squeeze": 21229, "safe": 21230, "\u0120prefix": 21231, "\u0120TM": 21232, "\u0120Nearly": 21233, "\u0120Clinical": 21234, "\u0120Mental": 21235, "otiation": 21236, "\u0120Unic": 21237, "antry": 21238, "\u0120Cir": 21239, "\u0120epit": 21240, "\u00c3\u00a6": 21241, "\u0120extracted": 21242, "versely": 21243, "riad": 21244, "\u0120strains": 21245, "\u0120tops": 21246, "\u0120poem": 21247, "\u0120Randy": 21248, "\u0120Maple": 21249, "THER": 21250, "upiter": 21251, "\u0120SSD": 21252, "\u013c\u00e9": 21253, "\u0120uncon": 21254, "pering": 21255, "\u0120slept": 21256, "iners": 21257, "\u0120underwater": 21258, "\u0120Evidence": 21259, "gone": 21260, "205": 21261, "\u0120historians": 21262, "\u0120synthesis": 21263, "\u0120frog": 21264, "basketball": 21265, "\u0120vibrant": 21266, "\u0120subord": 21267, "\u0120365": 21268, "\u0120Dial": 21269, "\u0120cooperate": 21270, "HAHA": 21271, "\u0120greeted": 21272, "158": 21273, "\u0120jazz": 21274, "\u0120intox": 21275, "\u0120Walking": 21276, "\u0120supervisor": 21277, "\u0120Fusion": 21278, "\u0120Mercedes": 21279, "send": 21280, "Ham": 21281, "sd": 21282, "nl": 21283, "\u0120tours": 21284, "\u0120FIFA": 21285, "\u0120culp": 21286, "gd": 21287, "304": 21288, "\u0120pleas": 21289, "\u0120illustrates": 21290, "\u0120Colombia": 21291, "\u0120highlighting": 21292, "\u0120Summary": 21293, "\u0120exposing": 21294, "\u0120Dru": 21295, "\u0120irony": 21296, "ritional": 21297, "\u0120Carroll": 21298, "\u0120Ellis": 21299, "Pict": 21300, "\u0120Rapt": 21301, "\u0120adapter": 21302, "\u0120unm": 21303, "\u0120corpse": 21304, "\u0120celebrities": 21305, "Den": 21306, "atum": 21307, "\u0120Apocalypse": 21308, "\u0120Wag": 21309, "lining": 21310, "\u0120hormones": 21311, "Rub": 21312, "\u0120Xi": 21313, "\u0120Vaults": 21314, "208": 21315, "alkyrie": 21316, "inosaur": 21317, "\u0120feeds": 21318, "vity": 21319, "\u0120defeating": 21320, "Wait": 21321, "\u0120emphasize": 21322, "\u0120Steelers": 21323, "yrinth": 21324, "leys": 21325, "\u0120Whenever": 21326, "Currently": 21327, "\u0120Clock": 21328, "\u0120collectively": 21329, "anyon": 21330, "\u0120JP": 21331, "\u0120mentality": 21332, "\u0120downloads": 21333, "\u0120surroundings": 21334, "\u0120Barnes": 21335, "\u0120flagship": 21336, "\u0120indicators": 21337, "\u0120grapp": 21338, "January": 21339, "\u0120Elemental": 21340, "\u0120Athena": 21341, "ibal": 21342, "\u0120sights": 21343, "\u0120capita": 21344, "\u0120Treaty": 21345, "\u0120voiced": 21346, "\u0120Gaz": 21347, "lette": 21348, "\u0120ya": 21349, "\u0120expired": 21350, "Legend": 21351, "Hot": 21352, "nature": 21353, "\u0120unstable": 21354, "\u0120280": 21355, "\u00c3\u00ba": 21356, "Comment": 21357, "ALE": 21358, "\u0120quests": 21359, "\u0120handler": 21360, "nis": 21361, "\u0120versatile": 21362, "\u0120conceal": 21363, "engeance": 21364, "\u0120Interactive": 21365, "\u0120obsessed": 21366, "\u0120Dogs": 21367, "\u0120cracked": 21368, "Sound": 21369, "sv": 21370, "\u0120Dylan": 21371, "roads": 21372, "fx": 21373, "\u0120Catholics": 21374, "\u0120Hag": 21375, "\u0120slammed": 21376, "\u0120glowing": 21377, "sale": 21378, "\u0120tissues": 21379, "\u0120Chi": 21380, "nee": 21381, "\u0120cher": 21382, "sic": 21383, "urrection": 21384, "\u0120bacon": 21385, "ulatory": 21386, ").\"": 21387, "\u0120irregular": 21388, "FORM": 21389, "assed": 21390, "\u0120intentional": 21391, "\u0120compensate": 21392, "\u0120Speaking": 21393, "\u0120Sets": 21394, "153": 21395, "\u0120conventions": 21396, "bands": 21397, "emade": 21398, "\u0120ecc": 21399, "\u0120Winston": 21400, "\u0120Assassin": 21401, "\u0120Belgian": 21402, "\u0120dependence": 21403, "\u0120niche": 21404, "\u0120bark": 21405, "\u0120Jazz": 21406, "\u0120disadvantage": 21407, "\u0120gasoline": 21408, "\u0120165": 21409, "\u00e7\u013c\u0126": 21410, "essa": 21411, "module": 21412, "angular": 21413, "OY": 21414, "\u0120Treatment": 21415, "itas": 21416, "olation": 21417, "\u0120Arnold": 21418, "\u0120feud": 21419, "\u0120Nest": 21420, "\u0120theatre": 21421, "ewater": 21422, "\u0120minors": 21423, "olicy": 21424, "\u0120Haven": 21425, "division": 21426, "\u0120trunk": 21427, "Far": 21428, "\u0120Pull": 21429, "\u0120capturing": 21430, "\u01201800": 21431, "\u0120Teen": 21432, "\u0120exempl": 21433, "\u0120clinics": 21434, "\u0120Burg": 21435, "\u0120substit": 21436, "\u0120payload": 21437, "\u0120Lav": 21438, "\u0120Troy": 21439, "\u0120Witness": 21440, "\u0120fragments": 21441, "\u0120passwords": 21442, "\u0120gospel": 21443, "\u0120Gin": 21444, "\u0120tenants": 21445, "olith": 21446, "Six": 21447, "Previous": 21448, "\u0120Ages": 21449, "\u0120Darwin": 21450, "\u0120blat": 21451, "\u0120empathy": 21452, "smith": 21453, "bag": 21454, "\u0120Echo": 21455, "\u0120Camb": 21456, "\u0120Madd": 21457, "\u0120Boo": 21458, "\u0120rede": 21459, "\u0120Burning": 21460, "\u0120smoothly": 21461, "\u0120Adrian": 21462, "\u0120Vampire": 21463, "\u0120Monsters": 21464, "steam": 21465, "Style": 21466, "Ma": 21467, "rea": 21468, "\u0120Dwar": 21469, "alyst": 21470, "ursor": 21471, "\u0120elimination": 21472, "\u0120crypto": 21473, "cht": 21474, "\u0120Eternal": 21475, "\u00e2\u0122\u00a6]": 21476, "\u0120Sorce": 21477, "Ill": 21478, "NER": 21479, "\u0120uh": 21480, "Conclusion": 21481, "wage": 21482, "\u0120respir": 21483, "\u0120reminis": 21484, "hetical": 21485, "\u0120gy": 21486, "\u0120utilized": 21487, "icidal": 21488, "\u01201900": 21489, "\u0120hunters": 21490, "\u0120Swan": 21491, "\u0120React": 21492, "\u0120visitor": 21493, "\u0120Thanksgiving": 21494, "308": 21495, "Posts": 21496, "\u0120hips": 21497, "1997": 21498, "omers": 21499, "\u0120knocking": 21500, "\u0120Vehicle": 21501, "\u0120til": 21502, "\u0120138": 21503, "\u0120mi": 21504, "\u0120Investigation": 21505, "\u0120Kenya": 21506, "\u0120casino": 21507, "\u0120motives": 21508, "\u0120regain": 21509, "rex": 21510, "\u0120weekends": 21511, "\u0120stabbed": 21512, "boro": 21513, "\u0120exploited": 21514, "\u0120HAVE": 21515, "\u0120Television": 21516, "cock": 21517, "\u0120preparations": 21518, "\u0120endeav": 21519, "\u0120Remote": 21520, "\u0120Maker": 21521, "\u0120Produ": 21522, "\u0120Evan": 21523, "\u0120informational": 21524, "\u0120Louisville": 21525, "154": 21526, "\u0120Dreams": 21527, "\u0120plots": 21528, "\u0120Runner": 21529, "\u0120hurting": 21530, "\u0120academy": 21531, "\u0120Montgomery": 21532, "nm": 21533, "\u0120Lanc": 21534, "\u0120Alz": 21535, "210": 21536, "elong": 21537, "\u0120retailer": 21538, "\u0120arising": 21539, "\u0120rebellion": 21540, "\u0120blonde": 21541, "played": 21542, "\u0120instrumental": 21543, "Cross": 21544, "\u0120retention": 21545, "\u0120therapeutic": 21546, "\u0120seas": 21547, "\u0120infantry": 21548, "\u0120Clint": 21549, "\u0120prompting": 21550, "\u0120bitch": 21551, "\u0120stems": 21552, "\u0120Kra": 21553, "\u0120thesis": 21554, "\u0120Bog": 21555, "rued": 21556, "\u0120kings": 21557, "\u0120clay": 21558, "ificent": 21559, "\u0120YES": 21560, "\u0120Thing": 21561, "\u0120Cubs": 21562, "veyard": 21563, "elsh": 21564, "inarily": 21565, "\u0120Ey": 21566, "\u0120Rolling": 21567, "\u0120evolving": 21568, "India": 21569, "\u0120recognizes": 21570, "\u0120graduation": 21571, "isers": 21572, "\u0120fertility": 21573, "\u0120Milan": 21574, "Command": 21575, "\u0120boxing": 21576, "\u01201943": 21577, "\u0120gluten": 21578, "\u0120Emir": 21579, "\u0120idol": 21580, "\u0120conceived": 21581, "\u0120Creation": 21582, "Merit": 21583, "uddy": 21584, "ussions": 21585, "\u0120Lieutenant": 21586, "ietal": 21587, "\u0120unchanged": 21588, "\u0120Scale": 21589, "\u0120Crimea": 21590, "balls": 21591, "atorial": 21592, "\u0120depths": 21593, "\u0120empirical": 21594, "\u0120transm": 21595, "\u0120unsafe": 21596, "missible": 21597, "comfort": 21598, "156": 21599, "\u0120mechanic": 21600, "002": 21601, "lins": 21602, "\u0120smoked": 21603, "Pos": 21604, "\u0120slowing": 21605, "\u0120lav": 21606, "Texas": 21607, "\u0120cheating": 21608, "\u0120Metropolitan": 21609, "ethyl": 21610, "\u0120discovering": 21611, "asse": 21612, "\u0120pencil": 21613, "\u0120Pyongyang": 21614, "\u0120closet": 21615, "\u0120Sheet": 21616, "\u0120Entry": 21617, "oustic": 21618, "\u0120myst": 21619, "erate": 21620, "ariat": 21621, "\u0120minerals": 21622, "\u0120musician": 21623, "\u0120Pul": 21624, "\u0120Maz": 21625, "249": 21626, "\u0120permissions": 21627, "\u0120iv": 21628, "enary": 21629, "ickers": 21630, "\u0120Bing": 21631, "hea": 21632, "enable": 21633, "\u0120griev": 21634, "\u0120asserted": 21635, "\u0120Colonel": 21636, "\u0120affidav": 21637, "wo": 21638, "\u0120seated": 21639, "\u0120Ride": 21640, "\u0120paintings": 21641, "\u0120Pix": 21642, "\u0120137": 21643, "ishi": 21644, "umbai": 21645, "gotten": 21646, "\u0120Earl": 21647, "\u0120inning": 21648, "\u0120census": 21649, "\u0120travelled": 21650, "\u0120Consult": 21651, "185": 21652, "bind": 21653, "\u0120simplicity": 21654, "\u0120overlooked": 21655, "\u0120Helpful": 21656, "\u0120monkey": 21657, "\u0120overwhelmingly": 21658, "Blood": 21659, "\u0120Flint": 21660, "\u0120Jama": 21661, "\u0120Present": 21662, "\u0120Rage": 21663, "\u0120TA": 21664, "ptive": 21665, "\u0120turnout": 21666, "wald": 21667, "\u0120Dolphins": 21668, "\u0120VPN": 21669, "\u0120onion": 21670, "\u0120crafting": 21671, "mma": 21672, "\u0120Mercury": 21673, "\u0120arrange": 21674, "\u0120alerts": 21675, "\u0120OT": 21676, "zbollah": 21677, "\u0120gases": 21678, "\u0120Richardson": 21679, "sal": 21680, "lar": 21681, "\u0120frost": 21682, "\u0120lowering": 21683, "\u0120acclaim": 21684, "\u0120startups": 21685, "\u0120Gain": 21686, "essment": 21687, "\u0120guardian": 21688, "\u00e4\u00ba\u00ba": 21689, "\u0120Pie": 21690, "\u0120Links": 21691, "\u0120merits": 21692, "\u0120awake": 21693, "\u0120parental": 21694, "\u0120exceeds": 21695, "\u0120idle": 21696, "\u0120Pilot": 21697, "\u0120eBay": 21698, "\u0120Accept": 21699, "ipeg": 21700, "Cam": 21701, "\u0120Kot": 21702, "\u0120traders": 21703, "olitics": 21704, "unker": 21705, "\u0120Pale": 21706, "osi": 21707, "anmar": 21708, "\u01201947": 21709, "\u0120Fell": 21710, "estial": 21711, "itating": 21712, "GF": 21713, "\u0120Sr": 21714, "ifted": 21715, "\u0120connector": 21716, "\u0120Bone": 21717, "illes": 21718, "260": 21719, "hma": 21720, "\u0120overlap": 21721, "\u0120GitHub": 21722, "\u0120cleaner": 21723, "\u0120Baptist": 21724, "\u0120WAS": 21725, "\u0120lungs": 21726, "\u00d1\u0123": 21727, "\u0120BUT": 21728, "\u0120cite": 21729, "\u0120pitched": 21730, "reatment": 21731, "\u0120trophies": 21732, "\u0120Nu": 21733, "386": 21734, "\u0120Pride": 21735, "\u0120attendees": 21736, "[]": 21737, "179": 21738, "\u0120spatial": 21739, "\u0120prizes": 21740, "\u0120Religion": 21741, "\u0120showcase": 21742, "\u0120Category": 21743, "vidia": 21744, "Target": 21745, "Property": 21746, "?,": 21747, "\u0120fusion": 21748, "pie": 21749, "\u0120UCLA": 21750, "\u0120soundtrack": 21751, "\u0120princess": 21752, "\u0120Caval": 21753, "should": 21754, "\u0120limbs": 21755, "Background": 21756, "\u0120lonely": 21757, "\u0120cores": 21758, "\u0120Tail": 21759, "sheet": 21760, "\u0120132": 21761, "Ra": 21762, "\u00e3\u0124\u00ab": 21763, "\u0120Bolt": 21764, "\u0120booked": 21765, "\u0120administer": 21766, "\u0120equals": 21767, "wy": 21768, "\u0120observing": 21769, "\u0120Baron": 21770, "\u0120Adobe": 21771, "\u0120virgin": 21772, "\u0120Socialist": 21773, "Move": 21774, "ghazi": 21775, "\u0120Linda": 21776, "212": 21777, "\u0120brewing": 21778, "\u0120merchants": 21779, "burse": 21780, "\u0120divor": 21781, "\u0120metals": 21782, "\u0120Ner": 21783, "\u0120sums": 21784, "\u0120Enemy": 21785, "\u0120envision": 21786, "\u0120granting": 21787, "\u0120Honey": 21788, "\u0120Skyrim": 21789, "\u0120socio": 21790, "graded": 21791, "\u0120selective": 21792, "WASHINGTON": 21793, "\u01201948": 21794, "\u0120Sirius": 21795, "\u0120Gross": 21796, "activity": 21797, "\u0120Ivan": 21798, "\u0120furious": 21799, "BSD": 21800, "\u0120Previous": 21801, "\u0120responsive": 21802, "\u0120charitable": 21803, "\u0120leaning": 21804, "\u0120Pew": 21805, "\u0120violates": 21806, "\\\\\\\\\\\\\\\\": 21807, "\u0120Coming": 21808, "wire": 21809, "\u0120poet": 21810, "\u0120resolutions": 21811, "command": 21812, "\u0120Portuguese": 21813, "\u0120nickname": 21814, "\u0120deaf": 21815, "February": 21816, "\u0120recognise": 21817, "\u0120entirety": 21818, "\u0120seasonal": 21819, "placed": 21820, "\u0120Telegraph": 21821, "\u0120microphone": 21822, "ouring": 21823, "\u0120grains": 21824, "\u0120governed": 21825, "\u0120postp": 21826, "\u0120Waters": 21827, "inement": 21828, "\u0120undocumented": 21829, "\u0120Comcast": 21830, "\u0120fox": 21831, "\u0120assaults": 21832, "reon": 21833, "many": 21834, "\u0120Jenkins": 21835, "\u0120Anyway": 21836, "\u0120assessments": 21837, "\u0120downs": 21838, "\u0120Mouse": 21839, "\u0120superb": 21840, "kt": 21841, "\u0120Dow": 21842, "\u0120taxation": 21843, "401": 21844, "\u0120smiles": 21845, "\u0120undertaken": 21846, "\u0120exh": 21847, "\u0120enthusiastic": 21848, "\u0120twent": 21849, "\u0120governmental": 21850, "\u0120autonomy": 21851, "\u0120Technologies": 21852, "\u0120Chain": 21853, "\u0120prevalent": 21854, "fb": 21855, "\u0120nicotine": 21856, "ogram": 21857, "job": 21858, "\u0120awaiting": 21859, "\u0120Menu": 21860, "\u0120deputies": 21861, "kov": 21862, "ishops": 21863, "Button": 21864, "\u0120Shanghai": 21865, "\u0120diesel": 21866, "\u0120Duck": 21867, "Ryan": 21868, "\u0120PCs": 21869, "NF": 21870, "jury": 21871, "ente": 21872, "\u0120inaccurate": 21873, "eddy": 21874, "Whatever": 21875, "\u0120showc": 21876, "\u0120Nad": 21877, "odus": 21878, "etr": 21879, "\u0120plaintiffs": 21880, "\u0120WOR": 21881, "\u0120Assange": 21882, "\u0120privat": 21883, "\u0120premiums": 21884, "\u0120tam": 21885, "URL": 21886, "\u0120elites": 21887, "\u0120Ranger": 21888, "ottenham": 21889, "\u0120Hoff": 21890, "\u0120Athens": 21891, "\u0120definite": 21892, "\u0120sighed": 21893, "\u0120evenly": 21894, "211": 21895, "\u0120Amber": 21896, "akia": 21897, "\u0120mailing": 21898, "\u0120crashing": 21899, "\u0120Confederate": 21900, "rugged": 21901, "Wal": 21902, "\u0120Depths": 21903, "\u0120juvenile": 21904, "\u0120reactor": 21905, "Introduction": 21906, "\u0120Deluxe": 21907, "1995": 21908, "\u0120Sanchez": 21909, "\u0120Mead": 21910, "ivable": 21911, ":-": 21912, "\u0120Planning": 21913, "\u0120Trap": 21914, "quin": 21915, "\u0120Protect": 21916, "vered": 21917, "Information": 21918, "\u0120kidney": 21919, "innamon": 21920, "las": 21921, "\u0120policing": 21922, "\u0120tolerate": 21923, "\u0120Qi": 21924, "\u0120biased": 21925, "Fort": 21926, "\u0120Ki": 21927, "save": 21928, "\u0120privileged": 21929, "\u0120beasts": 21930, "\u0120Glas": 21931, "\u0120Cinem": 21932, "\u0120comeback": 21933, "Sunday": 21934, "\u0120extinction": 21935, "hops": 21936, "\u0120transmit": 21937, "\u0120doubles": 21938, "\u0120Flat": 21939, "167": 21940, "\u0120disputed": 21941, "\u0120injustice": 21942, "foo": 21943, "Vict": 21944, "roleum": 21945, "\u0120Julie": 21946, "Context": 21947, "\u0120Rarity": 21948, "issue": 21949, "Component": 21950, "\u0120counseling": 21951, "anne": 21952, "dark": 21953, "\u0120objections": 21954, "uilt": 21955, "\u0120gast": 21956, "\u0120plac": 21957, "\u0120unused": 21958, "\u00e3\u0125\u0129": 21959, "\u0120Trial": 21960, "\u0120Jas": 21961, "hedral": 21962, "obb": 21963, "\u0120temporal": 21964, "\u0120PRO": 21965, "\u0120NW": 21966, "\u0120Anniversary": 21967, "Large": 21968, "\u0120therm": 21969, "\u0120david": 21970, "\u0120systemic": 21971, "\u0120Shir": 21972, "mut": 21973, "\u0120Nept": 21974, "address": 21975, "\u0120scanning": 21976, "\u0120understandable": 21977, "\u0120canvas": 21978, "Cat": 21979, "\u0120Zoo": 21980, "\u0120angels": 21981, "LO": 21982, "\u0120Statement": 21983, "\u0120Sig": 21984, "ovable": 21985, "\u0120Away": 21986, "sharing": 21987, "ocrats": 21988, "stated": 21989, "\u0120weighing": 21990, "Nor": 21991, "wild": 21992, "Bey": 21993, "\u0120astonishing": 21994, "\u0120Reynolds": 21995, "\u0120opener": 21996, "\u0120trainer": 21997, "\u0120surgical": 21998, "pn": 21999, "\u0120adjusting": 22000, "wheel": 22001, "\u0120frown": 22002, "ervative": 22003, "\u0120suspend": 22004, "Within": 22005, "tein": 22006, "\u0120obstacle": 22007, "\u0120liberties": 22008, "ymes": 22009, "\u0120uranium": 22010, "ansom": 22011, "anol": 22012, "uba": 22013, "\u0120Loss": 22014, "\u0120arous": 22015, "\u0120Henderson": 22016, "Wow": 22017, "spl": 22018, "cur": 22019, "\u0120\u00c2\u0143": 22020, "\u0120theirs": 22021, "Damage": 22022, "\u0120downloading": 22023, "\u0120discern": 22024, "\u0120Sto": 22025, "\u0120Fla": 22026, "\u0120hath": 22027, "\u0120Aj": 22028, "\u0120unpleasant": 22029, "European": 22030, "expensive": 22031, "\u0120screenshot": 22032, "\u0120UV": 22033, "\u0120allied": 22034, "\u0120Persian": 22035, "\u0120monopoly": 22036, "\u0120atom": 22037, "\u0120Redskins": 22038, "\"><": 22039, "\u0120cancell": 22040, "\u0120cinema": 22041, "131": 22042, "fair": 22043, "\u0120Alfred": 22044, "\u0120duck": 22045, "args": 22046, "223": 22047, "\u0120ISI": 22048, "\u0120signaling": 22049, "inar": 22050, "\u0120laughs": 22051, "\u0120forwards": 22052, "\u0120reckless": 22053, "\u0120listeners": 22054, "ativity": 22055, "\u0120vastly": 22056, "nant": 22057, "Less": 22058, "\u0120Hunting": 22059, "\u0120Scientific": 22060, "ITED": 22061, "\u0120knight": 22062, "\u0120HTC": 22063, "usa": 22064, "tmp": 22065, "\u0120rude": 22066, "\u0120Legendary": 22067, "\u0120arises": 22068, "Bad": 22069, "\u0120Claim": 22070, "peg": 22071, "\u0120realities": 22072, "Think": 22073, "\u0120\u00c2\u00b0": 22074, "\u0120rode": 22075, "\u0120strive": 22076, "\u0120anecd": 22077, "\u0120shorts": 22078, "\u0120hypothes": 22079, "\u0120coordinated": 22080, "\u0120Gandhi": 22081, "\u0120FPS": 22082, "RED": 22083, "\u0120susceptible": 22084, "\u0120shrink": 22085, "\u0120Chart": 22086, "Help": 22087, "\u0120ion": 22088, "deep": 22089, "ribes": 22090, "\u0120Kai": 22091, "\u0120Customer": 22092, "Summary": 22093, "\u0120cough": 22094, "wife": 22095, "\u0120lend": 22096, "\u0120positioning": 22097, "\u0120lottery": 22098, "\u0120Canyon": 22099, "\u0120fade": 22100, "\u0120bronze": 22101, "\u0120Kenny": 22102, "\u0120boasts": 22103, "\u0120Enhanced": 22104, "record": 22105, "\u0120emergence": 22106, "\u0120akin": 22107, "\u0120Bert": 22108, "itous": 22109, "\u00e2\u0138\u0133": 22110, "\u0120stip": 22111, "\u0120exchanged": 22112, "omore": 22113, "alsh": 22114, "\u0120reservoir": 22115, "\u0120standpoint": 22116, "WM": 22117, "\u0120initiate": 22118, "\u0120decay": 22119, "\u0120brewery": 22120, "\u0120terribly": 22121, "\u0120mortal": 22122, "levard": 22123, "\u0120revis": 22124, "NI": 22125, "elo": 22126, "\u0120confess": 22127, "\u0120MSNBC": 22128, "\u0120submissions": 22129, "Controller": 22130, "\u0120202": 22131, "\u0120Ruth": 22132, "});": 22133, "\u0120Azure": 22134, "\u0120.\"": 22135, "206": 22136, "\u0120Marketing": 22137, "\u0120laund": 22138, "iencies": 22139, "\u0120renowned": 22140, "\u0120Trou": 22141, "\u0120NGO": 22142, "blems": 22143, "\u0120terrified": 22144, "\u0120warns": 22145, "\u0120pert": 22146, "\u0120unsure": 22147, "480": 22148, "alez": 22149, "ultz": 22150, "\u0120Outside": 22151, "\u0120styl": 22152, "\u0120Underground": 22153, "\u0120panc": 22154, "\u0120dictionary": 22155, "\u0120foe": 22156, "riminal": 22157, "\u0120Norwegian": 22158, "\u0120jailed": 22159, "\u0120maternal": 22160, "\u00c3\u00a9e": 22161, "\u0120Lucy": 22162, "cop": 22163, "Cho": 22164, "\u0120unsigned": 22165, "\u0120Zelda": 22166, "\u0120Insider": 22167, "\u0120Continued": 22168, "\u0120133": 22169, "\u0120Naruto": 22170, "\u0120Majority": 22171, "169": 22172, "\u0120Wo": 22173, "\u00e3\u0124\u0135": 22174, "\u0120pastor": 22175, "\u0120informal": 22176, "\u00d0\u00bd": 22177, "anthrop": 22178, "join": 22179, "\u00e3\u0123\u0139": 22180, "itational": 22181, "NP": 22182, "\u0120Writing": 22183, "fn": 22184, "\u0120Bever": 22185, "195": 22186, "\u0120yelling": 22187, "\u0120drastically": 22188, "\u0120eject": 22189, "\u0120neut": 22190, "\u0120thrive": 22191, "\u0120Frequ": 22192, "oux": 22193, "\u0120possesses": 22194, "\u0120Senators": 22195, "\u0120DES": 22196, "\u0120Shakespeare": 22197, "\u0120Franco": 22198, "\u0120LB": 22199, "uchi": 22200, "\u0120incarn": 22201, "\u0120founders": 22202, "Function": 22203, "\u0120brightness": 22204, "\u0120BT": 22205, "\u0120whale": 22206, "\u0120Theater": 22207, "mass": 22208, "\u0120Doll": 22209, "Something": 22210, "\u0120echoed": 22211, "\u0120Hex": 22212, "crit": 22213, "afia": 22214, "\u0120goddess": 22215, "\u0120eleven": 22216, "\u0120Preview": 22217, "\u0120Aurora": 22218, "\u0120401": 22219, "ulsive": 22220, "\u0120Logan": 22221, "inburgh": 22222, "\u0120Centers": 22223, "\u0120ONLY": 22224, "\u0120Aid": 22225, "\u0120paradox": 22226, "\u0120hurd": 22227, "\u0120LC": 22228, "Due": 22229, "court": 22230, "\u0120offended": 22231, "\u0120evaluating": 22232, "\u0120Matthews": 22233, "\u0120tomb": 22234, "\u0120payroll": 22235, "\u0120extraction": 22236, "\u0120Hands": 22237, "ifi": 22238, "\u0120supernatural": 22239, "\u0120COMM": 22240, "]=": 22241, "dogs": 22242, "\u0120512": 22243, "\u0120Meeting": 22244, "Richard": 22245, "\u0120Maximum": 22246, "\u0120ideals": 22247, "Things": 22248, "mand": 22249, "\u0120Regardless": 22250, "\u0120humili": 22251, "buffer": 22252, "Little": 22253, "\u0120Dani": 22254, "\u0120Nak": 22255, "\u0120liberation": 22256, "\u0120Abe": 22257, "\u0120OL": 22258, "\u0120stuffed": 22259, "aca": 22260, "inda": 22261, "raphic": 22262, "\u0120mosqu": 22263, "\u0120campaigning": 22264, "\u0120occupy": 22265, "Squ": 22266, "rina": 22267, "\u0120Wel": 22268, "\u0120VS": 22269, "\u0120physic": 22270, "\u0120puls": 22271, "rint": 22272, "oaded": 22273, "ETF": 22274, "\u0120Archives": 22275, "\u0120venues": 22276, "hner": 22277, "\u0120Turbo": 22278, "\u0120lust": 22279, "\u0120appealed": 22280, "quez": 22281, "ilib": 22282, "\u0120Timothy": 22283, "\u0120omn": 22284, "dro": 22285, "\u0120obsession": 22286, "\u0120Savage": 22287, "1996": 22288, "Global": 22289, "Jes": 22290, "214": 22291, "\u0120sliding": 22292, "\u0120disappro": 22293, "\u0120Magical": 22294, "\u0120voluntarily": 22295, "gb": 22296, "aney": 22297, "\u0120prophet": 22298, "\u0120Rein": 22299, "\u0120Julia": 22300, "\u0120Worth": 22301, "aurus": 22302, "\u0120bounds": 22303, "ieu": 22304, ")))": 22305, "\u0120crore": 22306, "\u0120Citizen": 22307, "Sky": 22308, "\u0120columnist": 22309, "\u0120seekers": 22310, "ondo": 22311, "ISA": 22312, "\u0120Length": 22313, "\u0120nostalg": 22314, "\u0120newcom": 22315, "\u0120detrim": 22316, "entric": 22317, "375": 22318, "\u0120GE": 22319, "\u0120autop": 22320, "\u0120academics": 22321, "AppData": 22322, "\u0120Shen": 22323, "\u0120idiot": 22324, "\u0120Transit": 22325, "\u0120teaspoon": 22326, "Wil": 22327, "KO": 22328, "\u0120Comedy": 22329, ">,": 22330, "\u0120populated": 22331, "WD": 22332, "\u0120pigs": 22333, "\u0120Oculus": 22334, "\u0120sympathetic": 22335, "\u0120marathon": 22336, "198": 22337, "\u0120seizure": 22338, "sided": 22339, "\u0120dop": 22340, "irtual": 22341, "Land": 22342, "\u0120Floor": 22343, "osaurs": 22344, "...]": 22345, "\u0120los": 22346, "\u0120subsidiary": 22347, "EY": 22348, "\u0120Parts": 22349, "\u0120Stef": 22350, "\u0120Judiciary": 22351, "\u0120134": 22352, "\u0120mirrors": 22353, "\u0120ket": 22354, "times": 22355, "\u0120neurolog": 22356, "\u0120cav": 22357, "\u0120Guest": 22358, "\u0120tumor": 22359, "scill": 22360, "\u0120Lloyd": 22361, "Est": 22362, "\u0120clearer": 22363, "\u0120stereotypes": 22364, "\u0120dur": 22365, "nothing": 22366, "Reddit": 22367, "\u0120negotiated": 22368, "------------------------": 22369, "235": 22370, "\u0120flown": 22371, "\u0120Seoul": 22372, "\u0120Resident": 22373, "\u0120SCH": 22374, "\u0120disappearance": 22375, "\u0120Vince": 22376, "grown": 22377, "\u0120grabs": 22378, "ril": 22379, "\u0120Infinite": 22380, "\u0120Twenty": 22381, "\u0120pedestrian": 22382, "\u0120jersey": 22383, "\u0120Fur": 22384, "\u0120Infinity": 22385, "\u0120Elliott": 22386, "\u0120mentor": 22387, "\u0120morally": 22388, "\u0120obey": 22389, "secure": 22390, "iffe": 22391, "\u0120antibiotics": 22392, "angled": 22393, "\u0120Freeman": 22394, "\u0120Introduction": 22395, "Jun": 22396, "\u0120marsh": 22397, "icans": 22398, "\u0120EVENTS": 22399, "ochond": 22400, "Wall": 22401, "iculty": 22402, "\u0120misdemeanor": 22403, "\u0120ly": 22404, "Thomas": 22405, "\u0120Resolution": 22406, "\u0120animations": 22407, "\u0120Dry": 22408, "\u0120intercourse": 22409, "\u0120Newcastle": 22410, "\u0120Hog": 22411, "\u0120Equipment": 22412, "177": 22413, "\u0120territorial": 22414, "\u0120archives": 22415, "203": 22416, "Filter": 22417, "\u0120Munich": 22418, "\u0120commanded": 22419, "\u0120Wand": 22420, "\u0120pitches": 22421, "\u0120Croat": 22422, "\u0120ratios": 22423, "\u0120Mits": 22424, "\u0120accumulated": 22425, "\u0120Specifically": 22426, "\u0120gentleman": 22427, "acerb": 22428, "\u0120penn": 22429, "\u0120aka": 22430, "\u0120Fuk": 22431, "\u0120intervene": 22432, "\u0120Refuge": 22433, "\u0120Alzheimer": 22434, "\u0120succession": 22435, "ohan": 22436, "does": 22437, "Lord": 22438, "\u0120separat": 22439, "\u0120correspondence": 22440, "\u0120shiny": 22441, "Prior": 22442, "\u0120sulf": 22443, "\u0120miserable": 22444, "\u0120dedication": 22445, "().": 22446, "\u0120specialists": 22447, "\u0120defects": 22448, "\u0120Cult": 22449, "\u0120Xia": 22450, "\u0120jeopard": 22451, "\u0120Ore": 22452, "Ability": 22453, "\u0120lear": 22454, "\u0120ambitions": 22455, "\u0120BMI": 22456, "\u0120Arabs": 22457, "\u01201942": 22458, "\u0120preservation": 22459, "ificate": 22460, "\u0120ashamed": 22461, "loss": 22462, "\u0120Restaur": 22463, "\u0120resemble": 22464, "\u0120enrich": 22465, "\u0120KN": 22466, "\u0120Clan": 22467, "float": 22468, "\u0120playable": 22469, "ITT": 22470, "\u0120harmony": 22471, "arrison": 22472, "\u0120Weinstein": 22473, "were": 22474, "\u0120poisoning": 22475, "\u0120Comput": 22476, "\u0120WordPress": 22477, "major": 22478, "\u0120Valve": 22479, "Fan": 22480, "\u0120Throw": 22481, "\u0120Romans": 22482, "\u0120Depression": 22483, "ados": 22484, "\u0120tortured": 22485, "\u0120balancing": 22486, "bottom": 22487, "\u0120acquiring": 22488, "\u0120Monte": 22489, "ardi": 22490, "\u0120aura": 22491, "\u0120##": 22492, "\u0120Standing": 22493, "\u0120Atlas": 22494, "CF": 22495, "\u0120intrins": 22496, "\u0120Benghazi": 22497, "\u0120camping": 22498, "\u0120tapped": 22499, "blade": 22500, "strous": 22501, "\u0120Rabb": 22502, "\u0120Written": 22503, "tip": 22504, "\u0120Neigh": 22505, "sterdam": 22506, "\u0120Allow": 22507, "\u0120Healing": 22508, "\u0120Rhod": 22509, "num": 22510, "\u0120caffeine": 22511, "\u0120Percent": 22512, "\u0120boo": 22513, "\u0120apples": 22514, "305": 22515, "\u0120welcoming": 22516, "\u0120applaud": 22517, "\u0120austerity": 22518, "\u00c2\u00b1": 22519, "\u0120Reality": 22520, "efe": 22521, "\u00e5\u00ae": 22522, "\u0120sucks": 22523, "\u0120tabs": 22524, "\u0120PayPal": 22525, "\u0120backpack": 22526, "\u0120gifted": 22527, "abulary": 22528, "\u0120Scout": 22529, "irteen": 22530, "\u0120chin": 22531, "\u0120omitted": 22532, "\u0120negatively": 22533, "\u0120accessing": 22534, "\u0120Earn": 22535, "\u0120ambulance": 22536, "\u0120headphones": 22537, "\u0120205": 22538, "\u0120Refresh": 22539, "president": 22540, "\u0120Kitchen": 22541, "\u0120Entered": 22542, "\u0120Snyder": 22543, "005": 22544, "omical": 22545, "\u0120borrowed": 22546, "\u0120Nem": 22547, "\u0120aviation": 22548, "\u0120stall": 22549, "rimination": 22550, "\u0120uniforms": 22551, "itime": 22552, "\u0120Simmons": 22553, "energy": 22554, "ablished": 22555, "yy": 22556, "qualified": 22557, "\u0120rallies": 22558, "\u0120Stuart": 22559, "flight": 22560, "\u0120gangs": 22561, "rag": 22562, "\u0120vault": 22563, "lux": 22564, "\u0120Compar": 22565, "\u0120designation": 22566, "209": 22567, "\u0120Jos": 22568, "dollar": 22569, "zero": 22570, "\u0120wells": 22571, "303": 22572, "\u0120constituents": 22573, "\u0120heck": 22574, "\u0120cows": 22575, "\u0120commanders": 22576, "\u0120differential": 22577, "\u0120Catherine": 22578, "299": 22579, "\u0120valve": 22580, "\u0120brace": 22581, "\u0120perspectives": 22582, "cert": 22583, "fact": 22584, "icularly": 22585, "\u0120McN": 22586, "planes": 22587, "\u0120intric": 22588, "\u0120peas": 22589, "ovan": 22590, "\u0120tossed": 22591, "retch": 22592, "\u0120Lopez": 22593, "\u0120unfamiliar": 22594, "death": 22595, "\u0120Apart": 22596, "\u0120Chang": 22597, "\u0120relieved": 22598, "rophe": 22599, "\u0120airports": 22600, "\u0120freak": 22601, "util": 22602, "Mill": 22603, "\u0120Chin": 22604, "\u0120Owen": 22605, "male": 22606, "\u0120Broken": 22607, "\u0120Winds": 22608, "rob": 22609, "rising": 22610, "\u0120firefighters": 22611, "\u0120authoritarian": 22612, "\u0120148": 22613, "Bitcoin": 22614, "external": 22615, "\u0120browsers": 22616, "ichever": 22617, "orian": 22618, "\u0120unb": 22619, "\u0120poke": 22620, "\u0120Zot": 22621, "Mid": 22622, "\u0120Popular": 22623, "\u0120covert": 22624, "\u0120contributes": 22625, "\u0120650": 22626, "\u0120contention": 22627, "Gate": 22628, "\u0120consoles": 22629, "\u0120chromos": 22630, "\u0120IX": 22631, "\u0120visually": 22632, "\u0120Eisen": 22633, "\u0120jewelry": 22634, "\u0120delegation": 22635, "\u0120accelerate": 22636, "\u0120Riley": 22637, "\u0120slope": 22638, "\u0120indoor": 22639, "itially": 22640, "\u0120hugely": 22641, "\u0120tunnels": 22642, "\u0120fined": 22643, "\u0120directive": 22644, "\u0120forehead": 22645, "ustomed": 22646, "\u0120skate": 22647, "Music": 22648, "gas": 22649, "\u0120recognizing": 22650, "ambo": 22651, "\u0120overweight": 22652, "\u0120Grade": 22653, "\u00d9\u012c": 22654, "\u0120sounding": 22655, "\u0120locking": 22656, "\u0120REM": 22657, "Store": 22658, "\u0120excav": 22659, "\u0120Likewise": 22660, "\u0120Lights": 22661, "\u0120elbow": 22662, "\u0120Supply": 22663, "wic": 22664, "\u0120handsome": 22665, "1994": 22666, "Coll": 22667, "\u0120adequately": 22668, "\u0120Associate": 22669, "\u0120strips": 22670, "\u0120crackdown": 22671, "\u0120marvel": 22672, "\u0120Kun": 22673, "\u0120passages": 22674, "@@@@": 22675, "\u0120Tall": 22676, "\u0120thoughtful": 22677, "namese": 22678, "\u0120prostitution": 22679, "business": 22680, "\u0120ballistic": 22681, "personal": 22682, "cig": 22683, "izational": 22684, "Round": 22685, "\u0120\u00c2\u0142\u0120\u00c2\u0142\u0120\u00c2\u0142\u0120\u00c2\u0142": 22686, "\u0120Coleman": 22687, "\u0120admitting": 22688, "\u0120Plug": 22689, "\u0120bitcoins": 22690, "\u0120Suz": 22691, "\u0120fairness": 22692, "\u0120supplier": 22693, "\u0120catastrophic": 22694, "\u0120Helen": 22695, "oqu": 22696, "Marc": 22697, "\u0120Articles": 22698, "gie": 22699, "\u0120endangered": 22700, "\u0120destiny": 22701, "\u0120Volt": 22702, "olia": 22703, "axis": 22704, "\u0120cheat": 22705, "\u0120unified": 22706, "ICO": 22707, "quote": 22708, "302": 22709, "\u0120Sed": 22710, "\u0120suppression": 22711, "\u0120analyzing": 22712, "\u0120squat": 22713, "\u0120figuring": 22714, "\u0120coordinates": 22715, "\u0120chunks": 22716, "\u01201946": 22717, "\u0120subp": 22718, "\u0120wiki": 22719, "\u0120Forbes": 22720, "\u0120Jupiter": 22721, "\u0120Erik": 22722, "imer": 22723, "\u0120Commercial": 22724, "\\)": 22725, "\u0120legitimacy": 22726, "\u0120dental": 22727, "\u0120Mean": 22728, "\u0120deficits": 22729, "550": 22730, "Originally": 22731, "\u0120Horror": 22732, "\u0120contamination": 22733, "llah": 22734, "\u0120confisc": 22735, "\u0120Clare": 22736, "TB": 22737, "\u0120Failed": 22738, "aned": 22739, "\u0120ruler": 22740, "\u0120Controller": 22741, "\u0120feminists": 22742, "Fix": 22743, "gay": 22744, "207": 22745, "\u0120rabbit": 22746, "Third": 22747, "owntown": 22748, "\u0120glue": 22749, "\u0120volatile": 22750, "\u0120shining": 22751, "\u0120foll": 22752, "\u0120impaired": 22753, "\u0120supers": 22754, "\u00e6\u012a": 22755, "\u0120clutch": 22756, "\u013c\u00e9\u0128\u0134": 22757, "\u0120prolet": 22758, "\u0120(!": 22759, "\u0120yelled": 22760, "\u0120Kiev": 22761, "\u0120Ern": 22762, "\u0120Shock": 22763, "KB": 22764, "\u0120situated": 22765, "query": 22766, "\u0120Nas": 22767, "\u0120annex": 22768, "character": 22769, "\u0120Holiday": 22770, "\u0120automation": 22771, "\u0120Jill": 22772, "\u0120Remastered": 22773, "\u0120linem": 22774, "\u0120wilderness": 22775, "\u0120Horizon": 22776, "\u0120Guinea": 22777, "AZ": 22778, "\u0120mainland": 22779, "\u0120secrecy": 22780, "LEASE": 22781, "\u0120punk": 22782, "\u0120Province": 22783, "(),": 22784, "Speed": 22785, "\u0120handing": 22786, "\u0120Sebast": 22787, "Sir": 22788, "rase": 22789, "\u0120journals": 22790, "\u0120congest": 22791, "\u0120Tut": 22792, "irrel": 22793, "\u0120schizophrenia": 22794, "\u0120misogyn": 22795, "healthy": 22796, "Iron": 22797, "\u0120reacted": 22798, "-$": 22799, "252": 22800, "\u0120plural": 22801, "\u0120plum": 22802, "\u0120bargain": 22803, "\u0120grounded": 22804, "finder": 22805, "\u0120disse": 22806, "\u0120Laz": 22807, "OOD": 22808, "\u0120atroc": 22809, "Factory": 22810, "\u0120minions": 22811, "\u0120ori": 22812, "\u0120Brave": 22813, "\u0120PRE": 22814, "\u0120Myanmar": 22815, "\u0120Hod": 22816, "\u0120expedition": 22817, "\u0120explode": 22818, "\u0120Coord": 22819, "\u0120extr": 22820, "\u0120Brief": 22821, "\u0120ADHD": 22822, "\u0120hardcore": 22823, "feeding": 22824, "\u0120dile": 22825, "\u0120Fruit": 22826, "\u0120vaccination": 22827, "\u0120Mao": 22828, "osphere": 22829, "\u0120contests": 22830, "-|": 22831, "\u0120fren": 22832, "isphere": 22833, "Rom": 22834, "\u0120Sharp": 22835, "\u0120Trend": 22836, "\u0120disconnect": 22837, "\u00e2\u0122\u00a2\u00e2\u0122\u00a2": 22838, "\u0120persecution": 22839, "Earth": 22840, "\u0120healthier": 22841, "384": 22842, "\u0120cob": 22843, "\u0120Trinity": 22844, "OWS": 22845, "ANN": 22846, "\u0120specialty": 22847, "\u0120gru": 22848, "\u0120cooperative": 22849, "why": 22850, "Starting": 22851, "\u0120Issues": 22852, "stre": 22853, "ensor": 22854, "\u0120185": 22855, "Adv": 22856, "!?": 22857, "\u0120Revel": 22858, "emia": 22859, "\u0120Hulk": 22860, "\u0120celebrations": 22861, "\u0120Sou": 22862, "raud": 22863, "\u0120Klein": 22864, "\u0120unreal": 22865, "context": 22866, "\u0120partnerships": 22867, "\u0120adopting": 22868, "tical": 22869, "\u0120splash": 22870, "\u0120Hezbollah": 22871, "category": 22872, "cyclop": 22873, "xton": 22874, "\u0120Dot": 22875, "urdy": 22876, "tz": 22877, "\u0120envelope": 22878, "\u0120NL": 22879, "\u00e2\u0137": 22880, "\u0120wherein": 22881, "Spec": 22882, "184": 22883, "\u0120telev": 22884, "aliation": 22885, "\u0120myths": 22886, "\u00e5\u00b0": 22887, "\u0120rigorous": 22888, "\u0120communicating": 22889, "\u0120observer": 22890, "\u0120rehe": 22891, "\u0120Wash": 22892, "\u0120apologized": 22893, "\u0120Tin": 22894, "\u0120expenditures": 22895, "workers": 22896, "document": 22897, "\u0120hesitate": 22898, "\u0120Lenin": 22899, "\u0120unpredictable": 22900, "\u0120renewal": 22901, "cler": 22902, "okia": 22903, "\u0120CONT": 22904, "\u0120postseason": 22905, "Tokens": 22906, "\u0120exacerb": 22907, "\u0120betting": 22908, "\u0120147": 22909, "\u0120elevation": 22910, "Wood": 22911, "\u0120Solomon": 22912, "194": 22913, "004": 22914, "output": 22915, "\u0120redund": 22916, "\u0120Mumbai": 22917, "\u0120pH": 22918, "\u0120reproduce": 22919, "\u0120Duration": 22920, "MAX": 22921, "\u0120bog": 22922, "CBS": 22923, "\u0120Balance": 22924, "\u0120Sgt": 22925, "\u0120Recent": 22926, "\u0120cd": 22927, "\u0120popped": 22928, "\u0120incompet": 22929, "prop": 22930, "ayan": 22931, "guy": 22932, "Pacific": 22933, "\u0120tyr": 22934, "\u0120{{": 22935, "\u0120Mystic": 22936, "\u0120Dana": 22937, "\u0120masturb": 22938, "\u0120geometry": 22939, "\u00c3\u00a2": 22940, "\u0120Correct": 22941, "\u0120trajectory": 22942, "\u0120distracted": 22943, "\u0120foo": 22944, "\u0120Welsh": 22945, "Luc": 22946, "mith": 22947, "\u0120rugby": 22948, "\u0120respiratory": 22949, "\u0120triangle": 22950, "\u0120215": 22951, "\u0120undergraduate": 22952, "\u0120Superior": 22953, "changing": 22954, "_-": 22955, "\u0120rightly": 22956, "\u0120referee": 22957, "\u0120lucrative": 22958, "\u0120unauthorized": 22959, "\u0120resembles": 22960, "\u0120GNU": 22961, "\u0120Derby": 22962, "\u0120pathways": 22963, "\u0120Led": 22964, "\u0120endurance": 22965, "\u0120stint": 22966, "\u0120collector": 22967, "Fast": 22968, "\u0120dots": 22969, "\u0120nationals": 22970, "\u0120Securities": 22971, "\u0120whip": 22972, "Param": 22973, "\u0120learns": 22974, "Magic": 22975, "\u0120detailing": 22976, "moon": 22977, "\u0120broadcasting": 22978, "\u0120baked": 22979, "265": 22980, "holm": 22981, "\u0120Sah": 22982, "\u0120Hussein": 22983, "\u0120Courtesy": 22984, "174": 22985, "\u0120146": 22986, "\u0120geographic": 22987, "peace": 22988, "\u0120judging": 22989, "\u0120Stern": 22990, "Bur": 22991, "\u0120storyline": 22992, "Gun": 22993, "\u0120Stick": 22994, "245": 22995, "307": 22996, "\u00e3\u0124\u00b4\u00e3\u0125\u00b3": 22997, "\u0120Administrator": 22998, "\u0120burnt": 22999, "\u0120pave": 23000, "choes": 23001, "Exec": 23002, "\u0120campuses": 23003, "Result": 23004, "\u0120mutations": 23005, "\u0120Charter": 23006, "\u0120captures": 23007, "\u0120compares": 23008, "\u0120badge": 23009, "Scient": 23010, "\u0120erad": 23011, "iery": 23012, "oi": 23013, "ettes": 23014, "\u0120Estate": 23015, "\u0120strap": 23016, "\u0120proudly": 23017, "\u0120fried": 23018, "\u0120withdrawn": 23019, "\u0120Voy": 23020, "phony": 23021, "Items": 23022, "\u0120Pierce": 23023, "bard": 23024, "\u0120annotation": 23025, "anton": 23026, "illon": 23027, "Impro": 23028, "...)": 23029, "\u0120happier": 23030, "------": 23031, "adjust": 23032, "\u0120staffers": 23033, "\u0120activism": 23034, "\u0120perf": 23035, "\u0120alright": 23036, "Need": 23037, "\u0120commence": 23038, "\u0120opioid": 23039, "\u0120Amanda": 23040, "Es": 23041, "\u0120Pars": 23042, "\u0120Kaw": 23043, "Works": 23044, "248": 23045, "\u0120indo": 23046, "tc": 23047, "endant": 23048, "\u0120Moto": 23049, "\u0120legalization": 23050, "OTE": 23051, "\u0120tasked": 23052, "\u0120tsp": 23053, "\u0120ACTIONS": 23054, "166": 23055, "\u0120refreshing": 23056, "\u0120NR": 23057, "\u0120Perez": 23058, "\u0120infringement": 23059, "SY": 23060, "Listen": 23061, "inning": 23062, "ku": 23063, "\u0120rotate": 23064, "program": 23065, "arah": 23066, "Design": 23067, "\u0120(\u00c2\u00a3": 23068, "\u0120storing": 23069, "\u0120warrants": 23070, "\u0120judgement": 23071, "\u0120Brist": 23072, "usually": 23073, "photo": 23074, "\u0120Ran": 23075, "\u0120Pine": 23076, "\u0120outrageous": 23077, "\u0120Valentine": 23078, "luence": 23079, "\u0120Everybody": 23080, "Altern": 23081, "\u0120relevance": 23082, "\u0120terminated": 23083, "\u0120dessert": 23084, "\u0120fulfilled": 23085, "\u0120prosecuted": 23086, "\u0120Words": 23087, "\u0120migrant": 23088, "\u0120cultivation": 23089, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 23090, "idelity": 23091, "\u0120Vern": 23092, "\u0120Login": 23093, "\u0120metaphor": 23094, "\u0120Tip": 23095, "\u0120recruits": 23096, "\u0120Pig": 23097, "ribing": 23098, "\u0120enthusiasts": 23099, "exper": 23100, "\u0120frightening": 23101, "\u0120Hair": 23102, "anson": 23103, "strate": 23104, "\u0120hi": 23105, "Height": 23106, "\u0120owning": 23107, "none": 23108, "\u0120dislike": 23109, "\u0120knives": 23110, "pherd": 23111, "\u0120loudly": 23112, "\u0120APIs": 23113, "Display": 23114, "\u0120Lac": 23115, "\u0120USS": 23116, "abl": 23117, "verages": 23118, "Jew": 23119, "\u0120172": 23120, "\u0120Historical": 23121, "atoon": 23122, "\u0120Physics": 23123, "intern": 23124, "\u0120warmth": 23125, "\u0120topp": 23126, "DM": 23127, "\u0120gunman": 23128, "\u0120emperor": 23129, "odi": 23130, "\u00e3\u0125\u00a3": 23131, "inatory": 23132, "\u0120Rib": 23133, "\u0120131": 23134, "\u0120Saturn": 23135, "\u0120Shining": 23136, "\u0120waking": 23137, "Quotes": 23138, "\u0120comedian": 23139, "enberg": 23140, "\u00c2\u00bd": 23141, "\u0120believers": 23142, "\u0120paperwork": 23143, "custom": 23144, "\u0120lev": 23145, "\u0120lament": 23146, "\u0120pouring": 23147, "222": 23148, "political": 23149, "\u0120Supplement": 23150, "maid": 23151, "\u0120cruelty": 23152, "\u0120tread": 23153, "ysics": 23154, "Aw": 23155, "rites": 23156, "\u0120modifier": 23157, "\u0120Position": 23158, "Adam": 23159, "lb": 23160, "ubs": 23161, "\u0120imperfect": 23162, "\u0120clusters": 23163, "\u0120Engineer": 23164, "\u0120Cherry": 23165, "\u0120inauguration": 23166, "\u0120Sau": 23167, "\u0120embodiment": 23168, "\u0120Uncle": 23169, "\u0120overr": 23170, "\u0120explosions": 23171, "cule": 23172, "\u0120Princeton": 23173, "\u0120Andrea": 23174, "\u0120incorrectly": 23175, "\u0120earnest": 23176, "\u0120pilgr": 23177, "\u0120Sprint": 23178, "\u0120sleeve": 23179, "\u0120hears": 23180, "\u0120Amazing": 23181, "\u0120browsing": 23182, "agin": 23183, "\u0120homeland": 23184, "\u0120haw": 23185, "\u0120diving": 23186, "istered": 23187, "178": 23188, "\u0120bargaining": 23189, "\u0120Arcade": 23190, "\u0120delegate": 23191, "terson": 23192, "................................................................": 23193, "\u0120Jacksonville": 23194, "275": 23195, "\u0120stagn": 23196, "\u0120adam": 23197, "\u0120Sherman": 23198, "CB": 23199, "\u0120suburb": 23200, "\u0120Foods": 23201, "\u0120converting": 23202, "\u0120Arist": 23203, "\u0120chambers": 23204, "love": 23205, "\u0120amino": 23206, "\u0120Gan": 23207, "\u0120madness": 23208, "mc": 23209, "\u0120USE": 23210, "defined": 23211, "\u0120ultr": 23212, "indust": 23213, "\u0120wolves": 23214, "lance": 23215, "Additionally": 23216, "\u0120cracks": 23217, "asia": 23218, "\u0120Reason": 23219, "\u0120Pump": 23220, "\u0120accidental": 23221, "\u0120Laser": 23222, "\u0120Rid": 23223, "\u0120initialized": 23224, "elli": 23225, "\u0120unnamed": 23226, "\u0120noun": 23227, "\u0120Passed": 23228, "\u0120hostage": 23229, "\u0120Ethiop": 23230, "shirts": 23231, "\u0120unrel": 23232, "\u0120Embassy": 23233, "\u01201941": 23234, "\u0120atoms": 23235, "\u0120purported": 23236, "164": 23237, "\u0120Fi": 23238, "\u0120gallons": 23239, "\u0120Monica": 23240, "\u0120pg": 23241, "enment": 23242, "\u0120sorted": 23243, "\u0120Gospel": 23244, "\u0120heights": 23245, "\u0120traced": 23246, "\u0120undergoing": 23247, "Shell": 23248, "\u0120sacks": 23249, "\u0120proportions": 23250, "\u0120halluc": 23251, "Font": 23252, "acet": 23253, "\u0120warmer": 23254, "\u0120INTER": 23255, "\u0120grabbing": 23256, "Plug": 23257, "\u0120realization": 23258, "\u0120Burke": 23259, "\u0120enchant": 23260, "ATER": 23261, "\u0120Seed": 23262, "\u0120abundant": 23263, "FM": 23264, "\u0120civic": 23265, "Vs": 23266, "isi": 23267, "\u0120vow": 23268, "\u0120reper": 23269, "\u0120Partnership": 23270, "\u0120penetration": 23271, "\u0120axe": 23272, "\u0120shattered": 23273, "\u0120Zombies": 23274, "\u0120vinyl": 23275, "\u0120Alert": 23276, "eon": 23277, "\u0120obliged": 23278, "\u0120Illust": 23279, "\u0120Plaza": 23280, "\u0120Frontier": 23281, "\u0120davidjl": 23282, "\u0120Serial": 23283, "\u0120Hav": 23284, "\u0120Nutrition": 23285, "Bi": 23286, "\u0120\u00e2\u0138\u012a": 23287, "\u0120Jays": 23288, "linux": 23289, "\u0120hurry": 23290, "\u0120voy": 23291, "\u0120hopeless": 23292, "\u0120Stealth": 23293, "\u0120\u00e3\u0123": 23294, "essors": 23295, "ttle": 23296, "borg": 23297, "\u0120Safari": 23298, "fell": 23299, "\u0120wary": 23300, "due": 23301, "\u0120Above": 23302, "Ha": 23303, "ELL": 23304, "\u0120notor": 23305, "\u0120Won": 23306, "Too": 23307, "\u0120occupations": 23308, "\u0120possessions": 23309, "\u0120inviting": 23310, "\u0120predators": 23311, "\u0120accelerated": 23312, "\u0120157": 23313, "uterte": 23314, "\u0120Cube": 23315, "east": 23316, "account": 23317, "Give": 23318, "\u0120transplant": 23319, "redients": 23320, "idable": 23321, "\u0120screenshots": 23322, "\u0120Gund": 23323, "\u0120FS": 23324, "\u0120travelers": 23325, "\u0120sensory": 23326, "\u0120Fiat": 23327, "\u0120Rockets": 23328, "\u0130\u012d": 23329, "_{": 23330, "Friend": 23331, "\u0120charming": 23332, "ALS": 23333, "\u0120enjoyment": 23334, "mph": 23335, "\u01205000": 23336, "\u0120REG": 23337, "\u00d9\u0128": 23338, "bia": 23339, "\u0120compilation": 23340, "rost": 23341, "\u0120VP": 23342, "\u0120Schne": 23343, "2019": 23344, "\u0120copying": 23345, "MORE": 23346, "\u0120Flore": 23347, "falls": 23348, "215": 23349, "total": 23350, "\u0120disciples": 23351, "double": 23352, "\u0120exceeding": 23353, "\u0120smashed": 23354, "\u0120conceptual": 23355, "\u0120Romania": 23356, "\u0120Brent": 23357, "\u0120ICE": 23358, "\u0120Tou": 23359, "\u0120grap": 23360, "\u0120nails": 23361, "189": 23362, "\u00e3\u0125\u013a": 23363, "\u0120procure": 23364, "eur": 23365, "\u0120confirming": 23366, "\u0120Cec": 23367, "awi": 23368, "\u0120Eden": 23369, "\u0120ng": 23370, "\u0120engineered": 23371, "atics": 23372, "\u0120hooked": 23373, "\u0120disgusting": 23374, "\u0120Murder": 23375, "\u00e3\u0124\u00bf": 23376, "Library": 23377, "\u0120168": 23378, "Almost": 23379, "hematic": 23380, "Menu": 23381, "\u0120Notre": 23382, "\u0120Jur": 23383, "\u0120kidnapped": 23384, "\u0120hacker": 23385, "\u0120Jade": 23386, "\u0120creepy": 23387, "\u0120drawings": 23388, "\u0120Sponsor": 23389, "\u0120cyclists": 23390, "\u0120Goblin": 23391, "\u0120optimized": 23392, "\u0120staged": 23393, "\u0120McD": 23394, "between": 23395, "Age": 23396, "eno": 23397, "Sex": 23398, "\u0120Wide": 23399, "nings": 23400, "avis": 23401, "\u0120incapable": 23402, "\u0120Kob": 23403, "\u0120rewarding": 23404, "\u0120Lone": 23405, "olescent": 23406, "\u0120contracted": 23407, "\u0120sticky": 23408, "Jose": 23409, "Ball": 23410, "fest": 23411, "\u0120Input": 23412, "\u0120Recently": 23413, "\u0120tomat": 23414, "square": 23415, "Application": 23416, "\u0120nitrogen": 23417, "\u0120duplicate": 23418, "\u0120Recon": 23419, "\u0120Dear": 23420, "London": 23421, "\u0120intra": 23422, "\u0120dock": 23423, "\u0120outreach": 23424, "\u0120Million": 23425, "\u0120mammals": 23426, "ampton": 23427, "VAL": 23428, "\u0120snaps": 23429, "\u0120dos": 23430, "\u0120Whole": 23431, "\u0120Ready": 23432, "Try": 23433, "\u0120Winnipeg": 23434, "earance": 23435, "\u0120incurred": 23436, "renched": 23437, "\u0120NSW": 23438, "ilot": 23439, "raine": 23440, "\u0120cube": 23441, "got": 23442, "\u0120runway": 23443, "etermined": 23444, "\u0120Hawks": 23445, "\u0120survivor": 23446, "\u0120Wish": 23447, "\u0120Din": 23448, "\u0120DEF": 23449, "\u0120Vault": 23450, "187": 23451, "\u0120mushrooms": 23452, "\u0120crisp": 23453, "bey": 23454, "\u0120Discovery": 23455, "\u0120developmental": 23456, "\u0120paradigm": 23457, "\u0120chaotic": 23458, "\u0120Tsu": 23459, "\u0120333": 23460, "bons": 23461, "\u0120bacterial": 23462, "\u0120commits": 23463, "\u0120cosmic": 23464, "\u0120mega": 23465, "ocative": 23466, "\u0120Paint": 23467, "ophobic": 23468, "\u0120vain": 23469, "\u0120carved": 23470, "\u0120Thief": 23471, "\u0120Gul": 23472, "owship": 23473, "\u0120cites": 23474, "\u0120Edinburgh": 23475, "\u0120diminished": 23476, "\u0120acknowledges": 23477, "\u0120Kills": 23478, "\u0120microw": 23479, "\u0120Hera": 23480, "\u0120seniors": 23481, "\u0120whereby": 23482, "Hop": 23483, "atron": 23484, "\u0120unavailable": 23485, "\u0120Nate": 23486, "\u0120480": 23487, "\u0120slated": 23488, "\u0120Rebecca": 23489, "\u0120Battery": 23490, "\u0120grammar": 23491, "\u0120headset": 23492, "\u0120cursor": 23493, "\u0120excluding": 23494, "anye": 23495, "aundering": 23496, "ebin": 23497, "\u0120feasible": 23498, "\u0120Publishing": 23499, "\u0120Labs": 23500, "\u0120Cliff": 23501, "\u0120Ferrari": 23502, "\u0120pac": 23503, "visible": 23504, "marked": 23505, "pell": 23506, "\u0120polite": 23507, "\u0120staggering": 23508, "\u0120Galactic": 23509, "\u0120superst": 23510, "\u0120paran": 23511, "\u0120Officers": 23512, "\u00e3\u0122\u0123": 23513, "\u0120specifics": 23514, "ulus": 23515, "239": 23516, "\u0120Paste": 23517, "AMP": 23518, "\u0120Panama": 23519, "\u0120Delete": 23520, "anguard": 23521, "restrial": 23522, "\u0120heroic": 23523, "\u0120Dy": 23524, "\u00d8\u00a7\u00d9\u0126": 23525, "\u0120incumbent": 23526, "\u0120crunch": 23527, "tro": 23528, "\u0120scoop": 23529, "\u0120blogger": 23530, "\u0120sellers": 23531, "uren": 23532, "\u0120medicines": 23533, "\u0120Caps": 23534, "\u0120Animation": 23535, "oxy": 23536, "\u0120outward": 23537, "\u0120inquiries": 23538, "229": 23539, "\u0120psychologist": 23540, "\u0120Sask": 23541, "evil": 23542, "\u0120contaminated": 23543, "\u00e3\u0124\u00a8": 23544, "herence": 23545, "\u0120branded": 23546, "\u0120Abdul": 23547, "zh": 23548, "\u0120paragraphs": 23549, "\u0120mins": 23550, "\u0120correlated": 23551, "erb": 23552, "\u0120impart": 23553, "\u0120milestone": 23554, "\u0120Solutions": 23555, "otle": 23556, "\u0120undercover": 23557, "\u0120marched": 23558, "\u0120Chargers": 23559, "fax": 23560, "\u0120Secrets": 23561, "\u0120ruth": 23562, "weather": 23563, "\u0120feminine": 23564, "\u0120sham": 23565, "\u0120prestigious": 23566, "iggins": 23567, "\u0120sung": 23568, "history": 23569, "ettle": 23570, "ggie": 23571, "\u0120outdated": 23572, "oland": 23573, "\u0120perceptions": 23574, "\u0120Session": 23575, "\u0120Dodgers": 23576, "uj": 23577, "\u0120END": 23578, "Doc": 23579, "\u0120deficiency": 23580, "Grand": 23581, "\u0120Joker": 23582, "\u0120retrospect": 23583, "\u0120diagnostic": 23584, "\u0120harmless": 23585, "\u0120rogue": 23586, "\u0120Aval": 23587, "Equ": 23588, "\u0120transc": 23589, "\u0120Robertson": 23590, "\u0120Depending": 23591, "\u0120Burns": 23592, "ivo": 23593, "\u0120hostility": 23594, "Features": 23595, "\u0135\u013a": 23596, "\u0120discomfort": 23597, "\u0120LCD": 23598, "specified": 23599, "\u0120Expect": 23600, "340": 23601, "\u0120imperative": 23602, "\u0120Regular": 23603, "Chinese": 23604, "\u0120statewide": 23605, "\u0120symm": 23606, "\u0120loops": 23607, "\u0120autumn": 23608, "Nick": 23609, "\u0120shaping": 23610, "\u0120quot": 23611, "\u0120cherry": 23612, "\u0120Crossref": 23613, "\u00e8\u00a6\u013c\u00e9\u0128\u0134": 23614, "Standard": 23615, "heed": 23616, "\u0120Dell": 23617, "\u0120Vietnamese": 23618, "\u0120ost": 23619, "\u0120Valkyrie": 23620, "OA": 23621, "Assad": 23622, "\u0120rebound": 23623, "\u0120Traffic": 23624, "places": 23625, "\u00e6\u013a": 23626, "\u0120Buc": 23627, "172": 23628, "\u0120shelters": 23629, "\u0120insisting": 23630, "\u0120Certainly": 23631, "\u0120Kenneth": 23632, "\u0120TCP": 23633, "\u0120penal": 23634, "\u0120Replay": 23635, "heard": 23636, "\u0120dialect": 23637, "iza": 23638, "\u0120FY": 23639, "itcher": 23640, "\u0120DL": 23641, "\u0120spiral": 23642, "\u0120quarterbacks": 23643, "\u0120hull": 23644, "\u0120google": 23645, "\u0120todd": 23646, "\u0120Sterling": 23647, "\u0120Plate": 23648, "\u0120spying": 23649, "mbol": 23650, "\u0120Realm": 23651, "\u0120Proced": 23652, "\u0120Crash": 23653, "\u0120terminate": 23654, "\u0120protesting": 23655, "Center": 23656, "guided": 23657, "\u0120uncover": 23658, "\u0120boycott": 23659, "\u0120realizes": 23660, "sound": 23661, "\u0120pretending": 23662, "\u0120Vas": 23663, "1980": 23664, "\u0120framed": 23665, "\u0120139": 23666, "\u0120descended": 23667, "\u0120rehabilitation": 23668, "\u0120borrowing": 23669, "\u0120Buch": 23670, "\u0120blur": 23671, "Ron": 23672, "\u0120Frozen": 23673, "enza": 23674, "Chief": 23675, "\u0120Poor": 23676, "\u0120translates": 23677, "MIN": 23678, "\u0120212": 23679, "JECT": 23680, "\u0120erupted": 23681, "\u0120successes": 23682, "SEC": 23683, "\u0120plague": 23684, "\u0120gems": 23685, "doms": 23686, "\u0120stretches": 23687, "\u0120Spy": 23688, "\u0120storytelling": 23689, "Credit": 23690, "\u0120Push": 23691, "\u0120traction": 23692, "\u0120ineffective": 23693, "\u0120Luna": 23694, "\u0120tapes": 23695, "\u0120analytics": 23696, "ercise": 23697, "\u0120programmes": 23698, "\u0120Carbon": 23699, "\u0120behold": 23700, "heavy": 23701, "\u0120Conservation": 23702, "\u0120FIR": 23703, "\u0120sack": 23704, "termin": 23705, "ricks": 23706, "\u0120housed": 23707, "\u0120unusually": 23708, "Ice": 23709, "\u0120executing": 23710, "\u0120Moroc": 23711, "eday": 23712, "\u0120editions": 23713, "\u0120smarter": 23714, "\u0120BA": 23715, "\u0120outlaw": 23716, "\u0120vanished": 23717, "iba": 23718, "ALSE": 23719, "\u0120Silva": 23720, "238": 23721, "Could": 23722, "\u0120philosopher": 23723, "\u0120evacuated": 23724, "Secret": 23725, "142": 23726, "\u0120visas": 23727, "\u00e3\u0124\u00ac": 23728, "\u0120Malt": 23729, "\u0120Clearly": 23730, "\u0120Niger": 23731, "\u0120Cairo": 23732, "\u0120Fist": 23733, "380": 23734, "\u0120XML": 23735, "auto": 23736, "itant": 23737, "\u0120reinforced": 23738, "Record": 23739, "\u0120Survivor": 23740, "GHz": 23741, "\u0120screws": 23742, "parents": 23743, "\u0120oceans": 23744, "mares": 23745, "\u0120brakes": 23746, "vasive": 23747, "\u0120hello": 23748, "\u0120SIM": 23749, "rimp": 23750, "\u0120ore": 23751, "\u0120Armour": 23752, "247": 23753, "\u0120terrific": 23754, "\u0120tones": 23755, "141": 23756, "\u0120Minutes": 23757, "Episode": 23758, "\u0120curves": 23759, "\u0120inflammatory": 23760, "\u0120batting": 23761, "\u0120Beautiful": 23762, "Lay": 23763, "\u0120unpop": 23764, "vable": 23765, "\u0120riots": 23766, "\u0120Tactics": 23767, "baugh": 23768, "\u0120Cock": 23769, "\u0120orgasm": 23770, "\u0120Sas": 23771, "\u0120constructor": 23772, "etz": 23773, "Gov": 23774, "\u0120antagon": 23775, "\u0120theat": 23776, "\u0120deeds": 23777, "hao": 23778, "cuts": 23779, "\u0120McCl": 23780, "\u0120um": 23781, "\u0120Scientists": 23782, "\u0120grassroots": 23783, "yssey": 23784, "\"]=>": 23785, "\u0120surfaced": 23786, "\u0120shades": 23787, "\u0120neighbours": 23788, "\u0120advertis": 23789, "oya": 23790, "\u0120merged": 23791, "Upon": 23792, "\u0120gad": 23793, "\u0120anticipate": 23794, "Anyway": 23795, "\u0120slogan": 23796, "\u0120disrespect": 23797, "Iran": 23798, "\u0120TB": 23799, "acted": 23800, "\u0120subpoen": 23801, "mediately": 23802, "OOOO": 23803, "\u0120waiver": 23804, "\u0120vulnerabilities": 23805, "ottesville": 23806, "\u0120Huffington": 23807, "Josh": 23808, "\u0120DH": 23809, "Monday": 23810, "\u0120Ellen": 23811, "Know": 23812, "xon": 23813, "items": 23814, "228": 23815, "\u0120fills": 23816, "\u0120Nike": 23817, "\u0120cumulative": 23818, "andals": 23819, "Ir": 23820, "\u0120\u00ec": 23821, "\u0120friction": 23822, "igator": 23823, "\u0120scans": 23824, "\u0120Vienna": 23825, "ldom": 23826, "\u0120performers": 23827, "Prim": 23828, "\u0120bidding": 23829, "Mur": 23830, "\u0120leaned": 23831, "\u0120Prix": 23832, "alks": 23833, "\u0120[\u00e2\u0122\u00a6]": 23834, "\u0120Twitch": 23835, "\u0120Developer": 23836, "\u0120Gir": 23837, "\u0120callback": 23838, "Abstract": 23839, "\u0120accustomed": 23840, "\u0120freedoms": 23841, "\u0120PG": 23842, "uracy": 23843, "\u0120lump": 23844, "isman": 23845, ",,,,": 23846, "1992": 23847, "\u0120RED": 23848, "\u0120worm": 23849, "Match": 23850, "\u0120Platinum": 23851, "IJ": 23852, "\u0120Owner": 23853, "Trivia": 23854, "compl": 23855, "\u0120newborn": 23856, "\u0120fantas": 23857, "Own": 23858, "\u01201959": 23859, "\u0120sympath": 23860, "\u0120ubiqu": 23861, "\u0120outputs": 23862, "\u0120allev": 23863, "\u0120prag": 23864, "Kevin": 23865, "\u0120favors": 23866, "\u0120burial": 23867, "\u0120nurt": 23868, "solete": 23869, "cache": 23870, "\u0120156": 23871, "\u0120unlocks": 23872, "techn": 23873, "Making": 23874, "\u0120conquer": 23875, "adic": 23876, "\u00e6\u0138": 23877, "\u0120elf": 23878, "\u0120electorate": 23879, "\u0120Kurds": 23880, "\u0120Stack": 23881, "\u0120Samurai": 23882, "\u0120\u00e2\u013a\u0127": 23883, "\u0120{}": 23884, "\u0120Said": 23885, "\u0120Fallout": 23886, "\u0120kindness": 23887, "\u0120Customs": 23888, "\u0120Boulevard": 23889, "\u0120helicopters": 23890, "otics": 23891, "\u0120Veget": 23892, "comment": 23893, "\u0120criticised": 23894, "\u0120polished": 23895, "\u0120Remix": 23896, "\u0120Cultural": 23897, "\u0120recons": 23898, "\u0120doi": 23899, "atem": 23900, "Screen": 23901, "\u0120barred": 23902, "Comments": 23903, "\u0120Generally": 23904, "\u0120slap": 23905, "720": 23906, "Vari": 23907, "pine": 23908, "\u0120empt": 23909, "\u0120hats": 23910, "\u0120Playing": 23911, "lab": 23912, "average": 23913, "forms": 23914, "\u0120Cotton": 23915, "\u0120cans": 23916, "\u0120DON": 23917, "\u0120Somalia": 23918, "Crypt": 23919, "\u0120Increases": 23920, "Ever": 23921, "modern": 23922, "\u0120surgeon": 23923, "3000": 23924, "\u0120randomized": 23925, "================================================================": 23926, "Bern": 23927, "impl": 23928, "\u0120COR": 23929, "\u0120proclaim": 23930, "thouse": 23931, "\u0120toes": 23932, "\u0120ample": 23933, "\u0120preserving": 23934, "\u0120disbel": 23935, "grand": 23936, "Besides": 23937, "\u0120silk": 23938, "\u0120Pattern": 23939, "hm": 23940, "\u0120enterprises": 23941, "\u0120affidavit": 23942, "\u0120Advisory": 23943, "\u0120advertised": 23944, "\u0120Religious": 23945, "sections": 23946, "psych": 23947, "\u0120Fields": 23948, "aways": 23949, "\u0120hashtag": 23950, "\u0120Nightmare": 23951, "\u0120vampire": 23952, "\u0120forensic": 23953, "rossover": 23954, "nar": 23955, "\u0120navy": 23956, "\u0120vacant": 23957, "\u0120Duel": 23958, "\u0120hallway": 23959, "\u0120facebook": 23960, "identally": 23961, "\u0120NRA": 23962, "\u0120matt": 23963, "\u0120hurricane": 23964, "\u0120Kirby": 23965, "\u0120Puzzle": 23966, "\u0120skirt": 23967, "oust": 23968, "dullah": 23969, "\u0120analogy": 23970, "inion": 23971, "\u0120tomatoes": 23972, "\u0120NV": 23973, "\u0120Peak": 23974, "\u0120Meyer": 23975, "\u0120appointments": 23976, "\u0120masc": 23977, "\u0120alley": 23978, "rehend": 23979, "\u0120charities": 23980, "\u0120undo": 23981, "\u0120destinations": 23982, "\u0120Testing": 23983, "\">\"": 24618, "cats": 24619, "*.": 24620, "\u0120gestures": 24621, "general": 24622, "League": 24623, "\u0120packets": 24624, "\u0120Inspector": 24625, "\u0120Berg": 24626, "\u0120fraudulent": 24627, "\u0120criticize": 24628, "Fun": 24629, "\u0120blaming": 24630, "ndra": 24631, "\u0120slash": 24632, "\u0120Eston": 24633, "\u0120proposing": 24634, "\u0120whales": 24635, "\u0120therapist": 24636, "\u0120subset": 24637, "\u0120leisure": 24638, "ELD": 24639, "\u0120CVE": 24640, "\u0120Activity": 24641, "\u0120culmin": 24642, "shop": 24643, "\u0120DAY": 24644, "ischer": 24645, "\u0120Admiral": 24646, "\u0120Attacks": 24647, "\u01201958": 24648, "\u0120memoir": 24649, "\u0120folded": 24650, "\u0120sexist": 24651, "\u0120153": 24652, "\u0120LI": 24653, "\u0120readings": 24654, "\u0120embarrassment": 24655, "\u0120Employment": 24656, "wart": 24657, "chin": 24658, "\u0120continuation": 24659, "lia": 24660, "Recently": 24661, "\u0120duel": 24662, "\u0120evacuation": 24663, "\u0120Kashmir": 24664, "\u0120disposition": 24665, "\u0120Rig": 24666, "\u0120bolts": 24667, "\u0120insurers": 24668, "467": 24669, "Mex": 24670, "\u0120retaliation": 24671, "\u0120misery": 24672, "\u0120unreasonable": 24673, "raining": 24674, "Imm": 24675, "\u0120PU": 24676, "emer": 24677, "\u0120genital": 24678, "\u00e3\u0124\u00b3": 24679, "\u0120Candy": 24680, "\u0120onions": 24681, "\u0120Patt": 24682, "liner": 24683, "\u0120conceded": 24684, "\u0120fa": 24685, "\u0120forc": 24686, "\u0120Hernandez": 24687, "\u0120Geoff": 24688, "debian": 24689, "\u0120Teams": 24690, "\u0120cries": 24691, "\u0120homeowners": 24692, "237": 24693, "ABC": 24694, "\u0120stitch": 24695, "\u0120statistic": 24696, "\u0120headers": 24697, "\u0120Biology": 24698, "\u0120motors": 24699, "\u0120GEN": 24700, "\u0120Lip": 24701, "\u0120hates": 24702, "\u0120heel": 24703, "Self": 24704, "ipl": 24705, "EDIT": 24706, "orting": 24707, "\u0120annot": 24708, "\u0120Speech": 24709, "oldemort": 24710, "\u0120Javascript": 24711, "\u0120LeBron": 24712, "\u0120footprint": 24713, "\u0120fn": 24714, "\u0120seizures": 24715, "nas": 24716, "hide": 24717, "\u01201954": 24718, "\u0120Bee": 24719, "\u0120Declaration": 24720, "\u0120Katie": 24721, "\u0120reservations": 24722, "NR": 24723, "female": 24724, "\u0120saturated": 24725, "\u0120biblical": 24726, "\u0120trolls": 24727, "Device": 24728, "photos": 24729, "\u0120drums": 24730, "\u00e3\u0125\u012b\u00e3\u0125\u00a9\u00e3\u0124\u00b4\u00e3\u0125\u00b3": 24731, "Night": 24732, "fighter": 24733, "\u0120Hak": 24734, "riber": 24735, "\u0120cush": 24736, "\u0120disciplinary": 24737, "baum": 24738, "\u0120GH": 24739, "\u0120Schmidt": 24740, "ilibrium": 24741, "\u0120sixty": 24742, "\u0120Kushner": 24743, "rots": 24744, "\u0120pund": 24745, "\u0120Rac": 24746, "\u0120springs": 24747, "\u0120conve": 24748, "Business": 24749, "Fall": 24750, "\u0120qualifications": 24751, "\u0120verses": 24752, "\u0120narciss": 24753, "\u0120Koh": 24754, "\u0120Wow": 24755, "\u0120Charlottesville": 24756, "edo": 24757, "\u0120interrogation": 24758, "\u0120Wool": 24759, "365": 24760, "Brian": 24761, "\u0120\u00e2\u013e\u0135": 24762, "\u0120alleges": 24763, "onds": 24764, "idation": 24765, "\u0120Jackie": 24766, "yu": 24767, "\u0120lakes": 24768, "\u0120worthwhile": 24769, "\u0120crystals": 24770, "\u0120Juda": 24771, "\u0120comprehend": 24772, "\u0120flush": 24773, "\u0120absorption": 24774, "\u0120OC": 24775, "\u0120frightened": 24776, "\u0120Chocolate": 24777, "Martin": 24778, "\u0120buys": 24779, "\u0120bucks": 24780, "\u0120appell": 24781, "\u0120Championships": 24782, "\u0120listener": 24783, "\u0120Defensive": 24784, "\u0120cz": 24785, "uds": 24786, "\u0120Mate": 24787, "\u0120replay": 24788, "\u0120decorated": 24789, "\u0120sunk": 24790, "\u0120VIP": 24791, "\u0120Ank": 24792, "\u0120195": 24793, "aaaa": 24794, "Nobody": 24795, "\u0120Milk": 24796, "\u0120Gur": 24797, "\u0120Mk": 24798, "\u0120Sara": 24799, "\u0120seating": 24800, "\u0120Wid": 24801, "Track": 24802, "\u0120employs": 24803, "\u0120gigantic": 24804, "APP": 24805, "\u00e3\u0124\u00a7": 24806, "inventory": 24807, "\u0120towel": 24808, "atche": 24809, "lasting": 24810, "\u0120TL": 24811, "\u0120latency": 24812, "\u0120kne": 24813, "Ber": 24814, "meaning": 24815, "\u0120upheld": 24816, "\u0120playground": 24817, "\u0120mant": 24818, "Side": 24819, "\u0120stereo": 24820, "\u0120northwest": 24821, "\u0120exceptionally": 24822, "\u0120rays": 24823, "\u0120recurring": 24824, "Drive": 24825, "\u0120upright": 24826, "\u0120abduct": 24827, "\u0120Marathon": 24828, "\u0120goodbye": 24829, "\u0120alphabet": 24830, "hp": 24831, "\u0120courtroom": 24832, "rington": 24833, "othing": 24834, "Tag": 24835, "\u0120diplomats": 24836, "\u0120barbar": 24837, "\u0120Aqua": 24838, "183": 24839, "3333": 24840, "\u0120maturity": 24841, "\u0120instability": 24842, "\u0120Apache": 24843, "\u0120===": 24844, "\u0120fasting": 24845, "\u0120Grid": 24846, "ModLoader": 24847, "\u0120152": 24848, "Abs": 24849, "\u0120Operating": 24850, "etti": 24851, "\u0120acquaint": 24852, "Donnell": 24853, "\u0120Kem": 24854, "\u0120Forge": 24855, "\u0120armored": 24856, "Mil": 24857, "\u0120philosophers": 24858, "invest": 24859, "Players": 24860, "\u00e2\u012a": 24861, "\u0120myriad": 24862, "\u0120comrades": 24863, "Rot": 24864, "\u0120remembering": 24865, "\u0120corresponds": 24866, "\u0120programmers": 24867, "\u0120Lynn": 24868, "\u0120olig": 24869, "\u0120coherent": 24870, "ynchron": 24871, "\u0120Chemical": 24872, "\u0120jugg": 24873, "pair": 24874, "posts": 24875, "Eye": 24876, "\u0120Inner": 24877, "\u0120semester": 24878, "ottest": 24879, "\u0120Emirates": 24880, "ricanes": 24881, "orously": 24882, "mits": 24883, "\u0120Wis": 24884, "\u0120dodge": 24885, "location": 24886, "\u0120faded": 24887, "Amazon": 24888, "\u0120Proceed": 24889, "\u0120INFO": 24890, "journal": 24891, "\u0120Truck": 24892, "Ten": 24893, "\u0120217": 24894, "\u0120statutes": 24895, "mobile": 24896, "\u0120Types": 24897, "Recomm": 24898, "buster": 24899, "pex": 24900, "\u0120legends": 24901, "\u0120headache": 24902, "faced": 24903, "\u0120WiFi": 24904, "ifty": 24905, "\u0120HER": 24906, "\u0120circuits": 24907, "ERROR": 24908, "226": 24909, "olin": 24910, "\u0120cylinder": 24911, "ospace": 24912, "ikers": 24913, "Prem": 24914, "Quant": 24915, "\u0120conflicting": 24916, "\u0120slightest": 24917, "\u0120forged": 24918, "ionage": 24919, "Stephen": 24920, "\u0120Kub": 24921, "\u0120Opportun": 24922, "\u0120Heal": 24923, "\u0120blo": 24924, "\u0120rulers": 24925, "\u0120huh": 24926, "\u0120submarine": 24927, "fy": 24928, "asser": 24929, "\u0120allowance": 24930, "\u0120Kasich": 24931, "\u0120Tas": 24932, "\u0120Australians": 24933, "ForgeModLoader": 24934, "\u0120\u00e2\u0128\u0133": 24935, "\u0120Matrix": 24936, "amins": 24937, "\u01201200": 24938, "\u0120Acqu": 24939, "236": 24940, "Document": 24941, "\u0120Breaking": 24942, "193": 24943, "\u0120Subst": 24944, "\u0120Roller": 24945, "\u0120Properties": 24946, "\u0120NI": 24947, "tier": 24948, "\u0120crushing": 24949, "\u0120advocating": 24950, "Furthermore": 24951, "keepers": 24952, "\u0120sexism": 24953, "xd": 24954, "\u0120caller": 24955, "\u0120Sense": 24956, "chieve": 24957, "\u0120TF": 24958, "\u0120fueled": 24959, "\u0120reminiscent": 24960, "\u0120obsess": 24961, "urst": 24962, "\u0120uphold": 24963, "\u0120Fans": 24964, "hetics": 24965, "\u0120\u00e2\u0139": 24966, "\u0120Bath": 24967, "\u0120beverage": 24968, "\u0120oscill": 24969, "254": 24970, "\u0120poles": 24971, "\u0120gradual": 24972, "\u0120exting": 24973, "\u0120Suff": 24974, "\u0120Suddenly": 24975, "\u0120liking": 24976, "\u01201949": 24977, "unciation": 24978, "amination": 24979, "\u0120Omar": 24980, "\u0120LV": 24981, "\u0120Consequently": 24982, "\u0120synthes": 24983, "\u0120GIF": 24984, "\u0120pains": 24985, "\u0120interacting": 24986, "uously": 24987, "incre": 24988, "\u0120rumor": 24989, "\u0120Scientology": 24990, "197": 24991, "\u0120Zig": 24992, "\u0120spelling": 24993, "\u0120ASS": 24994, "\u0120extingu": 24995, "mson": 24996, "\u0120gh": 24997, "\u0120remarked": 24998, "\u0120Strategic": 24999, "\u0120MON": 25000, "\u00e5\u00a5": 25001, "gae": 25002, "\u0120WHAT": 25003, "Eric": 25004, "\u0120Campus": 25005, "\u0120methane": 25006, "\u0120imagin": 25007, "JUST": 25008, "\u0120Alm": 25009, "XT": 25010, "iq": 25011, "\u0120RSS": 25012, "\u0120wrongdoing": 25013, "atta": 25014, "\u0120bigot": 25015, "\u0120demonstrators": 25016, "\u0120Calvin": 25017, "\u0120Villa": 25018, "\u0120membrane": 25019, "\u0120Awesome": 25020, "\u0120benefic": 25021, "268": 25022, "\u0120magnificent": 25023, "\u0120Lots": 25024, "Greg": 25025, "\u0120Boris": 25026, "\u0120detainees": 25027, "\u0120Herman": 25028, "\u0120whispered": 25029, "\u0120awe": 25030, "Professor": 25031, "funding": 25032, "\u0120physiological": 25033, "\u0120Destruction": 25034, "\u0120limb": 25035, "\u0120manipulated": 25036, "\u0120bubbles": 25037, "\u0120pseud": 25038, "\u0120hydra": 25039, "\u0120Bristol": 25040, "\u0120stellar": 25041, "\u0120Expansion": 25042, "\u0120Kell": 25043, "\u0120Interestingly": 25044, "\u0120mans": 25045, "\u0120dragging": 25046, "\u0120ecological": 25047, "\u0120Fit": 25048, "\u0120gent": 25049, "\u0120benefited": 25050, "\u0120Haiti": 25051, "\u0120polyg": 25052, "\u00e3\u0125\u0130": 25053, "\u01202030": 25054, "\u0120prow": 25055, "\u0120reconstruction": 25056, "\u0120wast": 25057, "\u0120psychic": 25058, "\u0120Greeks": 25059, "Handler": 25060, "162": 25061, "\u0120Pulse": 25062, "\u0120solicit": 25063, "\u0120sys": 25064, "\u0120influx": 25065, "\u0120Gentle": 25066, "percent": 25067, "\u0120proliferation": 25068, "\u0120taxable": 25069, "\u0120disregard": 25070, "\u0120escaping": 25071, "\u0120ginger": 25072, "\u0120withstand": 25073, "\u0120devastated": 25074, "\u0120Dew": 25075, "series": 25076, "\u0120injected": 25077, "elaide": 25078, "\u0120turnover": 25079, "heat": 25080, "\u013b\u0124": 25081, "Happy": 25082, "\u0120Silent": 25083, "\u00e3\u0124\u0143": 25084, "ivism": 25085, "\u0120irrational": 25086, "AMA": 25087, "\u0120reef": 25088, "rub": 25089, "\u0120162": 25090, "\u0120bankers": 25091, "\u0120Ethics": 25092, "vv": 25093, "\u0120criticisms": 25094, "Kn": 25095, "186": 25096, "Movie": 25097, "\u0120Tories": 25098, "\u0120nood": 25099, "\u0120distortion": 25100, "False": 25101, "odore": 25102, "\u0120tasty": 25103, "Research": 25104, "\u0120UID": 25105, "-)": 25106, "\u0120divorced": 25107, "\u0120MU": 25108, "\u0120Hayes": 25109, "\u0120Isn": 25110, "iani": 25111, "\u0120HQ": 25112, "\u0120\"#": 25113, "ignant": 25114, "\u0120traumatic": 25115, "\u0120Ling": 25116, "Hun": 25117, "\u0120sabot": 25118, "online": 25119, "random": 25120, "\u0120renamed": 25121, "rared": 25122, "KA": 25123, "dead": 25124, "\u00c3\u00a9t": 25125, "\u0120Assistance": 25126, "\u0120seaf": 25127, "++++++++": 25128, "\u0120seldom": 25129, "\u0120Webb": 25130, "\u0120boolean": 25131, "ulet": 25132, "\u0120refrain": 25133, "\u0120DIY": 25134, "rule": 25135, "\u0120shutting": 25136, "\u0120utilizing": 25137, "loading": 25138, "\u0120Param": 25139, "coal": 25140, "ooter": 25141, "\u0120attracting": 25142, "\u0120Dol": 25143, "\u0120hers": 25144, "agnetic": 25145, "\u0120Reach": 25146, "imo": 25147, "\u0120discarded": 25148, "\u0120Pip": 25149, "015": 25150, "\u00c3\u00bcr": 25151, "\u0120mug": 25152, "Imagine": 25153, "COL": 25154, "\u0120cursed": 25155, "\u0120Shows": 25156, "\u0120Curtis": 25157, "\u0120Sachs": 25158, "speaking": 25159, "\u0120Vista": 25160, "\u0120Framework": 25161, "ongo": 25162, "\u0120subreddit": 25163, "\u0120crus": 25164, "\u0120Oval": 25165, "Row": 25166, "growing": 25167, "\u0120installment": 25168, "\u0120glac": 25169, "\u0120Advance": 25170, "ECK": 25171, "\u0120LGBTQ": 25172, "LEY": 25173, "\u0120acet": 25174, "\u0120successive": 25175, "\u0120Nicole": 25176, "\u01201957": 25177, "Quote": 25178, "\u0120circumstance": 25179, "ackets": 25180, "\u0120142": 25181, "ortium": 25182, "\u0120guessed": 25183, "\u0120Frame": 25184, "\u0120perpetrators": 25185, "\u0120Aviation": 25186, "\u0120Bench": 25187, "\u0120handc": 25188, "Ap": 25189, "\u01201956": 25190, "259": 25191, "rand": 25192, "NetMessage": 25193, "din": 25194, "urtles": 25195, "hig": 25196, "\u0120VIII": 25197, "ffiti": 25198, "\u0120Swords": 25199, "bial": 25200, "\u0120kidnapping": 25201, "device": 25202, "\u0120barn": 25203, "\u0120Eli": 25204, "aucas": 25205, "Send": 25206, "Constructed": 25207, "\u0120\u00c2\u00bd": 25208, "\u0120needles": 25209, "\u0120advertisements": 25210, "\u0120vou": 25211, "\u0120exhibited": 25212, "\u0120Fortress": 25213, "Ask": 25214, "Berry": 25215, "TYPE": 25216, "\u0120cancers": 25217, "umping": 25218, "\u0120Territory": 25219, "\u0120prud": 25220, "\u0120nas": 25221, "\u0120atheist": 25222, "\u0120balances": 25223, "\u00e3\u0123\u0141": 25224, "\u0120Shawn": 25225, "&&": 25226, "\u0120landsc": 25227, "\u0120RGB": 25228, "\u0120petty": 25229, "\u0120excellence": 25230, "\u0120translations": 25231, "\u0120parcel": 25232, "\u0120Chev": 25233, "East": 25234, "\u0120Output": 25235, "imi": 25236, "\u0120ambient": 25237, "\u0120Threat": 25238, "\u0120villains": 25239, "\u0120550": 25240, "ICA": 25241, "\u0120taller": 25242, "\u0120leaking": 25243, "cup": 25244, "\u0120polish": 25245, "\u0120infectious": 25246, "\u0120KC": 25247, "\u0120@@": 25248, "background": 25249, "\u0120bureaucracy": 25250, "\u0120Sai": 25251, "unless": 25252, "itious": 25253, "\u0120Skype": 25254, "Atl": 25255, "IDENT": 25256, "008": 25257, "\u0120hypocr": 25258, "\u0120pitchers": 25259, "\u0120guessing": 25260, "\u0120FINAL": 25261, "Between": 25262, "\u0120villagers": 25263, "\u0120252": 25264, "fashion": 25265, "\u0120Tunis": 25266, "Beh": 25267, "\u0120Exc": 25268, "\u0120MID": 25269, "288": 25270, "\u0120Haskell": 25271, "196": 25272, "\u0120NOR": 25273, "\u0120specs": 25274, "\u0120invari": 25275, "\u0120glut": 25276, "\u0120Cars": 25277, "\u0120impulse": 25278, "\u0120honors": 25279, "gel": 25280, "\u0120jurisdictions": 25281, "\u0120Bundle": 25282, "ulas": 25283, "California": 25284, "\u0120Increase": 25285, "\u0120pear": 25286, "\u0120singles": 25287, "\u0120cues": 25288, "\u0120underwent": 25289, "\u0120WS": 25290, "\u0120exaggerated": 25291, "\u0120dubious": 25292, "\u0120flashing": 25293, "LOG": 25294, ")].": 25295, "Journal": 25296, "tg": 25297, "Van": 25298, "\u0120Istanbul": 25299, "\u0120Insp": 25300, "\u0120Franken": 25301, "Draw": 25302, "\u0120sadness": 25303, "\u0120ironic": 25304, "\u0120Fry": 25305, "xc": 25306, "\u0120164": 25307, "isch": 25308, "Way": 25309, "\u0120Protestant": 25310, "horn": 25311, "\u0120unaff": 25312, "\u0120Viv": 25313, "illas": 25314, "\u0120Productions": 25315, "\u0120Hogan": 25316, "\u0120perimeter": 25317, "\u0120Sisters": 25318, "\u0120spontaneous": 25319, "\u0120downside": 25320, "\u0120descendants": 25321, "\u0120orn": 25322, "worm": 25323, "Japanese": 25324, "\u01201955": 25325, "\u0120151": 25326, "\u0120Doing": 25327, "elsen": 25328, "umbles": 25329, "\u0120radically": 25330, "\u0120Drum": 25331, "\u0120Bach": 25332, "\u0120liabilities": 25333, "\u0120OB": 25334, "\u0120Elementary": 25335, "\u0120meme": 25336, "ynes": 25337, "\u0120fingerprint": 25338, "\u0120Grab": 25339, "\u0120undertake": 25340, "Members": 25341, "\u0120Reader": 25342, "\u0120Sims": 25343, "god": 25344, "\u0120hypothetical": 25345, "scient": 25346, "\u0120AJ": 25347, "\u0120charism": 25348, "\u0120admissions": 25349, "\u0120Missile": 25350, "trade": 25351, "\u0120exercising": 25352, "\u0120Background": 25353, "Written": 25354, "\u0120vocals": 25355, "whether": 25356, "\u0120vi": 25357, "\u0120Winner": 25358, "\u0120litter": 25359, "\u0120Shooting": 25360, "STEM": 25361, "\u00e3\u0124\u00a1": 25362, "\u0120AFL": 25363, "\u0120variability": 25364, "\u0120eats": 25365, "\u0120DPS": 25366, "brow": 25367, "\u0120elephants": 25368, "\u0120strat": 25369, "\u0120\u00c5": 25370, "\u0120settlers": 25371, "Matthew": 25372, "\u0120inadvert": 25373, "HI": 25374, "\u0120IMF": 25375, "\u0120Goal": 25376, "\u0120nerves": 25377, "Johnson": 25378, "eye": 25379, "ablishment": 25380, "Thursday": 25381, "BILITY": 25382, "Had": 25383, "amoto": 25384, "hetamine": 25385, "eps": 25386, "\u0120mitochond": 25387, "\u0120compressed": 25388, "\u0120Trevor": 25389, "\u0120Animals": 25390, "Tool": 25391, "Lock": 25392, "\u0120tweak": 25393, "\u0120pinch": 25394, "\u0120cancellation": 25395, "Pot": 25396, "\u0120focal": 25397, "\u0120Astron": 25398, "173": 25399, "\u0120ASC": 25400, "\u0120OTHER": 25401, "umni": 25402, "\u0120demise": 25403, "dl": 25404, "\u00d9\u0127": 25405, "Semitism": 25406, "\u0120cracking": 25407, "\u0120collaborative": 25408, "\u0120explores": 25409, "sql": 25410, "\u0120herbs": 25411, "\u0120configurations": 25412, "mis": 25413, "\u0120Result": 25414, "acey": 25415, "\u0120Smoke": 25416, "\u0120sanct": 25417, "elia": 25418, "\u0120degener": 25419, "\u0120deepest": 25420, "\u0120screamed": 25421, "\u0120nap": 25422, "Software": 25423, "\u0120STAR": 25424, "EF": 25425, "\u0120Xin": 25426, "sponsored": 25427, "manship": 25428, "233": 25429, "\u0120primaries": 25430, "\u0120filtering": 25431, "\u0120assemble": 25432, "mil": 25433, "\u0120Myers": 25434, "bows": 25435, "\u0120punched": 25436, "Mic": 25437, "\u0120innovations": 25438, "\u0120func": 25439, "ando": 25440, "\u0120fracking": 25441, "\u0120Vul": 25442, "\u00d0\u00be\u00d0": 25443, "oshop": 25444, "\u0120Immun": 25445, "\u0120settling": 25446, "\u0120adolescents": 25447, "\u0120rebuilding": 25448, "\u0120transforming": 25449, "\u0120parole": 25450, "\u0120harbor": 25451, "\u0120booking": 25452, "otional": 25453, "ongevity": 25454, "\u0120Yo": 25455, "bug": 25456, "\u0120emerges": 25457, "\u0120Methods": 25458, "\u0120Chu": 25459, "Pres": 25460, "\u0120Dungeons": 25461, "\u0120trailing": 25462, "\u0120Rum": 25463, "\u0120Hugh": 25464, "\u00e5\u00a4\u00a9": 25465, "\u0120Era": 25466, "\u0120Battles": 25467, "Results": 25468, "\u0120Trading": 25469, "\u0120versa": 25470, "css": 25471, "axies": 25472, "heet": 25473, "\u0120greed": 25474, "1989": 25475, "\u0120gardens": 25476, "\u0120contingent": 25477, "Park": 25478, "\u0120Leafs": 25479, "hook": 25480, "robe": 25481, "\u0120diplomacy": 25482, "\u0120Fuel": 25483, "\u0120Invasion": 25484, "\u0120upgrading": 25485, "Male": 25486, "\u0120elic": 25487, "\u0120relentless": 25488, "\u0120Covenant": 25489, "apesh": 25490, "\u0120Trop": 25491, "Ty": 25492, "production": 25493, "arty": 25494, "\u0120punches": 25495, "ako": 25496, "cyclopedia": 25497, "\u0120Rabbit": 25498, "\u0120HDMI": 25499, "\u0120141": 25500, "\u0120foil": 25501, "ItemImage": 25502, "\u0120FG": 25503, "\u0120implementations": 25504, "\u0120Pom": 25505, "ixtures": 25506, "\u0120await": 25507, "\u0120330": 25508, "amus": 25509, "\u0120umbrella": 25510, "\u0120foresee": 25511, "separ": 25512, "\u0120circumcision": 25513, "\u0120peripheral": 25514, "Say": 25515, "\u0120Expert": 25516, "Inc": 25517, "\u0120withdrew": 25518, "\u0120Anders": 25519, "fried": 25520, "\u0120radioactive": 25521, "\u0120Opening": 25522, "\u0120boarding": 25523, "\u0120ND": 25524, "\u0120overthrow": 25525, "Activ": 25526, "WP": 25527, "\u0120Acts": 25528, "\u00d7\u013b": 25529, "\u0120motions": 25530, "vic": 25531, "\u0120Mighty": 25532, "\u0120Defender": 25533, "aer": 25534, "\u0120thankful": 25535, "\u0120Killing": 25536, "\u0120Bris": 25537, "moil": 25538, "\u0120predicting": 25539, "266": 25540, "choice": 25541, "\u0120killers": 25542, "\u0120incub": 25543, "\u0120Chest": 25544, "athering": 25545, "\u0120proclaimed": 25546, "flower": 25547, "ossom": 25548, "umbledore": 25549, "\u0120Cycling": 25550, "\u0120Occupy": 25551, "AGES": 25552, "Pen": 25553, "\u0120Yug": 25554, "\u0120packaged": 25555, "\u0120heightened": 25556, "cot": 25557, "stack": 25558, "Cond": 25559, "\u0120stamps": 25560, "mage": 25561, "\u0120persuaded": 25562, "\u0120ensl": 25563, "\u0120Cardinal": 25564, "\u0120solitary": 25565, "\u0120possessing": 25566, "\u0120Cork": 25567, "\u0120evid": 25568, "\u0120Tay": 25569, "\u0120blues": 25570, "\u0120extremism": 25571, "\u0120lunar": 25572, "\u0120clown": 25573, "Techn": 25574, "\u0120festivals": 25575, "\u0120PvP": 25576, "\u0120Lar": 25577, "\u0120consequently": 25578, "present": 25579, "\u0120someday": 25580, "\u00e7\u0130\u012d": 25581, "\u0120Meteor": 25582, "\u0120touring": 25583, "culture": 25584, "\u0120beaches": 25585, "Ship": 25586, "cause": 25587, "\u0120Flood": 25588, "\u00e3\u0125\u00af": 25589, "\u0120purity": 25590, "those": 25591, "\u0120emission": 25592, "bolt": 25593, "\u0120chord": 25594, "\u0120Scripture": 25595, "Lu": 25596, "\u0120${": 25597, "created": 25598, "Others": 25599, "258": 25600, "\u0120elemental": 25601, "\u0120annoyed": 25602, "\u0120AE": 25603, "dan": 25604, "\u0120Sag": 25605, "Researchers": 25606, "\u0120fairy": 25607, "\u00e2\u0122\u0135\u00e2\u0122\u0135": 25608, "============": 25609, "Smart": 25610, "GGGG": 25611, "\u0120skeletons": 25612, "\u0120pupils": 25613, "linked": 25614, "\u0120urgency": 25615, "enabled": 25616, "\u0120Fuck": 25617, "\u0120councill": 25618, "rab": 25619, "UAL": 25620, "TI": 25621, "\u0120lifes": 25622, "\u0120confessed": 25623, "Bug": 25624, "\u0120harmon": 25625, "\u0120CONFIG": 25626, "\u0120Neutral": 25627, "Double": 25628, "\u0120staple": 25629, "\u0120SHA": 25630, "British": 25631, "\u0120SNP": 25632, "ATOR": 25633, "oco": 25634, "\u0120swinging": 25635, "gex": 25636, "oleon": 25637, "plain": 25638, "\u0120Missing": 25639, "\u0120Trophy": 25640, "vari": 25641, "ranch": 25642, "\u0120301": 25643, "440": 25644, "0000000000000000": 25645, "\u0120restoring": 25646, "\u0120haul": 25647, "ucing": 25648, "nerg": 25649, "\u0120futures": 25650, "\u0120strategist": 25651, "question": 25652, "\u0120lateral": 25653, "\u0120Bard": 25654, "\u0120sor": 25655, "\u0120Rhodes": 25656, "\u0120Downtown": 25657, "?????-": 25658, "\u0120Lit": 25659, "\u0120Bened": 25660, "\u0120coil": 25661, "street": 25662, "\u0120Portal": 25663, "FILE": 25664, "\u0120Gru": 25665, "*,": 25666, "231": 25667, "neum": 25668, "\u0120sucked": 25669, "\u0120rapper": 25670, "\u0120tendencies": 25671, "\u0120Lauren": 25672, "cellaneous": 25673, "267": 25674, "\u0120browse": 25675, "\u0120overc": 25676, "header": 25677, "oise": 25678, "\u0120beet": 25679, "\u0120Gle": 25680, "Stay": 25681, "\u0120mum": 25682, "\u0120typed": 25683, "\u0120discounts": 25684, "Talk": 25685, "\u0120Og": 25686, "existing": 25687, "\u0120Sell": 25688, "uph": 25689, "CI": 25690, "\u0120Austrian": 25691, "\u0120Warm": 25692, "\u0120dismissal": 25693, "\u0120averages": 25694, "camera": 25695, "\u0120allegiance": 25696, "LAN": 25697, "=\"#": 25698, "\u0120commentators": 25699, "\u0120Setting": 25700, "\u0120Midwest": 25701, "\u0120pharmac": 25702, "\u0120EXP": 25703, "\u0120stainless": 25704, "Chicago": 25705, "\u0120tan": 25706, "244": 25707, "\u0120countryside": 25708, "\u0120Vac": 25709, "295": 25710, "\u0120pinned": 25711, "\u0120crises": 25712, "\u0120standardized": 25713, "Task": 25714, "\u0120Jail": 25715, "\u0120Docker": 25716, "colored": 25717, "forth": 25718, "\"},": 25719, "\u0120patrons": 25720, "\u0120spice": 25721, "\u0120mourn": 25722, "\u0120Mood": 25723, "\u0120laundry": 25724, "\u0120equip": 25725, "\u0120Mole": 25726, "yll": 25727, "\u0120THC": 25728, "nation": 25729, "\u0120Sherlock": 25730, "\u0120issu": 25731, "\u0120Kre": 25732, "\u0120Americas": 25733, "\u0120AAA": 25734, "\u0120systematically": 25735, "\u0120contra": 25736, "\u0120Sally": 25737, "\u0120rationale": 25738, "\u0120carriage": 25739, "\u0120peaks": 25740, "\u0120contradiction": 25741, "ensation": 25742, "\u0120Failure": 25743, "\u0120props": 25744, "\u0120namespace": 25745, "\u0120cove": 25746, "fields": 25747, "\u00e3\u0124\u012d": 25748, "\u0120wool": 25749, "\u0120Catch": 25750, "\u0120presumed": 25751, "\u0120Diana": 25752, "ragon": 25753, "igi": 25754, "\u0120hamm": 25755, "\u0120stunt": 25756, "\u0120GUI": 25757, "\u0120Observatory": 25758, "\u0120Shore": 25759, "\u0120smells": 25760, "annah": 25761, "\u0120cockpit": 25762, "\u0120Duterte": 25763, "850": 25764, "\u0120oppressed": 25765, "breaker": 25766, "\u0120Contribut": 25767, "\u0120Peru": 25768, "\u0120Monsanto": 25769, "\u0120Attempt": 25770, "\u0120commanding": 25771, "\u0120fridge": 25772, "\u0120Rin": 25773, "\u0120Chess": 25774, "uality": 25775, "\u0120ol": 25776, "Republican": 25777, "\u0120Glory": 25778, "\u0120WIN": 25779, ".......": 25780, "agent": 25781, "reading": 25782, "\u0120inh": 25783, "Jones": 25784, "\u0120clicks": 25785, "alan": 25786, "\u0120[];": 25787, "\u0120Majesty": 25788, "\u0120Ced": 25789, "opus": 25790, "atel": 25791, "\u00c3\u00aa": 25792, "ARC": 25793, "\u0120Ecuador": 25794, "\u00e3\u0125\u0142": 25795, "\u0120Kuro": 25796, "\u0120rituals": 25797, "\u0120captive": 25798, "\u0120ounce": 25799, "\u0120disagreement": 25800, "\u0120slog": 25801, "fuel": 25802, "Pet": 25803, "Mail": 25804, "\u0120exercised": 25805, "\u0120solic": 25806, "\u0120rainfall": 25807, "\u0120devotion": 25808, "\u0120Assessment": 25809, "\u0120robotic": 25810, "options": 25811, "\u0120RP": 25812, "\u0120Families": 25813, "\u0120Flames": 25814, "\u0120assignments": 25815, "007": 25816, "akedown": 25817, "\u0120vocabulary": 25818, "Reilly": 25819, "\u0120caval": 25820, "gars": 25821, "\u0120suppressed": 25822, "\u0120SET": 25823, "\u0120Johns": 25824, "\u0120warp": 25825, "broken": 25826, "\u0120statues": 25827, "\u0120advocated": 25828, "\u0120275": 25829, "\u0120peril": 25830, "omorph": 25831, "\u0120Femin": 25832, "perfect": 25833, "\u0120hatch": 25834, "Lib": 25835, "512": 25836, "\u0120lifelong": 25837, "313": 25838, "\u0120cheeks": 25839, "\u0120numbered": 25840, "\u0120Mug": 25841, "Body": 25842, "ravel": 25843, "Weight": 25844, "\u0120Jak": 25845, "\u0120Heath": 25846, "\u0120kissing": 25847, "\u0120JUST": 25848, "\u0120waving": 25849, "upload": 25850, "\u0120insider": 25851, "\u0120Progressive": 25852, "\u0120Filter": 25853, "tta": 25854, "\u0120Beam": 25855, "\u0120violently": 25856, "ipation": 25857, "\u0120skepticism": 25858, "\u01201918": 25859, "\u0120Annie": 25860, "\u0120SI": 25861, "\u0120genetics": 25862, "\u0120onboard": 25863, "atl": 25864, "\u0120Friedman": 25865, "\u0120Bri": 25866, "ceptive": 25867, "\u0120pirate": 25868, "\u0120Reporter": 25869, "278": 25870, "\u0120mythology": 25871, "\u0120eclipse": 25872, "\u0120skins": 25873, "\u0120glyph": 25874, "ingham": 25875, "Files": 25876, "Cour": 25877, "women": 25878, "\u0120regimes": 25879, "\u0120photographed": 25880, "Kat": 25881, "\u0120MAX": 25882, "Officials": 25883, "\u0120unexpectedly": 25884, "\u0120impressions": 25885, "Front": 25886, ";;;;;;;;": 25887, "\u0120supremacy": 25888, "\u0120sang": 25889, "\u0120aggravated": 25890, "\u0120abruptly": 25891, "\u0120Sector": 25892, "\u0120excuses": 25893, "\u0120costing": 25894, "idepress": 25895, "Stack": 25896, "\u0120RNA": 25897, "obil": 25898, "\u0120ghosts": 25899, "ldon": 25900, "atibility": 25901, "Topics": 25902, "\u0120reimburse": 25903, "\u0120HM": 25904, "\u0120Deg": 25905, "\u0120thief": 25906, "yet": 25907, "ogenesis": 25908, "leaning": 25909, "\u0120Kol": 25910, "\u0120Basketball": 25911, "\u0120fi": 25912, "\u0120Seeing": 25913, "\u0120recycling": 25914, "\u0120[-": 25915, "Congress": 25916, "\u0120lectures": 25917, "Psy": 25918, "\u0120nep": 25919, "\u0120maid": 25920, "\u0120oriented": 25921, "AX": 25922, "\u0120respectful": 25923, "rene": 25924, "flush": 25925, "\u0120Unloaded": 25926, "request": 25927, "grid": 25928, "\u0120Alternatively": 25929, "\u0120Hugo": 25930, "\u0120decree": 25931, "\u0120Buddhism": 25932, "andum": 25933, "Android": 25934, "\u0120Congo": 25935, "\u0120Joyce": 25936, "\u0120acknowledging": 25937, "hesive": 25938, "\u0120Tomorrow": 25939, "\u0120Hiro": 25940, "thren": 25941, "\u0120Maced": 25942, "\u0120hoax": 25943, "\u0120Increased": 25944, "\u0120Pradesh": 25945, "Wild": 25946, "______": 25947, "161": 25948, "\u0120aunt": 25949, "\u0120distributing": 25950, "\u0120Tucker": 25951, "\u0120SSL": 25952, "\u0120Wolves": 25953, "Building": 25954, "oult": 25955, "\u0120Luo": 25956, "\u0120Yas": 25957, "\u0120Spir": 25958, "\u0120Shape": 25959, "\u0120Cambod": 25960, "\u0120IPv": 25961, "\u0120ml": 25962, "\u0120extrad": 25963, "390": 25964, "\u0120Penny": 25965, "dream": 25966, "\u0120stationed": 25967, "optional": 25968, "eworthy": 25969, ".": 26700, "\u0120Workshop": 26701, "\u0120Retail": 26702, "\u0120Avatar": 26703, "625": 26704, "Na": 26705, "\u0120VC": 26706, "\u0120Secure": 26707, "MY": 26708, "1988": 26709, "ossip": 26710, "\u0120prostate": 26711, "\u0120unden": 26712, "\u0120gamer": 26713, "\u0120Contents": 26714, "\u0120Warhammer": 26715, "\u0120Sentinel": 26716, "310": 26717, "\u0120segregation": 26718, "\u0120Flex": 26719, "\u0120MAY": 26720, "\u0120drills": 26721, "\u0120Drugs": 26722, "Islamic": 26723, "\u0120spur": 26724, "\u0120cafe": 26725, "\u0120imaginary": 26726, "\u0120guiding": 26727, "\u0120swings": 26728, "\u0120Theme": 26729, "oby": 26730, "\u0120nud": 26731, "\u0120begging": 26732, "\u0120strongh": 26733, "\u0120rejecting": 26734, "\u0120pedestrians": 26735, "\u0120Prospect": 26736, "Rare": 26737, "sle": 26738, "\u0120concessions": 26739, "\u0120Constitutional": 26740, "\u0120beams": 26741, "\u0120fibers": 26742, "poon": 26743, "\u0120instincts": 26744, "property": 26745, "\u0120BIG": 26746, "Sanders": 26747, "imates": 26748, "\u0120coating": 26749, "\u0120corpses": 26750, "\u0120TRUE": 26751, "checked": 26752, "\u0120166": 26753, "Ash": 26754, "\u0120JS": 26755, "\u0120Fiction": 26756, "\u0120communal": 26757, "\u0120energetic": 26758, "oooooooo": 26759, "\u0120nowadays": 26760, "ILD": 26761, "ibo": 26762, "\u0120SUV": 26763, "Ren": 26764, "\u0120dwelling": 26765, "Silver": 26766, "\u0120tally": 26767, "\u0120Moving": 26768, "\u0120coward": 26769, "\u0120generals": 26770, "\u0120horns": 26771, "\u0120circulated": 26772, "\u0120robbed": 26773, "\u0120Unlimited": 26774, "\u0120harassed": 26775, "\u0120inhibit": 26776, "\u0120composer": 26777, "\u0120Spotify": 26778, "\u0120spreads": 26779, "364": 26780, "\u0120suicidal": 26781, "\u0120noises": 26782, "\u0120Stur": 26783, "\u0120saga": 26784, "\u0120Kag": 26785, "iso": 26786, "\u0120theoretically": 26787, "Money": 26788, "\u0120similarity": 26789, "\u0120sliced": 26790, "utils": 26791, "inges": 26792, "\"-": 26793, "\u0120anth": 26794, "\u0120imped": 26795, "Module": 26796, "Throughout": 26797, "\u0120menus": 26798, "committee": 26799, "andi": 26800, "obj": 26801, "inav": 26802, "fired": 26803, "\u0120Abdullah": 26804, "\u0120undead": 26805, "\u0120fonts": 26806, "Hold": 26807, "ENG": 26808, "\u0120sustainability": 26809, "\u0120flick": 26810, "\u0120razor": 26811, "\u0120Fest": 26812, "\u0120Characters": 26813, "\u0120wording": 26814, "\u0120populist": 26815, "\u0120criticizing": 26816, "\u0120muse": 26817, "vine": 26818, "\u0120cardboard": 26819, "\u0120kindly": 26820, "\u0120fringe": 26821, "\u0120Theft": 26822, "icultural": 26823, "\u0120governors": 26824, "\u0120\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 26825, "\u0120163": 26826, "\u0120timeout": 26827, "\u0120Auth": 26828, "Children": 26829, "AU": 26830, "\u0120redemption": 26831, "\u0120Alger": 26832, "\u01201914": 26833, "\u0120waved": 26834, "\u0120astronauts": 26835, "ograms": 26836, "\u0120swamp": 26837, "\u0120Finnish": 26838, "\u0120candle": 26839, "\u0120tonnes": 26840, "utm": 26841, "\u0120ray": 26842, "\u0120spun": 26843, "\u0120fearful": 26844, "articles": 26845, "\u0120caus": 26846, "orically": 26847, "\u0120Requires": 26848, "\u0120Gol": 26849, "\u0120pope": 26850, "\u0120inaugural": 26851, "\u0120gle": 26852, "ADA": 26853, "\u0120ISIL": 26854, "\u0120Offensive": 26855, "\u0120watchdog": 26856, "\u0120balcon": 26857, "entity": 26858, "\u0120Hoo": 26859, "\u0120gallon": 26860, "ACC": 26861, "\u0120doubling": 26862, "\u0120implication": 26863, "\u0120Sight": 26864, "\u0120doctr": 26865, "-------": 26866, "\u0120\\\\": 26867, "\u0120malt": 26868, "Roll": 26869, "\u0120\u00e2\u012b\u00a5": 26870, "\u0120recap": 26871, "adding": 26872, "uces": 26873, "\u0120Bend": 26874, "figure": 26875, "\u0120turkey": 26876, "\u0120societal": 26877, "\u0120Tickets": 26878, "\u0120commercially": 26879, "\u0120spicy": 26880, "\u0120216": 26881, "\u0120Ramp": 26882, "\u0120superiority": 26883, "\u00c3\u00af": 26884, "\u0120Tracker": 26885, "Carl": 26886, "\u0120Coy": 26887, "\u0120Patriot": 26888, "\u0120consulted": 26889, "\u0120listings": 26890, "\u0120slew": 26891, "reenshot": 26892, "\u0120Gone": 26893, "\u0120[...]": 26894, "309": 26895, "\u0120hottest": 26896, "\u00d8\u00b1": 26897, "\u0120rocky": 26898, "\u0120Diaz": 26899, "\u0120massage": 26900, "\u0120paraly": 26901, "\u0120pony": 26902, "Az": 26903, "\u0120cartridge": 26904, "\u0120NZ": 26905, "\u0120snack": 26906, "\u0120Lamar": 26907, "plement": 26908, "\u0120Leslie": 26909, "\u0120mater": 26910, "\u0120snipp": 26911, "246": 26912, "\u0120jointly": 26913, "\u0120Brisbane": 26914, "\u0120iPod": 26915, "\u0120pumping": 26916, "\u0120goat": 26917, "\u0120Sharon": 26918, "ealing": 26919, "\u0120coron": 26920, "\u0120anomal": 26921, "rahim": 26922, "\u0120Connection": 26923, "\u0120sculpture": 26924, "\u0120scheduling": 26925, "\u0120Daddy": 26926, "athing": 26927, "\u0120eyebrows": 26928, "\u0120curved": 26929, "\u0120sentiments": 26930, "\u0120drafting": 26931, "Drop": 26932, "([": 26933, "\u0120nominal": 26934, "\u0120Leadership": 26935, "\u0120Grow": 26936, "\u0120176": 26937, "\u0120constructive": 26938, "ivation": 26939, "\u0120corrupted": 26940, "gerald": 26941, "\u0120Cros": 26942, "\u0120Chester": 26943, "\u0120Lap": 26944, "\u00e3\u0123\u00aa": 26945, "OTH": 26946, "DATA": 26947, "\u0120almond": 26948, "probably": 26949, "Imp": 26950, "\u0120feast": 26951, "\u0120Warcraft": 26952, "Flor": 26953, "\u0120checkpoint": 26954, "\u0120transcription": 26955, "\u0120204": 26956, "\u0120tweaks": 26957, "\u0120relieve": 26958, "Science": 26959, "\u0120performer": 26960, "Zone": 26961, "\u0120turmoil": 26962, "igated": 26963, "hibit": 26964, "\u0120Cafe": 26965, "themed": 26966, "\u0120fluor": 26967, "bench": 26968, "\u0120decom": 26969, "\u0120Unt": 26970, "\u0120Barrett": 26971, "\u0120Facts": 26972, "\u0120tasting": 26973, "\u0120PTSD": 26974, "\u0120Seal": 26975, "\u0120Judaism": 26976, "\u0120Dynamic": 26977, "\u0120Cors": 26978, "Ve": 26979, "\u0120Ming": 26980, "\u0120Transform": 26981, "von": 26982, "\u0120Defenders": 26983, "\u0120Tactical": 26984, "\u0120Von": 26985, "\u0120Univers": 26986, "\u0120distorted": 26987, "\u0120Breath": 26988, "?'\"": 26989, "\u0120agon": 26990, "\u0120Deadly": 26991, "\u0120lan": 26992, "\u0120Cycle": 26993, "orned": 26994, "\u0120reliably": 26995, "\u0120glor": 26996, "\u0120Monkey": 26997, "\u00e3\u0125\u00a1": 26998, "\u0120adren": 26999, "\u0120microwave": 27000, "\u0120Alban": 27001, "ircraft": 27002, "digit": 27003, "smart": 27004, "\u0120Dread": 27005, "\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af": 27006, "{{": 27007, "\u0120Rochester": 27008, "\u0120simplified": 27009, "\u0120inflicted": 27010, "\u0120takeover": 27011, "\u0120yourselves": 27012, "aditional": 27013, "\u0120muscular": 27014, "KS": 27015, "\u0120ingen": 27016, "Tax": 27017, "\u0120Feature": 27018, "277": 27019, "\u0120cruc": 27020, "\u0120crate": 27021, "\u0120unidentified": 27022, "\u0120acclaimed": 27023, "\u0120Manga": 27024, "\u0120Frances": 27025, "\u0120Nepal": 27026, "\u0120Gerald": 27027, "\u0120Kuwait": 27028, "\u0120slain": 27029, "\u0120Heb": 27030, "\u0120Goku": 27031, "\u00e3\u0123\u00ae\u00e6": 27032, "286": 27033, "Mrs": 27034, "\u0120Cody": 27035, "\u0120Sanctuary": 27036, "016": 27037, "\u0120dismant": 27038, "\u0120dataset": 27039, "\u0120Hond": 27040, "buck": 27041, "\u0120Patterson": 27042, "\u0120palette": 27043, "\u0120GD": 27044, "icol": 27045, "\u0120Lodge": 27046, "\u0120planetary": 27047, "akin": 27048, "\u0120Registered": 27049, "abwe": 27050, "\u0120Petersburg": 27051, "\u0120hailed": 27052, "\u0120Piece": 27053, "Sche": 27054, "\u0120DOJ": 27055, "\u0120enumer": 27056, "181": 27057, "\u0120Observer": 27058, "\u0120Bold": 27059, "founded": 27060, "commerce": 27061, "\u0120exploits": 27062, "\u0120Finding": 27063, "URN": 27064, "\u0120Sne": 27065, "\u0120Acid": 27066, "ayette": 27067, "\u0120Values": 27068, "\u0120drastic": 27069, "\u0120architectural": 27070, "\u0120\".": 27071, "\u00d7\u0137": 27072, "umped": 27073, "\u0120wrapping": 27074, "\u0120widow": 27075, "\u0120Slayer": 27076, "lace": 27077, "once": 27078, "Germany": 27079, "avoid": 27080, "\u0120temples": 27081, "PAR": 27082, "\u00c3\u00b4": 27083, "\u0120Lucifer": 27084, "\u0120Flickr": 27085, "lov": 27086, "forces": 27087, "\u0120scouting": 27088, "\u0120louder": 27089, "tesy": 27090, "\u0120beforehand": 27091, "\u00c4\u0135": 27092, "\u0120Neon": 27093, "\u0120Wol": 27094, "\u0120Typically": 27095, "\u0120Politico": 27096, "-+-+": 27097, "\u0120builder": 27098, "\u0120derive": 27099, "Kill": 27100, "\u0120poker": 27101, "\u0120ambiguous": 27102, "\u0120lifts": 27103, "\u0120cyt": 27104, "\u0120ribs": 27105, "oodle": 27106, "\u0120Sounds": 27107, "hair": 27108, "\u0120Syndrome": 27109, "tf": 27110, "\u0120proportional": 27111, "uid": 27112, "\u0120pertaining": 27113, "\u0120Kindle": 27114, "\u0120Negro": 27115, "\u0120reiterated": 27116, "\u0120Tonight": 27117, "oths": 27118, "\u0120Cornell": 27119, "\u0120owing": 27120, "\u0120208": 27121, "elfare": 27122, "ocating": 27123, "\u0120Birds": 27124, "Subscribe": 27125, "\u0120essays": 27126, "\u0120burdens": 27127, "\u0120illustrations": 27128, "arious": 27129, "ERAL": 27130, "\u0120Calcul": 27131, "\u0120xen": 27132, "\u0120LinkedIn": 27133, "\u0120Jung": 27134, "\u0120redesign": 27135, "Connor": 27136, "296": 27137, "\u0120reversal": 27138, "\u0120Adelaide": 27139, "\u0120LL": 27140, "\u0120sinking": 27141, "\u0120gum": 27142, "USH": 27143, "capt": 27144, "\u0120Grimm": 27145, "\u0120footsteps": 27146, "\u0120CBD": 27147, "ispers": 27148, "\u0120prose": 27149, "Wednesday": 27150, "\u0120Movies": 27151, "edin": 27152, "\u0120overturned": 27153, "\u0120contentious": 27154, "USB": 27155, "~~~~~~~~~~~~~~~~": 27156, "\u0120Copper": 27157, "\u0120pointless": 27158, "NV": 27159, "values": 27160, "olphin": 27161, "dain": 27162, "\u0120deposited": 27163, "\u0120GW": 27164, "\u0120preceded": 27165, "\u0120Cla": 27166, "\u0120Golem": 27167, "\u0120Nim": 27168, "\u0120\u00ce\u00b2": 27169, "\u0120Engineers": 27170, "middle": 27171, "\u0120flatt": 27172, "operative": 27173, "\u0120councils": 27174, "imbabwe": 27175, "elin": 27176, "\u0120stressful": 27177, "\u0120LD": 27178, "\u0120resh": 27179, "lake": 27180, "\u0120wheelchair": 27181, "\u0120Alternative": 27182, "\u0120optimize": 27183, "operation": 27184, "\u0120peek": 27185, "\u0120oneself": 27186, "igil": 27187, "\u0120transitions": 27188, "opathy": 27189, "blank": 27190, "\u0120169": 27191, "171": 27192, "________________________________________________________________": 27193, "\u0120laundering": 27194, "Enc": 27195, "\u0120DEC": 27196, "\u0120workouts": 27197, "\u0120spikes": 27198, "\u0120dinosaurs": 27199, "\u0120discriminatory": 27200, "Pool": 27201, "Rather": 27202, "385": 27203, "RNA": 27204, "testers": 27205, "eto": 27206, "\u0120Identity": 27207, "\u0120vein": 27208, "\u0120Burton": 27209, "\u0120arcade": 27210, "420": 27211, "Ultimately": 27212, "\u0120Sadly": 27213, "\u00c3\u00b0": 27214, "pill": 27215, "\u0120cubic": 27216, "\u0120Spectrum": 27217, "these": 27218, "states": 27219, "\u0120unofficial": 27220, "hawks": 27221, "\u0120EVERY": 27222, "\u0120rainbow": 27223, "\u0120incarceration": 27224, "anding": 27225, "\u0120syll": 27226, "\u0120Everton": 27227, "\u0120179": 27228, "\u0120Serbia": 27229, "\u0120189": 27230, "meter": 27231, "\u0120Mickey": 27232, "\u0120antiqu": 27233, "\u0120factual": 27234, "neck": 27235, "\u0120Nare": 27236, "norm": 27237, "must": 27238, "\u0120highways": 27239, "\u0120glam": 27240, "\u0120dividing": 27241, "\u0120Squadron": 27242, "\u0120Martha": 27243, "\u0120births": 27244, "Cover": 27245, "////////////////": 27246, "\u0120Wong": 27247, "Phot": 27248, "\u0120ALS": 27249, "rio": 27250, "\u0120Nonetheless": 27251, "\u0120Lemon": 27252, "\u0120206": 27253, "\u0120EE": 27254, "\u0120derivative": 27255, "\u0120WWII": 27256, "vote": 27257, "\u0120therein": 27258, "\u0120separating": 27259, "446": 27260, "sync": 27261, "\u0120Streets": 27262, "\u0120ratt": 27263, "\u0120municipality": 27264, "\u0120Shortly": 27265, "\u0120monk": 27266, "),\"": 27267, "\u0120scrub": 27268, "\u0120operatives": 27269, "Neither": 27270, "Place": 27271, "\u0120Limit": 27272, "Female": 27273, "\u0120Actor": 27274, "Character": 27275, "\u0120constituted": 27276, "357": 27277, "\u0120protested": 27278, "\u0120Straw": 27279, "\u0120Height": 27280, "ilda": 27281, "\u0120Typh": 27282, "\u0120floods": 27283, "\u0120cosmetic": 27284, "WAY": 27285, "perture": 27286, "upon": 27287, "tons": 27288, "essing": 27289, "\u0120Pocket": 27290, "\u0120rooft": 27291, "\u0120Caucas": 27292, "\u0120antidepress": 27293, "\u0120incompatible": 27294, "ECD": 27295, "\u0120opera": 27296, "\u0120Contest": 27297, "\u0120generators": 27298, "lime": 27299, "Defense": 27300, "1987": 27301, "forum": 27302, "\u0120savage": 27303, "\u0120Hungarian": 27304, "nz": 27305, "\u0120metallic": 27306, "\u0120expelled": 27307, "\u0120residency": 27308, "\u0120dresses": 27309, "666": 27310, "\u0120Clement": 27311, "fires": 27312, "Category": 27313, "\u0120geek": 27314, "alis": 27315, "\u0120cemetery": 27316, "educated": 27317, "\u0120crawl": 27318, "\u0120Unable": 27319, "\u0120Tyson": 27320, "akis": 27321, "\u0120pardon": 27322, "\u0120Wra": 27323, "\u0120strengthened": 27324, "\u0120Fors": 27325, "335": 27326, "\u0120HC": 27327, "\u0120Mond": 27328, "\u0120visuals": 27329, "\u0120Beatles": 27330, "ettlement": 27331, "\u0120\u00ef": 27332, "gro": 27333, "\u0120bash": 27334, "\u0120poorest": 27335, "\u0120excel": 27336, "\u0120aspirations": 27337, "\u0120Municip": 27338, "ensible": 27339, "\u0120ceremonies": 27340, "\u0120intimidation": 27341, "\u0120CONTR": 27342, "beck": 27343, "\u0120Kap": 27344, "asu": 27345, "\u0120trademarks": 27346, "\u0120Sew": 27347, "\u0120Competition": 27348, "network": 27349, "\u0120Arri": 27350, "\u0120Tet": 27351, "Roaming": 27352, "WC": 27353, "Dat": 27354, "\u0120sob": 27355, "\u0120pairing": 27356, "\u0120overdose": 27357, "SAY": 27358, "aber": 27359, "\u0120revolt": 27360, "\u0120Fah": 27361, "acting": 27362, "eq": 27363, "estation": 27364, "Fight": 27365, "\u0120Marks": 27366, "273": 27367, "\u0120178": 27368, "Raw": 27369, "\u00e3\u0123\u012d": 27370, "349": 27371, "blocks": 27372, "\u0120verge": 27373, "estine": 27374, "\u0120Podesta": 27375, "\u0120invasive": 27376, "\u0120profoundly": 27377, "\u0120Ao": 27378, "each": 27379, "\u0120lest": 27380, "interpret": 27381, "\u0120shrinking": 27382, "\u0120errone": 27383, "\u0120chees": 27384, "lys": 27385, "\u0120Ivy": 27386, "\u0120Directory": 27387, "\u0120hinted": 27388, "VICE": 27389, "\u0120contacting": 27390, "\u0120Gent": 27391, "hei": 27392, "\u0120labeling": 27393, "\u0120mercury": 27394, "\u0120Lite": 27395, "\u0120expires": 27396, "\u0120destabil": 27397, "ritis": 27398, "cu": 27399, "\u0120feathers": 27400, "\u0120steer": 27401, "\u0120programmed": 27402, "\u0120Vader": 27403, "Going": 27404, "\u0120Elim": 27405, "\u0120yo": 27406, "\u0120Miche": 27407, "\u0120203": 27408, "\u0120sleeves": 27409, "\u0120bully": 27410, "\u0120Humans": 27411, "368": 27412, "\u0120compress": 27413, "\u0120Banner": 27414, "ARS": 27415, "\u0120awhile": 27416, "\u0120calib": 27417, "\u0120sponsorship": 27418, "\u0120Difficulty": 27419, "\u0120Papers": 27420, "\u0120identifier": 27421, "}.": 27422, "\u0120yog": 27423, "\u0120Shia": 27424, "\u0120cleanup": 27425, "\u0120vibe": 27426, "introdu": 27427, "imming": 27428, "Australia": 27429, "\u0120outlines": 27430, "\u0120Youtube": 27431, "train": 27432, "\u0120Makes": 27433, "\u0120deported": 27434, "\u0120centr": 27435, "\u0120Dug": 27436, "\u0120Boulder": 27437, "\u0120Buffy": 27438, "\u0120injunction": 27439, "\u0120Harley": 27440, "\u0120Groups": 27441, "\u0120Dumbledore": 27442, "\u0120Clara": 27443, "\u0120\"-": 27444, "\u0120sacrificed": 27445, "eph": 27446, "Shadow": 27447, "ibling": 27448, "\u0120freelance": 27449, "\u0120evidently": 27450, "phal": 27451, "\u0120retains": 27452, "Mir": 27453, "\u0120finite": 27454, "dar": 27455, "\u0120Cous": 27456, "\u0120repaired": 27457, "\u0120periodic": 27458, "\u0120championships": 27459, "\u0120asteroid": 27460, "blind": 27461, "\u0120expressly": 27462, "\u0120Astros": 27463, "\u0120scaled": 27464, "\u0120geographical": 27465, "\u0120Rapids": 27466, "Enjoy": 27467, "\u0120elastic": 27468, "\u0120Mohamed": 27469, "Market": 27470, "begin": 27471, "\u0120discovers": 27472, "\u0120telecommunications": 27473, "\u0120scanner": 27474, "\u0120enlarge": 27475, "\u0120sharks": 27476, "\u0120psychedel": 27477, "\u0120Rouge": 27478, "\u0120snapshot": 27479, "isine": 27480, "XP": 27481, "\u0120pesticides": 27482, "\u0120LSD": 27483, "\u0120Distribution": 27484, "really": 27485, "\u0120degradation": 27486, "\u0120disguise": 27487, "\u0120biom": 27488, "\u0120EXT": 27489, "\u0120equations": 27490, "\u0120hazards": 27491, "\u0120Compared": 27492, ")*": 27493, "\u0120virtues": 27494, "\u0120elders": 27495, "\u0120enhancing": 27496, "\u0120Across": 27497, "eros": 27498, "angling": 27499, "\u0120combust": 27500, "ucci": 27501, "\u0120concussion": 27502, "\u0120contraception": 27503, "\u0120Kang": 27504, "\u0120expresses": 27505, "\u0120aux": 27506, "\u0120Pione": 27507, "\u0120exhibits": 27508, "Debug": 27509, "OTAL": 27510, "\u0120Already": 27511, "\u0120Wheeler": 27512, "\u0120expands": 27513, "?:": 27514, "\u0120reconciliation": 27515, "\u0120pirates": 27516, "\u0120purse": 27517, "\u0120discourage": 27518, "\u0120spectacle": 27519, "Rank": 27520, "\u0120wraps": 27521, "\u0120Thought": 27522, "\u0120impending": 27523, "Opp": 27524, "\u0120Anglo": 27525, "\u0120EUR": 27526, "\u0120screwed": 27527, "retched": 27528, "\u0120encouragement": 27529, "models": 27530, "\u0120confuse": 27531, "mmm": 27532, "\u0120Vitamin": 27533, "\u00e2\u0138\u0133\u00e2\u0138\u0133": 27534, "Cru": 27535, "\u0120knights": 27536, "\u0120discard": 27537, "\u0120bishops": 27538, "\u0120Wear": 27539, "\u0120Garrett": 27540, "kan": 27541, "\u00e3\u0125\u0141": 27542, "\u0120masculine": 27543, "capital": 27544, "\u0120Aus": 27545, "\u0120fatally": 27546, "thanks": 27547, "\u0120AU": 27548, "\u0120Gut": 27549, "1200": 27550, "\u012000000000": 27551, "\u0120surrog": 27552, "\u0120BIOS": 27553, "raits": 27554, "\u0120Watts": 27555, "\u0120resurrection": 27556, "\u0120Electoral": 27557, "\u0120Tips": 27558, "4000": 27559, "\u0120nutrient": 27560, "\u0120depicting": 27561, "\u0120sprink": 27562, "\u0120muff": 27563, "\u0120LIM": 27564, "\u0120Sample": 27565, "psc": 27566, "ibi": 27567, "generated": 27568, "\u0120specimens": 27569, "\u0120dissatisf": 27570, "\u0120tailored": 27571, "\u0120holdings": 27572, "\u0120Monthly": 27573, "\u0120Eat": 27574, "poons": 27575, "\u0120nec": 27576, "\u0120Cage": 27577, "\u0120Lotus": 27578, "\u0120Lantern": 27579, "\u0120frontier": 27580, "\u0120pensions": 27581, "\u0120joked": 27582, "\u0120Hardy": 27583, "=-=-=-=-": 27584, "rade": 27585, "UID": 27586, "\u0120rails": 27587, "\u0120emit": 27588, "\u0120slate": 27589, "\u0120smug": 27590, "\u0120spit": 27591, "\u0120Calls": 27592, "\u0120Jacobs": 27593, "feat": 27594, "\u0120UE": 27595, "\u0120restruct": 27596, "\u0120regeneration": 27597, "\u0120energies": 27598, "\u0120Connor": 27599, "OHN": 27600, "\u0120Cheese": 27601, "\u0120ger": 27602, "\u0120resurrect": 27603, "management": 27604, "NW": 27605, "\u0120presently": 27606, "\u0120Bruins": 27607, "Member": 27608, "\u0120Mang": 27609, "idan": 27610, "\u0120boosting": 27611, "wyn": 27612, "+.": 27613, "requisite": 27614, "\u0120NYPD": 27615, "\u0120Megan": 27616, "\u0120Conditions": 27617, "\u0120pics": 27618, "nesium": 27619, "\u0120Rash": 27620, "\u0120174": 27621, "\u0120Ducks": 27622, "\u0120embro": 27623, "zu": 27624, "onian": 27625, "religious": 27626, "\u0120craz": 27627, "\u0120ACA": 27628, "\u0120Zucker": 27629, "EMA": 27630, "\u0120Pros": 27631, "Weapon": 27632, "\u0120Knox": 27633, "\u0120Arduino": 27634, "\u0120stove": 27635, "\u0120heavens": 27636, "\u0120Purchase": 27637, "\u0120herd": 27638, "\u0120fundraiser": 27639, "Digital": 27640, "5000": 27641, "\u0120proponents": 27642, "/\u00e2\u0122\u012d": 27643, "\u0120jelly": 27644, "\u0120Visa": 27645, "\u0120monks": 27646, "\u0120advancement": 27647, "\u0120Wer": 27648, "\u0120187": 27649, "eus": 27650, "ertility": 27651, "\u0120fetal": 27652, "\u01201936": 27653, "Lo": 27654, "\u0120outfits": 27655, "\u0120staircase": 27656, "bomb": 27657, "\u0120customized": 27658, "clair": 27659, "Tree": 27660, "\u0120mapped": 27661, "\u0120Considering": 27662, "\u0120Torres": 27663, "\u0120methyl": 27664, "\u0120approximate": 27665, "\u0120doom": 27666, "\u0120Hansen": 27667, "\u0120crossover": 27668, "\u0120standalone": 27669, "\u00e4\u00bc": 27670, "\u0120invites": 27671, "\u0120graveyard": 27672, "\u0120hp": 27673, "DonaldTrump": 27674, "\u0120escort": 27675, "Gar": 27676, "\u0120predecessors": 27677, "\u0120hay": 27678, "\u0120enzyme": 27679, "\u0120Straight": 27680, "visors": 27681, "Ing": 27682, "aneously": 27683, "\u0120Applied": 27684, "\u0120fec": 27685, "\u0120Durant": 27686, "\u0120outspoken": 27687, "orb": 27688, "\u0120zeal": 27689, "\u0120disgrace": 27690, "').": 27691, "\u0120Cheng": 27692, "289": 27693, "\u0120Rena": 27694, "\u0120Suicide": 27695, "294": 27696, "\u0120outraged": 27697, "\u0120Newman": 27698, "\u0120Nvidia": 27699, "\u0120Aber": 27700, "\u0120Bers": 27701, "\u0120recreation": 27702, "Window": 27703, "\u0120DP": 27704, "xe": 27705, "\u0120pedoph": 27706, "\u0120fallout": 27707, "amboo": 27708, "\u0120presentations": 27709, "\u0120Apps": 27710, "\u0120html": 27711, "345": 27712, "\u0120XXX": 27713, "\u0120rubbing": 27714, "\u0120Leather": 27715, "\u0120humidity": 27716, "seys": 27717, "established": 27718, "\u0120Units": 27719, "646": 27720, "\u0120respectable": 27721, "Auto": 27722, "\u0120thriving": 27723, "\u0120Innovation": 27724, "angs": 27725, "Extra": 27726, "regulation": 27727, "298": 27728, "pick": 27729, "Examples": 27730, "\u0120CJ": 27731, "Attack": 27732, "\u0120dracon": 27733, "LT": 27734, "\u0120sticker": 27735, "rers": 27736, "\u0120sunny": 27737, "Iss": 27738, "regulated": 27739, "dim": 27740, "\u0120Abstract": 27741, "\u0120husbands": 27742, "Office": 27743, "omination": 27744, "itars": 27745, "ANGE": 27746, "ascal": 27747, "\u0120Kris": 27748, "\u0120Infantry": 27749, "\u0120malf": 27750, "\u0120Athe": 27751, "\u0120Rally": 27752, "balanced": 27753, "........................": 27754, "OUP": 27755, "\u0120molecule": 27756, "metics": 27757, "\u0120Split": 27758, "\u0120Instructions": 27759, "\u0120Nights": 27760, "cards": 27761, "\u0120tug": 27762, "\u0120cone": 27763, "\u00e5\u0143": 27764, "\u0120tx": 27765, "\u0120Discussion": 27766, "\u0120catastrophe": 27767, "ppe": 27768, "gio": 27769, "\u0120communism": 27770, "\u0120halted": 27771, "\u0120Guant": 27772, "clean": 27773, "\u0120Sched": 27774, "\u0120Kanye": 27775, "\u0120wander": 27776, "\u0120Seriously": 27777, "\u0120188": 27778, "ennial": 27779, "follow": 27780, "productive": 27781, "\u0120Flow": 27782, "\u0120Sail": 27783, "\u0120craw": 27784, "\u0120simulations": 27785, "oru": 27786, "angles": 27787, "\u0120Nolan": 27788, "\u0120menstru": 27789, "470": 27790, "\u0120207": 27791, "aja": 27792, "\u0120casually": 27793, "boarding": 27794, "\u0120222": 27795, "ovy": 27796, "\u0120Numbers": 27797, "umat": 27798, "OE": 27799, "287": 27800, "\u0120Clemson": 27801, "\u0120certs": 27802, "\u0120slid": 27803, "\u0120Tribe": 27804, "\u0120toast": 27805, "\u0120fortunes": 27806, "\u0120fals": 27807, "\u0120Committees": 27808, "\u0120gp": 27809, "\u0120fiery": 27810, "\u0120Nets": 27811, "\u0120Anime": 27812, "Package": 27813, "\u0120Compare": 27814, "laughter": 27815, "infect": 27816, "\u0120atrocities": 27817, "\u0120justices": 27818, "\u0120insults": 27819, "\u0120Vernon": 27820, "\u0120shaken": 27821, "\u0120persona": 27822, "estamp": 27823, "367": 27824, "brain": 27825, "\u0120experimenting": 27826, "Ken": 27827, "\u0120Electronics": 27828, "\u0120161": 27829, "domain": 27830, "\u0120graphical": 27831, "bishop": 27832, "\u0120whopping": 27833, "\u0120Evangel": 27834, "\u0120advertisers": 27835, "\u0120Spear": 27836, "\u0120bids": 27837, "\u0120destroys": 27838, "utz": 27839, "\u0120undersc": 27840, "\u0120ADD": 27841, "\u0120ants": 27842, "\u0120Cum": 27843, "ipples": 27844, "\u0120Fill": 27845, "\u0120glanced": 27846, "\u0120indicted": 27847, "\u0120Eff": 27848, "\u0120miscon": 27849, "\u0120Desktop": 27850, "\u0120abide": 27851, "\u00e3\u0125\u0122": 27852, "\u0120Io": 27853, "\u0120Coul": 27854, "\u0120capsule": 27855, "\u0120Chrys": 27856, "MON": 27857, "\u0120undes": 27858, "\u0120IRA": 27859, "\u0120citation": 27860, "\u0120dictate": 27861, "\u0120Networks": 27862, "\u0120Conflict": 27863, "\u0120Stuff": 27864, "xa": 27865, "isec": 27866, "\u0120Chemistry": 27867, "\u0120quarterly": 27868, "Williams": 27869, "anan": 27870, "Opt": 27871, "\u0120Alexandria": 27872, "outheastern": 27873, "\u0120Springfield": 27874, "\u0120Blacks": 27875, "\u0120geography": 27876, "242": 27877, "\u0120utmost": 27878, "\u0120Exxon": 27879, "abouts": 27880, "EVA": 27881, "\u0120Enable": 27882, "\u0120Barr": 27883, "\u0120disagreed": 27884, "\u0120Cyprus": 27885, "\u0120dementia": 27886, "\u0120labs": 27887, "\u0120ubiquitous": 27888, "\u0120LOVE": 27889, "\u0120consolidated": 27890, "sr": 27891, "\u0120creamy": 27892, "\u0120Timber": 27893, "Regardless": 27894, "\u0120Certificate": 27895, "\u0120\"...": 27896, "ogenous": 27897, "Captain": 27898, "\u0120insulting": 27899, "\u0120Soros": 27900, "\u0120Instr": 27901, "\u0120Bulgaria": 27902, "better": 27903, "\u0120sucking": 27904, "\u0120Davidson": 27905, "atz": 27906, "\u0120collateral": 27907, "gif": 27908, "\u0120plagued": 27909, "\u0120Cancel": 27910, "\u0120Gardner": 27911, "RB": 27912, "\u0120sixteen": 27913, "Remove": 27914, "uristic": 27915, "cook": 27916, "Rod": 27917, "\u0120comprising": 27918, "fle": 27919, ")\u00e2\u0122\u0136": 27920, "\u0120Viking": 27921, "growth": 27922, "agonal": 27923, "\u0120srf": 27924, "afety": 27925, "mot": 27926, "Nearly": 27927, "stown": 27928, "\u0120Factor": 27929, "\u0120automobile": 27930, "\u0120procedural": 27931, "mask": 27932, "ampires": 27933, "\u0120disappears": 27934, "jab": 27935, "315": 27936, "\u01201951": 27937, "needed": 27938, "\u0120daring": 27939, "leader": 27940, "\u0120podium": 27941, "\u0120unhealthy": 27942, "\u0120mund": 27943, "\u0120pyramid": 27944, "ocre": 27945, "\u0120kissed": 27946, "\u0120dreamed": 27947, "\u0120Fantastic": 27948, "\u0120Gly": 27949, "\u00e5\u012c": 27950, "\u0120greatness": 27951, "\u0120spices": 27952, "\u0120metropolitan": 27953, "\u0120compuls": 27954, "iets": 27955, "1016": 27956, "\u0120Sham": 27957, "\u0120Pyr": 27958, "flies": 27959, "\u0120Midnight": 27960, "\u0120swallowed": 27961, "\u0120genres": 27962, "\u0120Lucky": 27963, "\u0120Rewards": 27964, "\u0120dispatch": 27965, "\u0120IPA": 27966, "\u0120Apply": 27967, "\u0120aven": 27968, "alities": 27969, "312": 27970, "things": 27971, "\u0120().": 27972, "\u0120mates": 27973, "\u0120Sz": 27974, "\u0120COP": 27975, "olate": 27976, "OFF": 27977, "\u0120recharge": 27978, "caps": 27979, "\u0120Yorker": 27980, "icone": 27981, "\u0120galaxies": 27982, "ileaks": 27983, "Dave": 27984, "\u0120Puzz": 27985, "\u0120Celtic": 27986, "\u0120AFC": 27987, "276": 27988, "\u0120Sons": 27989, "\u0120affirmative": 27990, "Hor": 27991, "\u0120tutorials": 27992, "\u0120CITY": 27993, "\u0120Rosa": 27994, "\u0120Extension": 27995, "Series": 27996, "\u0120fats": 27997, "\u0120rab": 27998, "lis": 27999, "\u0120unic": 28000, "\u0120eve": 28001, "\u0120Spin": 28002, "\u0120adulthood": 28003, "typ": 28004, "\u0120sectarian": 28005, "\u0120checkout": 28006, "\u0120Cycl": 28007, "Single": 28008, "\u0120martyr": 28009, "\u0120chilling": 28010, "888": 28011, "oufl": 28012, "\u0120];": 28013, "\u0120congestion": 28014, "mk": 28015, "\u0120Whereas": 28016, "\u01201938": 28017, "urrencies": 28018, "erion": 28019, "\u0120boast": 28020, "\u0120Patients": 28021, "\u0120chap": 28022, "\u0120BD": 28023, "realDonaldTrump": 28024, "\u0120examines": 28025, "hov": 28026, "\u0120startling": 28027, "\u0120Babylon": 28028, "wid": 28029, "omew": 28030, "brance": 28031, "\u0120Odyssey": 28032, "wig": 28033, "\u0120torch": 28034, "\u0120Vox": 28035, "\u0120Moz": 28036, "\u0120Troll": 28037, "\u0120Ans": 28038, "Similarly": 28039, "\u0120Ful": 28040, "006": 28041, "Unless": 28042, "\u0120Alone": 28043, "stead": 28044, "\u0120Publisher": 28045, "rights": 28046, "tu": 28047, "\u0120Doesn": 28048, "\u0120professionally": 28049, "\u0120clo": 28050, "icz": 28051, "\u0120steals": 28052, "\u0120\u00e1": 28053, "1986": 28054, "\u0120sturdy": 28055, "\u0120Johann": 28056, "\u0120medals": 28057, "\u0120filings": 28058, "\u0120Fraser": 28059, "done": 28060, "\u0120multinational": 28061, "\u0120feder": 28062, "\u0120worthless": 28063, "\u0120pest": 28064, "Yesterday": 28065, "ankind": 28066, "\u0120gays": 28067, "\u0120borne": 28068, "\u0120POS": 28069, "Picture": 28070, "\u0120percentages": 28071, "251": 28072, "rame": 28073, "\u0120potions": 28074, "AMD": 28075, "\u0120Lebanese": 28076, "\u0120rang": 28077, "\u0120LSU": 28078, "ongs": 28079, "\u0120peninsula": 28080, "\u0120Clause": 28081, "ALK": 28082, "oha": 28083, "\u0120MacBook": 28084, "\u0120unanimous": 28085, "\u0120lenders": 28086, "\u0120hangs": 28087, "\u0120franchises": 28088, "orers": 28089, "\u0120Updates": 28090, "\u0120isolate": 28091, "andro": 28092, "Soon": 28093, "\u0120disruptive": 28094, "\u0120Surve": 28095, "\u0120stitches": 28096, "\u0120Scorp": 28097, "\u0120Dominion": 28098, "\u0120supplying": 28099, "Arg": 28100, "\u0120turret": 28101, "\u0120Luk": 28102, "\u0120brackets": 28103, "*)": 28104, "\u0120Revolutionary": 28105, "\u0120Honest": 28106, "\u0120noticing": 28107, "\u0120Shannon": 28108, "\u0120afforded": 28109, "\u0120tha": 28110, "\u0120Janet": 28111, "!--": 28112, "\u0120Narendra": 28113, "\u0120Plot": 28114, "Hol": 28115, "sever": 28116, "eenth": 28117, "\u0120obstruction": 28118, "\u01201024": 28119, "staff": 28120, "jas": 28121, "orget": 28122, "scenes": 28123, "laughs": 28124, "\u0120Fargo": 28125, "crime": 28126, "\u0120orchestr": 28127, "\u0120delet": 28128, "iliary": 28129, "rieved": 28130, "\u0120militar": 28131, "\u0120Greene": 28132, "\u00e2\u0139\u0131": 28133, "\u00e3\u0123\u00a6": 28134, "\u0120Guards": 28135, "\u0120unleashed": 28136, "\u0120Weber": 28137, "\u0120adjustable": 28138, "\u0120caliber": 28139, "\u0120motivations": 28140, "\u0120\u00c3\u0142": 28141, "mAh": 28142, "\u0120Lanka": 28143, "handle": 28144, "\u0120pent": 28145, "\u0120Rav": 28146, "\u0120Angular": 28147, "\u0120Kau": 28148, "umbing": 28149, "\u0120philanthrop": 28150, "\u0120dehyd": 28151, "\u0120toxicity": 28152, "eer": 28153, "\u0120YORK": 28154, "witz": 28155, "\u00e5\u00bc": 28156, "\u0120IE": 28157, "community": 28158, "\u0120AH": 28159, "\u0120retali": 28160, "\u0120massively": 28161, "\u0120Daniels": 28162, "\u0120DEL": 28163, "\u0120carcin": 28164, "Url": 28165, "\u0120routing": 28166, "\u0120NPCs": 28167, "\u0120RAF": 28168, "ryce": 28169, "\u0120waived": 28170, "\u0120Guatem": 28171, "Everybody": 28172, "\u0120covenant": 28173, "\u0120173": 28174, "\u0120relaxing": 28175, "\u0120quart": 28176, "almost": 28177, "\u0120guarded": 28178, "\u0120Soldiers": 28179, "\u0120PLAY": 28180, "\u0120outgoing": 28181, "LAND": 28182, "\u0120rewrite": 28183, "\u0120MOV": 28184, "\u0120Imper": 28185, "\u0120Solution": 28186, "\u0120phenomenal": 28187, "\u0120longevity": 28188, "\u0120impat": 28189, "\u0120Nissan": 28190, "irie": 28191, "\u0120odor": 28192, "\u0120Zar": 28193, "oks": 28194, "\u0120militias": 28195, "\u0120SPEC": 28196, "\u0120tolerated": 28197, "arser": 28198, "\u0120Bradford": 28199, "+,": 28200, "\u0120surreal": 28201, "sf": 28202, "Canadian": 28203, "\u0120resemblance": 28204, "\u0120carbohydrate": 28205, "VIEW": 28206, "\u0120accessory": 28207, "meal": 28208, "largest": 28209, "iegel": 28210, "Someone": 28211, "\u0120toughest": 28212, "oso": 28213, "\u0120funnel": 28214, "\u0120condemnation": 28215, "luent": 28216, "\u0120wired": 28217, "\u0120Sunset": 28218, "Jesus": 28219, "\u0120PST": 28220, "\u0120Pages": 28221, "\u0120Tycoon": 28222, "\u0120PF": 28223, "\u0120selections": 28224, "\u0120\u00e0\u00a4": 28225, "partisan": 28226, "\u0120highs": 28227, "\u0120Rune": 28228, "\u0120crafts": 28229, "lead": 28230, "\u0120Parents": 28231, "\u0120reclaim": 28232, "eker": 28233, "\u0120Allied": 28234, "aeper": 28235, "\u0120looming": 28236, "\u0120beneficiaries": 28237, "\u0120Hull": 28238, "Students": 28239, "Jewish": 28240, "dj": 28241, "\u0120pact": 28242, "template": 28243, "\u0120Officials": 28244, "\u0120Baylor": 28245, "\u0120hemp": 28246, "\u0120youths": 28247, "\u0120Levels": 28248, "\u0120Xiao": 28249, "\u0120Ches": 28250, "\u0120endeavor": 28251, "\u0120Removed": 28252, "\u0120hippocamp": 28253, "Hell": 28254, "\u00e3\u0124\u012c": 28255, "805": 28256, "\u0120dinosaur": 28257, "\u0120Wrath": 28258, "\u0120Indonesian": 28259, "\u0120calculator": 28260, "\u0120Dictionary": 28261, "\u0120420": 28262, "\u0120MAG": 28263, "(_": 28264, "!,": 28265, "tarians": 28266, "\u0120restricting": 28267, "racuse": 28268, "\u0120weekday": 28269, "OUNT": 28270, "\u0120shrugged": 28271, "leground": 28272, "\u0120bald": 28273, "\u0120Doctors": 28274, "\u0120touted": 28275, "\u0120Maxwell": 28276, "\u0120214": 28277, "\u0120diplomat": 28278, "\u0120repression": 28279, "\u0120constituency": 28280, "vice": 28281, "ranked": 28282, "\u0120Napoleon": 28283, "gang": 28284, "\u0120Forever": 28285, "tun": 28286, "\u0120bulb": 28287, "\u0120PDT": 28288, "\u0120Cisco": 28289, "VEN": 28290, "\u0120resumed": 28291, "Steven": 28292, "\u0120Manitoba": 28293, "\u0120fabulous": 28294, "\u0120Agents": 28295, "1984": 28296, "\u0120amusing": 28297, "\u0120Mysteries": 28298, "\u0120orthodox": 28299, "floor": 28300, "\u0120questionnaire": 28301, "\u0120penetrate": 28302, "\u0120filmmakers": 28303, "\u0120Unc": 28304, "\u0120stamped": 28305, "\u0120thirteen": 28306, "\u0120outfield": 28307, "\u0120forwarded": 28308, "\u0120appra": 28309, "\u0120aided": 28310, "try": 28311, "\u0120unfocused": 28312, "\u0120Liz": 28313, "\u0120Wendy": 28314, "\u0120Scene": 28315, "Charg": 28316, "\u0120rejects": 28317, "\u0120leftist": 28318, "\u0120Providence": 28319, "\u0120Brid": 28320, "regn": 28321, "\u0120prophecy": 28322, "\u0120LIVE": 28323, "499": 28324, "\u0120forge": 28325, "\u0120FML": 28326, "\u0120intrinsic": 28327, "\u0120Frog": 28328, "\u0120wont": 28329, "\u0120Holt": 28330, "\u0120famed": 28331, "CLUS": 28332, "aepernick": 28333, "\u0120Hate": 28334, "\u0120Cay": 28335, "\u0120registering": 28336, "ortality": 28337, "ropy": 28338, "ocalyptic": 28339, "aan": 28340, "nav": 28341, "\u0120fascist": 28342, "IFIED": 28343, "\u0120implicated": 28344, "\u0120Resort": 28345, "\u0120Chandler": 28346, "\u0120Brick": 28347, "Pin": 28348, "ysc": 28349, "Usage": 28350, "\u0120Helm": 28351, "usra": 28352, "\u00e2\u013a\u0127\u00e2\u013a\u0127": 28353, "\u0120Abbas": 28354, "\u0120unanimously": 28355, "\u0120keeper": 28356, "\u0120addicted": 28357, "???": 28358, "\u0120helmets": 28359, "\u0120antioxid": 28360, "apsed": 28361, "808": 28362, "giene": 28363, "\u0120waits": 28364, "\u0120minion": 28365, "raved": 28366, "\u0120Porsche": 28367, "\u0120dreaming": 28368, "\u0120171": 28369, "\u0120Cain": 28370, "\u0120unfor": 28371, "asso": 28372, "\u0120Configuration": 28373, "kun": 28374, "hardt": 28375, "\u0120nested": 28376, "\u0120LDS": 28377, "LES": 28378, "\u0120tying": 28379, "enos": 28380, "\u0120cue": 28381, "\u0120Marqu": 28382, "skirts": 28383, "\u0120clicked": 28384, "\u0120expiration": 28385, "\u0120Accordingly": 28386, "\u0120WC": 28387, "\u0120blessings": 28388, "\u0120addictive": 28389, "\u0120Narr": 28390, "yx": 28391, "\u0120Jaguars": 28392, "\u0120rents": 28393, "\u0120Siber": 28394, "\u0120tipped": 28395, "ousse": 28396, "\u0120Fitzgerald": 28397, "\u0120hierarch": 28398, "outine": 28399, "\u0120wavelength": 28400, ">.": 28401, "chid": 28402, "\u0120Processing": 28403, "/+": 28404, "ranking": 28405, "Easy": 28406, "\u0120Construct": 28407, "\u0120tet": 28408, "insured": 28409, "HUD": 28410, "\u0120quoting": 28411, "\u0120communicated": 28412, "inx": 28413, "\u0120inmate": 28414, "\u0120erected": 28415, "\u0120Absolutely": 28416, "\u0120Surely": 28417, "\u0120unim": 28418, "\u0120Throne": 28419, "heid": 28420, "\u0120claws": 28421, "\u0120superstar": 28422, "\u0120Lenn": 28423, "\u0120Whis": 28424, "Uk": 28425, "abol": 28426, "\u0120sket": 28427, "\u0120Niet": 28428, "\u0120perks": 28429, "\u0120affinity": 28430, "\u0120openings": 28431, "phasis": 28432, "\u0120discriminate": 28433, "Tip": 28434, "vc": 28435, "\u0120grinding": 28436, "\u0120Jenny": 28437, "\u0120asthma": 28438, "holes": 28439, "\u0120Homer": 28440, "\u0120registers": 28441, "\u0120Glad": 28442, "\u0120creations": 28443, "\u0120lithium": 28444, "\u0120applause": 28445, "until": 28446, "Justice": 28447, "\u0120Turks": 28448, "\u0120scandals": 28449, "\u0120bake": 28450, "tank": 28451, "Mech": 28452, "\u0120Means": 28453, "\u0120Maid": 28454, "Republicans": 28455, "isal": 28456, "windows": 28457, "\u0120Santos": 28458, "\u0120vegetation": 28459, "338": 28460, "tri": 28461, "\u0120flux": 28462, "insert": 28463, "\u0120clarified": 28464, "\u0120mortg": 28465, "\u0120Chim": 28466, "\u0120Tort": 28467, "\u0120disclaim": 28468, "metal": 28469, "\u0120Aside": 28470, "\u0120induction": 28471, "\u0120infl": 28472, "\u0120atheists": 28473, "amph": 28474, "\u0120ether": 28475, "\u0120Vital": 28476, "\u0120Built": 28477, "Mind": 28478, "\u0120weaponry": 28479, "SET": 28480, "\u0120186": 28481, "admin": 28482, "gam": 28483, "contract": 28484, "afa": 28485, "\u0120derivatives": 28486, "\u0120snacks": 28487, "\u0120churn": 28488, "Econom": 28489, "\u0120capped": 28490, "\u0120Understanding": 28491, "\u0120Hers": 28492, "\u0120Iz": 28493, "\u0120duct": 28494, "IENT": 28495, "aughty": 28496, "\u0120\u00e2\u013e\u0136": 28497, "\u0120NP": 28498, "\u0120sailing": 28499, "Initialized": 28500, "\u0120ted": 28501, "\u0120reactors": 28502, "\u0120Lomb": 28503, "\u0120choke": 28504, "\u0120Worm": 28505, "\u0120admiration": 28506, "\u0120swung": 28507, "ensibly": 28508, "\u0120rash": 28509, "\u0120Goals": 28510, "\u0120Important": 28511, "Shot": 28512, "\u0120Ras": 28513, "\u0120trainers": 28514, "\u0120Bun": 28515, "Working": 28516, "\u0120harmed": 28517, "\u0120Pandora": 28518, "\u0120LTE": 28519, "\u0120mushroom": 28520, "\u0120CHAR": 28521, "\u0120Fee": 28522, "\u0120Moy": 28523, "Born": 28524, "oliberal": 28525, "\u0120Martial": 28526, "\u0120gentlemen": 28527, "\u0120lingering": 28528, "Official": 28529, "\u0120graffiti": 28530, "\u0120Names": 28531, "Der": 28532, "\u0120quint": 28533, "istrate": 28534, "azeera": 28535, "\u0120NOTICE": 28536, "\u0120Florence": 28537, "\u0120payable": 28538, "\u0120depicts": 28539, "\u0120Species": 28540, "Heart": 28541, "\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122": 28542, "\u0120enclosed": 28543, "Increases": 28544, "Daily": 28545, "\u0120Lis": 28546, "\u0120enactment": 28547, "\u0120Bacon": 28548, "\u0120Steele": 28549, "demand": 28550, "\u0120183": 28551, "\u0120mouths": 28552, "\u0120stranded": 28553, "\u0120enhancement": 28554, "011": 28555, "\u0120Whats": 28556, "\u0120healed": 28557, "eny": 28558, "\u0120Rab": 28559, "\u0120340": 28560, "\u0120Labyrinth": 28561, "roach": 28562, "\u0120Yosh": 28563, "\u0120Clippers": 28564, "\u0120concerts": 28565, "Internet": 28566, "355": 28567, "\u0120stickers": 28568, "\u0120termed": 28569, "\u0120Axe": 28570, "\u0120grandparents": 28571, "France": 28572, "\u0120Clim": 28573, "\u0120Uh": 28574, "ulic": 28575, "\u0120thrill": 28576, "centric": 28577, "\u0120Overview": 28578, "\u0120Conduct": 28579, "\u0120substantive": 28580, "\u0120182": 28581, "mur": 28582, "\u0120stray": 28583, "\u0120Coff": 28584, "\u0120repetitive": 28585, "\u0120Forgotten": 28586, "\u0120qualification": 28587, "ewitness": 28588, "\u0120Zimbabwe": 28589, "\u0120simulated": 28590, "\u0120JD": 28591, "253": 28592, "\u0120Ware": 28593, "\u0120unsc": 28594, "Times": 28595, "\u0120summons": 28596, "\u0120disconnected": 28597, "\u0120184": 28598, "cius": 28599, "\u0120Gujar": 28600, "odka": 28601, "\u0120erase": 28602, "\u0120Tobacco": 28603, "elected": 28604, "\u0120uncont": 28605, "\u0120Shepard": 28606, "\u0120Lamp": 28607, "\u0120alerted": 28608, "\u0120operative": 28609, "arna": 28610, "uint": 28611, "\u0120negligence": 28612, "acements": 28613, "\u0120supra": 28614, "\u0120prevail": 28615, "\u0120Shark": 28616, "\u0120belts": 28617, "\u00e3\u0123\u00ab": 28618, "\u0120tighter": 28619, "Engineers": 28620, "\u0120inactive": 28621, "\u0120exponent": 28622, "\u0120Willie": 28623, "aples": 28624, "\u0120heir": 28625, "\u0120Hits": 28626, "iann": 28627, "\u0120Says": 28628, "\u0120currents": 28629, "\u0120Bengal": 28630, "\u0120arist": 28631, "Buffer": 28632, "\u0120breeze": 28633, "\u0120Wesley": 28634, "Cola": 28635, "\u0120pronoun": 28636, "\u0120deed": 28637, "\u0120Kling": 28638, "\u0120oft": 28639, "\u0120inflict": 28640, "\u0120punishing": 28641, "\u0120nm": 28642, "iku": 28643, "ODUCT": 28644, "014": 28645, "\u0120subsidy": 28646, "\u0120DEA": 28647, "\u0120Herbert": 28648, "\u0120Jal": 28649, "Bank": 28650, "\u0120deferred": 28651, "\u0120shipment": 28652, "Bott": 28653, "\u0120alle": 28654, "bearing": 28655, "HTML": 28656, "Offline": 28657, "\u0120213": 28658, "\u0120scrolling": 28659, "\u0120scanned": 28660, "\u0120Libyan": 28661, "\u0120TOP": 28662, "chrom": 28663, "dt": 28664, "column": 28665, "PsyNetMessage": 28666, "Zero": 28667, "\u0120torso": 28668, "050": 28669, "\u00e2\u0137\u0132": 28670, "\u0120imperson": 28671, "\u0120Schwartz": 28672, "udic": 28673, "\u0120pissed": 28674, "\u0120Sapp": 28675, "257": 28676, "\u0120ISPs": 28677, "ogl": 28678, "\u0120supervised": 28679, "\u0120adolescent": 28680, "\u0120attained": 28681, "\u0120Delivery": 28682, "\u0120Bunny": 28683, "\u01201937": 28684, "\u0120miniature": 28685, "\u0120os": 28686, "\u0120370": 28687, "608": 28688, "\u0120Mourinho": 28689, "\u0120innate": 28690, "\u0120tempo": 28691, "\u0120NM": 28692, "\u0120Fallen": 28693, "009": 28694, "\u0120provocative": 28695, "Streamer": 28696, "\u0120Benedict": 28697, "\u0120Bolshe": 28698, "\u0120turtle": 28699, "\u0120PCB": 28700, "\u0120Equal": 28701, "Director": 28702, "\u0120Rend": 28703, "\u0120fluids": 28704, "Authorities": 28705, "\u0120cousins": 28706, "requency": 28707, "\u0120Neighbor": 28708, "sets": 28709, "shared": 28710, "Charles": 28711, "password": 28712, "\u0120gears": 28713, "\u0120211": 28714, "\u0120Hardware": 28715, "rika": 28716, "\u0120upstream": 28717, "Hom": 28718, "\u0120disproportionately": 28719, "ivities": 28720, "\u0120undefined": 28721, "\u0120electrons": 28722, "\u0120commemor": 28723, "Eventually": 28724, "\u0120><": 28725, "\u0120irresponsible": 28726, "218": 28727, "\u0120Released": 28728, "\u0120OVER": 28729, "\u0120IGN": 28730, "\u0120Bread": 28731, "stellar": 28732, "\u0120Sage": 28733, "tted": 28734, "damage": 28735, "edition": 28736, "\u0120Prec": 28737, "\u0120lime": 28738, "\u0120confinement": 28739, "\u0120calorie": 28740, "weapon": 28741, "\u0120differing": 28742, "\u0120Sina": 28743, "mys": 28744, "amd": 28745, "\u0120intricate": 28746, "kk": 28747, "\u0120PAT": 28748, "\u00c3\u00a3o": 28749, "stones": 28750, "links": 28751, "\u0120ranch": 28752, "Semitic": 28753, "\u0120differentiate": 28754, "\u0120Singer": 28755, "occupied": 28756, "\u0120fortress": 28757, "cmd": 28758, "\u0120interception": 28759, "\u0120Ankara": 28760, "\u0120rept": 28761, "\u0120Solitaire": 28762, "\u0120remake": 28763, "pred": 28764, "\u0120dared": 28765, "autions": 28766, "\u0120BACK": 28767, "Running": 28768, "\u0120debugging": 28769, "\u0120graphs": 28770, "399": 28771, "\u0120Nigel": 28772, "\u0120bun": 28773, "\u0120pillow": 28774, "\u0120progressed": 28775, "fashioned": 28776, "\u0120obedience": 28777, "ERN": 28778, "\u0120rehears": 28779, "Cell": 28780, "tl": 28781, "Sher": 28782, "\u0120herald": 28783, "\u0120Payment": 28784, "\u0120Cory": 28785, "\u0120Dept": 28786, "\u0120repent": 28787, "\u0120Weak": 28788, "uckland": 28789, "\u0120pleasing": 28790, "\u0120shortages": 28791, "\u0120jurors": 28792, "\u0120Kab": 28793, "qqa": 28794, "Anti": 28795, "\u0120wow": 28796, "\u0120RCMP": 28797, "\u0120tsun": 28798, "\u0120Sic": 28799, "\u0120comprises": 28800, "\u0120spies": 28801, "\u0120precinct": 28802, "nu": 28803, "\u0120urges": 28804, "\u0120timed": 28805, "\u0120stripes": 28806, "\u0120Boots": 28807, "\u0120yen": 28808, "Advanced": 28809, "\u0120discrete": 28810, "\u0120Archangel": 28811, "employment": 28812, "Diff": 28813, "\u0120monuments": 28814, "\u0120209": 28815, "worker": 28816, "\u0120196": 28817, "\u0120Ig": 28818, "utterstock": 28819, "TPS": 28820, "Jac": 28821, "\u0120homelessness": 28822, "\u0120commentator": 28823, "\u0120racially": 28824, "fing": 28825, "seed": 28826, "Ele": 28827, "ellation": 28828, "\u0120ethanol": 28829, "\u0120parish": 28830, "\u0120Dong": 28831, "\u0120Awakening": 28832, "\u0120deviation": 28833, "\u0120Bearing": 28834, "\u0120Tsuk": 28835, "\u0120recess": 28836, "\u0120lymph": 28837, "\u0120Cannabis": 28838, "\u00e5\u013e": 28839, "\u0120NEWS": 28840, "\u0120dra": 28841, "\u0120Stefan": 28842, "\u0120Wrong": 28843, "\u0120SAM": 28844, "\u0120loosely": 28845, "\u0120interpreter": 28846, "\u0120Plain": 28847, "Government": 28848, "\u0120bigotry": 28849, "\u0120grenades": 28850, "avez": 28851, "pictured": 28852, "\u0120mandated": 28853, "\u0120Monk": 28854, "\u0120Pedro": 28855, "\u0120lava": 28856, "274": 28857, "\u0120cynical": 28858, "\u0120Scrolls": 28859, "locks": 28860, "Mp": 28861, "\u0120congregation": 28862, "ornings": 28863, "phil": 28864, "\u0120Ibid": 28865, "\u0120ferv": 28866, "\u0120disappearing": 28867, "\u0120arrogant": 28868, "syn": 28869, "\u0120Maver": 28870, "\u0120Suit": 28871, "241": 28872, "\u0120abbre": 28873, "ackers": 28874, "Pa": 28875, "\u0120Yel": 28876, "Whenever": 28877, "\u0120235": 28878, "\u0120Vine": 28879, "\u0120Anat": 28880, "\u0120extinct": 28881, "LET": 28882, "\u0120executable": 28883, "VERS": 28884, "oxide": 28885, "DNA": 28886, "\u0120Prel": 28887, "\u0120resentment": 28888, "\u0120comprise": 28889, "\u0120Aviv": 28890, "\u0120interceptions": 28891, "\u0120prolific": 28892, "INA": 28893, "\u0120Erin": 28894, "thought": 28895, "219": 28896, "\u0120Psychiatry": 28897, "unky": 28898, "chemist": 28899, "Ho": 28900, "\u0120McCoy": 28901, "\u0120bricks": 28902, "Los": 28903, "rily": 28904, "\u0120USSR": 28905, "\u0120rud": 28906, "\u0120laud": 28907, "\u0120Wise": 28908, "\u0120Emerald": 28909, "\u0120revived": 28910, "\u0120damned": 28911, "\u0120Repair": 28912, "idem": 28913, "ctica": 28914, "\u0120patriarch": 28915, "\u0120Nurs": 28916, "meg": 28917, "\u0120cheapest": 28918, "reements": 28919, "empty": 28920, "\u0120Celebr": 28921, "\u0120deprivation": 28922, "chanted": 28923, "\u0120Thumbnails": 28924, "Energy": 28925, "\u0120Ethan": 28926, "\u0120Qing": 28927, "\u0120opposes": 28928, "WIND": 28929, "vik": 28930, "\u0120Mau": 28931, "\u0120SUB": 28932, "667": 28933, "GRE": 28934, "\u0120Volunte": 28935, "nton": 28936, "Cook": 28937, "\u00e5\u0132": 28938, "esque": 28939, "\u0120plummet": 28940, "\u0120suing": 28941, "\u0120pronounce": 28942, "\u0120resisting": 28943, "\u0120Fishing": 28944, "\u0120Trials": 28945, "\u0120yell": 28946, "\u0120310": 28947, "\u0120induct": 28948, "\u0120personalized": 28949, "often": 28950, "Reb": 28951, "EMBER": 28952, "\u0120viewpoint": 28953, "\u0120existential": 28954, "())": 28955, "remove": 28956, "MENTS": 28957, "lasses": 28958, "\u0120evapor": 28959, "\u0120aisle": 28960, "meta": 28961, "\u0120reflective": 28962, "\u0120entitlement": 28963, "\u0120devised": 28964, "music": 28965, "ascade": 28966, "\u0120winding": 28967, "offset": 28968, "\u0120accessibility": 28969, "kered": 28970, "Better": 28971, "\u0120Johnston": 28972, "thinking": 28973, "Snow": 28974, "\u0120Croatia": 28975, "\u0120Atomic": 28976, "271": 28977, "348": 28978, "\u0120textbook": 28979, "\u0120Sixth": 28980, "\u0120\u00d8\u00a7\u00d9\u0126": 28981, "\u0120slider": 28982, "\u0120Burger": 28983, "bol": 28984, "Sync": 28985, "\u0120grandchildren": 28986, "\u0120cerv": 28987, "+)": 28988, "\u0120eternity": 28989, "\u0120tweeting": 28990, "\u0120speculative": 28991, "\u0120pivotal": 28992, "\u0120WP": 28993, "\u0120TER": 28994, "ynamic": 28995, "\u0120upl": 28996, "\u0120Cats": 28997, "perhaps": 28998, "\u0120classmates": 28999, "\u0120blatant": 29000, "'-": 29001, "\u0120lakh": 29002, "antine": 29003, "\u0120Borg": 29004, "iom": 29005, "/(": 29006, "\u0120Athletic": 29007, "\u0120sar": 29008, "OTA": 29009, "\u0120Hoffman": 29010, "Nevertheless": 29011, "\u0120adorable": 29012, "\u0120spawned": 29013, "Associated": 29014, "\u0120Domestic": 29015, "\u0120implant": 29016, "\u0120Luxem": 29017, "\u0120Kens": 29018, "\u0120pumps": 29019, "\u0120SAT": 29020, "Attributes": 29021, "509": 29022, "avour": 29023, "\u0120centralized": 29024, "\u0120TN": 29025, "\u0120freshly": 29026, "\u0120Achieve": 29027, "\u0120outsiders": 29028, "herty": 29029, "\u0120Ree": 29030, "\u0120Towers": 29031, "\u0120Dart": 29032, "akable": 29033, "\u0120mp": 29034, "\u0120Heavenly": 29035, "\u0120ripe": 29036, "\u0120Caroline": 29037, "ryan": 29038, "\u0120classics": 29039, "\u0120retiring": 29040, "\u0120228": 29041, "\u0120ah": 29042, "\u0120dealings": 29043, "\u0120punching": 29044, "\u0120Chapman": 29045, "Options": 29046, "maxwell": 29047, "volume": 29048, "\u0120stal": 29049, "\u0120exported": 29050, "\u0120Quite": 29051, "\u0120numerical": 29052, "Burn": 29053, "Fact": 29054, "\u0120Keystone": 29055, "\u0120trending": 29056, "\u0120altering": 29057, "\u0120Africans": 29058, "478": 29059, "\u0120MN": 29060, "\u0120Knock": 29061, "\u0120temptation": 29062, "\u0120prestige": 29063, "Overview": 29064, "\u0120Traditional": 29065, "\u0120Bahrain": 29066, "Private": 29067, "\u0120HOU": 29068, "\u0120barr": 29069, "\u0120Tat": 29070, "Cube": 29071, "USD": 29072, "\u0120Grande": 29073, "\u0120Gat": 29074, "\u0120Flo": 29075, "\u0120resides": 29076, "\u0120indec": 29077, "volent": 29078, "\u0120perpetual": 29079, "ubes": 29080, "\u0120worldview": 29081, "\u0120Quantum": 29082, "\u0120filtered": 29083, "\u0120ensu": 29084, "orgetown": 29085, "ERSON": 29086, "\u0120Mild": 29087, "379": 29088, "OTT": 29089, "\u00c3\u00a5": 29090, "\u0120vitamins": 29091, "\u0120ribbon": 29092, "\u0120sincerely": 29093, "\u0120Hin": 29094, "\u0120eighteen": 29095, "\u0120contradictory": 29096, "\u0120glaring": 29097, "\u0120expectancy": 29098, "\u0120conspir": 29099, "\u0120monstrous": 29100, "\u0120380": 29101, "reci": 29102, "\u0120handic": 29103, "\u0120pumped": 29104, "\u0120indicative": 29105, "\u0120rapp": 29106, "\u0120avail": 29107, "\u0120LEGO": 29108, "\u0120Marijuana": 29109, "1985": 29110, "erton": 29111, "\u0120twentieth": 29112, "################################": 29113, "\u0120Swamp": 29114, "\u0120valuation": 29115, "\u0120affiliates": 29116, "adjusted": 29117, "\u0120Facility": 29118, "262": 29119, "\u0120enzymes": 29120, "itudinal": 29121, "\u0120imprint": 29122, "Site": 29123, "\u0120installer": 29124, "\u0120TRA": 29125, "mology": 29126, "linear": 29127, "\u0120Collective": 29128, "igating": 29129, "\u0120Token": 29130, "\u0120speculated": 29131, "KN": 29132, "\u0120Cly": 29133, "ority": 29134, "\u0120defer": 29135, "\u0120inspectors": 29136, "approved": 29137, "RM": 29138, "\u0120Suns": 29139, "\u0120informing": 29140, "\u0120Syracuse": 29141, "ibli": 29142, "765": 29143, "\u0120glove": 29144, "\u0120authorize": 29145, "\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6": 29146, "\u0120Cruise": 29147, "\u0120contracting": 29148, "shell": 29149, "IFE": 29150, "\u0120Jewel": 29151, "pract": 29152, "\u0120Photoshop": 29153, "\u0120Knowing": 29154, "harm": 29155, "\u0120attractions": 29156, "adan": 29157, "etus": 29158, "018": 29159, "wagen": 29160, "Alt": 29161, "\u0120multiply": 29162, "\u0120equilibrium": 29163, ":{": 29164, "\u0120Fighters": 29165, "\u0120Edgar": 29166, "\u0120fourteen": 29167, "Govern": 29168, "\u0120misuse": 29169, "\u0120abusing": 29170, "\u0120ancestry": 29171, "ramer": 29172, "644": 29173, "\u0120worms": 29174, "\u0120thicker": 29175, "\u0120Combine": 29176, "\u0120peasants": 29177, "\u0120vind": 29178, "\u0120conquest": 29179, "\u0120mocked": 29180, "\u0120cinnamon": 29181, "\u0120Cald": 29182, "\u0120Gallup": 29183, "\u0120avoidance": 29184, "\u0120incarnation": 29185, "\u0120Strat": 29186, "\u0120tasted": 29187, "enta": 29188, "\u0120Neal": 29189, "pared": 29190, "\u0120terminology": 29191, "jection": 29192, "Scientists": 29193, "\u0120INS": 29194, "\u0120Dee": 29195, "\u0120directories": 29196, "Road": 29197, "\u0120Shap": 29198, "bright": 29199, "\u0120Directors": 29200, "\u0120Column": 29201, "\u0120bob": 29202, "\u0120preferably": 29203, "\u0120glitch": 29204, "furt": 29205, "\u0120eg": 29206, "idis": 29207, "CBC": 29208, "\u0120surrendered": 29209, "\u0120testament": 29210, "336": 29211, "uggest": 29212, "\u0120Nil": 29213, "another": 29214, "\u0120pathetic": 29215, "\u0120Donna": 29216, "\u0120218": 29217, "\u0120Avery": 29218, "\u0120whiskey": 29219, "\u0120fixture": 29220, "\u0120Conquest": 29221, "\u0120bets": 29222, "Occ": 29223, "\u0120Leicester": 29224, "].\"": 29225, "\u0120));": 29226, "\u0120flashes": 29227, "456": 29228, "\u0120masked": 29229, "gebra": 29230, "\u0120computed": 29231, "chel": 29232, "auder": 29233, "\u0120defeats": 29234, "\u0120Liberation": 29235, "\u0120Osama": 29236, "\u0120Vive": 29237, "Changes": 29238, "Channel": 29239, "\u0120tariffs": 29240, "\u0120mage": 29241, "\u0120Sax": 29242, "\u0120inadvertently": 29243, "\u0120CRE": 29244, "\u0120Reaper": 29245, "inky": 29246, "grading": 29247, "\u0120stereotyp": 29248, "\u0120curl": 29249, "\u0120FANT": 29250, "\u0120frameworks": 29251, "Mom": 29252, "\u0120Anch": 29253, "\u0120flavour": 29254, "carbon": 29255, "\u0120permitting": 29256, "letcher": 29257, "\u0120Mozilla": 29258, "\u0120Parking": 29259, "\u0120Champ": 29260, "Scroll": 29261, "\u0120murderer": 29262, "\u0120rested": 29263, "\u0120owes": 29264, "\u0120Poss": 29265, "ADD": 29266, "IFF": 29267, "resolution": 29268, "\u0120Mining": 29269, "\u0120comparative": 29270, "Dim": 29271, "\u0120neighbouring": 29272, "\u0120AST": 29273, "\u0120Toxic": 29274, "\u0120biases": 29275, "\u0120gunfire": 29276, "urous": 29277, "\u0120Moment": 29278, "1983": 29279, "\u0120pervasive": 29280, "ttp": 29281, "\u0120Normally": 29282, "rir": 29283, "Sarah": 29284, "\u0120Albany": 29285, "\u0120unsett": 29286, "\u0120SMS": 29287, "ipers": 29288, "layer": 29289, "\u0120Whites": 29290, "uple": 29291, "\u0120turbo": 29292, "\u0120Leeds": 29293, "\u0120thats": 29294, "\u0120Miner": 29295, "MER": 29296, "\u0120Reign": 29297, "\u0120perme": 29298, "\u0120Blitz": 29299, "\u01201934": 29300, "\u0120intimidating": 29301, "tube": 29302, "\u0120eccentric": 29303, "abolic": 29304, "boxes": 29305, "\u0120Associates": 29306, "votes": 29307, "\u0120simulate": 29308, "umbo": 29309, "astery": 29310, "\u0120shipments": 29311, "FFFF": 29312, "anth": 29313, "\u0120seasoned": 29314, "\u0120experimentation": 29315, "\u00e2\u0138\u0142": 29316, "laws": 29317, "Meet": 29318, "iddles": 29319, "antics": 29320, "Rating": 29321, "ISIS": 29322, "hift": 29323, "\u0120fronts": 29324, "buf": 29325, "017": 29326, "\u0120unatt": 29327, "\u0120Dil": 29328, "leases": 29329, "\u0120Gardens": 29330, "777": 29331, "touch": 29332, "vell": 29333, "458": 29334, "\u0120=====": 29335, "saving": 29336, "\u0120erosion": 29337, "\u0120Quin": 29338, "\u0120earns": 29339, "\u0120accomplishment": 29340, "\u0120Wei": 29341, "\u0120<[": 29342, "_____": 29343, "\u0120irrig": 29344, "\u0120Teddy": 29345, "\u0120conquered": 29346, "\u0120Armored": 29347, "\u0120asserts": 29348, "\u0120manipulating": 29349, "r\u00c3\u00a9": 29350, "\u0120transcripts": 29351, "Gallery": 29352, "\u0120plotting": 29353, "Neil": 29354, "\u0120betrayal": 29355, "loader": 29356, "\u0120Sul": 29357, "\u0120displacement": 29358, "\u0120royalty": 29359, "\u0120WI": 29360, "heit": 29361, "\u0120Devices": 29362, "allel": 29363, "\u0120municipalities": 29364, "\u0120canal": 29365, "Stars": 29366, "\u0120UAE": 29367, "\u0120\"\u00e2\u0122\u00a6": 29368, "\u0120CU": 29369, "above": 29370, "\u0120resonance": 29371, "\u0120guiActiveUn": 29372, "added": 29373, "\u0120Braves": 29374, "\u0120Ibn": 29375, "\u0120hereby": 29376, "\u0120BRE": 29377, "\u0120shareholder": 29378, "\u0120Hir": 29379, "\u0120Ji": 29380, "\u0120strangely": 29381, "\u0120admired": 29382, "\u0120plight": 29383, "\u0120bachelor": 29384, "\u0120Pole": 29385, "ciplinary": 29386, "Tony": 29387, "\u0120Armenian": 29388, "\u0120unman": 29389, "\u0120Zionist": 29390, "Stage": 29391, "iscover": 29392, "\u0120automotive": 29393, "\u0120sidelines": 29394, "\u0120slick": 29395, "\u0120Renaissance": 29396, "\u0120FUN": 29397, "Images": 29398, "\u0120Haj": 29399, "\u0120ping": 29400, "\u0120shortcut": 29401, "\u0120Blvd": 29402, "\u0120Looks": 29403, "\u0120bursts": 29404, "\u0120clamp": 29405, "\u0120mish": 29406, "\u0120sorting": 29407, "\u0120patriot": 29408, "\u0120correctness": 29409, "\u0120Scandinav": 29410, "\u0120Cavaliers": 29411, "python": 29412, "azar": 29413, "\u0120375": 29414, "\u0120Jaune": 29415, "409": 29416, "\u0120detrimental": 29417, "\u0120stabbing": 29418, "\u0120poisoned": 29419, "\u0120fountain": 29420, "ocent": 29421, "orst": 29422, "\u0120Mari": 29423, "\u0120rains": 29424, "\u0120Overs": 29425, "\u0120Institution": 29426, "udget": 29427, "AMY": 29428, "tale": 29429, "\u0120KR": 29430, "\u0120Prices": 29431, "\u0120headaches": 29432, "\u0120landsl": 29433, "\u0120Aura": 29434, "Bonus": 29435, "\u0120Zhao": 29436, "\u0120Hip": 29437, "\u0120hops": 29438, "\u0120Kurdistan": 29439, "\u0120exploiting": 29440, "ryn": 29441, "\u0120hypocrisy": 29442, "opening": 29443, "\u0120gunshot": 29444, "\u0120wed": 29445, "interstitial": 29446, "Interstitial": 29447, "\u0120amen": 29448, "Breaking": 29449, "\u0120marketed": 29450, "Wire": 29451, "\u0120Crowd": 29452, "Continue": 29453, "\u0120Known": 29454, "\u0120Effective": 29455, "orean": 29456, "izons": 29457, "Joseph": 29458, "\u0120escalation": 29459, "username": 29460, "\u0120curtain": 29461, "ATES": 29462, "\u0120PAR": 29463, "\u0120Miy": 29464, "\u0120counterfe": 29465, "lene": 29466, "\u0120contenders": 29467, "daily": 29468, "\u0120Asc": 29469, "\u0120Phillip": 29470, "mostly": 29471, "\u0120filename": 29472, "hene": 29473, "\u0120resembling": 29474, "\u0120staging": 29475, "\u0120Chloe": 29476, "\u0120wiring": 29477, "Hon": 29478, "\u0120Renew": 29479, "ottage": 29480, "\u0120Hybrid": 29481, "much": 29482, "\u0120strokes": 29483, "\u0120policymakers": 29484, "APTER": 29485, "\u0120Arkham": 29486, "plot": 29487, "\u0120assistants": 29488, "\u0120deport": 29489, "\u0120Sega": 29490, "\u0120influenza": 29491, "\u0120Cursed": 29492, "\u0120Kobe": 29493, "\u0120skinny": 29494, "Provider": 29495, "\u0120Rip": 29496, "\u0120incremental": 29497, "products": 29498, "BF": 29499, "\u0120dome": 29500, "\u0120Credits": 29501, "\u0120losers": 29502, "ints": 29503, "\u0120Betty": 29504, "\u0120Talent": 29505, "\u0120DAM": 29506, "Lv": 29507, "Ess": 29508, "\u0120dens": 29509, "temp": 29510, "Judge": 29511, "odic": 29512, "\u0120'(": 29513, "URES": 29514, "etsk": 29515, "VO": 29516, "\u0120retrieved": 29517, "\u0120architects": 29518, "\u00d9\u0129": 29519, "\u0120ethic": 29520, "\u0120Secondary": 29521, "stocks": 29522, "adia": 29523, "\u0120325": 29524, "\u0120Opinion": 29525, "\u0120simultaneous": 29526, "\u0120dizz": 29527, "ulp": 29528, "\u0120smuggling": 29529, "ippery": 29530, "Random": 29531, "facing": 29532, "\u0120Das": 29533, "\u0120stockp": 29534, "\u0120disclosures": 29535, "pointer": 29536, "\u0120coral": 29537, "\u0120Selection": 29538, "\u0120Pike": 29539, "ivalent": 29540, "\u0120ruthless": 29541, "\u0120Rim": 29542, "\u0120ensuing": 29543, "\u0120Experiment": 29544, "\u0120congressman": 29545, "\u0120believer": 29546, "\u0120unspecified": 29547, "\u0120Mord": 29548, "\u0120knowledgeable": 29549, "\u0120VERY": 29550, "TX": 29551, "\u0120straps": 29552, "\u0120turf": 29553, "apeshifter": 29554, "\u0120marital": 29555, "\u0120flock": 29556, "\u00e3\u0123\u0128": 29557, "263": 29558, "AMES": 29559, "\u0120Opposition": 29560, "\u0120treasures": 29561, "\u0120GOD": 29562, "\u0120modeled": 29563, "\u0120WORLD": 29564, "\u0120([": 29565, "\u0120Usage": 29566, "HF": 29567, "\u0120$(": 29568, "ussed": 29569, "\u0120pioneer": 29570, "Eight": 29571, "parse": 29572, "bread": 29573, "ritz": 29574, "\u0120Miranda": 29575, "\u0120Kant": 29576, "++)": 29577, "oren": 29578, "\u0120provoked": 29579, "\u0120breeds": 29580, "\u0120Includes": 29581, "\u0120Pastebin": 29582, "\u0120Flip": 29583, "Java": 29584, "\u0120brink": 29585, "\u0120rumored": 29586, "\u0120unseen": 29587, "\u0120garnered": 29588, "\u0120Defin": 29589, "alted": 29590, "\u0120tattoos": 29591, "\u0120hesitation": 29592, "isitions": 29593, "\u0120Weaver": 29594, "\u0120Reporting": 29595, "\u0120therapies": 29596, "\u0120consultants": 29597, "\u0120residual": 29598, "\u0120Mali": 29599, "\u0120Roma": 29600, "iago": 29601, "\u0120Residents": 29602, "ubi": 29603, "\u0120remedies": 29604, "\u0120adaptive": 29605, "\u0120Alive": 29606, "\u0120Barcl": 29607, "\u0120wallets": 29608, "crypt": 29609, "etermination": 29610, "\u0120Pelosi": 29611, "\u0120slipping": 29612, "otonin": 29613, "\u0120alliances": 29614, "patrick": 29615, "iris": 29616, "\u0120orth": 29617, "\u0120Perkins": 29618, "\u0120DeV": 29619, "\u0120Gets": 29620, "\u0120drying": 29621, "gee": 29622, "forest": 29623, "\u0120Forget": 29624, "orem": 29625, "339": 29626, "\u0120vaguely": 29627, "\u0120Dion": 29628, "\u0120Porn": 29629, "\u0120HOW": 29630, "\u0120pneum": 29631, "\u0120rubble": 29632, "\u0120Taste": 29633, "encia": 29634, "\u0120Gel": 29635, "\u0120dst": 29636, "\u0120245": 29637, "\u0120Morocco": 29638, "inflamm": 29639, "\u0120Twins": 29640, "\u0120bots": 29641, "daughter": 29642, "\u0120Balk": 29643, "\u0120brethren": 29644, "\u0120logos": 29645, "\u0120gobl": 29646, "fps": 29647, "\u0120subdivision": 29648, "\u0120pawn": 29649, "\u0120squeezed": 29650, "\u0120morale": 29651, "\u0120DW": 29652, "'\"": 29653, "\u0120knot": 29654, "ooky": 29655, "\u0120divisive": 29656, "\u0120boosted": 29657, "chy": 29658, "\u00e3\u0125\u0132": 29659, "ifact": 29660, "\u0120newcomers": 29661, "\u0120Wrestling": 29662, "\u0120scouts": 29663, "wolves": 29664, "Rat": 29665, "\u0120nineteenth": 29666, "\u0120Osborne": 29667, "Stats": 29668, "\u0120empowered": 29669, "\u0120psychopath": 29670, "\u0120OEM": 29671, "uggage": 29672, "\u0120PK": 29673, "\u0120Mohammad": 29674, "Pak": 29675, "\u0120anarchists": 29676, "\u0120Extract": 29677, "esthes": 29678, "\u0120Stockholm": 29679, "loo": 29680, "\u0120Graph": 29681, "\u0120deploying": 29682, "\u0120Stranger": 29683, "\u0120Mold": 29684, "\u0120staffer": 29685, "\u0120discounted": 29686, "uckle": 29687, "please": 29688, "\u0120Landing": 29689, "\u00c3\u0143a": 29690, "\u0120193": 29691, "\u0120ante": 29692, "\u0120repetition": 29693, "\u0120+/-": 29694, "\u0120parody": 29695, "\u0120lively": 29696, "AAA": 29697, "\u0120Horus": 29698, "\u0120pits": 29699, "inders": 29700, "LOC": 29701, "\u0120Venice": 29702, "406": 29703, "\u0120Discover": 29704, "\u00e2\u0128": 29705, "ellectual": 29706, "\u0120pens": 29707, "\u0120eyel": 29708, "iguous": 29709, "Impl": 29710, "\u0120joking": 29711, "\u0120inval": 29712, "\u0120Belfast": 29713, "\u0120creditors": 29714, "\u0120Skywalker": 29715, "ovsky": 29716, "\u0120ceasefire": 29717, "\u0120seals": 29718, "isoft": 29719, ")).": 29720, "\u0120Felix": 29721, "ITS": 29722, "\u0120tresp": 29723, "\u0120Blockchain": 29724, "eware": 29725, "\u0120Schwar": 29726, "enne": 29727, "mounted": 29728, "\u0120Beacon": 29729, "lesh": 29730, "\u0120immensely": 29731, "\u0120cheering": 29732, "Employ": 29733, "scene": 29734, "ishly": 29735, "atchewan": 29736, "\u0120Nicolas": 29737, "\u0120drained": 29738, "\u0120Exit": 29739, "\u0120Azerb": 29740, "jun": 29741, "\u0120floated": 29742, "uania": 29743, "Deep": 29744, "\u0120superv": 29745, "\u0120mystical": 29746, "\u0120Dollar": 29747, "\u0120Apostle": 29748, "\u0120REL": 29749, "\u0120Provided": 29750, "\u0120Bucks": 29751, "\u00e3\u0125\u00b4": 29752, "cutting": 29753, "\u0120enhancements": 29754, "\u0120Penguins": 29755, "\u0120Isaiah": 29756, "\u0120jerk": 29757, "\u0120Wyn": 29758, "\u0120stalled": 29759, "\u0120cryptocurrencies": 29760, "\u0120Roland": 29761, "single": 29762, "\u0120lumin": 29763, "\u0120Fellow": 29764, "\u0120Capacity": 29765, "\u0120Kazakh": 29766, "WN": 29767, "\u0120financed": 29768, "389": 29769, "\u0120tid": 29770, "\u0120collusion": 29771, "\u0120Myr": 29772, "\u00ee\u0122": 29773, "Senator": 29774, "\u0120pediatric": 29775, "\u0120neatly": 29776, "\u0120sandwiches": 29777, "\u0120Architecture": 29778, "\u0120tucked": 29779, "\u0120balcony": 29780, "\u0120earthquakes": 29781, "quire": 29782, "Future": 29783, "\u0120hefty": 29784, "\u00e9\u0139": 29785, "\u0120specializes": 29786, "\u0120stresses": 29787, "\u0120sender": 29788, "\u0120misunderstanding": 29789, "\u0120epile": 29790, "\u0120provoke": 29791, "\u0120Colors": 29792, "\u0120dismay": 29793, "uko": 29794, "[_": 29795, "586": 29796, "neutral": 29797, "\u0120donating": 29798, "\u0120Randall": 29799, "Multi": 29800, "\u0120conveniently": 29801, "\u0120Sung": 29802, "\u0120Coca": 29803, "\u0120tents": 29804, "\u0120Acceler": 29805, "\u0120partnered": 29806, "272": 29807, "irming": 29808, "\u0120BAS": 29809, "sometimes": 29810, "\u0120objected": 29811, "ubric": 29812, "posed": 29813, "LCS": 29814, "grass": 29815, "\u0120attributable": 29816, "VIS": 29817, "Israeli": 29818, "\u0120repeats": 29819, "\u0120RM": 29820, "vag": 29821, "uta": 29822, "inous": 29823, "\u0120inert": 29824, "\u0120Miguel": 29825, "\u00e6\u0143": 29826, "\u0120Hawaiian": 29827, "Board": 29828, "\u0120artific": 29829, "\u0120Azerbai": 29830, "asio": 29831, "\u0120Rent": 29832, "AIN": 29833, "\u0120appliances": 29834, "\u0120nationality": 29835, "\u0120asshole": 29836, "\u0120Neb": 29837, "\u0120notch": 29838, "hani": 29839, "\u0120Bride": 29840, "Availability": 29841, "\u0120intercepted": 29842, "\u0120continental": 29843, "\u0120swelling": 29844, "\u0120Perspect": 29845, "bies": 29846, ".<": 29847, "ithmetic": 29848, "\u0120Lara": 29849, "\u0120tempting": 29850, "addr": 29851, "\u0120overseeing": 29852, "clad": 29853, "\u0120DV": 29854, "\u0120Gingrich": 29855, "\u0120mun": 29856, "\u0120Appropri": 29857, "\u0120alterations": 29858, "\u0120Patreon": 29859, "\u0120havoc": 29860, "\u0120disciplines": 29861, "\u0120notoriously": 29862, "akuya": 29863, "ieri": 29864, "?).": 29865, "\u0120Went": 29866, "\u0120silicon": 29867, "\u0120tremb": 29868, "Container": 29869, "Known": 29870, "\u0120mortar": 29871, "este": 29872, "icka": 29873, "Arthur": 29874, "\u0120Previously": 29875, "\u0120Marty": 29876, "\u0120sparse": 29877, "gins": 29878, "\u0120inward": 29879, "\u0120Participant": 29880, "Copy": 29881, "\u0120Misc": 29882, "\u0120antibiotic": 29883, "\u0120Retro": 29884, "\u0120elusive": 29885, "\u0120assail": 29886, "\u0120Battalion": 29887, "\u0120Bought": 29888, "\u0120diminish": 29889, "\u0120Europa": 29890, "session": 29891, "\u0120Dangerous": 29892, "iesel": 29893, "\u0120disbelief": 29894, "\u0120blasts": 29895, "extreme": 29896, "\u0120Boyd": 29897, "\u0120Projects": 29898, "\u0120Guys": 29899, "\u0120undergone": 29900, "\u0120grill": 29901, "\u0120Dwight": 29902, "\u0120197": 29903, "USER": 29904, "\u0120filesystem": 29905, "\u0120clocks": 29906, "Taylor": 29907, "\u0120wrapper": 29908, "\u0120folding": 29909, "ousand": 29910, "\u0120Philippine": 29911, "ATIONAL": 29912, "\u0120Perth": 29913, "\u0120ashes": 29914, "\u0120accumulate": 29915, "\u0120Gateway": 29916, "Shop": 29917, "orkshire": 29918, "Han": 29919, "\u0120Barrel": 29920, "\u0120Leh": 29921, "\u0120XV": 29922, "\u0120whim": 29923, "\u0120repo": 29924, "\u0120CG": 29925, "\u0120Mam": 29926, "\u0120incorporating": 29927, "\u0120bailout": 29928, "\u0120linguistic": 29929, "\u0120disinteg": 29930, "CLE": 29931, "\u0120cinematic": 29932, "\u0120Fiber": 29933, "Syn": 29934, "ilion": 29935, "\u0120Compos": 29936, "chens": 29937, "\u0120neoc": 29938, "\u0120boiled": 29939, "FINE": 29940, "ono": 29941, "uncle": 29942, "iken": 29943, "\u0120BM": 29944, "\u00ce\u00b9": 29945, "\u0120receipts": 29946, "\u0120disposed": 29947, "\u0120Thirty": 29948, "\u0120Rough": 29949, "\u0120ABS": 29950, "\u0120notwithstanding": 29951, "ollen": 29952, "#$": 29953, "\u0120unreliable": 29954, "\u0120bloom": 29955, "\u0120mediocre": 29956, "\u0120tram": 29957, "\u0120Tasman": 29958, "\u0120shakes": 29959, "\u0120manifesto": 29960, "\u0120MW": 29961, "\u0120satisfactory": 29962, "\u0120shores": 29963, "\u0120computation": 29964, "\u0120assertions": 29965, "ormons": 29966, "arag": 29967, "abit": 29968, "Democrats": 29969, "\u0120Loot": 29970, "\u0120Volks": 29971, "haired": 29972, "\u0120gravitational": 29973, "Sing": 29974, "\u0120Miz": 29975, "\u0120throttle": 29976, "\u0120tyranny": 29977, "\u0120Views": 29978, "\u0120robber": 29979, "\u0120Minority": 29980, "\u0120shrine": 29981, "scope": 29982, "purpose": 29983, "\u0120nucleus": 29984, "ourcing": 29985, "\u0120USDA": 29986, "\u0120DHS": 29987, "wra": 29988, "\u0120Bowie": 29989, "Scale": 29990, "\u0120BEL": 29991, "xi": 29992, "Iter": 29993, "\u0120(),": 29994, "wright": 29995, "\u0120sailors": 29996, "oused": 29997, "NASA": 29998, "\u0120Proof": 29999, "\u0120Mineral": 30000, "token": 30001, "\u0120FD": 30002, "Rew": 30003, "\u0120ell": 30004, "630": 30005, "\u0120chancellor": 30006, "\u0120Gos": 30007, "\u0120amounted": 30008, "\u0120Recre": 30009, "omez": 30010, "\u0120Optim": 30011, "\u0120Olive": 30012, "\u0120tracker": 30013, "owler": 30014, "\u0120Unique": 30015, "Root": 30016, "\u0120maritime": 30017, "\u0120Quran": 30018, "\u0120Adapt": 30019, "\u0120ecosystems": 30020, "\u0120Repeat": 30021, "\u0120Soy": 30022, "\u0120IMP": 30023, "\u0120graduating": 30024, "andem": 30025, "Pur": 30026, "\u0120Reset": 30027, "\u0120Trick": 30028, "\u0120Philly": 30029, "\u0120Tue": 30030, "\u0120Malaysian": 30031, "\u0120climax": 30032, "\u0120bury": 30033, "\u0120conspic": 30034, "\u0120Southampton": 30035, "\u0120Flowers": 30036, "\u0120escorted": 30037, "\u0120Educational": 30038, "\u0120IRC": 30039, "\u0120brutally": 30040, "eating": 30041, "\u0120pillar": 30042, "\u0120Sang": 30043, "\u0120Jude": 30044, "arling": 30045, "\u0120Amnesty": 30046, "\u0120reminding": 30047, "\u0120Administrative": 30048, "hesda": 30049, "\u0120flashed": 30050, "\u0120PBS": 30051, "perate": 30052, "feature": 30053, "\u0120swipe": 30054, "\u0120graves": 30055, "oultry": 30056, "261": 30057, "breaks": 30058, "\u0120Guer": 30059, "\u0120shrimp": 30060, "\u0120Voting": 30061, "quist": 30062, "\u0120analytical": 30063, "\u0120tablespoons": 30064, "\u0120SOU": 30065, "\u0120researched": 30066, "\u0120disrupted": 30067, "\u0120jour": 30068, "\u0120replica": 30069, "\u0120cartoons": 30070, "bians": 30071, "})": 30072, "copy": 30073, "Got": 30074, "ouched": 30075, "PUT": 30076, "\u0120swarm": 30077, "notations": 30078, "said": 30079, "\u0120rebuilt": 30080, "\u0120collaborate": 30081, "\u0120raging": 30082, "\u0120nar": 30083, "\u0120demographics": 30084, "\u0120DDR": 30085, "\u0120distrust": 30086, "ossier": 30087, "\u0120Kro": 30088, "\u0120pumpkin": 30089, "\u0120regrets": 30090, "\u0120fatalities": 30091, "\u0120Lens": 30092, "\u0120Ole": 30093, "pd": 30094, "\u0120puppet": 30095, "\u0120Outlook": 30096, "\u0120Stam": 30097, "Ol": 30098, "Fair": 30099, "UU": 30100, "\u0120rewritten": 30101, "\u00c4\u00b1": 30102, "\u0120fascinated": 30103, "\u0120vectors": 30104, "\u0120tribunal": 30105, "uay": 30106, "\u0120Mats": 30107, "\u0120Coins": 30108, "[[": 30109, "\u0120181": 30110, "\u0120renders": 30111, "\u0120Kaepernick": 30112, "\u0120espionage": 30113, "\u0120summ": 30114, "\u0120ditch": 30115, "Account": 30116, "\u0120spreadsheet": 30117, "\u0120mutant": 30118, "past": 30119, "407": 30120, "\u0120dye": 30121, "\u0120initiation": 30122, "\u01204000": 30123, "\u0120punishable": 30124, "\u0120thinner": 30125, "\u0120Khal": 30126, "\u0120intermedi": 30127, "Dun": 30128, "\u0120Gotham": 30129, "\u0120eagerly": 30130, "\u0120vaginal": 30131, "powers": 30132, "VW": 30133, "\u0120WATCHED": 30134, "\u0120predator": 30135, "amsung": 30136, "\u0120disparity": 30137, "\u0120[*": 30138, "\u0120amph": 30139, "\u0120outskirts": 30140, "\u0120Spirits": 30141, "\u0120skeletal": 30142, "\u00d0\u00bb": 30143, "\u0120Rear": 30144, "\u0120issuance": 30145, "\u0120Logic": 30146, "released": 30147, "ZZ": 30148, "\u0120Bound": 30149, "Entry": 30150, "\u0120exits": 30151, "isol": 30152, "\u0120Founder": 30153, "\u0120wre": 30154, "\u0120Greenland": 30155, "\u0120MMO": 30156, "taker": 30157, "INC": 30158, "\u00e3\u0123\u00be": 30159, "\u0120hourly": 30160, "henko": 30161, "\u0120fantasies": 30162, "\u0120disob": 30163, "\u0120demolition": 30164, "\u00e3\u0125\u012d": 30165, "\u0120enlisted": 30166, "ratulations": 30167, "\u0120misguided": 30168, "\u0120ensured": 30169, "\u0120discouraged": 30170, "mort": 30171, "\u0120flank": 30172, "\u0120cess": 30173, "\u0120reacts": 30174, "\u0120Sere": 30175, "sensitive": 30176, "\u0120Serpent": 30177, "assad": 30178, "\u0120247": 30179, "\u0120calmly": 30180, "busters": 30181, "\u0120bleed": 30182, "\u0120Stro": 30183, "\u0120amusement": 30184, "\u0120Antarctica": 30185, "\u0120scept": 30186, "\u0120Gaw": 30187, "aq": 30188, "asonic": 30189, "\u0120sprawling": 30190, "native": 30191, "aturated": 30192, "\u0120Battlefield": 30193, "IVERS": 30194, "EB": 30195, "\u0120Gems": 30196, "\u0120Northwestern": 30197, "\u0120Films": 30198, "\u0120Automatic": 30199, "\u0120apprehend": 30200, "\u00e3\u0123\u00a8": 30201, "\u0120guiName": 30202, "\u0120backend": 30203, "\u0120evidenced": 30204, "geant": 30205, "012": 30206, "\u0120Siege": 30207, "\u0120externalTo": 30208, "\u0120unfocusedRange": 30209, "\u0120guiActiveUnfocused": 30210, "\u0120guiIcon": 30211, "\u0120externalToEVA": 30212, "\u0120externalToEVAOnly": 30213, "Fri": 30214, "chard": 30215, "enaries": 30216, "\u0120chiefs": 30217, "\u0120cf": 30218, "\u0120HUD": 30219, "\u0120corrobor": 30220, "\u0120dB": 30221, "\u0120Taken": 30222, "\u0120Patricia": 30223, "rail": 30224, "\u0120Charm": 30225, "\u0120Libertarian": 30226, "rieve": 30227, "Personal": 30228, "\u0120OUR": 30229, "geries": 30230, "\u0120dumping": 30231, "\u0120neurological": 30232, "itimate": 30233, "\u0120Clintons": 30234, "rafted": 30235, "\u0120Molly": 30236, "\u0120terminals": 30237, "register": 30238, "\u0120flare": 30239, "\u0120encoded": 30240, "\u0120autopsy": 30241, "pel": 30242, "machine": 30243, "\u0120exemptions": 30244, "\u0120Royals": 30245, "distance": 30246, "\u0120drafts": 30247, "\u0120lame": 30248, "\u0120Cunning": 30249, "\u0120spouses": 30250, "\u0120Markets": 30251, "\u0120Carrier": 30252, "\u0120implying": 30253, "\u0120Yak": 30254, "sid": 30255, "\u0120loser": 30256, "\u0120vigilant": 30257, "\u0120impeachment": 30258, "\u0120augmented": 30259, "\u0120Employees": 30260, "\u0120unintended": 30261, "ternally": 30262, "\u0120Watt": 30263, "\u0120recognizable": 30264, "essim": 30265, "\u00e6\u013f": 30266, "\u0120coated": 30267, "rha": 30268, "\u0120lieutenant": 30269, "\u0120Legislation": 30270, "published": 30271, "444": 30272, "013": 30273, "\u0120ideally": 30274, "\u0120Password": 30275, "\u0120simplify": 30276, "\u0120Meta": 30277, "\u0120MRI": 30278, "\u0120pleading": 30279, "organized": 30280, "handler": 30281, "\u0120unravel": 30282, "correct": 30283, "\u0120icy": 30284, "\u0120paranoid": 30285, "\u0120passer": 30286, "\u0120inspections": 30287, "ofer": 30288, "\u0120Healthcare": 30289, "283": 30290, "\u0120Brut": 30291, "iola": 30292, "forge": 30293, "\u0120Medieval": 30294, "MSN": 30295, "ievers": 30296, "\u0120Programming": 30297, "\u00e5\u012b": 30298, "\u0120223": 30299, "mu": 30300, "\u0120CLE": 30301, "uga": 30302, "\u0120shoppers": 30303, "\u0120informative": 30304, "\u0120Plans": 30305, "\u0120supplementation": 30306, "\u0120Tests": 30307, "tyard": 30308, "ocytes": 30309, "\u0120Vega": 30310, "\u0120Gujarat": 30311, "ermanent": 30312, "Except": 30313, "\u0120LOT": 30314, "alla": 30315, "\u0120Cumm": 30316, "\u0120Osw": 30317, "\u0120venom": 30318, "\u0120Debt": 30319, "\u0120DOWN": 30320, "\u0120reunion": 30321, "\u0120muc": 30322, "\u0120Relief": 30323, "\u0120geop": 30324, "\u0120\u00f0\u0141\u013a": 30325, "alogue": 30326, "Anth": 30327, "echo": 30328, "\u0120corros": 30329, "\u0120replication": 30330, "\u0120Blazing": 30331, "\u0120Daughter": 30332, "\u0120inflic": 30333, "\u0120Lindsey": 30334, "\u00d9\u012a": 30335, "284": 30336, "Exit": 30337, "\u0120gloom": 30338, "TAIN": 30339, "\u0120undermining": 30340, "\u0120advising": 30341, "hidden": 30342, "\u0120overflow": 30343, "\u0120gor": 30344, "urdue": 30345, "\u0120echoes": 30346, "enhagen": 30347, "\u0120impuls": 30348, "drug": 30349, "cash": 30350, "\u0120async": 30351, "\u0120mirac": 30352, "atts": 30353, "punk": 30354, "\u0120pivot": 30355, "\u0120Legislative": 30356, "\u0120bloggers": 30357, "\u0120Claw": 30358, "sburg": 30359, "dyl": 30360, "\u0120Recommend": 30361, "\u0120verte": 30362, "\u0120prohibiting": 30363, "\u0120Panther": 30364, "Jonathan": 30365, "\u0120omin": 30366, "\u0120hateful": 30367, "281": 30368, "\u0120Orche": 30369, "\u0120Murdoch": 30370, "downs": 30371, "\u0120asymm": 30372, "GER": 30373, "Always": 30374, "\u0120informs": 30375, "\u0120WM": 30376, "\u0120Pony": 30377, "\u0120Appendix": 30378, "\u0120Arlington": 30379, "Jam": 30380, "\u0120medicinal": 30381, "\u0120Slam": 30382, "ITIES": 30383, "\u0120reaff": 30384, "\u0120Ri": 30385, "FG": 30386, "Spring": 30387, "bool": 30388, "\u0120thighs": 30389, "\u0120markings": 30390, "\u0120Raqqa": 30391, "\u0120Lak": 30392, "poll": 30393, "tsky": 30394, "\u0120Morty": 30395, "\u0120Definition": 30396, "\u0120debunk": 30397, "endered": 30398, "\u0120Leone": 30399, "avers": 30400, "\u0120mortgages": 30401, "Apparently": 30402, "Nic": 30403, "haus": 30404, "\u0120Thousands": 30405, "auld": 30406, "\u0120mash": 30407, "shoot": 30408, "\u0120diarr": 30409, "\u0120consciously": 30410, "Hero": 30411, "eas": 30412, "\u0120Naturally": 30413, "\u0120Destroyer": 30414, "\u0120dashboard": 30415, "services": 30416, "Rog": 30417, "\u0120millennials": 30418, "\u0120invade": 30419, "-(": 30420, "\u0120commissions": 30421, "\u0120Auckland": 30422, "\u0120broadcasts": 30423, "\u0120frontal": 30424, "\u0120crank": 30425, "\u0120Historic": 30426, "\u0120rumours": 30427, "CTV": 30428, "\u0120steril": 30429, "\u0120booster": 30430, "rocket": 30431, "\u00e3\u0124\u00bc": 30432, "utsche": 30433, "\u0120PI": 30434, "\u0120233": 30435, "\u0120Producer": 30436, "\u0120Analytics": 30437, "\u0120invaluable": 30438, "\u0120unintention": 30439, "\u0120CY": 30440, "\u0120scrutin": 30441, "\u0120gigg": 30442, "\u0120engulf": 30443, "\u0120proletariat": 30444, "\u0120hacks": 30445, "\u0120Hew": 30446, "arak": 30447, "\u0120Slime": 30448, "ielding": 30449, "agher": 30450, "\u0120Elliot": 30451, "\u0120telecom": 30452, "\u0120219": 30453, "ultan": 30454, "\u0120Arbor": 30455, "\u0120Scouts": 30456, "Ban": 30457, "\u0120lifespan": 30458, "\u0120blasp": 30459, "388": 30460, "\u0120judiciary": 30461, "\u0120Continental": 30462, "asking": 30463, "McC": 30464, "LED": 30465, "\u0120baggage": 30466, "\u0120Sorcerer": 30467, "\u0120remnants": 30468, "\u0120Griffith": 30469, "etsu": 30470, "\u0120Subaru": 30471, "\u0120Personality": 30472, "designed": 30473, "ushima": 30474, "agnar": 30475, "\u0120recoil": 30476, "\u0120passions": 30477, "\\\":": 30478, "\u0120tee": 30479, "\u0120abolition": 30480, "\u0120Creating": 30481, "jac": 30482, "\u0120194": 30483, "019": 30484, "\u0120pillars": 30485, "riched": 30486, "/\"": 30487, "tk": 30488, "\u0120livelihood": 30489, "\u0120roasted": 30490, "ahon": 30491, "\u0120Hutch": 30492, "assert": 30493, "\u0120dividend": 30494, "\u0120knit": 30495, "\u0120daunting": 30496, "\u0120disturbance": 30497, "\u0120shale": 30498, "\u0120cultivated": 30499, "\u0120refrigerator": 30500, "LB": 30501, "\u0120NET": 30502, "\u0120commercials": 30503, "\u0120thinkers": 30504, "455": 30505, "\u0120chop": 30506, "Broad": 30507, "\u0120suspicions": 30508, "\u0120tagged": 30509, "lifting": 30510, "\u0120stylish": 30511, "\u0120Shields": 30512, "Shortly": 30513, "\u0120tails": 30514, "Auth": 30515, "STE": 30516, "\u0120GAME": 30517, "\u0120seism": 30518, "\u0120Kis": 30519, "ologne": 30520, "\u0120cowork": 30521, "\u0120forcibly": 30522, "\u0120thyroid": 30523, "\u0120PB": 30524, "ANE": 30525, "married": 30526, "horse": 30527, "\u0120polymer": 30528, "\u0120Chal": 30529, "odor": 30530, "DEBUG": 30531, "\u0120Context": 30532, "\u0120bliss": 30533, "\u0120pinpoint": 30534, "\u0120Mathemat": 30535, "legram": 30536, "\u0120Weekend": 30537, "\u0120labelled": 30538, "\u0120bart": 30539, "itles": 30540, "\u0120estrogen": 30541, "\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136": 30542, "\"'": 30543, "\u0120visibly": 30544, "\u0120outsider": 30545, "aida": 30546, "Area": 30547, "\u0120dissemin": 30548, "\u0120dishonest": 30549, "\u0120Closed": 30550, "\u0120Bulletin": 30551, "\u0120Ramsey": 30552, "sword": 30553, "\u0120XI": 30554, "ourced": 30555, "Same": 30556, "346": 30557, "\u0120Repe": 30558, "\u0120Kou": 30559, "cake": 30560, "emis": 30561, "Cache": 30562, "\u0120Meaning": 30563, "\u0120Enlight": 30564, "onomy": 30565, "\u0120manifestation": 30566, "sworth": 30567, "Jay": 30568, "\u0120chore": 30569, "\u00c3\u00b6r": 30570, "Dream": 30571, "\u0120sanctioned": 30572, "\u0120culturally": 30573, "\u0120Ara": 30574, "Nav": 30575, "\u0120theological": 30576, "\u0120strut": 30577, "\u0120VO": 30578, "\u0120Handbook": 30579, "\u0120constructing": 30580, "\u0120\u00c2\u00b6": 30581, "\u0120Benefits": 30582, "\u0120Psychological": 30583, "sac": 30584, "\u00e5\u00b8": 30585, "policy": 30586, "\u0120Matters": 30587, "\u0120Reported": 30588, "\u0120Byte": 30589, "\u0120vitro": 30590, "\u0120Maiden": 30591, "\u0120lam": 30592, "\u0120Jennings": 30593, "\u0120garment": 30594, "\u0120Rutgers": 30595, "\u0120Stafford": 30596, "\u0120Wellington": 30597, "\u0120intermitt": 30598, "\u0120npm": 30599, "\u0120ordeal": 30600, "\u0120plugged": 30601, "ooming": 30602, "inished": 30603, "framework": 30604, "\u0120timber": 30605, "\u0120cass": 30606, "\u0120850": 30607, "iless": 30608, "\u0120Redux": 30609, "768": 30610, "Stre": 30611, "\u0120surpassed": 30612, "whel": 30613, "\u0120parallels": 30614, "\u0120veil": 30615, "\u0120GI": 30616, "\u0120REST": 30617, "\u0120readiness": 30618, "sort": 30619, "\u0120modifying": 30620, "\u0120Slate": 30621, "ruff": 30622, "\u0120marble": 30623, "\u0120infrared": 30624, "\u0120auditor": 30625, "\u0120FANTASY": 30626, "\u0120Poverty": 30627, "\u0120SPD": 30628, "\u0120\"(": 30629, "Ky": 30630, "RAY": 30631, "\u0120executions": 30632, "\u0120Beverly": 30633, "\u0120Marxism": 30634, "\u0120Burst": 30635, "\u0120Kali": 30636, "estones": 30637, "Clearly": 30638, "Ell": 30639, "\u00e3\u0123\u00a7": 30640, "\u0120Proceedings": 30641, "Token": 30642, "IFIC": 30643, "\u00c3\u00b1a": 30644, "Central": 30645, "\u0120Haley": 30646, "\u0120Drama": 30647, "\u0120formations": 30648, "ORN": 30649, "Books": 30650, "\u0120dominating": 30651, "\u0120Flyers": 30652, "\u0120Companion": 30653, "\u0120disciplined": 30654, "\u0120Yugoslav": 30655, "\u0120Spells": 30656, "\u0120vengeance": 30657, "\u0120landlords": 30658, "Len": 30659, "\u0120Ogre": 30660, "anoia": 30661, "\u0120piercing": 30662, "\u0120congreg": 30663, "\u0120scorer": 30664, "obia": 30665, "\u0120nickel": 30666, "\u0120Learns": 30667, "\u0120rejo": 30668, "\u0120masterpiece": 30669, "Flash": 30670, "\u0120inhabited": 30671, "\u0120OpenGL": 30672, "\u0120Dud": 30673, "\u0120ICO": 30674, "\u0120arter": 30675, "\u0120plur": 30676, "\u0120mastery": 30677, "\u0120longstanding": 30678, "sted": 30679, "\u0120wines": 30680, "\u0120televised": 30681, "\u0120Shrine": 30682, "\u0120Bayern": 30683, "\u0120\u00e2\u0135\u013a": 30684, "\u0120enclosure": 30685, "john": 30686, "\u0120prophets": 30687, "\u0120Resurrection": 30688, "\u0120Orders": 30689, "\u0120uneven": 30690, "rals": 30691, "\u0120dwind": 30692, "\u0120Lah": 30693, "\u0120Sloven": 30694, "378": 30695, "\u0120insistence": 30696, "affle": 30697, "\u0120Clone": 30698, "\u0120hardship": 30699, "\u0120Congressman": 30700, "\u0120plead": 30701, "\u0120reviewers": 30702, "\u0120cured": 30703, "\u01201935": 30704, "asley": 30705, "fake": 30706, "\u0120Thinking": 30707, "ydia": 30708, "PART": 30709, "\u0120Dota": 30710, "oit": 30711, "\u0120whipped": 30712, "\u0120bouncing": 30713, "\u0120Hispanics": 30714, "comings": 30715, "\u0120cannabin": 30716, "\u0120Chambers": 30717, "\u0120Zack": 30718, "Optional": 30719, "\u0120coats": 30720, "\u0120prowess": 30721, "\u0120Norton": 30722, "\u0120plainly": 30723, "\u0120freight": 30724, "\u0120inhibition": 30725, "\u0120clam": 30726, "\u0120303": 30727, "kef": 30728, "aleigh": 30729, "Luke": 30730, "\u0120psycho": 30731, "atorium": 30732, "MED": 30733, "\u0120treaties": 30734, "\u0120indisc": 30735, "\u0120dc": 30736, "OPS": 30737, "\u0120resilient": 30738, "\u0120Interstate": 30739, "\u0120slack": 30740, "\u0120mundane": 30741, "\u0120establishes": 30742, "359": 30743, "\u0120strained": 30744, "\u0120nond": 30745, "Sus": 30746, "\u0120caste": 30747, "arate": 30748, "ieving": 30749, "\u0120unfairly": 30750, "\u0120parser": 30751, "onial": 30752, "ursive": 30753, "Via": 30754, "\u0120Otto": 30755, "\u0120Authorities": 30756, "stroke": 30757, "KR": 30758, "\u0120Mercy": 30759, "\u0120furnished": 30760, "\u0120outset": 30761, "\u0120metic": 30762, "1982": 30763, "olithic": 30764, "\u0120Tent": 30765, "ogical": 30766, "\u0120Aircraft": 30767, "\u0120hides": 30768, "\u0120Became": 30769, "\u0120educators": 30770, "reaching": 30771, "\u0120volatility": 30772, "\u0120toddler": 30773, "\u0120NASCAR": 30774, "\u0120Twelve": 30775, "\u0120Highlights": 30776, "\u0120grape": 30777, "\u0120splits": 30778, "\u0120peasant": 30779, "\u0120reneg": 30780, "\u0120MSI": 30781, "Temp": 30782, "stars": 30783, "\u0120trek": 30784, "\u0120Hyde": 30785, "binding": 30786, "\u0120realism": 30787, "\u0120oxide": 30788, "\u0120Hos": 30789, "\u0120mounts": 30790, "\u0120biting": 30791, "\u0120collapsing": 30792, "\u0120postal": 30793, "\u0120museums": 30794, "\u0120detached": 30795, "\u0120respecting": 30796, "\u0120monopol": 30797, "\u0120workflow": 30798, "\u0120Cake": 30799, "Template": 30800, "\u0120Organisation": 30801, "\u0120persistence": 30802, "369": 30803, "Coming": 30804, "Brad": 30805, "\u0120redundant": 30806, "\u0120GTA": 30807, "\u0120bending": 30808, "\u0120revoked": 30809, "\u0120offending": 30810, "\u0120framing": 30811, "\u0120printf": 30812, "Commun": 30813, "members": 30814, "Outside": 30815, "\u0120construed": 30816, "\u0120coded": 30817, "FORE": 30818, "\u0120chast": 30819, "Chat": 30820, "Indian": 30821, "\u0120Yard": 30822, "?!\"": 30823, "\u0120Ports": 30824, "\u0120Xavier": 30825, "\u0120RET": 30826, "'.\"": 30827, "\u0120Boat": 30828, "ivated": 30829, "icht": 30830, "umerable": 30831, "Ds": 30832, "\u0120Dunn": 30833, "\u0120coffin": 30834, "\u0120securely": 30835, "\u0120Raptors": 30836, "\u0120Bes": 30837, "Installation": 30838, "\u0120inception": 30839, "\u0120Healthy": 30840, "endants": 30841, "\u0120psychologists": 30842, "\u0120Sheikh": 30843, "cultural": 30844, "\u0120BlackBerry": 30845, "shift": 30846, "Fred": 30847, "oche": 30848, "\u0120cakes": 30849, "\u0120SEO": 30850, "\u0120Gian": 30851, "\u0120Asians": 30852, "ogging": 30853, "element": 30854, "\u0120pundits": 30855, "\u0120Vaugh": 30856, "\u0120Gavin": 30857, "\u0120hitter": 30858, "\u0120drowned": 30859, "\u0120chalk": 30860, "\u0120Zika": 30861, "\u0120measles": 30862, "802": 30863, "\u00e2\u0122\u00a6..": 30864, "\u0120AWS": 30865, "]\"": 30866, "\u0120distort": 30867, "\u0120Mast": 30868, "\u0120antibodies": 30869, "\u0120Mash": 30870, "Memory": 30871, "\u0120Uganda": 30872, "\u0120Prob": 30873, "\u0120vomiting": 30874, "\u0120Turns": 30875, "\u0120occupying": 30876, "\u0120evasion": 30877, "\u0120Therapy": 30878, "\u0120promo": 30879, "\u0120electr": 30880, "\u0120blueprint": 30881, "\u0120Dre": 30882, "priced": 30883, "\u0120Depot": 30884, "\u0120alleviate": 30885, "\u0120Somali": 30886, "marg": 30887, "nine": 30888, "\u0120nostalgia": 30889, "\u0120Shepherd": 30890, "\u0120cavalry": 30891, "\u0120torped": 30892, "\u0120Bloody": 30893, "xb": 30894, "\u0120sank": 30895, "\u0120goalt": 30896, "reportprint": 30897, "embedreportprint": 30898, "cloneembedreportprint": 30899, "\u0120Initially": 30900, "\u0120Fischer": 30901, "\u0120noteworthy": 30902, "cern": 30903, "\u0120inefficient": 30904, "rawdownload": 30905, "rawdownloadcloneembedreportprint": 30906, "cation": 30907, "\u0120Dynasty": 30908, "lag": 30909, "DES": 30910, "\u0120distinctly": 30911, "\u0120Estonia": 30912, "\u0120openness": 30913, "\u0120gossip": 30914, "ruck": 30915, "Width": 30916, "\u0120Ibrahim": 30917, "\u0120petroleum": 30918, "\u0120avatar": 30919, "\u0120Hed": 30920, "atha": 30921, "\u0120Hogwarts": 30922, "\u0120caves": 30923, "678": 30924, "\u0120safeguard": 30925, "\u0120Mog": 30926, "isson": 30927, "\u0120Durham": 30928, "slaught": 30929, "\u0120Graduate": 30930, "\u0120subconscious": 30931, "\u0120Excellent": 30932, "\u0120Dum": 30933, "-----": 30934, "\u0120piles": 30935, "\u0120WORK": 30936, "\u0120Garn": 30937, "\u0120Fol": 30938, "\u0120ATM": 30939, "\u0120avoids": 30940, "\u0120Tul": 30941, "\u0120bleak": 30942, "ELY": 30943, "ivist": 30944, "lightly": 30945, "Pers": 30946, "\u0120Dob": 30947, "\u0120LS": 30948, "\u0120insanity": 30949, "\u00ce\u00b5": 30950, "atalie": 30951, "Enlarge": 30952, "\u0120twists": 30953, "\u0120faulty": 30954, "\u0120piracy": 30955, "\u0120impover": 30956, "\u0120rugged": 30957, "\u0120Fashion": 30958, "\u0120sands": 30959, "'?": 30960, "swick": 30961, "\u0120natives": 30962, "\u0120hen": 30963, "\u0120Noise": 30964, "\u00e3\u0125\u0139": 30965, "\u0120greens": 30966, "\u0120freezer": 30967, "\u0120dynasty": 30968, "\u0120Fathers": 30969, "\u0120Newark": 30970, "\u0120archaeological": 30971, "\u0120ot": 30972, "obar": 30973, "\u0120blockade": 30974, "\u0120allerg": 30975, "LV": 30976, "\u0120debit": 30977, "\u0120RFC": 30978, "\u0120Milton": 30979, "\u0120Pressure": 30980, "\u0120willingly": 30981, "\u0120disproportionate": 30982, "\u0120oppressive": 30983, "\u0120diamonds": 30984, "\u0120belongings": 30985, "1970": 30986, "\u0120bells": 30987, "\u0120imperialism": 30988, "\u0120227": 30989, "\u0120exploding": 30990, "\u0120Eclipse": 30991, "\u01201919": 30992, "\u0120rant": 30993, "\u0120nominations": 30994, "347": 30995, "\u0120peacefully": 30996, "rica": 30997, "\u0120FUCK": 30998, "\u0120vibration": 30999, "malink": 31000, "\u0120ropes": 31001, "\u0120Ivanka": 31002, "\u0120Brewery": 31003, "\u0120Booker": 31004, "\u0120Owens": 31005, "goers": 31006, "Services": 31007, "\u0120Snape": 31008, "\u0120191": 31009, "395": 31010, "\u0120299": 31011, "justice": 31012, "\u0120bri": 31013, "\u0120discs": 31014, "\u0120prominently": 31015, "\u0120vulgar": 31016, "\u0120skipping": 31017, "lves": 31018, "\u0120tsunami": 31019, "374": 31020, "\u0120Urug": 31021, "\u0120Eid": 31022, "recated": 31023, "phen": 31024, "\u0120faults": 31025, "\u0120Started": 31026, "950": 31027, "\u0120pi": 31028, "\u0120detector": 31029, "\u0120bastard": 31030, "\u0120validated": 31031, "SpaceEngineers": 31032, "OURCE": 31033, "\u0120(~": 31034, "\u0120unsur": 31035, "\u0120affirmed": 31036, "\u0120fascism": 31037, "\u0120resolving": 31038, "\u0120Chavez": 31039, "\u0120Cyn": 31040, "\u0120detract": 31041, "Lost": 31042, "\u0120rigged": 31043, "\u0120homage": 31044, "\u0120Bruno": 31045, "555": 31046, "eca": 31047, "\u0120presses": 31048, "\u0120humour": 31049, "\u0120spacing": 31050, "\u0120'/": 31051, "olkien": 31052, "Coun": 31053, "OPER": 31054, "Tre": 31055, "Son": 31056, "\u0120Cambodia": 31057, "ierre": 31058, "mong": 31059, "ozy": 31060, "\u0120liquidity": 31061, "\u0120Soviets": 31062, "\u0120Fernando": 31063, "\u0120229": 31064, "\u0120slug": 31065, "\u0120Catalan": 31066, "electric": 31067, "\u0120scenery": 31068, "\u0120Hearth": 31069, "\u0120constrained": 31070, "\u0120goalie": 31071, "\u0120Guidelines": 31072, "\u0120Ammo": 31073, "\u0120Pearson": 31074, "\u0120taxed": 31075, "\u0120fetus": 31076, "Response": 31077, "\u0120Alexis": 31078, "thia": 31079, "Guy": 31080, "\u0120reconstruct": 31081, "\u0120extremes": 31082, "\u0120concluding": 31083, "\u0120Peg": 31084, "ooks": 31085, "\u0120deductions": 31086, "Rose": 31087, "\u0120groundbreaking": 31088, "\u0120Targ": 31089, "\u00e3\u0125\u0123": 31090, "\u0120Reve": 31091, "resource": 31092, "\u0120moons": 31093, "\u0120electromagnetic": 31094, "\u0120amidst": 31095, "\u0120Viktor": 31096, "NESS": 31097, "BACK": 31098, "\u0120commute": 31099, "\u0120Anaheim": 31100, "\u0120fluctuations": 31101, "640": 31102, "\u0120noodles": 31103, "\u0120Copenhagen": 31104, "\u0120Tide": 31105, "\u0120Grizz": 31106, "\u0120SEE": 31107, "\u0120pipelines": 31108, "\u0120scars": 31109, "endo": 31110, "agus": 31111, "\u0120ETF": 31112, "/#": 31113, "\u0120Become": 31114, "448": 31115, "\u0120visc": 31116, "\u0120Recommended": 31117, "\u0120jumper": 31118, "\u0120cognition": 31119, "\u0120assassin": 31120, "\u0120witnessing": 31121, "\u0120Setup": 31122, "\u0120lac": 31123, "vim": 31124, "ISM": 31125, "pages": 31126, "SSL": 31127, "358": 31128, "\u0120adject": 31129, "industrial": 31130, "lore": 31131, "chery": 31132, "\u0120glitter": 31133, "\u0120calf": 31134, "Florida": 31135, "\u0120spoilers": 31136, "\u0120succeeds": 31137, "\u0120chanting": 31138, "\u0120slogans": 31139, "\u0120Tracy": 31140, "Visit": 31141, "rology": 31142, "\u0120mornings": 31143, "\u0120lineage": 31144, "\u0120sip": 31145, "\u0120intensely": 31146, "\u0120flourish": 31147, "\u0120Sleeping": 31148, "\u0120Fem": 31149, "orpor": 31150, "\u0120Klan": 31151, "\u0120Darth": 31152, "hack": 31153, "\u0120Nielsen": 31154, "\u0120tumors": 31155, "\u0120procurement": 31156, "\u0120Yorkshire": 31157, "\u0120raided": 31158, "KY": 31159, "Anna": 31160, "\u0120//[": 31161, "\u0120Disorder": 31162, "\u0120Mustang": 31163, "\u0120Wen": 31164, "\u0120Trying": 31165, "sq": 31166, "\u0120deliveries": 31167, "\u0120shutter": 31168, "\u0120cerebral": 31169, "\u0120bipolar": 31170, "\u0120CN": 31171, "lass": 31172, "jet": 31173, "\u0120debating": 31174, ">:": 31175, "\u0120eagle": 31176, "grades": 31177, "\u0120Dixon": 31178, "UGC": 31179, "MAS": 31180, "\u0120Draco": 31181, "\u0120Machines": 31182, "affer": 31183, "\u0120eman": 31184, "\u00c2\u00b2": 31185, "pron": 31186, "\u0120Gym": 31187, "\u0120comparatively": 31188, "\u0120Tribunal": 31189, "PRO": 31190, "\u0120lex": 31191, "\u0120fertile": 31192, "\u0120depressing": 31193, "\u0120superficial": 31194, "essential": 31195, "\u0120Hunters": 31196, "gp": 31197, "\u0120prominence": 31198, "Liber": 31199, "\u0120Ancest": 31200, "otechnology": 31201, "\u0120mocking": 31202, "\u0120Traff": 31203, "\u0138\u013c": 31204, "Medium": 31205, "Iraq": 31206, "\u0120psychiatrist": 31207, "Quantity": 31208, "\u0120Lect": 31209, "\u0120noisy": 31210, "520": 31211, "GY": 31212, "\u0120slapped": 31213, "\u0120MTV": 31214, "\u0120para": 31215, "pull": 31216, "Multiple": 31217, "asher": 31218, "\u0120nour": 31219, "\u0120Seg": 31220, "Spell": 31221, "vous": 31222, "ordial": 31223, "Senior": 31224, "\u0120Goldberg": 31225, "\u0120Plasma": 31226, "need": 31227, "\u0120messenger": 31228, "eret": 31229, "\u0120teamed": 31230, "\u0120literacy": 31231, "\u0120Leah": 31232, "\u0120Doyle": 31233, "\u0120emitted": 31234, "UX": 31235, "\u0120evade": 31236, "\u0120maze": 31237, "\u0120wrongly": 31238, "\u0120Lars": 31239, "\u0120stereotype": 31240, "\u0120pledges": 31241, "\u0120aroma": 31242, "\u0120MET": 31243, "\u0120acre": 31244, "\u0120OD": 31245, "\u0120ff": 31246, "\u0120breweries": 31247, "\u0120Hilton": 31248, "undle": 31249, "\u0120Kak": 31250, "\u0120Thankfully": 31251, "\u0120Canucks": 31252, "inctions": 31253, "\u0120Appears": 31254, "\u0120coer": 31255, "\u0120undermined": 31256, "rovers": 31257, "Andre": 31258, "\u0120blaze": 31259, "umers": 31260, "\u0120famine": 31261, "amphetamine": 31262, "ulkan": 31263, "Amount": 31264, "\u0120desperation": 31265, "wikipedia": 31266, "development": 31267, "\u0120Corinth": 31268, "ussia": 31269, "Jackson": 31270, "LI": 31271, "Native": 31272, "Rs": 31273, "Ohio": 31274, "\u0120Kathleen": 31275, "Fortunately": 31276, "\u0120attendant": 31277, "\u0120Preferred": 31278, "\u0120Didn": 31279, "\u0120Vs": 31280, "Mis": 31281, "\u0120respondent": 31282, "\u0120boun": 31283, "stable": 31284, "\u0120paved": 31285, "\u0120unexpl": 31286, "\u0120Cheney": 31287, "LM": 31288, "\u0120Cull": 31289, "blown": 31290, "\u0120confronting": 31291, "ocese": 31292, "serving": 31293, "Wi": 31294, "\u0120Lithuania": 31295, "anni": 31296, "\u0120stalk": 31297, "hd": 31298, "\u0120vener": 31299, "APH": 31300, "ynchronous": 31301, "URR": 31302, "umably": 31303, "historic": 31304, "Half": 31305, "Hay": 31306, "\u0120resilience": 31307, "spection": 31308, "\u0120abandoning": 31309, "Obs": 31310, "\u0120Debbie": 31311, "\u0120gradient": 31312, "\u0120Plaint": 31313, "\u0120Canal": 31314, "ARCH": 31315, "\u0120expansive": 31316, "\u0120fung": 31317, "\u0120bounced": 31318, "Und": 31319, "\u0120precautions": 31320, "\u0120clarification": 31321, "\u0120dagger": 31322, "\u0120grips": 31323, "\u0120\u00c2\u00b5": 31324, "\u0120Rivera": 31325, "\u0120Undead": 31326, "isites": 31327, "\u0120FIRST": 31328, "\u00c3\u00b1o": 31329, "audi": 31330, "\u0120hostages": 31331, "\u0120compliant": 31332, "\u0120alumni": 31333, "Seven": 31334, "\u0120cybersecurity": 31335, "either": 31336, "Collect": 31337, "\u0120invariably": 31338, "\u0120Soci": 31339, "\u0120lawmaker": 31340, "\u0120ale": 31341, "\u0120Personally": 31342, "Nazi": 31343, "\u0120customization": 31344, "\u0120Proc": 31345, "\u0120Saskatchewan": 31346, "eaturing": 31347, "\u0120spared": 31348, "\u0120discontinued": 31349, "\u0120computational": 31350, "\u0120Motorola": 31351, "\u0120supremacist": 31352, "governmental": 31353, "\u0120paradise": 31354, "\u0120Downing": 31355, "\u0120Nikon": 31356, "\u0120catalyst": 31357, "berra": 31358, "Toronto": 31359, "875": 31360, "beta": 31361, "\u0120Macron": 31362, "\u0120unrealistic": 31363, "vector": 31364, "\u0120Vehicles": 31365, "itiveness": 31366, "\u0120RV": 31367, "\u0120Colbert": 31368, "sin": 31369, "oji": 31370, "entin": 31371, "\u0120Krish": 31372, "hello": 31373, "ffield": 31374, "oky": 31375, "\u0120Tate": 31376, "\u0120maple": 31377, "\u0120aids": 31378, "chemical": 31379, "334": 31380, "nuts": 31381, "\u0120Warp": 31382, "\u0120xx": 31383, "\u0120Robb": 31384, "umerous": 31385, "_-_": 31386, "ftime": 31387, "\u0120VW": 31388, "\u0120winger": 31389, "\u0120Dome": 31390, "tools": 31391, "\u0120PV": 31392, "\u0120Georgetown": 31393, "\u0120geared": 31394, "\u0120jihadists": 31395, "\u0120cp": 31396, "\u0120steroids": 31397, "Mother": 31398, "clerosis": 31399, "\u0120DRM": 31400, "nesia": 31401, "\u0120linger": 31402, "\u0120immersive": 31403, "\u0120COUN": 31404, "\u0120outweigh": 31405, "ensual": 31406, "Band": 31407, "\u0120transforms": 31408, "matched": 31409, "psons": 31410, "\u0120Judicial": 31411, "factor": 31412, "\u0120referral": 31413, "\u0120oddly": 31414, "\u0120Wenger": 31415, "Bring": 31416, "\u0120Bows": 31417, "602": 31418, "ICLE": 31419, "\u0120lions": 31420, "\u0120Academic": 31421, "\u0120Thorn": 31422, "\u0120Raider": 31423, "kefeller": 31424, "Storage": 31425, "Lower": 31426, "\u0120Ort": 31427, "\u0120Equality": 31428, "ALT": 31429, "\u0120SOC": 31430, "Types": 31431, "\u0120lyn": 31432, "\u0120Asset": 31433, "coat": 31434, "TPP": 31435, "CVE": 31436, "\u0120Pioneer": 31437, "application": 31438, "Modern": 31439, "\u0120HK": 31440, "Environment": 31441, "Alright": 31442, "Rain": 31443, "IPP": 31444, "\u0120Shiite": 31445, "\u0120mound": 31446, "\u0120Abilities": 31447, "condition": 31448, "Staff": 31449, "\u0120competence": 31450, "\u0120Moor": 31451, "\u0120Diablo": 31452, "\u0120withheld": 31453, "\u0120ostensibly": 31454, "\u0120Brom": 31455, "\u0120msg": 31456, "\u0120denomin": 31457, "\u0120References": 31458, "\u0120FP": 31459, "\u0120plunged": 31460, "\u0120pamph": 31461, "moving": 31462, "central": 31463, "\u0120downright": 31464, "\u0120fading": 31465, "Tal": 31466, "Typ": 31467, "\u0120Thy": 31468, "ukes": 31469, "ithe": 31470, "\u0120ove": 31471, "\u0120battled": 31472, "\u0120seafood": 31473, "\u0120figur": 31474, "\u0120RD": 31475, "crop": 31476, "\u0120squads": 31477, "{\\": 31478, "\u00e0\u00b9": 31479, "\u0120Eh": 31480, "\u0120interviewing": 31481, "\u0120Qin": 31482, "\u0120aspiring": 31483, "PLIC": 31484, "\u0120clauses": 31485, "\u0120Gast": 31486, "\u0120Nir": 31487, "\u0120luggage": 31488, "\u0120hose": 31489, "\u0120systemd": 31490, "\u0120descending": 31491, "\u0120Revised": 31492, "\u0120Rails": 31493, "align": 31494, "709": 31495, "337": 31496, "\u0120fug": 31497, "charging": 31498, "tags": 31499, "\u0120uter": 31500, "kish": 31501, "WARNING": 31502, "490": 31503, "profits": 31504, "\u0120voyage": 31505, "\u0120ace": 31506, "\u0120Vanguard": 31507, "\u0120Tanks": 31508, "\u0120Muk": 31509, "\u0120226": 31510, "Safe": 31511, "Armor": 31512, "\u0120volcanic": 31513, "\u0120womb": 31514, "\u0120MIL": 31515, "\u0120beginner": 31516, "\u0120Recogn": 31517, "\u0120AAP": 31518, "PLAY": 31519, ")!": 31520, "\u0120detecting": 31521, "cn": 31522, "\u0120breaches": 31523, "Basically": 31524, "\u0120Pag": 31525, "\u0120Municipal": 31526, "\u0120Indie": 31527, "\u0120Laf": 31528, "\u0120Disable": 31529, "\u0120Olson": 31530, "\u0120restrained": 31531, "\u0120rulings": 31532, "\u0120humane": 31533, "events": 31534, "\u0120Cinema": 31535, "displayText": 31536, "\u0120Hatch": 31537, "actionDate": 31538, "onnaissance": 31539, "\u0120assaulting": 31540, "\u0120Lug": 31541, "CHAT": 31542, "\u0120vigorous": 31543, "\u0120Perse": 31544, "\u0120intolerance": 31545, "\u0120Snapchat": 31546, "\u0120Sharks": 31547, "\u0120dummy": 31548, "\u0120Diagn": 31549, "\u0120Guitar": 31550, "imeters": 31551, "403": 31552, "REG": 31553, "Ax": 31554, "\u0120separates": 31555, "\u0120Mahm": 31556, "\u0120tv": 31557, "jah": 31558, "OOL": 31559, "Circ": 31560, "\u0120Windsor": 31561, "ussian": 31562, "\u0120intuition": 31563, "\u0120disdain": 31564, "\u0120Donovan": 31565, "\u0120221": 31566, "Emb": 31567, "\u0120condemning": 31568, "\u0120generosity": 31569, "zzy": 31570, "\u0120panties": 31571, "\u0120Prevent": 31572, "ActionCode": 31573, "ANA": 31574, "342": 31575, "externalActionCode": 31576, "\u0120specifying": 31577, "\u0120crystall": 31578, "Jere": 31579, "\u0120rupt": 31580, "\u0120Apprentice": 31581, "\u0120profiling": 31582, "\u00d0\u00ba": 31583, "Strike": 31584, "\u0120sideline": 31585, "\u0120obligated": 31586, "\u0120occult": 31587, "\u0120bureaucratic": 31588, "antically": 31589, "rupted": 31590, "negative": 31591, "\u0120Ethiopia": 31592, "\u0120Civic": 31593, "\u0120insiders": 31594, "eligible": 31595, "\u0120TVs": 31596, "\u0120BAR": 31597, "\u0120TI": 31598, "iologist": 31599, "\u0120AIR": 31600, "\u0120substituted": 31601, "Arab": 31602, "\u0120Saul": 31603, "\u0120Yog": 31604, "prem": 31605, "\u0120builders": 31606, "\u0120stationary": 31607, "\u0120doubtful": 31608, "\u0120vigorously": 31609, "\u0120thrilling": 31610, "Physical": 31611, "\u0120Carey": 31612, "\u0120Hydra": 31613, "geoning": 31614, "\u0120Sly": 31615, "yton": 31616, "\u0120borrowers": 31617, "\u0120Parkinson": 31618, "\u0120\u00eb": 31619, "\u0120Jamaica": 31620, "\u0120satir": 31621, "\u0120insurgents": 31622, "\u0120Firm": 31623, "\u0120isot": 31624, "\u0120Karn": 31625, "ourning": 31626, "akens": 31627, "docs": 31628, "little": 31629, "\u0120Monaco": 31630, "CLASS": 31631, "Turkey": 31632, "Ly": 31633, "\u0120Conan": 31634, "assic": 31635, "\u0120starred": 31636, "\u0120Pacers": 31637, "eties": 31638, "\u0120tipping": 31639, "Moon": 31640, "\u0120Rw": 31641, "same": 31642, "\u0120cavity": 31643, "\u0120goof": 31644, "\u0120Zo": 31645, "Shock": 31646, "ummer": 31647, "\u0120emphasizes": 31648, "\u0120regrett": 31649, "\u0120novelty": 31650, "\u0120envy": 31651, "\u0120Passive": 31652, "rw": 31653, "505": 31654, "\u0120indifferent": 31655, "\u0120Rica": 31656, "\u0120Himself": 31657, "\u0120Freddie": 31658, "\u0120adip": 31659, "\u00e4\u00b8\u0122": 31660, "\u0120breakout": 31661, "\u0120hurried": 31662, "\u0120Huang": 31663, "\u0120Disk": 31664, "\u0120roaming": 31665, "?????-?????-": 31666, "UV": 31667, "\u0120Ricky": 31668, "\u0120Sigma": 31669, "\u0120marginalized": 31670, "\u0120edits": 31671, "\u0120304": 31672, "memory": 31673, "\u0120specimen": 31674, "293": 31675, "\u00e3\u0123\u00af": 31676, "\u0120vertically": 31677, "\u0120audition": 31678, "\u0120Heck": 31679, "\u0120caster": 31680, "\u0120Holdings": 31681, "adal": 31682, "\u0120Cron": 31683, "\u0120Liam": 31684, "\u0120deflect": 31685, "Pick": 31686, "\u0120Debug": 31687, "REF": 31688, "\u0120versatility": 31689, "othes": 31690, "classified": 31691, "\u0120Mahar": 31692, "\u0120Hort": 31693, "Counter": 31694, "stasy": 31695, "noticed": 31696, "331": 31697, "\u0120Shim": 31698, "fuck": 31699, "\u0120Bie": 31700, "\u0120airing": 31701, "\u0120Protein": 31702, "\u0120Holding": 31703, "\u0120spectators": 31704, "iliated": 31705, "\u0120Thatcher": 31706, "nosis": 31707, "\u00e3\u0125\u00bc\u00e3\u0125\u00b3": 31708, "Tele": 31709, "Boston": 31710, "\u0120Templ": 31711, "stay": 31712, "\u0120declarations": 31713, "479": 31714, "Volume": 31715, "\u0120Designer": 31716, "\u0120Overwatch": 31717, "idae": 31718, "\u0120onwards": 31719, "\u0120nets": 31720, "\u0120Manila": 31721, "particularly": 31722, "\u0120politic": 31723, "oother": 31724, "\u0120portraits": 31725, "\u0120pavement": 31726, "cffff": 31727, "\u0120saints": 31728, "\u0120beginners": 31729, "ESPN": 31730, "\u0120shortcomings": 31731, "\u00e2\u0137\u0132\u00e2\u0137\u0132": 31732, "\u0120comet": 31733, "\u0120Organic": 31734, "quel": 31735, "\u0120hospitalized": 31736, "Break": 31737, "\u0120peel": 31738, "dylib": 31739, "aspx": 31740, "urances": 31741, "\u0120TIM": 31742, "Pg": 31743, "\u0120readable": 31744, "\u0120Malik": 31745, "\u0120muzzle": 31746, "\u0120benchmarks": 31747, "dal": 31748, "\u0120Vacc": 31749, "\u0120Hicks": 31750, "609": 31751, "\u0120Biblical": 31752, "heng": 31753, "\u0120overload": 31754, "\u0120Civilization": 31755, "\u0120immoral": 31756, "\u0120fries": 31757, "\u00e3\u0124\u0134": 31758, "\u0120reproduced": 31759, "\u0120formulation": 31760, "jug": 31761, "irez": 31762, "gear": 31763, "\u0120coached": 31764, "MpServer": 31765, "\u0120SJ": 31766, "\u0120Kw": 31767, "Init": 31768, "deal": 31769, "\u0120Oro": 31770, "\u0120Loki": 31771, "\u0120Songs": 31772, "\u0120232": 31773, "\u0120Louise": 31774, "asionally": 31775, "\u0120uncond": 31776, "ollywood": 31777, "\u0120progressives": 31778, "\u0120Enough": 31779, "\u0120Doe": 31780, "\u0120wreckage": 31781, "\u0120brushed": 31782, "\u0120BaseType": 31783, "\u0120zoning": 31784, "ishable": 31785, "hetically": 31786, "\u0120Caucus": 31787, "\u0120Hue": 31788, "\u0120karma": 31789, "\u0120Sporting": 31790, "\u0120trader": 31791, "\u0120seeming": 31792, "\u0120Capture": 31793, "430": 31794, "bish": 31795, "\u0120tunes": 31796, "\u0120indoors": 31797, "\u0120Sphere": 31798, "\u0120Dancing": 31799, "TERN": 31800, "\u0120nob": 31801, "\u0120GST": 31802, "maps": 31803, "\u0120peppers": 31804, "Fit": 31805, "\u0120oversees": 31806, "\u0120Rabbi": 31807, "\u0120Ruler": 31808, "vertising": 31809, "office": 31810, "xxx": 31811, "\u0120raft": 31812, "Changed": 31813, "\u0120textbooks": 31814, "Links": 31815, "\u0120Omn": 31816, "\u00e3\u0122\u0133": 31817, "\u0120inconvenience": 31818, "\u0120Donetsk": 31819, "=~": 31820, "\u0120implicitly": 31821, "\u0120boosts": 31822, "\u0120Bones": 31823, "\u0120Boom": 31824, "Courtesy": 31825, "\u0120sensational": 31826, "ANY": 31827, "\u0120greedy": 31828, "eden": 31829, "\u0120inexper": 31830, "\u0120Ler": 31831, "\u0120Vale": 31832, "\u0120tighten": 31833, "\u0120EAR": 31834, "\u0120Num": 31835, "\u0120ancestor": 31836, "Sent": 31837, "\u0120Horde": 31838, "urgical": 31839, "allah": 31840, "\u0120sap": 31841, "amba": 31842, "\u0120Spread": 31843, "twitch": 31844, "\u0120grandson": 31845, "\u0120fracture": 31846, "\u0120moderator": 31847, "\u0120Seventh": 31848, "\u0120Reverse": 31849, "\u0120estimation": 31850, "Choose": 31851, "\u0120parach": 31852, "\u0120barric": 31853, "\u00e3\u0122\u0132": 31854, "\u0120compass": 31855, "\u0120allergic": 31856, "\u00e2\u0122\u0137": 31857, "OTHER": 31858, "errilla": 31859, "\u0120wagon": 31860, "\u0120zinc": 31861, "\u0120rubbed": 31862, "\u0120Fuller": 31863, "\u0120Luxembourg": 31864, "\u0120Hoover": 31865, "\u0120liar": 31866, "\u0120Evening": 31867, "\u0120Cobb": 31868, "esteem": 31869, "\u0120selector": 31870, "\u0120Brawl": 31871, "isance": 31872, "\u0120Ek": 31873, "\u0120troop": 31874, "\u0120guts": 31875, "\u0120Appeal": 31876, "\u0120Tibetan": 31877, "\u0120routines": 31878, "\u0120Ment": 31879, "\u0120summarized": 31880, "steamapps": 31881, "\u0120tranqu": 31882, "\u01201929": 31883, "oran": 31884, "\u0120Authent": 31885, "\u0120gmaxwell": 31886, "\u0120apprehens": 31887, "\u0120poems": 31888, "\u0120sausage": 31889, "\u0120Webster": 31890, "urus": 31891, "\u0120themed": 31892, "\u0120lounge": 31893, "\u0120charger": 31894, "Spoiler": 31895, "\u0120spilled": 31896, "hog": 31897, "\u0120Sunder": 31898, "\u0120Ain": 31899, "\u0120Angry": 31900, "\u0120disqual": 31901, "\u0120Frequency": 31902, "\u0120Ethernet": 31903, "\u0120helper": 31904, "Percent": 31905, "\u0120horrifying": 31906, "\u0120ail": 31907, "\u0120Allan": 31908, "EEE": 31909, "\u0120Crossing": 31910, "449": 31911, "\u0120holog": 31912, "\u0120Puzzles": 31913, "\u0120Goes": 31914, "erenn": 31915, "604": 31916, "\u00e3\u0123\u0131": 31917, "\u0120Rafael": 31918, "\u0120atten": 31919, "\u0120Emanuel": 31920, "\u0120upro": 31921, "\u0120Susp": 31922, "Psych": 31923, "\u0120Trainer": 31924, "\u0120NES": 31925, "\u0120Hunts": 31926, "becue": 31927, "\u0120counselor": 31928, "Rule": 31929, "\u0120toxins": 31930, "\u0120banners": 31931, "rifice": 31932, "\u0120greeting": 31933, "\u0120frenzy": 31934, "\u0120allocate": 31935, "\u0120*)": 31936, "expr": 31937, "503": 31938, "\u0120Chick": 31939, "\u0120Torn": 31940, "\u0120consolidation": 31941, "\u0120Fletcher": 31942, "switch": 31943, "frac": 31944, "clips": 31945, "\u0120McKin": 31946, "\u0120Lunar": 31947, "Month": 31948, "ITCH": 31949, "\u0120scholarly": 31950, "raped": 31951, "398": 31952, "\u01201910": 31953, "\u0120egreg": 31954, "\u0120insecure": 31955, "\u0120victorious": 31956, "cffffcc": 31957, "\u0120singled": 31958, "\u0120elves": 31959, "\u0120Wond": 31960, "burst": 31961, "\u0120camoufl": 31962, "\u0120BLACK": 31963, "\u0120conditioned": 31964, "\u00e7\u012b": 31965, "answered": 31966, "\u0120compulsory": 31967, "ascist": 31968, "\u0120podcasts": 31969, "\u0120Frankfurt": 31970, "bnb": 31971, "\u0120neoliberal": 31972, "\u0120Keyboard": 31973, "\u0120Belle": 31974, "warm": 31975, "\u0120trusts": 31976, "\u0120insured": 31977, "\u0120Bucc": 31978, "usable": 31979, "607": 31980, "\u0120Plains": 31981, "\u01201890": 31982, "\u0120sabotage": 31983, "\u0120lodged": 31984, "felt": 31985, "\u0120ga": 31986, "\u0120Narc": 31987, "\u0120Salem": 31988, "\u0120seventy": 31989, "\u0120Blank": 31990, "pocket": 31991, "\u0120whisper": 31992, "\u0120mating": 31993, "omics": 31994, "\u0120Salman": 31995, "\u0120Kad": 31996, "\u0120angered": 31997, "\u0120collisions": 31998, "\u0120extraordinarily": 31999, "\u0120coercion": 32000, "Ghost": 32001, "birds": 32002, "\u00e8\u0122": 32003, "kok": 32004, "\u0120permissible": 32005, "avorable": 32006, "\u0120pointers": 32007, "\u0120dissip": 32008, "aci": 32009, "\u0120theatrical": 32010, "\u0120Cosmic": 32011, "\u0120forgetting": 32012, "\u0120finalized": 32013, "\u00e5\u00a4\u00a7": 32014, "yout": 32015, "library": 32016, "\u0120booming": 32017, "\u0120Believe": 32018, "\u0120Teacher": 32019, "\u0120Liv": 32020, "\u0120GOODMAN": 32021, "\u0120Dominican": 32022, "ORED": 32023, "\u0120Parties": 32024, "\u0120precipitation": 32025, "\u0120Slot": 32026, "Roy": 32027, "\u0120Combined": 32028, "\u0120integrating": 32029, "\u0120chrome": 32030, "\u0120intestinal": 32031, "\u0120Rebell": 32032, "\u0120matchups": 32033, "\u0120blockbuster": 32034, "\u0120Loren": 32035, "\u0120Levy": 32036, "\u0120preaching": 32037, "\u0120Sending": 32038, "\u0120Purpose": 32039, "rax": 32040, "fif": 32041, "\u0120authoritative": 32042, "\u0120PET": 32043, "astical": 32044, "\u0120dishon": 32045, "\u0120chatting": 32046, "\u0120\"$:/": 32047, "Connection": 32048, "\u0120recreate": 32049, "\u0120delinqu": 32050, "\u0120broth": 32051, "\u0120Dirty": 32052, "\u0120Admin": 32053, "zman": 32054, "\u0120scholarships": 32055, "\u0120253": 32056, "contact": 32057, "alsa": 32058, "767": 32059, "creen": 32060, "abbage": 32061, "\u01201915": 32062, "\u0120blended": 32063, "\u0120alarmed": 32064, "Language": 32065, "356": 32066, "\u0120blends": 32067, "\u0120Changed": 32068, "Wolf": 32069, "\u0120hepat": 32070, "Creating": 32071, "\u0120persecut": 32072, "\u0120sweetness": 32073, "arte": 32074, "\u0120forfeiture": 32075, "\u0120Roberto": 32076, "impro": 32077, "NFL": 32078, "\u0120Magnet": 32079, "Detailed": 32080, "\u0120insignificant": 32081, "\u0120POLIT": 32082, "\u0120BBQ": 32083, "\u0120CPS": 32084, "\u0120seaw": 32085, "aminer": 32086, "mL": 32087, "endif": 32088, "finals": 32089, "\u0120265": 32090, "uish": 32091, "\u0120})": 32092, "\u0120Problems": 32093, "\u0120emblem": 32094, "\u0120seriousness": 32095, "\u0120parsing": 32096, "\u0120substitution": 32097, "\u0120pressured": 32098, "\u0120recycled": 32099, "aleb": 32100, "Ruby": 32101, "\u0120proficiency": 32102, "Driver": 32103, "\u0120Wester": 32104, ":'": 32105, "AFTA": 32106, "\u0120mantle": 32107, "\u0120Clayton": 32108, "flag": 32109, "\u0120practitioner": 32110, "covered": 32111, "\u0120Struct": 32112, "addafi": 32113, "425": 32114, "\u0120Township": 32115, "\u0120Hydro": 32116, "Louis": 32117, "343": 32118, "\u0120condo": 32119, "\u0120Tao": 32120, "\u0120utilization": 32121, "\u0120nausea": 32122, "\u0120Dems": 32123, "ridges": 32124, "pause": 32125, "\u0120formulas": 32126, "\u0120challenger": 32127, "376": 32128, "\u0120defective": 32129, "\u0120Railway": 32130, "\u0120PubMed": 32131, "\u0120yogurt": 32132, "lbs": 32133, "\u0120Norfolk": 32134, "OPE": 32135, "\u0120Moody": 32136, "\u0120distributor": 32137, "\u0120scrolls": 32138, "\u0120extracts": 32139, "Stan": 32140, "\u0120viability": 32141, "\u0120exposes": 32142, "\u0120starvation": 32143, "\u0120Steps": 32144, "\u0120Dodd": 32145, "few": 32146, "STD": 32147, "332": 32148, "\u0120closures": 32149, "\u0120complementary": 32150, "\u0120Sasha": 32151, "umpy": 32152, "\u0120monet": 32153, "\u0120articulate": 32154, "\u0120Doct": 32155, "killer": 32156, "\u0120scrim": 32157, "\u0120264": 32158, "\u0120prostitutes": 32159, "\u0120severed": 32160, "\u0120attachments": 32161, "\u0120cooled": 32162, "Lev": 32163, "\u0120Falk": 32164, "fail": 32165, "\u0120policeman": 32166, "\u0120Dag": 32167, "\u0120prayed": 32168, "\u0120Kernel": 32169, "\u0120clut": 32170, "\u0120cath": 32171, "\u0120anomaly": 32172, "Storm": 32173, "emaker": 32174, "\u0120Breakfast": 32175, "uli": 32176, "oire": 32177, "JJ": 32178, "hz": 32179, "Operation": 32180, "\u0120Sick": 32181, "354": 32182, "\u0120Guatemala": 32183, "Rate": 32184, "\u0120exposures": 32185, "faces": 32186, "\u0120Archae": 32187, "raf": 32188, "\u0120Mia": 32189, "\u01202025": 32190, "\u0120opaque": 32191, "\u0120disguised": 32192, "\u0120Headquarters": 32193, "Sah": 32194, "\u0120pots": 32195, "978": 32196, "\u0120Malf": 32197, "\u0120frowned": 32198, "\u0120poisonous": 32199, "\u0120Convers": 32200, "eeks": 32201, "\u0120crab": 32202, ".\"\"": 32203, "\u0120treason": 32204, "\u0120ranc": 32205, "\u0120escalating": 32206, "\u0120warr": 32207, "\u0120mobs": 32208, "\u0120lamps": 32209, "\u0120Sunshine": 32210, "\u0120Brunswick": 32211, "Phones": 32212, "\u0120spelled": 32213, "\u0120Skip": 32214, "\u01202050": 32215, "\u01201911": 32216, "\u0120Pluto": 32217, "\u0120Amend": 32218, "\u0120meats": 32219, "387": 32220, "\u0120stomp": 32221, "\u0120Zhou": 32222, "\u0120Leviathan": 32223, "\u0120Hazard": 32224, "adv": 32225, "\u0120Orwell": 32226, "\u0120aloud": 32227, "\u0120bumper": 32228, "\u0120Anarch": 32229, "ubuntu": 32230, "\u0120Serious": 32231, "fitting": 32232, "\u0120Optional": 32233, "\u0120Cecil": 32234, "REAM": 32235, "\u0120serotonin": 32236, "\u0120cultivate": 32237, "agogue": 32238, "}\\": 32239, "\u0120mosques": 32240, "\u0120Sunny": 32241, "\u0120reactive": 32242, "revolution": 32243, "\u0120Lup": 32244, "\u0120Fedora": 32245, "\u0120defenseman": 32246, "\u0120VID": 32247, "istine": 32248, "\u0120drowning": 32249, "\u0120Broadcasting": 32250, "\u0120thriller": 32251, "\u0120Scy": 32252, "\u0120accelerating": 32253, "\u0120directs": 32254, "odied": 32255, "bike": 32256, "duration": 32257, "\u0120painfully": 32258, "Redd": 32259, "\u0120productions": 32260, "\u0120gag": 32261, "\u0120whist": 32262, "\u0120sock": 32263, "\u0120infinitely": 32264, "\u0120Concern": 32265, "\u0120Citadel": 32266, "\u0120lieu": 32267, "\u0120candles": 32268, "ogeneous": 32269, "arger": 32270, "\u0120heavenly": 32271, "inflammatory": 32272, "Performance": 32273, "Cs": 32274, "ructose": 32275, "azaki": 32276, "\u0120pessim": 32277, "\u0120inference": 32278, "\u0120powd": 32279, "\u0120Zoe": 32280, "\u0120paints": 32281, "\u0120dazz": 32282, "pta": 32283, "-----------": 32284, "\u0120inspir": 32285, "\u0120Experimental": 32286, "\u0120Knife": 32287, "regor": 32288, "bors": 32289, "\u0120showers": 32290, "romeda": 32291, "\u0120saint": 32292, "\u0120benign": 32293, "\u0120Jiang": 32294, "\u0120envisioned": 32295, "\u0120shroud": 32296, "IFT": 32297, "HO": 32298, "\u0120shuff": 32299, "\u0120ICC": 32300, "\u0120segreg": 32301, "\u0120revisit": 32302, "ighthouse": 32303, "Li": 32304, "\u0120substrate": 32305, "\u0120Seas": 32306, "\u0120Reward": 32307, "\u0120Hep": 32308, "\u0120Brass": 32309, "sbm": 32310, "\u0120eliminates": 32311, "\u0120stamina": 32312, "\u0120VAT": 32313, "\u0120Loan": 32314, "\u0120constraint": 32315, "\u0120appropriated": 32316, "\u0120pes": 32317, "\u0120ALE": 32318, "ranging": 32319, "\u0120404": 32320, "392": 32321, "\u0120intellectuals": 32322, "achu": 32323, "\u0120restructuring": 32324, "\u0120Levin": 32325, "\u0120runes": 32326, "\u0120delightful": 32327, "\u0120carbohydrates": 32328, "\u0120Models": 32329, "\u0120Expo": 32330, "\u0120transporting": 32331, "alloc": 32332, "\u0120ringing": 32333, "Samsung": 32334, "\u0120scarcely": 32335, "\u0120URLs": 32336, "\u0120MAS": 32337, "\u0120prototypes": 32338, "\u0120narrator": 32339, "\u0120CPUs": 32340, "cdn": 32341, "\u0120Barton": 32342, "\u0120decidedly": 32343, "\u0120Shu": 32344, "ixir": 32345, "ocious": 32346, "\u0120Myst": 32347, "Nintendo": 32348, "\u0120reuse": 32349, "\u0120forgiven": 32350, "Few": 32351, "inical": 32352, "nat": 32353, "\u0120seamless": 32354, "\u0120Eva": 32355, "\u0120EVE": 32356, "\u0120JO": 32357, "landers": 32358, "\u0120softer": 32359, "negie": 32360, "\u0120transient": 32361, "\u0120orbital": 32362, "\u0120fulfil": 32363, "\u0120Kom": 32364, "Hopefully": 32365, "\u0120dynamically": 32366, "\u0120Hunger": 32367, "\u00e5\u013d": 32368, "\u0120Armenia": 32369, "elman": 32370, "berto": 32371, "\u0120pige": 32372, "\u0120IDs": 32373, "limit": 32374, "\u0120veins": 32375, "\u0120soaring": 32376, "packs": 32377, "Golden": 32378, "\u0120Crab": 32379, "istor": 32380, "\u0120RPM": 32381, "\u0120$$": 32382, "gression": 32383, "\u0120jihadist": 32384, "\u0120gamble": 32385, "\u0120careg": 32386, "\u0120inflated": 32387, "Face": 32388, "\u0120Firearms": 32389, "\u0120Emmanuel": 32390, "\u00e2\u013f": 32391, "\u0120shocks": 32392, "grab": 32393, "\u0120splend": 32394, "\u0120HPV": 32395, "abortion": 32396, "Above": 32397, "Entity": 32398, "players": 32399, "\u0120commenced": 32400, "ulence": 32401, "\u0120fulfillment": 32402, "\u0120embodiments": 32403, "\u0120Welfare": 32404, "\u0120hail": 32405, "\u0120<@": 32406, "tten": 32407, "\u0120catcher": 32408, "\u0120Jazeera": 32409, "\u0120volcano": 32410, "\u0120stabilize": 32411, "\u0120Handler": 32412, "\u0120intensified": 32413, "\u0120Abrams": 32414, "\u0120humiliation": 32415, "paced": 32416, "605": 32417, "\u0120CentOS": 32418, "Specific": 32419, "\u0120heed": 32420, "\u0120CAM": 32421, "\u0120Galile": 32422, "Die": 32423, "\u0120abolished": 32424, "\u0120Thomson": 32425, "\u0120Teachers": 32426, "\u0120Wass": 32427, "jong": 32428, "\u0120ISBN": 32429, "\u0120Allies": 32430, "shake": 32431, "\u00e5\u00b7": 32432, "vict": 32433, "Howard": 32434, "\u0120deem": 32435, "\u0120exceedingly": 32436, "\u0120Smartstocks": 32437, "ibe": 32438, "\u0120doorway": 32439, "\u0120competed": 32440, "igmat": 32441, "\u0120nationalists": 32442, "\u0120groom": 32443, "\u0120Keen": 32444, "\u0120disposable": 32445, "decl": 32446, "\u0120Tolkien": 32447, "\u0120Scheme": 32448, "\u0120biod": 32449, "\u0120avid": 32450, "\u0120Elon": 32451, "agar": 32452, "\u0120TSA": 32453, "Roman": 32454, "\u0120artificially": 32455, "\u0120advisors": 32456, "XL": 32457, "\u0120Inferno": 32458, "366": 32459, "\u0120tedious": 32460, "\u0120Photography": 32461, "\u0120Carrie": 32462, "\u0120trope": 32463, "\u0120Sandra": 32464, "\u0120decimal": 32465, "Queen": 32466, "\u0120Gundam": 32467, "\u0120OM": 32468, "otech": 32469, "NBA": 32470, "\u01201932": 32471, "\u0120entrenched": 32472, "\u0120Marion": 32473, "\u0120fraternity": 32474, "Labour": 32475, "Henry": 32476, "\u0120latitude": 32477, "Either": 32478, "\u0120enhances": 32479, "\u0120Potential": 32480, "\u0120shines": 32481, "idad": 32482, "\u0120breadth": 32483, "\u0120capacities": 32484, "\u0120\u00f0\u0141\u013b\u0124": 32485, "\u0120Bronx": 32486, "\u0120sexes": 32487, "\u0120differentiation": 32488, "\u0120heavyweight": 32489, "\u0120Taj": 32490, "dra": 32491, "\u0120migrate": 32492, "\u0120exhaustion": 32493, "\u0120RUN": 32494, "elsius": 32495, "\u0120Cuomo": 32496, "\u0120guitars": 32497, "\u0120clones": 32498, "\u0120Somew": 32499, "\u0120Pry": 32500, "-------------": 32501, "\u0120warranted": 32502, "cycles": 32503, "\u0120salvage": 32504, "\u0120disks": 32505, "RANT": 32506, "\u0120NGOs": 32507, "\u0120Martian": 32508, "\":[{\"": 32509, "\u0120addicts": 32510, "ojure": 32511, "illet": 32512, "\u0120amazingly": 32513, "artments": 32514, "pixel": 32515, "\u0120GPUs": 32516, "Layout": 32517, "\u00e8\u00a3": 32518, "\u0120Tamil": 32519, "\u0120Basil": 32520, "\u0120impartial": 32521, "\u0120Structure": 32522, "fork": 32523, "bryce": 32524, "\u0120ridge": 32525, "\u0120Hamburg": 32526, "rious": 32527, "\u0120blitz": 32528, "cigarettes": 32529, "\u0120canned": 32530, "402": 32531, "\u0120ironically": 32532, "\u0120compassionate": 32533, "\u0120Hawkins": 32534, ".#": 32535, "\u0120Cathedral": 32536, "\u0120rallied": 32537, "internal": 32538, "\u0120quota": 32539, "stakes": 32540, "TEXT": 32541, "mom": 32542, "\u0120completes": 32543, "\u0120238": 32544, "\u0120shrug": 32545, "\u00e3\u0125\u0133": 32546, "\u0120Ninth": 32547, "\u0120revise": 32548, "\u0120Provider": 32549, "\u0120treacher": 32550, "\u0120quasi": 32551, "\u0120PRES": 32552, "\u0120deposition": 32553, "\u0120confidentiality": 32554, "issors": 32555, "\u0120imbalance": 32556, "\u0120spanning": 32557, "\u0120angular": 32558, "\u0120Cul": 32559, "communication": 32560, "\u0120Nora": 32561, "\u0120Genius": 32562, "opter": 32563, "\u0120sacked": 32564, "Spot": 32565, "\u0120finely": 32566, "\u0120CHR": 32567, "282": 32568, "waves": 32569, "Palest": 32570, "\u0120Rohing": 32571, "NL": 32572, "\u00e8\u00bf": 32573, "\u0120shitty": 32574, "\u0120Scalia": 32575, "475": 32576, "Progress": 32577, "\u0120referencing": 32578, "\u0120classrooms": 32579, "abee": 32580, "\u0120sod": 32581, "hesion": 32582, "708": 32583, "\u0120Zuckerberg": 32584, "\u0120Finish": 32585, "\u0120Scotia": 32586, "\u0120Savior": 32587, "\u0120Installation": 32588, "antha": 32589, "(-": 32590, "\u0120302": 32591, "\u0120Punk": 32592, "\u0120crater": 32593, "youtu": 32594, "\u0120roast": 32595, "\u0120influencing": 32596, "\u0120dup": 32597, "\u0120JR": 32598, "\u0120Grav": 32599, "\u0120stature": 32600, "\u0120bathrooms": 32601, "Aside": 32602, "Wiki": 32603, "mean": 32604, "\u0120Zak": 32605, "\u0120Ones": 32606, "\u0120Nath": 32607, "\u0120hypert": 32608, "\u0120commencement": 32609, "Civil": 32610, "\u0120moderately": 32611, "\u0120distributors": 32612, "\u0120breastfeeding": 32613, "\u0120980": 32614, "\u0120Sik": 32615, "\u0120Cig": 32616, "\u0120AMER": 32617, "RIP": 32618, "\u0120Career": 32619, "usting": 32620, "\u0120messed": 32621, "\u0120eh": 32622, "\u0120Jensen": 32623, "/$": 32624, "\u0120blackmail": 32625, "\u0120conversions": 32626, "\u0120scientifically": 32627, "\u0120mantra": 32628, "paying": 32629, "\u0120ivory": 32630, "\u0120Courts": 32631, "OUGH": 32632, "auntlet": 32633, "Serial": 32634, "Brow": 32635, "\u0120Hundreds": 32636, "323": 32637, "\u0120pee": 32638, "\u0120linux": 32639, "\u0120submer": 32640, "\u0120Principal": 32641, "485": 32642, "\u0120DSL": 32643, "\u0120Cousins": 32644, "\u0120doctrines": 32645, "\u0120Athletics": 32646, "\u0120315": 32647, "\u0120Karma": 32648, "\u0120attent": 32649, "urger": 32650, "\u0120prescribe": 32651, "\u0120encaps": 32652, "\u0120Came": 32653, "\u0120secretive": 32654, "\u0120Crimes": 32655, "dn": 32656, "Clean": 32657, "\u0120Egyptians": 32658, "\u0120Carpenter": 32659, "\u0120ll": 32660, "Hum": 32661, "\u0120Milo": 32662, "\u0120capitalists": 32663, "\u0120briefed": 32664, "Twe": 32665, "\u0120Basin": 32666, "elvet": 32667, "Mos": 32668, "\u0120plunge": 32669, "\u0120Kaiser": 32670, "\u0120Fuj": 32671, "illin": 32672, "\u0120safeguards": 32673, "\u0120oste": 32674, "\u0120Opportunity": 32675, "\u0120Mafia": 32676, "\u0120Calling": 32677, "apa": 32678, "urban": 32679, "brush": 32680, "illard": 32681, "c\u00c3\u00a9": 32682, "intelligence": 32683, "\u0120Lob": 32684, "\u0120Druid": 32685, "\u0120smoother": 32686, "\u0120footing": 32687, "\u0120motorists": 32688, "arcity": 32689, "\u0120masculinity": 32690, "\u0120mism": 32691, "\u0120abdominal": 32692, "\u0120Tavern": 32693, "\u0120Roh": 32694, "\u0120escapes": 32695, "signed": 32696, "Anthony": 32697, "\u0120sacrificing": 32698, "\u0120intimacy": 32699, "\u0120anterior": 32700, "\u0120Kod": 32701, "\u0120motif": 32702, "\u0120graz": 32703, "\u0120visualization": 32704, "\u0120guitarist": 32705, "\u0120Trotsky": 32706, "magic": 32707, "Dar": 32708, "\u0120Mori": 32709, "\u0120wards": 32710, "\u0120toilets": 32711, "lest": 32712, "\u0120teleport": 32713, "\u0120Sundays": 32714, "\u0120Plat": 32715, "ETS": 32716, "\u0120eSports": 32717, "Patrick": 32718, "\u0120Katherine": 32719, "enko": 32720, "\u0120hassle": 32721, "\u0120Mick": 32722, "ggles": 32723, "\u0120hob": 32724, "aintain": 32725, "\u0120airborne": 32726, "\u0120spans": 32727, "\u0120chili": 32728, "\u0120aperture": 32729, "\u0120volunteered": 32730, "\u0120Incident": 32731, "\u0120Fres": 32732, "\u0120Veteran": 32733, "aughtered": 32734, "ingo": 32735, "\u0120uninsured": 32736, "CLOSE": 32737, "\u0120fuse": 32738, "\u0120erotic": 32739, "\u0120advertise": 32740, "raising": 32741, "Texture": 32742, "\u0120attends": 32743, "\u0120REAL": 32744, "uddled": 32745, "\u0120smoot": 32746, "\u0120305": 32747, "\u0120Willis": 32748, "\u0120blond": 32749, "Analysis": 32750, "\u0120VT": 32751, "onica": 32752, "\u0120stronghold": 32753, "RF": 32754, "NM": 32755, ".>>": 32756, "\u0120prosperous": 32757, "\u0120boasted": 32758, "292": 32759, "\u0120Manufacturing": 32760, "PRESS": 32761, "gren": 32762, "\u0120pharmacy": 32763, "\u0120Rockefeller": 32764, "kai": 32765, "\u0120thumbs": 32766, "\u0120Hut": 32767, "\u0120motherboard": 32768, "\u0120guardians": 32769, "\u0120Alter": 32770, "llular": 32771, "\u0120shack": 32772, "\u0120wisely": 32773, "\u0120backbone": 32774, "erva": 32775, "\u0120suicides": 32776, "\u0120McGregor": 32777, "ijah": 32778, "Emer": 32779, "\u0120Brav": 32780, "\u0120designate": 32781, "POST": 32782, "produced": 32783, "\u0120cleansing": 32784, "irlwind": 32785, "existent": 32786, "\u0120Humph": 32787, "\u0120Payne": 32788, "\u0120vested": 32789, "\u00c5\u00a1": 32790, "\u0120stringent": 32791, "iona": 32792, "\u0120unsub": 32793, "\u0120summed": 32794, "\u0120Hercules": 32795, "subject": 32796, "\u0120Ragnar": 32797, "\u0120Nos": 32798, "\u0120characterization": 32799, "\u0120savvy": 32800, "\u0120Dawson": 32801, "\u0120Casino": 32802, "\u0120fri": 32803, "\u0120Barrier": 32804, "\u0120misinformation": 32805, "\u0120insulation": 32806, "\u0120corridors": 32807, "\u0120airplanes": 32808, "\u0120Noct": 32809, "ahi": 32810, "\u01201916": 32811, "kb": 32812, "armac": 32813, "\u0120shun": 32814, "\u0120schema": 32815, "\u0120horrified": 32816, "\u0120239": 32817, "aunders": 32818, "NB": 32819, "iates": 32820, "erity": 32821, "\u0120Shard": 32822, "\u0120rarity": 32823, "\u0120grouped": 32824, "\u0120Ghana": 32825, "against": 32826, "\u0120Biological": 32827, "\u0120Aware": 32828, "owell": 32829, "\u00cf\u0126": 32830, "\u0120Beau": 32831, "shaw": 32832, "Hack": 32833, "\u0120Julius": 32834, "USS": 32835, "olson": 32836, "auna": 32837, "cru": 32838, "\u0120Maurice": 32839, "\u0120Ik": 32840, "\u0120sequencing": 32841, "\u0120radicals": 32842, "\u0120(?,": 32843, "virtual": 32844, "\u0120anyways": 32845, "\u0120reperc": 32846, "\u0120handlers": 32847, "\u0120hesitant": 32848, "\u00e9\u0125": 32849, "\u0120MF": 32850, "plementation": 32851, "associated": 32852, "\u0120campaigned": 32853, "\u0120Yue": 32854, "utations": 32855, "\u0120Yoga": 32856, "\u0120simmer": 32857, "\u0120rods": 32858, "\u0120melody": 32859, "\u0120convoy": 32860, "videos": 32861, "\u0120screened": 32862, "Neg": 32863, "ochemical": 32864, "\u0120())": 32865, "\u0120ultras": 32866, "\u0120antip": 32867, "\u0120Islanders": 32868, "704": 32869, "\u0120fetish": 32870, "\u0120ridiculously": 32871, "\u0120Kart": 32872, "\u0120mitochondrial": 32873, "\u0120interfering": 32874, "Builder": 32875, "\u0120overfl": 32876, "\u0120acne": 32877, "\u0120Mud": 32878, "\u0120Kerr": 32879, "flex": 32880, "\u0120Postal": 32881, "\u0120Baltic": 32882, "477": 32883, "\u0120Persons": 32884, "ourage": 32885, "HB": 32886, "\u0120Muse": 32887, "\u0120Immortal": 32888, "\u0120Driving": 32889, "\u0120petitions": 32890, "\u0120subscript": 32891, "\u0120sorce": 32892, "\u0120Processor": 32893, "uton": 32894, "Sony": 32895, "\u0120phon": 32896, "\u0120raced": 32897, "\u0120Anthrop": 32898, "\u0120daytime": 32899, "\u0120Exercise": 32900, "Adding": 32901, "\u0120engages": 32902, "\u0120Qualcomm": 32903, "\u0120miracles": 32904, "\u0120memes": 32905, "\u0120Drink": 32906, "\u0120Orioles": 32907, "\u0120hairs": 32908, "\u0120Polar": 32909, "athom": 32910, "\u0120slippery": 32911, "\u0120Remy": 32912, "\u0120caramel": 32913, "\u0120YEAR": 32914, "\u0120alk": 32915, "Ign": 32916, "aution": 32917, "\u0120Merlin": 32918, "\u0120Cran": 32919, "\u0120apologies": 32920, "\u0120410": 32921, "\u0120outing": 32922, "\u0120Memories": 32923, "appointed": 32924, "\u0120countered": 32925, "uld": 32926, "posing": 32927, "\u0120firewall": 32928, "\u0120Wast": 32929, "\u0120Wet": 32930, "worked": 32931, "seller": 32932, "\u0120repealed": 32933, "ereo": 32934, "assuming": 32935, "BLIC": 32936, "mite": 32937, "\u0120CEOs": 32938, "\u0120Chapel": 32939, "elligent": 32940, "________________________": 32941, "Dog": 32942, "\u0120wart": 32943, "\u0120subscriber": 32944, "sports": 32945, "\u0120begged": 32946, "\u0120MV": 32947, "\u0120semif": 32948, "ethical": 32949, "\u0120preach": 32950, "\u0120revital": 32951, "\u0120punitive": 32952, "\u0120shortcuts": 32953, "\u0120instituted": 32954, "\u0120Warsaw": 32955, "\u0120abdomen": 32956, "\u0120KING": 32957, "\u0120superintendent": 32958, "\u0120fry": 32959, "\u0120Geo": 32960, "TOR": 32961, "\u0120contradictions": 32962, "aptic": 32963, "\u0120landscapes": 32964, "bugs": 32965, "\u0120clust": 32966, "\u0120volley": 32967, "cribed": 32968, "\u0120tandem": 32969, "\u0120robes": 32970, "WHAT": 32971, "\u0120promoter": 32972, "\u0120eloqu": 32973, "reviewed": 32974, "\u0120DK": 32975, "\u0120Plato": 32976, "\u0120fps": 32977, "Tank": 32978, "\u0120Derrick": 32979, "\u0120prioritize": 32980, "asper": 32981, "\u0120Honduras": 32982, "\u0120Completed": 32983, "nec": 32984, "\u0120mog": 32985, "nir": 32986, "\u0120Mayo": 32987, "DEF": 32988, "stall": 32989, "inness": 32990, "\u0120Volkswagen": 32991, "\u0120precaution": 32992, "\u0120Mell": 32993, "iak": 32994, "istries": 32995, "\u0120248": 32996, "\u0120overlapping": 32997, "Senate": 32998, "\u0120Enhance": 32999, "resy": 33000, "racial": 33001, "ORTS": 33002, "\u0120Mormons": 33003, "Strong": 33004, "\u0120Coch": 33005, "Mexico": 33006, "\u0120Maduro": 33007, "\u0120jars": 33008, "\u0120cane": 33009, "Wik": 33010, "olla": 33011, "ifference": 33012, "\u0120physicist": 33013, "\u0120Maggie": 33014, "\u0120285": 33015, "\u0120depiction": 33016, "\u0120McLaren": 33017, "Ju": 33018, "\u0120slows": 33019, "\u0120commissioners": 33020, "\u0120Willow": 33021, "\u0120Explos": 33022, "hovah": 33023, "\u0120technician": 33024, "\u0120homicides": 33025, "\u0120Flav": 33026, "\u0120Truman": 33027, "\u012010000": 33028, "uctor": 33029, "\u0120shader": 33030, "Newsletter": 33031, "457": 33032, "\u0120rever": 33033, "\u0120hardened": 33034, "\u0120whereabouts": 33035, "\u0120redevelop": 33036, "\u0120carbs": 33037, "\u0120travers": 33038, "\u0120squirrel": 33039, "\u0120follower": 33040, "\u0120sings": 33041, "508": 33042, "\u0120rabbits": 33043, "emonium": 33044, "\u0120documenting": 33045, "\u0120misunderstood": 33046, ")'": 33047, "Rick": 33048, "ggies": 33049, "\u0120premie": 33050, "\u0120skating": 33051, "\u0120passports": 33052, "\u0120fists": 33053, "ageddon": 33054, "Haw": 33055, "ACP": 33056, "080": 33057, "\u0120Thoughts": 33058, "\u0120Carlson": 33059, "\u0120priesthood": 33060, "hua": 33061, "\u0120dungeons": 33062, "\u0120Loans": 33063, "\u0120antis": 33064, "\u0120familiarity": 33065, "\u0120Sabb": 33066, "opal": 33067, "\u0120Ink": 33068, "strike": 33069, "\u0120cram": 33070, "\u0120legalized": 33071, "\u0120cuisine": 33072, "\u0120fibre": 33073, "Travel": 33074, "\u0120Monument": 33075, "ODY": 33076, "ethy": 33077, "\u0120interstate": 33078, "\u0120PUR": 33079, "emporary": 33080, "\u0120Arabian": 33081, "developed": 33082, "\u0120saddle": 33083, "\u0120github": 33084, "\u0120Offer": 33085, "\u0120ISP": 33086, "rolet": 33087, "\u0120SUPER": 33088, "\u0120Denis": 33089, "\u0120multiplier": 33090, "\u0120stirred": 33091, "Interestingly": 33092, "\u0120customary": 33093, "\u0120billed": 33094, "hex": 33095, "\u0120multiplied": 33096, "\u0120flipping": 33097, "\u0120Crosby": 33098, "\u0120fundamentals": 33099, "iae": 33100, "\u0120Played": 33101, "\u0120Atom": 33102, "amazon": 33103, "\u0120Flam": 33104, "eez": 33105, "activated": 33106, "\u0120tablespoon": 33107, "\u0120liberalism": 33108, "\u0120Palin": 33109, "\u0120Patel": 33110, "Num": 33111, "\u0120TAM": 33112, "\u0120surn": 33113, "\u0120Reloaded": 33114, "\u0120coined": 33115, "\"],": 33116, "\u0120Clash": 33117, "\u0120Agu": 33118, "\u0120pragmatic": 33119, "\u0120Activate": 33120, "\u0120802": 33121, "\u0120trailers": 33122, "\u0120silhou": 33123, "\u0120probes": 33124, "\u0120circus": 33125, "\u0120Bain": 33126, "\u0120Lindsay": 33127, "\u0120Abbey": 33128, "Delivery": 33129, "\u0120concession": 33130, "\u0120gastro": 33131, "\u0120Sprite": 33132, "\u00c4\u0141": 33133, "andel": 33134, "\u0120gimm": 33135, "\u0120autobi": 33136, "\u0120Turtle": 33137, "\u0120wonderfully": 33138, "\u0120Haram": 33139, "\u0120Worldwide": 33140, "\u0120Handle": 33141, "\u0120theorists": 33142, "\u0120sleek": 33143, "\u0120Zhu": 33144, "ographically": 33145, "EGA": 33146, "\u0120Owners": 33147, "aths": 33148, "\u0120Antarctic": 33149, "natal": 33150, "=\"\"": 33151, "flags": 33152, "````": 33153, "\u0120sul": 33154, "Kh": 33155, "\u0120potassium": 33156, "\u0120lineman": 33157, "\u0120cereal": 33158, "\u0120Seasons": 33159, "\u01202022": 33160, "\u0120mathematic": 33161, "\u0120astronomers": 33162, "professional": 33163, "\u0120fares": 33164, "cknowled": 33165, "\u0120chi": 33166, "\u0120youngsters": 33167, "\u0120mistakenly": 33168, "\u0120hemisphere": 33169, "\u0120Divinity": 33170, "rone": 33171, "\u0120\",": 33172, "rings": 33173, "\u0120attracts": 33174, "vana": 33175, "\u00e5\u00b9": 33176, "CAP": 33177, "\u0120playlist": 33178, "\u0120porch": 33179, "\u00e3\u0123\u00a3": 33180, "\u0120incorporates": 33181, "\u0120soak": 33182, "\u0120asserting": 33183, "\u0120Terrorism": 33184, "\u0120Pablo": 33185, "Ja": 33186, "cester": 33187, "\u0120fearing": 33188, "\u0120Prayer": 33189, "\u0120escalated": 33190, "GW": 33191, "\u0120robe": 33192, "\u0120Brighton": 33193, "acists": 33194, "\u0120Symphony": 33195, "\u0120Dwarf": 33196, "\u0120Parade": 33197, "\u0120Lego": 33198, "\u0120inexpl": 33199, "\u0120lords": 33200, "leaf": 33201, "RAG": 33202, "liber": 33203, "\u0120cigars": 33204, "\u0120Jehovah": 33205, "606": 33206, "WINDOWS": 33207, "\u0120Liberia": 33208, "ebus": 33209, "Heavy": 33210, "\u0120lubric": 33211, "\u0120RW": 33212, "anguages": 33213, "\u0120narrowed": 33214, "computer": 33215, "\u0120Ember": 33216, "\u0120murdering": 33217, "\u0120downstream": 33218, "\u0120Tuls": 33219, "\u0120Tables": 33220, "Topic": 33221, "\u0120Accuracy": 33222, "=/": 33223, "lost": 33224, "\u0120Rei": 33225, "\u0120progresses": 33226, "bear": 33227, "\u0120establishments": 33228, "Justin": 33229, "\u0120Peach": 33230, "\u0120Gomez": 33231, "\u00e5\u00bf": 33232, "\u0120Triangle": 33233, "Ident": 33234, "\u0120Hive": 33235, "Resources": 33236, "\u0120mixes": 33237, "\u0120Assuming": 33238, "Mu": 33239, "\u0120hypoc": 33240, "\u0120sane": 33241, "\u0120Wan": 33242, "idious": 33243, "Success": 33244, "\u0120io": 33245, "Angel": 33246, "\u0120dangerously": 33247, "\u0120Creature": 33248, "WORK": 33249, ":[": 33250, "\u0120Katrina": 33251, "Listener": 33252, "Miller": 33253, "\u0120Idlib": 33254, "hang": 33255, "\u0120circumvent": 33256, "href": 33257, "\u0120celestial": 33258, "\u0120Weeks": 33259, "\u0120Pug": 33260, "\u0120Dalton": 33261, "\u0120subpoena": 33262, "uku": 33263, "\u0120persisted": 33264, "pei": 33265, "olding": 33266, "\u0120Documents": 33267, "\u0120Hast": 33268, "\u0120CENT": 33269, "\u0120primer": 33270, "\u0120synonymous": 33271, "\u0120nib": 33272, "ombs": 33273, "\u0120notation": 33274, "\u0120Dish": 33275, "\u0120Atmosp": 33276, "\u0120forbid": 33277, "\u0120ANG": 33278, "pattern": 33279, "los": 33280, "\u0120projectiles": 33281, "brown": 33282, ".\",": 33283, "\u0120Venom": 33284, "\u0120fiercely": 33285, "ublished": 33286, "\u0120Uran": 33287, "\u0120Nicarag": 33288, "410": 33289, "\u0120CAL": 33290, "OTOS": 33291, "\u0120Miracle": 33292, "\u0120Enchant": 33293, "\u0120guarding": 33294, "append": 33295, "Attach": 33296, "\u0120leveled": 33297, "\u0120condoms": 33298, "ihilation": 33299, "649": 33300, "\u0120nightmares": 33301, "\u0120THEY": 33302, "\u0120START": 33303, "\u0120Kinn": 33304, "\u0120roommate": 33305, "\u0120hygiene": 33306, "opping": 33307, "Job": 33308, "\u0120lvl": 33309, "\u0120VER": 33310, "\u0120Keeping": 33311, "abetic": 33312, "\u0120formatting": 33313, "erala": 33314, "\u0120revisions": 33315, "\u0120resurg": 33316, "Tel": 33317, "\u0120Goodman": 33318, "353": 33319, "pod": 33320, "\u0120indisp": 33321, "\u0120Translation": 33322, "\u0120gown": 33323, "\u0120Mund": 33324, "\u0120cis": 33325, "\u0120bystand": 33326, "collect": 33327, "\u0120Punjab": 33328, "actively": 33329, "\u0120Gamb": 33330, "tell": 33331, "\u0120importing": 33332, "gencies": 33333, "\u0120locom": 33334, "\u0120Brill": 33335, "Holy": 33336, "\u0120Berger": 33337, "\u0120showdown": 33338, "\u0120responders": 33339, "ILY": 33340, "\u0120takedown": 33341, "leted": 33342, "\u0120mattered": 33343, "\u0120predictive": 33344, "\u0120overlay": 33345, "GPU": 33346, "\u0120Vick": 33347, "\u0120conveyed": 33348, "Tab": 33349, "peer": 33350, "Scan": 33351, "\u0120defensively": 33352, "vae": 33353, "\u0120approving": 33354, "\u0120tiers": 33355, "\u0120Via": 33356, "querade": 33357, "\u0120Saudis": 33358, "\u0120demolished": 33359, "\u0120Prophe": 33360, "\u0120mono": 33361, "\u0120hospitality": 33362, "HAM": 33363, "\u0120Ariel": 33364, "MOD": 33365, "\u0120Torah": 33366, "\u0120blah": 33367, "\u0120Belarus": 33368, "erential": 33369, "\u0120Tuc": 33370, "\u0120banker": 33371, "397": 33372, "\u0120mosquit": 33373, "\u0120Scientist": 33374, "\u0120Musical": 33375, "\u0120hust": 33376, "Shift": 33377, "\u0120torment": 33378, "\u0120standoff": 33379, "Educ": 33380, "\u0120Fog": 33381, "\u0120amplifier": 33382, "Shape": 33383, "Instance": 33384, "\u0120Critics": 33385, "\u0120daemon": 33386, "Houston": 33387, "\u0120mattress": 33388, "\u0120IDF": 33389, "\u0120obscene": 33390, "\u0120Amer": 33391, "hetti": 33392, "\u0120compiling": 33393, "352": 33394, "verett": 33395, "\u0120Reduction": 33396, "istration": 33397, "\u0120Blessed": 33398, "\u0120Bachelor": 33399, "316": 33400, "\u0120prank": 33401, "\u0120Vulcan": 33402, "dding": 33403, "\u0120mourning": 33404, "\u0120Quint": 33405, "\u0120Blaster": 33406, "testing": 33407, "\u0120sediment": 33408, ">>>": 33409, "\u0120Eternity": 33410, "\u0120WHERE": 33411, "\u0120Maze": 33412, "\u0120reacting": 33413, "\u0120Alv": 33414, "omsday": 33415, "\u0120CRA": 33416, "\u0120translator": 33417, "\u0120bogus": 33418, "atu": 33419, "Website": 33420, "olls": 33421, "\u0120baptism": 33422, "\u0120sibling": 33423, "\u0120Autumn": 33424, "vez": 33425, "\u00e3\u0123\u00ae\u00e9": 33426, "guards": 33427, "Georg": 33428, "assadors": 33429, "\u0120Freud": 33430, "\u0120continents": 33431, "\u0120Registry": 33432, "Bernie": 33433, "\u0138\u013c\u00e5\u00a3\u00ab": 33434, "\u0120tolerant": 33435, "\u0120UW": 33436, "\u0120horribly": 33437, "995": 33438, "\u0120MIDI": 33439, "\u0120impatient": 33440, "ocado": 33441, "eri": 33442, "\u0120Worst": 33443, "\u0120Norris": 33444, "\u0120Talking": 33445, "\u0120defends": 33446, "ensable": 33447, "\u01202021": 33448, "\u0120anatomy": 33449, "Lew": 33450, "\u0120drawer": 33451, "\u0120Canberra": 33452, "\u0120patriotic": 33453, "\u00e9\u00be\u012f\u00e5\u0138\u013c\u00e5\u00a3\u00ab": 33454, "\u0120Avg": 33455, "ARM": 33456, "\u0120undisclosed": 33457, "\u0120farewell": 33458, "459": 33459, "bable": 33460, "\u0120Allison": 33461, "OLOG": 33462, "\u0120conco": 33463, "tight": 33464, "\u0120ACPI": 33465, "\u0120Mines": 33466, "lich": 33467, "\u0120\u00e2\u0136\u013e": 33468, "represented": 33469, "200000": 33470, "\u0120enthusiast": 33471, "OTS": 33472, "bil": 33473, "\u0120Ingredients": 33474, "\u0120inventor": 33475, "\u0120MySQL": 33476, "\u00c2\u0142\u00c2\u0142\u00c2\u0142": 33477, "\u0120ABOUT": 33478, "within": 33479, "\u0120mk": 33480, "Bul": 33481, "\u0120Fake": 33482, "\u0120draconian": 33483, "Wa": 33484, "helm": 33485, "\u0120Terran": 33486, "erville": 33487, "\u0120commonplace": 33488, "SIZE": 33489, "\u0120\"<": 33490, "replace": 33491, "ographs": 33492, "\u0120SELECT": 33493, "incible": 33494, "\u0120Mostly": 33495, "\u0120Sheffield": 33496, "\u0120IDE": 33497, "uggle": 33498, "\u0120citations": 33499, "hurst": 33500, "\u0120Unix": 33501, "\u0120unleash": 33502, "\u0120Piper": 33503, "\u0120Nano": 33504, "\u0120succumb": 33505, "\u0120reluctance": 33506, "\u01202500": 33507, "\u0120Merchant": 33508, "\u0120wiret": 33509, "\u0120combos": 33510, "\u0120Birthday": 33511, "\u0120charcoal": 33512, "\u0120UPS": 33513, "\u0120Fairfax": 33514, "\u0120driveway": 33515, "\u0120Tek": 33516, "\u0120Pitch": 33517, "overe": 33518, "\u0120technicians": 33519, "\u0120Actual": 33520, "flation": 33521, "\u0120Fiscal": 33522, "\u0120Empty": 33523, "anamo": 33524, "\u0120magnesium": 33525, "\u0120slut": 33526, "\u0120growers": 33527, "Investigators": 33528, "():": 33529, "\u0120Satellite": 33530, "\u0120Keynes": 33531, "missive": 33532, "lane": 33533, "\u0120borough": 33534, "344": 33535, "\u0120TEAM": 33536, "\u0120Bethesda": 33537, "CV": 33538, "hower": 33539, "\u0120RAD": 33540, "\u0120chant": 33541, "\u0120Riy": 33542, "\u0120compositions": 33543, "\u0120mildly": 33544, "\u0120meddling": 33545, "\u0120agility": 33546, "aneers": 33547, "501": 33548, "\u0120synth": 33549, "linger": 33550, "291": 33551, "\u0120exclaimed": 33552, "Party": 33553, "\u0120contamin": 33554, "\u0120Manor": 33555, "\u0120Respond": 33556, "\u0120praising": 33557, "\u0120manners": 33558, "fleet": 33559, "Summer": 33560, "\u0120Lynd": 33561, "\u0120Definitely": 33562, "grim": 33563, "\u0120bowling": 33564, "stri": 33565, "\u00e7\u013d": 33566, "ynt": 33567, "\u0120mandates": 33568, "DIV": 33569, "\u0120reconcile": 33570, "views": 33571, "\u0120Damon": 33572, "vette": 33573, "Flo": 33574, "\u0120Greatest": 33575, "ilon": 33576, "icia": 33577, "\u0120portrayal": 33578, "\u0120cushion": 33579, "504": 33580, "1979": 33581, "ossal": 33582, "Applic": 33583, "scription": 33584, "\u0120mitigation": 33585, "ATS": 33586, "pac": 33587, "\u0120erased": 33588, "\u0120deficiencies": 33589, "\u0120Hollande": 33590, "\u0120Xu": 33591, "\u0120bred": 33592, "\u0120pregnancies": 33593, "femin": 33594, "\u0120emph": 33595, "\u0120planners": 33596, "\u0120outper": 33597, "uttering": 33598, "\u0120perpetrator": 33599, "\u0120motto": 33600, "\u0120Ellison": 33601, "\u0120NEVER": 33602, "\u0120admittedly": 33603, "ARI": 33604, "\u0120Azerbaijan": 33605, "\u0120millisec": 33606, "\u0120combustion": 33607, "\u0120Bottle": 33608, "\u0120Lund": 33609, "\u0120Ps": 33610, "\u0120Dress": 33611, "\u0120fabricated": 33612, "\u0120battered": 33613, "\u0120sidel": 33614, "\u0120Notting": 33615, "Foreign": 33616, "\u0120Jerome": 33617, "020": 33618, "\u0120Arbit": 33619, "\u0120knots": 33620, "\u0120RIGHT": 33621, "Moving": 33622, "\u00e3\u0123\u013b": 33623, "\u0120surgeries": 33624, "\u0120courthouse": 33625, "\u0120mastered": 33626, "\u0120hovering": 33627, "\u0120Bran": 33628, "\u0120Alison": 33629, "\u0120safest": 33630, "military": 33631, "\u0120bullied": 33632, "\u0120barrage": 33633, "Reader": 33634, "ESE": 33635, "\u0120Geographic": 33636, "Tools": 33637, "314": 33638, "\u0120Geek": 33639, "roth": 33640, "glers": 33641, "\u0120FIN": 33642, "\u00cf\u0123": 33643, "\u0120Aston": 33644, "altern": 33645, "488": 33646, "\u0120veterin": 33647, "Gamer": 33648, "\u0120intel": 33649, "renches": 33650, "Shield": 33651, "\u0120amnesty": 33652, "\u0120Bhar": 33653, "\u0120piled": 33654, "\u0120honorable": 33655, "\u0120Institutes": 33656, "\u0120soaked": 33657, "\u0120coma": 33658, "\u0120EFF": 33659, "341": 33660, "bytes": 33661, "\u0120Gmail": 33662, "lein": 33663, "\u0120Canadiens": 33664, "material": 33665, "Il": 33666, "\u0120instructors": 33667, "\u0120KY": 33668, "\u0120conceive": 33669, "ubb": 33670, "\u0120Possible": 33671, "\u0120easing": 33672, "\u0120Christina": 33673, "\u0120caric": 33674, "\u0120HDR": 33675, "ROM": 33676, "\u0120shovel": 33677, "delete": 33678, "\u0120puff": 33679, "\u0120Changing": 33680, "\u0120seamlessly": 33681, "Attribute": 33682, "\u0120acquisitions": 33683, "akery": 33684, "\u0120EF": 33685, "\u0120autistic": 33686, "\u0120Takes": 33687, "\u0120Powder": 33688, "\u0120Stir": 33689, "510": 33690, "\u0120Bubble": 33691, "settings": 33692, "\u0120Fowler": 33693, "\u0120mustard": 33694, "\u0120moreover": 33695, "\u0120copyrighted": 33696, "\u0120LEDs": 33697, "1500": 33698, "\u00e6\u012b": 33699, "\u0120HIS": 33700, "enf": 33701, "\u0120custod": 33702, "\u0120Huck": 33703, "Gi": 33704, "\u0120img": 33705, "Answer": 33706, "Ct": 33707, "jay": 33708, "\u0120Infrastructure": 33709, "\u0120federally": 33710, "Loc": 33711, "\u0120microbes": 33712, "\u0120overrun": 33713, "dds": 33714, "otent": 33715, "adiator": 33716, ">>>>>>>>": 33717, "\u0120tornado": 33718, "\u0120adjud": 33719, "\u0120intrigued": 33720, "\u0120si": 33721, "\u0120Revelation": 33722, "progress": 33723, "\u0120burglary": 33724, "\u0120Saiyan": 33725, "\u0120Kathy": 33726, "\u0120serpent": 33727, "\u0120Andreas": 33728, "\u0120compel": 33729, "essler": 33730, "\u0120Plastic": 33731, "\u0120Advent": 33732, "\u0120Positive": 33733, "\u0120Qt": 33734, "\u0120Hindus": 33735, "registered": 33736, "ularity": 33737, "\u0120righteousness": 33738, "\u0120demonic": 33739, "uitive": 33740, "\u0120BDS": 33741, "\u0120Gregg": 33742, "cia": 33743, "\u0120Crusade": 33744, "\u0120Sinai": 33745, "WARE": 33746, "+(": 33747, "\u0120mell": 33748, "\u0120derail": 33749, "yards": 33750, "Ast": 33751, "\u0120noticeably": 33752, "\u0120Ober": 33753, "Ram": 33754, "\u0120unnoticed": 33755, "\u0120seq": 33756, "avage": 33757, "Ts": 33758, "\u0120640": 33759, "\u0120concede": 33760, "\u0120])": 33761, "Fill": 33762, "\u0120captivity": 33763, "\u0120Improvement": 33764, "\u0120Crusader": 33765, "araoh": 33766, "MAP": 33767, "\u00e6\u0139": 33768, "\u0120stride": 33769, "always": 33770, "Fly": 33771, "Nit": 33772, "\u0120algae": 33773, "\u0120Cooking": 33774, "\u0120Doors": 33775, "Malley": 33776, "\u0120policemen": 33777, "\u00e3\u0123\u012f": 33778, "\u0120astronaut": 33779, "accessible": 33780, "495": 33781, "\u0120RAW": 33782, "cliffe": 33783, "udicrous": 33784, "\u0120depended": 33785, "alach": 33786, "\u0120ventures": 33787, "rake": 33788, "\u0120tits": 33789, "\u0120Hou": 33790, "\u0120condom": 33791, "ormonal": 33792, "\u0120indent": 33793, "\u0120uploading": 33794, "Footnote": 33795, "Important": 33796, "\u0120271": 33797, "\u0120mindful": 33798, "\u0120contends": 33799, "Cra": 33800, "\u0120calibr": 33801, "\u0120OECD": 33802, "plugin": 33803, "Fat": 33804, "\u0120ISS": 33805, "\u0120Dynamics": 33806, "ansen": 33807, "686": 33808, "'),": 33809, "\u0120sprite": 33810, "\u0120handheld": 33811, "\u0120Hipp": 33812, "=~=~": 33813, "Trust": 33814, "\u0120semantics": 33815, "\u0120Bundes": 33816, "\u0120Reno": 33817, "\u0120Literature": 33818, "sense": 33819, "Gary": 33820, "\u0120Aeg": 33821, "\u0120Trin": 33822, "EEK": 33823, "\u0120cleric": 33824, "\u0120SSH": 33825, "\u0120christ": 33826, "\u0120invading": 33827, "ibu": 33828, "\u0120enum": 33829, "aura": 33830, "\u0120allege": 33831, "\u0120Incredible": 33832, "BBC": 33833, "\u0120thru": 33834, "\u0120sailed": 33835, "\u0120emulate": 33836, "\u0120insecurity": 33837, "\u0120crou": 33838, "\u0120accommodations": 33839, "\u0120incompetent": 33840, "\u0120slips": 33841, "\u0120Earthqu": 33842, "sama": 33843, "ILLE": 33844, "\u0120iPhones": 33845, "asaki": 33846, "\u0120bye": 33847, "\u0120ard": 33848, "\u0120extras": 33849, "\u0120slaughtered": 33850, "\u0120crowdfunding": 33851, "resso": 33852, "\u0120filib": 33853, "\u0120ERROR": 33854, "\u0120TLS": 33855, "egg": 33856, "\u0120Ital": 33857, "\u0120enlist": 33858, "\u0120Catalonia": 33859, "\u0120Scots": 33860, "\u0120sergeant": 33861, "\u0120dissolve": 33862, "NH": 33863, "\u0120standings": 33864, "rique": 33865, "IQ": 33866, "\u0120beneficiary": 33867, "\u0120aquarium": 33868, "YouTube": 33869, "\u0120PowerShell": 33870, "\u0120brightest": 33871, "\u0120Warrant": 33872, "Sold": 33873, "Writing": 33874, "\u0120beginnings": 33875, "\u0120Reserved": 33876, "\u0120Latinos": 33877, "heading": 33878, "\u0120440": 33879, "\u0120rooftop": 33880, "ATING": 33881, "\u0120390": 33882, "VPN": 33883, "Gs": 33884, "kernel": 33885, "turned": 33886, "\u0120preferable": 33887, "\u0120turnovers": 33888, "\u0120Hels": 33889, "Sa": 33890, "\u0120Shinji": 33891, "veh": 33892, "\u0120MODULE": 33893, "Viol": 33894, "\u0120exiting": 33895, "\u0120jab": 33896, "\u0120Vanilla": 33897, "\u0120acron": 33898, "\u0120Gap": 33899, "bern": 33900, "Ak": 33901, "\u0120McGu": 33902, "\u0120endlessly": 33903, "\u0120Farage": 33904, "\u0120Noel": 33905, "Va": 33906, "MK": 33907, "\u0120brute": 33908, "\u0120Kru": 33909, "\u0120ESV": 33910, "\u0120Olivia": 33911, "\u00e2\u0122\u0142": 33912, "\u0120Kaf": 33913, "\u0120trusting": 33914, "\u0120hots": 33915, "324": 33916, "\u0120malaria": 33917, "\u0120json": 33918, "\u0120pounding": 33919, "ortment": 33920, "Country": 33921, "\u0120postponed": 33922, "\u0120unequiv": 33923, "?),": 33924, "\u0120Rooney": 33925, "udding": 33926, "\u0120Leap": 33927, "urrence": 33928, "shapeshifter": 33929, "\u0120HAS": 33930, "osate": 33931, "\u0120cavern": 33932, "\u0120conservatism": 33933, "\u0120BAD": 33934, "\u0120mileage": 33935, "\u0120arresting": 33936, "Vaults": 33937, "\u0120mixer": 33938, "Democratic": 33939, "\u0120Benson": 33940, "\u0120authored": 33941, "8000": 33942, "\u0120proactive": 33943, "\u0120Spiritual": 33944, "tre": 33945, "\u0120incarcerated": 33946, "\u0120Sort": 33947, "\u0120peaked": 33948, "\u0120wielding": 33949, "reciation": 33950, "\u00d7\u013b\u00d7": 33951, "Patch": 33952, "\u0120Emmy": 33953, "\u0120exqu": 33954, "tto": 33955, "\u0120Ratio": 33956, "\u0120Picks": 33957, "\u0120Gry": 33958, "phant": 33959, "\u0120fret": 33960, "\u0120ethn": 33961, "\u0120archived": 33962, "%-": 33963, "cases": 33964, "\u0120Blaze": 33965, "\u0120imb": 33966, "cv": 33967, "yss": 33968, "imony": 33969, "\u0120countdown": 33970, "\u0120awakening": 33971, "\u0120Tunisia": 33972, "\u0120Refer": 33973, "\u0120MJ": 33974, "\u0120unnatural": 33975, "\u0120Carnegie": 33976, "izen": 33977, "\u0120Nuggets": 33978, "hess": 33979, "\u0120evils": 33980, "647": 33981, "\u0120introductory": 33982, "loving": 33983, "\u0120McMahon": 33984, "\u0120ambiguity": 33985, "Label": 33986, "\u0120Almighty": 33987, "\u0120coloring": 33988, "\u0120Claus": 33989, "setting": 33990, "NULL": 33991, "\u0120Favorite": 33992, "\u0120SIG": 33993, ">(": 33994, "\u0120Shiva": 33995, "\u0120Mayer": 33996, "\u0120stormed": 33997, "\u0120Coverage": 33998, "weapons": 33999, "igham": 34000, "\u0120unanswered": 34001, "\u0120leve": 34002, "\u0120coy": 34003, "cas": 34004, "bags": 34005, "asured": 34006, "Seattle": 34007, "\u0120Santorum": 34008, "serious": 34009, "\u0120courageous": 34010, "\u0120Soup": 34011, "\u0120confiscated": 34012, "\u0120///": 34013, "\u0120unconventional": 34014, "\u0120moms": 34015, "\u0120Rohingya": 34016, "\u0120Orchestra": 34017, "\u0120Potion": 34018, "\u0120discredit": 34019, "\u0120FIL": 34020, "fixed": 34021, "\u0120Deer": 34022, "doi": 34023, "\u0120Dimension": 34024, "\u0120bureaucrats": 34025, "eteen": 34026, "\u0120actionGroup": 34027, "ohm": 34028, "\u0120bumps": 34029, "\u0120Utility": 34030, "\u0120submarines": 34031, "renheit": 34032, "research": 34033, "\u0120Shapiro": 34034, "\u0120sketches": 34035, "\u0120deceptive": 34036, "\u0120Vil": 34037, "esame": 34038, "\u0120Essentially": 34039, "\u0120rampage": 34040, "isky": 34041, "\u0120muttered": 34042, "thritis": 34043, "\u0120236": 34044, "fet": 34045, "bars": 34046, "\u0120pupil": 34047, "\u0120Thou": 34048, "oS": 34049, "song": 34050, "\u0120fractured": 34051, "\u0120revert": 34052, "picture": 34053, "\u0120criterion": 34054, "usher": 34055, "\u0120repercussions": 34056, "\u0120Vintage": 34057, "\u0120Superintendent": 34058, "Officers": 34059, "\u0120flagged": 34060, "\u0120blames": 34061, "\u0120inverse": 34062, "ographers": 34063, "\u0120makeshift": 34064, "\u0120devoid": 34065, "\u0120fossils": 34066, "\u0120Aristotle": 34067, "\u0120Funds": 34068, "\u0120depleted": 34069, "\u0120Flu": 34070, "\u0120Yuan": 34071, "\u0120woes": 34072, "\u0120lipid": 34073, "\u0120situ": 34074, "requisites": 34075, "\u0120furnish": 34076, "\u0120Samar": 34077, "\u0120shameful": 34078, "\u0120adversely": 34079, "\u0120adept": 34080, "\u0120remorse": 34081, "\u0120murderous": 34082, "uckles": 34083, "\u0120ESL": 34084, "\u0120314": 34085, "sent": 34086, "\u0120redef": 34087, "\u0120Cache": 34088, "\u0120Purs": 34089, "igans": 34090, "\u0120460": 34091, "\u0120prescriptions": 34092, "\u0120fres": 34093, "Fuck": 34094, "ocrates": 34095, "Twenty": 34096, "\u0120Weird": 34097, "\u0120Toggle": 34098, "\u0120Called": 34099, "itizens": 34100, "\u0120poultry": 34101, "\u0120harvesting": 34102, "\u00e3\u0124\u00a6\u00e3\u0124\u00b9": 34103, "Bottom": 34104, "\u0120cautioned": 34105, "tn": 34106, "396": 34107, "\u0120Nikki": 34108, "\u0120evaluations": 34109, "\u0120harassing": 34110, "\u0120bindings": 34111, "\u0120Monetary": 34112, "\u0120hitters": 34113, "\u0120adversary": 34114, "unts": 34115, "\u0120setback": 34116, "\u0120encrypt": 34117, "\u0120Cait": 34118, "\u0120lows": 34119, "enges": 34120, "\u0120Norn": 34121, "\u0120bulbs": 34122, "\u0120bottled": 34123, "\u0120Voyager": 34124, "317": 34125, "\u0120spheres": 34126, "politics": 34127, "\u0120subtract": 34128, "\u0120sensations": 34129, "\u0120appalling": 34130, "\u0120316": 34131, "\u0120environmentally": 34132, "\u0120STEM": 34133, "\u0120publishes": 34134, "560": 34135, "\u0120diligence": 34136, "484": 34137, "\u0120advises": 34138, "\u0120petrol": 34139, "\u0120imagining": 34140, "\u0120patrols": 34141, "\u0120Integer": 34142, "\u0120Ashes": 34143, "actus": 34144, "\u0120Radiant": 34145, "\u0120LT": 34146, "itability": 34147, "htaking": 34148, "Setting": 34149, "\u0120nuanced": 34150, "\u0120Reef": 34151, "\u0120Developers": 34152, "Ni": 34153, "pieces": 34154, "990": 34155, "License": 34156, "\u0120lowers": 34157, "\u0120Ottoman": 34158, "327": 34159, "ooo": 34160, "\u0120quitting": 34161, "markets": 34162, "Behind": 34163, "\u0120basin": 34164, "\u0120docs": 34165, "anie": 34166, "flash": 34167, "ctl": 34168, "\u0120civilized": 34169, "\u0120Fukushima": 34170, "\"],\"": 34171, "\u0120KS": 34172, "\u0120Honestly": 34173, "arat": 34174, "\u0120constructs": 34175, "\u0120Lans": 34176, "\u0120Dire": 34177, "\u0120LIKE": 34178, "\u0120Trouble": 34179, "\u0120withholding": 34180, "\u0120Oblivion": 34181, "\u0120sanity": 34182, "anya": 34183, "Const": 34184, "\u0120grocer": 34185, "\u0120Celsius": 34186, "\u0120recounted": 34187, "\u0120Wife": 34188, "Border": 34189, "atered": 34190, "happy": 34191, "\u0120spoiler": 34192, "\u0120logically": 34193, "Hall": 34194, "\u0120succeeding": 34195, "\u0120polymorph": 34196, "\u0120axes": 34197, "\u0120Shotgun": 34198, "\u0120Slim": 34199, "\u0120Principles": 34200, "\u0120Leth": 34201, "arta": 34202, "\u0120scor": 34203, "Screenshot": 34204, "\u0120relaxation": 34205, "#$#$": 34206, "\u0120deterrent": 34207, "iddy": 34208, "\u0120powerless": 34209, "\u0120lesbians": 34210, "\u0120chords": 34211, "\u0120Edited": 34212, "selected": 34213, "\u0120separatists": 34214, "0002": 34215, "\u0120airspace": 34216, "\u0120turnaround": 34217, "\u0120cunning": 34218, "PATH": 34219, "Poly": 34220, "\u0120bombed": 34221, "\u0120tion": 34222, "xs": 34223, "\u0120withhold": 34224, "\u0120waged": 34225, "\u0120Liberties": 34226, "Flag": 34227, "\u0120comforting": 34228, "454": 34229, "\u0120Iris": 34230, "arers": 34231, "\u0120rag": 34232, "\u0120relocated": 34233, "\u0120Guarant": 34234, "\u0120strategically": 34235, "\u0120gamma": 34236, "uberty": 34237, "\u0120Lockheed": 34238, "gres": 34239, "\u0120grilled": 34240, "\u0120Lowe": 34241, "stats": 34242, "\u0120Rocks": 34243, "\u0120sensing": 34244, "\u0120renting": 34245, "\u0120Geological": 34246, "\u00d8\u00a7\u00d8": 34247, "otrop": 34248, "\u0120sew": 34249, "\u0120improperly": 34250, "486": 34251, "\u0120\u00e2\u0138\u0142": 34252, "\u0120starving": 34253, "\u0120Bj": 34254, "Discussion": 34255, "328": 34256, "\u0120Combo": 34257, "\u0120Fixes": 34258, "NAT": 34259, "\u0120striving": 34260, "thora": 34261, "\u0120harvested": 34262, "\u0120Ping": 34263, "\u0120playful": 34264, "\u0120avenues": 34265, "\u0120occupational": 34266, "\u0120wakes": 34267, "\u0120Courier": 34268, "\u0120drummer": 34269, "\u0120Browser": 34270, "\u0120Houth": 34271, "itu": 34272, "\u0120apparel": 34273, "paste": 34274, "\u0120hunted": 34275, "\u0120Secondly": 34276, "lain": 34277, "XY": 34278, "\u0120PIN": 34279, "icons": 34280, "\u0120cocktails": 34281, "\u0120sizable": 34282, "\u0120hurdles": 34283, "estinal": 34284, "\u0120Recreation": 34285, "\u0120eco": 34286, "648": 34287, "\u0120Died": 34288, "mint": 34289, "\u0120fingerprints": 34290, "\u0120dispose": 34291, "\u0120Bosnia": 34292, "tsy": 34293, "2200": 34294, "\u0120inspected": 34295, "\u0120Fou": 34296, "\u0120fuss": 34297, "\u0120ambush": 34298, "\u0120Rak": 34299, "\u0120manifested": 34300, "Prosecut": 34301, "\u0120suffice": 34302, "rences": 34303, "\u0120compensated": 34304, "\u0120Cyrus": 34305, "\u0120genus": 34306, "\u0120Wolverine": 34307, "\u0120Trends": 34308, "\u0120hikes": 34309, "\u0120Seen": 34310, "\u0120enrol": 34311, "Cold": 34312, "\u0120politely": 34313, "\u0120Slav": 34314, "\u0120Rupert": 34315, "\u0120eyewitness": 34316, "\u0120Alto": 34317, "\u0120uncomp": 34318, "\u0120posterior": 34319, "Must": 34320, "\u0120Herz": 34321, "\u0120progressively": 34322, "\u0120234": 34323, "\u0120indifference": 34324, "\u0120Cunningham": 34325, "\u0120academia": 34326, "\u0120sewer": 34327, "\u0120astounding": 34328, "\u0120AES": 34329, "rather": 34330, "\u0120eldest": 34331, "\u0120climbs": 34332, "\u0120Adds": 34333, "\u0120outcry": 34334, "\u0120contag": 34335, "\u0120Houses": 34336, "\u0120pept": 34337, "\u0120Melania": 34338, "interested": 34339, "\u0120UCH": 34340, "\u0120Roots": 34341, "\u0120Hubbard": 34342, "\u0120TBD": 34343, "\u0120Romanian": 34344, "filename": 34345, "Stone": 34346, "\u0120Impl": 34347, "\u0120chromosome": 34348, "Cle": 34349, "dx": 34350, "\u0120scrambled": 34351, "\u0120Pt": 34352, "\u0120242": 34353, "OPLE": 34354, "\u0120tremendously": 34355, "Street": 34356, "\u0120craving": 34357, "\u0120bundled": 34358, "\u0120RG": 34359, "pipe": 34360, "\u0120injuring": 34361, "\u0120arcane": 34362, "Particip": 34363, "\u0120Heroic": 34364, "sty": 34365, "\u0120topping": 34366, "\u0120Tempest": 34367, "rentices": 34368, "bh": 34369, "\u0120paranoia": 34370, "\u0120Unicode": 34371, "\u0120egregious": 34372, "\u0120\\'": 34373, "\u0120Oswald": 34374, "\u0120gravel": 34375, "\u0120Simpsons": 34376, "\u0120bland": 34377, "\u0120Guantanamo": 34378, "Writer": 34379, "liners": 34380, "\u0120Dice": 34381, "JC": 34382, "\u0120parity": 34383, "\u0120sided": 34384, "\u0120237": 34385, "\u0120Pyrrha": 34386, "atters": 34387, "dk": 34388, "Fine": 34389, "compan": 34390, "\u0120formulated": 34391, "\u0120Idol": 34392, "ilers": 34393, "hemoth": 34394, "\u0120Fav": 34395, "\u0120intrusion": 34396, "\u0120carrots": 34397, "\u0120Layer": 34398, "\u0120Hacker": 34399, "\u0120----------------": 34400, "\u0120moderation": 34401, "\u00e9\u0123": 34402, "ococ": 34403, "\u0120characterize": 34404, "\u0120Teresa": 34405, "\u0120socioeconomic": 34406, "\u0120perk": 34407, "\u0120Participation": 34408, "training": 34409, "\u0120Paulo": 34410, "phys": 34411, "\u0120trustworthy": 34412, "\u0120embodied": 34413, "\u0120Merch": 34414, "currency": 34415, "\u0120Priority": 34416, "\u0120teasing": 34417, "\u0120absorbing": 34418, "\u0120unfinished": 34419, "\u0120Comparison": 34420, "\u0120disple": 34421, "writers": 34422, "\u0120professions": 34423, "\u0120Penguin": 34424, "\u0120angrily": 34425, "\u0120LINK": 34426, "688": 34427, "\u0120Correspond": 34428, "\u0120prevailed": 34429, "\u0120cartel": 34430, "lp": 34431, "asms": 34432, "\u0120Redemption": 34433, "\u0120Islamists": 34434, "effects": 34435, "dose": 34436, "\u0120Latter": 34437, "\u0120Halifax": 34438, "\u0120vas": 34439, "\u0120Topics": 34440, "\u0120Named": 34441, "advertising": 34442, "zza": 34443, "ICES": 34444, "\u0120retarded": 34445, "achable": 34446, "\u0120Puppet": 34447, "\u0120ItemLevel": 34448, "\u0120retract": 34449, "\u0120identifiable": 34450, "Aaron": 34451, "\u0120Buster": 34452, "sol": 34453, "helle": 34454, "assemb": 34455, "Hope": 34456, "ranged": 34457, "Ba": 34458, "\u0120Purch": 34459, "\u00e9\u0122": 34460, "\u0120Siri": 34461, "\u0120arrivals": 34462, "\u01201912": 34463, "\u0120shortened": 34464, "\u0120312": 34465, "\u0120discrepancy": 34466, "\u0120Temperature": 34467, "\u0120Walton": 34468, "\u0120kinderg": 34469, "polit": 34470, "\u0120remix": 34471, "\u0120connectors": 34472, "\u00e3\u0125\u013a\u00e3\u0125\u00a9": 34473, "\u0120Kazakhstan": 34474, "dominated": 34475, "\u0120sugars": 34476, "imble": 34477, "\u0120Panic": 34478, "\u0120Demand": 34479, "\u0120Colony": 34480, "onen": 34481, "\u0120MER": 34482, "775": 34483, "uria": 34484, "azaar": 34485, "\u0120Degree": 34486, "Pri": 34487, "\u0120sunshine": 34488, "\u0120251": 34489, "\u0120psychedelic": 34490, "\u0120digitally": 34491, "\u0120Braun": 34492, "\u0120shimmer": 34493, "\u0120shave": 34494, "\u0120Telesc": 34495, "\u0120Astral": 34496, "\u0120Venezuelan": 34497, "\u0120OG": 34498, "\u0120crawling": 34499, "Integ": 34500, "\u0120Feather": 34501, "\u0120unfolding": 34502, "\u0120appropriation": 34503, "\u0120\u00e8\u00a3\u0131\u00e8": 34504, "\u0120Mobility": 34505, "\u0120Ney": 34506, "-.": 34507, "bilt": 34508, "LIN": 34509, "\u0120Tube": 34510, "\u0120Conversely": 34511, "\u0120keyboards": 34512, "\u0120Cao": 34513, "\u0120overth": 34514, "\u0120laure": 34515, ">>\\": 34516, "\u0120Viper": 34517, "acha": 34518, "Offset": 34519, "\u0120Raleigh": 34520, "\u0120Jae": 34521, "Jordan": 34522, "jp": 34523, "\u0120totalitarian": 34524, "Connector": 34525, "\u0120observes": 34526, "\u0120Spartan": 34527, "\u0120Immediately": 34528, "\u0120Scal": 34529, "Cool": 34530, "\u0120taps": 34531, "\u0120roar": 34532, "Past": 34533, "\u0120chars": 34534, "\u0120Bender": 34535, "\u0120Sheldon": 34536, "\u0120painter": 34537, "\u0120beacon": 34538, "\u0120Creatures": 34539, "\u0120downturn": 34540, "\u0120hinder": 34541, "\u0120Andromeda": 34542, "\u00c3\u013d": 34543, "ccoli": 34544, "\u0120Fitness": 34545, "etrical": 34546, "\u0120utilizes": 34547, "\u0120senate": 34548, "\u0120ensemble": 34549, "\u0120cheers": 34550, "TW": 34551, "\u0120affluent": 34552, "kil": 34553, "rylic": 34554, "ordering": 34555, "Computer": 34556, "\u0120gruesome": 34557, "ostics": 34558, "\u0120Ubisoft": 34559, "\u0120Kelley": 34560, "\u0120wrench": 34561, "\u0120bourgeoisie": 34562, "IBLE": 34563, "\u0120Preston": 34564, "worn": 34565, "arist": 34566, "reating": 34567, "\u0120stained": 34568, "arine": 34569, "\u0120slime": 34570, "ENN": 34571, "\u0120chests": 34572, "\u0120groundwater": 34573, "annot": 34574, "\u0120Tray": 34575, "\u0120Locke": 34576, "\u0120CTR": 34577, "\u0120dudes": 34578, "\u0120External": 34579, "\u0120Decoder": 34580, "\u0120paramed": 34581, "\u0120Medline": 34582, "809": 34583, "\u0120Dinner": 34584, "rupal": 34585, "gz": 34586, "\u0120Gum": 34587, "\u0120Demo": 34588, "jee": 34589, "\u0120dh": 34590, "berman": 34591, "archs": 34592, "\u0120enqu": 34593, "\u0120Epstein": 34594, "\u0120devastation": 34595, "\u0120friendships": 34596, "\u0120Ard": 34597, "\u0120231": 34598, "\u0120Rubin": 34599, "\u0120Distance": 34600, "\u0120spurred": 34601, "\u0120dossier": 34602, "\u0120overlooking": 34603, "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\": 34604, "Forest": 34605, "\u0120Comes": 34606, "\\\",": 34607, "\u0120Iranians": 34608, "\u0120fixtures": 34609, "Laughs": 34610, "\u0120curry": 34611, "\u0120Kingston": 34612, "\u0120squash": 34613, "\u0120catalogue": 34614, "\u0120abnormalities": 34615, "\u0120digestive": 34616, ".........": 34617, "\u0120subordinate": 34618, "ogly": 34619, "\u0120249": 34620, "Middle": 34621, "\u0120massac": 34622, "\u0120burgers": 34623, "\u0120downstairs": 34624, "\u01201931": 34625, "394": 34626, "\u0120VG": 34627, "\u0120lasers": 34628, "\u0120Sikh": 34629, "\u0120Alexa": 34630, "derived": 34631, "\u0120cyclist": 34632, "\u00e3\u0123\u00ae\u00e9\u0143\u0136": 34633, "oneliness": 34634, "!!!!!!!!": 34635, "\u0120buffs": 34636, "legate": 34637, "\u0120raping": 34638, "\u0120recommending": 34639, "rored": 34640, "\u0120multicultural": 34641, "unique": 34642, "\u0120businessmen": 34643, "\u0120uneasy": 34644, "\u0120MAP": 34645, "\u0120dispersed": 34646, "cipline": 34647, "Jess": 34648, "\u0120Kerala": 34649, "\u00e5\u00a7": 34650, "\u0120abstraction": 34651, "Surv": 34652, "Uh": 34653, "\u0120printers": 34654, "ija": 34655, "owder": 34656, "\u0120analogous": 34657, "\u0120ASP": 34658, "afer": 34659, "\u0120unfolded": 34660, "\u0120leveling": 34661, "\u0120breached": 34662, "\u0120Hearing": 34663, "\u0120nat": 34664, "\u0120translating": 34665, "critical": 34666, "\u0120antagonist": 34667, "\u0120Yesterday": 34668, "\u0120fuzzy": 34669, "wash": 34670, "mere": 34671, "\u0120bewild": 34672, "\u0120Mae": 34673, "Virgin": 34674, "phrase": 34675, "\u0120signaled": 34676, "\u0120HIGH": 34677, "\u0120protester": 34678, "\u0120garner": 34679, "unknown": 34680, "\u0120kay": 34681, "\u0120abducted": 34682, "\u0120stalking": 34683, "amn": 34684, "\u0120deserving": 34685, "\u0120Riv": 34686, "\u0120Jorge": 34687, "\u0120scratching": 34688, "\u0120Saving": 34689, "iping": 34690, "\u0120tease": 34691, "\u0120missionary": 34692, "\u0120Morrow": 34693, "TIME": 34694, "Present": 34695, "\u0120chemotherapy": 34696, "terness": 34697, "\u0120Homes": 34698, "\u0120Purdue": 34699, "\u0120staunch": 34700, "\u0120Whitney": 34701, "\u0120THERE": 34702, "\u00ce\u00bc": 34703, "iatus": 34704, "\u0120Ernest": 34705, "\u0120Deploy": 34706, "\u0120coveted": 34707, "FML": 34708, "\u0120Dialogue": 34709, "\u0120exited": 34710, "fruit": 34711, "\u0120nerd": 34712, "\":\"\",\"": 34713, "\u0120vivo": 34714, "ruly": 34715, "460": 34716, "\u0120Amen": 34717, "rehensible": 34718, "\u0120\u00e2\u013a": 34719, "DIR": 34720, "\u0120adherence": 34721, "\u0120chew": 34722, "\u0120Coke": 34723, "\u0120Sergei": 34724, "digital": 34725, "\u0120Neck": 34726, "gently": 34727, "enthal": 34728, "/)": 34729, "\u0120weary": 34730, "\u0120guise": 34731, "\u0120Concord": 34732, "\u0120Onion": 34733, "atcher": 34734, "\u0120binge": 34735, "\u0120Directive": 34736, "\u0120manned": 34737, "ansk": 34738, "\u0120illusions": 34739, "\u0120billionaires": 34740, "383": 34741, "olyn": 34742, "odynamic": 34743, "\u0120Wheat": 34744, "\u0120Alic": 34745, "\u0120coloured": 34746, "\u0120NAFTA": 34747, "abo": 34748, "\u0120macros": 34749, "independent": 34750, "sweet": 34751, "\u0120spac": 34752, "\u0120Kabul": 34753, "\u0120\u00c4": 34754, "eme": 34755, "\u0120dictated": 34756, "\u0120shouts": 34757, "={": 34758, "\u0120ripping": 34759, "\u0120Shay": 34760, "\u0120Cricket": 34761, "directed": 34762, "\u0120analysed": 34763, "\u0120WARRANT": 34764, "agons": 34765, "\u0120Blazers": 34766, "\u0120cheered": 34767, "\u0120arithmetic": 34768, "\u0120Tanz": 34769, "373": 34770, "\u0120Flags": 34771, "\u0120295": 34772, "\u0120witches": 34773, "\u0120Included": 34774, "\u0120Gained": 34775, "\u0120Blades": 34776, "Gam": 34777, "\u0120Samantha": 34778, "\u0120Atlantis": 34779, "\u0120Pratt": 34780, "\u0120spoiled": 34781, "\u0120IB": 34782, "\u0120Ramirez": 34783, "Probably": 34784, "rero": 34785, "\u0120Ng": 34786, "\u0120Warlock": 34787, "tp": 34788, "\u0120overhe": 34789, "\u0120administrations": 34790, "\u0120tint": 34791, "\u0120regiment": 34792, "\u0120pistols": 34793, "\u0120blankets": 34794, "\u0120epist": 34795, "\u0120bowls": 34796, "\u0120hydraulic": 34797, "\u0120dean": 34798, "\u0120jung": 34799, "\u0120ascend": 34800, "705": 34801, "\u0120Santiago": 34802, "\u00c3\u00ae": 34803, "\u0120unavoid": 34804, "\u0120Shaman": 34805, "reb": 34806, "\u0120stemming": 34807, "998": 34808, "\u0120MG": 34809, "sticks": 34810, "esthesia": 34811, "ERO": 34812, "\u0120morbid": 34813, "\u0120Grill": 34814, "\u0120Poe": 34815, "anyl": 34816, "\u0120deleting": 34817, "\u0120Surveillance": 34818, "\u0120directives": 34819, "\u0120iterations": 34820, "\u0120Rox": 34821, "\u0120Milky": 34822, "Father": 34823, "\u0120patented": 34824, "447": 34825, "\u0120precursor": 34826, "\u0120maiden": 34827, "\u0120Phen": 34828, "\u0120Vegan": 34829, "\u0120Patent": 34830, "Kelly": 34831, "Redditor": 34832, "\u0120nods": 34833, "\u0120ventilation": 34834, "\u0120Schwarz": 34835, "\u0120wizards": 34836, "\u0120ominous": 34837, "\u0120Heads": 34838, "\u0120BG": 34839, "\u0120lumber": 34840, "\u0120Spiel": 34841, "\u0120isEnabled": 34842, "\u0120ancestral": 34843, "\u0120Ships": 34844, "\u0120wrestler": 34845, "phi": 34846, "\u0120yuan": 34847, "\u0120Rebellion": 34848, "\u0120iceberg": 34849, "\u0120magically": 34850, "\u0120diversion": 34851, "arro": 34852, "ythm": 34853, "\u0120Riders": 34854, "\u0120Robbie": 34855, "\u0120Kara": 34856, "\u0120Maintenance": 34857, "\u0120Herb": 34858, "\u0120harms": 34859, "packed": 34860, "\u0120Feinstein": 34861, "\u0120marrying": 34862, "\u0120blending": 34863, "\u0120Rates": 34864, "\u01201880": 34865, "\u0120wrink": 34866, "\u0120Unch": 34867, "\u0120Torch": 34868, "described": 34869, "\u0120humanoid": 34870, "ilitating": 34871, "\u0120Conv": 34872, "\u0120Feld": 34873, "IGHTS": 34874, "\u0120whistleblower": 34875, "ortmund": 34876, "etsy": 34877, "arrett": 34878, "\u0120Mono": 34879, "\u0120Ike": 34880, "\u0120CNBC": 34881, "\u0120WAY": 34882, "\u0120MDMA": 34883, "\u0120Individuals": 34884, "\u0120supplemental": 34885, "\u0120powerhouse": 34886, "\u0120Stru": 34887, "Focus": 34888, "aphael": 34889, "\u0120Colleg": 34890, "atti": 34891, "ZA": 34892, "\u0120perenn": 34893, "\u0120Signature": 34894, "\u0120Rodney": 34895, "\u0120cubes": 34896, "iddled": 34897, "\u0120Dante": 34898, "\u0120INV": 34899, "ilingual": 34900, "\u0120Cth": 34901, "\u0120sofa": 34902, "\u0120intimidate": 34903, "\u0120Roe": 34904, "\u0120Diplom": 34905, "\u0120Countries": 34906, "ayson": 34907, "\u0120extradition": 34908, "\u0120disabling": 34909, "\u0120Cardiff": 34910, "\u0120memorandum": 34911, "\u0120Trace": 34912, "\u0120???": 34913, "sector": 34914, "\u0120Rouhani": 34915, "\u0120Yates": 34916, "\u0120Freeze": 34917, "\u0120bladder": 34918, "Motor": 34919, "\u0120Promise": 34920, "antasy": 34921, "\u0120foreseeable": 34922, "\u0120Cologne": 34923, "container": 34924, "\u0120Trees": 34925, "\u0120Gors": 34926, "\u0120Sinclair": 34927, "\u0120barring": 34928, "keye": 34929, "\u0120slashed": 34930, "\u0120Statistical": 34931, "\u00e9\u0129": 34932, "\u0120\u00e2\u0138\u00ba": 34933, "Allows": 34934, "\u0120humility": 34935, "\u0120drilled": 34936, "\u0120Furn": 34937, "443": 34938, "\u0120sewage": 34939, "\u0120homepage": 34940, "\u0120courtyard": 34941, "\u0120vile": 34942, "\u0120subsidiaries": 34943, "ajo": 34944, "directory": 34945, "\u0120ammon": 34946, "Vers": 34947, "charges": 34948, "\u0120}}": 34949, "\u0120Chains": 34950, "\u0120246": 34951, "nob": 34952, "\u0120percept": 34953, "\u0120grit": 34954, "\u0120fishermen": 34955, "\u0120Iraqis": 34956, "\u0120DISTR": 34957, "\u0120FULL": 34958, "\u0120Evaluation": 34959, "graph": 34960, "atial": 34961, "\u0120cooperating": 34962, "\u0120melan": 34963, "\u0120enlightened": 34964, "\u0120ali": 34965, "tailed": 34966, "\u0120salute": 34967, "\u0120weakest": 34968, "\u0120Bulldogs": 34969, "UA": 34970, "\u0120Alloy": 34971, "\u0120semen": 34972, "ocene": 34973, "\u0120Williamson": 34974, "spr": 34975, ",\u00e2\u0122\u0136": 34976, "\u0120GF": 34977, "ittens": 34978, "Beat": 34979, "\u0120Junk": 34980, "iphate": 34981, "\u0120Farmers": 34982, "\u0120Bitcoins": 34983, "igers": 34984, "dh": 34985, "\u0120Loyal": 34986, "payer": 34987, "\u0120entertained": 34988, "\u0120penned": 34989, "\u0120coupon": 34990, "Queue": 34991, "\u0120weakening": 34992, "carry": 34993, "\u0120underestimate": 34994, "\u0120shootout": 34995, "\u0120charismatic": 34996, "\u0120Procedure": 34997, "\u0120prudent": 34998, "inances": 34999, "\u0120riches": 35000, "\u0120cortical": 35001, "\u0120strides": 35002, "\u0120drib": 35003, "\u0120Oilers": 35004, "540": 35005, "\u0120Perform": 35006, "\u0120Bangkok": 35007, "\u0120euth": 35008, "SER": 35009, "\u0120simplistic": 35010, "tops": 35011, "campaign": 35012, "Quality": 35013, "\u0120impoverished": 35014, "\u0120Eisenhower": 35015, "\u0120augment": 35016, "\u0120Harden": 35017, "\u0120intervened": 35018, "\u0120listens": 35019, "\u0120Kok": 35020, "\u0120sage": 35021, "\u0120rubbish": 35022, "\u0120Ded": 35023, "\u0120mull": 35024, "pelling": 35025, "\u0120videot": 35026, "Production": 35027, "DJ": 35028, "miah": 35029, "\u0120adaptations": 35030, "\u0120medically": 35031, "\u0120boarded": 35032, "\u0120arrogance": 35033, "\u0120scrapped": 35034, "\u0120oppress": 35035, "FORMATION": 35036, "\u0120junction": 35037, "415": 35038, "EEEE": 35039, "Skill": 35040, "\u0120subdu": 35041, "\u0120Suggest": 35042, "\u0120Pett": 35043, "\u0120lett": 35044, "\u0120Manip": 35045, "\u0120Caf": 35046, "\u0120Cooperation": 35047, "Ther": 35048, "\u0120regained": 35049, "\u00b6\u00e6": 35050, "reflect": 35051, "\u0120thugs": 35052, "\u0120Shelby": 35053, "\u0120dictates": 35054, "\u0120Weiner": 35055, "\u0120Hale": 35056, "\u0120battleground": 35057, "schild": 35058, "\u0120condol": 35059, "hunt": 35060, "ositories": 35061, "\u0120accuses": 35062, "Filename": 35063, "\u0120shri": 35064, "\u0120motivate": 35065, "\u0120reflections": 35066, "Null": 35067, "\u0120Lobby": 35068, "\u00a5\u00b5": 35069, "\u0120SATA": 35070, "\u0120Backup": 35071, "\u00d1\u0125": 35072, "nin": 35073, "\u0120Correction": 35074, "\u0120juicy": 35075, "utra": 35076, "\u0120Pric": 35077, "\u0120restraining": 35078, "\u0120Airbnb": 35079, "\u0120Arrest": 35080, "\u0120appropriations": 35081, "\u0120slopes": 35082, "\u0120manslaughter": 35083, "\u0120workings": 35084, "\u0120Huss": 35085, "\u0120Frey": 35086, "Leave": 35087, "\u0120Harmony": 35088, "\u0120Feder": 35089, "\u0120430": 35090, "\u0120trench": 35091, "\u0120gladly": 35092, "\u0120bullpen": 35093, "\u0120Gau": 35094, "bones": 35095, "\u0120groove": 35096, "\u0120pretext": 35097, "\u00e3\u0127\u012d": 35098, "\u0120transmitter": 35099, "\u0120Component": 35100, "\u0120underage": 35101, "\u0120Empires": 35102, "Tile": 35103, "\u0120oy": 35104, "\u0120Marvin": 35105, "\u0120CAS": 35106, "\u0120bloss": 35107, "\u0120replicated": 35108, "\u0120Mariners": 35109, "Marcus": 35110, "\u0120Blocks": 35111, "\u0120liberated": 35112, "\u0120butterfly": 35113, "Feel": 35114, "\u0120fermentation": 35115, "\u0120youtube": 35116, "\u0120offend": 35117, "\u0120Term": 35118, "resist": 35119, "\u0120cessation": 35120, "\u0120insurgency": 35121, "\u0120bir": 35122, "\u0120Raise": 35123, "595": 35124, "\u0120hypotheses": 35125, "502": 35126, "\u0120plaque": 35127, "ocrat": 35128, "\u0120jackets": 35129, "\u0120HuffPost": 35130, "among": 35131, "\u0120confer": 35132, "487": 35133, "\u0120Lilly": 35134, "\u0120adapting": 35135, "\u0120Fay": 35136, "\u0120shoved": 35137, "vec": 35138, "\u0120refine": 35139, "\u0120gon": 35140, "\u0120gunmen": 35141, "zai": 35142, "\u0120Shuttle": 35143, "\u0120Izan": 35144, "\u01201913": 35145, "\u0120plethora": 35146, "\u00c2\u00b7\u00c2\u00b7": 35147, "\u0120510": 35148, "\u0120puberty": 35149, "\u0120241": 35150, "\u0120Wealth": 35151, "\u0120Alma": 35152, "\u0120MEM": 35153, "\u0120Adults": 35154, "Cas": 35155, "prison": 35156, "Race": 35157, "\u0120waterproof": 35158, "\u0120athleticism": 35159, "\u0120capitalize": 35160, "\u0120Juice": 35161, "\u0120illuminated": 35162, "\u0120Pascal": 35163, "\u0120irritation": 35164, "\u0120Witnesses": 35165, "adle": 35166, "\u0120Astro": 35167, "\u0120fax": 35168, "\u0120Elvis": 35169, "Primary": 35170, "\u0120Lich": 35171, "\u0120Elves": 35172, "\u0120residing": 35173, "\u0120stumble": 35174, "319": 35175, "\u0120PKK": 35176, "\u0120adversaries": 35177, "DOS": 35178, "\u0120Ritual": 35179, "\u0120smear": 35180, "\u0120arson": 35181, "idental": 35182, "\u0120scant": 35183, "\u0120monarchy": 35184, "\u0120halftime": 35185, "\u0120residue": 35186, "\u0120indign": 35187, "\u0120Shaun": 35188, "\u0120Elm": 35189, "auri": 35190, "Aff": 35191, "WATCH": 35192, "\u0120Lyon": 35193, "helps": 35194, "361": 35195, "\u0120lobbyist": 35196, "\u0120diminishing": 35197, "\u0120outbreaks": 35198, "\u0120goats": 35199, "favorite": 35200, "\u0120Nah": 35201, "sonian": 35202, "\u0120Booster": 35203, "\u0120sandbox": 35204, "\u0120Fare": 35205, "\u0120Malta": 35206, "\u0120attRot": 35207, "\u0120MOR": 35208, "lde": 35209, "\u0120navigating": 35210, "Touch": 35211, "\u0120untrue": 35212, "\u0120Disaster": 35213, "\u0120ludicrous": 35214, "Password": 35215, "\u0120JFK": 35216, "blogspot": 35217, "416": 35218, "\u0120UNDER": 35219, "ernal": 35220, "\u0120delaying": 35221, "TOP": 35222, "\u0120implants": 35223, "\u0120AVG": 35224, "\u0120Huge": 35225, "attr": 35226, "\u0120journalistic": 35227, "\u0120Peyton": 35228, "\u0120IA": 35229, "Rap": 35230, "goal": 35231, "\u0120Programme": 35232, "\u0120smashing": 35233, "wives": 35234, "println": 35235, "\u0120Plague": 35236, "inus": 35237, "EEP": 35238, "\u0120cruiser": 35239, "\u0120Parish": 35240, "uminium": 35241, "\u0120occupants": 35242, "\u0120Jihad": 35243, "mop": 35244, "\u0120pint": 35245, "\u0120hect": 35246, "\u0120Mecca": 35247, "director": 35248, "\u0120Funding": 35249, "\u0120Mixed": 35250, "\u0120stag": 35251, "Tier": 35252, "\u0120gust": 35253, "\u0120brightly": 35254, "orsi": 35255, "\u0120uphill": 35256, "RD": 35257, "\u0120lesions": 35258, "\u0120Bundy": 35259, "livious": 35260, "\u0120biologist": 35261, "\u0120Faculty": 35262, "\u0120Authorization": 35263, "\u0120244": 35264, "Allow": 35265, "\u00ef\u00b8": 35266, "\u0120Giul": 35267, "\u0120pertinent": 35268, "otaur": 35269, "esse": 35270, "\u0120Roof": 35271, "\u0120unmanned": 35272, "351": 35273, "\u0120Shak": 35274, "\u0120Orient": 35275, "\u0120endanger": 35276, "Dir": 35277, "\u0120replen": 35278, "edient": 35279, "\u0120tailor": 35280, "\u0120gadgets": 35281, "\u0120audible": 35282, "\u00e2\u013a\u0128": 35283, "Nice": 35284, "\u0120bombard": 35285, "\u0120Rape": 35286, "\u0120defiance": 35287, "\u0120TWO": 35288, "\u0120Filipino": 35289, "\u0120unaffected": 35290, "ervatives": 35291, "\u0120soared": 35292, "\u0120Bolton": 35293, "\u0120compromising": 35294, "\u0120Brewers": 35295, "RAL": 35296, "\u0120AHL": 35297, "icycle": 35298, "\u0120vampires": 35299, "\u0120dipped": 35300, "oyer": 35301, "\u0120XIII": 35302, "\u0120sideways": 35303, "\u0120Waste": 35304, "\u0120Diss": 35305, "\u0120\u00e2\u0136\u013e\u00e2\u0136\u0122\u00e2\u0136\u0122": 35306, "$.": 35307, "\u0120habitats": 35308, "\u0120Beef": 35309, "truth": 35310, "trained": 35311, "split": 35312, "Rus": 35313, "Andy": 35314, "\u0120Bram": 35315, "REP": 35316, "pid": 35317, "\u00e8\u00a3\u0127": 35318, "\u0120Mutant": 35319, "Anim": 35320, "\u0120Marina": 35321, "\u0120futile": 35322, "highest": 35323, "frequency": 35324, "\u0120epilepsy": 35325, "\u0120coping": 35326, "\u0120concise": 35327, "\u0120tracing": 35328, "\u0120SUN": 35329, "panel": 35330, "\u0120Sophie": 35331, "\u0120Crowley": 35332, "\u0120Adolf": 35333, "\u0120Shooter": 35334, "\u0120shaky": 35335, "\u0120IG": 35336, "\u0120Lies": 35337, "\u0120Barber": 35338, "pkg": 35339, "\u0120uptake": 35340, "\u0120predatory": 35341, "ULTS": 35342, "/**": 35343, "\u0120intoxicated": 35344, "\u0120Westbrook": 35345, "odder": 35346, "hement": 35347, "\u0120baseman": 35348, "APD": 35349, "storage": 35350, "\u0120Fifty": 35351, "editor": 35352, "GEN": 35353, "UTION": 35354, "irting": 35355, "\u0120sewing": 35356, "rift": 35357, "\u0120agony": 35358, "\u0120Sands": 35359, "\u0120254": 35360, "Cash": 35361, "\u0120lodge": 35362, "\u0120punt": 35363, "Natural": 35364, "\u0120Ideas": 35365, "\u0120erroneous": 35366, "\u0120Sensor": 35367, "\u0120Hannity": 35368, "\u01201921": 35369, "\u0120mould": 35370, "\u0120Gon": 35371, "kaya": 35372, "\u0120anonymously": 35373, "\u0120KEY": 35374, "\u0120simulator": 35375, "Winter": 35376, "\u0120streamed": 35377, "507": 35378, "?\",": 35379, "\u0120teased": 35380, "\u0120coefficient": 35381, "\u0120wartime": 35382, "\u0120THR": 35383, "''.": 35384, "\u0120Banking": 35385, "mpire": 35386, "\u0120fandom": 35387, "\u0120lia": 35388, "Ga": 35389, "\u0120downhill": 35390, "\u0120interpreting": 35391, "Individual": 35392, "Norm": 35393, "\u0120jealousy": 35394, "bitcoin": 35395, "\u0120pleasures": 35396, "\u0120Toys": 35397, "\u0120Chevrolet": 35398, "\u0120Advisor": 35399, "IZE": 35400, "\u0120receptions": 35401, "706": 35402, "Cro": 35403, "\u0120262": 35404, "\u0120citrus": 35405, "iru": 35406, "Reviewer": 35407, "jected": 35408, "UES": 35409, "anz": 35410, "1981": 35411, "\u0120Worker": 35412, "\u0120complied": 35413, "orescent": 35414, "continental": 35415, "Ton": 35416, "\u0120Prism": 35417, "\u0120Sheep": 35418, "\u0120288": 35419, "nox": 35420, "\u0120Vog": 35421, "Ord": 35422, "\u0120realms": 35423, "tek": 35424, "\u0120irrigation": 35425, "\u0120bicycles": 35426, "\u0120electronically": 35427, "poly": 35428, "tall": 35429, "());": 35430, "\u0120aesthetics": 35431, "\u0120Integrated": 35432, "Explore": 35433, "\u0120dunk": 35434, "476": 35435, "pain": 35436, "\u0120Jacques": 35437, "\u0120Dmit": 35438, "Frames": 35439, "\u0120reunited": 35440, "\u0120humid": 35441, "Dro": 35442, "Political": 35443, "\u0120youthful": 35444, "\u0120entails": 35445, "\u0120mosquito": 35446, "363": 35447, "species": 35448, "\u0120coordinating": 35449, "\u0120Mayhem": 35450, "\u0120Magnus": 35451, "Mount": 35452, "Improved": 35453, "\u0120STATE": 35454, "ATTLE": 35455, "\u0120flowed": 35456, "\u0120tackled": 35457, "\u0120fashioned": 35458, "\u0120reorgan": 35459, "ivari": 35460, "finger": 35461, "\u0120reluctantly": 35462, "etting": 35463, "\u0120Vand": 35464, "young": 35465, "\u0120Garland": 35466, "\u0120presumption": 35467, "\u0120amenities": 35468, "\u0120Pleasant": 35469, "onential": 35470, "\u0120Oxy": 35471, "\u0120morals": 35472, "\u0120Yah": 35473, "Ready": 35474, "Simon": 35475, "Enh": 35476, "Demon": 35477, "\u0120clich": 35478, "Monitor": 35479, "\u0120DU": 35480, "\u0120welcomes": 35481, "\u0120standout": 35482, "\u0120dreadful": 35483, "\u0120bananas": 35484, "\u0120balloons": 35485, "hooting": 35486, "basic": 35487, "\u0120suffix": 35488, "\u0120duly": 35489, "cano": 35490, "Chain": 35491, "atos": 35492, "\u0120geopolitical": 35493, "\u0120(&": 35494, "\u0120Gemini": 35495, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 35496, "\u0120acquitted": 35497, "Luck": 35498, "protect": 35499, "1024": 35500, "\u0120scarcity": 35501, "\u0120mindfulness": 35502, "ecided": 35503, "DN": 35504, "prime": 35505, "\u0120Presidents": 35506, "\u0120VIDEO": 35507, "\u0120(\u00e2\u012a\u0134": 35508, "addock": 35509, "NOR": 35510, "\u0120Pru": 35511, "pun": 35512, "\u0120LOL": 35513, "))))": 35514, "\u0120Liqu": 35515, "\u0120SAS": 35516, "\u0120styling": 35517, "\u0120punishments": 35518, "\u0120numb": 35519, "\u0120ascertain": 35520, "\u0120Rockies": 35521, "flu": 35522, "Thumbnail": 35523, "\u0120perpetrated": 35524, "\u0120Semi": 35525, "\u0120disarm": 35526, "\u0120Older": 35527, "\u0120Exception": 35528, "\u0120exponentially": 35529, "\u0120Communities": 35530, "\u0120abolish": 35531, "\u0120Partner": 35532, "ptoms": 35533, "\u0120777": 35534, "\u0120Foley": 35535, "\u0120Cases": 35536, "\u0120grease": 35537, "\u0120Rebirth": 35538, "Ground": 35539, "\u0120;)": 35540, "\u0120Doctrine": 35541, "ikini": 35542, "Ye": 35543, "\u0120Blossom": 35544, "\u0120persists": 35545, "bill": 35546, "\u0120infusion": 35547, "\u0120buddies": 35548, "911": 35549, "\u0120Patient": 35550, "\u0120demos": 35551, "\u0120acquaintance": 35552, "\u0120Paw": 35553, "atari": 35554, "\u0120xml": 35555, "\u0120fascination": 35556, "\u0120Serve": 35557, "\u00cf\u0124": 35558, "branded": 35559, "\u0120az": 35560, "Returns": 35561, "\u0120overshadow": 35562, "\u0120roam": 35563, "\u0120speedy": 35564, "numbered": 35565, "helial": 35566, "\u0120disciple": 35567, "\u0120assurances": 35568, "given": 35569, "pecting": 35570, "\u0120Natalie": 35571, "\u00e7\u0136\u00b0": 35572, "\u0120mosquitoes": 35573, "rotein": 35574, "\u0120numeric": 35575, "\u0120independents": 35576, "\u0120transitional": 35577, "\u0120reactionary": 35578, "\u0120Mechdragon": 35579, "doctor": 35580, "\u0120shortest": 35581, "\u0120sequential": 35582, "\u0120Bac": 35583, "\u0120Accounts": 35584, "\u00e3\u0123\u012e": 35585, "achy": 35586, "ractive": 35587, "\u0120Regiment": 35588, "\u0120breathtaking": 35589, "fficiency": 35590, "\u0120Bates": 35591, "\u0120311": 35592, "\u0120wardrobe": 35593, "fts": 35594, "\u0120Berk": 35595, "Simply": 35596, "\u0120Riverside": 35597, "ivering": 35598, "idential": 35599, "lucent": 35600, "\u0120enriched": 35601, "\u0120Conver": 35602, "\u0120Giving": 35603, "\u00e3\u0125\u013b": 35604, "\u0120legalize": 35605, "\u0120FTC": 35606, "\u0120freaking": 35607, "Mix": 35608, "\u0120terrestrial": 35609, "esian": 35610, "cients": 35611, "Wing": 35612, "LOAD": 35613, "\u0120ledge": 35614, "\u0120Violent": 35615, "\u0120Metall": 35616, "\u0120308": 35617, "\u0120southeastern": 35618, "hetto": 35619, "Meat": 35620, "\u0120slowdown": 35621, "\u0120retreated": 35622, "Jeremy": 35623, "endas": 35624, "*****": 35625, "eric": 35626, "\u0120reins": 35627, "oppable": 35628, "\u0120Humanity": 35629, "earances": 35630, "rigan": 35631, "Camera": 35632, "\u0120waivers": 35633, "soc": 35634, "\u0120alteration": 35635, "transform": 35636, "\u0120Cemetery": 35637, "506": 35638, "\u0120indefinite": 35639, "\u0120stimulating": 35640, "yg": 35641, "603": 35642, "\u0120Sop": 35643, "\u0120descriptive": 35644, "Phase": 35645, "\u0120Edmund": 35646, "\u0120pneumonia": 35647, "ventus": 35648, "Amb": 35649, "\u0120laboratories": 35650, "\u0120Exclusive": 35651, "ugar": 35652, "Were": 35653, "\u0120malfunction": 35654, "\u0120homosexuals": 35655, "\u0120-------": 35656, "uni": 35657, "\u0120turbines": 35658, "\u0120Equity": 35659, "Du": 35660, "\u0120minded": 35661, "\u0120RH": 35662, "\u0120Blackhawks": 35663, "\u0120feats": 35664, "\u01201700": 35665, "repl": 35666, "362": 35667, "laden": 35668, "\u0120indispensable": 35669, "lyss": 35670, "tti": 35671, "\u0120reel": 35672, "\u0120diverted": 35673, "\u0120likeness": 35674, "\u0120subscriptions": 35675, "\u0120fingert": 35676, "\u0120filthy": 35677, "destruct": 35678, "draft": 35679, "\u0120Bernardino": 35680, "launch": 35681, "\u0120perplex": 35682, "\u0120SUM": 35683, "carb": 35684, "\u0120sweater": 35685, "\u0120Venture": 35686, "\u0120Jag": 35687, "\u0120Celeb": 35688, "\u0120Voters": 35689, "\u0120steadfast": 35690, "\u0120athletics": 35691, "\u0120Hanson": 35692, "\u0120Drac": 35693, "Tracker": 35694, "\u0120commend": 35695, "\u0120Presidency": 35696, "\u0120DID": 35697, "informed": 35698, "\u0120webpage": 35699, "Pretty": 35700, "\u0120forcefully": 35701, "\u00e3\u0125\u0125\u00e3\u0124\u00af": 35702, "\u0120relocation": 35703, "\u0120satire": 35704, "\u00e2\u012b": 35705, "\u0120Sunderland": 35706, "\u00e6\u0126": 35707, "Voice": 35708, "????????": 35709, "\u0120informant": 35710, "\u0120bowel": 35711, "\u0120Uniform": 35712, "\u0120...\"": 35713, "\u0120purge": 35714, "\u0120picnic": 35715, "\u0120Umb": 35716, "\u0120UPDATE": 35717, "\u0120Sapphire": 35718, "\u0120Stall": 35719, "learn": 35720, "\u0120objectively": 35721, "\u0120obliter": 35722, "\u0120loophole": 35723, "\u0120journeys": 35724, "\u0120omission": 35725, "Pros": 35726, "\u0120Sidney": 35727, "ploma": 35728, "\u0120sprayed": 35729, "\u0120guru": 35730, "\u0120traitor": 35731, "\u0120timet": 35732, "\u0120snapping": 35733, "\u0120Sevent": 35734, "urnal": 35735, "\u0120Ukip": 35736, "\u0120bowed": 35737, "poral": 35738, "liberal": 35739, "Ros": 35740, "Questions": 35741, "iOS": 35742, "\u0120summarize": 35743, "STAT": 35744, "\u01201850": 35745, "apest": 35746, "\u0120lender": 35747, "\u0120Variable": 35748, "bringing": 35749, "\u0120LORD": 35750, ",)": 35751, "\u0120collapses": 35752, "xiety": 35753, "\u0120Ned": 35754, "YD": 35755, "\u0120Scha": 35756, "\u0120antibody": 35757, "\u0120disband": 35758, "yre": 35759, "illusion": 35760, "\u0120rover": 35761, "shed": 35762, "\u0120Hirosh": 35763, "cci": 35764, "\u0120calam": 35765, "\u0120Morton": 35766, "Pinterest": 35767, "\u01201928": 35768, "\u0120Euras": 35769, "ordes": 35770, "\u0120fences": 35771, "\u0120Inventory": 35772, "\u0120Valencia": 35773, "\u0120Ud": 35774, "\u0120Tiff": 35775, "\u0120sque": 35776, "\u0120quotation": 35777, "\u0120troublesome": 35778, "erker": 35779, "QUEST": 35780, "\u0120Kingdoms": 35781, "south": 35782, "\u0120levy": 35783, "Prince": 35784, "\u0120Sting": 35785, "\u0120nicknamed": 35786, "\u0120appe": 35787, "\u0120photographic": 35788, "\u0120corpus": 35789, "reference": 35790, "\u0120Trog": 35791, "Unt": 35792, ")=(": 35793, "\u0120Latvia": 35794, "\u0120activating": 35795, "\u0120licensee": 35796, "\u0120disparities": 35797, "\u0120Newsletter": 35798, "\u00e3\u0125\u0125\u00e3\u0125\u012a": 35799, "\u0120freeing": 35800, "\u0120Jeep": 35801, "\u0120Perception": 35802, "insk": 35803, "\u0120silicone": 35804, "\u0120Hayden": 35805, "Lean": 35806, "\u0120Suzuki": 35807, "ibrarian": 35808, "668": 35809, "\u0120spor": 35810, "\u0120correlations": 35811, "aghetti": 35812, "\u0120tuber": 35813, "\u0120IPCC": 35814, "ilus": 35815, "\u0120Vu": 35816, "\u0120wealthiest": 35817, "\u0120Carbuncle": 35818, "anza": 35819, "\u0120fooled": 35820, "\u0120Zur": 35821, "\u0120daddy": 35822, "rano": 35823, "ilian": 35824, "\u0120knockout": 35825, "fman": 35826, "required": 35827, "\u0120Wikileaks": 35828, "\u0120Duffy": 35829, "ONT": 35830, "\u0120insol": 35831, "\u0120Objects": 35832, "\u0120bou": 35833, "\u0120Nordic": 35834, "\u0120Insert": 35835, "scan": 35836, "\u0120dancers": 35837, "\u0120idiots": 35838, "majority": 35839, "\u0120Neville": 35840, "\u0120FreeBSD": 35841, "\u0120tart": 35842, "panic": 35843, "690": 35844, "\u0120cocoa": 35845, "\u0120sampled": 35846, "\u0120lookup": 35847, "Indust": 35848, "\u0120injections": 35849, "genre": 35850, "\u0120au": 35851, "\u0120roadway": 35852, "\u0120genitals": 35853, "Kind": 35854, "\u0120Examiner": 35855, "\u0120Yaz": 35856, "Fresh": 35857, "\u0120paralysis": 35858, "\u0120Aluminum": 35859, "\u0120reap": 35860, "ok\u00c3\u00a9": 35861, "\u0120sloppy": 35862, "\u0120Tunnel": 35863, "posium": 35864, "nery": 35865, "enic": 35866, "\u0120herbal": 35867, "\u0120Outer": 35868, "\u0120Builder": 35869, "\u0120incur": 35870, "\u0120ideologies": 35871, "\u0120backups": 35872, "consuming": 35873, "\u0120Detect": 35874, "deck": 35875, "\u0120KNOW": 35876, "\u0120Gret": 35877, "\u0120MIC": 35878, "\u0120toughness": 35879, "\u0120Exhibit": 35880, "\u0120hive": 35881, "Les": 35882, "\u0120SCHOOL": 35883, "\u0120Atari": 35884, "alde": 35885, "\u0120Null": 35886, "andestine": 35887, "mouse": 35888, "\u0120brigade": 35889, "489": 35890, "\u0120revol": 35891, "\u0120Lawson": 35892, "\u0120Wah": 35893, "opoly": 35894, "ebted": 35895, "\u0120Saunders": 35896, "\u0120313": 35897, "\u0120Winc": 35898, "\u0120taboo": 35899, "\u0120Helmet": 35900, "\u0120wedge": 35901, "chip": 35902, "\u0120Tina": 35903, "bg": 35904, "\u0120infuri": 35905, "rn": 35906, "\u0120anomalies": 35907, "\u0120Sync": 35908, "\u0120Exam": 35909, "\u0120Commit": 35910, "\u0120Diary": 35911, "\u0120ALSO": 35912, "\u0120Debor": 35913, "omedical": 35914, "\u0120comprehension": 35915, "655": 35916, "\u0120empowering": 35917, "\u0120ire": 35918, "\u0120juices": 35919, "\u0120ETH": 35920, "\u0120Boxing": 35921, "=\"/": 35922, "\u0120facilitated": 35923, "poke": 35924, "\u0120Parsons": 35925, "\u0120Moder": 35926, "travel": 35927, "\u0120civilizations": 35928, "\u0120libertarians": 35929, "\u0120rune": 35930, "\u0120Clarks": 35931, "athed": 35932, "\u0120campaigners": 35933, "\u0120Dispatch": 35934, "\u0120Fahrenheit": 35935, "\u0120Capcom": 35936, "----------": 35937, "\u0120lace": 35938, "\u0120draining": 35939, "\u0120liner": 35940, "\u0120Artificial": 35941, "\u00c3\u00a9n": 35942, "task": 35943, "]).": 35944, "\u0120GMO": 35945, "\u0120Operator": 35946, "ordinary": 35947, "\u0120Influence": 35948, "\u0120Ups": 35949, "\u0120potency": 35950, "ussen": 35951, "ospons": 35952, "\u0120Swim": 35953, "\u0120Deadline": 35954, "Unity": 35955, "\u0120culinary": 35956, "\u0120enlightenment": 35957, "\u0120wearer": 35958, "\u0120mined": 35959, "\u0120ply": 35960, "\u0120incest": 35961, "\u0120DVDs": 35962, "Walk": 35963, "BTC": 35964, "Trade": 35965, "\u0120deval": 35966, "iband": 35967, "\u0120Oversight": 35968, "Palestinian": 35969, "\u0120dart": 35970, "\u0120mul": 35971, "LR": 35972, "\u0120removable": 35973, "\u0120Realms": 35974, "\u00ec\u013f": 35975, "\u0120miscar": 35976, "\u0120Vulkan": 35977, "685": 35978, "\u00c3\u00a8re": 35979, "\u0120Sap": 35980, "\u0120merging": 35981, "\u0120Carly": 35982, "chester": 35983, "\u0120brisk": 35984, "\u0120luxurious": 35985, "\u0120Generator": 35986, "\u0120bitterness": 35987, "\u0120edible": 35988, "\u0120243": 35989, "TG": 35990, "\u0120rectangle": 35991, "WithNo": 35992, "below": 35993, "Jenn": 35994, "\u0120darkest": 35995, "\u0120hitch": 35996, "\u0120dosage": 35997, "\u0120scaven": 35998, "\u0120Keller": 35999, "\u0120Illustrated": 36000, "Certainly": 36001, "\u0120Mavericks": 36002, "Marginal": 36003, "\u0120diarrhea": 36004, "\u0120enormously": 36005, "\u0120999": 36006, "shr": 36007, "quart": 36008, "\u0120adamant": 36009, "\u0120Mew": 36010, "\u0120renovation": 36011, "\u0120cervical": 36012, "\u0120Percentage": 36013, "eners": 36014, "\u0120Kimber": 36015, "\u0120floats": 36016, "\u0120dex": 36017, "\u0120Witcher": 36018, "\u0120Swansea": 36019, "dm": 36020, "\u0120salty": 36021, "yellow": 36022, "\u0120cape": 36023, "\u0120Drain": 36024, "\u0120Paula": 36025, "\u0120Toledo": 36026, "lesi": 36027, "Magazine": 36028, "\u0120Wick": 36029, "\u0120Mn": 36030, "\u0120Ack": 36031, "\u0120Riding": 36032, "ASON": 36033, "\u0120homophobic": 36034, "ARP": 36035, "\u0120wandered": 36036, "CPU": 36037, "oodoo": 36038, "\u0120Pipe": 36039, "\u0120tightening": 36040, "\u0120Butt": 36041, "318": 36042, "\u0120deserted": 36043, "Session": 36044, "\u0120facilitating": 36045, "Jump": 36046, "\u0120emergencies": 36047, "OWER": 36048, "\u0120exhaustive": 36049, "\u0120AFTER": 36050, "\u0120heartbeat": 36051, "\u0120Label": 36052, "acky": 36053, "\u0120Certified": 36054, "iltration": 36055, "Ze": 36056, "\u0120Utt": 36057, "\u01201300": 36058, "\u0120presume": 36059, "\u0120Disp": 36060, "\u0120surged": 36061, "\u0120dolls": 36062, "Columb": 36063, "\u0120chimpan": 36064, "\u0120Razor": 36065, "\u0120ticks": 36066, "\u0120councillor": 36067, "\u0120pilgrimage": 36068, "\u0120Rebels": 36069, "\u0120QC": 36070, "\u0120Auction": 36071, "xia": 36072, "ikk": 36073, "bred": 36074, "\u0120insertion": 36075, "\u0120coarse": 36076, "dB": 36077, "SEE": 36078, "\u0120Zap": 36079, "\u0120Foo": 36080, "\u0120contempor": 36081, "\u0120Quarterly": 36082, "otions": 36083, "\u0120Alchemist": 36084, "\u0120Trey": 36085, "\u0120Duo": 36086, "Sweet": 36087, "804": 36088, "\u0120Giov": 36089, "\u0120funn": 36090, "Nin": 36091, "hoff": 36092, "\u0120ramifications": 36093, "\u01201922": 36094, "\u0120Experts": 36095, "azes": 36096, "\u0120garments": 36097, "arial": 36098, "\u0120Nab": 36099, "\u0120257": 36100, "\u0120Ved": 36101, "\u0120humorous": 36102, "\u0120Pompe": 36103, "\u0120nylon": 36104, "\u0120lurking": 36105, "\u0120Sergey": 36106, "\u0120Mattis": 36107, "\u0120misogyny": 36108, "\u0120Components": 36109, "\u0120Watching": 36110, "\u0120Folk": 36111, "ractical": 36112, "Bush": 36113, "\u0120taped": 36114, "\u0120grouping": 36115, "\u0120beads": 36116, "\u01202048": 36117, "\u0120condu": 36118, "querque": 36119, "Reading": 36120, "\u0120grievances": 36121, "Ultra": 36122, "\u0120endpoint": 36123, "Hig": 36124, "\u0120Static": 36125, "\u0120Scarborough": 36126, "Lua": 36127, "\u0120Messi": 36128, "aqu": 36129, "\u0120PsyNet": 36130, "\u0120Rudd": 36131, "\u0120avenue": 36132, "vp": 36133, "Jer": 36134, "\u0120shady": 36135, "\u0120Resist": 36136, "\u0120Artemis": 36137, "\u0120careless": 36138, "\u0120brokers": 36139, "\u0120temperament": 36140, "\u0120520": 36141, "Tags": 36142, "\u0120Turning": 36143, "\u0120uttered": 36144, "\u0120pedd": 36145, "\u0120improvised": 36146, "\u0120:(": 36147, "\u0120tabl": 36148, "\u0120plains": 36149, "1600": 36150, "pressure": 36151, "\u0120Essence": 36152, "margin": 36153, "friends": 36154, "\u0120Restoration": 36155, "\u0120pollut": 36156, "\u0120Poker": 36157, "\u0120Augustine": 36158, "\u0120CIS": 36159, "\u0120SEAL": 36160, "orama": 36161, "\u0120thwart": 36162, "seek": 36163, "\u0120pagan": 36164, "\u00c2\u00ba": 36165, "cpu": 36166, "\u0120garn": 36167, "\u0120assortment": 36168, "\u0120ILCS": 36169, "tower": 36170, "Recommended": 36171, "\u0120unborn": 36172, "\u0120RandomRedditor": 36173, "\u0120RandomRedditorWithNo": 36174, "\u0120paralyzed": 36175, "\u0120eruption": 36176, "\u0120intersect": 36177, "\u0120Stoke": 36178, "\u0120Sco": 36179, "Bind": 36180, "\u00e5\u00be": 36181, "\u0120PNG": 36182, "\u0120Negative": 36183, "\u0120NOAA": 36184, "Leon": 36185, "\u0120alloy": 36186, "\u0120Lama": 36187, "\u0120Diversity": 36188, "575": 36189, "\u0120underestimated": 36190, "\u0120Scor": 36191, "\u0120mural": 36192, "\u0120busted": 36193, "soon": 36194, "lif": 36195, "\u0120nonex": 36196, "\u0120allergy": 36197, "\u0120Underworld": 36198, "\u0120Rays": 36199, "\u0120Blasio": 36200, "\u0120hrs": 36201, "\u0120Dir": 36202, "\u0120327": 36203, "byter": 36204, "\u0120replacements": 36205, "\u0120activates": 36206, "rived": 36207, "MH": 36208, "\u0120pans": 36209, "\u0120HI": 36210, "\u0120longitudinal": 36211, "\u0120nuisance": 36212, "aler": 36213, "\u0120swell": 36214, "\u0120Signed": 36215, "sci": 36216, "\u0120Isles": 36217, "\u0120AGA": 36218, "\u0120defiant": 36219, "\u0120sonic": 36220, "ocon": 36221, "KC": 36222, "\u0120Aim": 36223, "tie": 36224, "ahah": 36225, "\u0120mL": 36226, "DX": 36227, "\u0120bisc": 36228, "\u0120Billboard": 36229, "\u0120SYSTEM": 36230, "NEY": 36231, "gaard": 36232, "\u0120distressed": 36233, "formerly": 36234, "Alan": 36235, "\u0120chefs": 36236, "\u0120optics": 36237, "\u0120Comet": 36238, "\u0120AMC": 36239, "\u0120redesigned": 36240, "irmation": 36241, "\u0120sightings": 36242, "382": 36243, "311": 36244, "\u0120WB": 36245, "\u0120contraction": 36246, "\u0120TOTAL": 36247, "Dual": 36248, "\u0120startled": 36249, "\u0120understandably": 36250, "\u0120sunglasses": 36251, "ETHOD": 36252, "\u0120docker": 36253, "\u0120surfing": 36254, "\u0120HEL": 36255, "\u0120Slack": 36256, "tones": 36257, "\u0120shalt": 36258, "Visual": 36259, "498": 36260, "Department": 36261, "cussion": 36262, "\u0120unrestricted": 36263, "\u0120tad": 36264, "\u0120rename": 36265, "employed": 36266, "\u0120educating": 36267, "\u0120grinned": 36268, "bedroom": 36269, "\u0120Activities": 36270, "\u0120Velvet": 36271, "\u0120SWAT": 36272, "\u0120shuffle": 36273, "igor": 36274, "\u0120saturation": 36275, "Finding": 36276, "cream": 36277, "icter": 36278, "\u0120vodka": 36279, "tracking": 36280, "tec": 36281, "\u0120foreground": 36282, "iesta": 36283, "\u0120vehement": 36284, "\u0120ECB": 36285, "\u0120Tie": 36286, "Ey": 36287, "\u0120turtles": 36288, "\u0120Railroad": 36289, "\u0120Katz": 36290, "\u0120Frames": 36291, "\u0120menace": 36292, "\u0120Fellowship": 36293, "\u0120Essential": 36294, "uggish": 36295, "\u0120drip": 36296, "chwitz": 36297, "\u0120Kyoto": 36298, "sb": 36299, "\u0120Nina": 36300, "Parameter": 36301, "\u0120alarms": 36302, "\u0120Claud": 36303, "\u0120pioneering": 36304, "\u0120chiefly": 36305, "\u0120Scream": 36306, "Collection": 36307, "\u0120thankfully": 36308, "\u0120Ronaldo": 36309, "\u00e5\u0143\u0132": 36310, "strip": 36311, "\u0120Disneyland": 36312, "commercial": 36313, "Seeing": 36314, "Soul": 36315, "\u0120evacuate": 36316, "\u0120civ": 36317, "\u0120Ashe": 36318, "\u0120divides": 36319, "\u0120Dagger": 36320, "rehensive": 36321, "\u0120berries": 36322, "\u0120DF": 36323, "\u0120sushi": 36324, "\u0120plurality": 36325, "WI": 36326, "\u0120disadvantaged": 36327, "\u0120battalion": 36328, "obiles": 36329, "451": 36330, "\u0120cling": 36331, "\u0120undeniable": 36332, "\u0120Lounge": 36333, "\u0120haunt": 36334, "phe": 36335, "\u0120quantify": 36336, "\u0120differed": 36337, "\u0120[*]": 36338, "\u0120Viz": 36339, "cum": 36340, "slave": 36341, "\u0120videog": 36342, "\u0120quar": 36343, "\u0120bundles": 36344, "\u0120Alonso": 36345, "tackle": 36346, "\u0120neuronal": 36347, "\u0120landslide": 36348, "confirmed": 36349, "\u0120Depth": 36350, "\u0120renewables": 36351, "Bear": 36352, "\u0120Macedonia": 36353, "\u0120jerseys": 36354, "\u0120bunk": 36355, "\u0120Spawn": 36356, "\u0120Controls": 36357, "\u0120Buchanan": 36358, "\u0120robotics": 36359, "\u0120emphasizing": 36360, "\u0120Tutorial": 36361, "hyp": 36362, "iston": 36363, "\u0120monumental": 36364, "\u00e6\u00b0": 36365, "\u0120Carry": 36366, "\u0120tbsp": 36367, "enance": 36368, "Hill": 36369, "arthed": 36370, "\u0120rotten": 36371, "Dean": 36372, "\u0120twisting": 36373, "\u0120goodwill": 36374, "\u0120immersion": 36375, "Living": 36376, "\u0120brushes": 36377, "\u0120CGI": 36378, "\u0120Atk": 36379, "traditional": 36380, "\u0120phantom": 36381, "\u0120Stamina": 36382, "\u0120expansions": 36383, "\u0120Marin": 36384, "\u0120embarked": 36385, "\u0120Eg": 36386, "intestinal": 36387, "\u0120PEOPLE": 36388, "\u0120Booth": 36389, "\u0120Appalach": 36390, "\u0120relegated": 36391, "VT": 36392, "MIT": 36393, "\u0120muster": 36394, "\u0120withdrawing": 36395, "\u0120microscope": 36396, "\u0120Gathering": 36397, "\u0120Crescent": 36398, "\u0120Argentine": 36399, "\u0120Decre": 36400, "\u0120Dominic": 36401, "\u0120buds": 36402, "antage": 36403, "\u0120Ion": 36404, "\u0120widened": 36405, "ONSORED": 36406, "\u0120Gloves": 36407, "iannopoulos": 36408, "razen": 36409, "feel": 36410, "\u0120repayment": 36411, "\u0120hindsight": 36412, "\u0120REALLY": 36413, "\u0120Pistol": 36414, "\u0120Brah": 36415, "\u0120watts": 36416, "\u0120survives": 36417, "\u0120flurry": 36418, "issy": 36419, "Alert": 36420, "\u0120Uruguay": 36421, "Phoenix": 36422, "Slow": 36423, "\u0120Grave": 36424, "\u0120Fir": 36425, "\u0120manageable": 36426, "\u0120tariff": 36427, "\u0120UDP": 36428, "\u0120Pistons": 36429, "\u0120Nigerian": 36430, "\u0120strikeouts": 36431, "\u0120cosmetics": 36432, "whelming": 36433, "fab": 36434, "cape": 36435, "proxy": 36436, "\u0120rethink": 36437, "\u0120overcoming": 36438, "simple": 36439, "\u0120woo": 36440, "\u0120distracting": 36441, "\u0120Stanton": 36442, "\u0120Tulsa": 36443, "\u0120Dock": 36444, "659": 36445, "\u0120discord": 36446, "\u0120Emacs": 36447, "\u0120Ves": 36448, "\u0120ROB": 36449, "\u0120reassuring": 36450, "\u0120consortium": 36451, "Muslims": 36452, "321": 36453, "\u0120prompts": 36454, "sei": 36455, "\u0120Hitch": 36456, "imposed": 36457, "\u0120Fool": 36458, "\u0120indiscrim": 36459, "wrong": 36460, "buquerque": 36461, "Davis": 36462, "!]": 36463, "\u0120timeless": 36464, "\u0120NEED": 36465, "\u0120pesticide": 36466, "\u0120rallying": 36467, "\u0120Calder": 36468, "\u0120\u00e5\u00a4": 36469, "\u0120xp": 36470, "\u0120Unle": 36471, "\u0120Export": 36472, "luaj": 36473, "Buff": 36474, ")[": 36937, "\u0120sqor": 36938, "Saudi": 36939, "\u0120istg": 36940, "\u0120indulge": 36941, "proc": 36942, "\u0120disgusted": 36943, "\u0120compounded": 36944, "\u0120nem": 36945, "\u0120schooling": 36946, "\u0120Cure": 36947, "processing": 36948, "Sol": 36949, "\u0120proverb": 36950, "itized": 36951, "\u0120Alvarez": 36952, "\u0120scarf": 36953, "\u0120rectangular": 36954, "reve": 36955, "\u0120hormonal": 36956, "\u0120Stress": 36957, "itizen": 36958, "\u0120425": 36959, "girls": 36960, "\u0120Noir": 36961, "\u0120Rapp": 36962, "\u0120marches": 36963, "church": 36964, "\u0120Uses": 36965, "\u0120405": 36966, "\u0120Berm": 36967, "\u0120ordinances": 36968, "\u0120Judgment": 36969, "Charges": 36970, "\u0120Zin": 36971, "\u0120dusty": 36972, "\u0120strawberries": 36973, "\u0120perce": 36974, "\u0120Thur": 36975, "\u0120Deborah": 36976, "netflix": 36977, "\u0120Lambert": 36978, "\u0120amused": 36979, "\u0120Guang": 36980, "YOU": 36981, "RGB": 36982, "\u0120CCTV": 36983, "\u0120fiat": 36984, "rang": 36985, "\u0120federation": 36986, "\u0120Mant": 36987, "\u0120Bust": 36988, "\u0120Mare": 36989, "respective": 36990, "\u0120Migration": 36991, "\u0120BIT": 36992, "590": 36993, "\u0120patriotism": 36994, "\u0120outlining": 36995, "region": 36996, "\u0120Jos\u00c3\u00a9": 36997, "\u0120blasting": 36998, "\u0120Ezra": 36999, "Bs": 37000, "\u0120undermines": 37001, "\u0120Smooth": 37002, "\u0120clashed": 37003, "radio": 37004, "\u0120transitioning": 37005, "\u0120Buccaneers": 37006, "\u0120Owl": 37007, "\u0120plugs": 37008, "\u0120hiatus": 37009, "\u0120Pinball": 37010, "\u0120mig": 37011, "\u0120Nutr": 37012, "\u0120Wolfe": 37013, "\u0120integers": 37014, "\u0120orbits": 37015, "\u0120Edwin": 37016, "\u0120DirectX": 37017, "bite": 37018, "\u0120blazing": 37019, "vr": 37020, "Edge": 37021, "\u0120PID": 37022, "exit": 37023, "\u0120Comed": 37024, "\u0120Pathfinder": 37025, "\u0120Guid": 37026, "\u0120Signs": 37027, "\u0120Zer": 37028, "\u0120Agenda": 37029, "\u0120reimbursement": 37030, "Mesh": 37031, "iPhone": 37032, "\u0120Marcos": 37033, "\u0120Sites": 37034, "hate": 37035, "enburg": 37036, "\u0120sockets": 37037, "pend": 37038, "Batman": 37039, "vir": 37040, "\u0120SHOW": 37041, "\u0120provisional": 37042, "conn": 37043, "\u0120Deaths": 37044, "ATIVE": 37045, "Profile": 37046, "sym": 37047, "JA": 37048, "\u0120ninja": 37049, "installed": 37050, "idates": 37051, "ebra": 37052, "\u0120Omaha": 37053, "\u0120seizing": 37054, "\u0120Beasts": 37055, "\u0120salts": 37056, "Mission": 37057, "Generally": 37058, "\u0120Trilogy": 37059, "heon": 37060, "legates": 37061, "\u0120dime": 37062, "\u0120faire": 37063, "parable": 37064, "Graph": 37065, "\u0120totaling": 37066, "\u0120diagrams": 37067, "\u0120Yanuk": 37068, "plet": 37069, "\u0120Meh": 37070, "\u0120mythical": 37071, "\u0120Stephens": 37072, "autical": 37073, "ochemistry": 37074, "\u0120kilograms": 37075, "\u0120elbows": 37076, "ancock": 37077, "\u0120BCE": 37078, "\u0120Prague": 37079, "\u0120improv": 37080, "\u0120Devin": 37081, "\u0120\"\\": 37082, "paralle": 37083, "\u0120supremacists": 37084, "\u0120Billion": 37085, "\u0120regimen": 37086, "innacle": 37087, "\u0120requisite": 37088, "angan": 37089, "\u0120Burlington": 37090, "ainment": 37091, "\u0120Objective": 37092, "omsky": 37093, "GV": 37094, "\u0120unilateral": 37095, "\u0120tc": 37096, "\u0120hires": 37097, "mental": 37098, "\u0120involuntary": 37099, "\u0120transpl": 37100, "\u0120ASCII": 37101, "\u00c2\u00a8": 37102, "Events": 37103, "\u0120doubted": 37104, "\u0120Kaplan": 37105, "\u0120Courage": 37106, "igon": 37107, "\u0120Managing": 37108, "\u0120Tart": 37109, "\u0120falsehood": 37110, "\u0120Violet": 37111, "\u0120airs": 37112, "\u0120fertilizer": 37113, "Britain": 37114, "\u0120aquatic": 37115, "ouf": 37116, "Words": 37117, "\u0120Hartford": 37118, "\u0120evenings": 37119, "\u0120Vengeance": 37120, "quite": 37121, "Gall": 37122, "\u0120Pret": 37123, "\u0120pdf": 37124, "\u0120LM": 37125, "\u0120Sochi": 37126, "\u0120Intercept": 37127, "920": 37128, "\u0120profitability": 37129, "\u0120Idle": 37130, "\u0120MacDonald": 37131, "\u0120Establishment": 37132, "umsy": 37133, "\u0120gatherings": 37134, "\u0120Naj": 37135, "Charlie": 37136, "\u0120ascent": 37137, "\u0120Protector": 37138, "\u0120algebra": 37139, "\u0120bios": 37140, "forums": 37141, "ELS": 37142, "Introduced": 37143, "\u0120335": 37144, "\u0120astronomy": 37145, "Contribut": 37146, "\u0120Polic": 37147, "Platform": 37148, "\u0120containment": 37149, "wrap": 37150, "\u0120coronary": 37151, "\u0120Jelly": 37152, "manager": 37153, "\u0120heartbreaking": 37154, "cair": 37155, "\u0120Chero": 37156, "cgi": 37157, "Medical": 37158, "\u0120Accountability": 37159, "!!\"": 37160, "ophile": 37161, "\u0120psychotic": 37162, "\u0120Restrict": 37163, "\u0120equitable": 37164, "issues": 37165, "\u01201905": 37166, "\u0120Nek": 37167, "cised": 37168, "\u0120Tracking": 37169, "\u0120ozone": 37170, "\u0120cooker": 37171, "rosis": 37172, "\u0120reopen": 37173, "\u0120infinity": 37174, "\u0120Pharmaceutical": 37175, "ensional": 37176, "Attempt": 37177, "\u0120Rory": 37178, "Marco": 37179, "\u0120awaits": 37180, "HOW": 37181, "treated": 37182, "\u0120bolst": 37183, "\u0120revered": 37184, "\u0120pods": 37185, "oppers": 37186, "0010": 37187, "\u0120amplitude": 37188, "rican": 37189, "SPONSORED": 37190, "\u0120trousers": 37191, "\u0120halves": 37192, "\u0120Kaine": 37193, "\u0120Cutler": 37194, "\u0120AUTH": 37195, "\u0120splendid": 37196, "\u0120preventive": 37197, "\u0120Dudley": 37198, "ifacts": 37199, "uminati": 37200, "\u0120Yin": 37201, "\u0120admon": 37202, "\u0120Vag": 37203, "\u0120inverted": 37204, "\u0120hastily": 37205, "\u0120Hague": 37206, "Lyn": 37207, "\u0120ledger": 37208, "\u0120astronomical": 37209, "getting": 37210, "\u0120circa": 37211, "\u0120Cic": 37212, "\u0120Tennis": 37213, "Limited": 37214, "\u0120dru": 37215, "\u0120BYU": 37216, "\u0120travellers": 37217, "\u0120pane": 37218, "\u0120Intro": 37219, "\u0120patiently": 37220, "\u0120aiding": 37221, "\u0120loos": 37222, "\u0120Tough": 37223, "\u0120293": 37224, "\u0120consumes": 37225, "SourceFile": 37226, "\u0120\"\"\"": 37227, "\u0120bonding": 37228, "\u0120tilted": 37229, "\u0120menstrual": 37230, "\u0120Celestial": 37231, "ULAR": 37232, "Plugin": 37233, "\u0120risking": 37234, "Naz": 37235, "\u0120Riyadh": 37236, "\u0120accredited": 37237, "\u0120skirm": 37238, "\u00e9\u013d": 37239, "\u0120examiner": 37240, "\u0120messing": 37241, "\u0120nearing": 37242, "\u0120Chern": 37243, "\u0120Beckham": 37244, "\u0120swapped": 37245, "\u0120goose": 37246, "Kay": 37247, "\u0120lofty": 37248, "\u0120Wallet": 37249, "\u0120['": 37250, "\u0120apocalypse": 37251, "\u0120bamboo": 37252, "\u0120SPACE": 37253, "\u0120Elena": 37254, "\u0120306": 37255, "acons": 37256, "\u0120tightened": 37257, "\u0120adolescence": 37258, "\u0120rainy": 37259, "\u0120vandalism": 37260, "\u0120Newtown": 37261, "\u0120conject": 37262, "cakes": 37263, "\u0120cheated": 37264, "\u0120moderators": 37265, "params": 37266, "EFF": 37267, "\u0120deceit": 37268, "\u0120STL": 37269, "\u0120Tanzania": 37270, "\u0120RI": 37271, "\u01201923": 37272, "\u0120Exile": 37273, "thel": 37274, "\u0120theolog": 37275, "\u0120quirky": 37276, "\u0120Irvine": 37277, "\u0120needy": 37278, "oris": 37279, "Um": 37280, "Ka": 37281, "\u0120mailbox": 37282, "322": 37283, "\u0120bos": 37284, "\u0120Petra": 37285, "KING": 37286, "\u0120enlarged": 37287, "Often": 37288, "\u0120badass": 37289, "\u0120343": 37290, "\u0120Places": 37291, "\u0120CAD": 37292, "\u0120pristine": 37293, "\u0120intervening": 37294, "direction": 37295, "\u0120laz": 37296, "\u0120DSM": 37297, "\u0120projecting": 37298, "\u0120Funk": 37299, "agog": 37300, "payment": 37301, "nov": 37302, "\u0120chatter": 37303, "ARB": 37304, "\u0120examinations": 37305, "\u0120Household": 37306, "\u0120Gus": 37307, "Ford": 37308, "414": 37309, "Boss": 37310, "\u0120mystic": 37311, "\u0120leaps": 37312, "\u0120Bav": 37313, "ulz": 37314, "budget": 37315, "Football": 37316, "\u0120subsidized": 37317, "\u0120firsthand": 37318, "\u0120coincide": 37319, "ocular": 37320, "Conn": 37321, "\u0120Collabor": 37322, "\u0120fools": 37323, "amura": 37324, "ahar": 37325, "rists": 37326, "\u0120swollen": 37327, "\u0120expended": 37328, "\u0120Pau": 37329, "sup": 37330, "\u0120spar": 37331, "\u0120keynote": 37332, "suff": 37333, "\u0120unequal": 37334, "\u0120progressing": 37335, "strings": 37336, "\u0120Gamergate": 37337, "Disney": 37338, "\u0120Eleven": 37339, "omnia": 37340, "\u0120scripted": 37341, "\u0120earners": 37342, "brother": 37343, "\u0120Enabled": 37344, "\u00e6\u00b3": 37345, "\u0120larvae": 37346, "\u0120LOC": 37347, "mess": 37348, "Wilson": 37349, "\u0120Template": 37350, "successfully": 37351, "\u0120paramount": 37352, "\u0120camouflage": 37353, "\u0120binds": 37354, "\u0120Quiet": 37355, "\u0120Shutterstock": 37356, "rush": 37357, "\u0120mascot": 37358, "fortune": 37359, "\u0120Colt": 37360, "\u0120Beyon": 37361, "habi": 37362, "\u0120hairc": 37363, "\u0120267": 37364, "\u0120Deus": 37365, "\u0120twitch": 37366, "\u0120concentrating": 37367, "\u0120nipples": 37368, "cible": 37369, "\u0120gir": 37370, "NZ": 37371, "Math": 37372, "nih": 37373, "Required": 37374, "\u0120ponder": 37375, "\u0120SAN": 37376, "\u0120weddings": 37377, "\u0120loneliness": 37378, "NES": 37379, "\u0120Mahjong": 37380, "695": 37381, "addle": 37382, "\u0120Garner": 37383, "\u0120COUR": 37384, "Bridge": 37385, "\u0120spree": 37386, "\u0120Caldwell": 37387, "\u0120bribery": 37388, "\u0120\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 37389, "plugins": 37390, "\u0120racket": 37391, "\u0120champagne": 37392, "versible": 37393, "Vote": 37394, "\u0120modifiers": 37395, "Mayor": 37396, "680": 37397, "\u0120assemblies": 37398, "\u0120Sultan": 37399, "\u0120Ning": 37400, "\u0120Ladies": 37401, "\u0120sulfur": 37402, "\u0120orbs": 37403, "\u0120-----": 37404, "_______": 37405, "\u0120Journalism": 37406, "\u0120esports": 37407, "\u0120lush": 37408, "\u0120hue": 37409, "\u0120spectral": 37410, "Honest": 37411, "\u00e3\u0125\u0131": 37412, "\u0120bushes": 37413, "\u0120reinforcement": 37414, "\u0120reopened": 37415, "\u0120Wheels": 37416, "\u0120Morg": 37417, "rieving": 37418, "\u0120auxiliary": 37419, "\u0120jQuery": 37420, "\u0120BAT": 37421, "tesque": 37422, "\u0120vertex": 37423, "pure": 37424, "frey": 37425, "\u00e3\u0124\u00ba": 37426, "dos": 37427, "\u0120typh": 37428, "\u0120cull": 37429, "\u0120eq": 37430, "\u0120decon": 37431, "\u0120tossing": 37432, "\u0120disparate": 37433, "\u0120Brigham": 37434, "printf": 37435, "ledged": 37436, "\u0120sund": 37437, "\u0120cozy": 37438, "\u0120hepatitis": 37439, "performing": 37440, "\u0120aval": 37441, "\u0120GG": 37442, "future": 37443, "\u0120petertodd": 37444, "\u0120Kosovo": 37445, "\u0120magnets": 37446, "Already": 37447, "\u0120Edison": 37448, "\u0120Ceres": 37449, "\u0120RAID": 37450, "\u0120brilliance": 37451, "576": 37452, "\u0120derives": 37453, "\u0120hypertension": 37454, "\u0120\u00ce\u0136": 37455, "\u0120lambda": 37456, "\u0120flair": 37457, "\u0120missionaries": 37458, "\u0120rapes": 37459, "\u0120Starter": 37460, "\u0120Months": 37461, "\u0120defy": 37462, "\u0120seismic": 37463, "\u0120Raphael": 37464, "\u0120eurozone": 37465, "656": 37466, "zsche": 37467, "\u0120scratched": 37468, "\u0120bows": 37469, "\u0120Lennon": 37470, "\u0120Gaia": 37471, "\u0120dripping": 37472, "facts": 37473, "Ale": 37474, "\u0120frogs": 37475, "\u0120Breast": 37476, "ogeneity": 37477, "\u0120Prosecutor": 37478, "\u0120amplified": 37479, "\u0120Hodg": 37480, "\u0120Fn": 37481, "Thousands": 37482, "\u0120NIH": 37483, "\u0120Monitoring": 37484, "FTWARE": 37485, "\u0120Priebus": 37486, "\u0120Growing": 37487, "hunter": 37488, "\u0120diagnose": 37489, "\u0120Mald": 37490, "\u0120LR": 37491, "\u0120crowned": 37492, "\u0120bursting": 37493, "\u0120dissolution": 37494, "javascript": 37495, "\u0120usefulness": 37496, "\u0120Execution": 37497, ":(": 37498, "\u0120Ivory": 37499, "aah": 37500, "\u0120persecuted": 37501, "violence": 37502, "istas": 37503, "\u0120Crate": 37504, "\u0120impulses": 37505, "\u0120Spani": 37506, "edes": 37507, "Handle": 37508, "\u0120Zerg": 37509, "thinkable": 37510, "Lastly": 37511, "\u0120spontaneously": 37512, "\u0120inconvenient": 37513, "\u0120dismissing": 37514, "\u0120plotted": 37515, "\u0120eighty": 37516, "\u0120737": 37517, "rish": 37518, "\u0120Thornton": 37519, "atham": 37520, "\u0120sitcom": 37521, "Ven": 37522, "Recipe": 37523, "tel": 37524, "lund": 37525, "\u0120clears": 37526, "\u0120Sasuke": 37527, "\u0120258": 37528, "\u0120opting": 37529, "\u0120enraged": 37530, "esthetic": 37531, "\u0120Ae": 37532, "uchs": 37533, "Prep": 37534, "Flow": 37535, "\u0120runoff": 37536, "\u0120Eating": 37537, "\u0120Giles": 37538, "\u0120Acting": 37539, "resources": 37540, "ibaba": 37541, "\u0120rpm": 37542, "\u0120skewed": 37543, "\u0120Blanc": 37544, "\u0120Sakuya": 37545, "\u0120hotter": 37546, "\u01201924": 37547, "opian": 37548, "cko": 37549, "\u0120crumbling": 37550, "\u0120captains": 37551, "\u0120Appropriations": 37552, "leaders": 37553, "dropping": 37554, "anuts": 37555, "\u0120reversing": 37556, "\u0120Pose": 37557, "\u0120Sek": 37558, "Scot": 37559, "\u0120Idea": 37560, "cise": 37561, "\u0120Slovenia": 37562, "\u0120317": 37563, "Doctor": 37564, "\u0120crocod": 37565, "aldi": 37566, "Sea": 37567, "\u0120Farrell": 37568, "\u0120mercenaries": 37569, "\u0120RNC": 37570, "\u0120Guess": 37571, "\u0120pacing": 37572, "Machine": 37573, "StreamerBot": 37574, "\u0120Charity": 37575, "\u0120298": 37576, "\u0120cannons": 37577, "\u0120Toby": 37578, "TPPStreamerBot": 37579, "\u0120Passion": 37580, "cfg": 37581, "Thom": 37582, "\u0120badges": 37583, "\u0120Bernstein": 37584, ".\u00e2\u0122\u0135": 37585, "\u0120POP": 37586, "\u0120Conj": 37587, "\u0120initialization": 37588, "\u0120biodiversity": 37589, "Dub": 37590, "\u0120feudal": 37591, "\u0120disclaimer": 37592, "\u0120crow": 37593, "\u0120ignition": 37594, "arf": 37595, "SHA": 37596, "\u0120kHz": 37597, "hazard": 37598, "\u0120Artists": 37599, "oeuv": 37600, "679": 37601, "\u0120Rudy": 37602, "Nine": 37603, "\u0120Ramadan": 37604, "\u00e5\u00bd": 37605, "itto": 37606, "\u0120adrenaline": 37607, "Cert": 37608, "\u0120smelled": 37609, "\u0120impunity": 37610, "\u0120agendas": 37611, "\u0120Reborn": 37612, "\u0120Concent": 37613, "\u0120Seems": 37614, "\u0120omega": 37615, "\u0120Dustin": 37616, "\u0120backer": 37617, "\u0120Sauce": 37618, "\u0120Boyle": 37619, "WIN": 37620, "\u0120spins": 37621, "\u0120pauses": 37622, "upt": 37623, "\u0120shredded": 37624, "\u0120strapped": 37625, "\u0120Corruption": 37626, "\u0120scratches": 37627, "\u0120ni": 37628, "\u0120attire": 37629, "\u0120SAF": 37630, "FactoryReloaded": 37631, "\u0120IPS": 37632, "\u0120(%": 37633, "\u0120seminar": 37634, "focus": 37635, "civil": 37636, "\u01201860": 37637, "intosh": 37638, "\u0120continual": 37639, "\u0120abbrevi": 37640, "\u0120Sok": 37641, "ocobo": 37642, "XM": 37643, "\u0120frantic": 37644, "\u0120unavoidable": 37645, "\u0120artery": 37646, "\u0120annotations": 37647, "bath": 37648, "Climate": 37649, "\u0120dors": 37650, "\u0120Slide": 37651, "coord": 37652, "\u0120Reload": 37653, "\u0120LDL": 37654, "\u0120Lovecraft": 37655, "\u0120unimagin": 37656, "\u0120resembled": 37657, "\u0120barracks": 37658, "np": 37659, "\u0120surrogate": 37660, "\u0120categorized": 37661, "\u00e3\u0124\u00a9": 37662, "\u0120vaccinated": 37663, "\u0120drainage": 37664, "\u0120indist": 37665, "\u0120WhatsApp": 37666, "\u01201870": 37667, "olerance": 37668, "invoke": 37669, "amorph": 37670, "\u0120reconnect": 37671, "\u0120emanc": 37672, "\u0120blindness": 37673, "\u01201280": 37674, "internet": 37675, "collar": 37676, "\u0120altru": 37677, "\u0120abyss": 37678, "\u0120TRI": 37679, "657": 37680, "\u0120infused": 37681, "HEAD": 37682, "\u0120forestry": 37683, "\u0120Woody": 37684, "\u0120Ci": 37685, "wi": 37686, "sam": 37687, "784": 37688, "holiday": 37689, "\u0120mogul": 37690, "\u0120Fees": 37691, "\u0120DEN": 37692, "Internal": 37693, "urbed": 37694, "fusc": 37695, "atom": 37696, "\u0120Illusion": 37697, "\u0120polled": 37698, "\u0120flap": 37699, "\u0120coax": 37700, "LGBT": 37701, "Analy": 37702, "\u0120Sections": 37703, "\u0120Californ": 37704, "emn": 37705, "\u0120hither": 37706, "\u0120NIGHT": 37707, "\u0120nailed": 37708, "\u0120Pipeline": 37709, "391": 37710, "oof": 37711, "\u0120Primal": 37712, "verend": 37713, "\u0120slashing": 37714, "\u0120retri": 37715, "aviour": 37716, "\u0120departing": 37717, "gil": 37718, "ISC": 37719, "\u0120midway": 37720, "\u0120ultrasound": 37721, "\u0120behaving": 37722, "\u0120Tara": 37723, "classes": 37724, "Virtual": 37725, "\u0120Colonial": 37726, "\u0120stripping": 37727, "\u0120orchestrated": 37728, "\u0120Graves": 37729, "452": 37730, "\u0120Ironically": 37731, "\u0120Writers": 37732, "\u0120lends": 37733, "\u0120Manz": 37734, "\u0120raven": 37735, "\u0120oxidative": 37736, "\u0120266": 37737, "ELF": 37738, "actually": 37739, "ascar": 37740, "Draft": 37741, "\u0120favourable": 37742, "\u0120humiliating": 37743, "\u0120fidelity": 37744, "\u0120Hof": 37745, "\u0120Xuan": 37746, "496": 37747, "\u0120layered": 37748, "atis": 37749, "790": 37750, "\u0120paycheck": 37751, "iton": 37752, "Kar": 37753, "\u0120VMware": 37754, "\u0120Farmer": 37755, "\u0120servic": 37756, "glomer": 37757, "\u0120slump": 37758, "\u0120Fabric": 37759, "\u0120DOC": 37760, "esting": 37761, "\u0120reassure": 37762, "\u0120phyl": 37763, "volt": 37764, "itory": 37765, "Rules": 37766, "\u0120oxidation": 37767, "\u0120prized": 37768, "\u0120mistress": 37769, "\u0120Django": 37770, "WARN": 37771, "\u00e5\u0133": 37772, "\u0120encode": 37773, "\u0120Feedback": 37774, "\u0120stupidity": 37775, "Ian": 37776, "\u0120Yugoslavia": 37777, "\u00d7\u00a8": 37778, "acl": 37779, "UTE": 37780, "1977": 37781, "\u0120qualifies": 37782, "\u0120pulses": 37783, "pretty": 37784, "\u0120froze": 37785, "\u0120ss": 37786, "Iterator": 37787, "\u0120urgently": 37788, "\u0120mailed": 37789, "\u0120Cham": 37790, "\u0120sustaining": 37791, "\u0120basil": 37792, "\u0120puppies": 37793, "ilant": 37794, "\u0120PLEASE": 37795, "lap": 37796, "aceous": 37797, "Fear": 37798, "\u0120Mastery": 37799, "automatic": 37800, "\u0120TAG": 37801, "\u0120antim": 37802, "agles": 37803, "473": 37804, "frames": 37805, "\u0120whispers": 37806, "\u0120Whoever": 37807, "\u0120bravery": 37808, "\u0120UKIP": 37809, "ractions": 37810, "\"\"\"": 37811, "\u0120tame": 37812, "\u0120parted": 37813, "everything": 37814, "CONT": 37815, "\u0120indebted": 37816, "\u0120addr": 37817, "rek": 37818, "IRED": 37819, "\u0120eminent": 37820, "clinton": 37821, "\u0120ousted": 37822, "\u0120reviewer": 37823, "\u0120meltdown": 37824, "\u0120rearr": 37825, "\u0120Yao": 37826, "thereal": 37827, "abyte": 37828, "\u0120stumbling": 37829, "\u0120batches": 37830, "\u0120259": 37831, "\u0120contraceptive": 37832, "\u0120prostitute": 37833, "ensis": 37834, "Decl": 37835, "\u0120Strikes": 37836, "Military": 37837, "\u0120Oath": 37838, "vacc": 37839, "ppings": 37840, "052": 37841, "\u0120partName": 37842, "amping": 37843, "Reports": 37844, "KI": 37845, "CHR": 37846, "\u0120subtly": 37847, "swers": 37848, "Blake": 37849, "usual": 37850, "\u0120contestants": 37851, "\u0120cartridges": 37852, "\u0120GREAT": 37853, "\u0120blush": 37854, "\u0120\u00e2\u0122\u00ba": 37855, "472": 37856, "\u0120reasoned": 37857, "\u00e3\u0125\u00a4": 37858, "paralleled": 37859, "\u0120dyn": 37860, "agate": 37861, "\u0120nightly": 37862, "\u00e5\u0128": 37863, "556": 37864, "\u0120semantic": 37865, "\u0120Advoc": 37866, "\u0120!!": 37867, "\u0120disagrees": 37868, "\u0120BW": 37869, "Veh": 37870, "\u0120harming": 37871, "\u0120embraces": 37872, "\u0120strives": 37873, "\u0120inland": 37874, "\u0120Kard": 37875, "\u0120heats": 37876, "\u0120Ginny": 37877, "utan": 37878, "ernaut": 37879, "ylene": 37880, "\u0120Elev": 37881, "JD": 37882, "\u0120hars": 37883, "\u0120Starr": 37884, "\u0120skysc": 37885, "\u0120collaborators": 37886, "Usually": 37887, "\u0120revolutions": 37888, "\u0120STATS": 37889, "\u0120dismantle": 37890, "\u0120confidently": 37891, "\u0120kinetic": 37892, "Ali": 37893, "\u0120percentile": 37894, "\u0120extracting": 37895, "illian": 37896, "estead": 37897, "\u0120physicists": 37898, "\u0120Marshal": 37899, "\u0120fellowship": 37900, "\u0120dashed": 37901, "\u0120UR": 37902, "\u0120Sioux": 37903, "\u0120Compact": 37904, "amide": 37905, "Python": 37906, "\u0120Leigh": 37907, "\u0120Pharmac": 37908, "istrates": 37909, "herical": 37910, "\u0120fue": 37911, "\u0120Emin": 37912, "\u0120({": 37913, "\u0120Neighborhood": 37914, "\u0120disrupting": 37915, "\u0120Dup": 37916, "\u0120gland": 37917, "\u0120Sev": 37918, "\u0120Marian": 37919, "argon": 37920, "\u0120Dund": 37921, "\u0120": 46904, "\u0120Philips": 46905, "\u0120Kafka": 46906, "\u0120upheaval": 46907, "\u0120sentimental": 46908, "\u0120sax": 46909, "\u0120Akira": 46910, "serial": 46911, "Matrix": 46912, "\u0120electing": 46913, "\u0120commenter": 46914, "\u0120Nebula": 46915, "plets": 46916, "\u0120Nadu": 46917, "\u0120Adren": 46918, "\u0120enshr": 46919, "\u0120RAND": 46920, "financial": 46921, "\u0120Clyde": 46922, "utherford": 46923, "\u0120signage": 46924, "\u0120deline": 46925, "\u0120phosphate": 46926, "roversial": 46927, "fascist": 46928, "\u0120Vall": 46929, "\u0120Bethlehem": 46930, "\u0120fors": 46931, "\u0120english": 46932, "Solid": 46933, "Nature": 46934, "\u0120va": 46935, "\u0120Guests": 46936, "\u0120tantal": 46937, "\u0120autoimmune": 46938, ";;;;;;;;;;;;": 46939, "\u0120Totally": 46940, "\u0120Ov": 46941, "\u0120defences": 46942, "\u0120Coconut": 46943, "\u0120tranquil": 46944, "\u0120ploy": 46945, "\u0120flavours": 46946, "\u0120Flask": 46947, "\u00e3\u0124\u00a8\u00e3\u0125\u00ab": 46948, "\u0120Weston": 46949, "\u0120Volvo": 46950, "870": 46951, "\u0120microphones": 46952, "verbal": 46953, "RPG": 46954, "\u0120iii": 46955, ";}": 46956, "028": 46957, "\u0120headlined": 46958, "\u0120primed": 46959, "\u0120hoard": 46960, "\u0120Shad": 46961, "\u0120ENTER": 46962, "\u0120triangular": 46963, "\u0120capit": 46964, "lik": 46965, "\u0120Ancients": 46966, "\u0120lash": 46967, "\u0120convol": 46968, "\u0120colonel": 46969, "enemy": 46970, "Gra": 46971, "\u0120pubs": 46972, "utters": 46973, "\u0120assigns": 46974, "\u0120Penet": 46975, "\u0120Monstrous": 46976, "\u0120Bowen": 46977, "ilver": 46978, "Haunted": 46979, "\u0120Ding": 46980, "started": 46981, "plin": 46982, "\u0120contaminants": 46983, "\u0120DOE": 46984, "ffen": 46985, "\u0120Technician": 46986, "Ry": 46987, "\u0120robbers": 46988, "\u0120hotline": 46989, "\u0120Guardiola": 46990, "\u0120Kaufman": 46991, "rower": 46992, "\u0120Dresden": 46993, "\u0120Alpine": 46994, "Elf": 46995, "\u0120fmt": 46996, "\u0120Sard": 46997, "urses": 46998, "gpu": 46999, "Unix": 47000, "\u0120unequivocally": 47001, "\u0120Citizenship": 47002, "quad": 47003, "mire": 47004, "\u0120Sweeney": 47005, "Battery": 47006, "615": 47007, "\u0120pancakes": 47008, "\u0120oats": 47009, "Maps": 47010, "\u0120Contrast": 47011, "mbudsman": 47012, "\u0120EPS": 47013, "\u0120subcommittee": 47014, "\u0120sourcing": 47015, "\u0120sizing": 47016, "\u0120Buffer": 47017, "\u0120Mandatory": 47018, "\u0120moderates": 47019, "\u0120Patterns": 47020, "\u0120Chocobo": 47021, "\u0120Zan": 47022, "\u0120STATES": 47023, "\u0120Judging": 47024, "\u0120Inher": 47025, "*:": 47026, "\u0120bil": 47027, "\u0120Yen": 47028, "\u0120exhilar": 47029, "ollower": 47030, "zers": 47031, "\u0120snug": 47032, "maximum": 47033, "\u0120despicable": 47034, "\u0120PACK": 47035, "\u0120Annex": 47036, "\u0120sarcastic": 47037, "\u0120latex": 47038, "\u0120tamp": 47039, "\u0120Sao": 47040, "bah": 47041, "\u0120Reverend": 47042, "\u0120Chinatown": 47043, "\u0120AUT": 47044, "documented": 47045, "\u0120GABA": 47046, "\u0120Canaan": 47047, "\u0120\u00d9\u0127": 47048, "\u0120governs": 47049, "prev": 47050, "Esc": 47051, "\u0120Estimates": 47052, "OSP": 47053, "\u0120endeavour": 47054, "\u0120Closing": 47055, "ometime": 47056, "everyone": 47057, "\u0120worsen": 47058, "\u0120scanners": 47059, "\u0120deviations": 47060, "\u0120Robotics": 47061, "\u0120Compton": 47062, "\u0120sorcerer": 47063, "\u0120endogenous": 47064, "\u0120emulation": 47065, "\u0120Piercing": 47066, "\u0120Aph": 47067, "\u0120Socket": 47068, "\u0120bould": 47069, "\u0120OU": 47070, "\u0120Borderlands": 47071, "\u01201863": 47072, "Gordon": 47073, "\u0120WTO": 47074, "\u0120restricts": 47075, "\u0120mosaic": 47076, "\u0120melodies": 47077, "\u00e7\u0126": 47078, "Tar": 47079, "\u0120disson": 47080, "\u0120Provides": 47081, "\u0120......": 47082, "bek": 47083, "FIX": 47084, "\u0120broom": 47085, "anship": 47086, "Doctors": 47087, "\u0120nerds": 47088, "\u0120Regions": 47089, "naissance": 47090, "\u0120mete": 47091, "\u0120crept": 47092, "plings": 47093, "\u0120girlfriends": 47094, "knit": 47095, "igent": 47096, "owe": 47097, "\u0120ushered": 47098, "\u0120Baz": 47099, "Mobil": 47100, "434": 47101, "\u0120Presents": 47102, "origin": 47103, "\u0120insomnia": 47104, "\u0120Aux": 47105, "439": 47106, "\u0120Chili": 47107, "irsch": 47108, "GAME": 47109, "\u0120gestation": 47110, "algia": 47111, "romising": 47112, "$,": 47113, "crow": 47114, "\u0120Inspection": 47115, "atomic": 47116, "Relations": 47117, "JOHN": 47118, "roman": 47119, "\u0120Clockwork": 47120, "\u0120Bakr": 47121, "mone": 47122, "MET": 47123, "\u0120thirsty": 47124, "\u0120bc": 47125, "\u0120faculties": 47126, "Rum": 47127, "\u0120nuance": 47128, "\u0120Darius": 47129, "pleting": 47130, "fters": 47131, "etchup": 47132, "Registration": 47133, "\u0120KE": 47134, "Rah": 47135, "\u0120preferential": 47136, "\u0120Lash": 47137, "\u0120HH": 47138, "Valid": 47139, "\u0120NAV": 47140, "\u0120starve": 47141, "\u0120Gong": 47142, "zynski": 47143, "\u0120Actress": 47144, "\u0120wik": 47145, "\u0120unaccompanied": 47146, "lvl": 47147, "Bride": 47148, "ADS": 47149, "\u0120Commando": 47150, "\u0120Vaughn": 47151, "Wallet": 47152, "\u0120hopping": 47153, "\u0120Vie": 47154, "\u0120caveats": 47155, "\u0120alas": 47156, "ifled": 47157, "abuse": 47158, "661": 47159, "\u0120ibn": 47160, "\u0120gul": 47161, "\u0120robbing": 47162, "til": 47163, "ILA": 47164, "\u0120mitigating": 47165, "\u0120aptly": 47166, "\u0120tyrant": 47167, "\u0120midday": 47168, "\u0120Gilmore": 47169, "\u0120Decker": 47170, "\u0120\u00c2\u00a7\u00c2\u00a7": 47171, "partial": 47172, "Exactly": 47173, "\u0120phenotype": 47174, "\u0120[+]": 47175, "\u0120Plex": 47176, "\u0120Ips": 47177, "versions": 47178, "\u0120ebook": 47179, "\u0120chic": 47180, "gross": 47181, "\":\"\"},{\"": 47182, "\u0120Surprisingly": 47183, "Morgan": 47184, "\u0120residues": 47185, "\u0120Confederation": 47186, "infeld": 47187, "\u0120lyr": 47188, "moderate": 47189, "\u0120perpendicular": 47190, "VK": 47191, "\u0120synchronized": 47192, "\u0120refreshed": 47193, "\u0120adore": 47194, "\u0120Torment": 47195, "olina": 47196, "\u01202600": 47197, "ItemTracker": 47198, "\u0120pies": 47199, "\u0120FAT": 47200, "\u0120RHP": 47201, "048": 47202, "\u0120RESP": 47203, "\u0120BJ": 47204, "allows": 47205, "Pand": 47206, "\u0120unwelcome": 47207, "\u0120Voc": 47208, "\u0120Bastard": 47209, "\u0120OW": 47210, "\u0120LAR": 47211, "\u0120Healer": 47212, "Environmental": 47213, "\u0120Kenyan": 47214, "\u0120Trance": 47215, "\u0120Pats": 47216, "\u0120aliases": 47217, "\u0120Garfield": 47218, "\u0120campaigner": 47219, "\u0120advancements": 47220, "\u0120Okinawa": 47221, "\u0120Coh": 47222, "owsky": 47223, "\u0120starved": 47224, "\u0120sizeable": 47225, "\u0120:-)": 47226, "\u0120mRNA": 47227, "\u0120suspensions": 47228, "istar": 47229, "Scotland": 47230, "Prin": 47231, "------------------------------------------------": 47232, "\u0120502": 47233, "\u0120teaspoons": 47234, "\u01201050": 47235, "\u0120coercive": 47236, "\u0120Masonic": 47237, "edded": 47238, "\u0120Passenger": 47239, "\u0120latt": 47240, "\u0120braces": 47241, "\u0120Steal": 47242, "\u0120NYT": 47243, "\u0120Kats": 47244, "\u0120Celest": 47245, "aez": 47246, "Tu": 47247, "\u0120Coulter": 47248, "\u00f0\u0141\u013a": 47249, "Flickr": 47250, "\u0120Wilmington": 47251, "iths": 47252, "++;": 47253, "\u0120vending": 47254, "\u0120negro": 47255, "\u0120Phi": 47256, "\u0120Yellowstone": 47257, "Callback": 47258, "\u0120shampoo": 47259, "\u0120Shades": 47260, "wat": 47261, "\u0120superhuman": 47262, "\u0120ridiculed": 47263, "\u0120holiest": 47264, "ombo": 47265, "\u0120interns": 47266, "\u0120hone": 47267, "\u0120Paragu": 47268, "URI": 47269, "\u0120dangling": 47270, "\u00e3\u0124\u00bb": 47271, "sov": 47272, "ictional": 47273, "availability": 47274, "\u0120revocation": 47275, "\u0120dow": 47276, "inic": 47277, "\u0120THEIR": 47278, "\u0120iso": 47279, "\u0120outings": 47280, "\u0120Lethal": 47281, "\u0120)))": 47282, "\u0120inaccur": 47283, "\u0120outlandish": 47284, "\u0120anus": 47285, "letico": 47286, "idon": 47287, "lol": 47288, "\u0120unregulated": 47289, "\u0120succumbed": 47290, "\u0120cuff": 47291, "\u0120Wasteland": 47292, "letal": 47293, "\u0120substr": 47294, "\u0120coffers": 47295, "\u0120automakers": 47296, "ovi": 47297, "\u0120Xue": 47298, "\u0120Daytona": 47299, "\u0120jarring": 47300, "\u0120fumes": 47301, "\u0120disbanded": 47302, "zik": 47303, "itton": 47304, "\u0120strikingly": 47305, "\u0120spores": 47306, "Adapter": 47307, ".):": 47308, "\u0120Lyndon": 47309, "ivalry": 47310, "\u0120orally": 47311, "\u0120tumultuous": 47312, "\u0120displeasure": 47313, "\u0120cones": 47314, "orrect": 47315, "\u0120appease": 47316, "\u0120derby": 47317, "\u0120Tripoli": 47318, "\u0120Aless": 47319, "\u0120poked": 47320, "\u0120Guilty": 47321, "vP": 47322, "Enough": 47323, "\u0120originals": 47324, "699": 47325, "\u0120rabbi": 47326, "\u0120proverbial": 47327, "\u0120postpone": 47328, "elope": 47329, "\u0120Misty": 47330, "\u0120staffed": 47331, "\u0120Unemployment": 47332, "reditary": 47333, "\u0120diligent": 47334, "recomm": 47335, "measures": 47336, "asin": 47337, "825": 47338, "\u0120ponds": 47339, "\u0120mmol": 47340, "\u0120SAR": 47341, "\u0120CARE": 47342, "\u0120371": 47343, "\u0120clenched": 47344, "\u0120Corsair": 47345, "\u0120caricature": 47346, "zn": 47347, "attach": 47348, "\u0120Schro": 47349, "speak": 47350, "painted": 47351, "\u0120Suc": 47352, "\u0120ENT": 47353, "\u0120cellul": 47354, "\u0120Paid": 47355, "diagn": 47356, "WHERE": 47357, "\u0120texted": 47358, "Barn": 47359, "\u0120retracted": 47360, "\u0120Referred": 47361, "Sav": 47362, "\u0120upkeep": 47363, "\u0120workplaces": 47364, "\u0120Tokens": 47365, "\u0120amplify": 47366, "clinical": 47367, "\u0120multic": 47368, "mberg": 47369, "\u0120convoluted": 47370, "Region": 47371, "565": 47372, "\u0120Topic": 47373, "\u0120snail": 47374, "\u0120saline": 47375, "\u0120insurrection": 47376, "\u0120Petr": 47377, "forts": 47378, "BAT": 47379, "\u0120Navajo": 47380, "\u0120rudimentary": 47381, "\u0120Laksh": 47382, "ONDON": 47383, "Measure": 47384, "\u0120transformer": 47385, "\u0120Goddard": 47386, "\u0120coincides": 47387, "irin": 47388, "Rex": 47389, "\u0120Bok": 47390, "quit": 47391, "\u0120shotguns": 47392, "\u0120proletarian": 47393, "\u0120scorp": 47394, "\u0120Ada": 47395, "514": 47396, "\u0120slander": 47397, "recorded": 47398, "\u0120embell": 47399, "risome": 47400, "\u0120apologizing": 47401, "\u0120Mulcair": 47402, "\u0120Gibraltar": 47403, "Cla": 47404, "\u0120allot": 47405, "\u0120Attention": 47406, "\u0120433": 47407, "leave": 47408, "\u0120whine": 47409, "\u0120Issa": 47410, "\u0120Faust": 47411, "\u0120Barron": 47412, "heny": 47413, "\u0120victimized": 47414, "Jews": 47415, "\u0120nurturing": 47416, "ettel": 47417, "Winged": 47418, "\u0120Subtle": 47419, "\u0120flavorful": 47420, "\u0120Reps": 47421, "enged": 47422, "callback": 47423, "\u0120directional": 47424, "\u0120clasp": 47425, "\u0120Directions": 47426, "planet": 47427, "iculture": 47428, "Helper": 47429, "icion": 47430, "acia": 47431, "\u0120\u00e7\u00a5\u0140": 47432, "\u0120surges": 47433, "\u0120canoe": 47434, "\u0120Premiership": 47435, "been": 47436, "\u0120defied": 47437, "\u0120Trooper": 47438, "\u0120tripod": 47439, "\u0120gasp": 47440, "\u0120Euph": 47441, "\u0120Ads": 47442, "vernight": 47443, "highly": 47444, "Role": 47445, "\u0120entangled": 47446, "\u0120Zeit": 47447, "618": 47448, "\u0120Rusty": 47449, "\u0120havens": 47450, "\u0120Vaughan": 47451, "HAEL": 47452, "\u0120SERVICE": 47453, "/,": 47454, "\u0120stricken": 47455, "\u0120delusions": 47456, "\u0120bis": 47457, "\u0120Haf": 47458, "\u0120gratification": 47459, "\u0120enticing": 47460, "UNCH": 47461, "Adams": 47462, "\u0120OLED": 47463, "\u0120Beetle": 47464, "\u01201899": 47465, "\u0120SOFTWARE": 47466, "ategor": 47467, "VL": 47468, "\u0120Totem": 47469, "\u0120Gators": 47470, "ATURES": 47471, "\u0120impedance": 47472, "Registered": 47473, "\u0120Cary": 47474, "\u0120Aerial": 47475, "onne": 47476, "enium": 47477, "\u0120dred": 47478, "\u0120Beg": 47479, "\u0120concurrently": 47480, "\u0120superpower": 47481, "\u0120Xan": 47482, "jew": 47483, "imester": 47484, "\u0120Dickinson": 47485, "\u00e2\u0136\u0123": 47486, "Fla": 47487, "\u0120pree": 47488, "\u0120Rollins": 47489, "\u00a9\u00b6\u00e6": 47490, "\u0120denomination": 47491, "\u0120Lana": 47492, "516": 47493, "\u0120inciting": 47494, "scribed": 47495, "juries": 47496, "\u0120Wonders": 47497, "approximately": 47498, "\u0120suspending": 47499, "\u0120mountainous": 47500, "\u0120Laugh": 47501, "oidal": 47502, "Ns": 47503, "Detect": 47504, ")=": 47505, "\u0120Luthor": 47506, "\u0120Schwarzenegger": 47507, "\u0120Muller": 47508, "\u0120Devi": 47509, "ecycle": 47510, "Jar": 47511, "613": 47512, "\u0120Longh": 47513, "Bah": 47514, "\u0120SPORTS": 47515, "nw": 47516, "\u0120refinement": 47517, "\u0120waterways": 47518, "\u0120diner": 47519, "Blade": 47520, "683": 47521, "Fac": 47522, "\u0120initials": 47523, "\u0120rog": 47524, "\u0120paranormal": 47525, "BUT": 47526, "\u0120[(": 47527, "\u0120Swanson": 47528, "\u0120Mesh": 47529, "\u00e2\u0138\u00ac": 47530, "Improve": 47531, "\u0120Radiation": 47532, "\u0120Esther": 47533, "\u0120Esk": 47534, "\u0120Aly": 47535, "iky": 47536, "\u0120irrad": 47537, "\u0120Buckingham": 47538, "\u0120refill": 47539, "\u0120._": 47540, "Repe": 47541, "CONCLUS": 47542, "\u0120differentiated": 47543, "\u0120chirop": 47544, "\u0120Atkins": 47545, "Pattern": 47546, "\u0120excise": 47547, "\u0120cabal": 47548, "NSA": 47549, "\u0120STA": 47550, "\u0120SIL": 47551, "\u0120Paraly": 47552, "\u0120rye": 47553, "\u0120Howell": 47554, "\u0120Countdown": 47555, "nesses": 47556, "alysed": 47557, "\u0120resize": 47558, "\u00e3\u0124\u00bd": 47559, "\u0120budgetary": 47560, "\u0120Stras": 47561, "wang": 47562, "\u0120apiece": 47563, "\u0120precincts": 47564, "\u0120peach": 47565, "\u0120skyline": 47566, "\u0120353": 47567, "popular": 47568, "Appearances": 47569, "\u0120Mechanics": 47570, "\u0120DevOnline": 47571, "Sullivan": 47572, "Zen": 47573, "\u0120pu": 47574, "opolis": 47575, "544": 47576, "\u0120deform": 47577, "\u0120counteract": 47578, "\u0120Lange": 47579, "\u0120417": 47580, "Console": 47581, "774": 47582, "\u0120nodding": 47583, "\u0120populism": 47584, "\u0120hep": 47585, "\u0120counselling": 47586, "compliance": 47587, "UFF": 47588, "\u0120undeniably": 47589, "\u0120railing": 47590, "\u0120Horowitz": 47591, "\u0120Simone": 47592, "\u0120Bungie": 47593, "\u0120ak": 47594, "\u0120Talks": 47595, "xff": 47596, "flake": 47597, "Crash": 47598, "\u0120sweaty": 47599, "\u0120banquet": 47600, "\u0120OFFIC": 47601, "\u0120inventive": 47602, "\u0120astronomer": 47603, "\u0120Stamford": 47604, "\u0120Scare": 47605, "\u0120GREEN": 47606, "olicited": 47607, "\u0120rusher": 47608, "\u0120centrist": 47609, "ighting": 47610, "\u0120subclass": 47611, "\u0120disav": 47612, "\u0120defund": 47613, "\u0120Nanto": 47614, "ociate": 47615, "mast": 47616, "\u0120pacif": 47617, "\u0120mend": 47618, "eers": 47619, "immigration": 47620, "ESSION": 47621, "\u0120numbering": 47622, "\u0120laughable": 47623, "\u0120Ended": 47624, "viation": 47625, "emark": 47626, "Pitt": 47627, "\u0120meticulous": 47628, "\u0120LF": 47629, "\u0120congratulated": 47630, "\u0120Birch": 47631, "\u0120swayed": 47632, "\u0120semifinals": 47633, "\u0120humankind": 47634, "matter": 47635, "\u0120Equip": 47636, "opausal": 47637, "Said": 47638, "\u0120Layout": 47639, "\u0120voicing": 47640, "\u0120thug": 47641, "\u0120pornographic": 47642, "IPS": 47643, "\u0120moaning": 47644, "\u0120grievance": 47645, "\u0120confessions": 47646, "escal": 47647, "TEXTURE": 47648, "Authent": 47649, "osaurus": 47650, "Purchase": 47651, "\u0120relegation": 47652, "alter": 47653, "\u0120\u00c2\u0142\u00c2\u0142": 47654, "\u0120riddled": 47655, "\u0120ogre": 47656, "\u0120Lowell": 47657, "Occup": 47658, "Eat": 47659, "\u0120Hyder": 47660, "\u0120Adviser": 47661, "Commerce": 47662, "Hunt": 47663, "\u0120Orth": 47664, "\u0120Competitive": 47665, "\u0120CLA": 47666, "CDC": 47667, "\u0120salads": 47668, "Fle": 47669, "\u0120industrialized": 47670, "`,": 47671, "\u0120OWN": 47672, "\u0120beck": 47673, "\u0120Particularly": 47674, "oubt": 47675, "\u0120mM": 47676, "\u0120Hussain": 47677, "\u0120Chennai": 47678, "\u0120920": 47679, "\u0120appointing": 47680, "\u0120Cullen": 47681, ",,,,,,,,": 47682, "\u0120pores": 47683, "verified": 47684, "\u0120biochemical": 47685, "emate": 47686, "\u0120cowardly": 47687, "\u0120Helsinki": 47688, "\u0120Ethiopian": 47689, "SOURCE": 47690, "ERC": 47691, "estro": 47692, "\u0120biotech": 47693, "\u0120Sour": 47694, "\u0120brewer": 47695, "Bloomberg": 47696, "\u0120intensify": 47697, "Glass": 47698, "anco": 47699, "\u0120FDR": 47700, "greSQL": 47701, "\u0120Fires": 47702, "\u00a9\u00b6\u00e6\u00a5\u00b5": 47703, "eco": 47704, "1001": 47705, "\u0120Homeless": 47706, "\u0120instantaneous": 47707, "\u0120Haste": 47708, "igel": 47709, "Diamond": 47710, "\u0120paving": 47711, "\u0120landfill": 47712, "\u0120dads": 47713, "houn": 47714, ":]": 47715, "\u0120incendiary": 47716, "\u0120Livingston": 47717, "\u0120Hilbert": 47718, "\u0120Checks": 47719, "styles": 47720, "inators": 47721, "\u0120Clive": 47722, "phrine": 47723, "\u0120chimpanzees": 47724, "\u0120pall": 47725, "\u0120JM": 47726, "\u0120Aadhaar": 47727, "\u00f0\u013f": 47728, "\u0120achievable": 47729, "disabled": 47730, "PET": 47731, "OOOOOOOO": 47732, "Mot": 47733, "\u0120intangible": 47734, "\u0120ballet": 47735, "\u0120Webs": 47736, "\u0120Estimated": 47737, "Effects": 47738, "\u0120bailed": 47739, "Joshua": 47740, "\u0120turbulence": 47741, "\u0120occupant": 47742, "\u0120Daylight": 47743, "\u0120361": 47744, "meet": 47745, "\u0120statically": 47746, "\u0120onlook": 47747, "\u0120ki": 47748, "illegal": 47749, "\u0120velvet": 47750, "\u0120dehydration": 47751, "\u0120acquies": 47752, "\u0120Rez": 47753, "akura": 47754, "\u0120Upton": 47755, "atro": 47756, "\u0120incomprehensible": 47757, "\u0120backdoor": 47758, "\u0120Rhino": 47759, "727": 47760, "\u0120maths": 47761, ")+": 47762, "\u0120heresy": 47763, "\u0120df": 47764, "\u0120Roche": 47765, "\u0120Lydia": 47766, "\u0120pancreat": 47767, "reply": 47768, "arrell": 47769, "\u0120solicitation": 47770, "\u0120circadian": 47771, "BIP": 47772, "\u0120foray": 47773, "\u0120cryptic": 47774, "izu": 47775, "imeo": 47776, "\u0120Tomato": 47777, "\u0120Homs": 47778, "examination": 47779, "\u0120quarry": 47780, "\u0120Valiant": 47781, "\u0120Jericho": 47782, "\u0120INCLUD": 47783, "\u01201840": 47784, "519": 47785, "\u0120resists": 47786, "\u0120snapshots": 47787, "\u0120Spur": 47788, "\u0120Antiqu": 47789, "Login": 47790, "\u0120bestselling": 47791, "\u0120antic": 47792, "\u0120Sutherland": 47793, "\u00e3\u0124\u00a2\u00e3\u0125\u00ab": 47794, "\u0120~/": 47795, "\u0120Parm": 47796, "\u00e8\u0125": 47797, "Pages": 47798, "intensity": 47799, "\u0120immobil": 47800, "\u01201865": 47801, "zzo": 47802, "\u0120nifty": 47803, "\u0120fentanyl": 47804, "\u0120Preservation": 47805, "ophen": 47806, "\u0120darts": 47807, "\u0120Dinosaur": 47808, "pointers": 47809, "\u0120Rite": 47810, "suggest": 47811, "awareness": 47812, "\u0120Sheridan": 47813, "\u0120stances": 47814, "\u0120sorcery": 47815, "\u0120perjury": 47816, "\u0120Nikola": 47817, "iever": 47818, "\u0120fiance": 47819, "\u0120Jordanian": 47820, "\u0120Balloon": 47821, "\u0120nab": 47822, "\u0120kb": 47823, "\u0120humanities": 47824, "\u0120Tanaka": 47825, "hillary": 47826, "\u0120consultancy": 47827, "\u0120Zub": 47828, "\u0120remission": 47829, "\u0120confid": 47830, "CHQ": 47831, "\u0120Fug": 47832, "\u0120improvis": 47833, "Yep": 47834, "/_": 47835, "\u0120unwillingness": 47836, "\u0120portfolios": 47837, "055": 47838, "\u0120Instructor": 47839, "aiman": 47840, "\u0120claimants": 47841, "Mbps": 47842, "\u0120Bye": 47843, "received": 47844, "Tweet": 47845, "\u0120indemn": 47846, "riz": 47847, "amara": 47848, "Nat": 47849, "\u0120evaluates": 47850, "\u0120Lur": 47851, "epad": 47852, "FOX": 47853, "\u0120Thro": 47854, "\u0120rusty": 47855, "\u0120bedrock": 47856, "\u0120Oprah": 47857, "JB": 47858, "\u0120manipulative": 47859, "\u0120willful": 47860, "\u0120relapse": 47861, "\u0120extant": 47862, "Theme": 47863, "Sensor": 47864, "\u0120Stability": 47865, "govern": 47866, "\u0120poppy": 47867, "\u0120knack": 47868, "\u0120insulated": 47869, "\u0120Tile": 47870, "\u0120Extrem": 47871, "\u0120untold": 47872, "\u0120converge": 47873, "\u0120refuel": 47874, "igroup": 47875, "\u0120distortions": 47876, "\u0120ravaged": 47877, "\u0120mechanically": 47878, "\u0120Reilly": 47879, "\u0120Nose": 47880, "\u0120Incarnation": 47881, "\u0120Becky": 47882, "abbling": 47883, "\u0120taco": 47884, "\u0120rake": 47885, "\u0120melancholy": 47886, "\u0120illustrious": 47887, "\u0120Dartmouth": 47888, "Guide": 47889, "\u0120Razer": 47890, "\u0120Benz": 47891, "Ultimate": 47892, "\u0120Surprise": 47893, "\u0120pageant": 47894, "offer": 47895, "Whoever": 47896, "\u0120wiser": 47897, "\u0120chemist": 47898, "\u0120HELL": 47899, "\u0120Bulk": 47900, "\u0120plutonium": 47901, "\u0120COVER": 47902, "\u00d6\u00bc": 47903, "failed": 47904, "\u0120tirelessly": 47905, "\u0120infertility": 47906, "\u0120Trident": 47907, "\u0120Showtime": 47908, "\u0120Civ": 47909, "Vice": 47910, "requires": 47911, "ittance": 47912, "\u0120uncontrolled": 47913, "interesting": 47914, "561": 47915, "\u0120innovate": 47916, "ategic": 47917, "Lie": 47918, "\u0120Selling": 47919, "Ul": 47920, "\u0120savior": 47921, "\u0120Tosh": 47922, "\u0120swast": 47923, "PASS": 47924, "\u0120rink": 47925, "\u0120cardio": 47926, "\u0120Iro": 47927, "udi": 47928, "\u0120vantage": 47929, "\u0120vans": 47930, "\u0120Ni\u00c3\u00b1o": 47931, "+=": 47932, "\u0120propagate": 47933, "": 49029, "\u0120leukemia": 49030, "\u0120eluc": 49031, "\u0120announcer": 49032, "\u0120Lithuan": 49033, "\u0120Armageddon": 49034, "\u00e5\u0129": 49035, "Lenin": 49036, "\u0120Ruk": 49037, "\u0120pepp": 49038, "\u0120Romantic": 49039, "\u0120PIT": 49040, "\u0120Interstellar": 49041, "\u0120Atkinson": 49042, "Raid": 49043, "Js": 49044, "Goal": 49045, "Course": 49046, "\u0120vanishing": 49047, "esley": 49048, "\u0120Rounds": 49049, "Elsa": 49050, "593": 49051, "\u0120redundancy": 49052, "\u0120STAND": 49053, "\u0120prophetic": 49054, "\u0120habitable": 49055, "ryu": 49056, "\u0120faintly": 49057, "MODE": 49058, "\u0120flanked": 49059, "IRC": 49060, "Awesome": 49061, "\u0120spurious": 49062, "\u0120Zah": 49063, "\u0120MSG": 49064, "\u0120shading": 49065, "\u0120motivational": 49066, "\u0120Santana": 49067, "\u0120SPR": 49068, "\u0120excruciating": 49069, "omial": 49070, "\u0120Miko": 49071, "\u0120Leopard": 49072, "Abyss": 49073, "\u0120[|": 49074, "dirty": 49075, "\u0120baths": 49076, "\u0120demoral": 49077, "andre": 49078, "PB": 49079, "\u0120unification": 49080, "\u0120sacrament": 49081, "\u0120[&": 49082, "\u0120priceless": 49083, "\u0120gelatin": 49084, "\u0120emanating": 49085, "\u0120Allaah": 49086, "986": 49087, "\u0120outburst": 49088, "\u0120eras": 49089, "\u0120XVI": 49090, "\u0120SPI": 49091, "Ott": 49092, "\u0120Lazarus": 49093, "PLIED": 49094, "Flying": 49095, "blogs": 49096, "Wisconsin": 49097, "Raven": 49098, "\u0120rebate": 49099, "\u0120creeps": 49100, "\u0120Span": 49101, "\u0120Painter": 49102, "\u0120Kira": 49103, "\u0120Amos": 49104, "\u0120Corvette": 49105, "Consumer": 49106, "\u0120Recover": 49107, "cki": 49108, "\u0120pesky": 49109, "\u0120Invention": 49110, "Companies": 49111, "\u0120challengers": 49112, "ademic": 49113, "\u0120Ukrainians": 49114, "\u0120Neurolog": 49115, "\u0120Forsaken": 49116, "\u0120entrants": 49117, "\u0120embattled": 49118, "\u0120defunct": 49119, "\u0120Glacier": 49120, "\u0120poisons": 49121, "\u0120Horses": 49122, "makes": 49123, "\u0120Dirt": 49124, "\u0120423": 49125, "hhh": 49126, "\u0120Transformation": 49127, "QUIRE": 49128, "..................": 49129, "\u0120traveller": 49130, "\u0120Sexy": 49131, "\u0120Kern": 49132, "ipolar": 49133, "\u0120ransomware": 49134, "oooooooooooooooo": 49135, "Ec": 49136, "ruby": 49137, "Professional": 49138, "\u0120Outbreak": 49139, "argument": 49140, "Grey": 49141, "\u0120Fifa": 49142, "\u0120CHO": 49143, "\u0120FORM": 49144, "\u0120Amtrak": 49145, "-[": 49146, "\u0120cradle": 49147, "\u0120antioxidants": 49148, "\u00e3\u0123\u00ae\u00e5\u00ae": 49149, "736": 49150, "\u0120NASL": 49151, "\u0120Contributions": 49152, "Indiana": 49153, "\u0120STEP": 49154, "CSS": 49155, "\u0120salient": 49156, "\u0120allocations": 49157, "yrights": 49158, "\u0120mashed": 49159, "\u0120Cutter": 49160, "Sexual": 49161, "\u0120pounded": 49162, "\u0120fanbase": 49163, "\u0120casc": 49164, "\u0120Transparency": 49165, "\u0120analytic": 49166, "\u0120Summoner": 49167, "\u00d7\u0140": 49168, "\u0120ADC": 49169, "detail": 49170, "\u0120vanquished": 49171, "\u0120crabs": 49172, "arie": 49173, "Destroy": 49174, "\u0120Sack": 49175, "\u0120transistor": 49176, "Alabama": 49177, "\u0120Koen": 49178, "\u0120Fisheries": 49179, "cone": 49180, "\u0120annexed": 49181, "\u0120MGM": 49182, "esa": 49183, "\u0120faked": 49184, "\u0120Congratulations": 49185, "\u0120hindered": 49186, "\u0120correctional": 49187, "\u0120ITV": 49188, "leeve": 49189, "\u0120inappropriately": 49190, "licks": 49191, "\u0120trespass": 49192, "\u0120paws": 49193, "\u0120negotiator": 49194, "\u0120Christensen": 49195, "limits": 49196, "\u0120Dianne": 49197, "\u0120elegance": 49198, "\u0120Contracts": 49199, "anke": 49200, "Obj": 49201, "\u0120vigilance": 49202, "\u0120castles": 49203, "\u0120NAD": 49204, "\u0120Holo": 49205, "\u0120emphatically": 49206, "\u0120Titus": 49207, "\u0120Serving": 49208, "\u0120Richie": 49209, "\u0120Pigs": 49210, "568": 49211, "\u0120animosity": 49212, "\u0120Attributes": 49213, "\u0120Uriel": 49214, "MQ": 49215, "myra": 49216, "\u0120Applicant": 49217, "\u0120psychiatrists": 49218, "\u0120Vij": 49219, "\u0120Abby": 49220, "agree": 49221, "Push": 49222, "\u0120kWh": 49223, "hiba": 49224, "\u0120incite": 49225, "\u0120Weasley": 49226, "\u0120Taxi": 49227, "ministic": 49228, "hyper": 49229, "\u0120Farn": 49230, "\u0120601": 49231, "\u0120Nationwide": 49232, "Fake": 49233, "952": 49234, "\u0120maize": 49235, "\u0120interacted": 49236, "\u0120transitioned": 49237, "\u0120parasitic": 49238, "\u0120harmonic": 49239, "\u0120decaying": 49240, "\u0120baseless": 49241, "nsics": 49242, "\u0120transpired": 49243, "\u0120abundantly": 49244, "\u0120Forensic": 49245, "\u0120treadmill": 49246, "\u0120Jav": 49247, "aband": 49248, "\u0120sshd": 49249, "\u0120frontman": 49250, "\u0120Jakarta": 49251, "oller": 49252, "drops": 49253, "\u0120SERVICES": 49254, "romptu": 49255, "ophical": 49256, "hospital": 49257, "bledon": 49258, "645": 49259, "\u0120midrange": 49260, "\u0120EVENT": 49261, "culated": 49262, "rawled": 49263, "\u0120perched": 49264, "\u0120overboard": 49265, "\u0120Peel": 49266, "\u0120Pwr": 49267, "\u0120Carth": 49268, "\u0120COMPLE": 49269, "coe": 49270, "shall": 49271, "\u0120deterrence": 49272, "METHOD": 49273, "\u0120Absent": 49274, "MEN": 49275, "\u0120sill": 49276, "\u0120LEVEL": 49277, "York": 49278, "\u0120sinners": 49279, "\u0120OPEC": 49280, "\u0120Nur": 49281, "\u0120Designs": 49282, "selection": 49283, "\u0120unworthy": 49284, "CHA": 49285, "\u0120strengthens": 49286, "883": 49287, "edly": 49288, "\u0120slicing": 49289, "\u0120malnutrition": 49290, "\u0120filmmaking": 49291, "\u0120Polk": 49292, "urated": 49293, "\u0120421": 49294, "breakers": 49295, "!'\"": 49296, "\u0120wetlands": 49297, "\u0120Discrimination": 49298, "\u0120allowable": 49299, "\u0120steered": 49300, "\u0120Sicily": 49301, "SAM": 49302, "\u0120mustache": 49303, "\u0120mids": 49304, "\u0120clipped": 49305, "\u0120circulate": 49306, "\u0120brittle": 49307, "\u0120Buildings": 49308, "raised": 49309, "\u0120Roundup": 49310, "\u0120wealthier": 49311, "\u0120overwrite": 49312, "\u0120overpowered": 49313, "\u0120Gerrard": 49314, "sites": 49315, "PDATED": 49316, "\u0120acutely": 49317, "\u0120Gamble": 49318, "\u0120pim": 49319, "\u0120Kus": 49320, "Typically": 49321, "Deploy": 49322, "\u0120Moroccan": 49323, "potion": 49324, "combe": 49325, "\u0120vigilante": 49326, "\u0120363": 49327, "Stew": 49328, "\u0120Bagg": 49329, "\u0120resided": 49330, "\u0120Spo": 49331, "\u0120remnant": 49332, "\u0120emptiness": 49333, "brainer": 49334, "\u0120outpatient": 49335, "priority": 49336, "\u0120leptin": 49337, "\u0120Payton": 49338, "\u0120Gleaming": 49339, "\u0120Shed": 49340, "\u0120Polo": 49341, "\u0120Mormonism": 49342, "restricted": 49343, "arlane": 49344, "wx": 49345, "\u0120creatine": 49346, "\u0120Anon": 49347, "\u0120STUD": 49348, "\u0120JUL": 49349, "\u0120Tee": 49350, "528": 49351, "089": 49352, "\u0120hatched": 49353, "Dispatch": 49354, "\u0120Composite": 49355, "\u0120451": 49356, "puff": 49357, "\u0120XCOM": 49358, "\u0120Orn": 49359, "\u0120THANK": 49360, "ENDED": 49361, "\u0120Asheville": 49362, "\u0120\u00c3\u013e": 49363, "\u0120mango": 49364, "\u0120Slightly": 49365, "worldly": 49366, "\u0120Wander": 49367, "\u0120Expand": 49368, "\u0120Chr": 49369, "Mist": 49370, "\u0120orthodoxy": 49371, "\u0120UNESCO": 49372, "regate": 49373, "Elsewhere": 49374, "kie": 49375, "irled": 49376, "\u0120topple": 49377, "\u0120adoptive": 49378, "\u0120Legs": 49379, "dress": 49380, "\u0120Sagan": 49381, "bare": 49382, "\u0120Glou": 49383, "Crunch": 49384, "\u0120helpers": 49385, "\u0120chronically": 49386, "\u0120Huma": 49387, "10000": 49388, "\u0120accommodating": 49389, "\u00e4\u00ba\u0136": 49390, "\u0120wrinkles": 49391, "\u0120dodged": 49392, "fourth": 49393, "\u0120precon": 49394, "\u0120compressor": 49395, "\u0120Kare": 49396, "\u0120evict": 49397, "\u0120Warwick": 49398, "imar": 49399, "\u0120modernization": 49400, "\u0120bandwagon": 49401, "\u0120refuted": 49402, "\u0120netted": 49403, "\u0120Naples": 49404, "\u0120Genie": 49405, "perors": 49406, "\u0120fielded": 49407, "\u0120dere": 49408, "\u0120Parables": 49409, "lees": 49410, "\u0120trout": 49411, "aspers": 49412, "\u0120nihil": 49413, "\u0120happiest": 49414, "\u0120floppy": 49415, "\u0120Loft": 49416, "\u0120Heard": 49417, "\u0120unison": 49418, "\u0120lug": 49419, "\u0120Redmond": 49420, "classic": 49421, "Supporters": 49422, "SHIP": 49423, "GMT": 49424, "\u0120fuelled": 49425, "\u00e7\u0132": 49426, "\u0120dd": 49427, "\u0120Eminem": 49428, "\u01201897": 49429, "NYSE": 49430, "\u0120secretaries": 49431, "\u0120FIA": 49432, "\u0120Canaveral": 49433, "Favorite": 49434, "\u0120pomp": 49435, "\u0120detainee": 49436, "ership": 49437, "aimon": 49438, "iour": 49439, "\u0120Apex": 49440, "\u0120plantations": 49441, "amia": 49442, "acion": 49443, "Rust": 49444, "\u0120towed": 49445, "\u0120Truly": 49446, "577": 49447, "\u0120sheltered": 49448, "rider": 49449, "Wo": 49450, "\u0120lair": 49451, "\u0120Intelligent": 49452, "improve": 49453, "matically": 49454, "\u0120etiquette": 49455, "adra": 49456, "allo": 49457, "\u0120Juno": 49458, "anything": 49459, "\u0120Struggle": 49460, "\u0120Predict": 49461, "\u0120Grimes": 49462, "\u0120AMERICA": 49463, "ctx": 49464, "\u0120Situation": 49465, "WOOD": 49466, "\u0120soluble": 49467, "meier": 49468, "\u0120intolerable": 49469, "angering": 49470, "\u0120uninterrupted": 49471, "\u0120tooltip": 49472, "\u0120interrogated": 49473, "\u0120gunned": 49474, "\u0120Sneak": 49475, "\u00e6\u0143\u00a6": 49476, "\u0120tether": 49477, "\u0120crumble": 49478, "Lens": 49479, "\u0120clustered": 49480, "\u0120Syl": 49481, "\u0120Hasan": 49482, "\u0120dystopian": 49483, "wana": 49484, "\u0120joystick": 49485, "\u0120Thib": 49486, "ammu": 49487, "Tomorrow": 49488, "546": 49489, "\u0120overcame": 49490, "\u0120minimized": 49491, "ceptor": 49492, "Runner": 49493, "ENGTH": 49494, "\u0120Brenda": 49495, "\u0120Achievements": 49496, "\u0120torches": 49497, "\u0120rapport": 49498, "\u0120Investigator": 49499, "\u0120Handling": 49500, "relation": 49501, "grey": 49502, "815": 49503, "\u0120kcal": 49504, "\u0120Commands": 49505, "dq": 49506, "\u0120curls": 49507, "\u0120bearer": 49508, "\u0120cynicism": 49509, "itri": 49510, "\u0120Useful": 49511, "Bee": 49512, "DCS": 49513, "\u0120abras": 49514, "Pract": 49515, "BILITIES": 49516, "712": 49517, "\u0120debugger": 49518, "\u0120debtor": 49519, "\u0120Lia": 49520, "\u0120Kers": 49521, "\u0120exacerbate": 49522, "\u0120Stacy": 49523, "\u0120Bland": 49524, "\u0120Scenes": 49525, "\u0120branching": 49526, "\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a": 49527, "apeake": 49528, "\u0120salsa": 49529, "\u0120mishand": 49530, "\u0120Konami": 49531, "\u0120Nib": 49532, "\u0120anecdote": 49533, "\u0120agreeable": 49534, "\u00cf\u012b": 49535, "\u0120Nathaniel": 49536, "\u0120Heisman": 49537, "\u0120Beware": 49538, "\u01201886": 49539, "spective": 49540, "691": 49541, "522": 49542, "\u0120inhibits": 49543, "\u0120hashing": 49544, "\u01201889": 49545, "\u00e5\u00b0\u0128": 49546, "vich": 49547, "Pure": 49548, "\u0120solidly": 49549, "\u0120aspirin": 49550, "imaru": 49551, "\u0120streetcar": 49552, "\u0120UCS": 49553, "\u0120Judd": 49554, "\u0120flashbacks": 49555, "pins": 49556, "\u01201440": 49557, "\u0120UNHCR": 49558, "\u0120Symptoms": 49559, "TIT": 49560, "538": 49561, "Fra": 49562, "%);": 49563, "\u0120ooz": 49564, "\u0120curfew": 49565, "\u0120calmed": 49566, "\u0120participates": 49567, "TeX": 49568, "\u0120nonsensical": 49569, "\u0120fullback": 49570, "\u0120DeL": 49571, "monkey": 49572, "hari": 49573, "\u0120metabolites": 49574, "\u0120looted": 49575, "\u0120ALWAYS": 49576, "\u0120BCC": 49577, "Lt": 49578, "ochet": 49579, "Bone": 49580, "\u0120vetoed": 49581, "\u0120gcc": 49582, "\u0120CLICK": 49583, "\u01201888": 49584, "saf": 49585, "\u0120stiffness": 49586, "\u0120lowly": 49587, "\u0120Geh": 49588, "verson": 49589, "orset": 49590, "\u0120unforeseen": 49591, "\u0120anesthesia": 49592, "\u0120Optical": 49593, "\u0120reconstructed": 49594, "\u0120Tup": 49595, "shows": 49596, "NEWS": 49597, "\u0120Newspaper": 49598, "\u0120ASA": 49599, "tera": 49600, "Numbers": 49601, "\u0120inexplicable": 49602, "\u00d7\u0133": 49603, "\u0120hardness": 49604, "untarily": 49605, "\u0120Acer": 49606, "gradient": 49607, "ARDIS": 49608, "\u0120woodland": 49609, "\u0120metaphors": 49610, "\u0120Wembley": 49611, "\u0120Pavel": 49612, "philis": 49613, "\u0120rewriting": 49614, "\u0120perceptual": 49615, "\u01201070": 49616, "worms": 49617, "\u0120Downs": 49618, "\u0120unsurprisingly": 49619, "\u0120tagging": 49620, "flame": 49621, "\u0120litres": 49622, "\u0120bounces": 49623, "\u0120Babe": 49624, "shut": 49625, "\u0120overdoses": 49626, "\u0120Sheila": 49627, "\u0120Chau": 49628, "\u0120Bless": 49629, "Capture": 49630, "\u0120Significant": 49631, "\u0120Scion": 49632, "\u0120389": 49633, "\u0120McH": 49634, "\u0120Titanium": 49635, "\u0120Meal": 49636, "ameda": 49637, "agents": 49638, "aggressive": 49639, "Billy": 49640, "763": 49641, "\u0120Saying": 49642, "DERR": 49643, "itone": 49644, "Collins": 49645, "Bound": 49646, "\u0120bolted": 49647, "\u0120DMCA": 49648, "953": 49649, "\u0120uniqueness": 49650, "\u0120epigen": 49651, "unci": 49652, "antam": 49653, "\u0120reckoning": 49654, "chairs": 49655, "OGR": 49656, "\u0120Senegal": 49657, "\u01201862": 49658, "relevant": 49659, "\u0120\u00c2\u00af": 49660, "\u0120pharmacies": 49661, "\u0120Geral": 49662, "vier": 49663, "Yan": 49664, "ORPG": 49665, "\u0120rabid": 49666, "bending": 49667, "\u0120UNITED": 49668, "\u0120465": 49669, "Assembly": 49670, "\u0120weep": 49671, "\u0120behest": 49672, "\u0120Mothers": 49673, "\u0120Jace": 49674, "hid": 49675, "\u0120whirlwind": 49676, "\u0120UNIVERS": 49677, "\u0120utopian": 49678, "\u0120kidnap": 49679, "Philipp": 49680, "Kin": 49681, "893": 49682, "\u0120livestream": 49683, "\u0120MISS": 49684, "\u0120subversive": 49685, "\u0120Techniques": 49686, "\u0120JUSTICE": 49687, "\u0120BASE": 49688, "\u0120387": 49689, "\u0120assailants": 49690, "\u0120Hardcore": 49691, "\u0120sprinkled": 49692, "\u0120Pse": 49693, "\u00e9\u013c": 49694, "printed": 49695, "\u0120Hau": 49696, "ORGE": 49697, "\u0120TOUR": 49698, "\u0120laced": 49699, "\u0120itch": 49700, "Giving": 49701, "\u0120ported": 49702, "781": 49703, "////////////////////////////////": 49704, "breeding": 49705, "\u0120logger": 49706, "\u0120HOL": 49707, "innie": 49708, "Firstly": 49709, "\u0120embryonic": 49710, "\u0120delegated": 49711, "pai": 49712, "OIL": 49713, "\u0120centrally": 49714, "\u0120Rx": 49715, "\u0120Scouting": 49716, "Dutch": 49717, "\u0120hereditary": 49718, "\u0120Cruiser": 49719, "sat": 49720, "529": 49721, "\u0120Marriott": 49722, "othermal": 49723, "\u0120prohibitions": 49724, "Earn": 49725, "\u0120Stab": 49726, "\u0120Colleges": 49727, "\u0120Belief": 49728, "stretched": 49729, "\u0120LH": 49730, "\u0120EntityItem": 49731, "CIA": 49732, "\u0120unrem": 49733, "\u0120laureate": 49734, "\u0120denominations": 49735, "summary": 49736, "hler": 49737, "Spect": 49738, "\u0120Klaus": 49739, "\u0120Beans": 49740, "\u0120insur": 49741, "\u0120PAX": 49742, "\u0120fielder": 49743, "\u0120Vet": 49744, "\u0120Sparrow": 49745, "zie": 49746, "\u0120SQ": 49747, "\u0120Mondays": 49748, "\u0120Offline": 49749, "\u0120Lerner": 49750, "\u0120Extensions": 49751, "Ireland": 49752, "\u0120patronage": 49753, "\u0120contrasted": 49754, "\u0120Mania": 49755, "hirt": 49756, "Moscow": 49757, "\u0120condemns": 49758, "\u0120Ange": 49759, "\u0120composing": 49760, "\u0120Pepe": 49761, "\u0120Paddock": 49762, "\u0120heterogeneity": 49763, "\u0120ideologically": 49764, "\u0120fishes": 49765, "\u0120cursing": 49766, "\u0120Rutherford": 49767, "\u0120Floating": 49768, "\u0120Amelia": 49769, "Tea": 49770, "Synopsis": 49771, "\u0120stunts": 49772, "\u0120bead": 49773, "\u0120stocking": 49774, "\u0120MILL": 49775, "obook": 49776, "massive": 49777, "\\<": 49778, "\u0120hump": 49779, "\u0120Preferences": 49780, "EngineDebug": 49781, "geist": 49782, "\u0120Nieto": 49783, "omever": 49784, "ishy": 49785, "evaluate": 49786, "colonial": 49787, "Alternative": 49788, "\u0120GoPro": 49789, "\u0120Vortex": 49790, "\u0120NETWORK": 49791, "ansky": 49792, "Secure": 49793, "\u0120Thrust": 49794, "Snake": 49795, "\u0120parcels": 49796, "\u0120samurai": 49797, "\u0120actresses": 49798, "Nap": 49799, "MF": 49800, "iferation": 49801, "Beer": 49802, "523": 49803, "\u0120Ily": 49804, "ointment": 49805, "Ping": 49806, "\u0120striped": 49807, "\u0120Mellon": 49808, "ossession": 49809, "\u0120neutron": 49810, "endium": 49811, "\u0120aph": 49812, "\u0120Flavoring": 49813, "\u0120383": 49814, "\u0120responsiveness": 49815, "\u0120Jindal": 49816, "\u0120Hitchcock": 49817, "Denver": 49818, "\u0120DRAGON": 49819, "smanship": 49820, "\u0120Dupl": 49821, "\u0120sly": 49822, "\u0120webcam": 49823, "\u0120Twain": 49824, "\u0120Darling": 49825, "iliate": 49826, "consumer": 49827, "DIT": 49828, "\u0120namesake": 49829, "\u0120unorthodox": 49830, "\u0120funer": 49831, "\u0120PLoS": 49832, "\u0120CONTROL": 49833, "ozyg": 49834, "oglobin": 49835, "FACE": 49836, "ERG": 49837, "\u0120Dia": 49838, "\u0120Fiesta": 49839, "cele": 49840, "034": 49841, "\u0120enclave": 49842, "\u00e2\u0138\u00ac\u00e2\u0138\u00ac": 49843, "onement": 49844, "alist": 49845, "Mand": 49846, "\u0120homegrown": 49847, "\u0120Fancy": 49848, "\u0120conceptions": 49849, "\u0120Contains": 49850, "ureen": 49851, "\u0120reiterate": 49852, "\u0120meager": 49853, "\u0120installments": 49854, "Spawn": 49855, "627": 49856, "\u0120photoc": 49857, "\u0120Cabrera": 49858, "\u0120Rosenthal": 49859, "\u0120Lansing": 49860, "isner": 49861, "\u0120invests": 49862, "\u0120UFOs": 49863, "EXP": 49864, "Hardware": 49865, "\u0120tragically": 49866, "\u0120concedes": 49867, "ieft": 49868, "cham": 49869, "borgh": 49870, "\u0120Schr": 49871, "\u0120Melanie": 49872, "\u0120Hoy": 49873, "\u0120visitation": 49874, "\u0120idiosyncr": 49875, "\u0120fractions": 49876, "\u0120foreskin": 49877, "obos": 49878, "\u0120poaching": 49879, "\u0120VIEW": 49880, "\u0120stimulates": 49881, "\u0120Gork": 49882, "canon": 49883, "MIC": 49884, "\u0120Nemesis": 49885, "\u0120Indra": 49886, "\u0120DMV": 49887, "\u0120529": 49888, "\u0120inspecting": 49889, "\u0120grandma": 49890, "\u0120Whedon": 49891, "\u0120Shant": 49892, "\u0120Purg": 49893, "ikan": 49894, "\u0120Teg": 49895, "\u0120CLR": 49896, "zac": 49897, "Victoria": 49898, "\u0120Verify": 49899, "ionics": 49900, "\u0120partying": 49901, "\u0120Mou": 49902, "colour": 49903, "\u0120testimonies": 49904, "lations": 49905, "\u0120pressuring": 49906, "hiro": 49907, "acers": 49908, "\u0120fid": 49909, "angler": 49910, "\u0120CSI": 49911, "\u0120hereafter": 49912, "\u0120dissidents": 49913, "reporting": 49914, "iphany": 49915, "chev": 49916, "\u0120solitude": 49917, "\u0120lobe": 49918, "\u0120indis": 49919, "\u0120credential": 49920, "recent": 49921, "adult": 49922, "\u0120Nirvana": 49923, "\u0120Franchise": 49924, "Layer": 49925, "Hyp": 49926, "\u0120Berkshire": 49927, "\u0120wills": 49928, "tif": 49929, "\u0120totem": 49930, "\u0120Judah": 49931, "repair": 49932, "Instant": 49933, "548": 49934, "\u0120embassies": 49935, "\u0120bottleneck": 49936, "\u0120bount": 49937, "\u0120typew": 49938, "\u0120Alvin": 49939, "jing": 49940, "imilar": 49941, "Rush": 49942, "\u0120brim": 49943, "\u0120HELP": 49944, "Aim": 49945, "]'": 49946, "\u0120passively": 49947, "\u0120bounded": 49948, "\u0120Rated": 49949, "\u0120criminality": 49950, "\u0120biomark": 49951, "\u0120dispatcher": 49952, "\u0120Towards": 49953, "\u0120+++": 49954, "righteous": 49955, "frog": 49956, "\u0120Panc": 49957, "Carter": 49958, "032": 49959, "\u00e6\u00a9\u0141": 49960, "\u0120ultraviolet": 49961, "\u0120Licensed": 49962, "\u0120Tata": 49963, "\u0120Blessing": 49964, "\u0120GAM": 49965, "\u0120chemically": 49966, "\u0120Seaf": 49967, "\u0120RELE": 49968, "\u0120Mercenary": 49969, "capitalist": 49970, "\u0120formulations": 49971, "\u0120annihilation": 49972, "\u0120Verb": 49973, "\u0120Argon": 49974, "\u0120unloaded": 49975, "\u0120morphed": 49976, "\u0120conquering": 49977, "backer": 49978, "IELD": 49979, "\u0120thefts": 49980, "\u0120frontrunner": 49981, "\u0120Royale": 49982, "\u0120Fundamental": 49983, "elight": 49984, "Chip": 49985, "necessary": 49986, "ayn": 49987, "\u0120Slip": 49988, "\u0120448": 49989, "cerned": 49990, "Pause": 49991, "\u0120shockingly": 49992, "\u0120ABV": 49993, "\u0120composure": 49994, "733": 49995, "\u0120Motorsport": 49996, "ahime": 49997, "Murray": 49998, "Mach": 49999, "\u0120grids": 50000, "\u0120debian": 50001, "\u0120furthermore": 50002, "\u0120dexterity": 50003, "\u0120Collections": 50004, "oslov": 50005, "ilage": 50006, "bj": 50007, "\u0120Monteneg": 50008, "\u0120strutConnector": 50009, "\u0120massacres": 50010, "\u0120briefs": 50011, "fetched": 50012, "uvian": 50013, "olition": 50014, "Failure": 50015, "emonic": 50016, "\u0120flared": 50017, "\u0120claimant": 50018, "\u0120cures": 50019, "\u0120giveaways": 50020, "\u0120Substance": 50021, "alions": 50022, "\u0120cringe": 50023, "\u0120Kul": 50024, "\u0120aristocracy": 50025, "\u0120Ulster": 50026, "olated": 50027, "housing": 50028, "\u0120MIS": 50029, "\u0120glared": 50030, "\u0120Wilhelm": 50031, "needs": 50032, "lambda": 50033, "builders": 50034, "\u0120VIS": 50035, "\u0120radiator": 50036, "\u0120Ghostbusters": 50037, "\u0120436": 50038, "actual": 50039, "\u0120herds": 50040, "\u00c3\u00a7a": 50041, "watching": 50042, "\u0120countering": 50043, "Charge": 50044, "\u0120charred": 50045, "\u0120warheads": 50046, "\u0120iodine": 50047, "\u0120Macy": 50048, "041": 50049, "\u0120departures": 50050, "\u0120Sins": 50051, "\u0120dyed": 50052, "\u0120Concepts": 50053, "gado": 50054, "713": 50055, "\u0120quotations": 50056, "\u0120gist": 50057, "\u0120Christy": 50058, "\u0120antigen": 50059, "\u0120Hemp": 50060, "\u0120Drawn": 50061, "\u0120Barg": 50062, "ezvous": 50063, "\u0120paternity": 50064, "\u0120ardu": 50065, "\u0120Anchorage": 50066, "\u0120Rik": 50067, "\u0120overloaded": 50068, "\u0120Username": 50069, "\u0120Tammy": 50070, "\u0120Nau": 50071, "\u0120Cellular": 50072, "\u0120waning": 50073, "\u0120rodent": 50074, "\u0120Worcester": 50075, "ilts": 50076, "\u0120Tad": 50077, "\u0120dwellings": 50078, "\u0120bullish": 50079, "431": 50080, "\u0120retaliate": 50081, "\u0120migraine": 50082, "\u0120Chevron": 50083, "CHECK": 50084, "\u0120donkey": 50085, "crim": 50086, "SPA": 50087, "\u0120Analog": 50088, "\u0120marquee": 50089, "\u0120Haas": 50090, "Bir": 50091, "\u0120GDDR": 50092, "\u0120Downloads": 50093, "\u0120willpower": 50094, "\u0120Forth": 50095, "\u0120Recorded": 50096, "\u0120impossibility": 50097, "\u0120Logged": 50098, "\u0120Franks": 50099, "\u0120Ratt": 50100, "initions": 50101, "\u0120cleaners": 50102, "\u0120sorely": 50103, "\u0120flickering": 50104, "\u0120Examination": 50105, "catching": 50106, "alloween": 50107, "Msg": 50108, "\u0120dunno": 50109, "Fa": 50110, "\u0120dysph": 50111, "crazy": 50112, ".''.": 50113, "\u0120mainline": 50114, "\u0120cs": 50115, "\u0120ptr": 50116, "\u0120Wally": 50117, "igun": 50118, "951": 50119, "\u0120Bigfoot": 50120, "fights": 50121, "\u0120retrieving": 50122, "Jr": 50123, "\u0120duplication": 50124, "\u0120Explan": 50125, "\u0120relational": 50126, "\u0120quaint": 50127, "\u0120biscuits": 50128, "\u0120ado": 50129, "\u0120shudder": 50130, "\u0120antidote": 50131, "blooded": 50132, "ksh": 50133, "\u0120sauces": 50134, "\u0120reinvest": 50135, "\u0120dispensary": 50136, "\u0120Diver": 50137, "\u01209000": 50138, "student": 50139, "\u0120insepar": 50140, "escap": 50141, "\u0120toddlers": 50142, "\u0120GPIO": 50143, "\u0120Assignment": 50144, "headers": 50145, "\u0120lackluster": 50146, "\u0120aback": 50147, "956": 50148, "\u0120toolbar": 50149, "745": 50150, "\u0120oust": 50151, "\u0120contemplation": 50152, "\u0120PRESIDENT": 50153, "\u0120458": 50154, "======": 50155, "\u0120guaranteeing": 50156, "\u0120Heist": 50157, "\u0120Cannes": 50158, "\u013b\u00bd": 50159, "\u0120collaborator": 50160, "\u0120Amp": 50161, "\u0120gou": 50162, "\u0120SHALL": 50163, "stories": 50164, "783": 50165, "\u0120mobilized": 50166, "\u0120brood": 50167, "\u0120LU": 50168, "\u0120\u00f0\u0141\u0133": 50169, "\u0120refin": 50170, "\u0120Anthropology": 50171, "vind": 50172, "illi": 50173, "\u0120warranties": 50174, "\u0120Babel": 50175, "\u0120swath": 50176, "\u0120caches": 50177, "\u0120antagonists": 50178, "artifacts": 50179, "\u0120hotly": 50180, "\u0120Starts": 50181, "\u0120G\u00c3\u00b6": 50182, "zag": 50183, "!!!!!": 50184, "\u0120scourge": 50185, "\u0120conspiring": 50186, "ruits": 50187, "reverse": 50188, "\u0120Sheen": 50189, "\u0120Jesuit": 50190, "\u0120Giovanni": 50191, "adies": 50192, "\u0120buttocks": 50193, "earcher": 50194, "acan": 50195, "\u0120volleyball": 50196, "\u0120shrouded": 50197, "\u0120scoreboard": 50198, "bats": 50199, "\u0120IPM": 50200, "\u0120asses": 50201, "\u0120deregulation": 50202, "\u0120Telegram": 50203, "\u0120Reboot": 50204, "\u01207000": 50205, "\u0120Canary": 50206, "\u0120kernels": 50207, "\u0120Fran\u00c3\u00a7ois": 50208, "\u0120Duff": 50209, "\u0120Pon": 50210, "\u0120Leica": 50211, "\u0120Garmin": 50212, "\u0120orphans": 50213, "\u0120Claudia": 50214, "\u0120calendars": 50215, "\u0120Leilan": 50216, "ento": 50217, "Rocket": 50218, "\u0120brunch": 50219, "\u0120Hawking": 50220, "ainers": 50221, "\u0120sensibilities": 50222, "\u0120kW": 50223, "\u0120Kand": 50224, "\u0120reclaimed": 50225, "\u0120interestingly": 50226, "\u00d7\u00a9": 50227, "romy": 50228, "JM": 50229, "\u0120Enhancement": 50230, "bush": 50231, "Skip": 50232, "\u0120rappers": 50233, "\u0120gazing": 50234, "pedia": 50235, "athlon": 50236, "Revolution": 50237, "\u0120snipers": 50238, "\u0120reverted": 50239, "\u0120conglomerate": 50240, "Terry": 50241, "794": 50242, "\u0120harsher": 50243, "\u0120desolate": 50244, "\u0120Hitman": 50245, "Commission": 50246, "\u0120(/": 50247, "\u00e2\u0122\u00a6.\"": 50248, "Compar": 50249, "\u0120amplification": 50250, "ominated": 50251, "\u0120regress": 50252, "\u0120Collider": 50253, "\u0120informants": 50254, "\u0120gazed": 50255, "<|endoftext|>": 50256} \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryException.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryException.cs deleted file mode 100644 index a21202fa1dff..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryException.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown by the Azure Cognitive Search connector -/// -public class AzureCognitiveSearchMemoryException : Exception -{ - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public AzureCognitiveSearchMemoryException(ErrorCodes errorCode, string? message) - : this(errorCode, message, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public AzureCognitiveSearchMemoryException(ErrorCodes errorCode, string? message, Exception? innerException) - : base(GetDefaultMessage(errorCode, message, innerException), innerException) - { - this.ErrorCode = errorCode; - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// Translate the error code into a default message. - private static string GetDefaultMessage(ErrorCodes errorCode, string? message, Exception? innerException) - { - if (message is not null) { return message; } - - var description = errorCode.ToString("G"); - return innerException is not null ? $"{description}: {innerException.Message}" : description; - } - - /// - /// Error codes for the Qdrant connector exceptions. - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError = -1, - - /// - /// Invalid embedding size, the value must be greater than zero - /// - InvalidEmbeddingSize, - - /// - /// Invalid index name - /// - InvalidIndexName, - - /// - /// Read failure - /// - ReadFailure, - - /// - /// Write failure - /// - WriteFailure, - } -} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryRecord.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryRecord.cs index 40ece7daea7f..2516e158d052 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryRecord.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryRecord.cs @@ -1,12 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; -using System.Linq; using System.Text; using System.Text.Json.Serialization; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch; @@ -16,16 +14,37 @@ namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch; /// public class AzureCognitiveSearchMemoryRecord { + /// + /// ID field name. + /// public const string IdField = "Id"; + /// + /// Text field name. + /// public const string TextField = "Text"; + /// + /// Embedding field name. + /// public const string EmbeddingField = "Embedding"; + /// + /// External source name field name. + /// public const string ExternalSourceNameField = "ExternalSourceName"; + /// + /// Description field name. + /// public const string DescriptionField = "Description"; + /// + /// Additional metadata field name. + /// public const string AdditionalMetadataField = "AdditionalMetadata"; + /// + /// Is reference field name. + /// public const string IsReferenceField = "IsReference"; /// - /// Record Id. + /// Record ID. /// The record is not filterable to save quota, also SK uses only semantic search. /// [JsonPropertyName(IdField)] @@ -41,7 +60,8 @@ public class AzureCognitiveSearchMemoryRecord /// Content embedding /// [JsonPropertyName(EmbeddingField)] - public List Embedding { get; set; } = Array.Empty().ToList(); + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Embedding { get; set; } /// /// Optional description of the content, e.g. a title. This can be useful when @@ -58,7 +78,7 @@ public class AzureCognitiveSearchMemoryRecord public string? AdditionalMetadata { get; set; } = string.Empty; /// - /// Name of the external source, in cases where the content and the Id are + /// Name of the external source, in cases where the content and the ID are /// referenced to external information. /// [JsonPropertyName(ExternalSourceNameField)] @@ -71,35 +91,54 @@ public class AzureCognitiveSearchMemoryRecord public bool IsReference { get; set; } = false; /// - /// Ctor required by JSON deserializer + /// Initializes a new instance of the class. + /// Required by JSON deserializer. /// public AzureCognitiveSearchMemoryRecord() { } + /// + /// Initializes a new instance of the class with the specified ID. + /// + /// The record ID. public AzureCognitiveSearchMemoryRecord(string id) { this.Id = EncodeId(id); } + /// + /// Initializes a new instance of the class with the specified parameters. + /// + /// The record ID. + /// The content stored in the record. + /// The name of the external source. + /// Whether the record references external information. + /// The content embedding. + /// The optional description of the content. + /// The additional metadata. public AzureCognitiveSearchMemoryRecord( string id, string text, string externalSourceName, bool isReference, - Embedding embedding, + ReadOnlyMemory embedding, string? description = null, string? additionalMetadata = null) { this.Id = EncodeId(id); this.IsReference = isReference; - this.Embedding = embedding.Vector.ToList(); + this.Embedding = embedding; this.Text = text; this.ExternalSourceName = externalSourceName; this.Description = description; this.AdditionalMetadata = additionalMetadata; } + /// + /// Converts the current instance to a object. + /// + /// A object. public MemoryRecordMetadata ToMemoryRecordMetadata() { return new MemoryRecordMetadata( @@ -111,6 +150,11 @@ public MemoryRecordMetadata ToMemoryRecordMetadata() additionalMetadata: this.AdditionalMetadata ?? string.Empty); } + /// + /// Creates a new object from the specified . + /// + /// The object. + /// A new object. public static AzureCognitiveSearchMemoryRecord FromMemoryRecord(MemoryRecord record) { return new AzureCognitiveSearchMemoryRecord( @@ -124,26 +168,37 @@ public static AzureCognitiveSearchMemoryRecord FromMemoryRecord(MemoryRecord rec ); } + /// + /// Converts the current instance to a object. + /// + /// Whether to include embeddings in the resulting . + /// A object. public MemoryRecord ToMemoryRecord(bool withEmbeddings = true) { return new MemoryRecord( metadata: this.ToMemoryRecordMetadata(), - embedding: new Embedding(withEmbeddings ? this.Embedding : Array.Empty()), + embedding: withEmbeddings ? this.Embedding : default, key: this.Id); } /// + /// Encodes the specified ID using a URL-safe algorithm. /// ACS keys can contain only letters, digits, underscore, dash, equal sign, recommending /// to encode values with a URL-safe algorithm. /// - /// Original Id - /// Encoded id + /// The original ID. + /// The encoded ID. protected internal static string EncodeId(string realId) { var bytes = Encoding.UTF8.GetBytes(realId); return Convert.ToBase64String(bytes); } + /// + /// Decodes the specified encoded ID. + /// + /// The encoded ID. + /// The decoded ID. private protected static string DecodeId(string encodedId) { var bytes = Convert.FromBase64String(encodedId); diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryStore.cs index e007c5c0a4ad..2f379e109966 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryStore.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -14,16 +15,16 @@ using Azure.Search.Documents.Indexes; using Azure.Search.Documents.Indexes.Models; using Azure.Search.Documents.Models; -using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch; +/// +/// AzureCognitiveSearchMemoryStore is a memory store implementation using Azure Cognitive Search. +/// public class AzureCognitiveSearchMemoryStore : IMemoryStore { - // Note: Azure max length 24 chars - private const string UserAgent = "Semantic-Kernel"; - /// /// Create a new instance of memory storage using Azure Cognitive Search. /// @@ -32,7 +33,7 @@ public class AzureCognitiveSearchMemoryStore : IMemoryStore public AzureCognitiveSearchMemoryStore(string endpoint, string apiKey) { AzureKeyCredential credentials = new(apiKey); - this._adminClient = new SearchIndexClient(new Uri(endpoint), credentials, ClientOptions()); + this._adminClient = new SearchIndexClient(new Uri(endpoint), credentials, GetClientOptions()); } /// @@ -42,7 +43,7 @@ public AzureCognitiveSearchMemoryStore(string endpoint, string apiKey) /// Azure service public AzureCognitiveSearchMemoryStore(string endpoint, TokenCredential credentials) { - this._adminClient = new SearchIndexClient(new Uri(endpoint), credentials, ClientOptions()); + this._adminClient = new SearchIndexClient(new Uri(endpoint), credentials, GetClientOptions()); } /// @@ -55,57 +56,73 @@ public Task CreateCollectionAsync(string collectionName, CancellationToken cance /// public IAsyncEnumerable GetCollectionsAsync(CancellationToken cancellationToken = default) { - return this.GetIndexesAsync(cancellationToken); + return RunMemoryStoreOperation(() => this.GetIndexesAsync(cancellationToken)); } /// public async Task DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default) { - string normalizeIndexName = this.NormalizeIndexName(collectionName); + var normalizedIndexName = this.NormalizeIndexName(collectionName); + + var indexes = RunMemoryStoreOperation(() => this.GetIndexesAsync(cancellationToken)); - return await this.GetIndexesAsync(cancellationToken) - .AnyAsync(index => string.Equals(index, collectionName, StringComparison.OrdinalIgnoreCase) - || string.Equals(index, normalizeIndexName, StringComparison.OrdinalIgnoreCase), - cancellationToken: cancellationToken).ConfigureAwait(false); + return await indexes + .AnyAsync(index => + string.Equals(index, collectionName, StringComparison.OrdinalIgnoreCase) || + string.Equals(index, normalizedIndexName, StringComparison.OrdinalIgnoreCase), + cancellationToken: cancellationToken + ) + .ConfigureAwait(false); } /// public Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) { - string normalizeIndexName = this.NormalizeIndexName(collectionName); + var normalizedIndexName = this.NormalizeIndexName(collectionName); - return this._adminClient.DeleteIndexAsync(normalizeIndexName, cancellationToken); + return RunMemoryStoreOperation(() => this._adminClient.DeleteIndexAsync(normalizedIndexName, cancellationToken)); } /// public Task UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default) { - collectionName = this.NormalizeIndexName(collectionName); - return this.UpsertRecordAsync(collectionName, AzureCognitiveSearchMemoryRecord.FromMemoryRecord(record), cancellationToken); + var normalizedIndexName = this.NormalizeIndexName(collectionName); + + return RunMemoryStoreOperation(() => this.UpsertRecordAsync(normalizedIndexName, AzureCognitiveSearchMemoryRecord.FromMemoryRecord(record), cancellationToken)); } /// public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IEnumerable records, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - collectionName = this.NormalizeIndexName(collectionName); - IList searchRecords = records.Select(AzureCognitiveSearchMemoryRecord.FromMemoryRecord).ToList(); - List result = await this.UpsertBatchAsync(collectionName, searchRecords, cancellationToken).ConfigureAwait(false); + var normalizedIndexName = this.NormalizeIndexName(collectionName); + + var searchRecords = records.Select(AzureCognitiveSearchMemoryRecord.FromMemoryRecord).ToList(); + + var result = await RunMemoryStoreOperation(() => this.UpsertBatchAsync(normalizedIndexName, searchRecords, cancellationToken)).ConfigureAwait(false); + foreach (var x in result) { yield return x; } } /// public async Task GetAsync(string collectionName, string key, bool withEmbedding = false, CancellationToken cancellationToken = default) { - collectionName = this.NormalizeIndexName(collectionName); - var client = this.GetSearchClient(collectionName); + var normalizedIndexName = this.NormalizeIndexName(collectionName); + var client = this.GetSearchClient(normalizedIndexName); + + var encodedId = AzureCognitiveSearchMemoryRecord.EncodeId(key); + Response? result; + try { - result = await client - .GetDocumentAsync(AzureCognitiveSearchMemoryRecord.EncodeId(key), cancellationToken: cancellationToken) - .ConfigureAwait(false); + result = await RunMemoryStoreOperation(async () => + { + return await client + .GetDocumentAsync(encodedId, cancellationToken: cancellationToken) + .ConfigureAwait(false); + }).ConfigureAwait(false); } - catch (RequestFailedException e) when (e.Status == 404) + catch (HttpOperationException e) when (e.StatusCode == System.Net.HttpStatusCode.NotFound) { // Index not found, no data to return return null; @@ -113,9 +130,7 @@ public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IE if (result?.Value == null) { - throw new AzureCognitiveSearchMemoryException( - AzureCognitiveSearchMemoryException.ErrorCodes.ReadFailure, - "Memory read returned null"); + throw new SKException("Memory read returned null"); } return result.Value.ToMemoryRecord(); @@ -138,7 +153,7 @@ public async IAsyncEnumerable GetBatchAsync( /// public async Task<(MemoryRecord, double)?> GetNearestMatchAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) @@ -151,45 +166,56 @@ public async IAsyncEnumerable GetBatchAsync( /// public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - collectionName = this.NormalizeIndexName(collectionName); + // Cosine similarity range: -1 .. +1 + minRelevanceScore = Math.Max(-1, Math.Min(1, minRelevanceScore)); + + var normalizedIndexName = this.NormalizeIndexName(collectionName); - var client = this.GetSearchClient(collectionName); + var client = this.GetSearchClient(normalizedIndexName); - SearchQueryVector vectorQuery = new() + RawVectorQuery vectorQuery = new() { KNearestNeighborsCount = limit, - Fields = AzureCognitiveSearchMemoryRecord.EmbeddingField, - Value = embedding.Vector.ToList() + Fields = { AzureCognitiveSearchMemoryRecord.EmbeddingField }, + Vector = MemoryMarshal.TryGetArray(embedding, out var array) && array.Count == embedding.Length ? array.Array! : embedding.ToArray(), + }; + + SearchOptions options = new() + { + VectorQueries = { vectorQuery } }; - SearchOptions options = new() { Vector = vectorQuery }; Response>? searchResult = null; try { - searchResult = await client - .SearchAsync(null, options, cancellationToken: cancellationToken) - .ConfigureAwait(false); + searchResult = await RunMemoryStoreOperation(async () => + { + return await client + .SearchAsync(null, options, cancellationToken: cancellationToken) + .ConfigureAwait(false); + }).ConfigureAwait(false); } - catch (RequestFailedException e) when (e.Status == 404) + catch (HttpOperationException e) when (e.StatusCode == System.Net.HttpStatusCode.NotFound) { // Index not found, no data to return } if (searchResult == null) { yield break; } + var minAzureSearchScore = CosineSimilarityToScore(minRelevanceScore); await foreach (SearchResult? doc in searchResult.Value.GetResultsAsync()) { - if (doc == null || doc.Score < minRelevanceScore) { continue; } + if (doc == null || doc.Score < minAzureSearchScore) { continue; } MemoryRecord memoryRecord = doc.Document.ToMemoryRecord(withEmbeddings); - yield return (memoryRecord, doc.Score ?? 0); + yield return (memoryRecord, ScoreToCosineSimilarity(doc.Score ?? 0)); } } @@ -202,16 +228,16 @@ public Task RemoveAsync(string collectionName, string key, CancellationToken can /// public async Task RemoveBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default) { - collectionName = this.NormalizeIndexName(collectionName); + var normalizedIndexName = this.NormalizeIndexName(collectionName); - var records = keys.Select(x => new List { new(x) }); + var records = keys.Select(x => new AzureCognitiveSearchMemoryRecord(x)); - var client = this.GetSearchClient(collectionName); + var client = this.GetSearchClient(normalizedIndexName); try { - await client.DeleteDocumentsAsync(records, cancellationToken: cancellationToken).ConfigureAwait(false); + await RunMemoryStoreOperation(() => client.DeleteDocumentsAsync(records, cancellationToken: cancellationToken)).ConfigureAwait(false); } - catch (RequestFailedException e) when (e.Status == 404) + catch (HttpOperationException e) when (e.StatusCode == System.Net.HttpStatusCode.NotFound) { // Index not found, no data to delete } @@ -246,12 +272,12 @@ private Task> CreateIndexAsync( { if (embeddingSize < 1) { - throw new AzureCognitiveSearchMemoryException( - AzureCognitiveSearchMemoryException.ErrorCodes.InvalidEmbeddingSize, - "Invalid embedding size: the value must be greater than zero."); + throw new SKException("Invalid embedding size: the value must be greater than zero."); } - var configName = "searchConfig"; + const string ProfileName = "searchProfile"; + const string AlgorithmName = "searchAlgorithm"; + var newIndex = new SearchIndex(indexName) { Fields = new List @@ -261,7 +287,7 @@ private Task> CreateIndexAsync( { IsSearchable = true, VectorSearchDimensions = embeddingSize, - VectorSearchConfiguration = configName + VectorSearchProfile = ProfileName }, new SearchField(AzureCognitiveSearchMemoryRecord.TextField, SearchFieldDataType.String) { IsFilterable = true, IsFacetable = true }, new SimpleField(AzureCognitiveSearchMemoryRecord.DescriptionField, SearchFieldDataType.String) { IsFilterable = true, IsFacetable = true }, @@ -271,13 +297,14 @@ private Task> CreateIndexAsync( }, VectorSearch = new VectorSearch { - AlgorithmConfigurations = + Algorithms = { - new HnswVectorSearchAlgorithmConfiguration(configName) + new HnswVectorSearchAlgorithmConfiguration(AlgorithmName) { Parameters = new HnswParameters { Metric = VectorSearchAlgorithmMetric.Cosine } } - } + }, + Profiles = { new VectorSearchProfile(ProfileName, AlgorithmName) } } }; @@ -311,7 +338,7 @@ private async Task> UpsertBatchAsync( if (records.Count < 1) { return keys; } - var embeddingSize = records[0].Embedding.Count; + var embeddingSize = records[0].Embedding.Length; var client = this.GetSearchClient(indexName); @@ -336,9 +363,7 @@ Task> UpsertCode() if (result == null || result.Value.Results.Count == 0) { - throw new AzureCognitiveSearchMemoryException( - AzureCognitiveSearchMemoryException.ErrorCodes.WriteFailure, - "Memory write returned null or an empty set"); + throw new SKException("Memory write returned null or an empty set"); } return result.Value.Results.Select(x => x.Key).ToList(); @@ -355,9 +380,7 @@ private string NormalizeIndexName(string indexName) { if (indexName.Length > 128) { - throw new AzureCognitiveSearchMemoryException( - AzureCognitiveSearchMemoryException.ErrorCodes.InvalidIndexName, - "The collection name is too long, it cannot exceed 128 chars."); + throw new SKException("The collection name is too long, it cannot exceed 128 chars."); } #pragma warning disable CA1308 // The service expects a lowercase string @@ -389,44 +412,46 @@ private SearchClient GetSearchClient(string indexName) /// Options used by the Azure Cognitive Search client, e.g. User Agent. /// See also https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/src/DiagnosticsOptions.cs /// - private static SearchClientOptions ClientOptions() + private static SearchClientOptions GetClientOptions() { return new SearchClientOptions { Diagnostics = { - IsTelemetryEnabled = IsTelemetryEnabled(), - ApplicationId = UserAgent, + IsTelemetryEnabled = Telemetry.IsTelemetryEnabled, + ApplicationId = Telemetry.HttpUserAgent, }, }; } /// - /// Source: https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/src/DiagnosticsOptions.cs - /// - private static bool IsTelemetryEnabled() - { - return !EnvironmentVariableToBool(Environment.GetEnvironmentVariable("AZURE_TELEMETRY_DISABLED")) ?? true; - } - - /// - /// Source: https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/src/DiagnosticsOptions.cs + /// Executes a memory store operation by invoking the provided operation delegate. /// - private static bool? EnvironmentVariableToBool(string? value) + /// The return type of the operation. + /// The operation delegate to be executed. + /// The result of the memory store operation. + private static T RunMemoryStoreOperation(Func operation) { - if (string.Equals(bool.TrueString, value, StringComparison.OrdinalIgnoreCase) || - string.Equals("1", value, StringComparison.OrdinalIgnoreCase)) + try { - return true; + return operation.Invoke(); } - - if (string.Equals(bool.FalseString, value, StringComparison.OrdinalIgnoreCase) || - string.Equals("0", value, StringComparison.OrdinalIgnoreCase)) + catch (RequestFailedException e) { - return false; + throw e.ToHttpOperationException(); } + } - return null; + private static double ScoreToCosineSimilarity(double score) + { + // Azure Cognitive Search score formula. The min value is 0.333 for cosine similarity -1. + score = Math.Max(score, 1.0 / 3); + return 2 - 1 / score; + } + + private static double CosineSimilarityToScore(double similarity) + { + return 1 / (2 - similarity); } #endregion diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/Connectors.Memory.AzureCognitiveSearch.csproj b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/Connectors.Memory.AzureCognitiveSearch.csproj index a9e9143a793c..2c5fd3b10aab 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/Connectors.Memory.AzureCognitiveSearch.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/Connectors.Memory.AzureCognitiveSearch.csproj @@ -25,7 +25,7 @@ - + diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/RequestFailedExceptionExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/RequestFailedExceptionExtensions.cs new file mode 100644 index 000000000000..6b1d3d99d598 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/RequestFailedExceptionExtensions.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net; +using Azure; +using Microsoft.SemanticKernel.Diagnostics; + +namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch; + +/// +/// Provides extension methods for the class. +/// +public static class RequestFailedExceptionExtensions +{ + /// + /// Converts a to an . + /// + /// The original . + /// An instance. + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "By design. See comment below.")] + public static HttpOperationException ToHttpOperationException(this RequestFailedException exception) + { + const int NoResponseReceived = 0; + + string? responseContent = null; + + try + { + responseContent = exception.GetRawResponse()?.Content?.ToString(); + } + catch { } // We want to suppress any exceptions that occur while reading the content, ensuring that an HttpOperationException is thrown instead. + + return new HttpOperationException( + exception.Status == NoResponseReceived ? null : (HttpStatusCode?)exception.Status, + responseContent, + exception.Message, + exception); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaBooleanConverter.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaBooleanConverter.cs deleted file mode 100644 index 6c4a9fc8fa42..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaBooleanConverter.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.SemanticKernel.Connectors.Memory.Chroma; - -/// -/// JSON Converter for Chroma boolean values. -/// -public class ChromaBooleanConverter : JsonConverter -{ - /// - public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - if (!reader.TryGetInt16(out short value)) - { - return false; - } - - return Convert.ToBoolean(value); - } - - /// - public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options) - { - writer.WriteNumberValue(Convert.ToDecimal(value)); - } -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaClient.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaClient.cs index d80cfb9833c9..63fdd8ef0b8e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaClient.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.Connectors.Memory.Chroma.Http.ApiSchema; using Microsoft.SemanticKernel.Connectors.Memory.Chroma.Http.ApiSchema.Internal; +using Microsoft.SemanticKernel.Diagnostics; namespace Microsoft.SemanticKernel.Connectors.Memory.Chroma; @@ -27,12 +28,12 @@ public class ChromaClient : IChromaClient /// Initializes a new instance of the class. /// /// Chroma server endpoint URL. - /// Optional logger instance. - public ChromaClient(string endpoint, ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + public ChromaClient(string endpoint, ILoggerFactory? loggerFactory = null) { this._httpClient = new HttpClient(NonDisposableHttpClientHandler.Instance, disposeHandler: false); this._endpoint = endpoint; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(ChromaClient)) : NullLogger.Instance; } /// @@ -40,18 +41,18 @@ public ChromaClient(string endpoint, ILogger? logger = null) /// /// The instance used for making HTTP requests. /// Chroma server endpoint URL. - /// Optional logger instance. - /// Occurs when doesn't have base address and endpoint parameter is not provided. - public ChromaClient(HttpClient httpClient, string? endpoint = null, ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + /// Occurs when doesn't have base address and endpoint parameter is not provided. + public ChromaClient(HttpClient httpClient, string? endpoint = null, ILoggerFactory? loggerFactory = null) { if (string.IsNullOrEmpty(httpClient.BaseAddress?.AbsoluteUri) && string.IsNullOrEmpty(endpoint)) { - throw new ChromaClientException("The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); + throw new SKException("The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); } this._httpClient = httpClient; this._endpoint = endpoint; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(ChromaClient)) : NullLogger.Instance; } /// @@ -106,7 +107,7 @@ public async IAsyncEnumerable ListCollectionsAsync([EnumeratorCancellati } /// - public async Task UpsertEmbeddingsAsync(string collectionId, string[] ids, float[][] embeddings, object[]? metadatas = null, CancellationToken cancellationToken = default) + public async Task UpsertEmbeddingsAsync(string collectionId, string[] ids, ReadOnlyMemory[] embeddings, object[]? metadatas = null, CancellationToken cancellationToken = default) { this._logger.LogDebug("Upserting embeddings to collection with id: {0}", collectionId); @@ -140,7 +141,7 @@ public async Task DeleteEmbeddingsAsync(string collectionId, string[] ids, Cance } /// - public async Task QueryEmbeddingsAsync(string collectionId, float[][] queryEmbeddings, int nResults, string[]? include = null, CancellationToken cancellationToken = default) + public async Task QueryEmbeddingsAsync(string collectionId, ReadOnlyMemory[] queryEmbeddings, int nResults, string[]? include = null, CancellationToken cancellationToken = default) { this._logger.LogDebug("Query embeddings in collection with id: {0}", collectionId); @@ -172,18 +173,20 @@ public async Task QueryEmbeddingsAsync(string collection request.RequestUri = new Uri(new Uri(endpoint), operationName); - HttpResponseMessage response = await this._httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + HttpResponseMessage? response = null; - string responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + string? responseContent = null; try { - response.EnsureSuccessStatusCode(); + response = await this._httpClient.SendWithSuccessCheckAsync(request, cancellationToken).ConfigureAwait(false); + + responseContent = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError(e, "{0} {1} operation failed: {2}, {3}", request.Method.Method, operationName, e.Message, responseContent); - throw new ChromaClientException($"{request.Method.Method} {operationName} operation failed: {e.Message}, {responseContent}", e); + this._logger.LogError(e, "{Method} {Path} operation failed: {Message}, {Response}", request.Method.Method, operationName, e.Message, e.ResponseContent); + throw; } return (response, responseContent); diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaClientException.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaClientException.cs deleted file mode 100644 index 535e7d3a3afb..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaClientException.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Globalization; - -namespace Microsoft.SemanticKernel.Connectors.Memory.Chroma; - -/// -/// Exception to identify issues in class. -/// -public class ChromaClientException : Exception -{ - private const string CollectionDoesNotExistErrorFormat = "Collection {0} does not exist"; - - /// - /// Initializes a new instance of the class. - /// - public ChromaClientException() : base() - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The message that describes the error. - public ChromaClientException(string message) : base(message) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The message that describes the error. - /// Instance of inner exception. - public ChromaClientException(string message, Exception innerException) : base(message, innerException) - { - } - - /// - /// Checks if Chroma API error means that collection does not exist. - /// - /// Collection name. - public bool CollectionDoesNotExistException(string collectionName) => - this.Message.Contains(string.Format(CultureInfo.InvariantCulture, CollectionDoesNotExistErrorFormat, collectionName)); -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaKernelBuilderExtensions.cs index f020d6c1905e..8366997c1ca0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaKernelBuilderExtensions.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using System.ComponentModel; using System.Net.Http; using Microsoft.SemanticKernel.Connectors.Memory.Chroma; @@ -10,6 +12,8 @@ namespace Microsoft.SemanticKernel; /// /// Provides extension methods for the class to configure Chroma memory connector. /// +[Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use ChromaMemoryBuilderExtensions instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class ChromaKernelBuilderExtensions { /// @@ -18,14 +22,16 @@ public static class ChromaKernelBuilderExtensions /// The instance. /// Chroma server endpoint URL. /// Self instance. + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use ChromaMemoryBuilderExtensions.WithChromaMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] public static KernelBuilder WithChromaMemoryStore(this KernelBuilder builder, string endpoint) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory, httpHandlerFactory) => { return new ChromaMemoryStore( - HttpClientProvider.GetHttpClient(parameters.Config, null, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, null, loggerFactory), endpoint, - parameters.Logger); + loggerFactory); }); return builder; @@ -38,16 +44,18 @@ public static KernelBuilder WithChromaMemoryStore(this KernelBuilder builder, st /// The instance used for making HTTP requests. /// Chroma server endpoint URL. If not specified, the base address of the HTTP client is used. /// Self instance. + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use ChromaMemoryBuilderExtensions.WithChromaMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] public static KernelBuilder WithChromaMemoryStore(this KernelBuilder builder, HttpClient httpClient, string? endpoint = null) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory, httpHandlerFactory) => { return new ChromaMemoryStore( - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), endpoint, - parameters.Logger); + loggerFactory); }); return builder; diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryBuilderExtensions.cs new file mode 100644 index 000000000000..cbe98e0a6748 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryBuilderExtensions.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.SemanticKernel.Plugins.Memory; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Chroma; + +/// +/// Provides extension methods for the class to configure Chroma memory connector. +/// +public static class ChromaMemoryBuilderExtensions +{ + /// + /// Registers Chroma memory connector. + /// + /// The instance. + /// Chroma server endpoint URL. + /// Updated Memory builder including Chroma memory connector. + public static MemoryBuilder WithChromaMemoryStore(this MemoryBuilder builder, string endpoint) + { + builder.WithMemoryStore((loggerFactory, httpHandlerFactory) => + { + return new ChromaMemoryStore( + HttpClientProvider.GetHttpClient(httpHandlerFactory, null, loggerFactory), + endpoint, + loggerFactory); + }); + + return builder; + } + + /// + /// Registers Chroma memory connector. + /// + /// The instance. + /// The instance used for making HTTP requests. + /// Chroma server endpoint URL. If not specified, the base address of the HTTP client is used. + /// Updated Memory builder including Chroma memory connector. + public static MemoryBuilder WithChromaMemoryStore( + this MemoryBuilder builder, + HttpClient httpClient, + string? endpoint = null) + { + builder.WithMemoryStore((loggerFactory, httpHandlerFactory) => + { + return new ChromaMemoryStore( + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + endpoint, + loggerFactory); + }); + + return builder; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryStore.cs index 723d54330567..28fa6cc6a063 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryStore.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; @@ -9,10 +11,10 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Chroma.Http.ApiSchema; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Chroma; @@ -25,9 +27,9 @@ public class ChromaMemoryStore : IMemoryStore /// Initializes a new instance of the class. /// /// Chroma server endpoint URL. - /// Optional logger instance. - public ChromaMemoryStore(string endpoint, ILogger? logger = null) - : this(new ChromaClient(endpoint, logger), logger) + /// The to use for logging. If null, no logging will be performed. + public ChromaMemoryStore(string endpoint, ILoggerFactory? loggerFactory = null) + : this(new ChromaClient(endpoint, loggerFactory), loggerFactory) { } @@ -36,9 +38,9 @@ public ChromaMemoryStore(string endpoint, ILogger? logger = null) /// /// The instance used for making HTTP requests. /// Chroma server endpoint URL. - /// Optional logger instance. - public ChromaMemoryStore(HttpClient httpClient, string? endpoint = null, ILogger? logger = null) - : this(new ChromaClient(httpClient, endpoint, logger), logger) + /// The to use for logging. If null, no logging will be performed. + public ChromaMemoryStore(HttpClient httpClient, string? endpoint = null, ILoggerFactory? loggerFactory = null) + : this(new ChromaClient(httpClient, endpoint, loggerFactory), loggerFactory) { } @@ -46,11 +48,11 @@ public ChromaMemoryStore(HttpClient httpClient, string? endpoint = null, ILogger /// Initializes a new instance of the class. /// /// Instance of implementation. - /// Optional logger instance. - public ChromaMemoryStore(IChromaClient client, ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + public ChromaMemoryStore(IChromaClient client, ILoggerFactory? loggerFactory = null) { this._chromaClient = client; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(ChromaMemoryStore)) : NullLogger.Instance; } /// @@ -70,10 +72,10 @@ public async Task DeleteCollectionAsync(string collectionName, CancellationToken { await this._chromaClient.DeleteCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); } - catch (ChromaClientException e) when (e.CollectionDoesNotExistException(collectionName)) + catch (HttpOperationException e) when (VerifyCollectionDoesNotExistMessage(e.ResponseContent, collectionName)) { this._logger.LogError("Cannot delete non-existent collection {0}", collectionName); - throw new ChromaMemoryStoreException($"Cannot delete non-existent collection {collectionName}", e); + throw new SKException($"Cannot delete non-existent collection {collectionName}", e); } } @@ -122,7 +124,7 @@ public IAsyncEnumerable GetCollectionsAsync(CancellationToken cancellati } /// - public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) { var results = this.GetNearestMatchesAsync( collectionName, @@ -138,13 +140,13 @@ public IAsyncEnumerable GetCollectionsAsync(CancellationToken cancellati } /// - public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(string collectionName, Embedding embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(string collectionName, ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) { Verify.NotNullOrWhiteSpace(collectionName); var collection = await this.GetCollectionOrThrowAsync(collectionName, cancellationToken).ConfigureAwait(false); - var queryEmbeddings = new float[][] { embedding.Vector.ToArray() }; + var queryEmbeddings = new[] { embedding }; var nResults = limit; var include = this.GetEmbeddingIncludeTypes(withEmbeddings: withEmbeddings, withDistances: true); @@ -202,13 +204,13 @@ public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IE var recordsLength = recordsArray.Length; var ids = new string[recordsLength]; - var embeddings = new float[recordsLength][]; + var embeddings = new ReadOnlyMemory[recordsLength]; var metadatas = new object[recordsLength]; for (var i = 0; i < recordsLength; i++) { ids[i] = recordsArray[i].Metadata.Id; - embeddings[i] = recordsArray[i].Embedding.Vector.ToArray(); + embeddings[i] = recordsArray[i].Embedding; metadatas[i] = recordsArray[i].Metadata; } @@ -230,16 +232,16 @@ public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IE private readonly IChromaClient _chromaClient; private readonly List _defaultEmbeddingIncludeTypes = new() { IncludeMetadatas }; - private readonly JsonSerializerOptions _jsonSerializerOptions = new() + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() { - Converters = { new ChromaBooleanConverter() } + Converters = { new ReadOnlyMemoryConverter() } }; private async Task GetCollectionOrThrowAsync(string collectionName, CancellationToken cancellationToken) { return await this.GetCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false) ?? - throw new ChromaMemoryStoreException($"Collection {collectionName} does not exist"); + throw new SKException($"Collection {collectionName} does not exist"); } private async Task GetCollectionAsync(string collectionName, CancellationToken cancellationToken) @@ -248,9 +250,9 @@ await this.GetCollectionAsync(collectionName, cancellationToken).ConfigureAwait( { return await this._chromaClient.GetCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); } - catch (ChromaClientException e) when (e.CollectionDoesNotExistException(collectionName)) + catch (HttpOperationException e) when (VerifyCollectionDoesNotExistMessage(e.ResponseContent, collectionName)) { - this._logger.LogError("Collection {0} does not exist", collectionName); + this._logger.LogDebug("Collection {0} does not exist", collectionName); return null; } @@ -305,21 +307,21 @@ private MemoryRecord GetMemoryRecordFromModel(List>? private MemoryRecordMetadata GetMetadataForMemoryRecord(List>? metadatas, int recordIndex) { - var serializedMetadata = metadatas != null ? JsonSerializer.Serialize(metadatas[recordIndex]) : string.Empty; + var serializedMetadata = metadatas != null ? JsonSerializer.Serialize(metadatas[recordIndex], s_jsonSerializerOptions) : string.Empty; return - JsonSerializer.Deserialize(serializedMetadata, this._jsonSerializerOptions) ?? - throw new ChromaMemoryStoreException("Unable to deserialize memory record metadata."); + JsonSerializer.Deserialize(serializedMetadata, ChromaMemoryStore.s_jsonSerializerOptions) ?? + throw new SKException("Unable to deserialize memory record metadata."); } - private Embedding GetEmbeddingForMemoryRecord(List? embeddings, int recordIndex) + private ReadOnlyMemory GetEmbeddingForMemoryRecord(List? embeddings, int recordIndex) { - return embeddings != null ? new Embedding(embeddings[recordIndex]) : Embedding.Empty; + return embeddings != null ? embeddings[recordIndex] : ReadOnlyMemory.Empty; } private double GetSimilarityScore(List? distances, int recordIndex) { - var similarityScore = distances != null ? 1 - distances[recordIndex] : default; + var similarityScore = distances != null ? 1.0 / (1.0 + distances[recordIndex]) : default; if (similarityScore < 0) { @@ -329,5 +331,15 @@ private double GetSimilarityScore(List? distances, int recordIndex) return similarityScore; } + /// + /// Checks if Chroma API error means that collection does not exist. + /// + /// Response content. + /// Collection name. + private static bool VerifyCollectionDoesNotExistMessage(string? responseContent, string collectionName) + { + return responseContent?.Contains(string.Format(CultureInfo.InvariantCulture, "Collection {0} does not exist", collectionName)) ?? false; + } + #endregion } diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryStoreException.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryStoreException.cs deleted file mode 100644 index cef6b78d1247..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaMemoryStoreException.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.Connectors.Memory.Chroma; - -/// -/// Exception to identify issues in class. -/// -public class ChromaMemoryStoreException : Exception -{ - /// - /// Initializes a new instance of the class. - /// - public ChromaMemoryStoreException() : base() - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The message that describes the error. - public ChromaMemoryStoreException(string message) : base(message) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The message that describes the error. - /// Instance of inner exception. - public ChromaMemoryStoreException(string message, Exception innerException) : base(message, innerException) - { - } -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/Connectors.Memory.Chroma.csproj b/dotnet/src/Connectors/Connectors.Memory.Chroma/Connectors.Memory.Chroma.csproj index a012d01d504f..4af82ec792a7 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/Connectors.Memory.Chroma.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/Connectors.Memory.Chroma.csproj @@ -14,7 +14,7 @@ Semantic Kernel - Chroma Connector - Chroma connector for Semantic Kernel skills and semantic memory + Chroma connector for Semantic Kernel plugins and semantic memory @@ -22,7 +22,8 @@ - + + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/Http/ApiSchema/Internal/QueryEmbeddingsRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/Http/ApiSchema/Internal/QueryEmbeddingsRequest.cs index 47c8c5486916..d350d6a14a8f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/Http/ApiSchema/Internal/QueryEmbeddingsRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/Http/ApiSchema/Internal/QueryEmbeddingsRequest.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Net.Http; using System.Text.Json.Serialization; @@ -11,7 +12,7 @@ internal sealed class QueryEmbeddingsRequest public string CollectionId { get; set; } [JsonPropertyName("query_embeddings")] - public float[][] QueryEmbeddings { get; set; } + public ReadOnlyMemory[] QueryEmbeddings { get; set; } [JsonPropertyName("n_results")] public int NResults { get; set; } @@ -19,7 +20,7 @@ internal sealed class QueryEmbeddingsRequest [JsonPropertyName("include")] public string[]? Include { get; set; } - public static QueryEmbeddingsRequest Create(string collectionId, float[][] queryEmbeddings, int nResults, string[]? include = null) + public static QueryEmbeddingsRequest Create(string collectionId, ReadOnlyMemory[] queryEmbeddings, int nResults, string[]? include = null) { return new QueryEmbeddingsRequest(collectionId, queryEmbeddings, nResults, include); } @@ -31,7 +32,7 @@ public HttpRequestMessage Build() #region private ================================================================================ - private QueryEmbeddingsRequest(string collectionId, float[][] queryEmbeddings, int nResults, string[]? include = null) + private QueryEmbeddingsRequest(string collectionId, ReadOnlyMemory[] queryEmbeddings, int nResults, string[]? include = null) { this.CollectionId = collectionId; this.QueryEmbeddings = queryEmbeddings; diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/Http/ApiSchema/Internal/UpsertEmbeddingsRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/Http/ApiSchema/Internal/UpsertEmbeddingsRequest.cs index bda2ca179c50..354c808f7bd0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/Http/ApiSchema/Internal/UpsertEmbeddingsRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/Http/ApiSchema/Internal/UpsertEmbeddingsRequest.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Net.Http; using System.Text.Json.Serialization; @@ -14,12 +15,12 @@ internal sealed class UpsertEmbeddingsRequest public string[] Ids { get; set; } [JsonPropertyName("embeddings")] - public float[][] Embeddings { get; set; } + public ReadOnlyMemory[] Embeddings { get; set; } [JsonPropertyName("metadatas")] public object[]? Metadatas { get; set; } - public static UpsertEmbeddingsRequest Create(string collectionId, string[] ids, float[][] embeddings, object[]? metadatas = null) + public static UpsertEmbeddingsRequest Create(string collectionId, string[] ids, ReadOnlyMemory[] embeddings, object[]? metadatas = null) { return new UpsertEmbeddingsRequest(collectionId, ids, embeddings, metadatas); } @@ -31,7 +32,7 @@ public HttpRequestMessage Build() #region private ================================================================================ - private UpsertEmbeddingsRequest(string collectionId, string[] ids, float[][] embeddings, object[]? metadatas = null) + private UpsertEmbeddingsRequest(string collectionId, string[] ids, ReadOnlyMemory[] embeddings, object[]? metadatas = null) { this.CollectionId = collectionId; this.Ids = ids; diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/IChromaClient.cs b/dotnet/src/Connectors/Connectors.Memory.Chroma/IChromaClient.cs index 9260bde408f3..e650c0b1388d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/IChromaClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/IChromaClient.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -49,7 +50,7 @@ public interface IChromaClient /// Array of embedding vectors. /// Array of embedding metadatas. /// The to monitor for cancellation requests. The default is . - Task UpsertEmbeddingsAsync(string collectionId, string[] ids, float[][] embeddings, object[]? metadatas = null, CancellationToken cancellationToken = default); + Task UpsertEmbeddingsAsync(string collectionId, string[] ids, ReadOnlyMemory[] embeddings, object[]? metadatas = null, CancellationToken cancellationToken = default); /// /// Returns embeddings from specified collection. @@ -78,5 +79,5 @@ public interface IChromaClient /// Array of entities to include in response (e.g. "embeddings", "metadatas" "documents"). For more information see: https://github.com/chroma-core/chroma/blob/main/chromadb/api/types.py /// The to monitor for cancellation requests. The default is . /// Instance of model. - Task QueryEmbeddingsAsync(string collectionId, float[][] queryEmbeddings, int nResults, string[]? include = null, CancellationToken cancellationToken = default); + Task QueryEmbeddingsAsync(string collectionId, ReadOnlyMemory[] queryEmbeddings, int nResults, string[]? include = null, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/Connectors/Connectors.Memory.Chroma/README.md b/dotnet/src/Connectors/Connectors.Memory.Chroma/README.md index 1cf92fd711ba..22706f210e15 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Chroma/README.md +++ b/dotnet/src/Connectors/Connectors.Memory.Chroma/README.md @@ -2,6 +2,8 @@ This assembly contains implementation of Semantic Kernel Memory Store using [Chroma](https://docs.trychroma.com/), open-source embedding database. +**Note:** Chroma connector is verified using Chroma version **0.4.10**. Any higher versions may introduce incompatibility. + ## Quickstart using local Chroma server 1. Clone Chroma: diff --git a/dotnet/src/Connectors/Connectors.Memory.CosmosDB/Connectors.Memory.CosmosDB.csproj b/dotnet/src/Connectors/Connectors.Memory.CosmosDB/Connectors.Memory.CosmosDB.csproj deleted file mode 100644 index 8761054a8de9..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.CosmosDB/Connectors.Memory.CosmosDB.csproj +++ /dev/null @@ -1,29 +0,0 @@ - - - - - Microsoft.SemanticKernel.Connectors.Memory.AzureCosmosDb - $(AssemblyName) - netstandard2.0 - - - - - - - - - Semantic Kernel - Azure Cosmos Db Connector - Azure Cosmos Db connector for Semantic Kernel skills and semantic memory - - - - - - - - - - - - diff --git a/dotnet/src/Connectors/Connectors.Memory.CosmosDB/CosmosMemoryRecord.cs b/dotnet/src/Connectors/Connectors.Memory.CosmosDB/CosmosMemoryRecord.cs deleted file mode 100644 index 1d2d10f0e40f..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.CosmosDB/CosmosMemoryRecord.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCosmosDb; - -/// -/// A Cosmos memory record. -/// -[JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))] -public class CosmosMemoryRecord -{ - /// - /// Unique identifier of the memory record. - /// - public string Id { get; set; } = string.Empty; - - /// - /// Unique identifier of the collection. - /// - public string CollectionId { get; set; } = string.Empty; - - /// - /// Optional timestamp. - /// - public DateTimeOffset? Timestamp { get; set; } - - /// - /// The embedding data as a string. - /// - public string EmbeddingString { get; set; } = string.Empty; - - /// - /// Metadata as a string. - /// - public string MetadataString { get; set; } = string.Empty; -} diff --git a/dotnet/src/Connectors/Connectors.Memory.CosmosDB/CosmosMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.CosmosDB/CosmosMemoryStore.cs deleted file mode 100644 index 76a35a73753b..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.CosmosDB/CosmosMemoryStore.cs +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.Cosmos; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Memory.Collections; - -namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCosmosDb; - -/// -/// An implementation of for Azure Cosmos DB. -/// -/// The Embedding data is saved to the Azure Cosmos DB database container specified in the constructor. -/// The embedding data persists between subsequent instances and has similarity search capability, handled by the client as Azure Cosmos DB is not a vector-native DB. -/// -public sealed class CosmosMemoryStore : IMemoryStore -{ - private Database _database; - private string _databaseName; - private ILogger _logger; - -#pragma warning disable CS8618 // Non-nullable field is uninitialized: Class instance is created and populated via factory method. - private CosmosMemoryStore() - { - } -#pragma warning restore CS8618 // Non-nullable field is uninitialized - - /// - /// Factory method to initialize a new instance of the class. - /// - /// Client with endpoint and authentication to the Azure CosmosDB Account. - /// The name of the database to back the memory store. - /// Optional logger. - /// The to monitor for cancellation requests. The default is . - /// - public static async Task CreateAsync(CosmosClient client, string databaseName, ILogger? logger = null, CancellationToken cancellationToken = default) - { - var newStore = new CosmosMemoryStore(); - - newStore._databaseName = databaseName; - newStore._logger = logger ?? NullLogger.Instance; - var response = await client.CreateDatabaseIfNotExistsAsync(newStore._databaseName, cancellationToken: cancellationToken).ConfigureAwait(false); - - if (response.StatusCode == HttpStatusCode.Created) - { - newStore._logger.LogDebug("Created database {0}", newStore._databaseName); - } - else if (response.StatusCode == HttpStatusCode.OK) - { - newStore._logger.LogDebug("Database {0}", newStore._databaseName); - } - else - { - throw new CosmosException("Database does not exist and was not created", response.StatusCode, 0, newStore._databaseName, 0); - } - - newStore._database = response.Database; - - return newStore; - } - - /// - public IAsyncEnumerable GetCollectionsAsync(CancellationToken cancellationToken = default) - { - // Azure Cosmos DB does not support listing all Containers, this does not break the interface but it is not ideal. - this._logger.LogWarning("Listing all containers is not supported by Azure Cosmos DB, returning empty list."); - - return Enumerable.Empty().ToAsyncEnumerable(); - } - - /// - public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default) - { - var response = await this._database.CreateContainerIfNotExistsAsync(collectionName, "/" + collectionName, cancellationToken: cancellationToken).ConfigureAwait(false); - - if (response.StatusCode == HttpStatusCode.Created) - { - this._logger.LogDebug("Created collection {0}", collectionName); - } - else if (response.StatusCode == HttpStatusCode.OK) - { - this._logger.LogDebug("Collection {0} already exists", collectionName); - } - else - { - throw new CosmosException("Collection does not exist and was not created", response.StatusCode, 0, collectionName, 0); - } - } - - /// - public Task DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default) - { - // Azure Cosmos DB does not support checking if container exists without attempting to create it. - // Note that CreateCollectionIfNotExistsAsync() is idempotent. This does not break the interface but it is not ideal. - return Task.FromResult(false); - } - - /// - public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) - { - var container = this._database.Client.GetContainer(this._databaseName, collectionName); - try - { - await container.DeleteContainerAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - } - catch (CosmosException ex) - { - this._logger.LogError(ex, "Failed to delete collection {0}: {2} - {3}", collectionName, ex.StatusCode, ex.Message); - } - } - - /// - public async Task GetAsync(string collectionName, string key, bool withEmbedding = false, CancellationToken cancellationToken = default) - { - var id = this.ToCosmosFriendlyId(key); - var partitionKey = PartitionKey.None; - - var container = this._database.Client.GetContainer(this._databaseName, collectionName); - MemoryRecord? memoryRecord = null; - - var response = await container.ReadItemAsync(id, partitionKey, cancellationToken: cancellationToken).ConfigureAwait(false); - - if (response == null) - { - this._logger?.LogWarning("Received no get response collection {1}", collectionName); - } - else if (response.StatusCode != HttpStatusCode.OK) - { - this._logger?.LogWarning("Failed to get record from collection {1} with status code {2}", collectionName, response.StatusCode); - } - else - { - var result = response.Resource; - - float[]? vector = withEmbedding ? System.Text.Json.JsonSerializer.Deserialize(result.EmbeddingString) : System.Array.Empty(); - - if (vector != null) - { - memoryRecord = MemoryRecord.FromJsonMetadata( - result.MetadataString, - new Embedding(vector, transferOwnership: true), - result.Id, - result.Timestamp); - } - } - - return memoryRecord; - } - - /// - public async IAsyncEnumerable GetBatchAsync(string collectionName, IEnumerable keys, bool withEmbeddings = false, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - foreach (var key in keys) - { - var record = await this.GetAsync(collectionName, key, withEmbeddings, cancellationToken).ConfigureAwait(false); - - if (record != null) - { - yield return record; - } - } - } - - /// - public async Task UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default) - { - record.Key = this.ToCosmosFriendlyId(record.Metadata.Id); - - var entity = new CosmosMemoryRecord - { - CollectionId = this.ToCosmosFriendlyId(collectionName), - Id = record.Key, - Timestamp = record.Timestamp, - EmbeddingString = System.Text.Json.JsonSerializer.Serialize(record.Embedding.Vector), - MetadataString = record.GetSerializedMetadata() - }; - - var container = this._database.Client.GetContainer(this._databaseName, collectionName); - - var response = await container.UpsertItemAsync(entity, cancellationToken: cancellationToken).ConfigureAwait(false); - - if (response.StatusCode is HttpStatusCode.OK or HttpStatusCode.Created) - { - this._logger.LogDebug("Upserted item to collection {0}", collectionName); - } - else - { - throw new CosmosException("Unable to upsert item collection", response.StatusCode, 0, collectionName, 0); - } - - return record.Key; - } - - /// - public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IEnumerable records, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - foreach (var r in records) - { - yield return await this.UpsertAsync(collectionName, r, cancellationToken).ConfigureAwait(false); - } - } - - /// - public async Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default) - { - var container = this._database.Client.GetContainer(this._databaseName, collectionName); - var response = await container.DeleteItemAsync( - key, - PartitionKey.None, - cancellationToken: cancellationToken).ConfigureAwait(false); - - if (response.StatusCode == HttpStatusCode.OK) - { - this._logger.LogDebug("Record deleted from {0}", collectionName); - } - else - { - throw new CosmosException("Unable to delete record", response.StatusCode, 0, collectionName, 0); - } - } - - /// - public async Task RemoveBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default) - { - await Task.WhenAll(keys.Select(k => this.RemoveAsync(collectionName, k, cancellationToken))).ConfigureAwait(false); - } - - /// - public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( - string collectionName, - Embedding embedding, - int limit, - double minRelevanceScore = 0, - bool withEmbeddings = false, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - { - if (limit <= 0) - { - yield break; - } - - var collectionMemories = new List(); - TopNCollection embeddings = new(limit); - - await foreach (var record in this.GetAllAsync(collectionName, cancellationToken)) - { - if (record != null) - { - double similarity = embedding - .AsReadOnlySpan() - .CosineSimilarity(record.Embedding.AsReadOnlySpan()); - if (similarity >= minRelevanceScore) - { - var entry = withEmbeddings ? record : MemoryRecord.FromMetadata(record.Metadata, Embedding.Empty, record.Key, record.Timestamp); - embeddings.Add(new(entry, similarity)); - } - } - } - - embeddings.SortByScore(); - - foreach (var item in embeddings) - { - yield return (item.Value, item.Score.Value); - } - } - } - - /// - public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding embedding, double minRelevanceScore = 0, bool withEmbedding = false, - CancellationToken cancellationToken = default) - { - return await this.GetNearestMatchesAsync( - collectionName: collectionName, - embedding: embedding, - limit: 1, - minRelevanceScore: minRelevanceScore, - withEmbeddings: withEmbedding, - cancellationToken: cancellationToken).FirstOrDefaultAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - } - - private async IAsyncEnumerable GetAllAsync(string collectionName, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - var container = this._database.Client.GetContainer(this._databaseName, collectionName); - var query = new QueryDefinition("SELECT * FROM c"); - - var iterator = container.GetItemQueryIterator(query); - - while (iterator.HasMoreResults) //read all result in batch - { - var items = await iterator.ReadNextAsync(cancellationToken).ConfigureAwait(false); - - foreach (var item in items) - { - var vector = System.Text.Json.JsonSerializer.Deserialize(item.EmbeddingString); - - if (vector != null) - { - yield return MemoryRecord.FromJsonMetadata( - item.MetadataString, - new Embedding(vector, transferOwnership: true), - item.Id, - item.Timestamp); - } - } - } - } - - private string ToCosmosFriendlyId(string id) - { - return $"{id.Trim().Replace(' ', '-').Replace('/', '_').Replace('\\', '_').Replace('?', '_').Replace('#', '_').ToUpperInvariant()}"; - } -} diff --git a/dotnet/src/Connectors/Connectors.Memory.DuckDB/Connectors.Memory.DuckDB.csproj b/dotnet/src/Connectors/Connectors.Memory.DuckDB/Connectors.Memory.DuckDB.csproj index 7b8a15313c8a..6b94fef2bb35 100644 --- a/dotnet/src/Connectors/Connectors.Memory.DuckDB/Connectors.Memory.DuckDB.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.DuckDB/Connectors.Memory.DuckDB.csproj @@ -14,7 +14,7 @@ Semantic Kernel - DuckDB Connector - DuckDB connector for Semantic Kernel skills and semantic memory + DuckDB connector for Semantic Kernel plugins and semantic memory @@ -23,7 +23,7 @@ - + diff --git a/dotnet/src/Connectors/Connectors.Memory.DuckDB/Database.cs b/dotnet/src/Connectors/Connectors.Memory.DuckDB/Database.cs index d58c933b89ec..0883f4f2e3f9 100644 --- a/dotnet/src/Connectors/Connectors.Memory.DuckDB/Database.cs +++ b/dotnet/src/Connectors/Connectors.Memory.DuckDB/Database.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; using System.Threading; @@ -18,13 +20,25 @@ internal struct DatabaseEntry public string EmbeddingString { get; set; } public string? Timestamp { get; set; } + + public float Score { get; set; } } internal sealed class Database { private const string TableName = "SKMemoryTable"; - public Database() { } + public Task CreateFunctionsAsync(DuckDBConnection conn, CancellationToken cancellationToken) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = @" + CREATE OR REPLACE MACRO cosine_similarity(a,b) AS (select sum (xy) from (select x * y as xy from (select UNNEST(a) as x, UNNEST(b) as y))) / sqrt(list_aggregate(list_transform(a, x -> x * x), 'sum') * list_aggregate(list_transform(b, x -> x * x), 'sum')); + CREATE OR REPLACE MACRO split_string_of_numbers(t) AS regexp_extract_all(regexp_replace(t,'(\[|\])', '', 'g'), '([+-]?([0-9]*[.])?[0-9]+)(\s*;\s*)?',1); + CREATE OR REPLACE MACRO number_vector_decoder(t) AS list_transform(split_string_of_numbers(t), x -> cast(x AS double)); + CREATE OR REPLACE MACRO encode_number_vector(t) AS concat('[',list_aggregate(list_transform(t, x -> cast(x AS string)), 'string_agg', '; '),']'); + "; + return cmd.ExecuteNonQueryAsync(cancellationToken); + } public Task CreateTableAsync(DuckDBConnection conn, CancellationToken cancellationToken = default) { @@ -34,7 +48,7 @@ public Task CreateTableAsync(DuckDBConnection conn, CancellationToken cancellati collection TEXT, key TEXT, metadata TEXT, - embedding TEXT, + embedding FLOAT[], timestamp TEXT, PRIMARY KEY(collection, key))"; return cmd.ExecuteNonQueryAsync(cancellationToken); @@ -50,27 +64,32 @@ public async Task CreateCollectionAsync(DuckDBConnection conn, string collection using var cmd = conn.CreateCommand(); cmd.CommandText = $@" - INSERT INTO {TableName} VALUES (?1,?2,?3,?4,?5 ); "; + INSERT INTO {TableName} VALUES (?1,?2,?3, [], ?4 ); "; cmd.Parameters.Add(new DuckDBParameter(collectionName)); cmd.Parameters.Add(new DuckDBParameter(string.Empty)); cmd.Parameters.Add(new DuckDBParameter(string.Empty)); cmd.Parameters.Add(new DuckDBParameter(string.Empty)); - cmd.Parameters.Add(new DuckDBParameter(string.Empty)); await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } + private static string EncodeFloatArrayToString(float[]? data) + { + var dataArrayString = $"[{string.Join(", ", (data ?? Array.Empty()).Select(n => n.ToString("F10", CultureInfo.InvariantCulture)))}]"; + return dataArrayString; + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "Internal method serializing array of float and numbers")] public async Task UpdateOrInsertAsync(DuckDBConnection conn, - string collection, string key, string? metadata, string? embedding, string? timestamp, CancellationToken cancellationToken = default) + string collection, string key, string? metadata, float[]? embedding, string? timestamp, CancellationToken cancellationToken = default) { + await this.DeleteAsync(conn, collection, key, cancellationToken).ConfigureAwait(true); + var embeddingArrayString = EncodeFloatArrayToString(embedding ?? Array.Empty()); using var cmd = conn.CreateCommand(); - cmd.CommandText = $@" - INSERT INTO {TableName} VALUES(?1, ?2, ?3, ?4, ?5) - ON CONFLICT (collection, key) DO UPDATE SET metadata=?3, embedding=?4, timestamp=?5; "; + cmd.CommandText = $"INSERT INTO {TableName} VALUES(?1, ?2, ?3, {embeddingArrayString}, ?4)"; cmd.Parameters.Add(new DuckDBParameter(collection)); cmd.Parameters.Add(new DuckDBParameter(key)); cmd.Parameters.Add(new DuckDBParameter(metadata ?? string.Empty)); - cmd.Parameters.Add(new DuckDBParameter(embedding ?? string.Empty)); cmd.Parameters.Add(new DuckDBParameter(timestamp ?? string.Empty)); await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } @@ -98,14 +117,23 @@ SELECT DISTINCT collection } } - public async IAsyncEnumerable ReadAllAsync(DuckDBConnection conn, + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "Internal method serializing array of float and numbers")] + public async IAsyncEnumerable GetNearestMatchesAsync( + DuckDBConnection conn, string collectionName, + float[]? embedding, + int limit, + double minRelevanceScore = 0, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + var embeddingArrayString = EncodeFloatArrayToString(embedding ?? Array.Empty()); + using var cmd = conn.CreateCommand(); cmd.CommandText = $@" - SELECT * FROM {TableName} - WHERE collection=?1;"; + SELECT key, metadata, timestamp, cast(embedding as string) as embeddingAsString, cast(cosine_similarity(embedding,{embeddingArrayString}) as FLOAT) as score FROM {TableName} + WHERE collection=?1 AND score >= {minRelevanceScore.ToString("F12", CultureInfo.InvariantCulture)} + ORDER BY score DESC + LIMIT {limit};"; cmd.Parameters.Add(new DuckDBParameter(collectionName)); using var dataReader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); @@ -116,10 +144,19 @@ public async IAsyncEnumerable ReadAllAsync(DuckDBConnection conn, { continue; } + string metadata = dataReader.GetString("metadata"); - string embedding = dataReader.GetString("embedding"); + string embeddingAsString = dataReader.GetString("embeddingAsString"); string timestamp = dataReader.GetString("timestamp"); - yield return new DatabaseEntry() { Key = key, MetadataString = metadata, EmbeddingString = embedding, Timestamp = timestamp }; + float score = dataReader.GetFloat("score"); + yield return new DatabaseEntry + { + Key = key, + MetadataString = metadata, + EmbeddingString = embeddingAsString, + Timestamp = timestamp, + Score = score + }; } } @@ -130,7 +167,7 @@ public async IAsyncEnumerable ReadAllAsync(DuckDBConnection conn, { using var cmd = conn.CreateCommand(); cmd.CommandText = $@" - SELECT * FROM {TableName} + SELECT metadata, timestamp, cast(embedding as string) as embeddingAsString FROM {TableName} WHERE collection=?1 AND key=?2; "; cmd.Parameters.Add(new DuckDBParameter(collectionName)); @@ -140,13 +177,13 @@ public async IAsyncEnumerable ReadAllAsync(DuckDBConnection conn, if (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) { string metadata = dataReader.GetString(dataReader.GetOrdinal("metadata")); - string embedding = dataReader.GetString(dataReader.GetOrdinal("embedding")); + string embeddingAsString = dataReader.GetString(dataReader.GetOrdinal("embeddingAsString")); string timestamp = dataReader.GetString(dataReader.GetOrdinal("timestamp")); - return new DatabaseEntry() + return new DatabaseEntry { Key = key, MetadataString = metadata, - EmbeddingString = embedding, + EmbeddingString = embeddingAsString, Timestamp = timestamp }; } @@ -175,15 +212,4 @@ DELETE FROM {TableName} cmd.Parameters.Add(new DuckDBParameter(key)); return cmd.ExecuteNonQueryAsync(cancellationToken); } - - public Task DeleteEmptyAsync(DuckDBConnection conn, string collectionName, CancellationToken cancellationToken = default) - { - using var cmd = conn.CreateCommand(); - cmd.CommandText = $@" - DELETE FROM {TableName} - WHERE collection=?1 - AND key IS NULL"; - cmd.Parameters.Add(new DuckDBParameter(collectionName)); - return cmd.ExecuteNonQueryAsync(cancellationToken); - } } diff --git a/dotnet/src/Connectors/Connectors.Memory.DuckDB/DuckDBExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.DuckDB/DuckDBExtensions.cs index 8724a64d0da8..ea7f25be415c 100644 --- a/dotnet/src/Connectors/Connectors.Memory.DuckDB/DuckDBExtensions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.DuckDB/DuckDBExtensions.cs @@ -11,4 +11,10 @@ public static string GetString(this DbDataReader reader, string fieldName) int ordinal = reader.GetOrdinal(fieldName); return reader.GetString(ordinal); } + + public static float GetFloat(this DbDataReader reader, string fieldName) + { + int ordinal = reader.GetOrdinal(fieldName); + return reader.GetFloat(ordinal); + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.DuckDB/DuckDBMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.DuckDB/DuckDBMemoryStore.cs index 3e79b72d1be9..b805129a4538 100644 --- a/dotnet/src/Connectors/Connectors.Memory.DuckDB/DuckDBMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.DuckDB/DuckDBMemoryStore.cs @@ -9,10 +9,8 @@ using System.Threading; using System.Threading.Tasks; using DuckDB.NET.Data; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Memory.Collections; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.DuckDB; @@ -22,7 +20,7 @@ namespace Microsoft.SemanticKernel.Connectors.Memory.DuckDB; /// The data is saved to a database file, specified in the constructor. /// The data persists between subsequent instances. Only one instance may access the file at a time. /// The caller is responsible for deleting the file. -public class DuckDBMemoryStore : IMemoryStore, IDisposable +public sealed class DuckDBMemoryStore : IMemoryStore, IDisposable { /// /// Connect a DuckDB database @@ -32,7 +30,7 @@ public class DuckDBMemoryStore : IMemoryStore, IDisposable public static async Task ConnectAsync(string filename, CancellationToken cancellationToken = default) { - var memoryStore = new DuckDBMemoryStore($"Data Source={filename}"); + var memoryStore = new DuckDBMemoryStore(filename); return await InitialiseMemoryStoreAsync(memoryStore, cancellationToken).ConfigureAwait(false); } @@ -40,11 +38,10 @@ public static async Task ConnectAsync(string filename, /// Connect a in memory DuckDB database /// /// The to monitor for cancellation requests. The default is . - public static async Task ConnectAsync( + public static Task ConnectAsync( CancellationToken cancellationToken = default) { - var memoryStore = new DuckDBMemoryStore(":memory:"); - return await InitialiseMemoryStoreAsync(memoryStore, cancellationToken).ConfigureAwait(false); + return ConnectAsync(":memory:", cancellationToken); } /// @@ -141,7 +138,7 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke /// public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, @@ -153,33 +150,26 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke } var collectionMemories = new List(); - TopNCollection embeddings = new(limit); + List<(MemoryRecord Record, double Score)> embeddings = new(); - await foreach (var record in this.GetAllAsync(collectionName, cancellationToken)) + await foreach (var dbEntry in this._dbConnector.GetNearestMatchesAsync(this._dbConnection, collectionName, embedding.ToArray(), limit, minRelevanceScore, cancellationToken)) { - if (record != null) - { - double similarity = embedding - .AsReadOnlySpan() - .CosineSimilarity(record.Embedding.AsReadOnlySpan()); - if (similarity >= minRelevanceScore) - { - var entry = withEmbeddings ? record : MemoryRecord.FromMetadata(record.Metadata, Embedding.Empty, record.Key, record.Timestamp); - embeddings.Add(new(entry, similarity)); - } - } + var entry = MemoryRecord.FromJsonMetadata( + json: dbEntry.MetadataString, + withEmbeddings ? JsonSerializer.Deserialize>(dbEntry.EmbeddingString, s_jsonSerializerOptions) : Array.Empty(), + dbEntry.Key, + ParseTimestamp(dbEntry.Timestamp)); + embeddings.Add(new(entry, dbEntry.Score)); } - embeddings.SortByScore(); - - foreach (var item in embeddings) + foreach (var item in embeddings.OrderByDescending(l => l.Score).Take(limit)) { - yield return (item.Value, item.Score.Value); + yield return (item.Record, item.Score); } } /// - public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding embedding, double minRelevanceScore = 0, bool withEmbedding = false, + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) { return await this.GetNearestMatchesAsync( @@ -200,7 +190,11 @@ public void Dispose() #region protected ================================================================================ - protected virtual void Dispose(bool disposing) + /// + /// Disposes the resources used by the instance. + /// + /// True to release both managed and unmanaged resources; false to release only unmanaged resources. + private void Dispose(bool disposing) { if (!this._disposedValue) { @@ -226,6 +220,7 @@ private static async Task InitialiseMemoryStoreAsync(DuckDBMe { await memoryStore._dbConnection.OpenAsync(cancellationToken).ConfigureAwait(false); await memoryStore._dbConnector.CreateTableAsync(memoryStore._dbConnection, cancellationToken).ConfigureAwait(false); + await memoryStore._dbConnector.CreateFunctionsAsync(memoryStore._dbConnection, cancellationToken).ConfigureAwait(false); return memoryStore; } @@ -240,16 +235,6 @@ private DuckDBMemoryStore(string filename) this._disposedValue = false; } - /// - /// Constructor - /// - private DuckDBMemoryStore() - { - this._dbConnector = new Database(); - this._dbConnection = new DuckDBConnection("Data Source=:memory:;"); - this._disposedValue = false; - } - /// /// Constructor /// @@ -277,22 +262,6 @@ private DuckDBMemoryStore(DuckDBConnection connection) return null; } - private async IAsyncEnumerable GetAllAsync(string collectionName, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - // delete empty entry in the database if it exists (see CreateCollection) - await this._dbConnector.DeleteEmptyAsync(this._dbConnection, collectionName, cancellationToken).ConfigureAwait(false); - - await foreach (DatabaseEntry dbEntry in this._dbConnector.ReadAllAsync(this._dbConnection, collectionName, cancellationToken)) - { - var dbEntryEmbeddingString = dbEntry.EmbeddingString; - Embedding? vector = JsonSerializer.Deserialize>(dbEntryEmbeddingString); - - var record = MemoryRecord.FromJsonMetadata(dbEntry.MetadataString, vector, dbEntry.Key, ParseTimestamp(dbEntry.Timestamp)); - - yield return record; - } - } - private async Task InternalUpsertAsync(DuckDBConnection connection, string collectionName, MemoryRecord record, CancellationToken cancellationToken) { record.Key = record.Metadata.Id; @@ -301,7 +270,7 @@ await this._dbConnector.UpdateOrInsertAsync(conn: connection, collection: collectionName, key: record.Key, metadata: record.GetSerializedMetadata(), - embedding: JsonSerializer.Serialize(record.Embedding), + embedding: record.Embedding.ToArray(), timestamp: ToTimestampString(record.Timestamp), cancellationToken: cancellationToken).ConfigureAwait(false); @@ -322,17 +291,26 @@ await this._dbConnector.UpdateOrInsertAsync(conn: connection, { return MemoryRecord.FromJsonMetadata( json: entry.Value.MetadataString, - JsonSerializer.Deserialize>(entry.Value.EmbeddingString), + JsonSerializer.Deserialize>(entry.Value.EmbeddingString, s_jsonSerializerOptions), entry.Value.Key, ParseTimestamp(entry.Value.Timestamp)); } return MemoryRecord.FromJsonMetadata( json: entry.Value.MetadataString, - Embedding.Empty, + ReadOnlyMemory.Empty, entry.Value.Key, ParseTimestamp(entry.Value.Timestamp)); } + private static readonly JsonSerializerOptions s_jsonSerializerOptions = CreateSerializerOptions(); + + private static JsonSerializerOptions CreateSerializerOptions() + { + var jso = new JsonSerializerOptions(); + jso.Converters.Add(new ReadOnlyMemoryConverter()); + return jso; + } + #endregion } diff --git a/dotnet/src/Connectors/Connectors.Memory.Kusto/Connectors.Memory.Kusto.csproj b/dotnet/src/Connectors/Connectors.Memory.Kusto/Connectors.Memory.Kusto.csproj new file mode 100644 index 000000000000..bb3ee4b14b04 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Kusto/Connectors.Memory.Kusto.csproj @@ -0,0 +1,31 @@ + + + + Microsoft.SemanticKernel.Connectors.Memory.Kusto + Microsoft.SemanticKernel.Connectors.Memory.Kusto + netstandard2.0 + + + NU5104 + + + + + + + + + Microsoft.SemanticKernel.Connectors.Memory.Kusto + Semantic Kernel - Azure Data Explorer (Kusto) Semantic Memory + Azure Data Explorer (Kusto) Semantic Memory connector for Semantic Kernel + + + + + + + + + + + diff --git a/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoMemoryRecord.cs b/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoMemoryRecord.cs new file mode 100644 index 000000000000..a7e8783f06f1 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoMemoryRecord.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; +using Kusto.Cloud.Platform.Utils; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Text; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Kusto; + +/// +/// Kusto memory record entity. +/// +public sealed class KustoMemoryRecord +{ + /// + /// Entity key. + /// + public string Key { get; set; } + + /// + /// Metadata associated with memory entity. + /// + public MemoryRecordMetadata Metadata { get; set; } + + /// + /// Source content embedding. + /// + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Embedding { get; set; } + + /// + /// Optional timestamp. + /// + public DateTimeOffset? Timestamp { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// Instance of . + public KustoMemoryRecord(MemoryRecord record) : this(record.Key, record.Metadata, record.Embedding, record.Timestamp) { } + + /// + /// Initializes a new instance of the class. + /// + /// Entity key. + /// Metadata associated with memory entity. + /// Source content embedding. + /// Optional timestamp. + public KustoMemoryRecord(string key, MemoryRecordMetadata metadata, ReadOnlyMemory embedding, DateTimeOffset? timestamp = null) + { + this.Key = key; + this.Metadata = metadata; + this.Embedding = embedding; + this.Timestamp = timestamp; + } + + /// + /// Initializes a new instance of the class. + /// + /// Entity key. + /// Serialized metadata associated with memory entity. + /// Source content embedding. + /// Optional timestamp. + public KustoMemoryRecord(string key, string metadata, string? embedding, string? timestamp = null) + { + this.Key = key; + this.Metadata = KustoSerializer.DeserializeMetadata(metadata); + this.Embedding = KustoSerializer.DeserializeEmbedding(embedding); + this.Timestamp = KustoSerializer.DeserializeDateTimeOffset(timestamp); + } + + /// + /// Returns instance of mapped . + /// + public MemoryRecord ToMemoryRecord() + { + return new MemoryRecord(this.Metadata, this.Embedding, this.Key, this.Timestamp); + } + + /// + /// Writes properties of instance to stream using . + /// + /// Instance of to write properties to stream. + public void WriteToCsvStream(CsvWriter streamWriter) + { + var jsonifiedMetadata = KustoSerializer.SerializeMetadata(this.Metadata); + var jsonifiedEmbedding = KustoSerializer.SerializeEmbedding(this.Embedding); + var isoFormattedDate = KustoSerializer.SerializeDateTimeOffset(this.Timestamp); + + streamWriter.WriteField(this.Key); + streamWriter.WriteField(jsonifiedMetadata); + streamWriter.WriteField(jsonifiedEmbedding); + streamWriter.WriteField(isoFormattedDate); + streamWriter.CompleteRecord(); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoMemoryStore.cs new file mode 100644 index 000000000000..bab93a542d99 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoMemoryStore.cs @@ -0,0 +1,419 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Kusto.Cloud.Platform.Utils; +using Kusto.Data; +using Kusto.Data.Common; +using Kusto.Data.Net.Client; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Memory; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Kusto; + +/// +/// An implementation of backed by a Kusto database. +/// +/// +/// The embedded data is saved to the Kusto database specified in the constructor. +/// Similarity search capability is provided through a cosine similarity function (added on first search operation). Use Kusto's "Table" to implement "Collection". +/// +public class KustoMemoryStore : IMemoryStore, IDisposable +{ + /// + /// Initializes a new instance of the class. + /// + /// Kusto Admin Client. + /// Kusto Query Client. + /// The database used for the tables. + public KustoMemoryStore(ICslAdminProvider cslAdminProvider, ICslQueryProvider cslQueryProvider, string database) + { + this._database = database; + this._queryClient = cslQueryProvider; + this._adminClient = cslAdminProvider; + + this._searchInitialized = false; + this._disposer = new Disposer(nameof(KustoMemoryStore), nameof(KustoMemoryStore)); + } + + /// + /// Initializes a new instance of the class. + /// + /// Kusto Connection String Builder. + /// The database used for the tables. + public KustoMemoryStore(KustoConnectionStringBuilder builder, string database) + : this(KustoClientFactory.CreateCslAdminProvider(builder), KustoClientFactory.CreateCslQueryProvider(builder), database) + { + // Dispose resources provided by this class + this._disposer.Add(this._queryClient); + this._disposer.Add(this._adminClient); + } + + /// + public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default) + { + using var resp = await this._adminClient + .ExecuteControlCommandAsync( + this._database, + CslCommandGenerator.GenerateTableCreateCommand(new TableSchema(GetTableName(collectionName, normalized: false), s_collectionColumns)), + GetClientRequestProperties() + ).ConfigureAwait(false); + } + + /// + public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) + { + using var resp = await this._adminClient + .ExecuteControlCommandAsync( + this._database, + CslCommandGenerator.GenerateTableDropCommand(GetTableName(collectionName, normalized: false)), + GetClientRequestProperties() + ).ConfigureAwait(false); + } + + /// + public async Task DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default) + { + var command = CslCommandGenerator.GenerateTablesShowCommand() + $" | where TableName == '{GetTableName(collectionName, normalized: false)}' | project TableName"; + var result = await this._adminClient + .ExecuteControlCommandAsync( + this._database, + command, + GetClientRequestProperties() + ).ConfigureAwait(false); + + return result.Count() == 1; + } + + /// + public async Task GetAsync(string collectionName, string key, bool withEmbedding = false, CancellationToken cancellationToken = default) + { + var result = this.GetBatchAsync(collectionName, new[] { key }, withEmbedding, cancellationToken); + return await result.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable GetBatchAsync( + string collectionName, + IEnumerable keys, + bool withEmbeddings = false, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var inClauseValue = string.Join(",", keys.Select(k => $"'{k}'")); + var query = $"{this.GetBaseQuery(collectionName)} " + + $"| where Key in ({inClauseValue}) " + + "| project " + + $"{s_keyColumn.Name}, " + + $"{s_metadataColumn.Name}=tostring({s_metadataColumn.Name}), " + + $"{s_timestampColumn.Name}, " + + $"{s_embeddingColumn.Name}=tostring({s_embeddingColumn.Name})"; + + if (!withEmbeddings) + { + // easiest way to ignore embeddings + query += " | extend Embedding = ''"; + } + + using var reader = await this._queryClient + .ExecuteQueryAsync( + this._database, + query, + GetClientRequestProperties(), + cancellationToken + ).ConfigureAwait(false); + + while (reader.Read()) + { + var key = reader.GetString(0); + var metadata = reader.GetString(1); + var timestamp = !reader.IsDBNull(2) ? reader.GetString(2) : null; + var embedding = withEmbeddings ? reader.GetString(3) : default; + + var kustoRecord = new KustoMemoryRecord(key, metadata, embedding, timestamp); + + yield return kustoRecord.ToMemoryRecord(); + } + } + + /// + public async IAsyncEnumerable GetCollectionsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var result = await this._adminClient + .ExecuteControlCommandAsync( + this._database, + CslCommandGenerator.GenerateTablesShowCommand(), + GetClientRequestProperties() + ).ConfigureAwait(false); + + foreach (var item in result) + { + yield return GetCollectionName(item.TableName); + } + } + + /// + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync( + string collectionName, + ReadOnlyMemory embedding, + double minRelevanceScore = 0, + bool withEmbedding = false, + CancellationToken cancellationToken = default) + { + var result = this.GetNearestMatchesAsync(collectionName, embedding, 1, minRelevanceScore, withEmbedding, cancellationToken); + return await result.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( + string collectionName, + ReadOnlyMemory embedding, + int limit, + double minRelevanceScore = 0, + bool withEmbeddings = false, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + this.InitializeVectorFunctions(); + + var similarityQuery = $"{this.GetBaseQuery(collectionName)} | extend similarity=series_cosine_similarity_fl('{KustoSerializer.SerializeEmbedding(embedding)}', {s_embeddingColumn.Name}, 1, 1)"; + + if (minRelevanceScore != 0) + { + similarityQuery += $" | where similarity > {minRelevanceScore}"; + } + + similarityQuery += $" | top {limit} by similarity desc"; + + // reorder to make it easier to ignore the embedding (key, metadata, timestamp, similarity, embedding) + // Using tostring to make it easier to parse the result. There are probably better ways we should explore. + similarityQuery += "| project " + + $"{s_keyColumn.Name}, " + + $"{s_metadataColumn.Name}=tostring({s_metadataColumn.Name}), " + + $"{s_timestampColumn.Name}, " + + "similarity, " + + $"{s_embeddingColumn.Name}=tostring({s_embeddingColumn.Name})"; + + if (!withEmbeddings) + { + similarityQuery += $" | project-away {s_embeddingColumn.Name} "; + } + + using var reader = await this._queryClient + .ExecuteQueryAsync( + this._database, + similarityQuery, + GetClientRequestProperties(), + cancellationToken + ).ConfigureAwait(false); + + while (reader.Read()) + { + var key = reader.GetString(0); + var metadata = reader.GetString(1); + var timestamp = !reader.IsDBNull(2) ? reader.GetString(2) : null; + var similarity = reader.GetDouble(3); + var recordEmbedding = withEmbeddings ? reader.GetString(4) : default; + + var kustoRecord = new KustoMemoryRecord(key, metadata, recordEmbedding, timestamp); + + yield return (kustoRecord.ToMemoryRecord(), similarity); + } + } + + /// + public Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default) + => this.RemoveBatchAsync(collectionName, new[] { key }, cancellationToken); + + /// + public async Task RemoveBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default) + { + if (keys != null) + { + var keysString = string.Join(",", keys.Select(k => $"'{k}'")); + using var resp = await this._adminClient + .ExecuteControlCommandAsync( + this._database, + CslCommandGenerator.GenerateDeleteTableRecordsCommand(GetTableName(collectionName), $"{GetTableName(collectionName)} | where Key in ({keysString})"), + GetClientRequestProperties() + ).ConfigureAwait(false); + } + } + + /// + public async Task UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default) + { + var result = this.UpsertBatchAsync(collectionName, new[] { record }, cancellationToken); + return await result.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) ?? string.Empty; + } + + /// + public async IAsyncEnumerable UpsertBatchAsync( + string collectionName, + IEnumerable records, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // In Kusto, upserts don't exist because it operates as an append-only store. + // Nevertheless, given that we have a straightforward primary key (PK), we can simply insert a new record. + // Our query always selects the latest row of that PK. + // An interesting scenario arises when performing deletion after many "upserts". + // This could turn out to be a heavy operation since, in theory, we might need to remove many outdated versions. + // Additionally, deleting these records equates to a "soft delete" operation. + // For simplicity, and under the assumption that upserts are relatively rare in most systems, + // we will overlook the potential accumulation of "garbage" records. + // Kusto is generally efficient with handling large volumes of data. + using var stream = new MemoryStream(); + using var streamWriter = new StreamWriter(stream); + var csvWriter = new FastCsvWriter(streamWriter); + + var keys = new List(); + var recordsAsList = records.ToList(); + + for (var i = 0; i < recordsAsList.Count; i++) + { + var record = recordsAsList[i]; + record.Key = record.Metadata.Id; + keys.Add(record.Key); + new KustoMemoryRecord(record).WriteToCsvStream(csvWriter); + } + + csvWriter.Flush(); + stream.Seek(0, SeekOrigin.Begin); + + var command = CslCommandGenerator.GenerateTableIngestPushCommand(GetTableName(collectionName), false, stream); + await this._adminClient + .ExecuteControlCommandAsync( + this._database, + command, + GetClientRequestProperties() + ).ConfigureAwait(false); + + foreach (var key in keys) + { + yield return key; + } + } + + /// + /// Disposes the instance. + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Disposes the resources used by the instance. + /// + /// True to release both managed and unmanaged resources; false to release only unmanaged resources. + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._disposer.Dispose(); + } + } + + #region private ================================================================================ + + private readonly Disposer _disposer; + private readonly object _lock = new(); + + private readonly string _database; + + private static ClientRequestProperties GetClientRequestProperties() => new() + { + Application = Telemetry.HttpUserAgent, + }; + + private bool _searchInitialized; + + private readonly ICslQueryProvider _queryClient; + private readonly ICslAdminProvider _adminClient; + + private static readonly ColumnSchema s_keyColumn = new("Key", typeof(string).FullName); + private static readonly ColumnSchema s_metadataColumn = new("Metadata", typeof(object).FullName); + private static readonly ColumnSchema s_embeddingColumn = new("Embedding", typeof(object).FullName); + private static readonly ColumnSchema s_timestampColumn = new("Timestamp", typeof(DateTime).FullName); + + private static readonly ColumnSchema[] s_collectionColumns = new ColumnSchema[] + { + s_keyColumn, + s_metadataColumn, + s_embeddingColumn, + s_timestampColumn + }; + + /// + /// Converts collection name to Kusto table name. + /// + /// + /// Kusto escaping rules for table names: https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/schema-entities/entity-names#identifier-quoting + /// + /// Kusto table name. + /// Boolean flag that indicates if table name normalization is needed. + private static string GetTableName(string collectionName, bool normalized = true) + => normalized ? CslSyntaxGenerator.NormalizeTableName(collectionName) : collectionName; + + /// + /// Converts Kusto table name to collection name. + /// + /// + /// Kusto escaping rules for table names: https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/schema-entities/entity-names#identifier-quoting + /// + /// Kusto table name. + private static string GetCollectionName(string tableName) + => tableName.Replace("['", "").Replace("']", ""); + + /// + /// Returns base Kusto query. + /// + /// + /// Kusto is an append-only store. Although deletions are possible, they are highly discourged, + /// and should only be used in rare cases (see: https://learn.microsoft.com/en-us/azure/data-explorer/kusto/concepts/data-soft-delete#use-cases). + /// As such, the recommended approach for dealing with row updates is versioning. + /// An easy way to achieve this is by using the ingestion time of the record (insertion time). + /// + /// Collection name. + private string GetBaseQuery(string collection) + => $"{GetTableName(collection)} | summarize arg_max(ingestion_time(), *) by {s_keyColumn.Name} "; + + /// + /// Initializes vector cosine similarity function for given database. + /// + /// + /// Cosine similarity function is created only once for better performance. + /// It's possible to run function creation multiple times, since .create-or-alter command is idempotent. + /// + private void InitializeVectorFunctions() + { + if (!this._searchInitialized) + { + lock (this._lock) + { + if (!this._searchInitialized) + { + var resp = this._adminClient + .ExecuteControlCommand( + this._database, + ".create-or-alter function with (docstring = 'Calculate the Cosine similarity of 2 numerical arrays',folder = 'Vector') series_cosine_similarity_fl(vec1:dynamic,vec2:dynamic,vec1_size:real=real(null),vec2_size:real=real(null)) {" + + " let dp = series_dot_product(vec1, vec2);" + + " let v1l = iff(isnull(vec1_size), sqrt(series_dot_product(vec1, vec1)), vec1_size);" + + " let v2l = iff(isnull(vec2_size), sqrt(series_dot_product(vec2, vec2)), vec2_size);" + + " dp/(v1l*v2l)" + + "}", + GetClientRequestProperties() + ); + + this._searchInitialized = true; + } + } + } + } + + #endregion private ================================================================================ +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoSerializer.cs b/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoSerializer.cs new file mode 100644 index 000000000000..30a21e61bc95 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Kusto/KustoSerializer.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Globalization; +using System.Text.Json; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Text; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Kusto; + +/// +/// Contains serialization/deserialization logic for memory record properties in Kusto. +/// +public static class KustoSerializer +{ + /// + /// Returns serialized string from an embedding instance. + /// + /// Instance of an embedding for serialization. + public static string SerializeEmbedding(ReadOnlyMemory embedding) + { + return JsonSerializer.Serialize(embedding, s_jsonSerializerOptions); + } + + /// + /// Returns deserialized instance of an embedding from serialized embedding. + /// + /// Serialized embedding. + public static ReadOnlyMemory DeserializeEmbedding(string? embedding) + { + return string.IsNullOrEmpty(embedding) ? + default : + JsonSerializer.Deserialize>(embedding!, s_jsonSerializerOptions); + } + + /// + /// Returns serialized string from instance. + /// + /// Instance of for serialization. + public static string SerializeMetadata(MemoryRecordMetadata metadata) + { + if (metadata == null) + { + return string.Empty; + } + + return JsonSerializer.Serialize(metadata); + } + + /// + /// Returns deserialized instance of from serialized metadata. + /// + /// Serialized metadata. + public static MemoryRecordMetadata DeserializeMetadata(string metadata) + { + return JsonSerializer.Deserialize(metadata)!; + } + + /// + /// Returns serialized string from instance. + /// + /// Instance of for serialization. + public static string SerializeDateTimeOffset(DateTimeOffset? dateTimeOffset) + { + if (dateTimeOffset == null) + { + return string.Empty; + } + + return dateTimeOffset.Value.DateTime.ToString(TimestampFormat, CultureInfo.InvariantCulture); + } + + /// + /// Returns deserialized instance of from serialized timestamp. + /// + /// Serialized timestamp. + public static DateTimeOffset? DeserializeDateTimeOffset(string? dateTimeOffset) + { + if (string.IsNullOrWhiteSpace(dateTimeOffset)) + { + return null; + } + + if (DateTimeOffset.TryParseExact(dateTimeOffset, TimestampFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTimeOffset result)) + { + return result; + } + + throw new InvalidCastException("Timestamp format cannot be parsed"); + } + + #region private ================================================================================ + + private const string TimestampFormat = "yyyy-MM-ddTHH:mm:ssZ"; + + private static readonly JsonSerializerOptions s_jsonSerializerOptions = CreateSerializerOptions(); + + private static JsonSerializerOptions CreateSerializerOptions() + { + var jso = new JsonSerializerOptions(); + jso.Converters.Add(new ReadOnlyMemoryConverter()); + return jso; + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Kusto/README.md b/dotnet/src/Connectors/Connectors.Memory.Kusto/README.md new file mode 100644 index 000000000000..6b33ab53ec81 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Kusto/README.md @@ -0,0 +1,42 @@ +# Microsoft.SemanticKernel.Connectors.Memory.Kusto + +This connector uses [Azure Data Explorer (Kusto)](https://learn.microsoft.com/en-us/azure/data-explorer/) to implement Semantic Memory. + +## Quick Start + +1. Create a cluster and database in Azure Data Explorer (Kusto) - see https://learn.microsoft.com/en-us/azure/data-explorer/create-cluster-and-database?tabs=free + +2. To use Kusto as a semantic memory store, use the following code: + +```csharp +using Kusto.Data; + +var connectionString = new KustoConnectionStringBuilder("https://kvc123.eastus.kusto.windows.net").WithAadUserPromptAuthentication(); +KustoMemoryStore memoryStore = new(connectionString, "MyDatabase"); + +IKernel kernel = Kernel.Builder + .WithLogger(ConsoleLogger.Log) + .WithOpenAITextCompletionService(modelId: TestConfiguration.OpenAI.ModelId, apiKey: TestConfiguration.OpenAI.ApiKey) + .WithOpenAITextEmbeddingGenerationService(modelId: TestConfiguration.OpenAI.EmbeddingModelId,apiKey: TestConfiguration.OpenAI.ApiKey) + .WithMemoryStorage(memoryStore) + .Build(); +``` + +## Important Notes + +### Cosine Similarity +As of now, cosine similarity is not built-in to Kusto. +A function to calculate cosine similarity is automatically added to the Kusto database during first search operation. +This function (`series_cosine_similarity_fl`) is not removed automatically. +You might want to delete it manually if you stop using the Kusto database as a semantic memory store. +If you want to delete the function, you can do it manually using the Kusto explorer. +The function is called `series_cosine_similarity_fl` and is located in the `Functions` folder of the database. + +### Append-Only Store +Kusto is an append-only store. This means that when a fact is updated, the old fact is not deleted. +This isn't a problem for the semantic memory connector, as it always utilizes the most recent fact. +This is made possible by using the [arg_max](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/arg-max-aggfunction) aggregation function in conjunction with the [ingestion_time](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/ingestiontimefunction) function. +However, users manually querying the underlying table should be aware of this behavior. + +### Authentication +Please note that the authentication used in the example above is not recommended for production use. You can find more details here: https://learn.microsoft.com/en-us/azure/data-explorer/kusto/api/connection-strings/kusto diff --git a/dotnet/src/Connectors/Connectors.Memory.Milvus/Connectors.Memory.Milvus.csproj b/dotnet/src/Connectors/Connectors.Memory.Milvus/Connectors.Memory.Milvus.csproj new file mode 100644 index 000000000000..530c85ec6423 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Milvus/Connectors.Memory.Milvus.csproj @@ -0,0 +1,32 @@ + + + + + Microsoft.SemanticKernel.Connectors.Memory.Milvus + $(AssemblyName) + net6.0;netstandard2.0 + enable + + + NU5104 + + + + + + + + + Semantic Kernel - Milvus Connector + Milvus connector for Semantic Kernel plugins and semantic memory + + + + + + + + + + + diff --git a/dotnet/src/Connectors/Connectors.Memory.Milvus/MilvusMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Milvus/MilvusMemoryStore.cs new file mode 100644 index 000000000000..278b4b52688e --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Milvus/MilvusMemoryStore.cs @@ -0,0 +1,560 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Memory; +using Milvus.Client; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Milvus; + +/// +/// An implementation of for the Milvus vector database. +/// +public class MilvusMemoryStore : IMemoryStore, IDisposable +{ + private readonly int _vectorSize; + private readonly SimilarityMetricType _metricType; + private readonly bool _ownsMilvusClient; + + private const string IsReferenceFieldName = "is_reference"; + private const string ExternalSourceNameFieldName = "external_source_name"; + private const string IdFieldName = "id"; + private const string DescriptionFieldName = "description"; + private const string TextFieldName = "text"; + private const string AdditionalMetadataFieldName = "additional_metadata"; + private const string EmbeddingFieldName = "embedding"; + private const string KeyFieldName = "key"; + private const string TimestampFieldName = "timestamp"; + + private const int DefaultMilvusPort = 19530; + private const ConsistencyLevel DefaultConsistencyLevel = ConsistencyLevel.Session; + private const int DefaultVarcharLength = 65_535; + + private readonly QueryParameters _queryParametersWithEmbedding = new() + { + OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, EmbeddingFieldName, KeyFieldName, TimestampFieldName } + }; + + private readonly QueryParameters _queryParametersWithoutEmbedding = new() + { + OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, KeyFieldName, TimestampFieldName } + }; + + private readonly SearchParameters _searchParameters = new() + { + OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, KeyFieldName, TimestampFieldName } + }; + + /// + /// Exposes the underlying used to communicate with Milvus. Can be used to execute operations not supported by the abstraction. + /// + public MilvusClient Client { get; } + + #region Constructors + + /// + /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. + /// For more advanced configuration opens, construct a instance and pass it to + /// . + /// + /// The hostname or IP address to connect to. + /// The port to connect to. Defaults to 19530. + /// Whether to use TLS/SSL. Defaults to false. + /// The database to connect to. Defaults to the default Milvus database. + /// The size of the vectors used in Milvus. Defaults to 1536. + /// The metric used to measure similarity between vectors. Defaults to . + /// An optional logger factory through which the Milvus client will log. + public MilvusMemoryStore( + string host, + int port = DefaultMilvusPort, + bool ssl = false, + string? database = null, + int vectorSize = 1536, + SimilarityMetricType metricType = SimilarityMetricType.Ip, + ILoggerFactory? loggerFactory = null) + : this(new MilvusClient(host, port, ssl, database, callOptions: default, loggerFactory), vectorSize, metricType) + { + this._ownsMilvusClient = true; + } + + /// + /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. + /// For more advanced configuration opens, construct a instance and pass it to + /// . + /// + /// The hostname or IP address to connect to. + /// The username to use for authentication. + /// The password to use for authentication. + /// The port to connect to. Defaults to 19530. + /// Whether to use TLS/SSL. Defaults to false. + /// The database to connect to. Defaults to the default Milvus database. + /// The size of the vectors used in Milvus. Defaults to 1536. + /// The metric used to measure similarity between vectors. Defaults to . + /// An optional logger factory through which the Milvus client will log. + public MilvusMemoryStore( + string host, + string username, + string password, + int port = DefaultMilvusPort, + bool ssl = false, + string? database = null, + int vectorSize = 1536, + SimilarityMetricType metricType = SimilarityMetricType.Ip, + ILoggerFactory? loggerFactory = null) + : this(new MilvusClient(host, username, password, port, ssl, database, callOptions: default, loggerFactory), vectorSize, metricType) + { + this._ownsMilvusClient = true; + } + + /// + /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. + /// For more advanced configuration opens, construct a instance and pass it to + /// . + /// + /// The hostname or IP address to connect to. + /// An API key to be used for authentication, instead of a username and password. + /// The port to connect to. Defaults to 19530. + /// Whether to use TLS/SSL. Defaults to false. + /// The database to connect to. Defaults to the default Milvus database. + /// The size of the vectors used in Milvus. Defaults to 1536. + /// The metric used to measure similarity between vectors. Defaults to . + /// An optional logger factory through which the Milvus client will log. + public MilvusMemoryStore( + string host, + string apiKey, + int port = DefaultMilvusPort, + bool ssl = false, + string? database = null, + int vectorSize = 1536, + SimilarityMetricType metricType = SimilarityMetricType.Ip, + ILoggerFactory? loggerFactory = null) + : this(new MilvusClient(host, apiKey, port, ssl, database, callOptions: default, loggerFactory), vectorSize, metricType) + { + this._ownsMilvusClient = true; + } + + /// + /// Initializes a new instance of over the given . + /// + /// A configured with the necessary endpoint and authentication information. + /// The size of the vectors used in Milvus. Defaults to 1536. + /// The metric used to measure similarity between vectors. Defaults to . + public MilvusMemoryStore( + MilvusClient client, + int vectorSize = 1536, + SimilarityMetricType metricType = SimilarityMetricType.Ip) + : this(client, ownsMilvusClient: false, vectorSize, metricType) + { + } + + private MilvusMemoryStore( + MilvusClient client, + bool ownsMilvusClient, + int vectorSize = 1536, + SimilarityMetricType metricType = SimilarityMetricType.Ip) + { + this.Client = client; + this._vectorSize = vectorSize; + this._metricType = metricType; + this._ownsMilvusClient = ownsMilvusClient; + } + + #endregion Constructors + + /// + public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default) + { + var exists = await this.Client.HasCollectionAsync(collectionName, cancellationToken: cancellationToken).ConfigureAwait(false); + if (!exists) + { + CollectionSchema schema = new() + { + Fields = + { + FieldSchema.CreateVarchar(IdFieldName, maxLength: DefaultVarcharLength, isPrimaryKey: true, autoId: false), + FieldSchema.CreateFloatVector(EmbeddingFieldName, this._vectorSize) + }, + EnableDynamicFields = true + }; + + MilvusCollection collection = await this.Client.CreateCollectionAsync(collectionName, schema, DefaultConsistencyLevel, cancellationToken: cancellationToken).ConfigureAwait(false); + + await collection.CreateIndexAsync(EmbeddingFieldName, metricType: this._metricType, cancellationToken: cancellationToken).ConfigureAwait(false); + await collection.WaitForIndexBuildAsync("float_vector", cancellationToken: cancellationToken).ConfigureAwait(false); + + await collection.LoadAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + await collection.WaitForCollectionLoadAsync(waitingInterval: TimeSpan.FromMilliseconds(100), timeout: TimeSpan.FromMinutes(1), cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + /// + public async IAsyncEnumerable GetCollectionsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (MilvusCollectionInfo collection in await this.Client.ListCollectionsAsync(cancellationToken: cancellationToken).ConfigureAwait(false)) + { + yield return collection.Name; + } + } + + /// + public Task DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default) + => this.Client.HasCollectionAsync(collectionName, cancellationToken: cancellationToken); + + /// + public Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) + => this.Client.GetCollection(collectionName).DropAsync(cancellationToken); + + /// + public async Task UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default) + { + MilvusCollection collection = this.Client.GetCollection(collectionName); + + await collection.DeleteAsync($@"{IdFieldName} in [""{record.Metadata.Id}""]", cancellationToken: cancellationToken).ConfigureAwait(false); + + var metadata = record.Metadata; + + List fieldData = new() + { + FieldData.Create(IdFieldName, new[] { metadata.Id }), + FieldData.CreateFloatVector(EmbeddingFieldName, new[] { record.Embedding }), + + FieldData.Create(IsReferenceFieldName, new[] { metadata.IsReference }, isDynamic: true), + FieldData.Create(ExternalSourceNameFieldName, new[] { metadata.ExternalSourceName }, isDynamic: true), + FieldData.Create(DescriptionFieldName, new[] { metadata.Description }, isDynamic: true), + FieldData.Create(TextFieldName, new[] { metadata.Text }, isDynamic: true), + FieldData.Create(AdditionalMetadataFieldName, new[] { metadata.AdditionalMetadata }, isDynamic: true), + FieldData.Create(KeyFieldName, new[] { record.Key }, isDynamic: true), + FieldData.Create(TimestampFieldName, new[] { record.Timestamp?.ToString(CultureInfo.InvariantCulture) ?? string.Empty }, isDynamic: true) + }; + + MutationResult result = await collection.InsertAsync(fieldData, cancellationToken: cancellationToken).ConfigureAwait(false); + + return result.Ids.StringIds![0]; + } + + /// + public async IAsyncEnumerable UpsertBatchAsync( + string collectionName, + IEnumerable records, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // TODO: Milvus v2.3.0 will have a 1st-class upsert API which we should use. + // In the meantime, we do delete+insert, following the Python connector's example. + + StringBuilder idString = new(); + + List isReferenceData = new(); + List externalSourceNameData = new(); + List idData = new(); + List descriptionData = new(); + List textData = new(); + List additionalMetadataData = new(); + List> embeddingData = new(); + List keyData = new(); + List timestampData = new(); + + foreach (MemoryRecord record in records) + { + var metadata = record.Metadata; + + if (idString.Length > 0) + { + idString.Append(','); + } + + idString.Append('"').Append(metadata.Id).Append('"'); + + isReferenceData.Add(metadata.IsReference); + externalSourceNameData.Add(metadata.ExternalSourceName); + idData.Add(record.Metadata.Id); + descriptionData.Add(metadata.Description); + textData.Add(metadata.Text); + additionalMetadataData.Add(metadata.AdditionalMetadata); + embeddingData.Add(record.Embedding); + keyData.Add(record.Key); + timestampData.Add(record.Timestamp?.ToString(CultureInfo.InvariantCulture) ?? string.Empty); + } + + MilvusCollection collection = this.Client.GetCollection(collectionName); + await collection.DeleteAsync($"{IdFieldName} in [{idString}]", cancellationToken: cancellationToken).ConfigureAwait(false); + + FieldData[] fieldData = + { + FieldData.Create(IdFieldName, idData), + FieldData.CreateFloatVector(EmbeddingFieldName, embeddingData), + + FieldData.Create(IsReferenceFieldName, isReferenceData, isDynamic: true), + FieldData.Create(ExternalSourceNameFieldName, externalSourceNameData, isDynamic: true), + FieldData.Create(DescriptionFieldName, descriptionData, isDynamic: true), + FieldData.Create(TextFieldName, textData, isDynamic: true), + FieldData.Create(AdditionalMetadataFieldName, additionalMetadataData, isDynamic: true), + FieldData.Create(KeyFieldName, keyData, isDynamic: true), + FieldData.Create(TimestampFieldName, timestampData, isDynamic: true) + }; + + MutationResult result = await collection.InsertAsync(fieldData, cancellationToken: cancellationToken).ConfigureAwait(false); + + foreach (var id in result.Ids.StringIds!) + { + yield return id; + } + } + + /// + public async Task GetAsync( + string collectionName, + string key, + bool withEmbedding = false, + CancellationToken cancellationToken = default) + { + await foreach (MemoryRecord record in this.GetBatchAsync(collectionName, new[] { key }, withEmbedding, cancellationToken)) + { + return record; + } + + return null; + } + + /// + public async IAsyncEnumerable GetBatchAsync( + string collectionName, + IEnumerable keys, + bool withEmbeddings = false, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + StringBuilder idString = new(); + + foreach (string key in keys) + { + if (idString.Length > 0) + { + idString.Append(','); + } + + idString.Append('"').Append(key).Append('"'); + } + + IReadOnlyList fields = await this.Client + .GetCollection(collectionName) + .QueryAsync($"{IdFieldName} in [{idString}]", withEmbeddings ? this._queryParametersWithEmbedding : this._queryParametersWithoutEmbedding, cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var rowCount = fields[0].RowCount; + + for (int rowNum = 0; rowNum < rowCount; rowNum++) + { + yield return this.ReadMemoryRecord(fields, rowNum); + } + } + + /// + public Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default) + => this.Client.GetCollection(collectionName) + .DeleteAsync($@"{IdFieldName} in [""{key}""]", cancellationToken: cancellationToken); + + /// + public Task RemoveBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default) + { + StringBuilder idString = new(); + + idString.Append(IdFieldName).Append(" in ["); + + bool first = true; + foreach (string id in keys) + { + if (first) + { + first = false; + } + else + { + idString.Append(','); + } + + idString.Append('"').Append(id).Append('"'); + } + + idString.Append(']'); + + return this.Client + .GetCollection(collectionName) + .DeleteAsync(idString.ToString(), cancellationToken: cancellationToken); + } + + /// + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync( + string collectionName, + ReadOnlyMemory embedding, + double minRelevanceScore = 0, + bool withEmbedding = false, + CancellationToken cancellationToken = default) + { + await foreach ((MemoryRecord, double) result in this.GetNearestMatchesAsync(collectionName, embedding, limit: 1, minRelevanceScore, withEmbedding, cancellationToken)) + { + return result; + } + + return null; + } + + /// + public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( + string collectionName, + ReadOnlyMemory embedding, + int limit, + double minRelevanceScore = 0, + bool withEmbeddings = false, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + MilvusCollection collection = this.Client.GetCollection(collectionName); + + SearchResults results = await collection + .SearchAsync(EmbeddingFieldName, new[] { embedding }, SimilarityMetricType.Ip, limit, this._searchParameters, cancellationToken) + .ConfigureAwait(false); + + IReadOnlyList ids = results.Ids.StringIds!; + int rowCount = ids.Count; + IReadOnlyList data = results.FieldsData; + + // Since Milvus does not support fetching vectors via the Search API, we do an extra call to fetch the ids and embeddings using the Query API, + // using the IDs returned from the Search above, populating a map from the IDs to the embedding. + // TODO: There's some support for fetching vectors from Search in Milvus 2.3, check that out. + Dictionary>? embeddingMap = null; + if (withEmbeddings) + { + StringBuilder filter = new(); + filter.Append(IdFieldName).Append(" in ["); + + for (int rowNum = 0; rowNum < ids.Count; rowNum++) + { + if (rowNum > 0) + { + filter.Append(','); + } + + filter.Append('"').Append(ids[rowNum]).Append('"'); + } + + filter.Append(']'); + + IReadOnlyList fieldData = await collection.QueryAsync( + filter.ToString(), + new() { OutputFields = { EmbeddingFieldName } }, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + IReadOnlyList idData = (fieldData[0] as FieldData ?? fieldData[1] as FieldData)!.Data; + IReadOnlyList> embeddingData = (fieldData[0] as FloatVectorFieldData ?? fieldData[1] as FloatVectorFieldData)!.Data; + + embeddingMap = new Dictionary>(ids.Count); + for (int rowNum = 0; rowNum < ids.Count; rowNum++) + { + embeddingMap[idData[rowNum]] = embeddingData[rowNum]; + } + } + + for (int rowNum = 0; rowNum < rowCount; rowNum++) + { + // TODO: Milvus 2.3 has range search, which will move this to the server. + if (results.Scores[rowNum] >= minRelevanceScore) + { + yield return ( + this.ReadMemoryRecord(data, rowNum, withEmbeddings ? embeddingMap![ids[rowNum]] : null), + results.Scores[rowNum]); + } + } + } + + private MemoryRecord ReadMemoryRecord(IReadOnlyList data, int rowNum, ReadOnlyMemory? externalEmbedding = null) + { + bool isReference = false; + string externalSourceName = string.Empty; + string id = string.Empty; + string description = string.Empty; + string text = string.Empty; + string additionalMetadata = string.Empty; + ReadOnlyMemory? embedding = null; + string key = string.Empty; + DateTimeOffset? timestamp = null; + + foreach (FieldData field in data) + { + switch (field.FieldName) + { + case IsReferenceFieldName when field is FieldData isReferenceField: + isReference = isReferenceField.Data[rowNum]; + break; + + case ExternalSourceNameFieldName when field is FieldData externalSourceNameField: + externalSourceName = externalSourceNameField.Data[rowNum]; + break; + + case IdFieldName when field is FieldData idField: + id = idField.Data[rowNum]; + break; + + case DescriptionFieldName when field is FieldData descriptionField: + description = descriptionField.Data[rowNum]; + break; + + case TextFieldName when field is FieldData textField: + text = textField.Data[rowNum]; + break; + + case AdditionalMetadataFieldName when field is FieldData additionalMetadataField: + additionalMetadata = additionalMetadataField.Data[rowNum]; + break; + + case EmbeddingFieldName when field is FloatVectorFieldData embeddingField: + Debug.Assert(externalEmbedding is null); + embedding = embeddingField.Data[rowNum]; + break; + + case KeyFieldName when field is FieldData keyField: + key = keyField.Data[rowNum]; + break; + + case TimestampFieldName when field is FieldData timestampField: + string timestampString = timestampField.Data[rowNum]; + timestamp = timestampString is { Length: > 0 } + ? DateTimeOffset.Parse(timestampString, CultureInfo.InvariantCulture) + : null; + break; + + default: + continue; // Unknown field - ignore + } + } + + return new MemoryRecord( + new MemoryRecordMetadata(isReference, id, text, description, externalSourceName, additionalMetadata), + embedding ?? externalEmbedding ?? Array.Empty(), + key, + timestamp); + } + + /// + /// Implements the dispose pattern. + /// + protected virtual void Dispose(bool disposing) + { + if (disposing && this._ownsMilvusClient) + { + this.Client.Dispose(); + } + } + + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Milvus/README.md b/dotnet/src/Connectors/Connectors.Memory.Milvus/README.md new file mode 100644 index 000000000000..a4aa4d35d9ff --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Milvus/README.md @@ -0,0 +1,33 @@ +# Microsoft.SemanticKernel.Connectors.Memory.Milvus + +This is an implementation of the Semantic Kernel Memory Store abstraction for the [Milvus vector database](https://milvus.io). + +**Note:** Currently, only Milvus v2.2 is supported. v2.3 is coming soon, older versions are untested. + +## Quickstart using a standalone Milvus installation + +1. Download the Milvus docker-compose.yml: + +```bash +wget https://github.com/milvus-io/milvus/releases/download/v2.2.14/milvus-standalone-docker-compose.yml -O docker-compose.yml +``` + +2. Start Milvus: + +```bash +docker-compose up -d +``` + +3. Use Semantic Kernel with Milvus, connecting to `localhost` with the default (gRPC) port of 1536: + +```csharp +using MilvusMemoryStore memoryStore = new("localhost"); + +IKernel kernel = Kernel.Builder + .WithLogger(logger) + .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", "OPENAI_API_KEY") + .WithMemoryStorage(memoryStore) + .Build(); +``` + +More information on setting up Milvus can be found [here](https://milvus.io/docs/v2.2.x/install_standalone-docker.md). The `MilvusMemoryStore` constructor provides additional configuration options, such as the vector size, the similarity metric type, etc. \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj index 200b77b095e0..da502d5cae4f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj @@ -14,7 +14,7 @@ Semantic Kernel - Pinecone Connector - Pinecone connector for Semantic Kernel skills and semantic memory + Pinecone connector for Semantic Kernel plugins and semantic memory @@ -22,7 +22,8 @@ - + + diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/QueryRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/QueryRequest.cs index bceee402a30d..446dac99d767 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/QueryRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/QueryRequest.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Net.Http; using System.Text.Json.Serialization; @@ -36,7 +37,7 @@ internal sealed class QueryRequest /// Vector dense data. This should be the same length as the dimension of the index being queried. /// [JsonPropertyName("vector")] - public IEnumerable? Vector { get; set; } + public ReadOnlyMemory Vector { get; set; } /// /// The unique ID of a vector @@ -106,7 +107,7 @@ public HttpRequestMessage Build() /// Initializes a new instance of the class. /// [JsonConstructor] - private QueryRequest(IEnumerable? values = null) + private QueryRequest(ReadOnlyMemory values) { this.Vector = values; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/UpdateVectorRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/UpdateVectorRequest.cs index c4eda878cd8e..008fb11594a6 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/UpdateVectorRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/UpdateVectorRequest.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Net.Http; using System.Text.Json.Serialization; @@ -25,7 +26,7 @@ internal sealed class UpdateVectorRequest /// Vector dense data. This should be the same length as the dimension of the index being queried. /// [JsonPropertyName("values")] - public IEnumerable? Values { get; set; } + public ReadOnlyMemory Values { get; set; } /// /// The sparse vector data @@ -77,7 +78,7 @@ public UpdateVectorRequest UpdateSparseValues(SparseVectorData? sparseValues) return this; } - public UpdateVectorRequest UpdateValues(IEnumerable? values) + public UpdateVectorRequest UpdateValues(ReadOnlyMemory values) { this.Values = values; return this; @@ -99,7 +100,7 @@ public HttpRequestMessage Build() /// Initializes a new instance of the class. /// [JsonConstructor] - private UpdateVectorRequest(string id, IEnumerable? values = default) + private UpdateVectorRequest(string id, ReadOnlyMemory values = default) { this.Id = id; this.Values = values; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/IPineconeClient.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/IPineconeClient.cs index ea8e7eeca6fb..48fc3753bdaa 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/IPineconeClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/IPineconeClient.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -59,7 +60,7 @@ public interface IPineconeClient /// Cancellation token. IAsyncEnumerable<(PineconeDocument, double)> GetMostRelevantAsync( string indexName, - IEnumerable vector, + ReadOnlyMemory vector, double threshold, int topK, bool includeValues, diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/IPineconeMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/IPineconeMemoryStore.cs index 53ccaac38697..dc079d9d9857 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/IPineconeMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/IPineconeMemoryStore.cs @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Memory; namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone; @@ -203,10 +203,10 @@ Task RemoveWithFilterAsync( CancellationToken cancellationToken = default); /// - /// Gets the nearest filtered matches to the of type + /// Gets the nearest filtered matches to an embedding of type /// /// The name associated with a collection of embeddings. - /// The to compare the collection's embeddings with. + /// The embedding to compare the collection's embeddings with. /// The maximum number of similarity results to return. /// The filter to apply to the collection. /// @@ -222,7 +222,7 @@ Task RemoveWithFilterAsync( /// IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesWithFilterAsync( string indexName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, Dictionary filter, double minRelevanceScore = 0.0, @@ -231,11 +231,11 @@ Task RemoveWithFilterAsync( CancellationToken cancellationToken = default); /// - /// Gets the nearest matches to the of type from the given namespace. + /// Gets the nearest matches to an embedding of type from the given namespace. /// /// The name associated with a collection of embeddings. /// The namespace associated with a collection of embeddings. - /// The to compare the collection's embeddings with. + /// The embedding to compare the collection's embeddings with. /// The maximum number of similarity results to return. /// The minimum relevance threshold for returned results. /// If true, the embeddings will be returned in the memory records. @@ -244,18 +244,18 @@ Task RemoveWithFilterAsync( IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesFromNamespaceAsync( string indexName, string indexNamespace, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0.0, bool withEmbeddings = false, CancellationToken cancellationToken = default); /// - /// Gets the nearest match to the of type from the given namespace. + /// Gets the nearest match to an embedding of type from the given namespace. /// /// The name associated with a collection of embeddings. /// The namespace associated with a collection of embeddings. - /// The to compare the collection's embeddings with. + /// The embedding to compare the collection's embeddings with. /// The minimum relevance threshold for returned results. /// If true, the embedding will be returned in the memory record. /// Cancellation token @@ -263,7 +263,7 @@ Task RemoveWithFilterAsync( Task<(MemoryRecord, double)?> GetNearestMatchFromNamespaceAsync( string indexName, string indexNamespace, - Embedding embedding, + ReadOnlyMemory embedding, double minRelevanceScore = 0.0, bool withEmbedding = false, CancellationToken cancellationToken = default); diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/PodType.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/PodType.cs index 2887fff969d6..25c825048375 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/PodType.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/PodType.cs @@ -15,6 +15,9 @@ namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; [JsonConverter(typeof(PodTypeJsonConverter))] public enum PodType { + /// + /// Represents an undefined or uninitialized PodType. + /// None = 0, /// @@ -87,7 +90,13 @@ public enum PodType /// Enum P2X8 for value: p2.x8 /// [EnumMember(Value = "p2.x8")] - P2X8 = 12 + P2X8 = 12, + + /// + /// Enum Starter for value: starter + /// + [EnumMember(Value = "starter")] + Starter = 13 } internal sealed class PodTypeJsonConverter : JsonConverter diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/Query.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/Query.cs index cb2345a5b7e2..bd549502b180 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/Query.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/Query.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; @@ -28,7 +30,8 @@ public sealed class Query /// /// Vector dense data. This should be the same length as the dimension of the index being queried. /// - public IEnumerable? Vector { get; set; } + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Vector { get; set; } /// /// The unique ID of a vector @@ -56,7 +59,7 @@ public static Query Create(int topK) /// Sets vector for instance. /// /// Vector dense data. This should be the same length as the dimension of the index being queried. - public Query WithVector(IEnumerable? vector) + public Query WithVector(ReadOnlyMemory vector) { this.Vector = vector; return this; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/SparseVectorData.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/SparseVectorData.cs index 4623cbf7da92..b4b9bf2d3815 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/SparseVectorData.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Model/SparseVectorData.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; /// -/// Vector sparse data. Represented as a list of indices and a list of corresponded values, which must be the same length. +/// Represents a sparse vector data, which is a list of indices and a list of corresponding values, both of the same length. /// public class SparseVectorData { @@ -22,9 +24,16 @@ public class SparseVectorData /// /// The corresponding values of the sparse data, which must be the same length as the indices. [JsonPropertyName("values")] - public IEnumerable Values { get; set; } + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Values { get; set; } - public static SparseVectorData CreateSparseVectorData(List indices, List values) + /// + /// Creates a new instance of the class with the specified indices and values. + /// + /// The indices of the sparse data. + /// The corresponding values of the sparse data, which must be the same length as the indices. + /// A new instance of the class. + public static SparseVectorData CreateSparseVectorData(List indices, ReadOnlyMemory values) { return new SparseVectorData(indices, values); } @@ -35,7 +44,7 @@ public static SparseVectorData CreateSparseVectorData(List indices, ListThe indices of the sparse data. (required). /// The corresponding values of the sparse data, which must be the same length as the indices. (required). [JsonConstructor] - public SparseVectorData(List indices, List values) + public SparseVectorData(List indices, ReadOnlyMemory values) { this.Indices = indices; this.Values = values; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeClient.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeClient.cs index 8089c1d62cc8..27a5cbcbc57b 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeClient.cs @@ -14,6 +14,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Http.ApiSchema; using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; +using Microsoft.SemanticKernel.Diagnostics; namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone; @@ -27,14 +28,14 @@ public sealed class PineconeClient : IPineconeClient /// /// The environment for Pinecone. /// The API key for accessing Pinecone services. - /// An optional logger instance for logging. + /// The to use for logging. If null, no logging will be performed. /// An optional HttpClient instance for making HTTP requests. - public PineconeClient(string pineconeEnvironment, string apiKey, ILogger? logger = null, HttpClient? httpClient = null) + public PineconeClient(string pineconeEnvironment, string apiKey, ILoggerFactory? loggerFactory = null, HttpClient? httpClient = null) { this._pineconeEnvironment = pineconeEnvironment; this._authHeader = new KeyValuePair("Api-Key", apiKey); this._jsonSerializerOptions = PineconeUtils.DefaultSerializerOptions; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(PineconeClient)) : NullLogger.Instance; this._httpClient = httpClient ?? new HttpClient(NonDisposableHttpClientHandler.Instance, disposeHandler: false); this._indexHostMapping = new ConcurrentDictionary(); } @@ -56,16 +57,15 @@ public PineconeClient(string pineconeEnvironment, string apiKey, ILogger? logger using HttpRequestMessage request = fetchRequest.Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(basePath, request, - cancellationToken).ConfigureAwait(false); + string? responseContent = null; try { - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError("Error occurred on Get Vectors request: {0}", e.Message); + this._logger.LogError(e, "Error occurred on Get Vectors request: {Message}", e.Message); yield break; } @@ -110,15 +110,15 @@ public PineconeClient(string pineconeEnvironment, string apiKey, ILogger? logger string basePath = await this.GetVectorOperationsApiBasePathAsync(indexName).ConfigureAwait(false); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); + string? responseContent = null; try { - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError("Error occurred on Query Vectors request: {0}", e.Message); + this._logger.LogError(e, "Error occurred on Query Vectors request: {Message}", e.Message); yield break; } @@ -145,7 +145,7 @@ public PineconeClient(string pineconeEnvironment, string apiKey, ILogger? logger /// public async IAsyncEnumerable<(PineconeDocument, double)> GetMostRelevantAsync( string indexName, - IEnumerable vector, + ReadOnlyMemory vector, double threshold, int topK, bool includeValues, @@ -217,15 +217,15 @@ public async Task UpsertAsync( using HttpRequestMessage request = batch.ToNamespace(indexNamespace).Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); + string? responseContent = null; try { - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError("Failed to upsert vectors {0}", e.Message); + this._logger.LogError(e, "Failed to upsert vectors {Message}", e.Message); throw; } @@ -258,9 +258,7 @@ public async Task DeleteAsync( { if (ids == null && string.IsNullOrEmpty(indexNamespace) && filter == null && !deleteAll) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToRemoveVectorData, - "Must provide at least one of ids, filter, or deleteAll"); + throw new SKException("Must provide at least one of ids, filter, or deleteAll"); } ids = ids?.ToList(); @@ -279,15 +277,13 @@ public async Task DeleteAsync( using HttpRequestMessage request = deleteRequest.Build(); - (HttpResponseMessage response, string _) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); - try { - response.EnsureSuccessStatusCode(); + await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError("Delete operation failed: {0}", e.Message); + this._logger.LogError(e, "Delete operation failed: {Message}", e.Message); throw; } } @@ -304,15 +300,13 @@ public async Task UpdateAsync(string indexName, PineconeDocument document, strin .InNamespace(indexNamespace) .Build(); - (HttpResponseMessage response, string _) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); - try { - response.EnsureSuccessStatusCode(); + await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogWarning("Vector update for Document {0} failed. Message: {1}", document.Id, e.Message); + this._logger.LogError(e, "Vector update for Document {Id} failed. {Message}", document.Id, e.Message); throw; } } @@ -331,15 +325,15 @@ public async Task UpdateAsync(string indexName, PineconeDocument document, strin .WithFilter(filter) .Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); + string? responseContent = null; try { - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(basePath, request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogDebug("Index not found {0}", e.Message); + this._logger.LogError(e, "Index not found {Message}", e.Message); throw; } @@ -386,25 +380,23 @@ public async Task CreateIndexAsync(IndexDefinition indexDefinition, Cancellation using HttpRequestMessage request = indexDefinition.Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(this.GetIndexOperationsApiBasePath(), request, cancellationToken).ConfigureAwait(false); - try { - response.EnsureSuccessStatusCode(); + await this.ExecuteHttpRequestAsync(this.GetIndexOperationsApiBasePath(), request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) when (response.StatusCode == HttpStatusCode.BadRequest) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.BadRequest) { - this._logger.LogError(e, "Bad Request: {0}, {1}", response.StatusCode, responseContent); + this._logger.LogError(e, "Bad Request: {StatusCode}, {Response}", e.StatusCode, e.ResponseContent); throw; } - catch (HttpRequestException e) when (response.StatusCode == HttpStatusCode.Conflict) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.Conflict) { - this._logger.LogError(e, "Index of given name already exists: {0}, {1}", response.StatusCode, responseContent); + this._logger.LogError(e, "Index of given name already exists: {StatusCode}, {Response}", e.StatusCode, e.ResponseContent); throw; } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError(e, "Creating index failed: {0}, {1}", e.Message, responseContent); + this._logger.LogError(e, "Creating index failed: {Message}, {Response}", e.Message, e.ResponseContent); throw; } } @@ -416,20 +408,18 @@ public async Task DeleteIndexAsync(string indexName, CancellationToken cancellat using HttpRequestMessage request = DeleteIndexRequest.Create(indexName).Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(this.GetIndexOperationsApiBasePath(), request, cancellationToken).ConfigureAwait(false); - try { - response.EnsureSuccessStatusCode(); + await this.ExecuteHttpRequestAsync(this.GetIndexOperationsApiBasePath(), request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) when (response.StatusCode == HttpStatusCode.NotFound) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.NotFound) { - this._logger.LogError(e, "Index Not Found: {0}, {1}", response.StatusCode, responseContent); + this._logger.LogError(e, "Index Not Found: {StatusCode}, {Response}", e.StatusCode, e.ResponseContent); throw; } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError(e, "Deleting index failed: {0}, {1}", e.Message, responseContent); + this._logger.LogError(e, "Deleting index failed: {Message}, {Response}", e.Message, e.ResponseContent); throw; } @@ -460,20 +450,20 @@ public async Task DoesIndexExistAsync(string indexName, CancellationToken using HttpRequestMessage request = DescribeIndexRequest.Create(indexName).Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(this.GetIndexOperationsApiBasePath(), request, cancellationToken).ConfigureAwait(false); + string? responseContent = null; try { - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(this.GetIndexOperationsApiBasePath(), request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) when (response.StatusCode == HttpStatusCode.BadRequest) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.BadRequest) { - this._logger.LogError(e, "Bad Request: {0}, {1}", response.StatusCode, responseContent); + this._logger.LogError(e, "Bad Request: {StatusCode}, {Response}", e.StatusCode, e.ResponseContent); throw; } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError(e, "Describe index failed: {0}, {1}", e.Message, responseContent); + this._logger.LogError(e, "Describe index failed: {Message}, {Response}", e.Message, e.ResponseContent); throw; } @@ -498,25 +488,23 @@ public async Task ConfigureIndexAsync(string indexName, int replicas = 1, PodTyp .NumberOfReplicas(replicas) .Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(this.GetIndexOperationsApiBasePath(), request, cancellationToken).ConfigureAwait(false); - try { - response.EnsureSuccessStatusCode(); + await this.ExecuteHttpRequestAsync(this.GetIndexOperationsApiBasePath(), request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) when (response.StatusCode == HttpStatusCode.BadRequest) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.BadRequest) { - this._logger.LogError(e, "Request exceeds quota or collection name is invalid. {0}", indexName); + this._logger.LogError(e, "Request exceeds quota or collection name is invalid. {Index}", indexName); throw; } - catch (HttpRequestException e) when (response.StatusCode == HttpStatusCode.NotFound) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.NotFound) { - this._logger.LogError(e, "Index not found. {0}", indexName); + this._logger.LogError(e, "Index not found. {Index}", indexName); throw; } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError(e, "Index configuration failed: {0}, {1}", e.Message, responseContent); + this._logger.LogError(e, "Index configuration failed: {Message}, {Response}", e.Message, e.ResponseContent); throw; } @@ -554,18 +542,9 @@ private string GetIndexOperationsApiBasePath() request.Headers.Add(this._authHeader.Key, this._authHeader.Value); request.RequestUri = new Uri(baseURL + request.RequestUri); - using HttpResponseMessage response = await this._httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); - - string responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + using HttpResponseMessage response = await this._httpClient.SendWithSuccessCheckAsync(request, cancellationToken).ConfigureAwait(false); - if (response.IsSuccessStatusCode) - { - this._logger.LogDebug("Pinecone responded successfully"); - } - else - { - this._logger.LogWarning("Pinecone responded with error"); - } + string responseContent = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); return (response, responseContent); } @@ -583,16 +562,12 @@ private async Task GetIndexHostAsync(string indexName, CancellationToken if (pineconeIndex == null) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.IndexNotFound, - "Index not found in Pinecone. Create index to perform operations with vectors."); + throw new SKException("Index not found in Pinecone. Create index to perform operations with vectors."); } if (string.IsNullOrWhiteSpace(pineconeIndex.Status.Host)) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.UnknownIndexHost, - $"Host of index {indexName} is unknown."); + throw new SKException($"Host of index {indexName} is unknown."); } this._logger.LogDebug("Found host {0} for index {1}", pineconeIndex.Status.Host, indexName); diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeDocument.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeDocument.cs index a602e5768921..397c5389a926 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeDocument.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeDocument.cs @@ -7,6 +7,7 @@ using System.Text.Json.Serialization; using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Http.ApiSchema; using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone; @@ -26,7 +27,8 @@ public class PineconeDocument /// Vector dense data. This should be the same length as the dimension of the index being queried. /// [JsonPropertyName("values")] - public IEnumerable Values { get; set; } + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Values { get; set; } /// /// The metadata associated with the document @@ -91,14 +93,14 @@ public class PineconeDocument /// [JsonConstructor] public PineconeDocument( - IEnumerable? values = null, + ReadOnlyMemory values = default, string? id = default, Dictionary? metadata = null, SparseVectorData? sparseValues = null, float? score = null) { this.Id = id ?? Guid.NewGuid().ToString(); - this.Values = values ?? Array.Empty(); + this.Values = values; this.Metadata = metadata ?? new Dictionary(); this.SparseValues = sparseValues; this.Score = score; @@ -109,7 +111,7 @@ public PineconeDocument( /// /// The unique ID of a vector. /// Vector dense data. This should be the same length as the dimension of the index being queried. - public static PineconeDocument Create(string? id = default, IEnumerable? values = default) + public static PineconeDocument Create(string? id = default, ReadOnlyMemory values = default) { return new PineconeDocument(values, id); } @@ -152,11 +154,20 @@ public string GetSerializedMetadata() .Where(x => !propertiesToSkip.Contains(x.Key)) .ToDictionary(x => x.Key, x => x.Value); - return JsonSerializer.Serialize(distinctMetadata); + return JsonSerializer.Serialize(distinctMetadata, s_jsonSerializerOptions); } internal UpdateVectorRequest ToUpdateRequest() { return UpdateVectorRequest.FromPineconeDocument(this); } + + private static readonly JsonSerializerOptions s_jsonSerializerOptions = CreateSerializerOptions(); + + private static JsonSerializerOptions CreateSerializerOptions() + { + var jso = new JsonSerializerOptions(); + jso.Converters.Add(new ReadOnlyMemoryConverter()); + return jso; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeDocumentExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeDocumentExtensions.cs index 3c4341b615f3..0962fd43ac44 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeDocumentExtensions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeDocumentExtensions.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Globalization; using System.Text.Json; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Memory; namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone; @@ -50,7 +49,7 @@ public static PineconeDocument ToPineconeDocument(this MemoryRecord memoryRecord } return PineconeDocument - .Create(key, memoryRecord.Embedding.Vector) + .Create(key, memoryRecord.Embedding) .WithMetadata(metadata); } @@ -70,7 +69,7 @@ public static MemoryRecord ToMemoryRecord(this PineconeDocument pineconeDocument /// Instance of . internal static MemoryRecord ToMemoryRecord(this PineconeDocument pineconeDocument, bool transferVectorOwnership) { - Embedding embedding = new(pineconeDocument.Values, transferVectorOwnership); + ReadOnlyMemory embedding = pineconeDocument.Values; string additionalMetadataJson = pineconeDocument.GetSerializedMetadata(); diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeKernelBuilderExtensions.cs index 000864986eea..2db0698f73d4 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeKernelBuilderExtensions.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using System.ComponentModel; using System.Net.Http; using Microsoft.SemanticKernel.Connectors.Memory.Pinecone; @@ -10,6 +12,8 @@ namespace Microsoft.SemanticKernel; /// /// Provides extension methods for the class to configure Pinecone connectors. /// +[Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use PineconeMemoryBuilderExtensions instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class PineconeKernelBuilderExtensions { /// @@ -20,20 +24,22 @@ public static class PineconeKernelBuilderExtensions /// The API key for accessing Pinecone services. /// An optional HttpClient instance for making HTTP requests. /// Self instance + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use PineconeMemoryBuilderExtensions.WithPineconeMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] public static KernelBuilder WithPineconeMemoryStore(this KernelBuilder builder, string environment, string apiKey, HttpClient? httpClient = null) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory, httpHandlerFactory) => { var client = new PineconeClient( environment, apiKey, - parameters.Logger, - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger)); + loggerFactory, + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory)); - return new PineconeMemoryStore(client, parameters.Logger); + return new PineconeMemoryStore(client, loggerFactory); }); return builder; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryBuilderExtensions.cs new file mode 100644 index 000000000000..984fd97e42db --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryBuilderExtensions.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.SemanticKernel.Plugins.Memory; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone; + +/// +/// Provides extension methods for the class to configure Pinecone connector. +/// +public static class PineconeMemoryBuilderExtensions +{ + /// + /// Registers Pinecone memory connector. + /// + /// The instance. + /// The environment for Pinecone. + /// The API key for accessing Pinecone services. + /// An optional HttpClient instance for making HTTP requests. + /// Updated Memory builder including Pinecone memory connector. + public static MemoryBuilder WithPineconeMemoryStore( + this MemoryBuilder builder, + string environment, + string apiKey, + HttpClient? httpClient = null) + { + builder.WithMemoryStore((loggerFactory, httpHandlerFactory) => + { + var client = new PineconeClient( + environment, + apiKey, + loggerFactory, + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory)); + + return new PineconeMemoryStore(client, loggerFactory); + }); + + return builder; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryException.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryException.cs deleted file mode 100644 index f4d4864457f0..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryException.cs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone; - -#pragma warning disable CA1032 // Implement standard exception constructors - -/// -/// Custom exceptions for the Pinecone connector. -/// -public class PineconeMemoryException : SKException -{ - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public PineconeMemoryException(ErrorCodes errorCode, string? message) - : this(errorCode: errorCode, message: message, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code and inner exception. - /// - /// The error code. - /// The exception that is the cause of the current exception. - public PineconeMemoryException(ErrorCodes errorCode, Exception? innerException) - : this(errorCode: errorCode, message: null, innerException: innerException) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public PineconeMemoryException(ErrorCodes errorCode, string? message = null, Exception? innerException = null) - : base(message: GetDefaultMessage(errorCode: errorCode, message: message, innerException: innerException), innerException: innerException) - { - this.ErrorCode = errorCode; - } - - protected PineconeMemoryException() : base() - { - } - - protected PineconeMemoryException(string? message) : base(message) - { - } - - protected PineconeMemoryException(string? message, Exception? innerException) : base(message, innerException) - { - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// Translate the error code into a default message. - private static string GetDefaultMessage(ErrorCodes errorCode, string? message, Exception? innerException) - { - if (message is not null) - { - return message; - } - - string description = errorCode switch - { - ErrorCodes.UnableToDeserializeDocumentMetadata => "Unable to deserialize document metadata", - ErrorCodes.FailedToUpsertVectors => "Failed to upsert vectors", - ErrorCodes.FailedToGetVectorData => "Failed to get vector data", - ErrorCodes.FailedToRemoveVectorData => "Failed to remove vector data", - ErrorCodes.FailedToConvertMemoryRecordToPineconeDocument => "Failed to convert memory record to Pinecone document", - ErrorCodes.FailedToConvertPineconeDocumentToMemoryRecord => "Failed to convert Pinecone document record to memory record", - _ => $"Unknown error ({errorCode:G})" - }; - - return innerException is not null ? $"{description}: {innerException.Message}" : description; - } - - /// - /// Error codes for the Pinecone connector exceptions. - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError, - - /// - /// The index is not found. - /// - IndexNotFound, - - /// - /// The index is not ready. - /// - IndexNotReady, - - /// - /// The index host is unknown. - /// - UnknownIndexHost, - - /// - /// Failed to deserialize the record payload. - /// - UnableToDeserializeDocumentMetadata, - - /// - /// Failed to upsert the vector. - /// - FailedToUpsertVectors, - - /// - /// Failed to get vector data from Pinecone. - /// - FailedToGetVectorData, - - /// - /// Failed to remove vector data from Pinecone. - /// - FailedToRemoveVectorData, - - /// - /// Failed to convert a memory record to a Pinecone vector record. - /// - FailedToConvertMemoryRecordToPineconeDocument, - - /// - /// Failed to convert a Pinecone vector record to a memory record. - /// - FailedToConvertPineconeDocumentToMemoryRecord - } -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryStore.cs index 3fbead30b116..d9d8f5e4468d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeMemoryStore.cs @@ -1,15 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Linq; -using System.Net.Http; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone; @@ -30,13 +30,13 @@ public class PineconeMemoryStore : IPineconeMemoryStore /// Initializes a new instance of the class. /// /// Instance of Pinecone client which implements interface. - /// Instance of logger. + /// The to use for logging. If null, no logging will be performed. public PineconeMemoryStore( IPineconeClient pineconeClient, - ILogger? logger = null) + ILoggerFactory? loggerFactory = null) { this._pineconeClient = pineconeClient; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(PineconeMemoryStore)) : NullLogger.Instance; } /// @@ -44,14 +44,14 @@ public PineconeMemoryStore( /// /// Pinecone project environment, see https://docs.pinecone.io/docs/projects#project-environment. /// Pinecone API key. - /// Instance of logger. + /// The to use for logging. If null, no logging will be performed. public PineconeMemoryStore( string pineconeEnvironment, string apiKey, - ILogger? logger = null) + ILoggerFactory? loggerFactory = null) { - this._pineconeClient = new PineconeClient(pineconeEnvironment, apiKey, logger); - this._logger = logger ?? NullLogger.Instance; + this._pineconeClient = new PineconeClient(pineconeEnvironment, apiKey, loggerFactory); + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(PineconeMemoryStore)) : NullLogger.Instance; } /// @@ -65,9 +65,7 @@ public async Task CreateCollectionAsync(string collectionName, CancellationToken { if (!await this.DoesCollectionExistAsync(collectionName, cancellationToken).ConfigureAwait(false)) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.IndexNotReady, - "Index creation is not supported within memory store. " + + throw new SKException("Index creation is not supported within memory store. " + $"It should be created manually or using {nameof(IPineconeClient.CreateIndexAsync)}. " + $"Ensure index state is {IndexState.Ready}."); } @@ -128,12 +126,10 @@ public async Task UpsertToNamespaceAsync(string indexName, string indexN { await request.ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToUpsertVectors, - $"Failed to upsert due to HttpRequestException: {ex.Message}", - ex); + this._logger.LogError(ex, "Failed to upsert: {Message}", ex.Message); + throw; } return vectorData.Id; @@ -211,12 +207,10 @@ public async IAsyncEnumerable UpsertBatchToNamespaceAsync( { await Task.WhenAll(tasks).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToUpsertVectors, - $"Failed to upsert due to HttpRequestException: {ex.Message}", - ex); + this._logger.LogError(ex, "Failed to upsert batch: {Message}", ex.Message); + throw; } foreach (PineconeDocument? v in vectorData) @@ -259,19 +253,10 @@ public async IAsyncEnumerable UpsertBatchToNamespaceAsync( return record?.ToMemoryRecord(transferVectorOwnership: true); } } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToGetVectorData, - $"Failed to get vector data from Pinecone: {ex.Message}", - ex); - } - catch (MemoryException ex) - { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToConvertPineconeDocumentToMemoryRecord, - $"Failed deserialize Pinecone response to Memory Record: {ex.Message}", - ex); + this._logger.LogError(ex, "Failed to get vector data from Pinecone: {Message}", ex.Message); + throw; } return null; @@ -323,7 +308,7 @@ public async IAsyncEnumerable GetBatchFromNamespaceAsync( /// If true, the embedding will be returned in the memory record. /// Cancellation token. /// - /// + /// public async IAsyncEnumerable GetWithDocumentIdAsync(string indexName, string documentId, int limit = 3, @@ -388,10 +373,10 @@ in documentIds.Select( .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException ex) { - this._logger.LogError(e, "Error getting batch with filter from Pinecone."); - yield break; + this._logger.LogError(ex, "Error getting batch with filter from Pinecone: {Message}", ex.Message); + throw; } foreach (PineconeDocument? record in vectorDataList) @@ -421,12 +406,10 @@ await this._pineconeClient.DeleteAsync(indexName, new[] indexNamespace, cancellationToken: cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToRemoveVectorData, - $"Failed to remove vector data from Pinecone {ex.Message}", - ex); + this._logger.LogError(ex, "Failed to remove vector data from Pinecone: {Message}", ex.Message); + throw; } } @@ -461,12 +444,10 @@ await this._pineconeClient.DeleteAsync( filter, cancellationToken: cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToRemoveVectorData, - $"Failed to remove vector data from Pinecone {ex.Message}", - ex); + this._logger.LogError(ex, "Failed to remove vector data from Pinecone: {Message}", ex.Message); + throw; } } @@ -478,7 +459,7 @@ await this._pineconeClient.DeleteAsync( /// The name associated with a collection of embeddings. /// Cancellation token. /// - /// + /// public async Task RemoveWithDocumentIdAsync(string indexName, string documentId, string indexNamespace, CancellationToken cancellationToken = default) { try @@ -488,12 +469,10 @@ public async Task RemoveWithDocumentIdAsync(string indexName, string documentId, { "document_Id", documentId } }, cancellationToken: cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToRemoveVectorData, - $"Failed to remove vector data from Pinecone {ex.Message}", - ex); + this._logger.LogError(ex, "Failed to remove vector data from Pinecone: {Message}", ex.Message); + throw; } } @@ -505,7 +484,7 @@ public async Task RemoveWithDocumentIdAsync(string indexName, string documentId, /// The name associated with a collection of embeddings. /// Cancellation token. /// - /// + /// public async Task RemoveWithDocumentIdBatchAsync( string indexName, IEnumerable documentIds, @@ -520,12 +499,10 @@ public async Task RemoveWithDocumentIdBatchAsync( await Task.WhenAll(tasks).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new PineconeMemoryException( - PineconeMemoryException.ErrorCodes.FailedToRemoveVectorData, - $"Error in batch removing data from Pinecone {ex.Message}", - ex); + this._logger.LogError(ex, "Error in batch removing data from Pinecone: {Message}", ex.Message); + throw; } } @@ -538,7 +515,7 @@ public async Task RemoveWithDocumentIdBatchAsync( /// public IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, @@ -558,7 +535,7 @@ public async Task RemoveWithDocumentIdBatchAsync( public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesFromNamespaceAsync( string indexName, string indexNamespace, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, @@ -566,7 +543,7 @@ public async Task RemoveWithDocumentIdBatchAsync( { IAsyncEnumerable<(PineconeDocument, double)> results = this._pineconeClient.GetMostRelevantAsync( indexName, - embedding.Vector, + embedding, minRelevanceScore, limit, withEmbeddings, @@ -589,7 +566,7 @@ public async Task RemoveWithDocumentIdBatchAsync( /// public async Task<(MemoryRecord, double)?> GetNearestMatchAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) @@ -607,7 +584,7 @@ public async Task RemoveWithDocumentIdBatchAsync( public async Task<(MemoryRecord, double)?> GetNearestMatchFromNamespaceAsync( string indexName, string indexNamespace, - Embedding embedding, + ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) @@ -629,7 +606,7 @@ public async Task RemoveWithDocumentIdBatchAsync( /// public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesWithFilterAsync( string indexName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, Dictionary filter, double minRelevanceScore = 0D, @@ -639,7 +616,7 @@ public async Task RemoveWithDocumentIdBatchAsync( { IAsyncEnumerable<(PineconeDocument, double)> results = this._pineconeClient.GetMostRelevantAsync( indexName, - embedding.Vector, + embedding, minRelevanceScore, limit, withEmbeddings, diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeUtils.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeUtils.cs index 1f90b75e2dbd..445234273aa3 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeUtils.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeUtils.cs @@ -249,6 +249,7 @@ public static string PodTypeToString(PodType podType) PodType.S1X2 => "s1x2", PodType.S1X4 => "s1x4", PodType.S1X8 => "s1x8", + PodType.Starter => "starter", _ => string.Empty }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/Connectors.Memory.Postgres.csproj b/dotnet/src/Connectors/Connectors.Memory.Postgres/Connectors.Memory.Postgres.csproj index b1972599424e..b8ce22772d9c 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/Connectors.Memory.Postgres.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/Connectors.Memory.Postgres.csproj @@ -14,7 +14,7 @@ Semantic Kernel - Postgres Connector - Postgres(with pgvector extension) connector for Semantic Kernel skills and semantic memory + Postgres(with pgvector extension) connector for Semantic Kernel plugins and semantic memory @@ -26,7 +26,8 @@ - + + diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresKernelBuilderExtensions.cs index 4baa6728c2df..5bbe95e28189 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresKernelBuilderExtensions.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using System.ComponentModel; using Microsoft.SemanticKernel.Connectors.Memory.Postgres; using Npgsql; @@ -10,22 +12,49 @@ namespace Microsoft.SemanticKernel; /// /// Provides extension methods for the class to configure Postgres connectors. /// +[Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use PostgresMemoryBuilderExtensions instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class PostgresKernelBuilderExtensions { + /// + /// Registers Postgres Memory Store. + /// + /// The instance + /// Postgres database connection string. + /// Embedding vector size. + /// Schema of collection tables. + /// Self instance + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use PostgresMemoryBuilderExtensions.WithPostgresMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public static KernelBuilder WithPostgresMemoryStore(this KernelBuilder builder, + string connectionString, + int vectorSize, + string schema = PostgresMemoryStore.DefaultSchema) + { + builder.WithMemoryStorage((loggerFactory) => + { + return new PostgresMemoryStore(connectionString, vectorSize, schema); + }); + + return builder; + } + /// /// Registers Postgres Memory Store. /// /// The instance /// Postgres data source. /// Embedding vector size. - /// Schema of collection tables. The default value is "public". + /// Schema of collection tables. /// Self instance + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use PostgresMemoryBuilderExtensions.WithPostgresMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] public static KernelBuilder WithPostgresMemoryStore(this KernelBuilder builder, NpgsqlDataSource dataSource, int vectorSize, string schema = PostgresMemoryStore.DefaultSchema) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory) => { return new PostgresMemoryStore(dataSource, vectorSize, schema); }); @@ -39,9 +68,11 @@ public static KernelBuilder WithPostgresMemoryStore(this KernelBuilder builder, /// The instance /// Postgres database client. /// Self instance + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use PostgresMemoryBuilderExtensions.WithPostgresMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] public static KernelBuilder WithPostgresMemoryStore(this KernelBuilder builder, IPostgresDbClient postgresDbClient) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory) => { return new PostgresMemoryStore(postgresDbClient); }); diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresMemoryBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresMemoryBuilderExtensions.cs new file mode 100644 index 000000000000..7037c3c74d6b --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresMemoryBuilderExtensions.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Plugins.Memory; +using Npgsql; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Postgres; + +/// +/// Provides extension methods for the class to configure Postgres connector. +/// +public static class PostgresMemoryBuilderExtensions +{ + /// + /// Registers Postgres memory connector. + /// + /// The instance. + /// Postgres database connection string. + /// Embedding vector size. + /// Schema of collection tables. + /// Updated Memory builder including Postgres memory connector. + public static MemoryBuilder WithPostgresMemoryStore( + this MemoryBuilder builder, + string connectionString, + int vectorSize, + string schema = PostgresMemoryStore.DefaultSchema) + { + builder.WithMemoryStore((_) => + { + return new PostgresMemoryStore(connectionString, vectorSize, schema); + }); + + return builder; + } + + /// + /// Registers Postgres memory connector. + /// + /// The instance. + /// Postgres data source. + /// Embedding vector size. + /// Schema of collection tables. + /// Updated Memory builder including Postgres memory connector. + public static MemoryBuilder WithPostgresMemoryStore( + this MemoryBuilder builder, + NpgsqlDataSource dataSource, + int vectorSize, + string schema = PostgresMemoryStore.DefaultSchema) + { + builder.WithMemoryStore((_) => + { + return new PostgresMemoryStore(dataSource, vectorSize, schema); + }); + + return builder; + } + + /// + /// Registers Postgres memory connector. + /// + /// The instance. + /// Postgres database client. + /// Updated Memory builder including Postgres memory connector. + public static MemoryBuilder WithPostgresMemoryStore( + this MemoryBuilder builder, + IPostgresDbClient postgresDbClient) + { + builder.WithMemoryStore((_) => + { + return new PostgresMemoryStore(postgresDbClient); + }); + + return builder; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresMemoryStore.cs index b95ac91f8fb1..18b5bbbb4035 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresMemoryStore.cs @@ -1,39 +1,60 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; using Npgsql; using Pgvector; +using Pgvector.Npgsql; namespace Microsoft.SemanticKernel.Connectors.Memory.Postgres; /// /// An implementation of backed by a Postgres database with pgvector extension. /// -/// The embedded data is saved to the Postgres database specified in the constructor. +/// +/// The embedded data is saved to the Postgres database specified in the constructor. /// Similarity search capability is provided through the pgvector extension. Use Postgres's "Table" to implement "Collection". /// -public class PostgresMemoryStore : IMemoryStore +public class PostgresMemoryStore : IMemoryStore, IDisposable { internal const string DefaultSchema = "public"; + /// + /// Initializes a new instance of the class. + /// + /// Postgres database connection string. + /// Embedding vector size. + /// Database schema of collection tables. + public PostgresMemoryStore(string connectionString, int vectorSize, string schema = DefaultSchema) + { + NpgsqlDataSourceBuilder dataSourceBuilder = new(connectionString); + dataSourceBuilder.UseVector(); + this._dataSource = dataSourceBuilder.Build(); + this._postgresDbClient = new PostgresDbClient(this._dataSource, schema, vectorSize); + } + /// /// Initializes a new instance of the class. /// /// Postgres data source. /// Embedding vector size. - /// Database schema of collection tables. The default value is "public". + /// Database schema of collection tables. public PostgresMemoryStore(NpgsqlDataSource dataSource, int vectorSize, string schema = DefaultSchema) : this(new PostgresDbClient(dataSource, schema, vectorSize)) { } + /// + /// Initializes a new instance of the class. + /// + /// An instance of . public PostgresMemoryStore(IPostgresDbClient postgresDbClient) { this._postgresDbClient = postgresDbClient; @@ -135,7 +156,7 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke /// public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, @@ -150,7 +171,7 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke IAsyncEnumerable<(PostgresMemoryEntry, double)> results = this._postgresDbClient.GetNearestMatchesAsync( tableName: collectionName, - embedding: new Vector(embedding.Vector.ToArray()), + embedding: new Vector(GetOrCreateArray(embedding)), limit: limit, minRelevanceScore: minRelevanceScore, withEmbeddings: withEmbeddings, @@ -163,7 +184,7 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke } /// - public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding embedding, double minRelevanceScore = 0, bool withEmbedding = false, + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) { return await this.GetNearestMatchesAsync( @@ -175,9 +196,28 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke cancellationToken: cancellationToken).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); } + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Disposes the managed resources. + /// + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._dataSource?.Dispose(); + } + } + #region private ================================================================================ private readonly IPostgresDbClient _postgresDbClient; + private readonly NpgsqlDataSource? _dataSource; private async Task InternalUpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken) { @@ -187,7 +227,7 @@ await this._postgresDbClient.UpsertAsync( tableName: collectionName, key: record.Key, metadata: record.GetSerializedMetadata(), - embedding: new Vector(record.Embedding.Vector.ToArray()), + embedding: new Vector(GetOrCreateArray(record.Embedding)), timestamp: record.Timestamp?.UtcDateTime, cancellationToken: cancellationToken).ConfigureAwait(false); @@ -198,10 +238,16 @@ private MemoryRecord GetMemoryRecordFromEntry(PostgresMemoryEntry entry) { return MemoryRecord.FromJsonMetadata( json: entry.MetadataString, - embedding: entry.Embedding != null ? new Embedding(entry.Embedding!.ToArray()) : Embedding.Empty, + embedding: entry.Embedding?.ToArray() ?? ReadOnlyMemory.Empty, key: entry.Key, timestamp: entry.Timestamp?.ToLocalTime()); } + private static float[] GetOrCreateArray(ReadOnlyMemory memory) => + MemoryMarshal.TryGetArray(memory, out ArraySegment array) && + array.Count == array.Array!.Length ? + array.Array : + memory.ToArray(); + #endregion } diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Connectors.Memory.Qdrant.csproj b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Connectors.Memory.Qdrant.csproj index 32ba6a905db0..361115c0f621 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Connectors.Memory.Qdrant.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Connectors.Memory.Qdrant.csproj @@ -14,7 +14,7 @@ Semantic Kernel - Qdrant Connector - Qdrant connector for Semantic Kernel skills and semantic memory + Qdrant connector for Semantic Kernel plugins and semantic memory @@ -22,7 +22,8 @@ - + + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Diagnostics/QdrantMemoryException.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Diagnostics/QdrantMemoryException.cs deleted file mode 100644 index b386c341c150..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Diagnostics/QdrantMemoryException.cs +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Diagnostics; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown for errors related to the Qdrant connector. -/// -public class QdrantMemoryException : SKException -{ - /// - /// Initializes a new instance of the class with a provided error code. - /// - /// The error code. - public QdrantMemoryException(ErrorCodes errorCode) - : this(errorCode, message: null, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public QdrantMemoryException(ErrorCodes errorCode, string? message) - : this(errorCode, message, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code and inner exception. - /// - /// The error code. - /// The exception that is the cause of the current exception. - public QdrantMemoryException(ErrorCodes errorCode, Exception? innerException) - : this(errorCode, message: null, innerException) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public QdrantMemoryException(ErrorCodes errorCode, string? message, Exception? innerException) - : base(GetDefaultMessage(errorCode, message, innerException), innerException) - { - this.ErrorCode = errorCode; - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// Translate the error code into a default message. - private static string GetDefaultMessage(ErrorCodes errorCode, string? message, Exception? innerException) - { - if (message is not null) - { - return message; - } - - string description = errorCode switch - { - ErrorCodes.UnableToDeserializeRecordPayload => "Unable to deserialize record payload", - ErrorCodes.FailedToUpsertVectors => "Failed to upsert vectors", - ErrorCodes.FailedToGetVectorData => "Failed to get vector data", - ErrorCodes.FailedToRemoveVectorData => "Failed to remove vector data", - ErrorCodes.FailedToConvertMemoryRecordToQdrantVectorRecord => "Failed to convert memory record to Qdrant vector record", - ErrorCodes.FailedToConvertQdrantVectorRecordToMemoryRecord => "Failed to convert Qdrant vector record to memory record", - _ => $"Unknown error ({errorCode:G})", - }; - - return innerException is not null ? $"{description}: {innerException.Message}" : description; - } - - /// - /// Error codes for the Qdrant connector exceptions. - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError = -1, - - /// - /// Failed to deserialize the record payload. - /// - UnableToDeserializeRecordPayload, - - /// - /// Failed to upsert the vector. - /// - FailedToUpsertVectors, - - /// - /// Failed to get vector data from Qdrant. - /// - FailedToGetVectorData, - - /// - /// Failed to remove vector data from Qdrant. - /// - FailedToRemoveVectorData, - - /// - /// Failed to convert a memory record to a Qdrant vector record. - /// - FailedToConvertMemoryRecordToQdrantVectorRecord, - - /// - /// Failed to convert a Qdrant vector record to a memory record. - /// - FailedToConvertQdrantVectorRecordToMemoryRecord - } -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/GetVectorsResponse.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/GetVectorsResponse.cs index 9acfea54abc4..49813b86e1f6 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/GetVectorsResponse.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/GetVectorsResponse.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; @@ -19,10 +20,10 @@ internal sealed class Record [JsonPropertyName("vector")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public IEnumerable? Vector { get; set; } + public ReadOnlyMemory? Vector { get; set; } [JsonConstructor] - public Record(string id, Dictionary? payload, IEnumerable? vector) + public Record(string id, Dictionary? payload, ReadOnlyMemory? vector) { this.Id = id; this.Payload = payload; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/SearchVectorsRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/SearchVectorsRequest.cs index 65003e619b2e..1800c3aca6c9 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/SearchVectorsRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/SearchVectorsRequest.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Net.Http; using System.Text.Json.Serialization; @@ -10,7 +11,7 @@ namespace Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Http.ApiSchema; internal sealed class SearchVectorsRequest : IValidatable { [JsonPropertyName("vector")] - public IEnumerable StartingVector { get; set; } = System.Array.Empty(); + public ReadOnlyMemory StartingVector { get; set; } [JsonPropertyName("filter")] public Filter Filters { get; set; } @@ -40,7 +41,7 @@ public static SearchVectorsRequest Create(string collectionName, int vectorSize) return new SearchVectorsRequest(collectionName).SimilarTo(new float[vectorSize]); } - public SearchVectorsRequest SimilarTo(IEnumerable vector) + public SearchVectorsRequest SimilarTo(ReadOnlyMemory vector) { this.StartingVector = vector; return this; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/SearchVectorsResponse.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/SearchVectorsResponse.cs index 8f15ba2b1e4e..4d2b7bd9ef30 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/SearchVectorsResponse.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/SearchVectorsResponse.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Http.ApiSchema; @@ -24,10 +26,11 @@ internal sealed class ScoredPoint public Dictionary Payload { get; set; } [JsonPropertyName("vector")] - public IEnumerable? Vector { get; } + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Vector { get; } [JsonConstructor] - public ScoredPoint(string id, double? score, Dictionary payload, IEnumerable vector) + public ScoredPoint(string id, double? score, Dictionary payload, ReadOnlyMemory vector) { this.Id = id; this.Score = score; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/UpsertVectorRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/UpsertVectorRequest.cs index a62d84346f9b..b258caee498e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/UpsertVectorRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Http/ApiSchema/UpsertVectorRequest.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Net.Http; using System.Text.Json.Serialization; @@ -50,7 +51,7 @@ internal sealed class BatchRequest public IList Ids { get; set; } [JsonPropertyName("vectors")] - public IList> Vectors { get; set; } + public IList> Vectors { get; set; } [JsonPropertyName("payloads")] public IList> Payloads { get; set; } @@ -58,7 +59,7 @@ internal sealed class BatchRequest internal BatchRequest() { this.Ids = new List(); - this.Vectors = new List>(); + this.Vectors = new List>(); this.Payloads = new List>(); } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/IQdrantVectorDbClient.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/IQdrantVectorDbClient.cs index f426f4dd5b3b..8bd6e89275e0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/IQdrantVectorDbClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/IQdrantVectorDbClient.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -68,7 +69,7 @@ public IAsyncEnumerable GetVectorsByIdAsync(string collectio /// The to monitor for cancellation requests. The default is . public IAsyncEnumerable<(QdrantVectorRecord, double)> FindNearestInCollectionAsync( string collectionName, - IEnumerable target, + ReadOnlyMemory target, double threshold, int top = 1, bool withVectors = false, diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantKernelBuilderExtensions.cs index 2e6e6e3a3934..0ed45e671fbb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantKernelBuilderExtensions.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using System.ComponentModel; using System.Net.Http; using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; @@ -10,6 +12,8 @@ namespace Microsoft.SemanticKernel; /// /// Provides extension methods for the class to configure Qdrant memory connector. /// +[Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use QdrantMemoryBuilderExtensions instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class QdrantKernelBuilderExtensions { /// @@ -19,19 +23,21 @@ public static class QdrantKernelBuilderExtensions /// The Qdrant Vector Database endpoint. /// The size of the vectors. /// Self instance + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use QdrantMemoryBuilderExtensions.WithQdrantMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] public static KernelBuilder WithQdrantMemoryStore(this KernelBuilder builder, string endpoint, int vectorSize) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory, httpHandlerFactory) => { var client = new QdrantVectorDbClient( - HttpClientProvider.GetHttpClient(parameters.Config, null, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, null, loggerFactory), vectorSize, endpoint, - parameters.Logger); + loggerFactory); - return new QdrantMemoryStore(client, parameters.Logger); + return new QdrantMemoryStore(client, loggerFactory); }); return builder; @@ -45,20 +51,22 @@ public static KernelBuilder WithQdrantMemoryStore(this KernelBuilder builder, /// The size of the vectors. /// The Qdrant Vector Database endpoint. If not specified, the base address of the HTTP client is used. /// Self instance + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use QdrantMemoryBuilderExtensions.WithQdrantMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] public static KernelBuilder WithQdrantMemoryStore(this KernelBuilder builder, HttpClient httpClient, int vectorSize, string? endpoint = null) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory, httpHandlerFactory) => { var client = new QdrantVectorDbClient( - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), vectorSize, endpoint, - parameters.Logger); + loggerFactory); - return new QdrantMemoryStore(client, parameters.Logger); + return new QdrantMemoryStore(client, loggerFactory); }); return builder; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantMemoryBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantMemoryBuilderExtensions.cs new file mode 100644 index 000000000000..8f18fbe4f850 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantMemoryBuilderExtensions.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.SemanticKernel.Plugins.Memory; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Qdrant; + +/// +/// Provides extension methods for the class to configure Qdrant connector. +/// +public static class QdrantMemoryBuilderExtensions +{ + /// + /// Registers Qdrant memory connector. + /// + /// The instance. + /// The Qdrant Vector Database endpoint. + /// The size of the vectors. + /// Updated Memory builder including Qdrant memory connector. + public static MemoryBuilder WithQdrantMemoryStore( + this MemoryBuilder builder, + string endpoint, + int vectorSize) + { + builder.WithMemoryStore((loggerFactory, httpHandlerFactory) => + { + var client = new QdrantVectorDbClient( + HttpClientProvider.GetHttpClient(httpHandlerFactory, null, loggerFactory), + vectorSize, + endpoint, + loggerFactory); + + return new QdrantMemoryStore(client, loggerFactory); + }); + + return builder; + } + + /// + /// Registers Qdrant memory connector. + /// + /// The instance. + /// The optional instance used for making HTTP requests. + /// The size of the vectors. + /// The Qdrant Vector Database endpoint. If not specified, the base address of the HTTP client is used. + /// Updated Memory builder including Qdrant memory connector. + public static MemoryBuilder WithQdrantMemoryStore( + this MemoryBuilder builder, + HttpClient httpClient, + int vectorSize, + string? endpoint = null) + { + builder.WithMemoryStore((loggerFactory, httpHandlerFactory) => + { + var client = new QdrantVectorDbClient( + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + vectorSize, + endpoint, + loggerFactory); + + return new QdrantMemoryStore(client, loggerFactory); + }); + + return builder; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantMemoryStore.cs index e6325f4ebe45..99809f4e7759 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantMemoryStore.cs @@ -8,8 +8,8 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Diagnostics; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; namespace Microsoft.SemanticKernel.Connectors.Memory.Qdrant; @@ -25,18 +25,18 @@ public class QdrantMemoryStore : IMemoryStore /// /// The Qdrant Vector Database memory store logger. /// - private readonly ILogger? _logger; + private readonly ILogger _logger; /// /// Initializes a new instance of the class. /// /// The Qdrant Vector Database endpoint. /// The size of the vectors used. - /// Optional logger instance. - public QdrantMemoryStore(string endpoint, int vectorSize, ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + public QdrantMemoryStore(string endpoint, int vectorSize, ILoggerFactory? loggerFactory = null) { - this._qdrantClient = new QdrantVectorDbClient(endpoint, vectorSize, logger); - this._logger = logger; + this._qdrantClient = new QdrantVectorDbClient(endpoint, vectorSize, loggerFactory); + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(QdrantMemoryStore)) : NullLogger.Instance; } /// @@ -45,22 +45,22 @@ public QdrantMemoryStore(string endpoint, int vectorSize, ILogger? logger = null /// The instance used for making HTTP requests. /// The size of the vectors used in the Qdrant Vector Database. /// The optional endpoint URL for the Qdrant Vector Database. If not specified, the base address of the HTTP client is used. - /// Optional logger instance. - public QdrantMemoryStore(HttpClient httpClient, int vectorSize, string? endpoint = null, ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + public QdrantMemoryStore(HttpClient httpClient, int vectorSize, string? endpoint = null, ILoggerFactory? loggerFactory = null) { - this._qdrantClient = new QdrantVectorDbClient(httpClient, vectorSize, endpoint, logger); - this._logger = logger; + this._qdrantClient = new QdrantVectorDbClient(httpClient, vectorSize, endpoint, loggerFactory); + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(QdrantMemoryStore)) : NullLogger.Instance; } /// /// Initializes a new instance of the class. /// /// The Qdrant Db client for interacting with Qdrant Vector Database. - /// Optional logger instance. - public QdrantMemoryStore(IQdrantVectorDbClient client, ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + public QdrantMemoryStore(IQdrantVectorDbClient client, ILoggerFactory? loggerFactory = null) { this._qdrantClient = client; - this._logger = logger; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(QdrantMemoryStore)) : NullLogger.Instance; } /// @@ -100,7 +100,7 @@ public async Task UpsertAsync(string collectionName, MemoryRecord record if (vectorData == null) { - throw new QdrantMemoryException(QdrantMemoryException.ErrorCodes.FailedToConvertMemoryRecordToQdrantVectorRecord); + throw new SKException("Failed to convert memory record to Qdrant vector record"); } try @@ -110,11 +110,10 @@ await this._qdrantClient.UpsertVectorsAsync( new[] { vectorData }, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToUpsertVectors, - ex); + this._logger.LogError(ex, "Failed to upsert vectors: {Message}", ex.Message); + throw; } return vectorData.PointId; @@ -134,13 +133,11 @@ await this._qdrantClient.UpsertVectorsAsync( vectorData, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToUpsertVectors, - ex); + this._logger.LogError(ex, "Failed to upsert vectors: {Message}", ex.Message); + throw; } - foreach (var v in vectorData) { yield return v.PointId; @@ -157,20 +154,13 @@ await this._qdrantClient.UpsertVectorsAsync( return MemoryRecord.FromJsonMetadata( json: vectorData.GetSerializedPayload(), - embedding: new Embedding(vectorData.Embedding, transferOwnership: true), + embedding: vectorData.Embedding, key: vectorData.PointId); } - catch (HttpRequestException ex) - { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToGetVectorData, - ex); - } - catch (MemoryException ex) + catch (HttpOperationException ex) { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToConvertQdrantVectorRecordToMemoryRecord, - ex); + this._logger.LogError(ex, "Failed to get vector data: {Message}", ex.Message); + throw; } } @@ -196,7 +186,7 @@ public async IAsyncEnumerable GetBatchAsync(string collectionName, /// If true, the embedding will be returned in the memory record. /// The to monitor for cancellation requests. The default is . /// Memory record - /// + /// public async Task GetWithPointIdAsync(string collectionName, string pointId, bool withEmbedding = false, CancellationToken cancellationToken = default) { @@ -211,19 +201,12 @@ public async IAsyncEnumerable GetBatchAsync(string collectionName, return MemoryRecord.FromJsonMetadata( json: vectorData.GetSerializedPayload(), - embedding: new Embedding(vectorData.Embedding, transferOwnership: true)); - } - catch (HttpRequestException ex) - { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToGetVectorData, - ex); + embedding: vectorData.Embedding); } - catch (MemoryException ex) + catch (HttpOperationException ex) { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToConvertQdrantVectorRecordToMemoryRecord, - ex); + this._logger.LogError(ex, "Failed to get vector data: {Message}", ex.Message); + throw; } } @@ -248,7 +231,7 @@ public async IAsyncEnumerable GetWithPointIdBatchAsync( { yield return MemoryRecord.FromJsonMetadata( json: vectorData.GetSerializedPayload(), - embedding: new Embedding(vectorData.Embedding, transferOwnership: true), + embedding: vectorData.Embedding, key: vectorData.PointId); } } @@ -260,11 +243,10 @@ public async Task RemoveAsync(string collectionName, string key, CancellationTok { await this._qdrantClient.DeleteVectorByPayloadIdAsync(collectionName, key, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToRemoveVectorData, - ex); + this._logger.LogError(ex, "Failed to remove vector data: {Message}", ex.Message); + throw; } } @@ -280,18 +262,17 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke /// The name associated with a collection of embeddings. /// The unique indexed ID associated with the Qdrant vector record to remove. /// The to monitor for cancellation requests. The default is . - /// + /// public async Task RemoveWithPointIdAsync(string collectionName, string pointId, CancellationToken cancellationToken = default) { try { await this._qdrantClient.DeleteVectorsByIdAsync(collectionName, new[] { pointId }, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToRemoveVectorData, - ex); + this._logger.LogError(ex, "Failed to remove vector data: {Message}", ex.Message); + throw; } } @@ -301,25 +282,24 @@ public async Task RemoveWithPointIdAsync(string collectionName, string pointId, /// The name associated with a collection of embeddings. /// The unique indexed IDs associated with the Qdrant vector records to remove. /// The to monitor for cancellation requests. The default is . - /// + /// public async Task RemoveWithPointIdBatchAsync(string collectionName, IEnumerable pointIds, CancellationToken cancellationToken = default) { try { await this._qdrantClient.DeleteVectorsByIdAsync(collectionName, pointIds, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException ex) + catch (HttpOperationException ex) { - throw new QdrantMemoryException( - QdrantMemoryException.ErrorCodes.FailedToRemoveVectorData, - ex); + this._logger.LogError(ex, "Failed to remove vector data: {Message}", ex.Message); + throw; } } /// public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, @@ -328,7 +308,7 @@ public async Task RemoveWithPointIdBatchAsync(string collectionName, IEnumerable IAsyncEnumerator<(QdrantVectorRecord, double)> enumerator = this._qdrantClient .FindNearestInCollectionAsync( collectionName: collectionName, - target: embedding.Vector, + target: embedding, threshold: minRelevanceScore, top: limit, withVectors: withEmbeddings, @@ -352,9 +332,9 @@ public async Task RemoveWithPointIdBatchAsync(string collectionName, IEnumerable result = null; } } - catch (HttpRequestException ex) when (ex.Message.Contains("404")) + catch (HttpOperationException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) { - this._logger?.LogWarning("NotFound when calling {0}::FindNearestInCollectionAsync - the collection '{1}' may not exist yet", + this._logger.LogWarning("NotFound when calling {QdrantMemoryStore}::FindNearestInCollectionAsync - the collection '{Name}' may not exist yet", nameof(QdrantMemoryStore), collectionName); hasResult = false; } @@ -364,7 +344,7 @@ public async Task RemoveWithPointIdBatchAsync(string collectionName, IEnumerable yield return ( MemoryRecord.FromJsonMetadata( json: result.Value.Item1.GetSerializedPayload(), - embedding: new Embedding(result.Value.Item1.Embedding, transferOwnership: true)), + embedding: result.Value.Item1.Embedding), result.Value.Item2); } } while (hasResult); @@ -373,7 +353,7 @@ public async Task RemoveWithPointIdBatchAsync(string collectionName, IEnumerable /// public async Task<(MemoryRecord, double)?> GetNearestMatchAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) @@ -434,12 +414,12 @@ private async Task ConvertFromMemoryRecordAsync( var vectorData = QdrantVectorRecord.FromJsonMetadata( pointId: pointId, - embedding: record.Embedding.Vector, + embedding: record.Embedding, json: record.GetSerializedMetadata()); if (vectorData == null) { - throw new QdrantMemoryException(QdrantMemoryException.ErrorCodes.FailedToConvertMemoryRecordToQdrantVectorRecord); + throw new SKException("Failed to convert memory record to Qdrant vector record"); } return vectorData; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorDbClient.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorDbClient.cs index bbd70d7cbef3..7803aad8b16b 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorDbClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorDbClient.cs @@ -11,9 +11,9 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Diagnostics; using Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Http.ApiSchema; +using Microsoft.SemanticKernel.Diagnostics; +using Verify = Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Diagnostics.Verify; namespace Microsoft.SemanticKernel.Connectors.Memory.Qdrant; @@ -30,16 +30,16 @@ public sealed class QdrantVectorDbClient : IQdrantVectorDbClient /// /// The Qdrant Vector Database endpoint. /// The size of the vectors used in the Qdrant Vector Database. - /// Optional logger instance. + /// The to use for logging. If null, no logging will be performed. public QdrantVectorDbClient( string endpoint, int vectorSize, - ILogger? logger = null) + ILoggerFactory? loggerFactory = null) { this._vectorSize = vectorSize; this._httpClient = new HttpClient(NonDisposableHttpClientHandler.Instance, disposeHandler: false); this._httpClient.BaseAddress = SanitizeEndpoint(endpoint); - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(QdrantVectorDbClient)) : NullLogger.Instance; } /// @@ -48,24 +48,22 @@ public QdrantVectorDbClient( /// The instance used for making HTTP requests. /// The size of the vectors used in the Qdrant Vector Database. /// The optional endpoint URL for the Qdrant Vector Database. If not specified, the base address of the HTTP client is used. - /// Optional logger instance. + /// The to use for logging. If null, no logging will be performed. public QdrantVectorDbClient( HttpClient httpClient, int vectorSize, string? endpoint = null, - ILogger? logger = null) + ILoggerFactory? loggerFactory = null) { if (string.IsNullOrEmpty(httpClient.BaseAddress?.AbsoluteUri) && string.IsNullOrEmpty(endpoint)) { - throw new AIException( - AIException.ErrorCodes.InvalidConfiguration, - "The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); + throw new SKException("The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); } this._httpClient = httpClient; this._vectorSize = vectorSize; this._endpointOverride = string.IsNullOrEmpty(endpoint) ? null : SanitizeEndpoint(endpoint!); - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(QdrantVectorDbClient)) : NullLogger.Instance; } /// @@ -80,15 +78,16 @@ public async IAsyncEnumerable GetVectorsByIdAsync(string col .WithVectors(withVectors) .Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + string? responseContent = null; + try { - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogDebug("Vectors not found {0}", e.Message); - yield break; + this._logger.LogError(e, "Vectors not found {Message}", e.Message); + throw; } var data = JsonSerializer.Deserialize(responseContent); @@ -112,7 +111,7 @@ public async IAsyncEnumerable GetVectorsByIdAsync(string col { yield return new QdrantVectorRecord( pointId: record.Id, - embedding: record.Vector ?? Array.Empty(), + embedding: record.Vector ?? default, record.Payload, tags: null); } @@ -130,16 +129,21 @@ public async IAsyncEnumerable GetVectorsByIdAsync(string col .IncludeVectorData(withVector) .Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + string? responseContent = null; + try { - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.NotFound) { - this._logger.LogDebug("Request for vector with payload ID failed {0}", e.Message); return null; } + catch (HttpOperationException e) + { + this._logger.LogError(e, "Request for vector with payload ID failed {Message}", e.Message); + throw; + } var data = JsonSerializer.Deserialize(responseContent); @@ -159,7 +163,7 @@ public async IAsyncEnumerable GetVectorsByIdAsync(string col var record = new QdrantVectorRecord( pointId: point.Id, - embedding: point.Vector ?? Array.Empty(), + embedding: point.Vector, payload: point.Payload, tags: null); this._logger.LogDebug("Vector found}"); @@ -178,24 +182,27 @@ public async Task DeleteVectorsByIdAsync(string collectionName, IEnumerable(responseContent); - if (result?.Status == "ok") - { - this._logger.LogDebug("Vector being deleted"); - } - else - { - this._logger.LogWarning("Vector delete failed"); - } + (_, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HttpOperationException e) + { + this._logger.LogError(e, "Vector delete request failed: {Message}", e.Message); + throw; } - catch (HttpRequestException e) + + var result = JsonSerializer.Deserialize(responseContent); + if (result?.Status == "ok") { - this._logger.LogError(e, "Vector delete request failed: {0}", e.Message); + this._logger.LogDebug("Vector being deleted"); + } + else + { + this._logger.LogWarning("Vector delete failed"); } } @@ -217,24 +224,26 @@ public async Task DeleteVectorByPayloadIdAsync(string collectionName, string met .DeleteVector(existingRecord.PointId) .Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + string? responseContent = null; try { - response.EnsureSuccessStatusCode(); - var result = JsonSerializer.Deserialize(responseContent); - if (result?.Status == "ok") - { - this._logger.LogDebug("Vector being deleted"); - } - else - { - this._logger.LogWarning("Vector delete failed"); - } + (_, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HttpOperationException e) + { + this._logger.LogError(e, "Vector delete request failed: {Message}", e.Message); + throw; } - catch (HttpRequestException e) + + var result = JsonSerializer.Deserialize(responseContent); + if (result?.Status == "ok") + { + this._logger.LogDebug("Vector being deleted"); + } + else { - this._logger.LogError(e, "Vector delete request failed: {0}", e.Message); + this._logger.LogWarning("Vector delete failed"); } } @@ -248,31 +257,34 @@ public async Task UpsertVectorsAsync(string collectionName, IEnumerable(responseContent); - if (result?.Status == "ok") - { - this._logger.LogDebug("Vectors upserted"); - } - else - { - this._logger.LogWarning("Vector upserts failed"); - } + (_, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError(e, "Vector upserts request failed: {0}", e.Message); + this._logger.LogError(e, "Vector upserts request failed: {Message}", e.Message); + throw; + } + + var result = JsonSerializer.Deserialize(responseContent); + if (result?.Status == "ok") + { + this._logger.LogDebug("Vectors upserted"); + } + else + { + this._logger.LogWarning("Vector upserts failed"); } } /// public async IAsyncEnumerable<(QdrantVectorRecord, double)> FindNearestInCollectionAsync( string collectionName, - IEnumerable target, + ReadOnlyMemory target, double threshold, int top = 1, bool withVectors = false, @@ -293,15 +305,17 @@ public async Task UpsertVectorsAsync(string collectionName, IEnumerable(responseContent); @@ -323,7 +337,7 @@ public async Task UpsertVectorsAsync(string collectionName, IEnumerable(), + embedding: v.Vector, payload: v.Payload); result.Add((record, v.Score ?? 0.0)); @@ -346,21 +360,24 @@ public async Task CreateCollectionAsync(string collectionName, CancellationToken .Create(collectionName, this._vectorSize, QdrantDistanceType.Cosine) .Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - - // Creation is idempotent, ignore error (and for now ignore vector size) - if (response.StatusCode == HttpStatusCode.BadRequest) + try { - if (responseContent.IndexOf("already exists", StringComparison.OrdinalIgnoreCase) >= 0) { return; } + await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); } - - try + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.BadRequest) { - response.EnsureSuccessStatusCode(); + // Creation is idempotent, ignore error (and for now ignore vector size) + if (e.ResponseContent?.IndexOf("already exists", StringComparison.OrdinalIgnoreCase) >= 0) + { + return; + } + + this._logger.LogError(e, "Collection creation failed: {Message}, {Response}", e.Message, e.ResponseContent); + throw; } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError(e, "Collection upsert failed: {0}, {1}", e.Message, responseContent); + this._logger.LogError(e, "Collection creation failed: {Message}, {Response}", e.Message, e.ResponseContent); throw; } } @@ -371,21 +388,18 @@ public async Task DeleteCollectionAsync(string collectionName, CancellationToken this._logger.LogDebug("Deleting collection {0}", collectionName); using var request = DeleteCollectionRequest.Create(collectionName).Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - // Deletion is idempotent, ignore error - if (response.StatusCode == HttpStatusCode.NotFound) + try { - return; + await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); } - - try + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.NotFound) { - response.EnsureSuccessStatusCode(); + return; // Deletion is idempotent, ignore error } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError(e, "Collection deletion failed: {0}, {1}", e.Message, responseContent); + this._logger.LogError(e, "Collection deletion failed: {Message}, {Response}", e.Message, e.ResponseContent); throw; } } @@ -396,20 +410,22 @@ public async Task DoesCollectionExistAsync(string collectionName, Cancella this._logger.LogDebug("Fetching collection {0}", collectionName); using var request = GetCollectionsRequest.Create(collectionName).Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - if (response.IsSuccessStatusCode) + try { + await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + return true; } - else if (response.StatusCode == HttpStatusCode.NotFound) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.NotFound) { + this._logger.LogDebug(e, "Collection {Name} not found: {Message}, {Response}", collectionName, e.Message, e.ResponseContent); return false; } - else + catch (HttpOperationException e) { - this._logger.LogError("Collection fetch failed: {0}, {1}", response.StatusCode, responseContent); - return false; + this._logger.LogError(e, "Collection fetch failed: {Message}, {Response}", e.Message, e.ResponseContent); + throw; } } @@ -419,7 +435,18 @@ public async IAsyncEnumerable ListCollectionsAsync([EnumeratorCancellati this._logger.LogDebug("Listing collections"); using var request = ListCollectionsRequest.Create().Build(); - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + + string? responseContent = null; + + try + { + (_, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HttpOperationException e) + { + this._logger.LogError(e, "Collection listing failed: {Message}, {Response}", e.Message, e.ResponseContent); + throw; + } var collections = JsonSerializer.Deserialize(responseContent); @@ -434,7 +461,7 @@ public async IAsyncEnumerable ListCollectionsAsync([EnumeratorCancellati private readonly ILogger _logger; private readonly HttpClient _httpClient; private readonly int _vectorSize; - private readonly Uri? _endpointOverride = null; + private readonly Uri? _endpointOverride; private static Uri SanitizeEndpoint(string endpoint, int? port = null) { @@ -456,18 +483,9 @@ private static Uri SanitizeEndpoint(string endpoint, int? port = null) request.RequestUri = new Uri(this._endpointOverride, request.RequestUri); } - HttpResponseMessage response = await this._httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + HttpResponseMessage response = await this._httpClient.SendWithSuccessCheckAsync(request, cancellationToken).ConfigureAwait(false); - string responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - - if (response.IsSuccessStatusCode) - { - this._logger.LogDebug("Qdrant responded successfully"); - } - else - { - this._logger.LogWarning("Qdrant responded with error"); - } + string responseContent = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); return (response, responseContent); } diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorRecord.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorRecord.cs index 43714997a326..5dd1ad3a43eb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorRecord.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorRecord.cs @@ -1,9 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; -using Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Diagnostics; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Qdrant; @@ -22,7 +24,8 @@ public class QdrantVectorRecord /// The embedding data. /// [JsonPropertyName("embedding")] - public IEnumerable Embedding { get; } + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Embedding { get; } /// /// The metadata. @@ -43,7 +46,7 @@ public class QdrantVectorRecord /// /// /// - public QdrantVectorRecord(string pointId, IEnumerable embedding, Dictionary payload, List? tags = null) + public QdrantVectorRecord(string pointId, ReadOnlyMemory embedding, Dictionary payload, List? tags = null) { this.PointId = pointId; this.Embedding = embedding; @@ -68,8 +71,8 @@ public string GetSerializedPayload() /// /// /// Vector record - /// Qdrant exception - public static QdrantVectorRecord FromJsonMetadata(string pointId, IEnumerable embedding, string json, List? tags = null) + /// Qdrant exception + public static QdrantVectorRecord FromJsonMetadata(string pointId, ReadOnlyMemory embedding, string json, List? tags = null) { var payload = JsonSerializer.Deserialize>(json); if (payload != null) @@ -77,6 +80,6 @@ public static QdrantVectorRecord FromJsonMetadata(string pointId, IEnumerable Semantic Kernel - Redis Connector - Redis connector for Semantic Kernel skills and semantic memory + Redis connector for Semantic Kernel plugins and semantic memory + + + + - + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStore.cs index 265a7e4b30c3..b275660260c5 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStore.cs @@ -7,13 +7,14 @@ using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; using NRedisStack; using NRedisStack.RedisStackCommands; using NRedisStack.Search; using NRedisStack.Search.Literals.Enums; using StackExchange.Redis; +using static NRedisStack.Search.Schema.VectorField; namespace Microsoft.SemanticKernel.Connectors.Memory.Redis; @@ -23,18 +24,65 @@ namespace Microsoft.SemanticKernel.Connectors.Memory.Redis; /// The embedded data is saved to the Redis server database specified in the constructor. /// Similarity search capability is provided through the RediSearch module. Use RediSearch's "Index" to implement "Collection". /// -public sealed class RedisMemoryStore : IMemoryStore +public class RedisMemoryStore : IMemoryStore, IDisposable { /// /// Create a new instance of semantic memory using Redis. /// - /// The database of the redis server. - /// Embedding vector size - public RedisMemoryStore(IDatabase database, int vectorSize) + /// The database of the Redis server. + /// Embedding vector size, defaults to 1536 + /// Indexing algorithm for vectors, defaults to "HNSW" + /// Metric for measuring vector distances, defaults to "COSINE" + /// Query dialect, must be 2 or greater for vector similarity searching, defaults to 2 + public RedisMemoryStore( + IDatabase database, + int vectorSize = DefaultVectorSize, + VectorAlgo vectorIndexAlgorithm = DefaultIndexAlgorithm, + VectorDistanceMetric vectorDistanceMetric = DefaultDistanceMetric, + int queryDialect = DefaultQueryDialect) { + if (vectorSize <= 0) + { + throw new ArgumentException( + $"Invalid vector size: {vectorSize}. Vector size must be a positive integer.", nameof(vectorSize)); + } + this._database = database; this._vectorSize = vectorSize; this._ft = database.FT(); + this._vectorIndexAlgorithm = vectorIndexAlgorithm; + this._vectorDistanceMetric = vectorDistanceMetric.ToString(); + this._queryDialect = queryDialect; + } + + /// + /// Create a new instance of semantic memory using Redis. + /// + /// Provide connection URL to a Redis instance + /// Embedding vector size, defaults to 1536 + /// Indexing algorithm for vectors, defaults to "HNSW" + /// Metric for measuring vector distances, defaults to "COSINE" + /// Query dialect, must be 2 or greater for vector similarity searching, defaults to 2 + public RedisMemoryStore( + string connectionString, + int vectorSize = DefaultVectorSize, + VectorAlgo vectorIndexAlgorithm = DefaultIndexAlgorithm, + VectorDistanceMetric vectorDistanceMetric = DefaultDistanceMetric, + int queryDialect = DefaultQueryDialect) + { + if (vectorSize <= 0) + { + throw new ArgumentException( + $"Invalid vector size: {vectorSize}. Vector size must be a positive integer.", nameof(vectorSize)); + } + + this._connection = ConnectionMultiplexer.Connect(connectionString); + this._database = this._connection.GetDatabase(); + this._vectorSize = vectorSize; + this._ft = this._database.FT(); + this._vectorIndexAlgorithm = vectorIndexAlgorithm; + this._vectorDistanceMetric = vectorDistanceMetric.ToString(); + this._queryDialect = queryDialect; } /// @@ -54,10 +102,10 @@ public async Task CreateCollectionAsync(string collectionName, CancellationToken .AddTextField("key") .AddTextField("metadata") .AddNumericField("timestamp") - .AddVectorField("embedding", VECTOR_INDEX_ALGORITHM, new Dictionary { - {"TYPE", VECTOR_TYPE}, + .AddVectorField("embedding", this._vectorIndexAlgorithm, new Dictionary { + {"TYPE", DefaultVectorType}, {"DIM", this._vectorSize}, - {"DISTANCE_METRIC", VECTOR_DISTANCE_METRIC}, + {"DISTANCE_METRIC", this._vectorDistanceMetric}, }); await this._ft.CreateAsync(collectionName, ftCreateParams, schema).ConfigureAwait(false); @@ -71,7 +119,7 @@ public async Task DoesCollectionExistAsync(string collectionName, Cancella await this._ft.InfoAsync(collectionName).ConfigureAwait(false); return true; } - catch (RedisServerException ex) when (ex.Message == MESSAGE_WHEN_INDEX_DOES_NOT_EXIST) + catch (RedisServerException ex) when (ex.Message.Equals(IndexDoesNotExistErrorMessage, StringComparison.OrdinalIgnoreCase)) { return false; } @@ -112,7 +160,7 @@ public async Task UpsertAsync(string collectionName, MemoryRecord record await this._database.HashSetAsync(GetRedisKey(collectionName, record.Key), new[] { new HashEntry("key", record.Key), new HashEntry("metadata", record.GetSerializedMetadata()), - new HashEntry("embedding", MemoryMarshal.Cast(record.Embedding.AsReadOnlySpan()).ToArray()), + new HashEntry("embedding", this.ConvertEmbeddingToBytes(record.Embedding)), new HashEntry("timestamp", ToTimestampLong(record.Timestamp)) }, flags: CommandFlags.None).ConfigureAwait(false); @@ -143,7 +191,7 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke /// public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, @@ -155,11 +203,11 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke } var query = new Query($"*=>[KNN {limit} @embedding $embedding AS vector_score]") - .AddParam("embedding", MemoryMarshal.Cast(embedding.AsReadOnlySpan()).ToArray()) + .AddParam("embedding", this.ConvertEmbeddingToBytes(embedding)) .SetSortBy("vector_score") .ReturnFields("key", "metadata", "embedding", "timestamp", "vector_score") .Limit(0, limit) - .Dialect(QUERY_DIALECT); + .Dialect(this._queryDialect); var results = await this._ft.SearchAsync(collectionName, query).ConfigureAwait(false); @@ -171,11 +219,11 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke yield break; } - Embedding convertedEmbedding = withEmbeddings && document["embedding"].HasValue + ReadOnlyMemory convertedEmbedding = withEmbeddings && document["embedding"].HasValue ? - new Embedding(MemoryMarshal.Cast((byte[])document["embedding"]!).ToArray()) + MemoryMarshal.Cast((byte[])document["embedding"]!).ToArray() : - Embedding.Empty; + ReadOnlyMemory.Empty; yield return (MemoryRecord.FromJsonMetadata( json: document["metadata"]!, @@ -186,7 +234,7 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke } /// - public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding embedding, double minRelevanceScore = 0, bool withEmbedding = false, + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) { return await this.GetNearestMatchesAsync( @@ -198,43 +246,70 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke cancellationToken: cancellationToken).FirstOrDefaultAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - #region constants ================================================================================ + /// + /// Disposes the instance. + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Disposes the resources used by the instance. + /// + /// True to release both managed and unmanaged resources; false to release only unmanaged resources. + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._connection?.Dispose(); + } + } + + #region private ================================================================================ /// - /// Vector similarity index algorithm. The default value is "HNSW". - /// + /// Vector similarity index algorithm. Supported algorithms are {FLAT, HNSW}. The default value is "HNSW". + /// /// - internal const Schema.VectorField.VectorAlgo VECTOR_INDEX_ALGORITHM = Schema.VectorField.VectorAlgo.HNSW; + private const VectorAlgo DefaultIndexAlgorithm = VectorAlgo.HNSW; /// - /// Vector type. Supported types are FLOAT32 and FLOAT64. The default value is "FLOAT32". + /// Vector type. Available values are {FLOAT32, FLOAT64}. + /// Value "FLOAT32" is used by default based on type. /// - internal const string VECTOR_TYPE = "FLOAT32"; + private const string DefaultVectorType = "FLOAT32"; /// - /// Supported distance metric, one of {L2, IP, COSINE}. The default value is "COSINE". + /// Supported distance metrics are {L2, IP, COSINE}. The default value is "COSINE". /// - internal const string VECTOR_DISTANCE_METRIC = "COSINE"; + private const VectorDistanceMetric DefaultDistanceMetric = VectorDistanceMetric.COSINE; /// /// Query dialect. To use a vector similarity query, specify DIALECT 2 or higher. The default value is "2". - /// + /// /// - internal const int QUERY_DIALECT = 2; + private const int DefaultQueryDialect = 2; /// - /// Message when index does not exist. - /// + /// Embedding vector size. /// - internal const string MESSAGE_WHEN_INDEX_DOES_NOT_EXIST = "Unknown Index name"; + private const int DefaultVectorSize = 1536; - #endregion - - #region private ================================================================================ + /// + /// Message when index does not exist. + /// + /// + private const string IndexDoesNotExistErrorMessage = "Unknown index name"; private readonly IDatabase _database; private readonly int _vectorSize; private readonly SearchCommands _ft; + private readonly ConnectionMultiplexer? _connection; + private readonly Schema.VectorField.VectorAlgo _vectorIndexAlgorithm; + private readonly string _vectorDistanceMetric; + private readonly int _queryDialect; private static long ToTimestampLong(DateTimeOffset? timestamp) { @@ -271,14 +346,14 @@ private static RedisKey GetRedisKey(string collectionName, string key) RedisValue embedding = hashEntries.FirstOrDefault(x => x.Name == "embedding").Value; return MemoryRecord.FromJsonMetadata( json: hashEntries.FirstOrDefault(x => x.Name == "metadata").Value!, - embedding: embedding.HasValue ? new Embedding(MemoryMarshal.Cast((byte[])embedding!).ToArray()) : Embedding.Empty, + embedding: embedding.HasValue ? MemoryMarshal.Cast((byte[])embedding!).ToArray() : ReadOnlyMemory.Empty, key: hashEntries.FirstOrDefault(x => x.Name == "key").Value, timestamp: ParseTimestamp((long?)hashEntries.FirstOrDefault(x => x.Name == "timestamp").Value)); } return MemoryRecord.FromJsonMetadata( json: hashEntries.FirstOrDefault(x => x.Name == "metadata").Value!, - embedding: Embedding.Empty, + embedding: ReadOnlyMemory.Empty, key: hashEntries.FirstOrDefault(x => x.Name == "key").Value, timestamp: ParseTimestamp((long?)hashEntries.FirstOrDefault(x => x.Name == "timestamp").Value)); } @@ -289,11 +364,16 @@ private double GetSimilarity(Document document) if (vectorScoreValue.IsNullOrEmpty || !vectorScoreValue.TryParse(out double vectorScore)) { - throw new RedisMemoryStoreException("Invalid or missing vector score value."); + throw new SKException("Invalid or missing vector score value."); } return 1 - vectorScore; } + private byte[] ConvertEmbeddingToBytes(ReadOnlyMemory embedding) + { + return MemoryMarshal.AsBytes(embedding.Span).ToArray(); + } + #endregion } diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStoreException.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStoreException.cs deleted file mode 100644 index 80454a9beecf..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStoreException.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.Connectors.Memory.Redis; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown by the Redis connector -/// -public class RedisMemoryStoreException : Exception -{ - /// - /// Initializes a new instance of the class. - /// - /// Exception message. - internal RedisMemoryStoreException(string? message) : base(message) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// Exception message. - /// Inner exception. - internal RedisMemoryStoreException(string? message, Exception? innerException) : base(message, innerException) - { - } -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorDistanceMetric.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorDistanceMetric.cs new file mode 100644 index 000000000000..2d5ff71c900c --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorDistanceMetric.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Connectors.Memory.Redis; + +/// +/// Supported distance metrics are {L2, IP, COSINE}. The default value is "COSINE". +/// +/// +public enum VectorDistanceMetric +{ + /// + /// Euclidean distance between two vectors + /// + L2, + + /// + /// Inner product of two vectors + /// + IP, + + /// + /// Cosine distance of two vectors + /// + COSINE, +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/Connectors.Memory.Sqlite.csproj b/dotnet/src/Connectors/Connectors.Memory.Sqlite/Connectors.Memory.Sqlite.csproj index 2c6512e435dd..8625483cf0ab 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/Connectors.Memory.Sqlite.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/Connectors.Memory.Sqlite.csproj @@ -14,15 +14,16 @@ Semantic Kernel - SQLite Connector - SQLite connector for Semantic Kernel skills and semantic memory + SQLite connector for Semantic Kernel plugins and semantic memory + - + diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteMemoryStore.cs index d0c001c04115..d15c47bf5ab7 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteMemoryStore.cs @@ -4,15 +4,14 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Data.Sqlite; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Memory.Collections; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Sqlite; @@ -120,7 +119,7 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke /// public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, @@ -132,33 +131,29 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke } var collectionMemories = new List(); - TopNCollection embeddings = new(limit); + List<(MemoryRecord Record, double Score)> embeddings = new(); await foreach (var record in this.GetAllAsync(collectionName, cancellationToken)) { if (record != null) { - double similarity = embedding - .AsReadOnlySpan() - .CosineSimilarity(record.Embedding.AsReadOnlySpan()); + double similarity = TensorPrimitives.CosineSimilarity(embedding.Span, record.Embedding.Span); if (similarity >= minRelevanceScore) { - var entry = withEmbeddings ? record : MemoryRecord.FromMetadata(record.Metadata, Embedding.Empty, record.Key, record.Timestamp); + var entry = withEmbeddings ? record : MemoryRecord.FromMetadata(record.Metadata, ReadOnlyMemory.Empty, record.Key, record.Timestamp); embeddings.Add(new(entry, similarity)); } } } - embeddings.SortByScore(); - - foreach (var item in embeddings) + foreach (var item in embeddings.OrderByDescending(l => l.Score).Take(limit)) { - yield return (item.Value, item.Score.Value); + yield return (item.Record, item.Score); } } /// - public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding embedding, double minRelevanceScore = 0, bool withEmbedding = false, + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) { return await this.GetNearestMatchesAsync( @@ -178,7 +173,10 @@ public void Dispose() } #region protected ================================================================================ - + /// + /// Disposes the resources used by the instance. + /// + /// True to release both managed and unmanaged resources; false to release only unmanaged resources. protected virtual void Dispose(bool disposing) { if (!this._disposedValue) @@ -236,7 +234,7 @@ private async IAsyncEnumerable GetAllAsync(string collectionName, await foreach (DatabaseEntry dbEntry in this._dbConnector.ReadAllAsync(this._dbConnection, collectionName, cancellationToken)) { - Embedding? vector = JsonSerializer.Deserialize>(dbEntry.EmbeddingString); + ReadOnlyMemory vector = JsonSerializer.Deserialize>(dbEntry.EmbeddingString, s_jsonSerializerOptions); var record = MemoryRecord.FromJsonMetadata(dbEntry.MetadataString, vector, dbEntry.Key, ParseTimestamp(dbEntry.Timestamp)); @@ -254,7 +252,7 @@ await this._dbConnector.UpdateAsync( collection: collectionName, key: record.Key, metadata: record.GetSerializedMetadata(), - embedding: JsonSerializer.Serialize(record.Embedding), + embedding: JsonSerializer.Serialize(record.Embedding, s_jsonSerializerOptions), timestamp: ToTimestampString(record.Timestamp), cancellationToken: cancellationToken).ConfigureAwait(false); @@ -264,7 +262,7 @@ await this._dbConnector.InsertOrIgnoreAsync( collection: collectionName, key: record.Key, metadata: record.GetSerializedMetadata(), - embedding: JsonSerializer.Serialize(record.Embedding), + embedding: JsonSerializer.Serialize(record.Embedding, s_jsonSerializerOptions), timestamp: ToTimestampString(record.Timestamp), cancellationToken: cancellationToken).ConfigureAwait(false); @@ -285,17 +283,26 @@ await this._dbConnector.InsertOrIgnoreAsync( { return MemoryRecord.FromJsonMetadata( json: entry.Value.MetadataString, - JsonSerializer.Deserialize>(entry.Value.EmbeddingString), + JsonSerializer.Deserialize>(entry.Value.EmbeddingString, s_jsonSerializerOptions), entry.Value.Key, ParseTimestamp(entry.Value.Timestamp)); } return MemoryRecord.FromJsonMetadata( json: entry.Value.MetadataString, - Embedding.Empty, + ReadOnlyMemory.Empty, entry.Value.Key, ParseTimestamp(entry.Value.Timestamp)); } + private static readonly JsonSerializerOptions s_jsonSerializerOptions = CreateSerializerOptions(); + + private static JsonSerializerOptions CreateSerializerOptions() + { + var jso = new JsonSerializerOptions(); + jso.Converters.Add(new ReadOnlyMemoryConverter()); + return jso; + } + #endregion } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj index 6b97b6e1375e..b142a7b5ba2c 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj @@ -14,7 +14,7 @@ Semantic Kernel - Weaviate Connector - Weaviate connector for Semantic Kernel skills and semantic memory + Weaviate connector for Semantic Kernel plugins and semantic memory @@ -22,7 +22,8 @@ - + + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Diagnostics/WeaviateMemoryException.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Diagnostics/WeaviateMemoryException.cs deleted file mode 100644 index 64079e7b696b..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Diagnostics/WeaviateMemoryException.cs +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Diagnostics; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown for errors related to the Weaviate connector. -/// -public class WeaviateMemoryException : SKException -{ - /// - /// Initializes a new instance of the class with a provided error code. - /// - /// The error code. - public WeaviateMemoryException(ErrorCodes errorCode) - : this(errorCode, message: null, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public WeaviateMemoryException(ErrorCodes errorCode, string? message) - : this(errorCode, message, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code and inner exception. - /// - /// The error code. - /// The exception that is the cause of the current exception. - public WeaviateMemoryException(ErrorCodes errorCode, Exception? innerException) - : this(errorCode, message: null, innerException) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public WeaviateMemoryException(ErrorCodes errorCode, string? message, Exception? innerException) - : base(GetDefaultMessage(errorCode, message, innerException), innerException) - { - this.ErrorCode = errorCode; - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// Translate the error code into a default message. - private static string GetDefaultMessage(ErrorCodes errorCode, string? message, Exception? innerException) - { - if (message is not null) - { - return message; - } - - string description = errorCode switch - { - ErrorCodes.FailedToUpsertVectors => "Failed to upsert vectors", - ErrorCodes.FailedToGetVectorData => "Failed to get vector data", - ErrorCodes.FailedToRemoveVectorData => "Failed to remove vector data", - ErrorCodes.CollectionNameConflict => "Naming conflict for the collection name", - ErrorCodes.FailedToCreateCollection => "Failed to create the collection", - ErrorCodes.FailedToDeleteCollection => "Failed to delete the collection", - ErrorCodes.FailedToListCollections => "Failed to list collections", - ErrorCodes.FailedToGetClass => "Failed to get class", - _ => $"Unknown error ({errorCode:G})", - }; - - return innerException is not null ? $"{description}: {innerException.Message}" : description; - } - - /// - /// Error codes for the Weaviate connector exceptions. - /// - public enum ErrorCodes - { - /// - /// Failed to upsert the vector. - /// - FailedToUpsertVectors, - - /// - /// Failed to get vector data from Weaviate. - /// - FailedToGetVectorData, - - /// - /// Failed to remove vector data from Weaviate. - /// - FailedToRemoveVectorData, - - /// - /// Failed to create a collection. - /// - FailedToCreateCollection, - - // ReSharper disable once CommentTypo - /// - /// Naming conflict for the collection name. - /// For example a collectionName of '__this_collection' and 'this_collection' are - /// both transformed to the class name of SKthiscollection - even though - /// semantic kernel would consider them as unique collection names. - /// - CollectionNameConflict, - - /// - /// Failed to delete a collection. - /// - FailedToDeleteCollection, - - /// - /// Failed to list collections. - /// - FailedToListCollections, - - /// - /// Failed to get a Weaviate class. - /// - FailedToGetClass - } -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/BatchRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/BatchRequest.cs index 1bbab81403db..0d2c45943bdb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/BatchRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/BatchRequest.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System.Collections.Generic; -using System.Linq; using System.Net.Http; using Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Model; using Microsoft.SemanticKernel.Memory; @@ -38,7 +37,7 @@ public void Add(MemoryRecord record) { Class = this._class, Id = record.Key, - Vector = record.Embedding.Vector.ToArray(), + Vector = record.Embedding, Properties = new() { { "sk_timestamp", record.Timestamp! }, diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/CreateGraphRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/CreateGraphRequest.cs index e30debedf287..d937eb78019d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/CreateGraphRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/CreateGraphRequest.cs @@ -1,7 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; +using System; using System.Net.Http; +using System.Runtime.InteropServices; namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Http.ApiSchema; @@ -10,7 +11,7 @@ internal sealed class CreateGraphRequest { #pragma warning disable CS8618 public string Class { get; set; } - public IEnumerable Vector { get; set; } + public ReadOnlyMemory Vector { get; set; } #pragma warning restore CS8618 public int Limit { get; set; } public bool WithVector { get; set; } @@ -19,7 +20,7 @@ internal sealed class CreateGraphRequest public HttpRequestMessage Build() { string payload = $"{{Get{{{this.Class}(" + - $"nearVector:{{vector:[{string.Join(",", this.Vector)}] " + + $"nearVector:{{vector:[{string.Join(",", MemoryMarshal.ToEnumerable(this.Vector))}] " + $"distance:{this.Distance}}} " + $"limit:{this.Limit}){{{(this.WithVector ? "_additional{vector}" : string.Empty)} " + "_additional{id distance} sk_timestamp sk_id sk_description sk_text sk_additional_metadata}}}"; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs index ec72b4d9580c..60a51cf482e1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/HttpRequest.cs @@ -4,16 +4,13 @@ using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Http; internal static class HttpRequest { - private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; + private static readonly JsonSerializerOptions s_jsonSerializerOptions = CreateSerializerOptions(); public static HttpRequestMessage CreateGetRequest(string url, object? payload = null) { @@ -46,4 +43,15 @@ public static HttpRequestMessage CreateDeleteRequest(string url) string strPayload = payload as string ?? JsonSerializer.Serialize(payload, s_jsonSerializerOptions); return new(strPayload, Encoding.UTF8, "application/json"); } + + private static JsonSerializerOptions CreateSerializerOptions() + { + var jso = new JsonSerializerOptions() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + jso.Converters.Add(new ReadOnlyMemoryConverter()); + return jso; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Model/WeaviateObject.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Model/WeaviateObject.cs index 9462a8586e48..f3c60fb3a461 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Model/WeaviateObject.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Model/WeaviateObject.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Model; @@ -9,5 +10,5 @@ internal class WeaviateObject public string? Id { get; set; } public string? Class { get; set; } public Dictionary? Properties { get; set; } - public float[]? Vector { get; set; } + public ReadOnlyMemory Vector { get; set; } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateKernelBuilderExtensions.cs index a56ad06145d5..e11047eb2288 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateKernelBuilderExtensions.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using System.ComponentModel; using System.Net.Http; using Microsoft.SemanticKernel.Connectors.Memory.Weaviate; @@ -10,6 +12,8 @@ namespace Microsoft.SemanticKernel; /// /// Provides extension methods for the class to configure Weaviate memory connector. /// +[Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use WeaviateMemoryBuilderExtensions instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class WeaviateKernelBuilderExtensions { /// @@ -18,16 +22,24 @@ public static class WeaviateKernelBuilderExtensions /// The instance. /// The Weaviate server endpoint URL. /// The API key for accessing Weaviate server. + /// The API version to use. /// Self instance - public static KernelBuilder WithWeaviateMemoryStore(this KernelBuilder builder, string endpoint, string? apiKey) + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use WeaviateMemoryBuilderExtensions.WithWeaviateMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public static KernelBuilder WithWeaviateMemoryStore( + this KernelBuilder builder, + string endpoint, + string? apiKey, + string? apiVersion = null) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory, httpHandlerFactory) => { return new WeaviateMemoryStore( - HttpClientProvider.GetHttpClient(parameters.Config, null, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, null, loggerFactory), apiKey, endpoint, - parameters.Logger); + apiVersion, + loggerFactory); }); return builder; @@ -40,19 +52,24 @@ public static KernelBuilder WithWeaviateMemoryStore(this KernelBuilder builder, /// The optional instance used for making HTTP requests. /// The Weaviate server endpoint URL. If not specified, the base address of the HTTP client is used. /// The API key for accessing Weaviate server. + /// The API version to use. /// Self instance + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. Use WeaviateMemoryBuilderExtensions.WithWeaviateMemoryStore instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] public static KernelBuilder WithWeaviateMemoryStore(this KernelBuilder builder, HttpClient httpClient, string? endpoint = null, - string? apiKey = null) + string? apiKey = null, + string? apiVersion = null) { - builder.WithMemoryStorage((parameters) => + builder.WithMemoryStorage((loggerFactory, httpHandlerFactory) => { return new WeaviateMemoryStore( - HttpClientProvider.GetHttpClient(parameters.Config, httpClient, parameters.Logger), + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), apiKey, endpoint, - parameters.Logger); + apiVersion, + loggerFactory); }); return builder; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateMemoryBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateMemoryBuilderExtensions.cs new file mode 100644 index 000000000000..b1dbd2686707 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateMemoryBuilderExtensions.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.SemanticKernel.Plugins.Memory; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate; + +/// +/// Provides extension methods for the class to configure Weaviate connector. +/// +public static class WeaviateMemoryBuilderExtensions +{ + /// + /// Registers Weaviate memory connector. + /// + /// The instance. + /// The Weaviate server endpoint URL. + /// The API key for accessing Weaviate server. + /// The API version to use. + /// Updated Memory builder including Weaviate memory connector. + public static MemoryBuilder WithWeaviateMemoryStore( + this MemoryBuilder builder, + string endpoint, + string? apiKey, + string? apiVersion = null) + { + builder.WithMemoryStore((loggerFactory, httpHandlerFactory) => + { + return new WeaviateMemoryStore( + HttpClientProvider.GetHttpClient(httpHandlerFactory, null, loggerFactory), + apiKey, + endpoint, + apiVersion, + loggerFactory); + }); + + return builder; + } + + /// + /// Registers Weaviate memory connector. + /// + /// The instance. + /// The optional instance used for making HTTP requests. + /// The Weaviate server endpoint URL. If not specified, the base address of the HTTP client is used. + /// The API key for accessing Weaviate server. + /// The API version to use. + /// Updated Memory builder including Weaviate memory connector. + public static MemoryBuilder WithWeaviateMemoryStore( + this MemoryBuilder builder, + HttpClient httpClient, + string? endpoint = null, + string? apiKey = null, + string? apiVersion = null) + { + builder.WithMemoryStore((loggerFactory, httpHandlerFactory) => + { + return new WeaviateMemoryStore( + HttpClientProvider.GetHttpClient(httpHandlerFactory, httpClient, loggerFactory), + apiKey, + endpoint, + apiVersion, + loggerFactory); + }); + + return builder; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateMemoryStore.cs index 52ceb58a05c9..821200b61968 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateMemoryStore.cs @@ -15,13 +15,11 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Diagnostics; using Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Http.ApiSchema; using Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Model; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate; @@ -45,30 +43,40 @@ public class WeaviateMemoryStore : IMemoryStore // https://weaviate.io/developers/weaviate/configuration/schema-configuration#class private static readonly Regex s_classNameRegEx = new("[^0-9a-zA-Z]+", RegexOptions.Compiled); + private const string DefaultApiVersion = "v1"; + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = { new ReadOnlyMemoryConverter() } }; private readonly HttpClient _httpClient; private readonly ILogger _logger; private readonly Uri? _endpoint = null; - private string? _apiKey; + private readonly string? _apiVersion; + private readonly string? _apiKey; /// /// Initializes a new instance of the class. /// /// The Weaviate server endpoint URL. /// The API key for accessing Weaviate server. - /// Optional logger instance. - public WeaviateMemoryStore(string endpoint, string? apiKey = null, ILogger? logger = null) + /// The API version to use. + /// The to use for logging. If null, no logging will be performed. + public WeaviateMemoryStore( + string endpoint, + string? apiKey = null, + string? apiVersion = null, + ILoggerFactory? loggerFactory = null) { Verify.NotNullOrWhiteSpace(endpoint); this._endpoint = new Uri(endpoint); this._apiKey = apiKey; - this._logger = logger ?? NullLogger.Instance; + this._apiVersion = apiVersion; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(WeaviateMemoryStore)) : NullLogger.Instance; this._httpClient = new HttpClient(NonDisposableHttpClientHandler.Instance, disposeHandler: false); } @@ -78,21 +86,26 @@ public WeaviateMemoryStore(string endpoint, string? apiKey = null, ILogger? logg /// The instance used for making HTTP requests. /// The API key for accessing Weaviate server. /// The optional Weaviate server endpoint URL. If not specified, the base address of the HTTP client is used. - /// Optional logger instance. - public WeaviateMemoryStore(HttpClient httpClient, string? apiKey = null, string? endpoint = null, ILogger? logger = null) + /// The API version to use. + /// The to use for logging. If null, no logging will be performed. + public WeaviateMemoryStore( + HttpClient httpClient, + string? apiKey = null, + string? endpoint = null, + string? apiVersion = null, + ILoggerFactory? loggerFactory = null) { Verify.NotNull(httpClient); if (string.IsNullOrEmpty(httpClient.BaseAddress?.AbsoluteUri) && string.IsNullOrEmpty(endpoint)) { - throw new AIException( - AIException.ErrorCodes.InvalidConfiguration, - "The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); + throw new SKException("The HttpClient BaseAddress and endpoint are both null or empty. Please ensure at least one is provided."); } this._apiKey = apiKey; + this._apiVersion = apiVersion; this._endpoint = string.IsNullOrEmpty(endpoint) ? null : new Uri(endpoint); - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(WeaviateMemoryStore)) : NullLogger.Instance; this._httpClient = httpClient; } @@ -111,21 +124,20 @@ public async Task CreateCollectionAsync(string collectionName, CancellationToken try { (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + CreateClassSchemaResponse? result = JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); - response.EnsureSuccessStatusCode(); if (result == null || result.Description != description) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.CollectionNameConflict, - $"Name conflict for collection: {collectionName} with class name: {className}"); + throw new SKException($"Name conflict for collection: {collectionName} with class name: {className}"); } this._logger.LogDebug("Created collection: {0}, with class name: {1}", collectionName, className); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToCreateCollection, - $"Unable to create collection: {collectionName}, with class name: {className}", e); + this._logger.LogError(e, "Unable to create collection: {CollectionName}, with class name: {ClassName}", collectionName, className); + throw; } } @@ -136,39 +148,37 @@ public async Task DoesCollectionExistAsync(string collectionName, Cancella string className = ToWeaviateFriendlyClassName(collectionName); - this._logger.LogDebug("Does collection exist: {0}, with class name: {1}:", collectionName, className); + this._logger.LogDebug("Does collection exist: {CollectionName}, with class name: {ClassName}:", collectionName, className); using HttpRequestMessage request = GetClassRequest.Create(className).Build(); try { - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + (_, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - // Needs to return a non-404 AND collection name should match - bool exists = response.StatusCode != HttpStatusCode.NotFound; - if (!exists) - { - this._logger.LogDebug("Collection: {0}, with class name: {1}, does not exist.", collectionName, className); - } - else + GetClassResponse? existing = JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); + + if (existing != null && existing.Description != ToWeaviateFriendlyClassDescription(collectionName)) { - GetClassResponse? existing = JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); - if (existing != null && existing.Description != ToWeaviateFriendlyClassDescription(collectionName)) - { - // ReSharper disable once CommentTypo - // Check that we don't have an accidental conflict. - // For example a collectionName of '__this_collection' and 'this_collection' are - // both transformed to the class name of thiscollection - even though the external - // system could consider them as unique collection names. - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.CollectionNameConflict, $"Unable to verify existing collection: {collectionName} with class name: {className}"); - } + // ReSharper disable once CommentTypo + // Check that we don't have an accidental conflict. + // For example a collectionName of '__this_collection' and 'this_collection' are + // both transformed to the class name of thiscollection - even though the external + // system could consider them as unique collection names. + throw new SKException($"Unable to verify existing collection: {collectionName} with class name: {className}"); } - return exists; + return true; } - catch (Exception e) + catch (HttpOperationException e) when (e.StatusCode == HttpStatusCode.NotFound) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToGetClass, "Unable to get class from Weaviate", e); + this._logger.LogDebug(e, "Collection: {CollectionName}, with class name: {ClassName}, does not exist.", collectionName, className); + return false; + } + catch (HttpOperationException e) + { + this._logger.LogError(e, "Request to check collection: {CollectionName}, with class name: {ClassName} existence failed.", collectionName, className); + throw; } } @@ -178,21 +188,23 @@ public async IAsyncEnumerable GetCollectionsAsync([EnumeratorCancellatio this._logger.LogDebug("Listing collections"); using HttpRequestMessage request = GetSchemaRequest.Create().Build(); + string responseContent; + try { (HttpResponseMessage response, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); } - catch (Exception e) + catch (HttpOperationException e) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToListCollections, "Unable to list collections", e); + this._logger.LogError(e, "Request to list collections failed."); + throw; } GetSchemaResponse? getSchemaResponse = JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); if (getSchemaResponse == null) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToListCollections, "Unable to deserialize list collections response"); + throw new SKException("Unable to deserialize list collections response"); } foreach (GetClassResponse? @class in getSchemaResponse.Classes!) @@ -212,15 +224,16 @@ public async Task DeleteCollectionAsync(string collectionName, CancellationToken if (await this.DoesCollectionExistAsync(collectionName, cancellationToken).ConfigureAwait(false)) { + using HttpRequestMessage request = DeleteSchemaRequest.Create(className).Build(); + try { - using HttpRequestMessage request = DeleteSchemaRequest.Create(className).Build(); - (HttpResponseMessage response, string _) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); } - catch (Exception e) + catch (HttpOperationException e) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToDeleteCollection, "Collection deletion failed", e); + this._logger.LogError(e, "Request to delete collection: {CollectionName}, with class name: {ClassName} failed.", collectionName, className); + throw; } } } @@ -251,21 +264,22 @@ public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IE using HttpRequestMessage request = requestBuilder.Build(); string responseContent; + try { - (HttpResponseMessage response, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToUpsertVectors, e); + this._logger.LogError(e, "Request to upsert vectors to collection: {CollectionName}, with class name: {ClassName} failed.", collectionName, className); + throw; } BatchResponse[]? result = JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); if (result == null) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToUpsertVectors, "Unable to deserialize batch response"); + throw new SKException("Unable to deserialize batch response"); } foreach (BatchResponse batchResponse in result) @@ -287,14 +301,14 @@ public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IE }.Build(); string responseContent; + try { - (HttpResponseMessage response, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + (_, responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - this._logger.LogError("Request for vector failed {0}", e.Message); + this._logger.LogError(e, "Request to get vector from collection: {CollectionName} failed.", collectionName); return null; } @@ -314,7 +328,7 @@ public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IE MemoryRecord record = new( key: weaviateObject.Id!, timestamp: timestamp, - embedding: new(weaviateObject.Vector ?? Array.Empty()), + embedding: weaviateObject.Vector, metadata: ToMetadata(weaviateObject)); this._logger.LogDebug("Vector found with key: {0}", key); @@ -360,14 +374,14 @@ public async Task RemoveAsync(string collectionName, string key, CancellationTok try { - (HttpResponseMessage response, string _) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); this._logger.LogDebug("Vector deleted"); } - catch (HttpRequestException e) + catch (HttpOperationException e) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToRemoveVectorData, "Vector delete request failed", e); + this._logger.LogError(e, "Request to delete collection: {CollectionName}, with class name: {ClassName} failed.", collectionName, className); + throw; } } @@ -380,7 +394,7 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke /// public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, @@ -388,14 +402,14 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke { Verify.NotNull(embedding, "The given vector is NULL"); - this._logger.LogDebug("Searching top {0} nearest vectors", limit); + this._logger.LogDebug("Searching top {Limit} nearest vectors", limit); string className = ToWeaviateFriendlyClassName(collectionName); using HttpRequestMessage request = new CreateGraphRequest { Class = className, - Vector = embedding.Vector, + Vector = embedding, Distance = minRelevanceScore, Limit = limit, WithVector = withEmbeddings @@ -404,8 +418,8 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke List<(MemoryRecord, double)> result = new(); try { - (HttpResponseMessage response, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + (_, string responseContent) = await this.ExecuteHttpRequestAsync(request, cancellationToken).ConfigureAwait(false); + GraphResponse? data = JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); if (data == null) @@ -424,9 +438,10 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke result.Add((memoryRecord, distance)); } } - catch (Exception e) + catch (HttpOperationException e) { - throw new WeaviateMemoryException(WeaviateMemoryException.ErrorCodes.FailedToGetVectorData, "Unable to deserialize Weaviate object", e); + this._logger.LogError(e, "Request to find nearest vector in collection: {CollectionName}, with class name: {ClassName} failed.", collectionName, className); + throw; } foreach ((MemoryRecord, double) kv in result) @@ -438,11 +453,10 @@ public async Task RemoveBatchAsync(string collectionName, IEnumerable ke private static MemoryRecord DeserializeToMemoryRecord(JsonNode? json) { string id = json!["_additional"]!["id"]!.GetValue(); - Embedding vector = Embedding.Empty; - if (json["_additional"]!["vector"] != null) + ReadOnlyMemory vector = ReadOnlyMemory.Empty; + if (json["_additional"]!["vector"] is JsonArray jsonArray) { - IEnumerable floats = json["_additional"]!["vector"]!.AsArray().Select(a => a!.GetValue()); - vector = new(floats); + vector = jsonArray.Select(a => a!.GetValue()).ToArray(); } string text = json["sk_text"]!.GetValue(); @@ -467,7 +481,7 @@ private static MemoryRecord DeserializeToMemoryRecord(JsonNode? json) /// public async Task<(MemoryRecord, double)?> GetNearestMatchAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) @@ -511,22 +525,31 @@ private static string ToWeaviateFriendlyClassName(string collectionName) HttpRequestMessage request, CancellationToken cancel = default) { - if (this._endpoint != null) - { - request.RequestUri = new Uri(this._endpoint, request.RequestUri); - } + var apiVersion = !string.IsNullOrWhiteSpace(this._apiVersion) ? this._apiVersion : DefaultApiVersion; + var baseAddress = this._endpoint ?? this._httpClient.BaseAddress; + + request.RequestUri = new Uri(baseAddress, $"{apiVersion}/{request.RequestUri}"); if (!string.IsNullOrEmpty(this._apiKey)) { request.Headers.Add(AuthorizationHeaderName, this._apiKey); } - HttpResponseMessage response = await this._httpClient.SendAsync(request, cancel).ConfigureAwait(false); - string? responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + try + { + HttpResponseMessage response = await this._httpClient.SendWithSuccessCheckAsync(request, cancel).ConfigureAwait(false); + + string? responseContent = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); - this._logger.LogDebug("Weaviate responded with {0}", response.StatusCode); + this._logger.LogDebug("Weaviate responded with {StatusCode}", response.StatusCode); - return (response, responseContent); + return (response, responseContent); + } + catch (HttpOperationException e) + { + this._logger.LogError(e, "Weaviate responded with {StatusCode}", e.StatusCode); + throw; + } } private static MemoryRecordMetadata ToMetadata(WeaviateObject weaviateObject) diff --git a/dotnet/src/Connectors/Connectors.UnitTests/.editorconfig b/dotnet/src/Connectors/Connectors.UnitTests/.editorconfig index 8f4c52fa9f51..394eef685f21 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/.editorconfig +++ b/dotnet/src/Connectors/Connectors.UnitTests/.editorconfig @@ -2,4 +2,5 @@ [*.cs] dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave - +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj b/dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj index 62baaacb23ed..7ec3d3599b19 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj +++ b/dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj @@ -20,6 +20,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + @@ -29,12 +30,13 @@ - + + diff --git a/dotnet/src/Connectors/Connectors.UnitTests/HuggingFace/TextCompletion/HuggingFaceTextCompletionTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/HuggingFace/TextCompletion/HuggingFaceTextCompletionTests.cs index 84f02d98280a..19d0b975acdf 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/HuggingFace/TextCompletion/HuggingFaceTextCompletionTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/HuggingFace/TextCompletion/HuggingFaceTextCompletionTests.cs @@ -5,7 +5,6 @@ using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; using Xunit; @@ -16,56 +15,56 @@ namespace SemanticKernel.Connectors.UnitTests.HuggingFace.TextCompletion; /// public sealed class HuggingFaceTextCompletionTests : IDisposable { - private HttpMessageHandlerStub messageHandlerStub; - private HttpClient httpClient; + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; public HuggingFaceTextCompletionTests() { - this.messageHandlerStub = new HttpMessageHandlerStub(); - this.messageHandlerStub.ResponseToReturn.Content = new StringContent(HuggingFaceTestHelper.GetTestResponse("completion_test_response.json")); + this._messageHandlerStub = new HttpMessageHandlerStub(); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(HuggingFaceTestHelper.GetTestResponse("completion_test_response.json")); - this.httpClient = new HttpClient(this.messageHandlerStub, false); + this._httpClient = new HttpClient(this._messageHandlerStub, false); } [Fact] public async Task SpecifiedModelShouldBeUsedAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - Assert.EndsWith("/fake-model", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.EndsWith("/fake-model", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task NoAuthorizationHeaderShouldBeAddedIfApiKeyIsNotProvidedAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", apiKey: null, httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", apiKey: null, httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - Assert.False(this.messageHandlerStub.RequestHeaders?.Contains("Authorization")); + Assert.False(this._messageHandlerStub.RequestHeaders?.Contains("Authorization")); } [Fact] public async Task AuthorizationHeaderShouldBeAddedIfApiKeyIsProvidedAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", apiKey: "fake-api-key", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", apiKey: "fake-api-key", httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - Assert.True(this.messageHandlerStub.RequestHeaders?.Contains("Authorization")); + Assert.True(this._messageHandlerStub.RequestHeaders?.Contains("Authorization")); - var values = this.messageHandlerStub.RequestHeaders!.GetValues("Authorization"); + var values = this._messageHandlerStub.RequestHeaders!.GetValues("Authorization"); var value = values.SingleOrDefault(); Assert.Equal("Bearer fake-api-key", value); @@ -75,85 +74,85 @@ public async Task AuthorizationHeaderShouldBeAddedIfApiKeyIsProvidedAsync() public async Task UserAgentHeaderShouldBeUsedAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - Assert.True(this.messageHandlerStub.RequestHeaders?.Contains("User-Agent")); + Assert.True(this._messageHandlerStub.RequestHeaders?.Contains("User-Agent")); - var values = this.messageHandlerStub.RequestHeaders!.GetValues("User-Agent"); + var values = this._messageHandlerStub.RequestHeaders!.GetValues("User-Agent"); var value = values.SingleOrDefault(); - Assert.Equal("Microsoft-Semantic-Kernel", value); + Assert.Equal("Semantic-Kernel", value); } [Fact] public async Task ProvidedEndpointShouldBeUsedAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", endpoint: "https://fake-random-test-host/fake-path", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", endpoint: "https://fake-random-test-host/fake-path", httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - Assert.StartsWith("https://fake-random-test-host/fake-path", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith("https://fake-random-test-host/fake-path", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task HttpClientBaseAddressShouldBeUsedAsync() { //Arrange - this.httpClient.BaseAddress = new Uri("https://fake-random-test-host/fake-path"); + this._httpClient.BaseAddress = new Uri("https://fake-random-test-host/fake-path"); - var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - Assert.StartsWith("https://fake-random-test-host/fake-path", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith("https://fake-random-test-host/fake-path", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task DefaultAddressShouldBeUsedAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - Assert.StartsWith("https://api-inference.huggingface.co/models", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith("https://api-inference.huggingface.co/models", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task ModelUrlShouldBeBuiltSuccessfullyAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", endpoint: "https://fake-random-test-host/fake-path", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", endpoint: "https://fake-random-test-host/fake-path", httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - Assert.Equal("https://fake-random-test-host/fake-path/fake-model", this.messageHandlerStub.RequestUri?.AbsoluteUri); + Assert.Equal("https://fake-random-test-host/fake-path/fake-model", this._messageHandlerStub.RequestUri?.AbsoluteUri); } [Fact] public async Task ShouldSendPromptToServiceAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", httpClient: this._httpClient); //Act - await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + await sut.GetCompletionsAsync("fake-text"); //Assert - var requestPayload = JsonSerializer.Deserialize(this.messageHandlerStub.RequestContent); + var requestPayload = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); Assert.NotNull(requestPayload); Assert.Equal("fake-text", requestPayload.Input); @@ -163,10 +162,10 @@ public async Task ShouldSendPromptToServiceAsync() public async Task ShouldHandleServiceResponseAsync() { //Arrange - var sut = new HuggingFaceTextCompletion("fake-model", endpoint: "https://fake-random-test-host/fake-path", httpClient: this.httpClient); + var sut = new HuggingFaceTextCompletion("fake-model", endpoint: "https://fake-random-test-host/fake-path", httpClient: this._httpClient); //Act - var result = await sut.GetCompletionsAsync("fake-text", new CompleteRequestSettings()); + var result = await sut.GetCompletionsAsync("fake-text"); //Assert Assert.NotNull(result); @@ -180,7 +179,7 @@ public async Task ShouldHandleServiceResponseAsync() public void Dispose() { - this.httpClient.Dispose(); - this.messageHandlerStub.Dispose(); + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/HuggingFace/TextEmbedding/HuggingFaceEmbeddingGenerationTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/HuggingFace/TextEmbedding/HuggingFaceEmbeddingGenerationTests.cs index dfdf49ae2dc4..6a4a973408b6 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/HuggingFace/TextEmbedding/HuggingFaceEmbeddingGenerationTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/HuggingFace/TextEmbedding/HuggingFaceEmbeddingGenerationTests.cs @@ -16,101 +16,101 @@ namespace SemanticKernel.Connectors.UnitTests.HuggingFace.TextEmbedding; /// public sealed class HuggingFaceEmbeddingGenerationTests : IDisposable { - private HttpMessageHandlerStub messageHandlerStub; - private HttpClient httpClient; + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; public HuggingFaceEmbeddingGenerationTests() { - this.messageHandlerStub = new HttpMessageHandlerStub(); - this.messageHandlerStub.ResponseToReturn.Content = new StringContent(HuggingFaceTestHelper.GetTestResponse("embeddings_test_response.json")); + this._messageHandlerStub = new HttpMessageHandlerStub(); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(HuggingFaceTestHelper.GetTestResponse("embeddings_test_response.json")); - this.httpClient = new HttpClient(this.messageHandlerStub, false); + this._httpClient = new HttpClient(this._messageHandlerStub, false); } [Fact] public async Task SpecifiedModelShouldBeUsedAsync() { //Arrange - var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this.httpClient, "https://fake-random-test-host/fake-path"); + var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this._httpClient, "https://fake-random-test-host/fake-path"); //Act await sut.GenerateEmbeddingsAsync(new List()); //Assert - Assert.EndsWith("/fake-model", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.EndsWith("/fake-model", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task UserAgentHeaderShouldBeUsedAsync() { //Arrange - var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this.httpClient, "https://fake-random-test-host/fake-path"); + var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this._httpClient, "https://fake-random-test-host/fake-path"); //Act await sut.GenerateEmbeddingsAsync(new List()); //Assert - Assert.True(this.messageHandlerStub.RequestHeaders?.Contains("User-Agent")); + Assert.True(this._messageHandlerStub.RequestHeaders?.Contains("User-Agent")); - var values = this.messageHandlerStub.RequestHeaders!.GetValues("User-Agent"); + var values = this._messageHandlerStub.RequestHeaders!.GetValues("User-Agent"); var value = values.SingleOrDefault(); - Assert.Equal("Microsoft-Semantic-Kernel", value); + Assert.Equal("Semantic-Kernel", value); } [Fact] public async Task ProvidedEndpointShouldBeUsedAsync() { //Arrange - var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this.httpClient, "https://fake-random-test-host/fake-path"); + var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this._httpClient, "https://fake-random-test-host/fake-path"); //Act await sut.GenerateEmbeddingsAsync(new List()); //Assert - Assert.StartsWith("https://fake-random-test-host/fake-path", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith("https://fake-random-test-host/fake-path", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task HttpClientBaseAddressShouldBeUsedAsync() { //Arrange - this.httpClient.BaseAddress = new Uri("https://fake-random-test-host/fake-path"); + this._httpClient.BaseAddress = new Uri("https://fake-random-test-host/fake-path"); - var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this.httpClient); + var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this._httpClient); //Act await sut.GenerateEmbeddingsAsync(new List()); //Assert - Assert.StartsWith("https://fake-random-test-host/fake-path", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith("https://fake-random-test-host/fake-path", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task ModelUrlShouldBeBuiltSuccessfullyAsync() { //Arrange - var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this.httpClient, endpoint: "https://fake-random-test-host/fake-path"); + var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this._httpClient, endpoint: "https://fake-random-test-host/fake-path"); //Act await sut.GenerateEmbeddingsAsync(new List()); //Assert - Assert.Equal("https://fake-random-test-host/fake-path/fake-model", this.messageHandlerStub.RequestUri?.AbsoluteUri); + Assert.Equal("https://fake-random-test-host/fake-path/fake-model", this._messageHandlerStub.RequestUri?.AbsoluteUri); } [Fact] public async Task ShouldSendDataToServiceAsync() { //Arrange - var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this.httpClient, "https://fake-random-test-host/fake-path"); + var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this._httpClient, "https://fake-random-test-host/fake-path"); var data = new List() { "test_string_1", "test_string_2", "test_string_3" }; //Act await sut.GenerateEmbeddingsAsync(data); //Assert - var requestPayload = JsonSerializer.Deserialize(this.messageHandlerStub.RequestContent); + var requestPayload = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); Assert.NotNull(requestPayload); Assert.Equivalent(data, requestPayload.Input); @@ -120,7 +120,7 @@ public async Task ShouldSendDataToServiceAsync() public async Task ShouldHandleServiceResponseAsync() { //Arrange - var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this.httpClient, "https://fake-random-test-host/fake-path"); + var sut = new HuggingFaceTextEmbeddingGeneration("fake-model", this._httpClient, "https://fake-random-test-host/fake-path"); //Act var embeddings = await sut.GenerateEmbeddingsAsync(new List()); @@ -128,13 +128,13 @@ public async Task ShouldHandleServiceResponseAsync() //Assert Assert.NotNull(embeddings); - Assert.Equal(1, embeddings.Count); - Assert.Equal(8, embeddings.First().Count); + Assert.Single(embeddings); + Assert.Equal(8, embeddings.First().Length); } public void Dispose() { - this.httpClient.Dispose(); - this.messageHandlerStub.Dispose(); + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Chroma/ChromaMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Chroma/ChromaMemoryStoreTests.cs index 7c16caa58040..ced200105aab 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Chroma/ChromaMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Chroma/ChromaMemoryStoreTests.cs @@ -7,9 +7,9 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Chroma; using Microsoft.SemanticKernel.Connectors.Memory.Chroma.Http.ApiSchema; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; using Moq; using Xunit; @@ -27,7 +27,6 @@ public sealed class ChromaMemoryStoreTests : IDisposable private readonly HttpMessageHandlerStub _messageHandlerStub; private readonly HttpClient _httpClient; private readonly Mock _chromaClientMock; - private readonly JsonSerializerOptions _serializerOptions; public ChromaMemoryStoreTests() { @@ -38,35 +37,30 @@ public ChromaMemoryStoreTests() this._chromaClientMock .Setup(client => client.GetCollectionAsync(CollectionName, CancellationToken.None)) .ReturnsAsync(new ChromaCollectionModel { Id = CollectionId, Name = CollectionName }); - - this._serializerOptions = new JsonSerializerOptions - { - Converters = { new ChromaBooleanConverter() } - }; } [Fact] public async Task ItUsesProvidedEndpointFromConstructorAsync() { // Arrange - const string endpoint = "https://fake-random-test-host/fake-path/"; - var store = new ChromaMemoryStore(this._httpClient, endpoint); + const string Endpoint = "https://fake-random-test-host/fake-path/"; + var store = new ChromaMemoryStore(this._httpClient, Endpoint); // Act await store.GetAsync("fake-collection", "fake-key"); // Assert - Assert.StartsWith(endpoint, this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith(Endpoint, this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task ItUsesBaseAddressFromHttpClientAsync() { // Arrange - const string baseAddress = "https://fake-random-test-host/fake-path/"; + const string BaseAddress = "https://fake-random-test-host/fake-path/"; using var httpClient = this.GetHttpClientStub(); - httpClient.BaseAddress = new Uri(baseAddress); + httpClient.BaseAddress = new Uri(BaseAddress); var store = new ChromaMemoryStore(httpClient); @@ -74,7 +68,7 @@ public async Task ItUsesBaseAddressFromHttpClientAsync() await store.GetAsync("fake-collection", "fake-key"); // Assert - Assert.StartsWith(baseAddress, this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith(BaseAddress, this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -107,22 +101,22 @@ public async Task ItCanDeleteCollectionAsync() public async Task ItThrowsExceptionOnNonExistentCollectionDeletionAsync() { // Arrange - const string collectionName = "non-existent-collection"; - const string collectionDoesNotExistErrorMessage = $"Collection {collectionName} does not exist"; - const string expectedExceptionMessage = $"Cannot delete non-existent collection {collectionName}"; + const string CollectionName = "non-existent-collection"; + const string CollectionDoesNotExistErrorMessage = $"Collection {CollectionName} does not exist"; + const string ExpectedExceptionMessage = $"Cannot delete non-existent collection {CollectionName}"; this._chromaClientMock - .Setup(client => client.DeleteCollectionAsync(collectionName, CancellationToken.None)) - .Throws(new ChromaClientException(collectionDoesNotExistErrorMessage)); + .Setup(client => client.DeleteCollectionAsync(CollectionName, CancellationToken.None)) + .Throws(new HttpOperationException { ResponseContent = CollectionDoesNotExistErrorMessage }); var store = new ChromaMemoryStore(this._chromaClientMock.Object); // Act - var exception = await Record.ExceptionAsync(() => store.DeleteCollectionAsync(collectionName)); + var exception = await Record.ExceptionAsync(() => store.DeleteCollectionAsync(CollectionName)); // Assert - Assert.IsType(exception); - Assert.Equal(expectedExceptionMessage, exception.Message); + Assert.IsType(exception); + Assert.Equal(ExpectedExceptionMessage, exception.Message); } [Fact] @@ -142,17 +136,17 @@ public async Task ItReturnsTrueWhenCollectionExistsAsync() public async Task ItReturnsFalseWhenCollectionDoesNotExistAsync() { // Arrange - const string collectionName = "non-existent-collection"; - const string collectionDoesNotExistErrorMessage = $"Collection {collectionName} does not exist"; + const string CollectionName = "non-existent-collection"; + const string CollectionDoesNotExistErrorMessage = $"Collection {CollectionName} does not exist"; this._chromaClientMock - .Setup(client => client.GetCollectionAsync(collectionName, CancellationToken.None)) - .Throws(new ChromaClientException(collectionDoesNotExistErrorMessage)); + .Setup(client => client.GetCollectionAsync(CollectionName, CancellationToken.None)) + .Throws(new HttpOperationException { ResponseContent = CollectionDoesNotExistErrorMessage }); var store = new ChromaMemoryStore(this._chromaClientMock.Object); // Act - var doesCollectionExist = await store.DoesCollectionExistAsync(collectionName); + var doesCollectionExist = await store.DoesCollectionExistAsync(CollectionName); // Assert Assert.False(doesCollectionExist); @@ -183,16 +177,16 @@ public async Task ItCanGetMemoryRecordFromCollectionAsync() public async Task ItReturnsNullWhenMemoryRecordDoesNotExistAsync() { // Arrange - const string memoryRecordKey = "fake-record-key"; + const string MemoryRecordKey = "fake-record-key"; this._chromaClientMock - .Setup(client => client.GetEmbeddingsAsync(CollectionId, new[] { memoryRecordKey }, It.IsAny(), CancellationToken.None)) + .Setup(client => client.GetEmbeddingsAsync(CollectionId, new[] { MemoryRecordKey }, It.IsAny(), CancellationToken.None)) .ReturnsAsync(new ChromaEmbeddingsModel()); var store = new ChromaMemoryStore(this._chromaClientMock.Object); // Act - var actualMemoryRecord = await store.GetAsync(CollectionName, memoryRecordKey, withEmbedding: true); + var actualMemoryRecord = await store.GetAsync(CollectionName, MemoryRecordKey, withEmbedding: true); // Assert Assert.Null(actualMemoryRecord); @@ -202,22 +196,22 @@ public async Task ItReturnsNullWhenMemoryRecordDoesNotExistAsync() public async Task ItThrowsExceptionOnGettingMemoryRecordFromNonExistingCollectionAsync() { // Arrange - const string collectionName = "non-existent-collection"; - const string memoryRecordKey = "fake-record-key"; - const string collectionDoesNotExistErrorMessage = $"Collection {collectionName} does not exist"; + const string CollectionName = "non-existent-collection"; + const string MemoryRecordKey = "fake-record-key"; + const string CollectionDoesNotExistErrorMessage = $"Collection {CollectionName} does not exist"; this._chromaClientMock - .Setup(client => client.GetCollectionAsync(collectionName, CancellationToken.None)) - .Throws(new ChromaClientException(collectionDoesNotExistErrorMessage)); + .Setup(client => client.GetCollectionAsync(CollectionName, CancellationToken.None)) + .Throws(new SKException(CollectionDoesNotExistErrorMessage)); var store = new ChromaMemoryStore(this._chromaClientMock.Object); // Act - var exception = await Record.ExceptionAsync(() => store.GetAsync(collectionName, memoryRecordKey, withEmbedding: true)); + var exception = await Record.ExceptionAsync(() => store.GetAsync(CollectionName, MemoryRecordKey, withEmbedding: true)); // Assert - Assert.IsType(exception); - Assert.Equal(collectionDoesNotExistErrorMessage, exception.Message); + Assert.IsType(exception); + Assert.Equal(CollectionDoesNotExistErrorMessage, exception.Message); } [Fact] @@ -286,7 +280,7 @@ public void Dispose() private void AssertMemoryRecordEqual(MemoryRecord expectedRecord, MemoryRecord actualRecord) { Assert.Equal(expectedRecord.Key, actualRecord.Key); - Assert.Equal(expectedRecord.Embedding.Vector, actualRecord.Embedding.Vector); + Assert.True(expectedRecord.Embedding.Span.SequenceEqual(actualRecord.Embedding.Span)); Assert.Equal(expectedRecord.Metadata.Id, actualRecord.Metadata.Id); Assert.Equal(expectedRecord.Metadata.Text, actualRecord.Metadata.Text); Assert.Equal(expectedRecord.Metadata.Description, actualRecord.Metadata.Description); @@ -300,10 +294,10 @@ private HttpClient GetHttpClientStub() return new HttpClient(this._messageHandlerStub, false); } - private MemoryRecord GetRandomMemoryRecord(Embedding? embedding = null) + private MemoryRecord GetRandomMemoryRecord(ReadOnlyMemory? embedding = null) { var id = Guid.NewGuid().ToString(); - var memoryEmbedding = embedding ?? new Embedding(new[] { 1f, 3f, 5f }); + var memoryEmbedding = embedding ?? new[] { 1f, 3f, 5f }; return MemoryRecord.LocalRecord( id: id, @@ -316,7 +310,7 @@ private MemoryRecord GetRandomMemoryRecord(Embedding? embedding = null) private Dictionary GetEmbeddingMetadataFromMemoryRecord(MemoryRecord memoryRecord) { - var serialized = JsonSerializer.Serialize(memoryRecord.Metadata, this._serializerOptions); + var serialized = JsonSerializer.Serialize(memoryRecord.Metadata); return JsonSerializer.Deserialize>(serialized)!; } @@ -325,7 +319,7 @@ private ChromaEmbeddingsModel GetEmbeddingsModelFromMemoryRecords(MemoryRecord[] var embeddingsModel = new ChromaEmbeddingsModel(); embeddingsModel.Ids.AddRange(memoryRecords.Select(l => l.Key)); - embeddingsModel.Embeddings.AddRange(memoryRecords.Select(l => l.Embedding.Vector.ToArray())); + embeddingsModel.Embeddings.AddRange(memoryRecords.Select(l => l.Embedding.ToArray())); embeddingsModel.Metadatas.AddRange(memoryRecords.Select(this.GetEmbeddingMetadataFromMemoryRecord)); return embeddingsModel; diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/DuckDB/DuckDBMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/DuckDB/DuckDBMemoryStoreTests.cs index 8e809a5dd4e9..f2930cc47a34 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/DuckDB/DuckDBMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/DuckDB/DuckDBMemoryStoreTests.cs @@ -5,7 +5,6 @@ using System.Collections.Immutable; using System.Linq; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.DuckDB; using Microsoft.SemanticKernel.Memory; using Xunit; @@ -32,7 +31,7 @@ private IEnumerable CreateBatchRecords(int numRecords) id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); records = records.Append(testRecord); } @@ -42,7 +41,7 @@ private IEnumerable CreateBatchRecords(int numRecords) externalId: "test" + i, sourceName: "sourceName" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); records = records.Append(testRecord); } @@ -139,7 +138,7 @@ public async Task ItCanInsertIntoNonExistentCollectionAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); @@ -151,7 +150,7 @@ public async Task ItCanInsertIntoNonExistentCollectionAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -167,7 +166,7 @@ public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection" + this._collectionNum; @@ -182,8 +181,8 @@ public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() // Assert Assert.NotNull(actualDefault); Assert.NotNull(actualWithEmbedding); - Assert.Empty(actualDefault.Embedding.Vector); - Assert.NotEmpty(actualWithEmbedding.Embedding.Vector); + Assert.True(actualDefault.Embedding.IsEmpty); + Assert.Equal(actualWithEmbedding.Embedding.ToArray(), testRecord.Embedding.ToArray()); } [Fact] @@ -195,7 +194,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection" + this._collectionNum; @@ -210,7 +209,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -226,7 +225,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: DateTimeOffset.UtcNow); string collection = "test_collection" + this._collectionNum; @@ -241,7 +240,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -258,12 +257,12 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() id: commonId, text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); MemoryRecord testRecord2 = MemoryRecord.LocalRecord( id: commonId, text: "text2", description: "description2", - embedding: new Embedding(new float[] { 1, 2, 4 })); + embedding: new float[] { 1, 2, 4 }); string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -277,8 +276,8 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord2.Metadata.Id, actual.Key); - Assert.NotEqual(testRecord.Embedding.Vector, actual.Embedding.Vector); - Assert.Equal(testRecord2.Embedding.Vector, actual.Embedding.Vector); + Assert.False(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); + Assert.True(testRecord2.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.NotEqual(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord2.Metadata.Description, actual.Metadata.Description); } @@ -292,7 +291,7 @@ public async Task ExistingRecordCanBeRemovedAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -359,7 +358,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() { // Arrange using var db = await DuckDBMemoryStore.ConnectAsync(); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -369,7 +368,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -377,7 +376,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -385,7 +384,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -393,7 +392,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -401,7 +400,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await db.UpsertAsync(collection, testRecord); // Act @@ -422,7 +421,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( { // Arrange using var db = await DuckDBMemoryStore.ConnectAsync(); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection" + this._collectionNum; this._collectionNum++; await db.CreateCollectionAsync(collection); @@ -431,7 +430,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -439,7 +438,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -447,7 +446,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -455,7 +454,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -463,7 +462,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await db.UpsertAsync(collection, testRecord); // Act @@ -474,8 +473,8 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( // Assert Assert.NotNull(topNResultDefault); Assert.NotNull(topNResultWithEmbedding); - Assert.Empty(topNResultDefault.Value.Item1.Embedding.Vector); - Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.Vector); + Assert.True(topNResultDefault.Value.Item1.Embedding.IsEmpty); + Assert.False(topNResultWithEmbedding.Value.Item1.Embedding.IsEmpty); } [Fact] @@ -483,7 +482,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() { // Arrange using var db = await DuckDBMemoryStore.ConnectAsync(); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection" + this._collectionNum; this._collectionNum++; await db.CreateCollectionAsync(collection); @@ -492,7 +491,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -500,7 +499,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -508,7 +507,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -516,7 +515,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -524,7 +523,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await db.UpsertAsync(collection, testRecord); // Act @@ -542,7 +541,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() { // Arrange using var db = await DuckDBMemoryStore.ConnectAsync(); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -554,7 +553,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await db.UpsertAsync(collection, testRecord); } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Kusto/KustoMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Kusto/KustoMemoryStoreTests.cs new file mode 100644 index 000000000000..e840cd936a56 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Kusto/KustoMemoryStoreTests.cs @@ -0,0 +1,405 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Kusto.Cloud.Platform.Utils; +using Kusto.Data.Common; +using Microsoft.SemanticKernel.Connectors.Memory.Kusto; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Memory; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.Memory.Kusto; + +/// +/// Unit tests for class. +/// +public class KustoMemoryStoreTests +{ + private const string CollectionName = "fake_collection"; + private const string DatabaseName = "FakeDb"; + private readonly Mock _cslQueryProviderMock; + private readonly Mock _cslAdminProviderMock; + + public KustoMemoryStoreTests() + { + this._cslQueryProviderMock = new Mock(); + this._cslAdminProviderMock = new Mock(); + + this._cslAdminProviderMock + .Setup(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.IsAny(), + It.IsAny())) + .ReturnsAsync(FakeEmptyResult()); + + this._cslAdminProviderMock + .Setup(client => client.ExecuteControlCommand( + DatabaseName, + It.IsAny(), + It.IsAny())) + .Returns(FakeEmptyResult()); + + this._cslQueryProviderMock + .Setup(client => client.ExecuteQueryAsync( + DatabaseName, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(FakeEmptyResult()); + } + + [Fact] + public async Task ItCanCreateCollectionAsync() + { + // Arrange + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + await store.CreateCollectionAsync(CollectionName); + + // Assert + this._cslAdminProviderMock + .Verify(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => s.StartsWith($".create table {CollectionName}")), + It.Is(crp => string.Equals(crp.Application, Telemetry.HttpUserAgent, StringComparison.Ordinal)) + ), Times.Once()); + } + + [Fact] + public async Task ItCanDeleteCollectionAsync() + { + // Arrange + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + await store.DeleteCollectionAsync(CollectionName); + + // Assert + // Assert + this._cslAdminProviderMock + .Verify(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => s.StartsWith($".drop table {CollectionName}")), + It.Is(crp => string.Equals(crp.Application, Telemetry.HttpUserAgent, StringComparison.Ordinal)) + ), Times.Once()); + } + + [Fact] + public async Task ItReturnsTrueWhenCollectionExistsAsync() + { + // Arrange + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + this._cslAdminProviderMock + .Setup(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => s.StartsWith(CslCommandGenerator.GenerateTablesShowCommand())), + It.IsAny())) + .ReturnsAsync(CollectionToSingleColumnDataReader(new[] { CollectionName })); + + // Act + var doesCollectionExist = await store.DoesCollectionExistAsync(CollectionName); + + // Assert + Assert.True(doesCollectionExist); + } + + [Fact] + public async Task ItReturnsFalseWhenCollectionDoesNotExistAsync() + { + // Arrange + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + this._cslAdminProviderMock + .Setup(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => s.StartsWith(CslCommandGenerator.GenerateTablesShowCommand())), + It.IsAny())) + .ReturnsAsync(FakeEmptyResult()); + + // Act + var doesCollectionExist = await store.DoesCollectionExistAsync(CollectionName); + + // Assert + Assert.False(doesCollectionExist); + } + + [Fact] + public async Task ItCanUpsertAsync() + { + // Arrange + var expectedMemoryRecord = this.GetRandomMemoryRecord(); + var kustoMemoryEntry = new KustoMemoryRecord(expectedMemoryRecord); + + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + var actualMemoryRecordKey = await store.UpsertAsync(CollectionName, expectedMemoryRecord); + + // Assert + this._cslAdminProviderMock.Verify(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => s.StartsWith($".ingest inline into table {CollectionName}", StringComparison.Ordinal) && s.Contains(actualMemoryRecordKey, StringComparison.Ordinal)), + It.IsAny()), Times.Once()); + Assert.Equal(expectedMemoryRecord.Key, actualMemoryRecordKey); + } + + [Fact] + public async Task ItCanUpsertBatchAsyncAsync() + { + // Arrange + var memoryRecord1 = this.GetRandomMemoryRecord(); + var memoryRecord2 = this.GetRandomMemoryRecord(); + var memoryRecord3 = this.GetRandomMemoryRecord(); + + var batchUpsertMemoryRecords = new[] { memoryRecord1, memoryRecord2, memoryRecord3 }; + var expectedMemoryRecordKeys = batchUpsertMemoryRecords.Select(l => l.Key).ToList(); + + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + var actualMemoryRecordKeys = await store.UpsertBatchAsync(CollectionName, batchUpsertMemoryRecords).ToListAsync(); + + // Assert + this._cslAdminProviderMock + .Verify(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => + s.StartsWith($".ingest inline into table {CollectionName}", StringComparison.Ordinal) && + batchUpsertMemoryRecords.All(r => s.Contains(r.Key, StringComparison.Ordinal))), + It.IsAny() + ), Times.Once()); + + for (int i = 0; i < expectedMemoryRecordKeys.Count; i++) + { + Assert.Equal(expectedMemoryRecordKeys[i], actualMemoryRecordKeys[i]); + } + } + + [Fact] + public async Task ItCanGetMemoryRecordFromCollectionAsync() + { + // Arrange + var expectedMemoryRecord = this.GetRandomMemoryRecord(); + var kustoMemoryEntry = new KustoMemoryRecord(expectedMemoryRecord); + + this._cslQueryProviderMock + .Setup(client => client.ExecuteQueryAsync( + DatabaseName, + It.Is(s => s.Contains(CollectionName) && s.Contains(expectedMemoryRecord.Key)), + It.IsAny(), + CancellationToken.None)) + .ReturnsAsync(CollectionToDataReader(new string[][] { + new string[] { + expectedMemoryRecord.Key, + KustoSerializer.SerializeMetadata(expectedMemoryRecord.Metadata), + KustoSerializer.SerializeDateTimeOffset(expectedMemoryRecord.Timestamp), + KustoSerializer.SerializeEmbedding(expectedMemoryRecord.Embedding), + }})); + + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + var actualMemoryRecord = await store.GetAsync(CollectionName, expectedMemoryRecord.Key, withEmbedding: true); + + // Assert + Assert.NotNull(actualMemoryRecord); + this.AssertMemoryRecordEqual(expectedMemoryRecord, actualMemoryRecord); + } + + [Fact] + public async Task ItReturnsNullWhenMemoryRecordDoesNotExistAsync() + { + // Arrange + const string MemoryRecordKey = "fake-record-key"; + + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + var actualMemoryRecord = await store.GetAsync(CollectionName, MemoryRecordKey, withEmbedding: true); + + // Assert + Assert.Null(actualMemoryRecord); + } + + [Fact] + public async Task ItCanGetMemoryRecordBatchFromCollectionAsync() + { + // Arrange + var memoryRecord1 = this.GetRandomMemoryRecord(); + var memoryRecord2 = this.GetRandomMemoryRecord(); + var memoryRecord3 = this.GetRandomMemoryRecord(); + + var batchUpsertMemoryRecords = new[] { memoryRecord1, memoryRecord2, memoryRecord3 }; + var expectedMemoryRecordKeys = batchUpsertMemoryRecords.Select(l => l.Key).ToList(); + + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + this._cslQueryProviderMock + .Setup(client => client.ExecuteQueryAsync( + DatabaseName, + It.Is(s => + s.Contains(CollectionName, StringComparison.Ordinal) && + batchUpsertMemoryRecords.All(r => s.Contains(r.Key, StringComparison.Ordinal))), + It.IsAny(), + CancellationToken.None)) + .ReturnsAsync(CollectionToDataReader(batchUpsertMemoryRecords.Select(r => new string[] { + r.Key, + KustoSerializer.SerializeMetadata(r.Metadata), + KustoSerializer.SerializeDateTimeOffset(r.Timestamp), + KustoSerializer.SerializeEmbedding(r.Embedding), + }).ToArray())); + + // Act + var actualMemoryRecords = await store.GetBatchAsync(CollectionName, expectedMemoryRecordKeys, withEmbeddings: true).ToListAsync(); + + // Assert + Assert.NotNull(actualMemoryRecords); + for (var i = 0; i < actualMemoryRecords.Count; i++) + { + this.AssertMemoryRecordEqual(batchUpsertMemoryRecords[i], actualMemoryRecords[i]); + } + } + + [Fact] + public async Task ItCanReturnCollectionsAsync() + { + // Arrange + var expectedCollections = new List { "fake-collection-1", "fake-collection-2", "fake-collection-3" }; + + this._cslAdminProviderMock + .Setup(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => s.StartsWith(CslCommandGenerator.GenerateTablesShowCommand(), StringComparison.Ordinal)), + It.IsAny()) + ).ReturnsAsync(CollectionToSingleColumnDataReader(expectedCollections)); + + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + var actualCollections = await store.GetCollectionsAsync().ToListAsync(); + + // Assert + Assert.Equal(expectedCollections.Count, actualCollections.Count); + + for (var i = 0; i < expectedCollections.Count; i++) + { + Assert.Equal(expectedCollections[i], actualCollections[i]); + } + } + + [Fact] + public async Task ItCanRemoveAsync() + { + // Arrange + const string MemoryRecordKey = "fake-record-key"; + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + await store.RemoveAsync(CollectionName, MemoryRecordKey); + + // Assert + this._cslAdminProviderMock + .Verify(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => s.Replace(" ", " ").StartsWith($".delete table {CollectionName}") && s.Contains(MemoryRecordKey)), // Replace double spaces with single space to account for the fact that the query is formatted with double spaces and to be future proof + It.IsAny() + ), Times.Once()); + } + + [Fact] + public async Task ItCanRemoveBatchAsync() + { + // Arrange + string[] memoryRecordKeys = new string[] { "fake-record-key1", "fake-record-key2", "fake-record-key3" }; + using var store = new KustoMemoryStore(this._cslAdminProviderMock.Object, this._cslQueryProviderMock.Object, DatabaseName); + + // Act + await store.RemoveBatchAsync(CollectionName, memoryRecordKeys); + + // Assert + this._cslAdminProviderMock + .Verify(client => client.ExecuteControlCommandAsync( + DatabaseName, + It.Is(s => s.Replace(" ", " ").StartsWith($".delete table {CollectionName}") && memoryRecordKeys.All(r => s.Contains(r, StringComparison.OrdinalIgnoreCase))), + It.IsAny() + ), Times.Once()); + } + + #region private ================================================================================ + + private void AssertMemoryRecordEqual(MemoryRecord expectedRecord, MemoryRecord actualRecord) + { + Assert.Equal(expectedRecord.Key, actualRecord.Key); + Assert.Equal(expectedRecord.Timestamp, actualRecord.Timestamp); + Assert.True(expectedRecord.Embedding.Span.SequenceEqual(actualRecord.Embedding.Span)); + Assert.Equal(expectedRecord.Metadata.Id, actualRecord.Metadata.Id); + Assert.Equal(expectedRecord.Metadata.Text, actualRecord.Metadata.Text); + Assert.Equal(expectedRecord.Metadata.Description, actualRecord.Metadata.Description); + Assert.Equal(expectedRecord.Metadata.AdditionalMetadata, actualRecord.Metadata.AdditionalMetadata); + Assert.Equal(expectedRecord.Metadata.IsReference, actualRecord.Metadata.IsReference); + Assert.Equal(expectedRecord.Metadata.ExternalSourceName, actualRecord.Metadata.ExternalSourceName); + } + + private MemoryRecord GetRandomMemoryRecord(ReadOnlyMemory? embedding = null) + { + var id = Guid.NewGuid().ToString(); + var memoryEmbedding = embedding ?? new[] { 1f, 3f, 5f }; + + return MemoryRecord.LocalRecord( + id: id, + text: "text-" + Guid.NewGuid().ToString(), + description: "description-" + Guid.NewGuid().ToString(), + embedding: memoryEmbedding, + additionalMetadata: "metadata-" + Guid.NewGuid().ToString(), + key: id, + timestamp: new DateTimeOffset(2023, 8, 4, 23, 59, 59, TimeSpan.Zero)); + } + + private static DataTableReader FakeEmptyResult() => Array.Empty().ToDataTable().CreateDataReader(); + + private static DataTableReader CollectionToSingleColumnDataReader(IEnumerable collection) + { + using var table = new DataTable(); + table.Columns.Add("Column1", typeof(string)); + + foreach (var item in collection) + { + table.Rows.Add(item); + } + + return table.CreateDataReader(); + } + + private static DataTableReader CollectionToDataReader(string[][] data) + { + using var table = new DataTable(); + + if (data != null) + { + data = data.ToArrayIfNotAlready(); + if (data[0] != null) + { + for (int i = 0; i < data[0].Length; i++) + { + table.Columns.Add($"Column{i + 1}", typeof(string)); + } + } + + for (int i = 0; i < data.Length; i++) + { + table.Rows.Add(data[i]); + } + } + + return table.CreateDataReader(); + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeKernelBuilderExtensionsTests.cs index 2fc1abd2d1e7..4a624fb4ca13 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeKernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeKernelBuilderExtensionsTests.cs @@ -13,42 +13,46 @@ namespace SemanticKernel.Connectors.UnitTests.Memory.Pinecone; public sealed class PineconeKernelBuilderExtensionsTests : IDisposable { - private HttpMessageHandlerStub messageHandlerStub; - private HttpClient httpClient; + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; public PineconeKernelBuilderExtensionsTests() { - this.messageHandlerStub = new HttpMessageHandlerStub(); + this._messageHandlerStub = new HttpMessageHandlerStub(); - this.httpClient = new HttpClient(this.messageHandlerStub, false); + this._httpClient = new HttpClient(this._messageHandlerStub, false); } [Fact] - public async Task PineconeMemoryStoreShouldBeProperlyInitialized() + public async Task PineconeMemoryStoreShouldBeProperlyInitializedAsync() { //Arrange - this.messageHandlerStub.ResponseToReturn.Content = new StringContent("[\"fake-index1\"]", Encoding.UTF8, MediaTypeNames.Application.Json); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent("[\"fake-index1\"]", Encoding.UTF8, MediaTypeNames.Application.Json); var builder = new KernelBuilder(); - builder.WithPineconeMemoryStore("fake-environment", "fake-api-key", this.httpClient); +#pragma warning disable CS0618 // This will be removed in a future release. + builder.WithPineconeMemoryStore("fake-environment", "fake-api-key", this._httpClient); +#pragma warning restore CS0618 // This will be removed in a future release. builder.WithAzureTextEmbeddingGenerationService("fake-deployment-name", "https://fake-random-test-host/fake-path", "fake -api-key"); var kernel = builder.Build(); //This call triggers the internal factory registered by WithPineconeMemoryStore method to create an instance of the PineconeMemoryStore class. //Act +#pragma warning disable CS0618 // This will be removed in a future release. await kernel.Memory.GetCollectionsAsync(); //This call triggers a subsequent call to Pinecone memory store. +#pragma warning restore CS0618 // This will be removed in a future release. //Assert - Assert.Equal("https://controller.fake-environment.pinecone.io/databases", this.messageHandlerStub?.RequestUri?.AbsoluteUri); + Assert.Equal("https://controller.fake-environment.pinecone.io/databases", this._messageHandlerStub?.RequestUri?.AbsoluteUri); var headerValues = Enumerable.Empty(); - var headerExists = this.messageHandlerStub?.RequestHeaders?.TryGetValues("Api-Key", out headerValues); + var headerExists = this._messageHandlerStub?.RequestHeaders?.TryGetValues("Api-Key", out headerValues); Assert.True(headerExists); Assert.Contains(headerValues!, (value) => value == "fake-api-key"); } public void Dispose() { - this.httpClient.Dispose(); - this.messageHandlerStub.Dispose(); + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeMemoryBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeMemoryBuilderExtensionsTests.cs new file mode 100644 index 000000000000..abfc4e85f6d1 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeMemoryBuilderExtensionsTests.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Linq; +using System.Net.Http; +using System.Net.Mime; +using System.Text; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Connectors.Memory.Pinecone; +using Microsoft.SemanticKernel.Plugins.Memory; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.Memory.Pinecone; + +public sealed class PineconeMemoryBuilderExtensionsTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; + + public PineconeMemoryBuilderExtensionsTests() + { + this._messageHandlerStub = new HttpMessageHandlerStub(); + + this._httpClient = new HttpClient(this._messageHandlerStub, false); + } + + [Fact] + public async Task PineconeMemoryStoreShouldBeProperlyInitializedAsync() + { + // Arrange + var embeddingGenerationMock = Mock.Of(); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent("[\"fake-index1\"]", Encoding.UTF8, MediaTypeNames.Application.Json); + + var builder = new MemoryBuilder(); + builder.WithPineconeMemoryStore("fake-environment", "fake-api-key", this._httpClient); + builder.WithTextEmbeddingGeneration(embeddingGenerationMock); + + var memory = builder.Build(); //This call triggers the internal factory registered by WithPineconeMemoryStore method to create an instance of the PineconeMemoryStore class. + + // Act + await memory.GetCollectionsAsync(); //This call triggers a subsequent call to Pinecone memory store. + + // Assert + Assert.Equal("https://controller.fake-environment.pinecone.io/databases", this._messageHandlerStub?.RequestUri?.AbsoluteUri); + + var headerValues = Enumerable.Empty(); + var headerExists = this._messageHandlerStub?.RequestHeaders?.TryGetValues("Api-Key", out headerValues); + Assert.True(headerExists); + Assert.Contains(headerValues!, (value) => value == "fake-api-key"); + } + + public void Dispose() + { + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeMemoryStoreTests.cs index 73d518cef69a..aef930aa8bca 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Pinecone/PineconeMemoryStoreTests.cs @@ -1,13 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Pinecone; using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; using Moq; using Xunit; @@ -28,26 +29,26 @@ public class PineconeMemoryStoreTests private readonly string _description2 = "description2"; private readonly string _description3 = "description3"; - private readonly Embedding _embedding = new(new float[] { 1, 1, 1 }); - private readonly Embedding _embedding2 = new(new float[] { 2, 2, 2 }); - private readonly Embedding _embedding3 = new(new float[] { 3, 3, 3 }); + private readonly ReadOnlyMemory _embedding = new float[] { 1, 1, 1 }; + private readonly ReadOnlyMemory _embedding2 = new float[] { 2, 2, 2 }; + private readonly ReadOnlyMemory _embedding3 = new float[] { 3, 3, 3 }; private readonly Mock _mockPineconeClient; - private readonly Mock> _mockLogger = new(); + private readonly Mock _mockLoggerFactory = new(); private readonly PineconeMemoryStore _pineconeMemoryStore; public PineconeMemoryStoreTests() { this._mockPineconeClient = new Mock(); - this._pineconeMemoryStore = new PineconeMemoryStore(this._mockPineconeClient.Object, this._mockLogger.Object); + this._pineconeMemoryStore = new PineconeMemoryStore(this._mockPineconeClient.Object, this._mockLoggerFactory.Object); } [Fact] public void ConnectionCanBeInitialized() { // Arrange & Act - PineconeMemoryStore memoryStore = new(this._mockPineconeClient.Object, this._mockLogger.Object); + PineconeMemoryStore memoryStore = new(this._mockPineconeClient.Object, this._mockLoggerFactory.Object); // Assert Assert.NotNull(memoryStore); @@ -62,13 +63,13 @@ public async Task ItThrowsExceptionOnIndexCreationAsync() .ReturnsAsync(false); // Act - var exception = await Assert.ThrowsAsync(async () => await this._pineconeMemoryStore.CreateCollectionAsync("test")); + var exception = await Assert.ThrowsAsync(async () => await this._pineconeMemoryStore.CreateCollectionAsync("test")); // Assert this._mockPineconeClient .Verify>(x => x.DoesIndexExistAsync("test", It.IsAny()), Times.Once()); - Assert.Equal(PineconeMemoryException.ErrorCodes.IndexNotReady, exception.ErrorCode); + Assert.NotNull(exception); } [Fact] @@ -221,7 +222,7 @@ public async Task TestRemoveAsync() public async Task TestGetNearestMatchesAsync() { // Arrange - Embedding embedding = new(new float[] { 0.1f, 0.2f }); + ReadOnlyMemory embedding = new float[] { 0.1f, 0.2f }; List<(PineconeDocument, double)> queryResults = new() { @@ -232,20 +233,20 @@ public async Task TestGetNearestMatchesAsync() { { "document_Id", "value1" }, }, - Values = this._embedding.Vector + Values = this._embedding }, 0.9), new(new() { Id = this._id2, Metadata = new Dictionary { { "document_Id", "value2" } }, - Values = this._embedding2.Vector, + Values = this._embedding2, }, 0.5) }; this._mockPineconeClient .Setup>(x => x.GetMostRelevantAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -258,7 +259,7 @@ public async Task TestGetNearestMatchesAsync() // Act List<(MemoryRecord, double)> results = await this._pineconeMemoryStore.GetNearestMatchesAsync( "indexName", - new Embedding(new[] { 0.1f, 0.2f, 0.3f }), + new[] { 0.1f, 0.2f, 0.3f }, 2, 0.5, true).ToListAsync(); diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Postgres/PostgresMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Postgres/PostgresMemoryStoreTests.cs index e358466ed960..cf7fa84ea835 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Postgres/PostgresMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Postgres/PostgresMemoryStoreTests.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Postgres; using Microsoft.SemanticKernel.Memory; using Moq; @@ -34,7 +33,7 @@ public PostgresMemoryStoreTests() public async Task ItCanCreateCollectionAsync() { // Arrange - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act await store.CreateCollectionAsync(CollectionName); @@ -47,7 +46,7 @@ public async Task ItCanCreateCollectionAsync() public async Task ItCanDeleteCollectionAsync() { // Arrange - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act await store.DeleteCollectionAsync(CollectionName); @@ -60,7 +59,7 @@ public async Task ItCanDeleteCollectionAsync() public async Task ItReturnsTrueWhenCollectionExistsAsync() { // Arrange - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act var doesCollectionExist = await store.DoesCollectionExistAsync(CollectionName); @@ -73,16 +72,16 @@ public async Task ItReturnsTrueWhenCollectionExistsAsync() public async Task ItReturnsFalseWhenCollectionDoesNotExistAsync() { // Arrange - const string collectionName = "non-existent-collection"; + const string CollectionName = "non-existent-collection"; this._postgresDbClientMock - .Setup(client => client.DoesTableExistsAsync(collectionName, CancellationToken.None)) + .Setup(client => client.DoesTableExistsAsync(CollectionName, CancellationToken.None)) .ReturnsAsync(false); - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act - var doesCollectionExist = await store.DoesCollectionExistAsync(collectionName); + var doesCollectionExist = await store.DoesCollectionExistAsync(CollectionName); // Assert Assert.False(doesCollectionExist); @@ -95,7 +94,7 @@ public async Task ItCanUpsertAsync() var expectedMemoryRecord = this.GetRandomMemoryRecord(); var postgresMemoryEntry = this.GetPostgresMemoryEntryFromMemoryRecord(expectedMemoryRecord)!; - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act var actualMemoryRecordKey = await store.UpsertAsync(CollectionName, expectedMemoryRecord); @@ -116,7 +115,7 @@ public async Task ItCanUpsertBatchAsyncAsync() var batchUpsertMemoryRecords = new[] { memoryRecord1, memoryRecord2, memoryRecord3 }; var expectedMemoryRecordKeys = batchUpsertMemoryRecords.Select(l => l.Key).ToList(); - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act var actualMemoryRecordKeys = await store.UpsertBatchAsync(CollectionName, batchUpsertMemoryRecords).ToListAsync(); @@ -145,7 +144,7 @@ public async Task ItCanGetMemoryRecordFromCollectionAsync() .Setup(client => client.ReadAsync(CollectionName, expectedMemoryRecord.Key, true, CancellationToken.None)) .ReturnsAsync(postgresMemoryEntry); - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act var actualMemoryRecord = await store.GetAsync(CollectionName, expectedMemoryRecord.Key, withEmbedding: true); @@ -159,16 +158,16 @@ public async Task ItCanGetMemoryRecordFromCollectionAsync() public async Task ItReturnsNullWhenMemoryRecordDoesNotExistAsync() { // Arrange - const string memoryRecordKey = "fake-record-key"; + const string MemoryRecordKey = "fake-record-key"; this._postgresDbClientMock - .Setup(client => client.ReadAsync(CollectionName, memoryRecordKey, true, CancellationToken.None)) + .Setup(client => client.ReadAsync(CollectionName, MemoryRecordKey, true, CancellationToken.None)) .ReturnsAsync((PostgresMemoryEntry?)null); - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act - var actualMemoryRecord = await store.GetAsync(CollectionName, memoryRecordKey, withEmbedding: true); + var actualMemoryRecord = await store.GetAsync(CollectionName, MemoryRecordKey, withEmbedding: true); // Assert Assert.Null(actualMemoryRecord); @@ -200,7 +199,7 @@ public async Task ItCanGetMemoryRecordBatchFromCollectionAsync() .Setup(client => client.ReadBatchAsync(CollectionName, memoryRecordKeys, true, CancellationToken.None)) .Returns(expectedMemoryRecords.Select(memoryRecord => this.GetPostgresMemoryEntryFromMemoryRecord(memoryRecord)).ToAsyncEnumerable()); - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act var actualMemoryRecords = await store.GetBatchAsync(CollectionName, memoryRecordKeys, withEmbeddings: true).ToListAsync(); @@ -225,7 +224,7 @@ public async Task ItCanReturnCollectionsAsync() .Setup(client => client.GetTablesAsync(CancellationToken.None)) .Returns(expectedCollections.ToAsyncEnumerable()); - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act var actualCollections = await store.GetCollectionsAsync().ToListAsync(); @@ -243,14 +242,14 @@ public async Task ItCanReturnCollectionsAsync() public async Task ItCanRemoveAsync() { // Arrange - const string memoryRecordKey = "fake-record-key"; - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + const string MemoryRecordKey = "fake-record-key"; + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act - await store.RemoveAsync(CollectionName, memoryRecordKey); + await store.RemoveAsync(CollectionName, MemoryRecordKey); // Assert - this._postgresDbClientMock.Verify(client => client.DeleteAsync(CollectionName, memoryRecordKey, CancellationToken.None), Times.Once()); + this._postgresDbClientMock.Verify(client => client.DeleteAsync(CollectionName, MemoryRecordKey, CancellationToken.None), Times.Once()); } [Fact] @@ -258,7 +257,7 @@ public async Task ItCanRemoveBatchAsync() { // Arrange string[] memoryRecordKeys = new string[] { "fake-record-key1", "fake-record-key2", "fake-record-key3" }; - var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); + using var store = new PostgresMemoryStore(this._postgresDbClientMock.Object); // Act await store.RemoveBatchAsync(CollectionName, memoryRecordKeys); @@ -272,7 +271,7 @@ public async Task ItCanRemoveBatchAsync() private void AssertMemoryRecordEqual(MemoryRecord expectedRecord, MemoryRecord actualRecord) { Assert.Equal(expectedRecord.Key, actualRecord.Key); - Assert.Equal(expectedRecord.Embedding.Vector, actualRecord.Embedding.Vector); + Assert.True(expectedRecord.Embedding.Span.SequenceEqual(actualRecord.Embedding.Span)); Assert.Equal(expectedRecord.Metadata.Id, actualRecord.Metadata.Id); Assert.Equal(expectedRecord.Metadata.Text, actualRecord.Metadata.Text); Assert.Equal(expectedRecord.Metadata.Description, actualRecord.Metadata.Description); @@ -281,10 +280,10 @@ private void AssertMemoryRecordEqual(MemoryRecord expectedRecord, MemoryRecord a Assert.Equal(expectedRecord.Metadata.ExternalSourceName, actualRecord.Metadata.ExternalSourceName); } - private MemoryRecord GetRandomMemoryRecord(Embedding? embedding = null) + private MemoryRecord GetRandomMemoryRecord(ReadOnlyMemory? embedding = null) { var id = Guid.NewGuid().ToString(); - var memoryEmbedding = embedding ?? new Embedding(new[] { 1f, 3f, 5f }); + var memoryEmbedding = embedding ?? new[] { 1f, 3f, 5f }; return MemoryRecord.LocalRecord( id: id, @@ -301,7 +300,7 @@ private PostgresMemoryEntry GetPostgresMemoryEntryFromMemoryRecord(MemoryRecord return new PostgresMemoryEntry() { Key = memoryRecord.Key, - Embedding = new Pgvector.Vector(memoryRecord.Embedding.Vector.ToArray()), + Embedding = new Pgvector.Vector(memoryRecord.Embedding.ToArray()), MetadataString = memoryRecord.GetSerializedMetadata(), Timestamp = memoryRecord.Timestamp?.UtcDateTime }; diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantKernelBuilderExtensionsTests.cs index aee72a868d39..e7d1052faadb 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantKernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantKernelBuilderExtensionsTests.cs @@ -12,38 +12,42 @@ namespace SemanticKernel.Connectors.UnitTests.Memory.Qdrant; public sealed class QdrantKernelBuilderExtensionsTests : IDisposable { - private HttpMessageHandlerStub messageHandlerStub; - private HttpClient httpClient; + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; public QdrantKernelBuilderExtensionsTests() { - this.messageHandlerStub = new HttpMessageHandlerStub(); + this._messageHandlerStub = new HttpMessageHandlerStub(); - this.httpClient = new HttpClient(this.messageHandlerStub, false); + this._httpClient = new HttpClient(this._messageHandlerStub, false); } [Fact] - public async Task QdrantMemoryStoreShouldBeProperlyInitialized() + public async Task QdrantMemoryStoreShouldBeProperlyInitializedAsync() { //Arrange - this.httpClient.BaseAddress = new Uri("https://fake-random-qdrant-host"); - this.messageHandlerStub.ResponseToReturn.Content = new StringContent("{\"result\":{\"collections\":[]}}", Encoding.UTF8, MediaTypeNames.Application.Json); + this._httpClient.BaseAddress = new Uri("https://fake-random-qdrant-host"); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent("{\"result\":{\"collections\":[]}}", Encoding.UTF8, MediaTypeNames.Application.Json); var builder = new KernelBuilder(); - builder.WithQdrantMemoryStore(this.httpClient, 123); +#pragma warning disable CS0618 // This will be removed in a future release. + builder.WithQdrantMemoryStore(this._httpClient, 123); +#pragma warning restore CS0618 // This will be removed in a future release. builder.WithAzureTextEmbeddingGenerationService("fake-deployment-name", "https://fake-random-text-embedding-generation-host/fake-path", "fake-api-key"); var kernel = builder.Build(); //This call triggers the internal factory registered by WithQdrantMemoryStore method to create an instance of the QdrantMemoryStore class. //Act +#pragma warning disable CS0618 // This will be removed in a future release. await kernel.Memory.GetCollectionsAsync(); //This call triggers a subsequent call to Qdrant memory store. +#pragma warning restore CS0618 // This will be removed in a future release. //Assert - Assert.Equal("https://fake-random-qdrant-host/collections", this.messageHandlerStub?.RequestUri?.AbsoluteUri); + Assert.Equal("https://fake-random-qdrant-host/collections", this._messageHandlerStub?.RequestUri?.AbsoluteUri); } public void Dispose() { - this.httpClient.Dispose(); - this.messageHandlerStub.Dispose(); + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryBuilderExtensionsTests.cs new file mode 100644 index 000000000000..37ed036b54f6 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryBuilderExtensionsTests.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Net.Mime; +using System.Text; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; +using Microsoft.SemanticKernel.Plugins.Memory; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.Memory.Qdrant; + +public sealed class QdrantMemoryBuilderExtensionsTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; + + public QdrantMemoryBuilderExtensionsTests() + { + this._messageHandlerStub = new HttpMessageHandlerStub(); + + this._httpClient = new HttpClient(this._messageHandlerStub, false); + } + + [Fact] + public async Task QdrantMemoryStoreShouldBeProperlyInitializedAsync() + { + // Arrange + var embeddingGenerationMock = Mock.Of(); + + this._httpClient.BaseAddress = new Uri("https://fake-random-qdrant-host"); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent("{\"result\":{\"collections\":[]}}", Encoding.UTF8, MediaTypeNames.Application.Json); + + var builder = new MemoryBuilder(); + builder.WithQdrantMemoryStore(this._httpClient, 123); + builder.WithTextEmbeddingGeneration(embeddingGenerationMock); + var memory = builder.Build(); //This call triggers the internal factory registered by WithQdrantMemoryStore method to create an instance of the QdrantMemoryStore class. + + // Act + await memory.GetCollectionsAsync(); //This call triggers a subsequent call to Qdrant memory store. + + // Assert + Assert.Equal("https://fake-random-qdrant-host/collections", this._messageHandlerStub?.RequestUri?.AbsoluteUri); + } + + public void Dispose() + { + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests.cs index db8b0b8c2401..3c4315ff8fa2 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests.cs @@ -3,14 +3,11 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.Connectors.Memory.Pinecone; using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; -using Microsoft.SemanticKernel.Connectors.Memory.Qdrant.Diagnostics; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; using Moq; using Xunit; @@ -31,10 +28,17 @@ public class QdrantMemoryStoreTests private readonly string _description = "description"; private readonly string _description2 = "description2"; private readonly string _description3 = "description3"; - private readonly Embedding _embedding = new(new float[] { 1, 1, 1 }); - private readonly Embedding _embedding2 = new(new float[] { 2, 2, 2 }); - private readonly Embedding _embedding3 = new(new float[] { 3, 3, 3 }); - private readonly Mock> _mockLogger = new(); + private readonly ReadOnlyMemory _embedding = new float[] { 1, 1, 1 }; + private readonly ReadOnlyMemory _embedding2 = new float[] { 2, 2, 2 }; + private readonly ReadOnlyMemory _embedding3 = new float[] { 3, 3, 3 }; + private readonly Mock _mockLoggerFactory = new(); + + public QdrantMemoryStoreTests() + { + this._mockLoggerFactory + .Setup(f => f.CreateLogger(It.IsAny())) + .Returns(new Mock().Object); + } [Fact] public async Task ItCreatesNewCollectionAsync() @@ -47,7 +51,7 @@ public async Task ItCreatesNewCollectionAsync() mockQdrantClient .Setup(x => x.CreateCollectionAsync(It.IsAny(), It.IsAny())); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act await vectorStore.CreateCollectionAsync("test"); @@ -70,7 +74,7 @@ public async Task ItWillNotOverwriteExistingCollectionAsync() mockQdrantClient .Setup(x => x.CreateCollectionAsync(It.IsAny(), It.IsAny())); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act await vectorStore.CreateCollectionAsync("test"); @@ -91,7 +95,7 @@ public async Task ItListsCollectionsAsync() .Setup>(x => x.ListCollectionsAsync(It.IsAny())) .Returns((new string[] { "test1", "test2" }).ToAsyncEnumerable()); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act var collections = await vectorStore.GetCollectionsAsync().ToListAsync(); @@ -114,7 +118,7 @@ public async Task ItDeletesCollectionAsync() .Setup(x => x.DoesCollectionExistAsync(It.IsAny(), It.IsAny())) .Returns(Task.FromResult(true)); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act await vectorStore.DeleteCollectionAsync("test"); @@ -146,12 +150,12 @@ public async Task ItThrowsIfUpsertRequestFailsAsync() .Returns(AsyncEnumerable.Empty()); mockQdrantClient .Setup(x => x.UpsertVectorsAsync(It.IsAny(), It.IsAny>(), It.IsAny())) - .Throws(); + .Throws(); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Assert - await Assert.ThrowsAsync(() => vectorStore.UpsertAsync("test_collection", memoryRecord)); + await Assert.ThrowsAsync(() => vectorStore.UpsertAsync("test_collection", memoryRecord)); } [Fact] @@ -178,7 +182,7 @@ public async Task InsertIntoNonExistentCollectionDoesNotCallCreateCollectionAsyn mockQdrantClient .Setup(x => x.UpsertVectorsAsync(It.IsAny(), It.IsAny>(), It.IsAny())); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act string guidString = await vectorStore.UpsertAsync("test_collection", memoryRecord); @@ -203,7 +207,7 @@ public async Task ItUpdatesExistingDataEntryBasedOnMetadataIdAsync() var qdrantVectorRecord = QdrantVectorRecord.FromJsonMetadata( key, - memoryRecord.Embedding.Vector, + memoryRecord.Embedding, memoryRecord.GetSerializedMetadata()); var mockQdrantClient = new Mock(); @@ -220,7 +224,7 @@ public async Task ItUpdatesExistingDataEntryBasedOnMetadataIdAsync() mockQdrantClient .Setup(x => x.UpsertVectorsAsync(It.IsAny(), It.IsAny>(), It.IsAny())); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act string guidString = await vectorStore.UpsertAsync("test_collection", memoryRecord); @@ -272,7 +276,7 @@ public async Task ItGeneratesIdsForQdrantUntilUniqueIdIsFoundAsync() mockQdrantClient .Setup(x => x.UpsertVectorsAsync(It.IsAny(), It.IsAny>(), It.IsAny())); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act string guidString = await vectorStore.UpsertAsync("test_collection", memoryRecord); @@ -323,7 +327,7 @@ public async Task ItUpdatesExistingDataEntryBasedOnKnownDatabaseKeyAsync() mockQdrantClient .Setup(x => x.UpsertVectorsAsync(It.IsAny(), It.IsAny>(), It.IsAny())); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act string guidString = await vectorStore.UpsertAsync("test_collection", memoryRecord); @@ -375,7 +379,7 @@ public async Task ItCanBatchUpsertAsync() mockQdrantClient .Setup(x => x.UpsertVectorsAsync(It.IsAny(), It.IsAny>(), It.IsAny())); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act var keys = await vectorStore.UpsertBatchAsync("test_collection", new[] { memoryRecord, memoryRecord2, memoryRecord3 }).ToListAsync(); diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests2.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests2.cs index 2b173851d5be..abd56fbcee3d 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests2.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests2.cs @@ -6,8 +6,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.Connectors.Memory.Pinecone; using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; using Microsoft.SemanticKernel.Memory; using Moq; @@ -29,10 +27,10 @@ public class QdrantMemoryStoreTests2 private readonly string _description = "description"; private readonly string _description2 = "description2"; private readonly string _description3 = "description3"; - private readonly Embedding _embedding = new(new float[] { 1, 1, 1 }); - private readonly Embedding _embedding2 = new(new float[] { 2, 2, 2 }); - private readonly Embedding _embedding3 = new(new float[] { 3, 3, 3 }); - private readonly Mock> _mockLogger = new(); + private readonly ReadOnlyMemory _embedding = new float[] { 1, 1, 1 }; + private readonly ReadOnlyMemory _embedding2 = new float[] { 2, 2, 2 }; + private readonly ReadOnlyMemory _embedding3 = new float[] { 3, 3, 3 }; + private readonly Mock _mockLogger = new(); [Fact] public async Task GetAsyncCallsDoNotRequestVectorsUnlessSpecifiedAsync() @@ -53,7 +51,7 @@ public async Task GetAsyncCallsDoNotRequestVectorsUnlessSpecifiedAsync() // this information will not be verified var qdrantVectorRecord = QdrantVectorRecord.FromJsonMetadata( guidString, - memoryRecord.Embedding.Vector, + memoryRecord.Embedding, memoryRecord.GetSerializedMetadata()); mockQdrantClient @@ -132,7 +130,7 @@ public async Task GetAsyncSearchesByMetadataIdReturnsMemoryRecordIfFoundAsync() var qdrantVectorRecord = QdrantVectorRecord.FromJsonMetadata( memoryRecord.Key, - memoryRecord.Embedding.Vector, + memoryRecord.Embedding, memoryRecord.GetSerializedMetadata()); var mockQdrantClient = new Mock(); @@ -154,7 +152,7 @@ public async Task GetAsyncSearchesByMetadataIdReturnsMemoryRecordIfFoundAsync() Assert.Equal(memoryRecord.Metadata.Description, getResult.Metadata.Description); Assert.Equal(memoryRecord.Metadata.ExternalSourceName, getResult.Metadata.ExternalSourceName); Assert.Equal(memoryRecord.Metadata.IsReference, getResult.Metadata.IsReference); - Assert.Equal(memoryRecord.Embedding.Vector, getResult.Embedding.Vector); + Assert.True(memoryRecord.Embedding.Span.SequenceEqual(getResult.Embedding.Span)); } [Fact] @@ -183,15 +181,15 @@ public async Task GetBatchAsyncSearchesByMetadataIdReturnsAllResultsIfAllFoundAs var qdrantVectorRecord = QdrantVectorRecord.FromJsonMetadata( key, - memoryRecord.Embedding.Vector, + memoryRecord.Embedding, memoryRecord.GetSerializedMetadata()); var qdrantVectorRecord2 = QdrantVectorRecord.FromJsonMetadata( key2, - memoryRecord2.Embedding.Vector, + memoryRecord2.Embedding, memoryRecord2.GetSerializedMetadata()); var qdrantVectorRecord3 = QdrantVectorRecord.FromJsonMetadata( key3, - memoryRecord3.Embedding.Vector, + memoryRecord3.Embedding, memoryRecord3.GetSerializedMetadata()); var mockQdrantClient = new Mock(); @@ -252,11 +250,11 @@ public async Task GetBatchAsyncSearchesByMetadataIdReturnsOnlyNonNullResultsAsyn var qdrantVectorRecord = QdrantVectorRecord.FromJsonMetadata( key, - memoryRecord.Embedding.Vector, + memoryRecord.Embedding, memoryRecord.GetSerializedMetadata()); var qdrantVectorRecord2 = QdrantVectorRecord.FromJsonMetadata( key2, - memoryRecord2.Embedding.Vector, + memoryRecord2.Embedding, memoryRecord2.GetSerializedMetadata()); var mockQdrantClient = new Mock(); @@ -366,7 +364,7 @@ public async Task GetByQdrantPointIdReturnsMemoryRecordIfFoundAsync() var qdrantVectorRecord = QdrantVectorRecord.FromJsonMetadata( memoryRecord.Key, - memoryRecord.Embedding.Vector, + memoryRecord.Embedding, memoryRecord.GetSerializedMetadata()); var mockQdrantClient = new Mock(); @@ -391,7 +389,7 @@ public async Task GetByQdrantPointIdReturnsMemoryRecordIfFoundAsync() Assert.Equal(memoryRecord.Metadata.Description, getResult.Metadata.Description); Assert.Equal(memoryRecord.Metadata.ExternalSourceName, getResult.Metadata.ExternalSourceName); Assert.Equal(memoryRecord.Metadata.IsReference, getResult.Metadata.IsReference); - Assert.Equal(memoryRecord.Embedding.Vector, getResult.Embedding.Vector); + Assert.True(memoryRecord.Embedding.Span.SequenceEqual(getResult.Embedding.Span)); } [Fact] @@ -420,15 +418,15 @@ public async Task GetBatchByQdrantPointIdsReturnsAllResultsIfFoundAsync() var qdrantVectorRecord = QdrantVectorRecord.FromJsonMetadata( key, - memoryRecord.Embedding.Vector, + memoryRecord.Embedding, memoryRecord.GetSerializedMetadata()); var qdrantVectorRecord2 = QdrantVectorRecord.FromJsonMetadata( key2, - memoryRecord2.Embedding.Vector, + memoryRecord2.Embedding, memoryRecord2.GetSerializedMetadata()); var qdrantVectorRecord3 = QdrantVectorRecord.FromJsonMetadata( key3, - memoryRecord3.Embedding.Vector, + memoryRecord3.Embedding, memoryRecord3.GetSerializedMetadata()); var mockQdrantClient = new Mock(); diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests3.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests3.cs index 39a4982d75ff..7c9cd1adf2c9 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests3.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantMemoryStoreTests3.cs @@ -9,8 +9,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.Connectors.Memory.Pinecone; using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; using Microsoft.SemanticKernel.Memory; using Moq; @@ -27,8 +25,8 @@ public class QdrantMemoryStoreTests3 private readonly string _id = "Id"; private readonly string _text = "text"; private readonly string _description = "description"; - private readonly Embedding _embedding = new(new float[] { 1, 1, 1 }); - private readonly Mock> _mockLogger = new(); + private readonly ReadOnlyMemory _embedding = new float[] { 1, 1, 1 }; + private readonly Mock _mockLoggerFactory = new(); [Fact] public async Task GetNearestMatchesAsyncCallsDoNotReturnVectorsUnlessSpecifiedAsync() @@ -38,7 +36,7 @@ public async Task GetNearestMatchesAsyncCallsDoNotReturnVectorsUnlessSpecifiedAs mockQdrantClient .Setup>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -46,7 +44,7 @@ public async Task GetNearestMatchesAsyncCallsDoNotReturnVectorsUnlessSpecifiedAs It.IsAny())) .Returns(AsyncEnumerable.Empty<(QdrantVectorRecord, double)>()); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act _ = await vectorStore.GetNearestMatchAsync( @@ -73,7 +71,7 @@ public async Task GetNearestMatchesAsyncCallsDoNotReturnVectorsUnlessSpecifiedAs // Assert mockQdrantClient.Verify>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), 1, false, @@ -82,7 +80,7 @@ public async Task GetNearestMatchesAsyncCallsDoNotReturnVectorsUnlessSpecifiedAs Times.Once()); mockQdrantClient.Verify>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), 1, true, @@ -91,7 +89,7 @@ public async Task GetNearestMatchesAsyncCallsDoNotReturnVectorsUnlessSpecifiedAs Times.Once()); mockQdrantClient.Verify>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), 3, false, @@ -100,7 +98,7 @@ public async Task GetNearestMatchesAsyncCallsDoNotReturnVectorsUnlessSpecifiedAs Times.Once()); mockQdrantClient.Verify>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), 3, true, @@ -117,7 +115,7 @@ public async Task ItReturnsEmptyTupleIfNearestMatchNotFoundAsync() mockQdrantClient .Setup>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -125,7 +123,7 @@ public async Task ItReturnsEmptyTupleIfNearestMatchNotFoundAsync() It.IsAny())) .Returns(AsyncEnumerable.Empty<(QdrantVectorRecord, double)>()); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act var similarityResult = await vectorStore.GetNearestMatchAsync( @@ -136,7 +134,7 @@ public async Task ItReturnsEmptyTupleIfNearestMatchNotFoundAsync() // Assert mockQdrantClient.Verify>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -162,14 +160,14 @@ public async Task ItWillReturnTheNearestMatchAsATupleAsync() var qdrantVectorRecord = QdrantVectorRecord.FromJsonMetadata( memoryRecord.Key, - memoryRecord.Embedding.Vector, + memoryRecord.Embedding, memoryRecord.GetSerializedMetadata()); var mockQdrantClient = new Mock(); mockQdrantClient .Setup>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -177,7 +175,7 @@ public async Task ItWillReturnTheNearestMatchAsATupleAsync() It.IsAny())) .Returns(new[] { (qdrantVectorRecord, 0.5) }.ToAsyncEnumerable()); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act var similarityResult = await vectorStore.GetNearestMatchAsync( @@ -188,7 +186,7 @@ public async Task ItWillReturnTheNearestMatchAsATupleAsync() // Assert mockQdrantClient.Verify>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -199,7 +197,7 @@ public async Task ItWillReturnTheNearestMatchAsATupleAsync() Assert.Equal(this._id, similarityResult.Value.Item1.Metadata.Id); Assert.Equal(this._text, similarityResult.Value.Item1.Metadata.Text); Assert.Equal(this._description, similarityResult.Value.Item1.Metadata.Description); - Assert.Equal(this._embedding.Vector, similarityResult.Value.Item1.Embedding.Vector); + Assert.True(this._embedding.Span.SequenceEqual(similarityResult.Value.Item1.Embedding.Span)); Assert.Equal(0.5, similarityResult.Value.Item2); } @@ -211,7 +209,7 @@ public async Task ItReturnsEmptyListIfNearestMatchesNotFoundAsync() mockQdrantClient .Setup>(x => x.FindNearestInCollectionAsync( It.IsAny(), - It.IsAny>(), + It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -219,7 +217,7 @@ public async Task ItReturnsEmptyListIfNearestMatchesNotFoundAsync() It.IsAny())) .Returns(AsyncEnumerable.Empty<(QdrantVectorRecord, double)>()); - var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLogger.Object); + var vectorStore = new QdrantMemoryStore(mockQdrantClient.Object, this._mockLoggerFactory.Object); // Act var similarityResults = await vectorStore.GetNearestMatchesAsync( @@ -233,7 +231,7 @@ public async Task ItReturnsEmptyListIfNearestMatchesNotFoundAsync() } [Fact] - public async Task ScoredVectorSupportsIntegerIds() + public async Task ScoredVectorSupportsIntegerIdsAsync() { // Arrange var payloadId = "payloadId"; @@ -266,7 +264,7 @@ public async Task ScoredVectorSupportsIntegerIds() var result = await client.GetVectorByPayloadIdAsync(payloadId, metadataId); //Assert - Assert.Equal(result!.PointId, expectedId.ToString(CultureInfo.InvariantCulture)); + Assert.Equal(result!.PointId, expectedId.ToString(CultureInfo.InvariantCulture)); } } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantVectorDbClientTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantVectorDbClientTests.cs index 09e29e49f369..0a689e18a307 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantVectorDbClientTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Qdrant/QdrantVectorDbClientTests.cs @@ -10,49 +10,49 @@ namespace SemanticKernel.Connectors.UnitTests.Memory.Qdrant; public sealed class QdrantVectorDbClientTests : IDisposable { - private HttpMessageHandlerStub messageHandlerStub; - private HttpClient httpClient; + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; public QdrantVectorDbClientTests() { - this.messageHandlerStub = new HttpMessageHandlerStub(); + this._messageHandlerStub = new HttpMessageHandlerStub(); - this.httpClient = new HttpClient(this.messageHandlerStub, false); + this._httpClient = new HttpClient(this._messageHandlerStub, false); } [Fact] - public async Task BaseAddressOfHttpClientShouldBeUsedIfNotOverrideProvided() + public async Task BaseAddressOfHttpClientShouldBeUsedIfNotOverrideProvidedAsync() { //Arrange - this.httpClient.BaseAddress = new Uri("https://fake-random-test-host:123/fake-path/"); + this._httpClient.BaseAddress = new Uri("https://fake-random-test-host:123/fake-path/"); - var sut = new QdrantVectorDbClient(this.httpClient, 123); + var sut = new QdrantVectorDbClient(this._httpClient, 123); //Act await sut.DoesCollectionExistAsync("fake-collection"); //Assert - Assert.Equal("https://fake-random-test-host:123/fake-path/collections/fake-collection", this.messageHandlerStub.RequestUri?.AbsoluteUri); + Assert.Equal("https://fake-random-test-host:123/fake-path/collections/fake-collection", this._messageHandlerStub.RequestUri?.AbsoluteUri); } [Fact] - public async Task EndpointOverrideShouldBeUsedIfProvided() + public async Task EndpointOverrideShouldBeUsedIfProvidedAsync() { //Arrange - this.httpClient.BaseAddress = new Uri("https://fake-random-test-host:123/fake-path/"); + this._httpClient.BaseAddress = new Uri("https://fake-random-test-host:123/fake-path/"); - var sut = new QdrantVectorDbClient(this.httpClient, 123, "https://fake-random-test-host-override:123/"); + var sut = new QdrantVectorDbClient(this._httpClient, 123, "https://fake-random-test-host-override:123/"); //Act await sut.DoesCollectionExistAsync("fake-collection"); //Assert - Assert.Equal("https://fake-random-test-host-override:123/collections/fake-collection", this.messageHandlerStub.RequestUri?.AbsoluteUri); + Assert.Equal("https://fake-random-test-host-override:123/collections/fake-collection", this._messageHandlerStub.RequestUri?.AbsoluteUri); } public void Dispose() { - this.httpClient.Dispose(); - this.messageHandlerStub.Dispose(); + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Redis/RedisMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Redis/RedisMemoryStoreTests.cs index 2d2c3dad7cc1..e8f09a5f2f2f 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Redis/RedisMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Redis/RedisMemoryStoreTests.cs @@ -4,13 +4,12 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Numerics.Tensors; using System.Runtime.InteropServices; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; using Microsoft.SemanticKernel.Connectors.Memory.Redis; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Memory.Collections; using Moq; using StackExchange.Redis; using Xunit; @@ -35,14 +34,14 @@ public RedisMemoryStoreTests() public void ConnectionCanBeInitialized() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); } [Fact] public async Task ItCanCreateAndGetCollectionAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); string collection = "test_collection"; this.MockCreateIndex(collection); @@ -59,7 +58,7 @@ public async Task ItCanCreateAndGetCollectionAsync() public async Task ItCanCheckIfCollectionExistsAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); string collection = "my_collection"; this.MockCreateIndex(collection); @@ -75,7 +74,7 @@ public async Task ItCanCheckIfCollectionExistsAsync() public async Task CollectionsCanBeDeletedAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); string collection = "test_collection"; this.MockCreateIndex(collection, () => { @@ -101,17 +100,17 @@ public async Task CollectionsCanBeDeletedAsync() public async Task ItCanInsertIntoNonExistentCollectionAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "random collection"; string redisKey = $"{collection}:{testRecord.Metadata.Id}"; - byte[] embedding = MemoryMarshal.Cast(testRecord.Embedding.AsReadOnlySpan()).ToArray(); + byte[] embedding = MemoryMarshal.Cast(testRecord.Embedding.Span).ToArray(); this._mockDatabase .Setup(x => x.HashSetAsync( It.Is(x => x == redisKey), @@ -141,7 +140,7 @@ public async Task ItCanInsertIntoNonExistentCollectionAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -152,12 +151,12 @@ public async Task ItCanInsertIntoNonExistentCollectionAsync() public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection"; @@ -177,20 +176,20 @@ public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() // Assert Assert.NotNull(actualDefault); Assert.NotNull(actualWithEmbedding); - Assert.Empty(actualDefault.Embedding.Vector); - Assert.NotEmpty(actualWithEmbedding.Embedding.Vector); + Assert.True(actualDefault.Embedding.IsEmpty); + Assert.False(actualWithEmbedding.Embedding.IsEmpty); } [Fact] public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection"; @@ -208,7 +207,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -219,12 +218,12 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: DateTimeOffset.UtcNow); string collection = "test_collection"; @@ -242,7 +241,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -253,18 +252,18 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() public async Task UpsertReplacesExistingRecordWithSameIdAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); string commonId = "test"; MemoryRecord testRecord = MemoryRecord.LocalRecord( id: commonId, text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); MemoryRecord testRecord2 = MemoryRecord.LocalRecord( id: commonId, text: "text2", description: "description2", - embedding: new Embedding(new float[] { 1, 2, 4 })); + embedding: new float[] { 1, 2, 4 }); string collection = "test_collection"; this.MockCreateIndex(collection, () => { @@ -282,8 +281,8 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord2.Metadata.Id, actual.Key); - Assert.NotEqual(testRecord.Embedding.Vector, actual.Embedding.Vector); - Assert.Equal(testRecord2.Embedding.Vector, actual.Embedding.Vector); + Assert.False(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); + Assert.True(testRecord2.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.NotEqual(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord2.Metadata.Description, actual.Metadata.Description); } @@ -292,12 +291,12 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() public async Task ExistingRecordCanBeRemovedAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); string collection = "test_collection"; this.MockCreateIndex(collection, () => { @@ -321,7 +320,7 @@ public async Task ExistingRecordCanBeRemovedAsync() public async Task RemovingNonExistingRecordDoesNothingAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); string collection = "test_collection"; this.MockCreateIndex(collection, () => { @@ -341,7 +340,7 @@ public async Task RemovingNonExistingRecordDoesNothingAsync() public async Task ItCanListAllDatabaseCollectionsAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); string[] testCollections = { "random_collection1", "random_collection2", "random_collection3" }; foreach (var collection in testCollections) { @@ -378,18 +377,18 @@ public async Task ItCanListAllDatabaseCollectionsAsync() public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection"; int topN = 4; double threshold = -1; var testEmbeddings = new[] { - new Embedding(new float[] { 1, 1, 1 }), - new Embedding(new float[] { -1, -1, -1 }), - new Embedding(new float[] { 1, 2, 3 }), - new Embedding(new float[] { -1, -2, -3 }), - new Embedding(new float[] { 1, -1, -2 }) + new float[] { 1, 1, 1 }, + new float[] { -1, -1, -1 }, + new float[] { 1, 2, 3 }, + new float[] { -1, -2, -3 }, + new float[] { 1, -1, -2 } }; var testRecords = new List(); for (int i = 0; i < testEmbeddings.Length; i++) @@ -436,18 +435,18 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection"; int topN = 1; double threshold = 0.75; var testEmbeddings = new[] { - new Embedding(new float[] { 1, 1, 1 }), - new Embedding(new float[] { -1, -1, -1 }), - new Embedding(new float[] { 1, 2, 3 }), - new Embedding(new float[] { -1, -2, -3 }), - new Embedding(new float[] { 1, -1, -2 }) + new float[] { 1, 1, 1 }, + new float[] { -1, -1, -1 }, + new float[] { 1, 2, 3 }, + new float[] { -1, -2, -3 }, + new float[] { 1, -1, -2 } }; var testRecords = new List(); for (int i = 0; i < testEmbeddings.Length; i++) @@ -488,26 +487,26 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( // Assert Assert.NotNull(topNResultDefault); Assert.NotNull(topNResultWithEmbedding); - Assert.Empty(topNResultDefault.Value.Item1.Embedding.Vector); - Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.Vector); + Assert.True(topNResultDefault.Value.Item1.Embedding.IsEmpty); + Assert.False(topNResultWithEmbedding.Value.Item1.Embedding.IsEmpty); } [Fact] public async Task GetNearestMatchAsyncReturnsExpectedAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection"; int topN = 1; double threshold = 0.75; var testEmbeddings = new[] { - new Embedding(new float[] { 1, 1, 1 }), - new Embedding(new float[] { -1, -1, -1 }), - new Embedding(new float[] { 1, 2, 3 }), - new Embedding(new float[] { -1, -2, -3 }), - new Embedding(new float[] { 1, -1, -2 }) + new float[] { 1, 1, 1 }, + new float[] { -1, -1, -1 }, + new float[] { 1, 2, 3 }, + new float[] { -1, -2, -3 }, + new float[] { 1, -1, -2 } }; var testRecords = new List(); for (int i = 0; i < testEmbeddings.Length; i++) @@ -554,8 +553,8 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; double threshold = 0.75; string collection = "test_collection"; @@ -566,7 +565,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 }))); + embedding: new float[] { 1, 1, 1 })); } this.MockCreateIndex(collection, () => { @@ -610,7 +609,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() public async Task ItCanBatchUpsertRecordsAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); int numRecords = 10; string collection = "test_collection"; IEnumerable records = this.CreateBatchRecords(numRecords); @@ -637,7 +636,7 @@ public async Task ItCanBatchUpsertRecordsAsync() public async Task ItCanBatchGetRecordsAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); int numRecords = 10; string collection = "test_collection"; IEnumerable records = this.CreateBatchRecords(numRecords); @@ -664,7 +663,7 @@ public async Task ItCanBatchGetRecordsAsync() public async Task ItCanBatchRemoveRecordsAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); int numRecords = 10; string collection = "test_collection"; IEnumerable records = this.CreateBatchRecords(numRecords); @@ -699,18 +698,18 @@ public async Task ItCanBatchRemoveRecordsAsync() public async Task GetNearestMatchAsyncThrowsExceptionOnInvalidVectorScoreAsync() { // Arrange - RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection"; int topN = 1; double threshold = 0.75; var testEmbeddings = new[] { - new Embedding(new float[] { 1, 1, 1 }), - new Embedding(new float[] { -1, -1, -1 }), - new Embedding(new float[] { 1, 2, 3 }), - new Embedding(new float[] { -1, -2, -3 }), - new Embedding(new float[] { 1, -1, -2 }) + new float[] { 1, 1, 1 }, + new float[] { -1, -1, -1 }, + new float[] { 1, 2, 3 }, + new float[] { -1, -2, -3 }, + new float[] { 1, -1, -2 } }; var testRecords = new List(); for (int i = 0; i < testEmbeddings.Length; i++) @@ -745,12 +744,12 @@ public async Task GetNearestMatchAsyncThrowsExceptionOnInvalidVectorScoreAsync() } // Assert - RedisMemoryStoreException ex = await Assert.ThrowsAsync(async () => + var ex = await Assert.ThrowsAsync(async () => { // Act await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold); }); - Assert.Equal(ex.Message, "Invalid or missing vector score value."); + Assert.Equal("Invalid or missing vector score value.", ex.Message); } #region private @@ -823,7 +822,7 @@ private void MockDropIndex(string collection, Action? callback = null) private void MockHashSet(string collection, MemoryRecord record, Action? callback = null) { string redisKey = $"{collection}:{record.Metadata.Id}"; - byte[] embedding = MemoryMarshal.Cast(record.Embedding.AsReadOnlySpan()).ToArray(); + byte[] embedding = MemoryMarshal.Cast(record.Embedding.Span).ToArray(); long timestamp = record.Timestamp?.ToUnixTimeMilliseconds() ?? -1; this._mockDatabase @@ -899,24 +898,22 @@ private void MockKeyDelete(string collection, IEnumerable keys, Action? }); } - private void MockSearch(string collection, Embedding compareEmbedding, int topN, double threshold, bool returnStringVectorScore = false) + private void MockSearch(string collection, ReadOnlyMemory compareEmbedding, int topN, double threshold, bool returnStringVectorScore = false) { - TopNCollection embeddings = new(topN); + List<(MemoryRecord Record, double Score)> embeddings = new(); List records = this._collections.TryGetValue(collection, out var value) ? value : new(); foreach (var record in records) { - double similarity = compareEmbedding - .AsReadOnlySpan() - .CosineSimilarity(record.Embedding.AsReadOnlySpan()); + double similarity = TensorPrimitives.CosineSimilarity(compareEmbedding.Span, record.Embedding.Span); if (similarity >= threshold) { embeddings.Add(new(record, similarity)); } } - embeddings.SortByScore(); + embeddings = embeddings.OrderByDescending(l => l.Score).Take(topN).ToList(); string redisKey = $"{collection}"; @@ -925,22 +922,22 @@ private void MockSearch(string collection, Embedding compareEmbedding, in foreach (var item in embeddings) { - long timestamp = item.Value.Timestamp?.ToUnixTimeMilliseconds() ?? -1; - byte[] embedding = MemoryMarshal.Cast(item.Value.Embedding.AsReadOnlySpan()).ToArray(); - redisResults.Add(RedisResult.Create($"{collection}:{item.Value.Metadata.Id}", ResultType.BulkString)); + long timestamp = item.Record.Timestamp?.ToUnixTimeMilliseconds() ?? -1; + byte[] embedding = MemoryMarshal.Cast(item.Record.Embedding.Span).ToArray(); + redisResults.Add(RedisResult.Create($"{collection}:{item.Record.Metadata.Id}", ResultType.BulkString)); redisResults.Add(RedisResult.Create( new RedisResult[] { RedisResult.Create("key", ResultType.BulkString), - RedisResult.Create(item.Value.Metadata.Id, ResultType.BulkString), + RedisResult.Create(item.Record.Metadata.Id, ResultType.BulkString), RedisResult.Create("metadata", ResultType.BulkString), - RedisResult.Create(item.Value.GetSerializedMetadata(), ResultType.BulkString), + RedisResult.Create(item.Record.GetSerializedMetadata(), ResultType.BulkString), RedisResult.Create("embedding", ResultType.BulkString), RedisResult.Create(embedding, ResultType.BulkString), RedisResult.Create("timestamp", ResultType.BulkString), RedisResult.Create(timestamp, ResultType.BulkString), RedisResult.Create("vector_score", ResultType.BulkString), - RedisResult.Create(returnStringVectorScore ? $"score:{1-item.Score.Value}" : 1-item.Score.Value, ResultType.BulkString), + RedisResult.Create(returnStringVectorScore ? $"score:{1-item.Score}" : 1-item.Score, ResultType.BulkString), }) ); } @@ -965,7 +962,7 @@ private IEnumerable CreateBatchRecords(int numRecords) id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); records = records.Append(testRecord); } @@ -975,7 +972,7 @@ private IEnumerable CreateBatchRecords(int numRecords) externalId: "test" + i, sourceName: "sourceName" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); records = records.Append(testRecord); } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Sqlite/SqliteMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Sqlite/SqliteMemoryStoreTests.cs index c8bda27876ca..1ceb459a5c18 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Sqlite/SqliteMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Sqlite/SqliteMemoryStoreTests.cs @@ -6,7 +6,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Sqlite; using Microsoft.SemanticKernel.Memory; using Xunit; @@ -66,7 +65,7 @@ private IEnumerable CreateBatchRecords(int numRecords) id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); records = records.Append(testRecord); } @@ -76,7 +75,7 @@ private IEnumerable CreateBatchRecords(int numRecords) externalId: "test" + i, sourceName: "sourceName" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); records = records.Append(testRecord); } @@ -173,7 +172,7 @@ public async Task ItCanInsertIntoNonExistentCollectionAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); @@ -185,7 +184,7 @@ public async Task ItCanInsertIntoNonExistentCollectionAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -201,7 +200,7 @@ public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection" + this._collectionNum; @@ -216,8 +215,8 @@ public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() // Assert Assert.NotNull(actualDefault); Assert.NotNull(actualWithEmbedding); - Assert.Empty(actualDefault.Embedding.Vector); - Assert.NotEmpty(actualWithEmbedding.Embedding.Vector); + Assert.True(actualDefault.Embedding.IsEmpty); + Assert.False(actualWithEmbedding.Embedding.IsEmpty); } [Fact] @@ -229,7 +228,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection" + this._collectionNum; @@ -244,7 +243,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -260,7 +259,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: DateTimeOffset.UtcNow); string collection = "test_collection" + this._collectionNum; @@ -275,7 +274,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -292,12 +291,12 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() id: commonId, text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); MemoryRecord testRecord2 = MemoryRecord.LocalRecord( id: commonId, text: "text2", description: "description2", - embedding: new Embedding(new float[] { 1, 2, 4 })); + embedding: new float[] { 1, 2, 4 }); string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -311,8 +310,8 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord2.Metadata.Id, actual.Key); - Assert.NotEqual(testRecord.Embedding.Vector, actual.Embedding.Vector); - Assert.Equal(testRecord2.Embedding.Vector, actual.Embedding.Vector); + Assert.False(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); + Assert.True(testRecord2.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.NotEqual(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord2.Metadata.Description, actual.Metadata.Description); } @@ -326,7 +325,7 @@ public async Task ExistingRecordCanBeRemovedAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -393,7 +392,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() { // Arrange using SqliteMemoryStore db = await SqliteMemoryStore.ConnectAsync(DatabaseFile); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -403,7 +402,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -411,7 +410,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new ReadOnlyMemory(new float[] { -1, -1, -1 })); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -419,7 +418,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -427,7 +426,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new ReadOnlyMemory(new float[] { -1, -2, -3 })); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -435,7 +434,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new ReadOnlyMemory(new float[] { 1, -1, -2 })); _ = await db.UpsertAsync(collection, testRecord); // Act @@ -456,7 +455,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( { // Arrange using SqliteMemoryStore db = await SqliteMemoryStore.ConnectAsync(DatabaseFile); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection" + this._collectionNum; this._collectionNum++; await db.CreateCollectionAsync(collection); @@ -465,7 +464,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -473,7 +472,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new ReadOnlyMemory(new float[] { -1, -1, -1 })); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -481,7 +480,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -489,7 +488,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new ReadOnlyMemory(new float[] { -1, -2, -3 })); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -497,7 +496,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new ReadOnlyMemory(new float[] { 1, -1, -2 })); _ = await db.UpsertAsync(collection, testRecord); // Act @@ -508,8 +507,8 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( // Assert Assert.NotNull(topNResultDefault); Assert.NotNull(topNResultWithEmbedding); - Assert.Empty(topNResultDefault.Value.Item1.Embedding.Vector); - Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.Vector); + Assert.True(topNResultDefault.Value.Item1.Embedding.IsEmpty); + Assert.False(topNResultWithEmbedding.Value.Item1.Embedding.IsEmpty); } [Fact] @@ -517,7 +516,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() { // Arrange using SqliteMemoryStore db = await SqliteMemoryStore.ConnectAsync(DatabaseFile); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection" + this._collectionNum; this._collectionNum++; await db.CreateCollectionAsync(collection); @@ -526,7 +525,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -534,7 +533,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new ReadOnlyMemory(new float[] { -1, -1, -1 })); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -542,7 +541,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -550,7 +549,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new ReadOnlyMemory(new float[] { -1, -2, -3 })); _ = await db.UpsertAsync(collection, testRecord); i++; @@ -558,7 +557,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new ReadOnlyMemory(new float[] { 1, -1, -2 })); _ = await db.UpsertAsync(collection, testRecord); // Act @@ -576,7 +575,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() { // Arrange using SqliteMemoryStore db = await SqliteMemoryStore.ConnectAsync(DatabaseFile); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -588,7 +587,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await db.UpsertAsync(collection, testRecord); } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateKernelBuilderExtensionsTests.cs index 6062311c468b..0d771ef95194 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateKernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateKernelBuilderExtensionsTests.cs @@ -15,18 +15,20 @@ namespace SemanticKernel.Connectors.UnitTests.Memory.Weaviate; public sealed class WeaviateKernelBuilderExtensionsTests : IDisposable { - private HttpMessageHandlerStub messageHandlerStub; - private HttpClient httpClient; + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; public WeaviateKernelBuilderExtensionsTests() { - this.messageHandlerStub = new HttpMessageHandlerStub(); + this._messageHandlerStub = new HttpMessageHandlerStub(); - this.httpClient = new HttpClient(this.messageHandlerStub, false); + this._httpClient = new HttpClient(this._messageHandlerStub, false); } - [Fact] - public async Task WeaviateMemoryStoreShouldBeProperlyInitialized() + [Theory] + [InlineData(null, "https://fake-random-test-weaviate-host/v1/objects/fake-key")] + [InlineData("v2", "https://fake-random-test-weaviate-host/v2/objects/fake-key")] + public async Task WeaviateMemoryStoreShouldBeProperlyInitializedAsync(string? apiVersion, string expectedAddress) { //Arrange var getResponse = new @@ -39,28 +41,32 @@ public async Task WeaviateMemoryStoreShouldBeProperlyInitialized() } }; - this.messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }), Encoding.UTF8, MediaTypeNames.Application.Json); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }), Encoding.UTF8, MediaTypeNames.Application.Json); var builder = new KernelBuilder(); - builder.WithWeaviateMemoryStore(this.httpClient, "https://fake-random-test-weaviate-host", "fake-api-key"); +#pragma warning disable CS0618 // This will be removed in a future release. + builder.WithWeaviateMemoryStore(this._httpClient, "https://fake-random-test-weaviate-host", "fake-api-key", apiVersion); +#pragma warning restore CS0618 // This will be removed in a future release. builder.WithAzureTextEmbeddingGenerationService("fake-deployment-name", "https://fake-random-test-host/fake-path", "fake -api-key"); var kernel = builder.Build(); //This call triggers the internal factory registered by WithWeaviateMemoryStore method to create an instance of the WeaviateMemoryStore class. //Act +#pragma warning disable CS0618 // This will be removed in a future release. await kernel.Memory.GetAsync("fake-collection", "fake-key"); //This call triggers a subsequent call to Weaviate memory store. +#pragma warning restore CS0618 // This will be removed in a future release. //Assert - Assert.Equal("https://fake-random-test-weaviate-host/objects/fake-key", this.messageHandlerStub?.RequestUri?.AbsoluteUri); + Assert.Equal(expectedAddress, this._messageHandlerStub?.RequestUri?.AbsoluteUri); var headerValues = Enumerable.Empty(); - var headerExists = this.messageHandlerStub?.RequestHeaders?.TryGetValues("Authorization", out headerValues); + var headerExists = this._messageHandlerStub?.RequestHeaders?.TryGetValues("Authorization", out headerValues); Assert.True(headerExists); Assert.Contains(headerValues!, (value) => value == "fake-api-key"); } public void Dispose() { - this.httpClient.Dispose(); - this.messageHandlerStub.Dispose(); + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryBuilderExtensionsTests.cs new file mode 100644 index 000000000000..d9e90cbeba42 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryBuilderExtensionsTests.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Mime; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Connectors.Memory.Weaviate; +using Microsoft.SemanticKernel.Plugins.Memory; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.Memory.Weaviate; + +public sealed class WeaviateMemoryBuilderExtensionsTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; + + public WeaviateMemoryBuilderExtensionsTests() + { + this._messageHandlerStub = new HttpMessageHandlerStub(); + + this._httpClient = new HttpClient(this._messageHandlerStub, false); + } + + [Theory] + [InlineData(null, "https://fake-random-test-weaviate-host/v1/objects/fake-key")] + [InlineData("v2", "https://fake-random-test-weaviate-host/v2/objects/fake-key")] + public async Task WeaviateMemoryStoreShouldBeProperlyInitializedAsync(string? apiVersion, string expectedAddress) + { + // Arrange + var embeddingGenerationMock = Mock.Of(); + + var getResponse = new + { + Properties = new Dictionary { + { "sk_id", "fake_id" }, + { "sk_description", "fake_description" }, + { "sk_text", "fake_text" }, + { "sk_additional_metadata", "fake_additional_metadata" } + } + }; + + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }), Encoding.UTF8, MediaTypeNames.Application.Json); + + var builder = new MemoryBuilder(); + builder.WithWeaviateMemoryStore(this._httpClient, "https://fake-random-test-weaviate-host", "fake-api-key", apiVersion); + builder.WithTextEmbeddingGeneration(embeddingGenerationMock); + + var memory = builder.Build(); //This call triggers the internal factory registered by WithWeaviateMemoryStore method to create an instance of the WeaviateMemoryStore class. + + // Act + await memory.GetAsync("fake-collection", "fake-key"); //This call triggers a subsequent call to Weaviate memory store. + + // Assert + Assert.Equal(expectedAddress, this._messageHandlerStub?.RequestUri?.AbsoluteUri); + + var headerValues = Enumerable.Empty(); + var headerExists = this._messageHandlerStub?.RequestHeaders?.TryGetValues("Authorization", out headerValues); + Assert.True(headerExists); + Assert.Contains(headerValues!, (value) => value == "fake-api-key"); + } + + public void Dispose() + { + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryStoreTests.cs index bf64563e5276..fa47bff5ee10 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Weaviate/WeaviateMemoryStoreTests.cs @@ -18,12 +18,12 @@ namespace SemanticKernel.Connectors.UnitTests.Memory.Weaviate; /// public sealed class WeaviateMemoryStoreTests : IDisposable { - private HttpMessageHandlerStub messageHandlerStub; - private HttpClient httpClient; + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; public WeaviateMemoryStoreTests() { - this.messageHandlerStub = new HttpMessageHandlerStub(); + this._messageHandlerStub = new HttpMessageHandlerStub(); var getResponse = new { @@ -35,37 +35,37 @@ public WeaviateMemoryStoreTests() } }; - this.messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }), Encoding.UTF8, MediaTypeNames.Application.Json); + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(JsonSerializer.Serialize(getResponse, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }), Encoding.UTF8, MediaTypeNames.Application.Json); - this.httpClient = new HttpClient(this.messageHandlerStub, false); + this._httpClient = new HttpClient(this._messageHandlerStub, false); } [Fact] public async Task NoAuthorizationHeaderShouldBeAddedIfApiKeyIsNotProvidedAsync() { //Arrange - var sut = new WeaviateMemoryStore(this.httpClient, null, "https://fake-random-test-host/fake-path"); + var sut = new WeaviateMemoryStore(this._httpClient, null, "https://fake-random-test-host/fake-path"); //Act await sut.GetAsync("fake-collection", "fake-key"); //Assert - Assert.False(this.messageHandlerStub.RequestHeaders?.Contains("Authorization")); + Assert.False(this._messageHandlerStub.RequestHeaders?.Contains("Authorization")); } [Fact] public async Task AuthorizationHeaderShouldBeAddedIfApiKeyIsProvidedAsync() { //Arrange - var sut = new WeaviateMemoryStore(this.httpClient, "fake-api-key", "https://fake-random-test-host/fake-path"); + var sut = new WeaviateMemoryStore(this._httpClient, "fake-api-key", "https://fake-random-test-host/fake-path"); //Act await sut.GetAsync("fake-collection", "fake-key"); //Assert - Assert.True(this.messageHandlerStub.RequestHeaders?.Contains("Authorization")); + Assert.True(this._messageHandlerStub.RequestHeaders?.Contains("Authorization")); - var values = this.messageHandlerStub.RequestHeaders!.GetValues("Authorization"); + var values = this._messageHandlerStub.RequestHeaders!.GetValues("Authorization"); var value = values.SingleOrDefault(); Assert.Equal("fake-api-key", value); @@ -75,33 +75,33 @@ public async Task AuthorizationHeaderShouldBeAddedIfApiKeyIsProvidedAsync() public async Task ProvidedEndpointShouldBeUsedAsync() { //Arrange - var sut = new WeaviateMemoryStore(this.httpClient, "fake-api-key", "https://fake-random-test-host/fake-path/"); + var sut = new WeaviateMemoryStore(this._httpClient, "fake-api-key", "https://fake-random-test-host/fake-path/"); //Act await sut.GetAsync("fake-collection", "fake-key"); //Assert - Assert.StartsWith("https://fake-random-test-host/fake-path", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith("https://fake-random-test-host/fake-path", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } [Fact] public async Task HttpClientBaseAddressShouldBeUsedAsync() { //Arrange - this.httpClient.BaseAddress = new Uri("https://fake-random-test-host/fake-path/"); + this._httpClient.BaseAddress = new Uri("https://fake-random-test-host/fake-path/"); - var sut = new WeaviateMemoryStore(this.httpClient, "fake-api-key"); + var sut = new WeaviateMemoryStore(this._httpClient, "fake-api-key"); //Act await sut.GetAsync("fake-collection", "fake-key"); //Assert - Assert.StartsWith("https://fake-random-test-host/fake-path", this.messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); + Assert.StartsWith("https://fake-random-test-host/fake-path", this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); } public void Dispose() { - this.httpClient.Dispose(); - this.messageHandlerStub.Dispose(); + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Oobabooga/OobaboogaWebSocketTestServer.cs b/dotnet/src/Connectors/Connectors.UnitTests/Oobabooga/OobaboogaWebSocketTestServer.cs index d9210603a8fd..7a4608d4569b 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Oobabooga/OobaboogaWebSocketTestServer.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Oobabooga/OobaboogaWebSocketTestServer.cs @@ -15,6 +15,7 @@ namespace SemanticKernel.Connectors.UnitTests.Oobabooga; /// The server accepts WebSocket connections, receives requests, and generates responses based on the Oobabooga text completion logic. /// The OobaboogaWebSocketTestServer class uses a delegate to handle the request and response logic, allowing customization of the behavior. /// +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] internal sealed class OobaboogaWebSocketTestServer : WebSocketTestServer { public OobaboogaWebSocketTestServer(string url, Func> stringHandler, ILogger? logger = null) diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Oobabooga/TextCompletion/OobaboogaTextCompletionTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Oobabooga/TextCompletion/OobaboogaTextCompletionTests.cs index 65810789802d..e9503a5915dd 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Oobabooga/TextCompletion/OobaboogaTextCompletionTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/Oobabooga/TextCompletion/OobaboogaTextCompletionTests.cs @@ -22,6 +22,7 @@ namespace SemanticKernel.Connectors.UnitTests.Oobabooga.TextCompletion; /// /// Unit tests for class. /// +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] public sealed class OobaboogaTextCompletionTests : IDisposable { private readonly XunitLogger _logger; @@ -31,10 +32,10 @@ public sealed class OobaboogaTextCompletionTests : IDisposable private const string CompletionText = "fake-test"; private const string CompletionMultiText = "Hello, my name is"; - private HttpMessageHandlerStub _messageHandlerStub; - private HttpClient _httpClient; - private Uri _endPointUri; - private string _streamCompletionResponseStub; + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; + private readonly Uri _endPointUri; + private readonly string _streamCompletionResponseStub; public OobaboogaTextCompletionTests(ITestOutputHelper output) { @@ -57,7 +58,7 @@ public async Task UserAgentHeaderShouldBeUsedAsync() logger: this._logger); //Act - await sut.GetCompletionsAsync(CompletionText, new CompleteRequestSettings()); + await sut.GetCompletionsAsync(CompletionText, null); //Assert Assert.True(this._messageHandlerStub.RequestHeaders?.Contains("User-Agent")); @@ -65,7 +66,7 @@ public async Task UserAgentHeaderShouldBeUsedAsync() var values = this._messageHandlerStub.RequestHeaders!.GetValues("User-Agent"); var value = values.SingleOrDefault(); - Assert.Equal(OobaboogaTextCompletion.HttpUserAgent, value); + Assert.Equal("Semantic-Kernel", value); } [Fact] @@ -78,7 +79,7 @@ public async Task ProvidedEndpointShouldBeUsedAsync() logger: this._logger); //Act - await sut.GetCompletionsAsync(CompletionText, new CompleteRequestSettings()); + await sut.GetCompletionsAsync(CompletionText, null); //Assert Assert.StartsWith(EndPoint, this._messageHandlerStub.RequestUri?.AbsoluteUri, StringComparison.OrdinalIgnoreCase); @@ -94,7 +95,7 @@ public async Task BlockingUrlShouldBeBuiltSuccessfullyAsync() logger: this._logger); //Act - await sut.GetCompletionsAsync(CompletionText, new CompleteRequestSettings()); + await sut.GetCompletionsAsync(CompletionText); var expectedUri = new UriBuilder(this._endPointUri) { Path = OobaboogaTextCompletion.BlockingUriPath, @@ -115,7 +116,7 @@ public async Task ShouldSendPromptToServiceAsync() logger: this._logger); //Act - await sut.GetCompletionsAsync(CompletionText, new CompleteRequestSettings()); + await sut.GetCompletionsAsync(CompletionText, null); //Assert var requestPayload = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); @@ -134,7 +135,7 @@ public async Task ShouldHandleServiceResponseAsync() logger: this._logger); //Act - var result = await sut.GetCompletionsAsync(CompletionText, new CompleteRequestSettings()); + var result = await sut.GetCompletionsAsync(CompletionText, null); //Assert Assert.NotNull(result); @@ -154,7 +155,7 @@ public async Task ShouldHandleStreamingServicePersistentWebSocketResponseAsync() await this.RunWebSocketMultiPacketStreamingTestAsync( requestMessage: requestMessage, expectedResponse: expectedResponse, - isPersistent: true).ConfigureAwait(false); + isPersistent: true); } [Fact] @@ -164,7 +165,7 @@ public async Task ShouldHandleStreamingServiceTransientWebSocketResponseAsync() var expectedResponse = new List { this._streamCompletionResponseStub }; await this.RunWebSocketMultiPacketStreamingTestAsync( requestMessage: requestMessage, - expectedResponse: expectedResponse).ConfigureAwait(false); + expectedResponse: expectedResponse); } [Fact] @@ -186,7 +187,7 @@ public async Task ShouldHandleConcurrentWebSocketConnectionsAsync() // Simulate different responses for each request var responseIndex = int.Parse(Encoding.UTF8.GetString(request.ToArray()), CultureInfo.InvariantCulture); byte[] bytes = Encoding.UTF8.GetBytes(expectedResponses[responseIndex]); - var toReturn = new List> { new ArraySegment(bytes) }; + var toReturn = new List> { new(bytes) }; return toReturn; }); @@ -208,7 +209,7 @@ public async Task ShouldHandleConcurrentWebSocketConnectionsAsync() // Receive the response from the server var responseBytes = new byte[1024]; var responseResult = await client.ReceiveAsync(new ArraySegment(responseBytes), CancellationToken.None); - await client.CloseAsync(WebSocketCloseStatus.NormalClosure, "Close connection after message received", CancellationToken.None).ConfigureAwait(false); + await client.CloseAsync(WebSocketCloseStatus.NormalClosure, "Close connection after message received", CancellationToken.None); var response = Encoding.UTF8.GetString(responseBytes, 0, responseResult.Count); @@ -219,7 +220,7 @@ public async Task ShouldHandleConcurrentWebSocketConnectionsAsync() // Assert for (int i = 0; i < expectedResponses.Count; i++) { - var response = await tasks[i].ConfigureAwait(false); + var response = await tasks[i]; Assert.Equal(expectedResponses[i], response); } } @@ -227,25 +228,25 @@ public async Task ShouldHandleConcurrentWebSocketConnectionsAsync() [Fact] public async Task ShouldHandleMultiPacketStreamingServiceTransientWebSocketResponseAsync() { - await this.RunWebSocketMultiPacketStreamingTestAsync().ConfigureAwait(false); + await this.RunWebSocketMultiPacketStreamingTestAsync(); } [Fact] public async Task ShouldHandleMultiPacketStreamingServicePersistentWebSocketResponseBroadcastBlockAsync() { - await this.RunWebSocketMultiPacketStreamingTestAsync(isPersistent: true).ConfigureAwait(false); + await this.RunWebSocketMultiPacketStreamingTestAsync(isPersistent: true); } [Fact] public async Task ShouldHandleConcurrentMultiPacketStreamingServiceTransientWebSocketResponseAsync() { - await this.RunWebSocketMultiPacketStreamingTestAsync(nbConcurrentCalls: 10).ConfigureAwait(false); + await this.RunWebSocketMultiPacketStreamingTestAsync(nbConcurrentCalls: 10); } [Fact] public async Task ShouldHandleConcurrentMultiPacketStreamingServicePersistentWebSocketResponseAsync() { - await this.RunWebSocketMultiPacketStreamingTestAsync(nbConcurrentCalls: 10, isPersistent: true).ConfigureAwait(false); + await this.RunWebSocketMultiPacketStreamingTestAsync(nbConcurrentCalls: 10, isPersistent: true); } /// @@ -264,7 +265,7 @@ await this.RunWebSocketMultiPacketStreamingTestAsync( keepAliveWebSocketsDuration: 100, concurrentCallsTicksDelay: 0, enforcedConcurrentCallSemaphore: enforcedConcurrentCallSemaphore, - maxExpectedNbClients: 20).ConfigureAwait(false); + maxExpectedNbClients: 20); } private async Task RunWebSocketMultiPacketStreamingTestAsync( @@ -339,26 +340,21 @@ ClientWebSocket IncrementFactory() for (int i = 0; i < nbConcurrentCalls; i++) { - tasks.Add(Task.Run(() => + tasks.Add(Task.FromResult(sut.CompleteStreamAsync(requestMessage, new TextCompletionRequest() { - var localResponse = sut.CompleteStreamAsync(requestMessage, new CompleteRequestSettings() - { - Temperature = 0.01, - MaxTokens = 7, - TopP = 0.1, - }, cancellationToken: cleanupToken.Token); - return localResponse; - })); + Temperature = 0.01, + MaxNewTokens = 7, + TopP = 0.1, + }, cancellationToken: cleanupToken.Token))); } var callEnumerationTasks = new List>>(); - await Task.WhenAll(tasks).ConfigureAwait(false); + var results = await Task.WhenAll(tasks); - foreach (var callTask in tasks) + foreach (var completion in results) { callEnumerationTasks.AddRange(Enumerable.Range(0, nbConcurrentEnumeration).Select(_ => Task.Run(async () => { - var completion = await callTask.ConfigureAwait(false); var result = new List(); await foreach (var chunk in completion) { @@ -369,10 +365,10 @@ ClientWebSocket IncrementFactory() }))); // Introduce a delay between creating each WebSocket client - await Task.Delay(delayTimeSpan).ConfigureAwait(false); + await Task.Delay(delayTimeSpan); } - var allResults = await Task.WhenAll(callEnumerationTasks).ConfigureAwait(false); + var allResults = await Task.WhenAll(callEnumerationTasks); var elapsed = sw.ElapsedMilliseconds; if (maxExpectedNbClients > 0) diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AIServicesOpenAIExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AIServicesOpenAIExtensionsTests.cs index 36664cef1dd7..ec88b607171f 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AIServicesOpenAIExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AIServicesOpenAIExtensionsTests.cs @@ -79,7 +79,7 @@ public void ItCanOverwriteServices() targetBuilder.WithAIService("one", new OpenAITextCompletion("model", "key")); targetBuilder.WithAIService("one", new OpenAITextCompletion("model", "key")); - targetBuilder.WithAIService("one", (_) => new OpenAITextCompletion("model", "key")); - targetBuilder.WithAIService("one", (_) => new OpenAITextCompletion("model", "key")); + targetBuilder.WithAIService("one", (loggerFactory) => new OpenAITextCompletion("model", "key")); + targetBuilder.WithAIService("one", (loggerFactory) => new OpenAITextCompletion("model", "key")); } } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/ChatCompletionWithData/AzureChatCompletionWithDataTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/ChatCompletionWithData/AzureChatCompletionWithDataTests.cs new file mode 100644 index 000000000000..e3cdc12654e6 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/ChatCompletionWithData/AzureChatCompletionWithDataTests.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.OpenAI.ChatCompletionWithData; + +/// +/// Unit tests for +/// +public sealed class AzureChatCompletionWithDataTests : IDisposable +{ + private readonly AzureChatCompletionWithDataConfig _config; + + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; + + public AzureChatCompletionWithDataTests() + { + this._config = this.GetConfig(); + + this._messageHandlerStub = new HttpMessageHandlerStub(); + this._httpClient = new HttpClient(this._messageHandlerStub, false); + } + + [Fact] + public async Task SpecifiedConfigurationShouldBeUsedAsync() + { + // Arrange + const string ExpectedUri = "https://fake-completion-endpoint/openai/deployments/fake-completion-model-id/extensions/chat/completions?api-version=fake-api-version"; + var chatCompletion = new AzureChatCompletionWithData(this._config, this._httpClient); + + // Act + await chatCompletion.GetChatCompletionsAsync(new ChatHistory()); + + // Assert + var actualUri = this._messageHandlerStub.RequestUri?.AbsoluteUri; + var actualRequestHeaderValues = this._messageHandlerStub.RequestHeaders!.GetValues("Api-Key"); + var actualRequestContent = Encoding.UTF8.GetString(this._messageHandlerStub.RequestContent!); + + Assert.Equal(ExpectedUri, actualUri); + + Assert.Contains("fake-completion-api-key", actualRequestHeaderValues); + Assert.Contains("https://fake-data-source-endpoint", actualRequestContent, StringComparison.OrdinalIgnoreCase); + Assert.Contains("fake-data-source-api-key", actualRequestContent, StringComparison.OrdinalIgnoreCase); + Assert.Contains("fake-data-source-index", actualRequestContent, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task DefaultApiVersionShouldBeUsedAsync() + { + // Arrange + var config = this.GetConfig(); + config.CompletionApiVersion = string.Empty; + + var chatCompletion = new AzureChatCompletionWithData(config, this._httpClient); + + // Act + await chatCompletion.GetChatCompletionsAsync(new ChatHistory()); + + // Assert + var actualUri = this._messageHandlerStub.RequestUri?.AbsoluteUri; + + Assert.Contains("2023-06-01-preview", actualUri, StringComparison.OrdinalIgnoreCase); + } + + public void Dispose() + { + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } + + private AzureChatCompletionWithDataConfig GetConfig() + { + return new AzureChatCompletionWithDataConfig + { + CompletionModelId = "fake-completion-model-id", + CompletionEndpoint = "https://fake-completion-endpoint", + CompletionApiKey = "fake-completion-api-key", + CompletionApiVersion = "fake-api-version", + DataSourceEndpoint = "https://fake-data-source-endpoint", + DataSourceApiKey = "fake-data-source-api-key", + DataSourceIndex = "fake-data-source-index" + }; + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/FunctionViewExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/FunctionViewExtensionsTests.cs new file mode 100644 index 000000000000..1da8ccb3ee1f --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/FunctionViewExtensionsTests.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.OpenAI.FunctionCalling; +public sealed class FunctionViewExtensionsTests +{ + [Fact] + public void ItCanConvertToOpenAIFunctionNoParameters() + { + // Arrange + var sut = new FunctionView( + Name: "foo", + PluginName: "bar", + Description: "baz"); + + // Act + var result = sut.ToOpenAIFunction(); + + // Assert + Assert.Equal(sut.Name, result.FunctionName); + Assert.Equal(sut.PluginName, result.PluginName); + Assert.Equal(sut.Description, result.Description); + Assert.Equal($"{sut.PluginName}-{sut.Name}", result.FullyQualifiedName); + } + + [Fact] + public void ItCanConvertToOpenAIFunctionNoPluginName() + { + // Arrange + var sut = new FunctionView( + Name: "foo", + PluginName: string.Empty, + Description: "baz"); + + // Act + var result = sut.ToOpenAIFunction(); + + // Assert + Assert.Equal(sut.Name, result.FunctionName); + Assert.Equal(sut.PluginName, result.PluginName); + Assert.Equal(sut.Description, result.Description); + Assert.Equal(sut.Name, result.FullyQualifiedName); + } + + [Fact] + public void ItCanConvertToOpenAIFunctionWithParameter() + { + // Arrange + var param1 = new ParameterView( + Name: "param1", + Description: "This is param1", + DefaultValue: "1", + Type: new ParameterViewType("int"), + IsRequired: false); + + var sut = new FunctionView( + Name: "foo", + PluginName: "bar", + Description: "baz", + Parameters: new List { param1 }); + + // Act + var result = sut.ToOpenAIFunction(); + var outputParam = result.Parameters.First(); + + // Assert + Assert.Equal("int", outputParam.Type); + Assert.Equal(param1.Name, outputParam.Name); + Assert.Equal("This is param1 (default value: 1)", outputParam.Description); + Assert.Equal(param1.IsRequired, outputParam.IsRequired); + } + + [Fact] + public void ItCanConvertToOpenAIFunctionWithParameterNoType() + { + // Arrange + var param1 = new ParameterView( + Name: "param1", + Description: "This is param1", + Type: null, + IsRequired: false); + + var sut = new FunctionView( + Name: "foo", + PluginName: "bar", + Description: "baz", + Parameters: new List { param1 }); + + // Act + var result = sut.ToOpenAIFunction(); + var outputParam = result.Parameters.First(); + + // Assert + Assert.Equal("string", outputParam.Type); + Assert.Equal(param1.Name, outputParam.Name); + Assert.Equal(param1.Description, outputParam.Description); + Assert.Equal(param1.IsRequired, outputParam.IsRequired); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionResponseTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionResponseTests.cs new file mode 100644 index 000000000000..ee2b6d35dbd4 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionResponseTests.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Azure.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.OpenAI.FunctionCalling; +public sealed class OpenAIFunctionResponseTests +{ + [Fact] + public void ItCanConvertFromFunctionCallWithPluginName() + { + // Arrange + var sut = new FunctionCall("foo-bar", "{}"); + + // Act + var result = OpenAIFunctionResponse.FromFunctionCall(sut); + + // Assert + Assert.Equal("foo", result.PluginName); + Assert.Equal("bar", result.FunctionName); + } + + [Fact] + public void ItCanConvertFromFunctionCallWithNoPluginName() + { + // Arrange + var sut = new FunctionCall("foo", "{}"); + + // Act + var result = OpenAIFunctionResponse.FromFunctionCall(sut); + + // Assert + Assert.Equal(string.Empty, result.PluginName); + Assert.Equal("foo", result.FunctionName); + } + + [Fact] + public void ItCanConvertFromFunctionCallWithNoParameters() + { + // Arrange + var sut = new FunctionCall("foo", "{}"); + + // Act + var result = OpenAIFunctionResponse.FromFunctionCall(sut); + + // Assert + Assert.Equal(new Dictionary(), result.Parameters); + } + + [Fact] + public void ItCanConvertFromFunctionCallWithParameters() + { + // Arrange + var sut = new FunctionCall("foo", "{ \"param1\": \"bar\", \"param2\": 5 }"); + + // Act + var result = OpenAIFunctionResponse.FromFunctionCall(sut); + + // Assert + Assert.True(result.Parameters.TryGetValue("param1", out object? value1)); + Assert.Equal("bar", value1.ToString()); + Assert.True(result.Parameters.TryGetValue("param2", out object? value2)); + Assert.Equal("5", value2.ToString()); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionTests.cs new file mode 100644 index 000000000000..112f73a17c69 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Azure.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.OpenAI.FunctionCalling; +public sealed class OpenAIFunctionTests +{ + [Fact] + public void ItCanConvertToFunctionDefinitionWithNoPluginName() + { + // Arrange + var sut = new OpenAIFunction + { + FunctionName = "myfunc", + PluginName = string.Empty, + Description = "This is a description of the function.", + }; + + // Act + FunctionDefinition result = sut.ToFunctionDefinition(); + + // Assert + Assert.Equal(sut.FunctionName, result.Name); + Assert.Equal(sut.Description, result.Description); + } + + [Fact] + public void ItCanConvertToFunctionDefinitionWithPluginName() + { + // Arrange + var sut = new OpenAIFunction + { + FunctionName = "myfunc", + PluginName = "myplugin", + Description = "This is a description of the function.", + }; + + // Act + FunctionDefinition result = sut.ToFunctionDefinition(); + + // Assert + Assert.Equal("myplugin-myfunc", result.Name); + Assert.Equal(sut.Description, result.Description); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/OpenAIRequestSettingsConverterTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/OpenAIRequestSettingsConverterTests.cs new file mode 100644 index 000000000000..a7d1feec60da --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/OpenAIRequestSettingsConverterTests.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.OpenAI; + +/// +/// Unit tests of OpenAIRequestSettingsConverter +/// +public class OpenAIRequestSettingsConverterTests +{ + [Fact] + public void ItDeserialisesOpenAIRequestSettingsWithCorrectDefaults() + { + // Arrange + JsonSerializerOptions options = new(); + options.Converters.Add(new OpenAIRequestSettingsConverter()); + var json = "{}"; + + // Act + var requestSettings = JsonSerializer.Deserialize(json, options); + + // Assert + Assert.NotNull(requestSettings); + Assert.Equal(0, requestSettings.Temperature); + Assert.Equal(0, requestSettings.TopP); + Assert.Equal(0, requestSettings.FrequencyPenalty); + Assert.Equal(0, requestSettings.PresencePenalty); + Assert.Equal(1, requestSettings.ResultsPerPrompt); + Assert.Equal(Array.Empty(), requestSettings.StopSequences); + Assert.Equal(new Dictionary(), requestSettings.TokenSelectionBiases); + Assert.Null(requestSettings.ServiceId); + Assert.Null(requestSettings.MaxTokens); + } + + [Fact] + public void ItDeserialisesOpenAIRequestSettingsWithSnakeCaseNaming() + { + // Arrange + JsonSerializerOptions options = new(); + options.Converters.Add(new OpenAIRequestSettingsConverter()); + var json = @"{ + ""temperature"": 0.7, + ""top_p"": 0.7, + ""frequency_penalty"": 0.7, + ""presence_penalty"": 0.7, + ""results_per_prompt"": 2, + ""stop_sequences"": [ ""foo"", ""bar"" ], + ""token_selection_biases"": { ""1"": 2, ""3"": 4 }, + ""service_id"": ""service"", + ""max_tokens"": 128 +}"; + + // Act + var requestSettings = JsonSerializer.Deserialize(json, options); + + // Assert + Assert.NotNull(requestSettings); + Assert.Equal(0.7, requestSettings.Temperature); + Assert.Equal(0.7, requestSettings.TopP); + Assert.Equal(0.7, requestSettings.FrequencyPenalty); + Assert.Equal(0.7, requestSettings.PresencePenalty); + Assert.Equal(2, requestSettings.ResultsPerPrompt); + Assert.Equal(new string[] { "foo", "bar" }, requestSettings.StopSequences); + Assert.Equal(new Dictionary() { { 1, 2 }, { 3, 4 } }, requestSettings.TokenSelectionBiases); + Assert.Equal("service", requestSettings.ServiceId); + Assert.Equal(128, requestSettings.MaxTokens); + } + + [Fact] + public void ItDeserialisesOpenAIRequestSettingsWithPascalCaseNaming() + { + // Arrange + JsonSerializerOptions options = new(); + options.Converters.Add(new OpenAIRequestSettingsConverter()); + var json = @"{ + ""Temperature"": 0.7, + ""TopP"": 0.7, + ""FrequencyPenalty"": 0.7, + ""PresencePenalty"": 0.7, + ""ResultsPerPrompt"": 2, + ""StopSequences"": [ ""foo"", ""bar"" ], + ""TokenSelectionBiases"": { ""1"": 2, ""3"": 4 }, + ""ServiceId"": ""service"", + ""MaxTokens"": 128 +}"; + + // Act + var requestSettings = JsonSerializer.Deserialize(json, options); + + // Assert + Assert.NotNull(requestSettings); + Assert.Equal(0.7, requestSettings.Temperature); + Assert.Equal(0.7, requestSettings.TopP); + Assert.Equal(0.7, requestSettings.FrequencyPenalty); + Assert.Equal(0.7, requestSettings.PresencePenalty); + Assert.Equal(2, requestSettings.ResultsPerPrompt); + Assert.Equal(new string[] { "foo", "bar" }, requestSettings.StopSequences); + Assert.Equal(new Dictionary() { { 1, 2 }, { 3, 4 } }, requestSettings.TokenSelectionBiases); + Assert.Equal("service", requestSettings.ServiceId); + Assert.Equal(128, requestSettings.MaxTokens); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/OpenAIRequestSettingsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/OpenAIRequestSettingsTests.cs new file mode 100644 index 000000000000..bcdbb64603a4 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/OpenAIRequestSettingsTests.cs @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Text; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.OpenAI; + +/// +/// Unit tests of OpenAIRequestSettings +/// +public class OpenAIRequestSettingsTests +{ + [Fact] + public void ItCreatesOpenAIRequestSettingsWithCorrectDefaults() + { + // Arrange + // Act + OpenAIRequestSettings requestSettings = OpenAIRequestSettings.FromRequestSettings(null, 128); + + // Assert + Assert.NotNull(requestSettings); + Assert.Equal(0, requestSettings.Temperature); + Assert.Equal(0, requestSettings.TopP); + Assert.Equal(0, requestSettings.FrequencyPenalty); + Assert.Equal(0, requestSettings.PresencePenalty); + Assert.Equal(1, requestSettings.ResultsPerPrompt); + Assert.Equal(Array.Empty(), requestSettings.StopSequences); + Assert.Equal(new Dictionary(), requestSettings.TokenSelectionBiases); + Assert.Null(requestSettings.ServiceId); + Assert.Equal(128, requestSettings.MaxTokens); + } + + [Fact] + public void ItUsesExistingOpenAIRequestSettings() + { + // Arrange + OpenAIRequestSettings actualSettings = new() + { + Temperature = 0.7, + TopP = 0.7, + FrequencyPenalty = 0.7, + PresencePenalty = 0.7, + ResultsPerPrompt = 2, + StopSequences = new string[] { "foo", "bar" }, + ChatSystemPrompt = "chat system prompt", + MaxTokens = 128, + ServiceId = "service", + TokenSelectionBiases = new Dictionary() { { 1, 2 }, { 3, 4 } }, + }; + + // Act + OpenAIRequestSettings requestSettings = OpenAIRequestSettings.FromRequestSettings(actualSettings); + + // Assert + Assert.NotNull(requestSettings); + Assert.Equal(actualSettings, requestSettings); + } + + [Fact] + public void ItCanUseOpenAIRequestSettings() + { + // Arrange + AIRequestSettings actualSettings = new() + { + ServiceId = "service", + }; + + // Act + OpenAIRequestSettings requestSettings = OpenAIRequestSettings.FromRequestSettings(actualSettings, null); + + // Assert + Assert.NotNull(requestSettings); + Assert.Equal(actualSettings.ServiceId, requestSettings.ServiceId); + } + + [Fact] + public void ItCreatesOpenAIRequestSettingsFromExtraPropertiesSnakeCase() + { + // Arrange + AIRequestSettings actualSettings = new() + { + ServiceId = "service", + ExtensionData = new Dictionary() + { + { "temperature", 0.7 }, + { "top_p", 0.7 }, + { "frequency_penalty", 0.7 }, + { "presence_penalty", 0.7 }, + { "results_per_prompt", 2 }, + { "stop_sequences", new [] { "foo", "bar" } }, + { "chat_system_prompt", "chat system prompt" }, + { "max_tokens", 128 }, + { "service_id", "service" }, + { "token_selection_biases", new Dictionary() { { 1, 2 }, { 3, 4 } } } + } + }; + + // Act + OpenAIRequestSettings requestSettings = OpenAIRequestSettings.FromRequestSettings(actualSettings, null); + + // Assert + AssertRequestSettings(requestSettings); + } + + [Fact] + public void ItCreatesOpenAIRequestSettingsFromExtraPropertiesPascalCase() + { + // Arrange + AIRequestSettings actualSettings = new() + { + ServiceId = "service", + ExtensionData = new Dictionary() + { + { "Temperature", 0.7 }, + { "TopP", 0.7 }, + { "FrequencyPenalty", 0.7 }, + { "PresencePenalty", 0.7 }, + { "ResultsPerPrompt", 2 }, + { "StopSequences", new[] { "foo", "bar" } }, + { "ChatSystemPrompt", "chat system prompt" }, + { "MaxTokens", 128 }, + { "ServiceId", "service" }, + { "TokenSelectionBiases", new Dictionary() { { 1, 2 }, { 3, 4 } } } + } + }; + + // Act + OpenAIRequestSettings requestSettings = OpenAIRequestSettings.FromRequestSettings(actualSettings); + + // Assert + AssertRequestSettings(requestSettings); + } + + [Fact] + public void ItCreatesOpenAIRequestSettingsFromJsonSnakeCase() + { + // Arrange + var json = @"{ + ""temperature"": 0.7, + ""top_p"": 0.7, + ""frequency_penalty"": 0.7, + ""presence_penalty"": 0.7, + ""results_per_prompt"": 2, + ""stop_sequences"": [ ""foo"", ""bar"" ], + ""chat_system_prompt"": ""chat system prompt"", + ""token_selection_biases"": { ""1"": 2, ""3"": 4 }, + ""service_id"": ""service"", + ""max_tokens"": 128 +}"; + var actualSettings = Json.Deserialize(json); + + // Act + OpenAIRequestSettings requestSettings = OpenAIRequestSettings.FromRequestSettings(actualSettings); + + // Assert + AssertRequestSettings(requestSettings); + } + + [Fact] + public void ItCreatesOpenAIRequestSettingsFromJsonPascalCase() + { + // Arrange + var json = @"{ + ""Temperature"": 0.7, + ""TopP"": 0.7, + ""FrequencyPenalty"": 0.7, + ""PresencePenalty"": 0.7, + ""ResultsPerPrompt"": 2, + ""StopSequences"": [ ""foo"", ""bar"" ], + ""ChatSystemPrompt"": ""chat system prompt"", + ""TokenSelectionBiases"": { ""1"": 2, ""3"": 4 }, + ""ServiceId"": ""service"", + ""MaxTokens"": 128 +}"; + var actualSettings = Json.Deserialize(json); + + // Act + OpenAIRequestSettings requestSettings = OpenAIRequestSettings.FromRequestSettings(actualSettings); + + // Assert + AssertRequestSettings(requestSettings); + } + + private static void AssertRequestSettings(OpenAIRequestSettings requestSettings) + { + Assert.NotNull(requestSettings); + Assert.Equal(0.7, requestSettings.Temperature); + Assert.Equal(0.7, requestSettings.TopP); + Assert.Equal(0.7, requestSettings.FrequencyPenalty); + Assert.Equal(0.7, requestSettings.PresencePenalty); + Assert.Equal(2, requestSettings.ResultsPerPrompt); + Assert.Equal(new string[] { "foo", "bar" }, requestSettings.StopSequences); + Assert.Equal("chat system prompt", requestSettings.ChatSystemPrompt); + Assert.Equal(new Dictionary() { { 1, 2 }, { 3, 4 } }, requestSettings.TokenSelectionBiases); + Assert.Equal("service", requestSettings.ServiceId); + Assert.Equal(128, requestSettings.MaxTokens); + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/Tokenizers/GPT3TokenizerTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/Tokenizers/GPT3TokenizerTests.cs deleted file mode 100644 index f8b0b94f6497..000000000000 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/Tokenizers/GPT3TokenizerTests.cs +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Text.Json; -using Microsoft.SemanticKernel.Connectors.AI.OpenAI.Tokenizers; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.Connectors.UnitTests.OpenAI.Tokenizers; - -#pragma warning disable CA5394 -public class GPT3TokenizerTests -{ - private readonly ITestOutputHelper _logger; - - public GPT3TokenizerTests(ITestOutputHelper logger) - { - this._logger = logger; - } - - // ReSharper disable StringLiteralTypo - [Theory] - [InlineData("", 0)] - [InlineData("a", 1)] - [InlineData("abbccd", 3)] - [InlineData("ab bc cd", 3)] - [InlineData("ab + bc + cd = 10.", 8)] - [InlineData("Array.prototype.slice()", 6)] - [InlineData("const animals = ['ant', 'bison', 'camel', 'duck', 'elephant'];", 23)] - [InlineData(" c o n s t a n i m a l s = [ ' a n t ' , ' b i s o n ' , ' c a m e l ' , ' d u c k ' , ' e l e p h a n t ' ] ; ", 70)] - [InlineData("Many words map to one token, but some don't: indivisible.", 16)] - [InlineData("Unicode characters like emojis may be split into many tokens containing the underlying bytes: 🤚🏾", 25)] - [InlineData("Sequences of characters commonly found next to each other may be grouped together: 1234567890", 19)] - [InlineData("ἀμφὶ Ποσειδάωτα, μέγαν θεόν, ἄρχομ᾽ ἀείδειν,", 58)] - [InlineData("This is a test 𝓣𝓱𝓲𝓼 𝓲𝓼 𝓪 𝓽𝓮𝓼𝓽", 41)] - [InlineData("This.▶︎ is🎶 a😀 test🐼", 17)] - [InlineData( - "在计算机编程中,单元测试(英語:Unit Testing)又称为模块测试 [來源請求] ,是针对程序模块(软件设计的最小单位)来进行正确性检验的测试工作。程序单元是应用的最小可测试部件。在过程化编程中,一个单元就是单个程序、函数、过程等;对于面向对象编程,最小单元就是方法,包括基类(超类)、抽象类、或者派生类(子类)中的方法。 ", - 334)] - [InlineData(@"En programación, una prueba unitaria o test unitario (del inglés: unit test) -es una forma efectiva de comprobar el correcto funcionamiento de las unidades individuales -más pequeñas de los programas informáticos", 68)] - // ReSharper restore StringLiteralTypo - public void ItReturnsTheCorrectNumberOfTokens(string text, int tokenCount) - { - // Act-Assert - Assert.Equal(tokenCount, GPT3Tokenizer.Encode(text).Count); - Assert.Equal(tokenCount, GPT3Tokenizer.Encode(new StringBuilder(text)).Count); - Assert.Equal(tokenCount, GPT3Tokenizer.Encode(text.ToArray()).Count); - Assert.Equal(tokenCount, GPT3Tokenizer.Encode(text.ToCharArray()).Count); - Assert.Equal(tokenCount, GPT3Tokenizer.Encode(text.ToCharArray().ToList()).Count); - } - - // TODO: check actual token IDs// ReSharper disable StringLiteralTypo - [Theory] - [InlineData("", "[]")] - [InlineData("a", "[64]")] - [InlineData("abbccd", "[6485, 535, 67]")] - [InlineData("ab bc cd", "[397, 47125, 22927]")] - [InlineData("January 1st, 2000", "[21339, 352, 301, 11, 4751]")] - [InlineData("ab + bc + cd = 10.", "[397, 1343, 47125, 1343, 22927, 796, 838, 13]")] - [InlineData("Array.prototype.slice()", "[19182, 13, 38124, 13, 48369, 3419]")] - [InlineData("const animals = ['ant', 'bison', 'camel', 'duck', 'elephant'];", - "[9979,4695,796,37250,415,3256,705,65,1653,3256,705,66,17983,3256,705,646,694,3256,705,11129,33959,6,11208]")] - [InlineData(" c o n s t a n i m a l s = [ ' a n t ' , ' b i s o n ' , ' c a m e l ' , ' d u c k ' , ' e l e p h a n t ' ] ; ", - "[269,267,299,264,256,220,220,257,299,1312,285,257,300,264,220,220,796,220,220,685,705,257,299,256,705,837,220,220,705,275,1312,264,267,299,705,837,220,220,705,269,257,285,304,300,705,837,220,220,705,288,334,269,479,705,837,220,220,705,304,300,304,279,289,257,299,256,705,2361,2162,220]")] - [InlineData("This is a test 𝓣𝓱𝓲𝓼 𝓲𝓼 𝓪 𝓽𝓮𝓼𝓽", - "[1212,318,257,1332,220,47728,241,96,47728,241,109,47728,241,110,47728,241,120,220,47728,241,110,47728,241,120,220,47728,241,103,220,47728,241,121,47728,241,106,47728,241,120,47728,241,121]")] - [InlineData("This.▶︎ is🎶 a😀 test🐼", "[1212,13,5008,114,35266,236,318,8582,236,114,257,47249,222,1332,8582,238,120]")] - [InlineData("Many words map to one token, but some don't: indivisible.", "[7085,2456,3975,284,530,11241,11,475,617,836,470,25,773,452,12843,13]")] - [InlineData("Unicode characters like emojis may be split into many tokens containing the underlying bytes: 🤚🏾", - "[3118,291,1098,3435,588,795,13210,271,743,307,6626,656,867,16326,7268,262,10238,9881,25,12520,97,248,8582,237,122]")] - [InlineData("Sequences of characters commonly found next to each other may be grouped together: 1234567890", - "[44015,3007,286,3435,8811,1043,1306,284,1123,584,743,307,32824,1978,25,17031,2231,30924,3829]")] - [InlineData("ἀμφὶ Ποσειδάωτα, μέγαν θεόν, ἄρχομ᾽ ἀείδειν,", - "[157,120,222,34703,139,228,45495,114,7377,254,26517,38392,30950,29945,138,112,138,105,49535,32830,17394,11,18919,138,255,42063,17394,26180,7377,116,30950,139,234,26180,11,28053,120,226,33643,139,229,26517,34703,157,122,121,28053,120,222,30950,138,107,138,112,30950,29945,26180,11]")] - // ReSharper restore StringLiteralTypo - public void ItReturnsTheCorrectTokens(string text, string tokens) - { - // Arrange - List expectedTokens = JsonSerializer.Deserialize>(tokens)!; - - // Act - List actualTokens = GPT3Tokenizer.Encode(text); - - // Assert - Assert.Equal(expectedTokens.Count, actualTokens.Count); - Assert.Equal(tokens.Replace(" ", "", StringComparison.OrdinalIgnoreCase), - JsonSerializer.Serialize(actualTokens).Replace(" ", "", StringComparison.OrdinalIgnoreCase)); - } - - [Fact] - // ReSharper disable StringLiteralTypo - public void ItDoesntTimeOutForABigText() - { - // Arrange - const int OneMb = 1000 * 1000; - var watch = new Stopwatch(); - Random rnd = new(); - StringBuilder text = new(); - watch.Start(); - var count = 0; - while (text.Length < OneMb) - { - text.Append(GenerateWord(rnd, count++)); - } - - watch.Stop(); - this._logger.WriteLine($"Text size: {text.Length}. Text generated in: {watch.ElapsedMilliseconds} msecs."); - - // Act + Assert no exception occurs - watch.Restart(); - GPT3Tokenizer.Encode(text); - watch.Stop(); - this._logger.WriteLine($"Text size: {text.Length}. Text tokenized in: {watch.ElapsedMilliseconds} msecs."); - } - - // Try to generate some random text to reduce the tokenizer cache hits. - private static string GenerateWord(Random rnd, int count) - { - string[] group1 = { "test🐼", "bytes", " ", "- =", "llllllllllllllllllllllllllllllllllll", "%", "This.▶ ", " group ", "ἄρχομ " }; - string[] group2 = { "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z", "\n" }; - string[] group3 = { "a", "e", "i", "o", "u", " " }; - - var result = new StringBuilder(); - - var length = rnd.Next(1, 200); - if (length <= 2) - { - var word1 = group1[rnd.Next(0, group1.Length)]; - var word2 = group1[rnd.Next(0, group1.Length)]; - result.Append(word1); - result.Append(word2); - } - else - { - while (result.Length < length) - { - result.Append(group2[rnd.Next(0, group2.Length)]); - result.Append(group3[rnd.Next(0, group3.Length)]); - result.Append(rnd.Next(0, 2) == 0 ? group2[rnd.Next(0, group2.Length)] : group3[rnd.Next(0, group3.Length)]); - } - } - - return $"{result}{count}"; - } -} -#pragma warning restore CA5394 diff --git a/dotnet/src/Connectors/Connectors.UnitTests/WebSocketTestServer.cs b/dotnet/src/Connectors/Connectors.UnitTests/WebSocketTestServer.cs index 11eafcb24ef2..c833797f3219 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/WebSocketTestServer.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/WebSocketTestServer.cs @@ -21,12 +21,14 @@ internal class WebSocketTestServer : IDisposable private readonly CancellationTokenSource _socketCancellationTokenSource; private bool _serverIsRunning; - private Func, List>> _arraySegmentHandler; + private readonly Func, List>> _arraySegmentHandler; private readonly ConcurrentDictionary> _requestContentQueues; private readonly ConcurrentBag _runningTasks = new(); private readonly ConcurrentDictionary _clients = new(); + private readonly Task? _handleRequestTask = null; + public TimeSpan RequestProcessingDelay { get; set; } = TimeSpan.Zero; public TimeSpan SegmentMessageDelay { get; set; } = TimeSpan.Zero; @@ -55,7 +57,10 @@ public WebSocketTestServer(string url, Func, List)this.HandleRequestsAsync, this._mainCancellationTokenSource.Token); + if (this._handleRequestTask is null || this._handleRequestTask.IsCompleted) + { + this._handleRequestTask = Task.Run((Func)this.HandleRequestsAsync, this._mainCancellationTokenSource.Token); + } } private async Task HandleRequestsAsync() @@ -216,6 +221,7 @@ public async ValueTask DisposeAsync() } } +#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits public void Dispose() { this.DisposeAsync().AsTask().GetAwaiter().GetResult(); diff --git a/dotnet/src/Connectors/Connectors.UnitTests/XunitLogger.cs b/dotnet/src/Connectors/Connectors.UnitTests/XunitLogger.cs index 1521dac75bed..6a6ad5c22bc6 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/XunitLogger.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/XunitLogger.cs @@ -9,7 +9,7 @@ namespace SemanticKernel.Connectors.UnitTests; /// /// A logger that writes to the Xunit test output /// -internal sealed class XunitLogger : ILogger, IDisposable +internal sealed class XunitLogger : ILogger, IDisposable { private readonly ITestOutputHelper _output; diff --git a/dotnet/src/Extensions/Extensions.UnitTests/.editorconfig b/dotnet/src/Extensions/Extensions.UnitTests/.editorconfig index 8f4c52fa9f51..394eef685f21 100644 --- a/dotnet/src/Extensions/Extensions.UnitTests/.editorconfig +++ b/dotnet/src/Extensions/Extensions.UnitTests/.editorconfig @@ -2,4 +2,5 @@ [*.cs] dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave - +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Extensions.UnitTests.csproj b/dotnet/src/Extensions/Extensions.UnitTests/Extensions.UnitTests.csproj index 27580b44a75b..5724cc90ffc4 100644 --- a/dotnet/src/Extensions/Extensions.UnitTests/Extensions.UnitTests.csproj +++ b/dotnet/src/Extensions/Extensions.UnitTests/Extensions.UnitTests.csproj @@ -1,4 +1,4 @@ - + SemanticKernel.Extensions.UnitTests @@ -28,9 +28,9 @@ - - - + + + diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Planning/ActionPlanner/ActionPlannerTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/Planning/ActionPlanner/ActionPlannerTests.cs deleted file mode 100644 index 4a38ff333bca..000000000000 --- a/dotnet/src/Extensions/Extensions.UnitTests/Planning/ActionPlanner/ActionPlannerTests.cs +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; - -namespace SemanticKernel.Extensions.UnitTests.Planning.ActionPlanner; - -public sealed class ActionPlannerTests -{ - [Fact] - public async Task ExtractsAndDeserializesWellFormedJsonFromPlannerResult() - { - // Arrange - var functions = new List<(string name, string skillName, string description, bool isSemantic)>() - { - ("SendEmail", "email", "Send an e-mail", false), - ("PullsList", "GitHubSkill", "List pull requests", true) - }; - - var functionsView = new FunctionsView(); - var skills = new Mock(); - foreach (var (name, skillName, description, isSemantic) in functions) - { - var functionView = new FunctionView(name, skillName, description, new List(), isSemantic, true); - var mockFunction = CreateMockFunction(functionView); - functionsView.AddFunction(functionView); - - mockFunction.Setup(x => - x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns((context, settings, CancellationToken) => - { - context.Variables.Update("MOCK FUNCTION CALLED"); - return Task.FromResult(context); - }); - - skills.Setup(x => x.GetFunction(It.Is(s => s == skillName), It.Is(s => s == name))) - .Returns(mockFunction.Object); - ISKFunction? outFunc = mockFunction.Object; - skills.Setup(x => x.TryGetFunction(It.Is(s => s == skillName), It.Is(s => s == name), out outFunc)).Returns(true); - } - - skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); - - string planString = "Here is a possible plan to accomplish the user intent:\n\n{\"plan\":{\n\"rationale\": \"the list contains a function that allows to list pull requests\",\n\"function\": \"GitHubSkill.PullsList\",\n\"parameters\": {\n\"owner\": \"microsoft\",\n\"repo\": \"semantic-kernel\",\n\"state\": \"open\"\n}}}\n\nThis plan uses the `GitHubSkill.PullsList` function to list the open pull requests for the `semantic-kernel` repository owned by `microsoft`. The `state` parameter is set to `\"open\"` to filter the results to only show open pull requests."; - - var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(planString, skills); - - var planner = new Microsoft.SemanticKernel.Planning.ActionPlanner(kernel.Object); - - // Act - var plan = await planner.CreatePlanAsync("goal"); - - // Assert - Assert.Equal("goal", plan.Description); - - Assert.Equal(plan.Steps.Count, 1); - Assert.Equal(plan.Steps[0].SkillName, "GitHubSkill"); - Assert.Equal(plan.Steps[0].Name, "PullsList"); - } - - [Fact] - public async Task InvalidJsonThrowsAsync() - { - // Arrange - string invalidJsonString = "<>"; - - var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(invalidJsonString); - - var planner = new Microsoft.SemanticKernel.Planning.ActionPlanner(kernel.Object); - - // Act & Assert - await Assert.ThrowsAsync(async () => await planner.CreatePlanAsync("goal")); - } - - [Fact] - public async Task MalformedJsonThrowsAsync() - { - // Arrange - - // Extra opening brace before rationale - string invalidJsonString = "Here is a possible plan to accomplish the user intent:\n\n{\"plan\": { {\n\"rationale\": \"the list contains a function that allows to list pull requests\",\n\"function\": \"GitHubSkill.PullsList\",\n\"parameters\": {\n\"owner\": \"microsoft\",\n\"repo\": \"semantic-kernel\",\n\"state\": \"open\"\n}}}\n\nThis plan uses the `GitHubSkill.PullsList` function to list the open pull requests for the `semantic-kernel` repository owned by `microsoft`. The `state` parameter is set to `\"open\"` to filter the results to only show open pull requests."; - - var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(invalidJsonString); - - var planner = new Microsoft.SemanticKernel.Planning.ActionPlanner(kernel.Object); - - // Act & Assert - await Assert.ThrowsAsync(async () => await planner.CreatePlanAsync("goal")); - } - - private Mock CreateMockKernelAndFunctionFlowWithTestString(string testPlanString, Mock? skills = null) - { - var kernel = new Mock(); - - if (skills is null) - { - skills = new Mock(); - - var functionsView = new FunctionsView(); - skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); - } - - var returnContext = new SKContext( - new ContextVariables(testPlanString), - skills.Object - ); - - var context = new SKContext( - skills: skills.Object - ); - - var mockFunctionFlowFunction = new Mock(); - mockFunctionFlowFunction.Setup(x => x.InvokeAsync( - It.IsAny(), - null, - default - )).Callback( - (c, s, ct) => c.Variables.Update("Hello world!") - ).Returns(() => Task.FromResult(returnContext)); - - // Mock Skills - kernel.Setup(x => x.Skills).Returns(skills.Object); - kernel.Setup(x => x.CreateNewContext()).Returns(context); - - kernel.Setup(x => x.RegisterSemanticFunction( - It.IsAny(), - It.IsAny(), - It.IsAny() - )).Returns(mockFunctionFlowFunction.Object); - - return kernel; - } - - // Method to create Mock objects - private static Mock CreateMockFunction(FunctionView functionView) - { - var mockFunction = new Mock(); - mockFunction.Setup(x => x.Describe()).Returns(functionView); - mockFunction.Setup(x => x.Name).Returns(functionView.Name); - mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); - return mockFunction; - } -} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SKContextExtensionsTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SKContextExtensionsTests.cs deleted file mode 100644 index 79ad8405a4df..000000000000 --- a/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SKContextExtensionsTests.cs +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning.Sequential; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using SemanticKernel.Extensions.UnitTests.XunitHelpers; -using Xunit; - -namespace SemanticKernel.Extensions.UnitTests.Planning.SequentialPlanner; - -public class SKContextExtensionsTests -{ - [Fact] - public async Task CanCallGetAvailableFunctionsWithNoFunctionsAsync() - { - // Arrange - var variables = new ContextVariables(); - var skills = new SkillCollection(); - var logger = TestConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - - // Arrange Mock Memory and Result - var memory = new Mock(); - var memoryQueryResult = new MemoryQueryResult( - new MemoryRecordMetadata( - isReference: false, - id: "id", - text: "text", - description: "description", - externalSourceName: "sourceName", - additionalMetadata: "value"), - relevance: 0.8, - embedding: null); - var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); - memory.Setup(x => - x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(asyncEnumerable); - - // Arrange GetAvailableFunctionsAsync parameters - var context = new SKContext(variables, skills, logger); - var config = new SequentialPlannerConfig() { Memory = memory.Object }; - var semanticQuery = "test"; - - // Act - var result = await context.GetAvailableFunctionsAsync(config, semanticQuery, cancellationToken); - - // Assert - Assert.NotNull(result); - memory.Verify( - x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), - Times.Never); - } - - [Fact] - public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() - { - // Arrange - var variables = new ContextVariables(); - var logger = TestConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - - // Arrange FunctionView - var functionMock = new Mock(); - var functionsView = new FunctionsView(); - var functionView = new FunctionView("functionName", "skillName", "description", new List(), true, false); - var nativeFunctionView = new FunctionView("nativeFunctionName", "skillName", "description", new List(), false, false); - functionsView.AddFunction(functionView); - functionsView.AddFunction(nativeFunctionView); - - // Arrange Mock Memory and Result - var skills = new Mock(); - var memoryQueryResult = - new MemoryQueryResult( - new MemoryRecordMetadata( - isReference: false, - id: functionView.ToFullyQualifiedName(), - text: "text", - description: "description", - externalSourceName: "sourceName", - additionalMetadata: "value"), - relevance: 0.8, - embedding: null); - var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); - var memory = new Mock(); - memory.Setup(x => - x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(asyncEnumerable); - - skills.Setup(x => x.TryGetFunction(It.IsAny(), It.IsAny(), out It.Ref.IsAny)).Returns(true); - skills.Setup(x => x.GetFunction(It.IsAny(), It.IsAny())).Returns(functionMock.Object); - skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); - - // Arrange GetAvailableFunctionsAsync parameters - var context = new SKContext(variables, skills.Object, logger); - var config = new SequentialPlannerConfig() { Memory = memory.Object }; - var semanticQuery = "test"; - - // Act - var result = (await context.GetAvailableFunctionsAsync(config, semanticQuery, cancellationToken)).ToList(); - - // Assert - Assert.NotNull(result); - Assert.Equal(2, result.Count); - Assert.Equal(functionView, result[0]); - - // Arrange update IncludedFunctions - config.IncludedFunctions.UnionWith(new List { "nativeFunctionName" }); - - // Act - result = (await context.GetAvailableFunctionsAsync(config, semanticQuery)).ToList(); - - // Assert - Assert.NotNull(result); - Assert.Equal(2, result.Count); // IncludedFunctions should be added to the result - Assert.Equal(functionView, result[0]); - Assert.Equal(nativeFunctionView, result[1]); - } - - [Fact] - public async Task CanCallGetAvailableFunctionsWithFunctionsWithRelevancyAsync() - { - // Arrange - var variables = new ContextVariables(); - var logger = TestConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - - // Arrange FunctionView - var functionMock = new Mock(); - var functionsView = new FunctionsView(); - var functionView = new FunctionView("functionName", "skillName", "description", new List(), true, false); - var nativeFunctionView = new FunctionView("nativeFunctionName", "skillName", "description", new List(), false, false); - functionsView.AddFunction(functionView); - functionsView.AddFunction(nativeFunctionView); - - // Arrange Mock Memory and Result - var skills = new Mock(); - var memoryQueryResult = - new MemoryQueryResult( - new MemoryRecordMetadata( - isReference: false, - id: functionView.ToFullyQualifiedName(), - text: "text", - description: "description", - externalSourceName: "sourceName", - additionalMetadata: "value"), - relevance: 0.8, - embedding: null); - var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); - var memory = new Mock(); - memory.Setup(x => - x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(asyncEnumerable); - - skills.Setup(x => x.TryGetFunction(It.IsAny(), It.IsAny(), out It.Ref.IsAny)).Returns(true); - skills.Setup(x => x.GetFunction(It.IsAny(), It.IsAny())).Returns(functionMock.Object); - skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); - - // Arrange GetAvailableFunctionsAsync parameters - var context = new SKContext(variables, skills.Object, logger); - var config = new SequentialPlannerConfig { RelevancyThreshold = 0.78, Memory = memory.Object }; - var semanticQuery = "test"; - - // Act - var result = (await context.GetAvailableFunctionsAsync(config, semanticQuery, cancellationToken)).ToList(); - - // Assert - Assert.NotNull(result); - Assert.Single(result); - Assert.Equal(functionView, result[0]); - - // Arrange update IncludedFunctions - config.IncludedFunctions.UnionWith(new List { "nativeFunctionName" }); - - // Act - result = (await context.GetAvailableFunctionsAsync(config, semanticQuery)).ToList(); - - // Assert - Assert.NotNull(result); - Assert.Equal(2, result.Count); // IncludedFunctions should be added to the result - Assert.Equal(functionView, result[0]); - Assert.Equal(nativeFunctionView, result[1]); - } - - [Fact] - public async Task CanCallGetAvailableFunctionsAsyncWithDefaultRelevancyAsync() - { - // Arrange - var variables = new ContextVariables(); - var skills = new SkillCollection(); - var logger = TestConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - - // Arrange Mock Memory and Result - var memory = new Mock(); - var memoryQueryResult = - new MemoryQueryResult( - new MemoryRecordMetadata( - isReference: false, - id: "id", - text: "text", - description: "description", - externalSourceName: "sourceName", - additionalMetadata: "value"), - relevance: 0.8, - embedding: null); - var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); - memory.Setup(x => - x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(asyncEnumerable); - - // Arrange GetAvailableFunctionsAsync parameters - var context = new SKContext(variables, skills, logger); - var config = new SequentialPlannerConfig { RelevancyThreshold = 0.78, Memory = memory.Object }; - var semanticQuery = "test"; - - // Act - var result = await context.GetAvailableFunctionsAsync(config, semanticQuery, cancellationToken); - - // Assert - Assert.NotNull(result); - memory.Verify( - x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), - Times.Once); - } -} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SequentialPlanParserTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SequentialPlanParserTests.cs deleted file mode 100644 index 474184f84261..000000000000 --- a/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SequentialPlanParserTests.cs +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.Planning.Sequential; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.Extensions.UnitTests.Planning.SequentialPlanner; - -public class SequentialPlanParserTests -{ - private readonly ITestOutputHelper _testOutputHelper; - - public SequentialPlanParserTests(ITestOutputHelper testOutputHelper) - { - this._testOutputHelper = testOutputHelper; - } - - private Mock CreateKernelMock( - out Mock semanticMemoryMock, - out Mock mockSkillCollection, - out Mock mockLogger) - { - semanticMemoryMock = new Mock(); - mockSkillCollection = new Mock(); - mockLogger = new Mock(); - - var kernelMock = new Mock(); - kernelMock.SetupGet(k => k.Skills).Returns(mockSkillCollection.Object); - kernelMock.SetupGet(k => k.Logger).Returns(mockLogger.Object); - kernelMock.SetupGet(k => k.Memory).Returns(semanticMemoryMock.Object); - - return kernelMock; - } - - private SKContext CreateSKContext( - IKernel kernel, - ContextVariables? variables = null) - { - return new SKContext(variables, kernel.Skills, kernel.Logger); - } - - private static Mock CreateMockFunction(FunctionView functionView, string result = "") - { - var mockFunction = new Mock(); - mockFunction.Setup(x => x.Describe()).Returns(functionView); - mockFunction.Setup(x => x.Name).Returns(functionView.Name); - mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); - return mockFunction; - } - - private void CreateKernelAndFunctionCreateMocks(List<(string name, string skillName, string description, bool isSemantic, string result)> functions, - out IKernel kernel) - { - var kernelMock = this.CreateKernelMock(out _, out var skills, out _); - kernel = kernelMock.Object; - - // For Create - kernelMock.Setup(k => k.CreateNewContext()).Returns(this.CreateSKContext(kernel)); - - var functionsView = new FunctionsView(); - foreach (var (name, skillName, description, isSemantic, resultString) in functions) - { - var functionView = new FunctionView(name, skillName, description, new List(), isSemantic, true); - var mockFunction = CreateMockFunction(functionView); - functionsView.AddFunction(functionView); - - var result = this.CreateSKContext(kernel); - result.Variables.Update(resultString); - mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .ReturnsAsync(result); - - if (string.IsNullOrEmpty(name)) - { - kernelMock.Setup(x => x.RegisterSemanticFunction( - It.IsAny(), - It.IsAny(), - It.IsAny() - )).Returns(mockFunction.Object); - } - else - { - skills.Setup(x => x.GetFunction(It.Is(s => s == skillName), It.Is(s => s == name))) - .Returns(mockFunction.Object); - ISKFunction? outFunc = mockFunction.Object; - skills.Setup(x => x.TryGetFunction(It.Is(s => s == skillName), It.Is(s => s == name), out outFunc)).Returns(true); - } - } - - skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); - } - - [Fact] - public void CanCallToPlanFromXml() - { - // Arrange - var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() - { - ("Summarize", "SummarizeSkill", "Summarize an input", true, "This is the summary."), - ("Translate", "WriterSkill", "Translate to french", true, "Bonjour!"), - ("GetEmailAddressAsync", "email", "Get email address", false, "johndoe@email.com"), - ("SendEmailAsync", "email", "Send email", false, "Email sent."), - }; - this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); - - var planString = - @" - - - - - -"; - var goal = "Summarize an input, translate to french, and e-mail to John Doe"; - - // Act - var plan = planString.ToPlanFromXml(goal, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext())); - - // Assert - Assert.NotNull(plan); - Assert.Equal("Summarize an input, translate to french, and e-mail to John Doe", plan.Description); - - Assert.Equal(4, plan.Steps.Count); - Assert.Collection(plan.Steps, - step => - { - Assert.Equal("SummarizeSkill", step.SkillName); - Assert.Equal("Summarize", step.Name); - }, - step => - { - Assert.Equal("WriterSkill", step.SkillName); - Assert.Equal("Translate", step.Name); - Assert.Equal("French", step.Parameters["language"]); - Assert.True(step.Outputs.Contains("TRANSLATED_SUMMARY")); - }, - step => - { - Assert.Equal("email", step.SkillName); - Assert.Equal("GetEmailAddressAsync", step.Name); - Assert.Equal("John Doe", step.Parameters["input"]); - Assert.True(step.Outputs.Contains("EMAIL_ADDRESS")); - }, - step => - { - Assert.Equal("email", step.SkillName); - Assert.Equal("SendEmailAsync", step.Name); - Assert.Equal("$TRANSLATED_SUMMARY", step.Parameters["input"]); - Assert.Equal("$EMAIL_ADDRESS", step.Parameters["email_address"]); - } - ); - } - - private const string GoalText = "Solve the equation x^2 = 2."; - - [Fact] - public void InvalidPlanExecutePlanReturnsInvalidResult() - { - // Arrange - this.CreateKernelAndFunctionCreateMocks(new(), out var kernel); - var planString = ""; - - // Act - Assert.Throws(() => planString.ToPlanFromXml(GoalText, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext()))); - } - - // Test that contains a #text node in the plan - [Theory] - [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner - - - This is some text - ")] - public void CanCreatePlanWithTextNodes(string goalText, string planText) - { - // Arrange - var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() - { - ("Echo", "MockSkill", "Echo an input", true, "Mock Echo Result"), - }; - this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); - - // Act - var plan = planText.ToPlanFromXml(goalText, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext())); - - // Assert - Assert.NotNull(plan); - Assert.Equal(goalText, plan.Description); - Assert.Equal(1, plan.Steps.Count); - Assert.Equal("MockSkill", plan.Steps[0].SkillName); - Assert.Equal("Echo", plan.Steps[0].Name); - } - - // Test that contains a #text node in the plan - [Theory] - [InlineData(@" - - - - ", true)] - [InlineData(@" - - - - ", false)] - public void CanCreatePlanWithInvalidFunctionNodes(string planText, bool allowMissingFunctions) - { - // Arrange - var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() - { - ("Echo", "MockSkill", "Echo an input", true, "Mock Echo Result"), - }; - this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); - - // Act - if (allowMissingFunctions) - { - // it should not throw - var plan = planText.ToPlanFromXml(string.Empty, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext()), allowMissingFunctions); - - // Assert - Assert.NotNull(plan); - Assert.Equal(2, plan.Steps.Count); - - Assert.Equal("MockSkill", plan.Steps[0].SkillName); - Assert.Equal("Echo", plan.Steps[0].Name); - Assert.Null(plan.Steps[0].Description); - - Assert.Equal(plan.GetType().FullName, plan.Steps[1].SkillName); - Assert.Equal(string.Empty, plan.Steps[1].Name); - Assert.Equal("MockSkill.DoesNotExist", plan.Steps[1].Description); - } - else - { - Assert.Throws(() => planText.ToPlanFromXml(string.Empty, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext()), allowMissingFunctions)); - } - } - - [Theory] - [InlineData("Test the functionFlowRunner", @"Possible result: Test the functionFlowRunner - - - This is some text - ")] - [InlineData("Test the functionFlowRunner", @" - - - This is some text - - - plan end")] - [InlineData("Test the functionFlowRunner", @" - - - This is some text - - - plan end")] - public void CanCreatePlanWithOtherText(string goalText, string planText) - { - // Arrange - var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() - { - ("Echo", "MockSkill", "Echo an input", true, "Mock Echo Result"), - }; - this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); - - // Act - var plan = planText.ToPlanFromXml(goalText, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext())); - - // Assert - Assert.NotNull(plan); - Assert.Equal(goalText, plan.Description); - Assert.Equal(1, plan.Steps.Count); - Assert.Equal("MockSkill", plan.Steps[0].SkillName); - Assert.Equal("Echo", plan.Steps[0].Name); - } - - [Theory] - [InlineData(@" ")] - [InlineData("\n \n")] - [InlineData("\n \n")] - public void CanCreatePlanWithOpenApiPlugin(string planText) - { - // Arrange - var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() - { - ("codesearchresults_post", "CodeSearch", "Echo an input", true, "Mock Echo Result"), - }; - this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); - - // Act - var plan = planText.ToPlanFromXml(string.Empty, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext())); - - // Assert - Assert.NotNull(plan); - Assert.Equal(1, plan.Steps.Count); - Assert.Equal("CodeSearch", plan.Steps[0].SkillName); - Assert.Equal("codesearchresults_post", plan.Steps[0].Name); - } - - // test that a that is not will just get skipped - [Theory] - [InlineData("Test the functionFlowRunner", @" - - Some other tag - - ")] - public void CanCreatePlanWithIgnoredNodes(string goalText, string planText) - { - // Arrange - var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() - { - ("Echo", "MockSkill", "Echo an input", true, "Mock Echo Result"), - }; - this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); - - // Act - var plan = planText.ToPlanFromXml(goalText, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext())); - - // Assert - Assert.NotNull(plan); - Assert.Equal(goalText, plan.Description); - Assert.Equal(2, plan.Steps.Count); - Assert.Equal("MockSkill", plan.Steps[0].SkillName); - Assert.Equal("Echo", plan.Steps[0].Name); - Assert.Equal(0, plan.Steps[1].Steps.Count); - Assert.Equal("MockSkill", plan.Steps[1].SkillName); - Assert.Equal("Echo", plan.Steps[1].Name); - } -} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SequentialPlannerTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SequentialPlannerTests.cs deleted file mode 100644 index c913fc028f75..000000000000 --- a/dotnet/src/Extensions/Extensions.UnitTests/Planning/SequentialPlanner/SequentialPlannerTests.cs +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; - -namespace SemanticKernel.Extensions.UnitTests.Planning.SequentialPlanner; - -public sealed class SequentialPlannerTests -{ - [Theory] - [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] - public async Task ItCanCreatePlanAsync(string goal) - { - // Arrange - var kernel = new Mock(); - kernel.Setup(x => x.Logger).Returns(new Mock().Object); - - var input = new List<(string name, string skillName, string description, bool isSemantic)>() - { - ("SendEmail", "email", "Send an e-mail", false), - ("GetEmailAddress", "email", "Get an e-mail address", false), - ("Translate", "WriterSkill", "Translate something", true), - ("Summarize", "SummarizeSkill", "Summarize something", true) - }; - - var functionsView = new FunctionsView(); - var skills = new Mock(); - foreach (var (name, skillName, description, isSemantic) in input) - { - var functionView = new FunctionView(name, skillName, description, new List(), isSemantic, true); - var mockFunction = CreateMockFunction(functionView); - functionsView.AddFunction(functionView); - - mockFunction.Setup(x => - x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns((context, settings, cancellationToken) => - { - context.Variables.Update("MOCK FUNCTION CALLED"); - return Task.FromResult(context); - }); - - skills.Setup(x => x.GetFunction(It.Is(s => s == skillName), It.Is(s => s == name))) - .Returns(mockFunction.Object); - ISKFunction? outFunc = mockFunction.Object; - skills.Setup(x => x.TryGetFunction(It.Is(s => s == skillName), It.Is(s => s == name), out outFunc)).Returns(true); - } - - skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); - - var expectedFunctions = input.Select(x => x.name).ToList(); - var expectedSkills = input.Select(x => x.skillName).ToList(); - - var context = new SKContext( - new ContextVariables(), - skills.Object, - new Mock().Object - ); - - var returnContext = new SKContext( - new ContextVariables(), - skills.Object, - new Mock().Object - ); - var planString = - @" - - - - - -"; - - returnContext.Variables.Update(planString); - - var mockFunctionFlowFunction = new Mock(); - mockFunctionFlowFunction.Setup(x => x.InvokeAsync( - It.IsAny(), - null, - default - )).Callback( - (c, s, ct) => c.Variables.Update("Hello world!") - ).Returns(() => Task.FromResult(returnContext)); - - // Mock Skills - kernel.Setup(x => x.Skills).Returns(skills.Object); - kernel.Setup(x => x.CreateNewContext()).Returns(context); - - kernel.Setup(x => x.RegisterSemanticFunction( - It.IsAny(), - It.IsAny(), - It.IsAny() - )).Returns(mockFunctionFlowFunction.Object); - - var planner = new Microsoft.SemanticKernel.Planning.SequentialPlanner(kernel.Object); - - // Act - var plan = await planner.CreatePlanAsync(goal, default); - - // Assert - Assert.Equal(goal, plan.Description); - - Assert.Contains( - plan.Steps, - step => - expectedFunctions.Contains(step.Name) && - expectedSkills.Contains(step.SkillName)); - - foreach (var expectedFunction in expectedFunctions) - { - Assert.Contains( - plan.Steps, - step => step.Name == expectedFunction); - } - - foreach (var expectedSkill in expectedSkills) - { - Assert.Contains( - plan.Steps, - step => step.SkillName == expectedSkill); - } - } - - [Fact] - public async Task EmptyGoalThrowsAsync() - { - // Arrange - var kernel = new Mock(); - // kernel.Setup(x => x.Logger).Returns(new Mock().Object); - - var planner = new Microsoft.SemanticKernel.Planning.SequentialPlanner(kernel.Object); - - // Act - await Assert.ThrowsAsync(async () => await planner.CreatePlanAsync("")); - } - - [Fact] - public async Task InvalidXMLThrowsAsync() - { - // Arrange - var kernel = new Mock(); - var skills = new Mock(); - - var functionsView = new FunctionsView(); - skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); - - var planString = "notvalid<"; - var returnContext = new SKContext( - new ContextVariables(planString), - skills.Object, - new Mock().Object - ); - - var context = new SKContext( - new ContextVariables(), - skills.Object, - new Mock().Object - ); - - var mockFunctionFlowFunction = new Mock(); - mockFunctionFlowFunction.Setup(x => x.InvokeAsync( - It.IsAny(), - null, - default - )).Callback( - (c, s, ct) => c.Variables.Update("Hello world!") - ).Returns(() => Task.FromResult(returnContext)); - - // Mock Skills - kernel.Setup(x => x.Skills).Returns(skills.Object); - kernel.Setup(x => x.CreateNewContext()).Returns(context); - - kernel.Setup(x => x.RegisterSemanticFunction( - It.IsAny(), - It.IsAny(), - It.IsAny() - )).Returns(mockFunctionFlowFunction.Object); - - var planner = new Microsoft.SemanticKernel.Planning.SequentialPlanner(kernel.Object); - - // Act - await Assert.ThrowsAsync(async () => await planner.CreatePlanAsync("goal")); - } - - // Method to create Mock objects - private static Mock CreateMockFunction(FunctionView functionView) - { - var mockFunction = new Mock(); - mockFunction.Setup(x => x.Describe()).Returns(functionView); - mockFunction.Setup(x => x.Name).Returns(functionView.Name); - mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); - return mockFunction; - } -} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs deleted file mode 100644 index 2e9712f24d56..000000000000 --- a/dotnet/src/Extensions/Extensions.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; - -namespace SemanticKernel.Extensions.UnitTests.Planning.StepwisePlanner; - -public sealed class ParseResultTests -{ - [Theory] - [InlineData("[FINAL ANSWER] 42", "42")] - [InlineData("[FINAL ANSWER]42", "42")] - [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42", "42")] - [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42\n", "42")] - [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42\n\n", "42")] - [InlineData("I think I have everything I need.\n[FINAL ANSWER]42\n\n\n", "42")] - [InlineData("I think I have everything I need.\n[FINAL ANSWER]\n 42\n\n\n", "42")] - public void WhenInputIsFinalAnswerReturnsFinalAnswer(string input, string expected) - { - // Arrange - var kernel = new Mock(); - kernel.Setup(x => x.Logger).Returns(new Mock().Object); - - var planner = new Microsoft.SemanticKernel.Planning.StepwisePlanner(kernel.Object); - - // Act - var result = planner.ParseResult(input); - - // Assert - Assert.Equal(expected, result.FinalAnswer); - } - - [Theory] - [InlineData("To answer the first part of the question, I need to search for Leo DiCaprio's girlfriend on the web. To answer the second part, I need to find her current age and use a calculator to raise it to the 0.43 power.\n[ACTION]\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"Leo DiCaprio's girlfriend\"}\n}", "Search", "input", "Leo DiCaprio's girlfriend")] - [InlineData("To answer the first part of the question, I need to search the web for Leo DiCaprio's girlfriend. To answer the second part, I need to find her current age and use the calculator tool to raise it to the 0.43 power.\n[ACTION]\n```\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"Leo DiCaprio's girlfriend\"}\n}\n```", "Search", "input", "Leo DiCaprio's girlfriend")] - [InlineData("The web search result is a snippet from a Wikipedia article that says Leo DiCaprio's girlfriend is Camila Morrone, an Argentine-American model and actress. I need to find out her current age, which might be in the same article or another source. I can use the WebSearch.Search function again to search for her name and age.\n\n[ACTION] {\n \"action\": \"WebSearch.Search\",\n \"action_variables\": {\"input\": \"Camila Morrone age\", \"count\": \"1\"}\n}", "WebSearch.Search", "input", - "Camila Morrone age", "count", "1")] - public void ParseActionReturnsAction(string input, string expectedAction, params string[] expectedVariables) - { - Dictionary? expectedDictionary = null; - for (int i = 0; i < expectedVariables.Length; i += 2) - { - expectedDictionary ??= new Dictionary(); - expectedDictionary.Add(expectedVariables[i], expectedVariables[i + 1]); - } - - // Arrange - var kernel = new Mock(); - kernel.Setup(x => x.Logger).Returns(new Mock().Object); - - var planner = new Microsoft.SemanticKernel.Planning.StepwisePlanner(kernel.Object); - - // Act - var result = planner.ParseResult(input); - - // Assert - Assert.Equal(expectedAction, result.Action); - Assert.Equal(expectedDictionary, result.ActionVariables); - } - - // Method to create Mock objects - private static Mock CreateMockFunction(FunctionView functionView) - { - var mockFunction = new Mock(); - mockFunction.Setup(x => x.Describe()).Returns(functionView); - mockFunction.Setup(x => x.Name).Returns(functionView.Name); - mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); - return mockFunction; - } -} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Basic/BasicHttpRetryHandlerTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Basic/BasicHttpRetryHandlerTests.cs new file mode 100644 index 000000000000..94751de1b100 --- /dev/null +++ b/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Basic/BasicHttpRetryHandlerTests.cs @@ -0,0 +1,678 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Reliability.Basic; +using Moq; +using Moq.Protected; +using Xunit; + +namespace SemanticKernel.Extensions.UnitTests.Reliability.Basic; + +public class BasicHttpRetryHandlerTests +{ + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task NoMaxRetryCountCallsOnceForStatusAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = new BasicHttpRetryHandler(new BasicRetryConfig() { MaxRetryCount = 0 }, NullLoggerFactory.Instance); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task ItRetriesOnceOnRetryableStatusAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesOnceOnRetryableExceptionAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task NoMaxRetryCountCallsOnceForExceptionAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig() { MaxRetryCount = 0 }); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task ItRetriesOnceOnTransientStatusWithExponentialBackoffAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig() { UseExponentialBackoff = true }); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesOnceOnRetryableExceptionWithExponentialBackoffAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig() { UseExponentialBackoff = true }); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesExponentiallyWithExponentialBackoffAsync(HttpStatusCode statusCode) + { + // Arrange + var currentTime = DateTimeOffset.UtcNow; + var mockTimeProvider = new Mock(); + var mockDelayProvider = new Mock(); + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(510)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(1015)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(1520)); + using var retry = ConfigureRetryHandler(new BasicRetryConfig() + { + UseExponentialBackoff = true, MaxRetryCount = 3, + MinRetryDelay = TimeSpan.FromMilliseconds(500) + }, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(1000), It.IsAny()), Times.Once); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(2000), It.IsAny()), Times.Once); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesOnceOnTransientStatusCodeWithRetryValueAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig(), null); + using var mockResponse = new HttpResponseMessage() + { + StatusCode = statusCode, + Headers = { RetryAfter = new RetryConditionHeaderValue(new TimeSpan(0, 0, 0, 1)) }, + }; + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + using var testContent = new StringContent("test"); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + Assert.Equal(new TimeSpan(0, 0, 0, 1), response.Headers.RetryAfter?.Delta); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesStatusCustomCountAsync(HttpStatusCode expectedStatus) + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(expectedStatus); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(expectedStatus, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesExceptionsCustomCountAsync(Type expectedException) + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig() { MaxRetryCount = 3 }, null); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(expectedException); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(expectedException, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task NoExceptionNoRetryAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ItDoesNotExecuteOnCancellationTokenAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + var cancellationToken = new CancellationToken(true); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Never(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoestExecuteOnFalseCancellationTokenAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new BasicRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + var cancellationToken = new CancellationToken(false); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ItRetriesWithMinRetryDelayAsync() + { + var BasicRetryConfig = new BasicRetryConfig + { + MinRetryDelay = TimeSpan.FromMilliseconds(500) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime.AddMilliseconds(5)) + .Returns(() => currentTime.AddMilliseconds(510)); + + mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) + .Returns(() => Task.CompletedTask); + + using var retry = ConfigureRetryHandler(BasicRetryConfig, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesWithMaxRetryDelayAsync() + { + var BasicRetryConfig = new BasicRetryConfig + { + MinRetryDelay = TimeSpan.FromMilliseconds(1), + MaxRetryDelay = TimeSpan.FromMilliseconds(500) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime.AddMilliseconds(5)) + .Returns(() => currentTime.AddMilliseconds(505)); + + mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) + .Returns(() => Task.CompletedTask); + + using var retry = ConfigureRetryHandler(BasicRetryConfig, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(2000)) } + }; + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + Assert.Equal(TimeSpan.FromMilliseconds(2000), response.Headers.RetryAfter?.Delta); + } + + [Theory] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.RequestTimeout)] + public async Task ItRetriesWithMaxTotalDelayAsync(HttpStatusCode statusCode) + { + // Arrange + var BasicRetryConfig = new BasicRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(350) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(BasicRetryConfig, mockTimeProvider, mockDelayProvider); + + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(6)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(5)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(6), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Fact] + public async Task ItRetriesFewerWithMaxTotalDelayAsync() + { + // Arrange + var BasicRetryConfig = new BasicRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(100) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(BasicRetryConfig, mockTimeProvider, mockDelayProvider); + + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); // 1 initial, 2 retries, 1 for logging time taken. + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(1)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesFewerWithMaxTotalDelayOnExceptionAsync() + { + // Arrange + var BasicRetryConfig = new BasicRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(100) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(BasicRetryConfig, mockTimeProvider, mockDelayProvider); + var mockHandler = GetHttpMessageHandlerMock(typeof(HttpRequestException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(() => httpClient.GetAsync(new Uri("https://www.microsoft.com"), CancellationToken.None)); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); // 1 initial, 2 retries, 1 for logging time taken. + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(1)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItRetriesOnRetryableStatusCodesAsync() + { + // Arrange + var config = new BasicRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; + using var retry = ConfigureRetryHandler(config); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.Unauthorized); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ItDoesNotRetryOnNonRetryableStatusCodesAsync() + { + // Arrange + var config = new BasicRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; + using var retry = ConfigureRetryHandler(config); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesOnRetryableExceptionsAsync() + { + // Arrange + var config = new BasicRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; + using var retry = ConfigureRetryHandler(config); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(typeof(InvalidOperationException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoesNotRetryOnNonRetryableExceptionsAsync() + { + // Arrange + var config = new BasicRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; + using var retry = ConfigureRetryHandler(config); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(typeof(ArgumentException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + private static BasicHttpRetryHandler ConfigureRetryHandler(BasicRetryConfig? config = null, + Mock? timeProvider = null, Mock? delayProvider = null) + { + delayProvider ??= new Mock(); + timeProvider ??= new Mock(); + + var retry = new BasicHttpRetryHandler(config ?? new BasicRetryConfig(), null, delayProvider.Object, timeProvider.Object); + return retry; + } + + private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(mockResponse); + return mockHandler; + } + + private static Mock GetHttpMessageHandlerMock(Type exceptionType) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ThrowsAsync(Activator.CreateInstance(exceptionType) as Exception); + return mockHandler; + } +} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Basic/BasicRetryConfigTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Basic/BasicRetryConfigTests.cs new file mode 100644 index 000000000000..f210722cdf34 --- /dev/null +++ b/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Basic/BasicRetryConfigTests.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Reliability.Basic; +using Xunit; + +namespace SemanticKernel.Extensions.UnitTests.Reliability.Basic; + +/// +/// Unit tests of . +/// +public class BasicRetryConfigTests +{ + [Fact] + public async Task NegativeMaxRetryCountThrowsAsync() + { + // Act + await Assert.ThrowsAsync(() => + { + var BasicRetryConfig = new BasicRetryConfig() { MaxRetryCount = -1 }; + return Task.CompletedTask; + }); + } + + [Fact] + public void SetDefaultBasicRetryConfig() + { + // Arrange + var builder = new KernelBuilder(); + var basicRetryConfig = new BasicRetryConfig() { MaxRetryCount = 3 }; + builder.WithRetryBasic(basicRetryConfig); + + // Act + var kernel = builder.Build(); + + // Assert + Assert.IsType(kernel.HttpHandlerFactory); + var httpHandlerFactory = kernel.HttpHandlerFactory as BasicHttpRetryHandlerFactory; + Assert.NotNull(httpHandlerFactory); + Assert.Equal(basicRetryConfig, httpHandlerFactory.Config); + } + + [Fact] + public void SetDefaultBasicRetryConfigToDefaultIfNotSet() + { + // Arrange + var retryConfig = new BasicRetryConfig(); + var builder = new KernelBuilder(); + builder.WithRetryBasic(retryConfig); + + // Act + var kernel = builder.Build(); + + // Assert + Assert.IsType(kernel.HttpHandlerFactory); + var httpHandlerFactory = kernel.HttpHandlerFactory as BasicHttpRetryHandlerFactory; + Assert.NotNull(httpHandlerFactory); + Assert.Equal(retryConfig.MaxRetryCount, httpHandlerFactory.Config.MaxRetryCount); + Assert.Equal(retryConfig.MaxRetryDelay, httpHandlerFactory.Config.MaxRetryDelay); + Assert.Equal(retryConfig.MinRetryDelay, httpHandlerFactory.Config.MinRetryDelay); + Assert.Equal(retryConfig.MaxTotalRetryTime, httpHandlerFactory.Config.MaxTotalRetryTime); + Assert.Equal(retryConfig.UseExponentialBackoff, httpHandlerFactory.Config.UseExponentialBackoff); + Assert.Equal(retryConfig.RetryableStatusCodes, httpHandlerFactory.Config.RetryableStatusCodes); + Assert.Equal(retryConfig.RetryableExceptionTypes, httpHandlerFactory.Config.RetryableExceptionTypes); + } +} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Polly/PollyHttpRetryHandlerTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Polly/PollyHttpRetryHandlerTests.cs new file mode 100644 index 000000000000..a42f5f052959 --- /dev/null +++ b/dotnet/src/Extensions/Extensions.UnitTests/Reliability/Polly/PollyHttpRetryHandlerTests.cs @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Reliability.Polly; +using Moq; +using Moq.Protected; +using Polly; +using Polly.Utilities; +using Xunit; + +namespace SemanticKernel.Extensions.UnitTests.Reliability.Polly; + +public sealed class PollyHttpRetryHandlerTests : IDisposable +{ + public PollyHttpRetryHandlerTests() + { + SystemClock.SleepAsync = (_, _) => Task.CompletedTask; + SystemClock.Sleep = (_, _) => { }; + } + + public void Dispose() + { + SystemClock.Reset(); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task CustomPolicyNoOpShouldNotAvoidSendRequestsAsync(HttpStatusCode statusCode) + { + // Arrange + var asyncPolicy = Policy.NoOpAsync(); + var (mockLoggerFactory, mockLogger) = GetLoggerMocks(); + using var retry = new PollyHttpRetryHandler(asyncPolicy); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task CustomPolicyStatusDontMatchNeverTriggersAsync(HttpStatusCode statusCode) + { + // Arrange + var asyncPolicy = Policy + .HandleResult(result => result.StatusCode != statusCode) + .WaitAndRetryAsync( + retryCount: 1, + sleepDurationProvider: (retryTimes) => TimeSpan.FromMilliseconds(10)); + + var (mockLoggerFactory, mockLogger) = GetLoggerMocks(); + using var retry = new PollyHttpRetryHandler(asyncPolicy); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout, HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.ServiceUnavailable, HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.GatewayTimeout, HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.TooManyRequests, HttpStatusCode.TooManyRequests)] + public async Task CustomPolicyRetryStatusShouldTriggerRetrialsAsync(HttpStatusCode statusCode, HttpStatusCode retryStatusCode) + { + // Arrange + var retryCount = 3; + var asyncPolicy = Policy + .HandleResult(result => result.StatusCode == retryStatusCode) + .WaitAndRetryAsync( + retryCount, + (retryNumber) => TimeSpan.FromMilliseconds(10)); + + var (mockLoggerFactory, mockLogger) = GetLoggerMocks(); + using var retry = new PollyHttpRetryHandler(asyncPolicy); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + var expectedSendAsyncTimes = (statusCode == retryStatusCode) + ? retryCount + 1 + : 1; + + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(expectedSendAsyncTimes), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(ApplicationException), typeof(HttpRequestException))] + [InlineData(typeof(HttpRequestException), typeof(HttpRequestException))] + public async Task CustomPolicyRetryExceptionsShouldTriggerRetrialsAsync(Type exceptionType, Type retryExceptionType) + { + // Arrange + var retryCount = 1; + var asyncPolicy = Policy.Handle(exception => exception.GetType() == retryExceptionType) + .WaitAndRetryAsync( + retryCount, + (retryNumber) => TimeSpan.FromMilliseconds(10)); + + var (mockLoggerFactory, mockLogger) = GetLoggerMocks(); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + using var retry = new PollyHttpRetryHandler(asyncPolicy); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + var expectedSendAsyncTimes = (exceptionType == retryExceptionType) + ? retryCount + 1 + : 1; + + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(expectedSendAsyncTimes), ItExpr.IsAny(), ItExpr.IsAny()); + } + + private static (Mock, Mock) GetLoggerMocks() + { + var mockLoggerFactory = new Mock(); + var mockLogger = new Mock(); + mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny())).Returns(mockLogger.Object); + + return (mockLoggerFactory, mockLogger); + } + + private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(mockResponse); + return mockHandler; + } + + private static Mock GetHttpMessageHandlerMock(Type exceptionType) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ThrowsAsync(Activator.CreateInstance(exceptionType) as Exception); + return mockHandler; + } +} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/CodeBlockTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/CodeBlockTests.cs new file mode 100644 index 000000000000..c6a35e87ffbe --- /dev/null +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/CodeBlockTests.cs @@ -0,0 +1,369 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; +using Moq; +using Xunit; + +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt.Blocks; + +public class CodeBlockTests +{ + private readonly Mock _functions; + private readonly ILoggerFactory _logger = NullLoggerFactory.Instance; + private readonly Mock _functionRunner = new(); + + public CodeBlockTests() + { + this._functions = new Mock(); + } + + [Fact] + public async Task ItThrowsIfAFunctionDoesntExistAsync() + { + // Arrange + var functionRunner = new Mock(); + var context = new SKContext(this._functionRunner.Object); + var target = new CodeBlock("functionName", this._logger); + + this._functionRunner.Setup(r => r.RunAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((pluginName, functionName, variables, cancellationToken) => + { + throw new SKException("No function was found"); + }); + + // Act & Assert + await Assert.ThrowsAsync(() => target.RenderCodeAsync(context)); + } + + [Fact] + public async Task ItThrowsIfAFunctionCallThrowsAsync() + { + // Arrange + var context = new SKContext(this._functionRunner.Object, functions: this._functions.Object); + var function = new Mock(); + function + .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Throws(new RuntimeWrappedException("error")); + + this.MockFunctionRunner(function.Object); + + var target = new CodeBlock("functionName", this._logger); + + // Act & Assert + await Assert.ThrowsAsync(() => target.RenderCodeAsync(context)); + } + + [Fact] + public void ItHasTheCorrectType() + { + // Act + var target = new CodeBlock("", NullLoggerFactory.Instance); + + // Assert + Assert.Equal(BlockTypes.Code, target.Type); + } + + [Fact] + public void ItTrimsSpaces() + { + // Act + Assert + Assert.Equal("aa", new CodeBlock(" aa ", NullLoggerFactory.Instance).Content); + } + + [Fact] + public void ItChecksValidityOfInternalBlocks() + { + // Arrange + var validBlock1 = new FunctionIdBlock("x"); + var validBlock2 = new ValBlock("''"); + var invalidBlock = new VarBlock(""); + + // Act + var codeBlock1 = new CodeBlock(new List { validBlock1, validBlock2 }, "", NullLoggerFactory.Instance); + var codeBlock2 = new CodeBlock(new List { validBlock1, invalidBlock }, "", NullLoggerFactory.Instance); + + // Assert + Assert.True(codeBlock1.IsValid(out _)); + Assert.False(codeBlock2.IsValid(out _)); + } + + [Fact] + public void ItRequiresAValidFunctionCall() + { + // Arrange + var funcId = new FunctionIdBlock("funcName"); + var valBlock = new ValBlock("'value'"); + var varBlock = new VarBlock("$var"); + var namedArgBlock = new NamedArgBlock("varName='foo'"); + + // Act + var codeBlock1 = new CodeBlock(new List { funcId, valBlock }, "", NullLoggerFactory.Instance); + var codeBlock2 = new CodeBlock(new List { funcId, varBlock }, "", NullLoggerFactory.Instance); + var codeBlock3 = new CodeBlock(new List { funcId, funcId }, "", NullLoggerFactory.Instance); + var codeBlock4 = new CodeBlock(new List { funcId, varBlock, varBlock }, "", NullLoggerFactory.Instance); + var codeBlock5 = new CodeBlock(new List { funcId, varBlock, namedArgBlock }, "", NullLoggerFactory.Instance); + var codeBlock6 = new CodeBlock(new List { varBlock, valBlock }, "", NullLoggerFactory.Instance); + var codeBlock7 = new CodeBlock(new List { namedArgBlock }, "", NullLoggerFactory.Instance); + + // Assert + Assert.True(codeBlock1.IsValid(out _)); + Assert.True(codeBlock2.IsValid(out _)); + + // Assert - Can't pass a function to a function + Assert.False(codeBlock3.IsValid(out var errorMessage3)); + Assert.Equal("The first arg of a function must be a quoted string, variable or named argument", errorMessage3); + + // Assert - Can't pass more than one unnamed param + Assert.False(codeBlock4.IsValid(out var errorMessage4)); + Assert.Equal("Functions only support named arguments after the first argument. Argument 2 is not named.", errorMessage4); + + // Assert - Can pass one unnamed param and named args + Assert.True(codeBlock5.IsValid(out var errorMessage5)); + Assert.Empty(errorMessage5); + + // Assert - Can't use > 1 block if not a function call + Assert.False(codeBlock6.IsValid(out var errorMessage6)); + Assert.Equal("Unexpected second token found: 'value'", errorMessage6); + + // Assert - Can't use a named argument without a function block + Assert.False(codeBlock7.IsValid(out var errorMessage7)); + Assert.Equal("Unexpected named argument found. Expected function name first.", errorMessage7); + } + + [Fact] + public async Task ItRendersCodeBlockConsistingOfJustAVarBlock1Async() + { + // Arrange + var variables = new ContextVariables { ["varName"] = "foo" }; + var context = new SKContext(this._functionRunner.Object, variables, functions: this._functions.Object); + + // Act + var codeBlock = new CodeBlock("$varName", NullLoggerFactory.Instance); + var result = await codeBlock.RenderCodeAsync(context); + + // Assert + Assert.Equal("foo", result); + } + + [Fact] + public async Task ItRendersCodeBlockConsistingOfJustAVarBlock2Async() + { + // Arrange + var variables = new ContextVariables { ["varName"] = "bar" }; + var context = new SKContext(this._functionRunner.Object, variables, functions: this._functions.Object); + var varBlock = new VarBlock("$varName"); + + // Act + var codeBlock = new CodeBlock(new List { varBlock }, "", NullLoggerFactory.Instance); + var result = await codeBlock.RenderCodeAsync(context); + + // Assert + Assert.Equal("bar", result); + } + + [Fact] + public async Task ItRendersCodeBlockConsistingOfJustAValBlock1Async() + { + // Arrange + var context = new SKContext(this._functionRunner.Object); + + // Act + var codeBlock = new CodeBlock("'ciao'", NullLoggerFactory.Instance); + var result = await codeBlock.RenderCodeAsync(context); + + // Assert + Assert.Equal("ciao", result); + } + + [Fact] + public async Task ItRendersCodeBlockConsistingOfJustAValBlock2Async() + { + // Arrange + var kernel = new Mock(); + var context = new SKContext(this._functionRunner.Object); + var valBlock = new ValBlock("'arrivederci'"); + + // Act + var codeBlock = new CodeBlock(new List { valBlock }, "", NullLoggerFactory.Instance); + var result = await codeBlock.RenderCodeAsync(context); + + // Assert + Assert.Equal("arrivederci", result); + } + + [Fact] + public async Task ItInvokesFunctionCloningAllVariablesAsync() + { + // Arrange + const string Func = "funcName"; + const string Plugin = "pluginName"; + + var variables = new ContextVariables { ["input"] = "zero", ["var1"] = "uno", ["var2"] = "due" }; + var context = new SKContext(this._functionRunner.Object, variables, functions: this._functions.Object); + var funcId = new FunctionIdBlock(Func); + + var canary0 = string.Empty; + var canary1 = string.Empty; + var canary2 = string.Empty; + var function = new Mock(); + function + .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((context, _, _) => + { + canary0 = context!.Variables["input"]; + canary1 = context.Variables["var1"]; + canary2 = context.Variables["var2"]; + + context.Variables["input"] = "overridden"; + context.Variables["var1"] = "overridden"; + context.Variables["var2"] = "overridden"; + }) + .ReturnsAsync((SKContext inputcontext, object _, CancellationToken _) => new FunctionResult(Func, Plugin, inputcontext)); + + this.MockFunctionRunner(function.Object); + + // Act + var codeBlock = new CodeBlock(new List { funcId }, "", NullLoggerFactory.Instance); + string result = await codeBlock.RenderCodeAsync(context); + + // Assert - Values are received + Assert.Equal("zero", canary0); + Assert.Equal("uno", canary1); + Assert.Equal("due", canary2); + + // Assert - Original context is intact + Assert.Equal("zero", variables["input"]); + Assert.Equal("uno", variables["var1"]); + Assert.Equal("due", variables["var2"]); + } + + [Fact] + public async Task ItInvokesFunctionWithCustomVariableAsync() + { + // Arrange + const string Func = "funcName"; + const string Plugin = "pluginName"; + const string Var = "varName"; + const string VarValue = "varValue"; + + var variables = new ContextVariables { [Var] = VarValue }; + var context = new SKContext(this._functionRunner.Object, variables, functions: this._functions.Object); + var funcId = new FunctionIdBlock(Func); + var varBlock = new VarBlock($"${Var}"); + + var canary = string.Empty; + var function = new Mock(); + function + .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((context, _, _) => + { + canary = context!.Variables["input"]; + }) + .ReturnsAsync((SKContext inputcontext, object _, CancellationToken _) => new FunctionResult(Func, Plugin, inputcontext)); + + this.MockFunctionRunner(function.Object); + + // Act + var codeBlock = new CodeBlock(new List { funcId, varBlock }, "", NullLoggerFactory.Instance); + string result = await codeBlock.RenderCodeAsync(context); + + // Assert + Assert.Equal(VarValue, result); + Assert.Equal(VarValue, canary); + } + + [Fact] + public async Task ItInvokesFunctionWithCustomValueAsync() + { + // Arrange + const string Func = "funcName"; + const string Plugin = "pluginName"; + const string Value = "value"; + + var context = new SKContext(this._functionRunner.Object, variables: null, functions: this._functions.Object); + var funcId = new FunctionIdBlock(Func); + var valBlock = new ValBlock($"'{Value}'"); + + var canary = string.Empty; + var function = new Mock(); + function + .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((context, _, _) => + { + canary = context!.Variables["input"]; + }) + .ReturnsAsync((SKContext inputcontext, object _, CancellationToken _) => new FunctionResult(Func, Plugin, inputcontext)); + + this.MockFunctionRunner(function.Object); + + // Act + var codeBlock = new CodeBlock(new List { funcId, valBlock }, "", NullLoggerFactory.Instance); + string result = await codeBlock.RenderCodeAsync(context); + + // Assert + Assert.Equal(Value, result); + Assert.Equal(Value, canary); + } + + [Fact] + public async Task ItInvokesFunctionWithNamedArgsAsync() + { + // Arrange + const string Func = "funcName"; + const string Plugin = "pluginName"; + const string Value = "value"; + const string FooValue = "bar"; + const string BobValue = "bob's value"; + + var variables = new ContextVariables(); + variables.Set("bob", BobValue); + variables.Set("input", Value); + var context = new SKContext(this._functionRunner.Object, variables: variables, functions: this._functions.Object); + var funcId = new FunctionIdBlock(Func); + var namedArgBlock1 = new NamedArgBlock($"foo='{FooValue}'"); + var namedArgBlock2 = new NamedArgBlock("baz=$bob"); + + var foo = string.Empty; + var baz = string.Empty; + var function = new Mock(); + function + .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((context, _, _) => + { + foo = context!.Variables["foo"]; + baz = context!.Variables["baz"]; + }) + .ReturnsAsync((SKContext inputcontext, object _, CancellationToken _) => new FunctionResult(Func, Plugin, inputcontext)); + + this.MockFunctionRunner(function.Object); + + // Act + var codeBlock = new CodeBlock(new List { funcId, namedArgBlock1, namedArgBlock2 }, "", NullLoggerFactory.Instance); + string result = await codeBlock.RenderCodeAsync(context); + + // Assert + Assert.Equal(FooValue, foo); + Assert.Equal(BobValue, baz); + Assert.Equal(Value, result); + } + + private void MockFunctionRunner(ISKFunction function) + { + this._functionRunner.Setup(r => r.RunAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((pluginName, functionName, variables, cancellationToken) => + { + var context = new SKContext(this._functionRunner.Object, variables); + return function.InvokeAsync(context, null, cancellationToken); + }); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/FunctionIdBlockTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/FunctionIdBlockTests.cs similarity index 83% rename from dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/FunctionIdBlockTests.cs rename to dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/FunctionIdBlockTests.cs index d4839cb33012..b5b7e884da88 100644 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/FunctionIdBlockTests.cs +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/FunctionIdBlockTests.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; using Xunit; -namespace SemanticKernel.UnitTests.TemplateEngine.Blocks; +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt.Blocks; public class FunctionIdBlockTests { @@ -13,7 +13,7 @@ public class FunctionIdBlockTests public void ItHasTheCorrectType() { // Act - var target = new FunctionIdBlock("", NullLogger.Instance); + var target = new FunctionIdBlock("", NullLoggerFactory.Instance); // Assert Assert.Equal(BlockTypes.FunctionId, target.Type); @@ -23,7 +23,7 @@ public void ItHasTheCorrectType() public void ItTrimsSpaces() { // Act + Assert - Assert.Equal("aa", new FunctionIdBlock(" aa ", NullLogger.Instance).Content); + Assert.Equal("aa", new FunctionIdBlock(" aa ", NullLoggerFactory.Instance).Content); } [Theory] @@ -85,8 +85,8 @@ public void ItAllowsOnlyOneDot() { // Arrange var target1 = new FunctionIdBlock("functionName"); - var target2 = new FunctionIdBlock("skillName.functionName"); - Assert.Throws(() => new FunctionIdBlock("foo.skillName.functionName")); + var target2 = new FunctionIdBlock("pluginName.functionName"); + Assert.Throws(() => new FunctionIdBlock("foo.pluginName.functionName")); // Act + Assert Assert.True(target1.IsValid(out _)); diff --git a/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/NamedArgBlockTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/NamedArgBlockTests.cs new file mode 100644 index 000000000000..507f0af0c178 --- /dev/null +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/NamedArgBlockTests.cs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; +using Xunit; + +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt.Blocks; + +public class NamedArgBlockTests +{ + [Fact] + public void ItHasTheCorrectType() + { + // Act + var target = new NamedArgBlock("a=$b", NullLoggerFactory.Instance); + + // Assert + Assert.Equal(BlockTypes.NamedArg, target.Type); + } + + [Theory] + [InlineData(" a=$b ", "a=$b")] + [InlineData(" a = $b ", "a=$b")] + [InlineData(" a=\"b\" ", "a=\"b\"")] + [InlineData(" a = \"b\" ", "a=\"b\"")] + [InlineData(" a='b' ", "a='b'")] + [InlineData("a = 'b' ", "a='b'")] + public void ItTrimsSpaces(string input, string expected) + { + // Act + Assert + Assert.Equal(expected, new NamedArgBlock(input, NullLoggerFactory.Instance).Content); + } + + [Theory] + [InlineData("0='val'", true)] + [InlineData("1='val'", true)] + [InlineData("a='val'", true)] + [InlineData("_='val'", true)] + [InlineData("01='val'", true)] + [InlineData("01a='val'", true)] + [InlineData("a01='val'", true)] + [InlineData("_0='val'", true)] + [InlineData("a01_='val'", true)] + [InlineData("_a01='val'", true)] + [InlineData(".='val'", false)] + [InlineData("-='val'", false)] + [InlineData("a b='val'", false)] + [InlineData("a\nb='val'", false)] + [InlineData("a\tb='val'", false)] + [InlineData("a\rb='val'", false)] + [InlineData("a.b='val'", false)] + [InlineData("a,b='val'", false)] + [InlineData("a-b='val'", false)] + [InlineData("a+b='val'", false)] + [InlineData("a~b='val'", false)] + [InlineData("a`b='val'", false)] + [InlineData("a!b='val'", false)] + [InlineData("a@b='val'", false)] + [InlineData("a#b='val'", false)] + [InlineData("a$b='val'", false)] + [InlineData("a%b='val'", false)] + [InlineData("a^b='val'", false)] + [InlineData("a*b='val'", false)] + [InlineData("a(b='val'", false)] + [InlineData("a)b='val'", false)] + [InlineData("a|b='val'", false)] + [InlineData("a{b='val'", false)] + [InlineData("a}b='val'", false)] + [InlineData("a[b='val'", false)] + [InlineData("a]b='val'", false)] + [InlineData("a:b='val'", false)] + [InlineData("a;b='val'", false)] + [InlineData("a'b='val'", false)] + [InlineData("a\"b='val'", false)] + [InlineData("ab='val'", false)] + [InlineData("a/b='val'", false)] + [InlineData("a\\b='val'", false)] + [InlineData("a ='val'", true)] + public void ArgNameAllowsUnderscoreLettersAndDigits(string name, bool isValid) + { + // Arrange + var target = new NamedArgBlock($" {name} "); + + // Act + Assert + Assert.Equal(isValid, target.IsValid(out _)); + } + + [Theory] + [InlineData("name ='value'")] + [InlineData("name= 'value'")] + public void AllowsAnyNumberOfSpacesBeforeAndAfterEqualSign(string input) + { + // Arrange + var target = new NamedArgBlock(input); + + // Act + Assert + Assert.True(target.IsValid(out _)); + Assert.Equal("name", target.Name); + Assert.Equal("value", target.GetValue(null)); + } + + [Fact] + public void ArgValueNeedsQuoteOrDollarSignPrefix() + { + // Arrange + var target = new NamedArgBlock("a=b"); + + // Act + Assert + Assert.False(target.IsValid(out var error)); + Assert.Equal("There was an issue with the named argument value for 'a': A value must have single quotes or double quotes on both sides", error); + } + + [Fact] + public void ArgNameShouldBeNonEmpty() + { + // Arrange + var target = new NamedArgBlock("='b'"); + + // Act + Assert + Assert.False(target.IsValid(out var error)); + Assert.Equal("A named argument must have a name", error); + } + + [Fact] + public void ArgValueShouldBeNonEmpty() + { + Assert.Throws(() => new NamedArgBlock("a=")); + } + + [Theory] + [InlineData("!@#^='b'", "The argument name '!@#^' contains invalid characters. Only alphanumeric chars and underscore are allowed.")] + [InlineData("a=$!@#^", "There was an issue with the named argument value for 'a': The variable name '!@#^' contains invalid characters. Only alphanumeric chars and underscore are allowed.")] + public void ArgNameAndVariableShouldBeAValidVariableName(string content, string expectedError) + { + // Arrange + var target = new NamedArgBlock(content); + + // Act + Assert + Assert.False(target.IsValid(out var error)); + Assert.Equal(expectedError, error); + } + + [Theory] + [InlineData("0='val'", true)] + [InlineData("0=\"val\"", true)] + [InlineData("0='val\"", false)] + [InlineData("0=\"val'", false)] + [InlineData("0= 'val'", true)] + public void ArgValueAllowsConsistentlyQuotedValues(string name, bool isValid) + { + // Arrange + var target = new NamedArgBlock($" {name} "); + + // Act + Assert + Assert.Equal(isValid, target.IsValid(out _)); + } + + [Theory] + [InlineData("0", true)] + [InlineData("1", true)] + [InlineData("a", true)] + [InlineData("_", true)] + [InlineData("01", true)] + [InlineData("01a", true)] + [InlineData("a01", true)] + [InlineData("_0", true)] + [InlineData("a01_", true)] + [InlineData("_a01", true)] + [InlineData(".", false)] + [InlineData("-", false)] + [InlineData("a b", false)] + [InlineData("a\nb", false)] + [InlineData("a\tb", false)] + [InlineData("a\rb", false)] + [InlineData("a.b", false)] + [InlineData("a,b", false)] + [InlineData("a-b", false)] + [InlineData("a+b", false)] + [InlineData("a~b", false)] + [InlineData("a`b", false)] + [InlineData("a!b", false)] + [InlineData("a@b", false)] + [InlineData("a#b", false)] + [InlineData("a$b", false)] + [InlineData("a%b", false)] + [InlineData("a^b", false)] + [InlineData("a*b", false)] + [InlineData("a(b", false)] + [InlineData("a)b", false)] + [InlineData("a|b", false)] + [InlineData("a{b", false)] + [InlineData("a}b", false)] + [InlineData("a[b", false)] + [InlineData("a]b", false)] + [InlineData("a:b", false)] + [InlineData("a;b", false)] + [InlineData("a'b", false)] + [InlineData("a\"b", false)] + [InlineData("ab", false)] + [InlineData("a/b", false)] + [InlineData("a\\b", false)] + public void ArgValueAllowsVariablesWithUnderscoreLettersAndDigits(string name, bool isValid) + { + // Arrange + var target = new NamedArgBlock($"a=${name}"); + var variables = new ContextVariables { [name] = "value" }; + + // Act + Assert + Assert.Equal(isValid, target.IsValid(out _)); + } + + [Fact] + public void ItRequiresOneEquals() + { + // Arrange + var target1 = new NamedArgBlock("a='b'"); + var target2 = new NamedArgBlock("a=$b"); + var target3 = new NamedArgBlock("a=\"b\""); + Assert.Throws(() => new NamedArgBlock("foo")); + Assert.Throws(() => new NamedArgBlock("foo=$bar=$baz")); + + // Act + Assert + Assert.True(target1.IsValid(out _)); + Assert.True(target2.IsValid(out _)); + Assert.True(target3.IsValid(out _)); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/TextBlockTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/TextBlockTests.cs similarity index 91% rename from dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/TextBlockTests.cs rename to dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/TextBlockTests.cs index 2660eac50bb5..278efa5e108d 100644 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/TextBlockTests.cs +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/TextBlockTests.cs @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -using Microsoft.SemanticKernel.TemplateEngine.Blocks; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; using Xunit; -namespace SemanticKernel.UnitTests.TemplateEngine.Blocks; +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt.Blocks; public class TextBlockTests { diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/ValBlockTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/ValBlockTests.cs similarity index 92% rename from dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/ValBlockTests.cs rename to dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/ValBlockTests.cs index 1add88f9528c..90e5c10d9dbb 100644 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/ValBlockTests.cs +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/ValBlockTests.cs @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -using Microsoft.SemanticKernel.TemplateEngine.Blocks; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; using Xunit; -namespace SemanticKernel.UnitTests.TemplateEngine.Blocks; +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt.Blocks; public class ValBlockTests { diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/VarBlockTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/VarBlockTests.cs similarity index 92% rename from dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/VarBlockTests.cs rename to dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/VarBlockTests.cs index 1d777488cb6a..2e7e2d53eda9 100644 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/VarBlockTests.cs +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/Blocks/VarBlockTests.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; using Xunit; -namespace SemanticKernel.UnitTests.TemplateEngine.Blocks; +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt.Blocks; public class VarBlockTests { @@ -96,8 +96,7 @@ public void ItThrowsIfTheVarNameIsEmpty() var target = new VarBlock(" $ "); // Act + Assert - var ex = Assert.Throws(() => target.Render(variables)); - Assert.Equal(TemplateException.ErrorCodes.SyntaxError, ex.ErrorCode); + Assert.Throws(() => target.Render(variables)); } [Theory] diff --git a/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/CodeTokenizerTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/CodeTokenizerTests.cs new file mode 100644 index 000000000000..34afa3d0e35e --- /dev/null +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/CodeTokenizerTests.cs @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine.Basic; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; +using Xunit; + +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt; + +public class CodeTokenizerTests +{ + private readonly CodeTokenizer _target; + + public CodeTokenizerTests() + { + this._target = new CodeTokenizer(); + } + + [Fact] + public void ItParsesEmptyText() + { + // Act + Assert + Assert.Empty(this._target.Tokenize(null)); + Assert.Empty(this._target.Tokenize("")); + Assert.Empty(this._target.Tokenize(" ")); + Assert.Empty(this._target.Tokenize(" \n ")); + } + + [Theory] + [InlineData("$", "$")] + [InlineData(" $ ", "$")] + [InlineData("$foo", "$foo")] + [InlineData("$foo ", "$foo")] + [InlineData(" $foo", "$foo")] + [InlineData(" $bar ", "$bar")] + public void ItParsesVarBlocks(string template, string content) + { + // Act + var blocks = this._target.Tokenize(template); + + // Assert + Assert.Single(blocks); + Assert.Equal(content, blocks[0].Content); + Assert.Equal(BlockTypes.Variable, blocks[0].Type); + } + + [Theory] + [InlineData("'", "'")] + [InlineData(" \" ", "\"")] + [InlineData("'foo'", "'foo'")] + [InlineData("'foo' ", "'foo'")] + [InlineData(" 'foo'", "'foo'")] + [InlineData(" \"bar\" ", "\"bar\"")] + public void ItParsesValBlocks(string template, string content) + { + // Act + var blocks = this._target.Tokenize(template); + + // Assert + Assert.Single(blocks); + Assert.Equal(content, blocks[0].Content); + Assert.Equal(BlockTypes.Value, blocks[0].Type); + } + + [Theory] + [InlineData("f", "f")] + [InlineData(" x ", "x")] + [InlineData("foo", "foo")] + [InlineData("fo.o ", "fo.o")] + [InlineData(" f.oo", "f.oo")] + [InlineData(" bar ", "bar")] + public void ItParsesFunctionIdBlocks(string template, string content) + { + // Act + var blocks = this._target.Tokenize(template); + + // Assert + Assert.Single(blocks); + Assert.Equal(content, blocks[0].Content); + Assert.Equal(BlockTypes.FunctionId, blocks[0].Type); + } + + [Fact] + public void ItParsesFunctionCalls() + { + // Arrange + var template1 = "x.y $foo"; + var template2 = "xy $foo"; + var template3 = "xy '$value'"; + + // Act + var blocks1 = this._target.Tokenize(template1); + var blocks2 = this._target.Tokenize(template2); + var blocks3 = this._target.Tokenize(template3); + + // Assert + Assert.Equal(2, blocks1.Count); + Assert.Equal(2, blocks2.Count); + Assert.Equal(2, blocks3.Count); + + Assert.Equal("x.y", blocks1[0].Content); + Assert.Equal("xy", blocks2[0].Content); + Assert.Equal("xy", blocks3[0].Content); + + Assert.Equal(BlockTypes.FunctionId, blocks1[0].Type); + Assert.Equal(BlockTypes.FunctionId, blocks2[0].Type); + Assert.Equal(BlockTypes.FunctionId, blocks3[0].Type); + + Assert.Equal("$foo", blocks1[1].Content); + Assert.Equal("$foo", blocks2[1].Content); + Assert.Equal("'$value'", blocks3[1].Content); + + Assert.Equal(BlockTypes.Variable, blocks1[1].Type); + Assert.Equal(BlockTypes.Variable, blocks2[1].Type); + Assert.Equal(BlockTypes.Value, blocks3[1].Type); + } + + [Fact] + public void ItParsesMultiNamedArgFunctionCalls() + { + // Arrange + var template1 = "x.y first=$foo second='bar'"; + var parameters = new ContextVariables(); + parameters.Set("foo", "fooValue"); + + // Act + var blocks1 = this._target.Tokenize(template1); + + // Assert + Assert.Equal(3, blocks1.Count); + + var firstBlock = blocks1[0]; + var secondBlock = blocks1[1] as NamedArgBlock; + var thirdBlock = blocks1[2] as NamedArgBlock; + + Assert.Equal("x.y", firstBlock.Content); + Assert.Equal(BlockTypes.FunctionId, firstBlock.Type); + + Assert.Equal("first=$foo", secondBlock?.Content); + Assert.Equal(BlockTypes.NamedArg, secondBlock?.Type); + Assert.Equal("first", secondBlock?.Name); + Assert.Equal("fooValue", secondBlock?.GetValue(parameters)); + + Assert.Equal("second='bar'", thirdBlock?.Content); + Assert.Equal(BlockTypes.NamedArg, thirdBlock?.Type); + Assert.Equal("second", thirdBlock?.Name); + Assert.Equal("bar", thirdBlock?.GetValue(parameters)); + } + + [Fact] + public void ItSupportsEscaping() + { + // Arrange + var template = "func 'f\\'oo'"; + + // Act + var blocks = this._target.Tokenize(template); + + // Assert + Assert.Equal(2, blocks.Count); + Assert.Equal("func", blocks[0].Content); + Assert.Equal("'f\'oo'", blocks[1].Content); + } + + [Fact] + public void ItSupportsEscapingNamedArgs() + { + // Arrange + var template = "func name='f\\'oo'"; + + // Act + var blocks = this._target.Tokenize(template); + + // Assert + Assert.Equal(2, blocks.Count); + Assert.Equal("func", blocks[0].Content); + Assert.Equal("name='f\'oo'", blocks[1].Content); + var namedArg = blocks[1] as NamedArgBlock; + Assert.NotNull(namedArg); + Assert.Equal("f'oo", namedArg.GetValue(null)); + } + + [Fact] + public void ItSupportsSpacesInNamedArguments() + { + // Arrange + var template = "func name = 'foo'"; + + // Act + var blocks = this._target.Tokenize(template); + + // Assert + Assert.Equal(2, blocks.Count); + Assert.Equal("func", blocks[0].Content); + Assert.Equal("name='foo'", blocks[1].Content); + var namedArg = blocks[1] as NamedArgBlock; + Assert.NotNull(namedArg); + Assert.Equal("foo", namedArg.GetValue(null)); + Assert.Equal("name", namedArg.Name); + } + + [Theory] + [InlineData(@"call 'f\\'xy'")] + [InlineData(@"call 'f\\'x")] + [InlineData("f name")] + public void ItThrowsWhenSeparatorsAreMissing(string template) + { + // Act & Assert + Assert.Throws(() => this._target.Tokenize(template)); + } + + [Theory] + [InlineData("f a =", "A function named argument must contain a quoted value or variable after the '=' character.")] + [InlineData("f a='b' arg2", "A function named argument must contain a name and value separated by a '=' character.")] + public void ItThrowsWhenArgValueIsMissing(string template, string expectedErrorMessage) + { + // Act & Assert + var exception = Assert.Throws(() => this._target.Tokenize(template)); + Assert.Equal(expectedErrorMessage, exception.Message); + } +} diff --git a/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/PromptTemplateEngineTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/PromptTemplateEngineTests.cs new file mode 100644 index 000000000000..c222beacd834 --- /dev/null +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/PromptTemplateEngineTests.cs @@ -0,0 +1,402 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine.Basic; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; +using Moq; +using SemanticKernel.Extensions.UnitTests.XunitHelpers; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt; + +public sealed class PromptTemplateEngineTests +{ + private const string DateFormat = "M/d/yyyy"; + private readonly BasicPromptTemplateEngine _target; + private readonly ContextVariables _variables; + private readonly Mock _functions; + private readonly ITestOutputHelper _logger; + private readonly Mock _kernel; + private readonly Mock _functionRunner; + + public PromptTemplateEngineTests(ITestOutputHelper testOutputHelper) + { + this._logger = testOutputHelper; + this._target = new BasicPromptTemplateEngine(TestConsoleLogger.LoggerFactory); + this._variables = new ContextVariables(Guid.NewGuid().ToString("X")); + this._functions = new Mock(); + this._kernel = new Mock(); + this._functionRunner = new Mock(); + } + + [Fact] + public void ItRendersVariables() + { + // Arrange + var template = "{$x11} This {$a} is {$_a} a {{$x11}} test {{$x11}} " + + "template {{foo}}{{bar $a}}{{baz $_a}}{{yay $x11}}{{food a='b' c = $d}}"; + + // Act + var blocks = this._target.ExtractBlocks(template); + var updatedBlocks = this._target.RenderVariables(blocks, this._variables); + + // Assert + Assert.Equal(10, blocks.Count); + Assert.Equal(10, updatedBlocks.Count); + + Assert.Equal("$x11", blocks[1].Content); + Assert.Equal("", updatedBlocks[1].Content); + Assert.Equal(BlockTypes.Variable, blocks[1].Type); + Assert.Equal(BlockTypes.Text, updatedBlocks[1].Type); + + Assert.Equal("$x11", blocks[3].Content); + Assert.Equal("", updatedBlocks[3].Content); + Assert.Equal(BlockTypes.Variable, blocks[3].Type); + Assert.Equal(BlockTypes.Text, updatedBlocks[3].Type); + + Assert.Equal("foo", blocks[5].Content); + Assert.Equal("foo", updatedBlocks[5].Content); + Assert.Equal(BlockTypes.Code, blocks[5].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[5].Type); + + Assert.Equal("bar $a", blocks[6].Content); + Assert.Equal("bar $a", updatedBlocks[6].Content); + Assert.Equal(BlockTypes.Code, blocks[6].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[6].Type); + + Assert.Equal("baz $_a", blocks[7].Content); + Assert.Equal("baz $_a", updatedBlocks[7].Content); + Assert.Equal(BlockTypes.Code, blocks[7].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[7].Type); + + Assert.Equal("yay $x11", blocks[8].Content); + Assert.Equal("yay $x11", updatedBlocks[8].Content); + Assert.Equal(BlockTypes.Code, blocks[8].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[8].Type); + + Assert.Equal("food a='b' c = $d", blocks[9].Content); + Assert.Equal("food a='b' c = $d", updatedBlocks[9].Content); + Assert.Equal(BlockTypes.Code, blocks[9].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[9].Type); + + // Arrange + this._variables.Set("x11", "x11 value"); + this._variables.Set("a", "a value"); + this._variables.Set("_a", "_a value"); + this._variables.Set("c", "c value"); + this._variables.Set("d", "d value"); + + // Act + blocks = this._target.ExtractBlocks(template); + updatedBlocks = this._target.RenderVariables(blocks, this._variables); + + // Assert + Assert.Equal(10, blocks.Count); + Assert.Equal(10, updatedBlocks.Count); + + Assert.Equal("$x11", blocks[1].Content); + Assert.Equal("x11 value", updatedBlocks[1].Content); + Assert.Equal(BlockTypes.Variable, blocks[1].Type); + Assert.Equal(BlockTypes.Text, updatedBlocks[1].Type); + + Assert.Equal("$x11", blocks[3].Content); + Assert.Equal("x11 value", updatedBlocks[3].Content); + Assert.Equal(BlockTypes.Variable, blocks[3].Type); + Assert.Equal(BlockTypes.Text, updatedBlocks[3].Type); + + Assert.Equal("foo", blocks[5].Content); + Assert.Equal("foo", updatedBlocks[5].Content); + Assert.Equal(BlockTypes.Code, blocks[5].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[5].Type); + + Assert.Equal("bar $a", blocks[6].Content); + Assert.Equal("bar $a", updatedBlocks[6].Content); + Assert.Equal(BlockTypes.Code, blocks[6].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[6].Type); + + Assert.Equal("baz $_a", blocks[7].Content); + Assert.Equal("baz $_a", updatedBlocks[7].Content); + Assert.Equal(BlockTypes.Code, blocks[7].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[7].Type); + + Assert.Equal("yay $x11", blocks[8].Content); + Assert.Equal("yay $x11", updatedBlocks[8].Content); + Assert.Equal(BlockTypes.Code, blocks[8].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[8].Type); + + Assert.Equal("food a='b' c = $d", blocks[9].Content); + Assert.Equal("food a='b' c = $d", updatedBlocks[9].Content); + Assert.Equal(BlockTypes.Code, blocks[9].Type); + Assert.Equal(BlockTypes.Code, updatedBlocks[9].Type); + } + + [Fact] + public async Task ItRendersCodeUsingInputAsync() + { + // Arrange + string MyFunctionAsync(SKContext context) + { + this._logger.WriteLine("MyFunction call received, input: {0}", context.Variables.Input); + return $"F({context.Variables.Input})"; + } + + List functions = new() + { + SKFunction.FromNativeMethod(Method(MyFunctionAsync), this), + }; + + Assert.NotNull(functions[0]); + + this._variables.Update("INPUT-BAR"); + var template = "foo-{{function}}-baz"; + + this.MockFunctionRunner(functions[0]); + + var context = this.MockContext(); + + // Act + var result = await this._target.RenderAsync(template, context); + + // Assert + Assert.Equal("foo-F(INPUT-BAR)-baz", result); + } + + [Fact] + public async Task ItRendersCodeUsingVariablesAsync() + { + // Arrange + string MyFunctionAsync(SKContext context) + { + this._logger.WriteLine("MyFunction call received, input: {0}", context.Variables.Input); + return $"F({context.Variables.Input})"; + } + + var func = SKFunction.FromNativeMethod(Method(MyFunctionAsync), this); + + Assert.NotNull(func); + + this._variables.Set("myVar", "BAR"); + var template = "foo-{{function $myVar}}-baz"; + + this.MockFunctionRunner(func); + var context = this.MockContext(); + + // Act + var result = await this._target.RenderAsync(template, context); + + // Assert + Assert.Equal("foo-F(BAR)-baz", result); + } + + [Fact] + public async Task ItRendersCodeUsingNamedVariablesAsync() + { + // Arrange + string MyFunctionAsync( + [Description("Name"), SKName("input")] string name, + [Description("Age"), SKName("age")] int age, + [Description("Slogan"), SKName("slogan")] string slogan, + [Description("Date"), SKName("date")] DateTime date) + { + var dateStr = date.ToString(PromptTemplateEngineTests.DateFormat, CultureInfo.InvariantCulture); + this._logger.WriteLine("MyFunction call received, name: {0}, age: {1}, slogan: {2}, date: {3}", name, age, slogan, date); + return $"[{dateStr}] {name} ({age}): \"{slogan}\""; + } + + var func = SKFunction.FromNativeMethod(Method(MyFunctionAsync), this); + + Assert.NotNull(func); + + this._variables.Set("input", "Mario"); + this._variables.Set("someDate", "2023-08-25T00:00:00"); + var template = "foo-{{function input=$input age='42' slogan='Let\\'s-a go!' date=$someDate}}-baz"; + + this.MockFunctionRunner(func); + var context = this.MockContext(); + + // Act + var result = await this._target.RenderAsync(template, context); + + // Assert + Assert.Equal("foo-[8/25/2023] Mario (42): \"Let's-a go!\"-baz", result); + } + + [Fact] + public async Task ItHandlesSyntaxErrorsAsync() + { + // Arrange + string MyFunctionAsync( + [Description("Name"), SKName("input")] string name, + [Description("Age"), SKName("age")] int age, + [Description("Slogan"), SKName("slogan")] string slogan, + [Description("Date"), SKName("date")] DateTime date) + { + var dateStr = date.ToString(PromptTemplateEngineTests.DateFormat, CultureInfo.InvariantCulture); + this._logger.WriteLine("MyFunction call received, name: {0}, age: {1}, slogan: {2}, date: {3}", name, age, slogan, date); + return $"[{dateStr}] {name} ({age}): \"{slogan}\""; + } + + ISKFunction func = SKFunction.FromNativeMethod(Method(MyFunctionAsync), this); + Assert.NotNull(func); + + this._variables.Set("input", "Mario"); + this._variables.Set("someDate", "2023-08-25T00:00:00"); + var template = "foo-{{function input=$input age=42 slogan='Let\\'s-a go!' date=$someDate}}-baz"; + var context = this.MockContext(); + + // Act + var result = await Assert.ThrowsAsync(() => this._target.RenderAsync(template, context)); + Assert.Equal($"Named argument values need to be prefixed with a quote or {Symbols.VarPrefix}.", result.Message); + } + + [Fact] + public async Task ItRendersCodeUsingImplicitInputAndNamedVariablesAsync() + { + // Arrange + string MyFunctionAsync( + [Description("Input"), SKName("input")] string name, + [Description("Age"), SKName("age")] int age, + [Description("Slogan"), SKName("slogan")] string slogan, + [Description("Date"), SKName("date")] DateTime date) + { + this._logger.WriteLine("MyFunction call received, name: {0}, age: {1}, slogan: {2}, date: {3}", name, age, slogan, date); + var dateStr = date.ToString(PromptTemplateEngineTests.DateFormat, CultureInfo.InvariantCulture); + return $"[{dateStr}] {name} ({age}): \"{slogan}\""; + } + + ISKFunction func = SKFunction.FromNativeMethod(Method(MyFunctionAsync), this); + + Assert.NotNull(func); + + this._variables.Set("input", "Mario"); + this._variables.Set("someDate", "2023-08-25T00:00:00"); + + var template = "foo-{{function $input age='42' slogan='Let\\'s-a go!' date=$someDate}}-baz"; + + this.MockFunctionRunner(func); + + var context = this.MockContext(); + + // Act + var result = await this._target.RenderAsync(template, context); + + // Assert + Assert.Equal("foo-[8/25/2023] Mario (42): \"Let's-a go!\"-baz", result); + } + + [Fact] + public async Task ItRendersAsyncCodeUsingImmutableVariablesAsync() + { + // Arrange + var template = "{{func1}} {{func2}} {{func3 $myVar}}"; + this._variables.Update("BAR"); + this._variables.Set("myVar", "BAZ"); + + string MyFunction1Async(SKContext context) + { + this._logger.WriteLine("MyFunction1 call received, input: {0}", context.Variables.Input); + context.Variables.Update("foo"); + return "F(OUTPUT-FOO)"; + } + string MyFunction2Async(SKContext context) + { + // Input value should be "BAR" because the variable $input is immutable in MyFunction1 + this._logger.WriteLine("MyFunction2 call received, input: {0}", context.Variables.Input); + context.Variables.Set("myVar", "bar"); + return context.Variables.Input; + } + string MyFunction3Async(SKContext context) + { + // Input value should be "BAZ" because the variable $myVar is immutable in MyFunction2 + this._logger.WriteLine("MyFunction3 call received, input: {0}", context.Variables.Input); + return context.Variables.TryGetValue("myVar", out string? value) ? value : ""; + } + + var functions = new List() + { + SKFunction.FromNativeMethod(Method(MyFunction1Async), this, "func1"), + SKFunction.FromNativeMethod(Method(MyFunction2Async), this, "func2"), + SKFunction.FromNativeMethod(Method(MyFunction3Async), this, "func3") + }; + + this.MockFunctionRunner(functions); + + // Act + var result = await this._target.RenderAsync(template, this.MockContext()); + + // Assert + Assert.Equal("F(OUTPUT-FOO) BAR BAZ", result); + } + + [Fact] + public async Task ItRendersAsyncCodeUsingVariablesAsync() + { + // Arrange + Task MyFunctionAsync(SKContext context) + { + // Input value should be "BAR" because the variable $myVar is passed in + this._logger.WriteLine("MyFunction call received, input: {0}", context.Variables.Input); + return Task.FromResult(context.Variables.Input); + } + + ISKFunction func = SKFunction.FromNativeMethod(Method(MyFunctionAsync), this); + Assert.NotNull(func); + + this._variables.Set("myVar", "BAR"); + + var template = "foo-{{function $myVar}}-baz"; + var context = this.MockContext(); + + // Act + var result = await this._target.RenderAsync(template, context); + + // Assert + Assert.Equal("foo-BAR-baz", result); + } + + private static MethodInfo Method(Delegate method) + { + return method.Method; + } + + private void MockFunctionRunner(ISKFunction function) + { + this._functionRunner.Setup(r => r.RunAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((pluginName, functionName, variables, cancellationToken) => + { + var context = new SKContext(this._functionRunner.Object, variables); + return function.InvokeAsync(context, null, cancellationToken); + }); + } + + private void MockFunctionRunner(List functions) + { + this._functionRunner.Setup(r => r.RunAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((pluginName, functionName, variables, cancellationToken) => + { + var context = new SKContext(this._functionRunner.Object, variables); + var function = functions.First(f => f.PluginName == functionName); + + return function.InvokeAsync(context, null, cancellationToken); + }); + } + + private SKContext MockContext() + { + return new SKContext( + this._functionRunner.Object, + this._variables, + this._functions.Object); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/TemplateTokenizerTests.cs b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/TemplateTokenizerTests.cs similarity index 82% rename from dotnet/src/SemanticKernel.UnitTests/TemplateEngine/TemplateTokenizerTests.cs rename to dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/TemplateTokenizerTests.cs index 77fd89b0be27..53e36f3046fa 100644 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/TemplateTokenizerTests.cs +++ b/dotnet/src/Extensions/Extensions.UnitTests/TemplateEngine/Prompt/TemplateTokenizerTests.cs @@ -1,10 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.TemplateEngine.Basic; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; using Xunit; -namespace SemanticKernel.UnitTests.TemplateEngine; +namespace SemanticKernel.Extensions.UnitTests.TemplateEngine.Prompt; public class TemplateTokenizerTests { @@ -35,7 +36,7 @@ internal void ItParsesTextWithoutCode(string? text, BlockTypes type) var blocks = this._target.Tokenize(text); // Assert - Assert.Equal(1, blocks.Count); + Assert.Single(blocks); Assert.Equal(type, blocks[0].Type); } @@ -66,7 +67,7 @@ internal void ItParsesBasicBlocks(string? text, BlockTypes type) var blocks = this._target.Tokenize(text); // Assert - Assert.Equal(1, blocks.Count); + Assert.Single(blocks); Assert.Equal(type, blocks[0].Type); } @@ -97,9 +98,9 @@ public void ItTokenizesEdgeCasesCorrectly1() // Assert - Count Assert.Equal(2, blocks1.Count); - Assert.Equal(1, blocks2.Count); - Assert.Equal(1, blocks3.Count); - Assert.Equal(1, blocks4.Count); + Assert.Single(blocks2); + Assert.Single(blocks3); + Assert.Single(blocks4); // Assert - Type Assert.Equal(BlockTypes.Text, blocks1[0].Type); @@ -182,7 +183,7 @@ public void ItTokenizesEdgeCasesCorrectly4(string template) var blocks = this._target.Tokenize(template); // Assert - Assert.Equal(1, blocks.Count); + Assert.Single(blocks); Assert.Equal(BlockTypes.Code, blocks[0].Type); Assert.Equal(template[2..^2].Trim(), blocks[0].Content); } @@ -224,4 +225,38 @@ public void ItTokenizesATypicalPrompt() Assert.Equal("and 'values'", blocks[7].Content); Assert.Equal(BlockTypes.Code, blocks[7].Type); } + + [Fact] + public void ItTokenizesAFunctionCallWithMultipleArguments() + { + // Arrange + var template = "this is a {{ function with='many' named=$arguments }}"; + + // Act + var blocks = this._target.Tokenize(template); + + // Assert + Assert.Equal(2, blocks.Count); + + Assert.Equal("this is a ", blocks[0].Content); + Assert.Equal(BlockTypes.Text, blocks[0].Type); + + Assert.Equal("function with='many' named=$arguments", blocks[1].Content); + Assert.Equal(BlockTypes.Code, blocks[1].Type); + } + + [Fact] + public void ItThrowsWhenCodeBlockStartsWithNamedArg() + { + // Arrange + var template = "{{ not='valid' }}"; + + // Assert + var ex = Assert.Throws(() => + { + // Act + this._target.Tokenize(template); + }); + Assert.Equal("Code tokenizer returned an incorrect first token type NamedArg", ex.Message); + } } diff --git a/dotnet/src/Extensions/Extensions.UnitTests/XunitHelpers/TestConsoleLogger.cs b/dotnet/src/Extensions/Extensions.UnitTests/XunitHelpers/TestConsoleLogger.cs index 3a375bed388c..8971cd44a83b 100644 --- a/dotnet/src/Extensions/Extensions.UnitTests/XunitHelpers/TestConsoleLogger.cs +++ b/dotnet/src/Extensions/Extensions.UnitTests/XunitHelpers/TestConsoleLogger.cs @@ -10,14 +10,14 @@ namespace SemanticKernel.Extensions.UnitTests.XunitHelpers; /// internal static class TestConsoleLogger { - internal static ILogger Log => LogFactory.CreateLogger(); + internal static ILogger Log => LoggerFactory.CreateLogger(); - private static ILoggerFactory LogFactory => s_loggerFactory.Value; + internal static ILoggerFactory LoggerFactory => s_loggerFactory.Value; private static readonly Lazy s_loggerFactory = new(LogBuilder); private static ILoggerFactory LogBuilder() { - return LoggerFactory.Create(builder => + return Microsoft.Extensions.Logging.LoggerFactory.Create(builder => { builder.SetMinimumLevel(LogLevel.Trace); // builder.AddFilter("Microsoft", LogLevel.Trace); diff --git a/dotnet/src/Extensions/Planning.ActionPlanner/ActionPlanner.cs b/dotnet/src/Extensions/Planning.ActionPlanner/ActionPlanner.cs deleted file mode 100644 index 89ca3218048d..000000000000 --- a/dotnet/src/Extensions/Planning.ActionPlanner/ActionPlanner.cs +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning.Action; -using Microsoft.SemanticKernel.SkillDefinition; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using NS of Plan -namespace Microsoft.SemanticKernel.Planning; -#pragma warning restore IDE0130 - -/// -/// Action Planner allows to select one function out of many, to achieve a given goal. -/// The planner implement the Intent Detection pattern, uses the functions registered -/// in the kernel to see if there's a relevant one, providing instructions to call the -/// function and the rationale used to select it. The planner can also return -/// "no function" is nothing relevant is available. -/// The rationale is currently available only in the prompt, we might include it in -/// the Plan object in future. -/// -public sealed class ActionPlanner : IActionPlanner -{ - private const string StopSequence = "#END-OF-PLAN"; - private const string SkillName = "this"; - - /// - /// The regular expression for extracting serialized plan. - /// - private static readonly Regex PlanRegex = new("^[^{}]*(((?'Open'{)[^{}]*)+((?'Close-Open'})[^{}]*)+)*(?(Open)(?!))", RegexOptions.Singleline | RegexOptions.Compiled); - - // Planner semantic function - private readonly ISKFunction _plannerFunction; - - // Context used to access the list of functions in the kernel - private readonly SKContext _context; - private readonly IKernel _kernel; - private readonly ILogger _logger; - - // TODO: allow to inject skill store - /// - /// Initialize a new instance of the class. - /// - /// The semantic kernel instance. - /// Optional prompt override - /// Optional logger - public ActionPlanner( - IKernel kernel, - string? prompt = null, - ILogger? logger = null) - { - Verify.NotNull(kernel); - - this._logger = logger ?? new NullLogger(); - - string promptTemplate = prompt ?? EmbeddedResource.Read("skprompt.txt"); - - this._plannerFunction = kernel.CreateSemanticFunction( - skillName: SkillName, - promptTemplate: promptTemplate, - maxTokens: 1024, - stopSequences: new[] { StopSequence }); - - kernel.ImportSkill(this, skillName: SkillName); - - this._kernel = kernel; - this._context = kernel.CreateNewContext(); - } - - /// - public async Task CreatePlanAsync(string goal, CancellationToken cancellationToken = default) - { - if (string.IsNullOrEmpty(goal)) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidGoal, "The goal specified is empty"); - } - - this._context.Variables.Update(goal); - - SKContext result = await this._plannerFunction.InvokeAsync(this._context, cancellationToken: cancellationToken).ConfigureAwait(false); - ActionPlanResponse? planData = this.ParsePlannerResult(result); - - if (planData == null) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, "The plan deserialized to a null object"); - } - - // Build and return plan - Plan plan; - if (planData.Plan.Function.Contains(".")) - { - var parts = planData.Plan.Function.Split('.'); - plan = new Plan(goal, this._context.Skills!.GetFunction(parts[0], parts[1])); - } - else if (!string.IsNullOrWhiteSpace(planData.Plan.Function)) - { - plan = new Plan(goal, this._context.Skills!.GetFunction(planData.Plan.Function)); - } - else - { - // No function was found - return a plan with no steps. - plan = new Plan(goal); - } - - // Create a plan using the function and the parameters suggested by the planner - foreach (KeyValuePair p in planData.Plan.Parameters) - { - if (p.Value != null) - { - plan.Parameters[p.Key] = p.Value.ToString(); - } - } - - return plan; - } - - // TODO: use goal to find relevant functions in a skill store - /// - /// Native function returning a list of all the functions in the current context, - /// excluding functions in the planner itself. - /// - /// Currently unused. Will be used to handle long lists of functions. - /// Function execution context - /// List of functions, formatted accordingly to the prompt - [SKFunction, Description("List all functions available in the kernel")] - public string ListOfFunctions( - [Description("The current goal processed by the planner")] string goal, - SKContext context) - { - Verify.NotNull(context.Skills); - var functionsAvailable = context.Skills.GetFunctionsView(); - - // Prepare list using the format used by skprompt.txt - var list = new StringBuilder(); - this.PopulateList(list, functionsAvailable.NativeFunctions); - this.PopulateList(list, functionsAvailable.SemanticFunctions); - - return list.ToString(); - } - - // TODO: generate string programmatically - // TODO: use goal to find relevant examples - [SKFunction, Description("List a few good examples of plans to generate")] - public string GoodExamples( - [Description("The current goal processed by the planner")] string goal, - SKContext context) - { - return @" -[EXAMPLE] -- List of functions: -// Read a file. -FileIOSkill.ReadAsync -Parameter ""path"": Source file. -// Write a file. -FileIOSkill.WriteAsync -Parameter ""path"": Destination file. (default value: sample.txt) -Parameter ""content"": File content. -// Get the current time. -TimeSkill.Time -No parameters. -// Makes a POST request to a uri. -HttpSkill.PostAsync -Parameter ""body"": The body of the request. -- End list of functions. -Goal: create a file called ""something.txt"". -{""plan"":{ -""rationale"": ""the list contains a function that allows to create files"", -""function"": ""FileIOSkill.WriteAsync"", -""parameters"": { -""path"": ""something.txt"", -""content"": null -}}} -#END-OF-PLAN -"; - } - - // TODO: generate string programmatically - [SKFunction, Description("List a few edge case examples of plans to handle")] - public string EdgeCaseExamples( - [Description("The current goal processed by the planner")] string goal, - SKContext context) - { - return @" -[EXAMPLE] -- List of functions: -// Get the current time. -TimeSkill.Time -No parameters. -// Write a file. -FileIOSkill.WriteAsync -Parameter ""path"": Destination file. (default value: sample.txt) -Parameter ""content"": File content. -// Makes a POST request to a uri. -HttpSkill.PostAsync -Parameter ""body"": The body of the request. -// Read a file. -FileIOSkill.ReadAsync -Parameter ""path"": Source file. -- End list of functions. -Goal: tell me a joke. -{""plan"":{ -""rationale"": ""the list does not contain functions to tell jokes or something funny"", -""function"": """", -""parameters"": { -}}} -#END-OF-PLAN -"; - } - - #region private ================================================================================ - - /// - /// Native function that filters out good JSON from planner result in case additional text is present - /// using a similar regex to the balancing group regex defined here: https://learn.microsoft.com/en-us/dotnet/standard/base-types/grouping-constructs-in-regular-expressions#balancing-group-definitions - /// - /// Result context of planner function. - /// Instance of object deserialized from extracted JSON. - private ActionPlanResponse? ParsePlannerResult(SKContext plannerResult) - { - Match match = PlanRegex.Match(plannerResult.ToString()); - - if (match.Success && match.Groups["Close"].Length > 0) - { - string planJson = $"{{{match.Groups["Close"]}}}"; - try - { - return JsonSerializer.Deserialize(planJson, new JsonSerializerOptions - { - AllowTrailingCommas = true, - DictionaryKeyPolicy = null, - DefaultIgnoreCondition = JsonIgnoreCondition.Never, - PropertyNameCaseInsensitive = true, - }); - } - catch (Exception e) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, - "Plan parsing error, invalid JSON", e); - } - } - else - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, $"Failed to extract valid json string from planner result: '{plannerResult}'"); - } - } - - private void PopulateList(StringBuilder list, IDictionary> functions) - { - foreach (KeyValuePair> skill in functions) - { - // Skip this planner skills - if (string.Equals(skill.Key, SkillName, StringComparison.OrdinalIgnoreCase)) { continue; } - - foreach (FunctionView func in skill.Value) - { - // Function description - if (func.Description != null) - { - list.AppendLine($"// {AddPeriod(func.Description)}"); - } - else - { - this._logger.LogWarning("{0}.{1} is missing a description", func.SkillName, func.Name); - list.AppendLine($"// Function {func.SkillName}.{func.Name}."); - } - - // Function name - list.AppendLine($"{func.SkillName}.{func.Name}"); - - // Function parameters - foreach (var p in func.Parameters) - { - var description = string.IsNullOrEmpty(p.Description) ? p.Name : p.Description!; - var defaultValueString = string.IsNullOrEmpty(p.DefaultValue) ? string.Empty : $" (default value: {p.DefaultValue})"; - list.AppendLine($"Parameter \"{p.Name}\": {AddPeriod(description)} {defaultValueString}"); - } - } - } - } - - private static string AddPeriod(string x) - { - return x.EndsWith(".", StringComparison.Ordinal) ? x : $"{x}."; - } - - #endregion -} diff --git a/dotnet/src/Extensions/Planning.ActionPlanner/ActionPlannerExtensions.cs b/dotnet/src/Extensions/Planning.ActionPlanner/ActionPlannerExtensions.cs deleted file mode 100644 index 618c1e647af1..000000000000 --- a/dotnet/src/Extensions/Planning.ActionPlanner/ActionPlannerExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Logging; - -namespace Microsoft.SemanticKernel.Planning.Action; - -/// -/// Extension methods for class. -/// -public static class ActionPlannerExtensions -{ - /// - /// Returns decorated instance of with enabled instrumentation. - /// - /// Instance of to decorate. - /// Optional logger. - public static IActionPlanner WithInstrumentation(this IActionPlanner planner, ILogger? logger = null) - { - return new InstrumentedActionPlanner(planner, logger); - } -} diff --git a/dotnet/src/Extensions/Planning.ActionPlanner/EmbeddedResource.cs b/dotnet/src/Extensions/Planning.ActionPlanner/EmbeddedResource.cs deleted file mode 100644 index ac8a49d5d285..000000000000 --- a/dotnet/src/Extensions/Planning.ActionPlanner/EmbeddedResource.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.IO; -using System.Reflection; - -namespace Microsoft.SemanticKernel.Planning.Action; - -internal static class EmbeddedResource -{ - private static readonly string? s_namespace = typeof(EmbeddedResource).Namespace; - - internal static string Read(string name) - { - var assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly; - if (assembly == null) { throw new PlanningException(PlanningException.ErrorCodes.InvalidConfiguration, $"[{s_namespace}] {name} assembly not found"); } - - using Stream? resource = assembly.GetManifestResourceStream($"{s_namespace}." + name); - if (resource == null) { throw new PlanningException(PlanningException.ErrorCodes.InvalidConfiguration, $"[{s_namespace}] {name} resource not found"); } - - using var reader = new StreamReader(resource); - return reader.ReadToEnd(); - } -} diff --git a/dotnet/src/Extensions/Planning.ActionPlanner/IActionPlanner.cs b/dotnet/src/Extensions/Planning.ActionPlanner/IActionPlanner.cs deleted file mode 100644 index ce844303a7af..000000000000 --- a/dotnet/src/Extensions/Planning.ActionPlanner/IActionPlanner.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.SemanticKernel.Planning.Action; - -/// -/// Interface for planner that uses a set of semantic functions to select one function out of many and create a plan. -/// -public interface IActionPlanner -{ - /// - /// Create a plan for a goal. - /// - /// The goal to create a plan for. - /// The to monitor for cancellation requests. The default is . - /// The plan. - /// Thrown when the plan cannot be created. - Task CreatePlanAsync(string goal, CancellationToken cancellationToken = default); -} diff --git a/dotnet/src/Extensions/Planning.ActionPlanner/Planning.ActionPlanner.csproj b/dotnet/src/Extensions/Planning.ActionPlanner/Planning.ActionPlanner.csproj deleted file mode 100644 index 1058a819a159..000000000000 --- a/dotnet/src/Extensions/Planning.ActionPlanner/Planning.ActionPlanner.csproj +++ /dev/null @@ -1,31 +0,0 @@ - - - - - Microsoft.SemanticKernel.Planning.ActionPlanner - Microsoft.SemanticKernel.Planning.Action - netstandard2.0 - - - - - - - - Semantic Kernel - Action Planner - Semantic Kernel Action Planner - - - - - - - - - - - Always - - - - diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/EmbeddedResource.cs b/dotnet/src/Extensions/Planning.SequentialPlanner/EmbeddedResource.cs deleted file mode 100644 index 54eb4b0187b6..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/EmbeddedResource.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.IO; -using System.Reflection; - -namespace Microsoft.SemanticKernel.Planning.Sequential; - -internal sealed class EmbeddedResource -{ - private static readonly string? s_namespace = typeof(EmbeddedResource).Namespace; - - internal static string Read(string name) - { - var assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly; - if (assembly == null) { throw new PlanningException(PlanningException.ErrorCodes.InvalidConfiguration, $"[{s_namespace}] {name} assembly not found"); } - - using Stream? resource = assembly.GetManifestResourceStream($"{s_namespace}." + name); - if (resource == null) { throw new PlanningException(PlanningException.ErrorCodes.InvalidConfiguration, $"[{s_namespace}] {name} resource not found"); } - - using var reader = new StreamReader(resource); - return reader.ReadToEnd(); - } -} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/FunctionViewExtensions.cs b/dotnet/src/Extensions/Planning.SequentialPlanner/FunctionViewExtensions.cs deleted file mode 100644 index 01453a99a930..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/FunctionViewExtensions.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Linq; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Planning.Sequential; - -internal static class FunctionViewExtensions -{ - /// - /// Create a manual-friendly string for a function. - /// - /// The function to create a manual-friendly string for. - /// A manual-friendly string for a function. - internal static string ToManualString(this FunctionView function) - { - var inputs = string.Join("\n", function.Parameters.Select(parameter => - { - var defaultValueString = string.IsNullOrEmpty(parameter.DefaultValue) ? string.Empty : $" (default value: {parameter.DefaultValue})"; - return $" - {parameter.Name}: {parameter.Description}{defaultValueString}"; - })); - - return $@"{function.ToFullyQualifiedName()}: - description: {function.Description} - inputs: - {inputs}"; - } - - /// - /// Create a fully qualified name for a function. - /// - /// The function to create a fully qualified name for. - /// A fully qualified name for a function. - internal static string ToFullyQualifiedName(this FunctionView function) - { - return $"{function.SkillName}.{function.Name}"; - } - - /// - /// Create a string for generating an embedding for a function. - /// - /// The function to create a string for generating an embedding for. - /// A string for generating an embedding for a function. - internal static string ToEmbeddingString(this FunctionView function) - { - var inputs = string.Join("\n", function.Parameters.Select(p => $" - {p.Name}: {p.Description}")); - return $"{function.Name}:\n description: {function.Description}\n inputs:\n{inputs}"; - } -} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/ISequentialPlanner.cs b/dotnet/src/Extensions/Planning.SequentialPlanner/ISequentialPlanner.cs deleted file mode 100644 index 7df5180c5cb1..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/ISequentialPlanner.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.SemanticKernel.Planning.Sequential; - -/// -/// Interface for planner that uses a set of semantic functions to create a sequential plan. -/// -public interface ISequentialPlanner -{ - /// - /// Create a plan for a goal. - /// - /// The goal to create a plan for. - /// The to monitor for cancellation requests. The default is . - /// The plan. - /// Thrown when the plan cannot be created. - Task CreatePlanAsync(string goal, CancellationToken cancellationToken = default); -} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/Planning.SequentialPlanner.csproj b/dotnet/src/Extensions/Planning.SequentialPlanner/Planning.SequentialPlanner.csproj deleted file mode 100644 index 336e419d099a..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/Planning.SequentialPlanner.csproj +++ /dev/null @@ -1,36 +0,0 @@ - - - - - Microsoft.SemanticKernel.Planning.SequentialPlanner - Microsoft.SemanticKernel.Planning.Sequential - netstandard2.0 - - - - - - - - Semantic Kernel - Sequential Planner - Semantic Kernel Sequential Planner - - - - - - - - - - - - - - - - Always - - - - diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/SKContextSequentialPlannerExtensions.cs b/dotnet/src/Extensions/Planning.SequentialPlanner/SKContextSequentialPlannerExtensions.cs deleted file mode 100644 index 418dc7b2087b..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/SKContextSequentialPlannerExtensions.cs +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Planning.Sequential; -using Microsoft.SemanticKernel.SkillDefinition; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using NS of SKContext -namespace Microsoft.SemanticKernel.Orchestration; -#pragma warning restore IDE0130 - -public static class SKContextSequentialPlannerExtensions -{ - internal const string PlannerMemoryCollectionName = "Planning.SKFunctionsManual"; - - internal const string PlanSKFunctionsAreRemembered = "Planning.SKFunctionsAreRemembered"; - - /// - /// Returns a string containing the manual for all available functions. - /// - /// The SKContext to get the functions manual for. - /// The semantic query for finding relevant registered functions - /// The planner skill config. - /// The to monitor for cancellation requests. The default is . - /// A string containing the manual for all available functions. - public static async Task GetFunctionsManualAsync( - this SKContext context, - string? semanticQuery = null, - SequentialPlannerConfig? config = null, - CancellationToken cancellationToken = default) - { - config ??= new SequentialPlannerConfig(); - - // Use configured function provider if available, otherwise use the default SKContext function provider. - IOrderedEnumerable functions = config.GetAvailableFunctionsAsync is null ? - await context.GetAvailableFunctionsAsync(config, semanticQuery, cancellationToken).ConfigureAwait(false) : - await config.GetAvailableFunctionsAsync(config, semanticQuery, cancellationToken).ConfigureAwait(false); - - return string.Join("\n\n", functions.Select(x => x.ToManualString())); - } - - /// - /// Returns a list of functions that are available to the user based on the semantic query and the excluded skills and functions. - /// - /// The SKContext - /// The planner config. - /// The semantic query for finding relevant registered functions - /// The to monitor for cancellation requests. The default is . - /// A list of functions that are available to the user based on the semantic query and the excluded skills and functions. - public static async Task> GetAvailableFunctionsAsync( - this SKContext context, - SequentialPlannerConfig config, - string? semanticQuery = null, - CancellationToken cancellationToken = default) - { - var functionsView = context.Skills.GetFunctionsView(); - - var availableFunctions = functionsView.SemanticFunctions - .Concat(functionsView.NativeFunctions) - .SelectMany(x => x.Value) - .Where(s => !config.ExcludedSkills.Contains(s.SkillName) && !config.ExcludedFunctions.Contains(s.Name)) - .ToList(); - - List? result = null; - if (string.IsNullOrEmpty(semanticQuery) || config.Memory is NullMemory || config.RelevancyThreshold is null) - { - // If no semantic query is provided, return all available functions. - // If a Memory provider has not been registered, return all available functions. - result = availableFunctions; - } - else - { - result = new List(); - - // Remember functions in memory so that they can be searched. - await RememberFunctionsAsync(context, config.Memory, availableFunctions, cancellationToken).ConfigureAwait(false); - - // Search for functions that match the semantic query. - var memories = config.Memory.SearchAsync( - PlannerMemoryCollectionName, - semanticQuery!, - config.MaxRelevantFunctions, - config.RelevancyThreshold.Value, - cancellationToken: cancellationToken); - - // Add functions that were found in the search results. - result.AddRange(await GetRelevantFunctionsAsync(context, availableFunctions, memories, cancellationToken).ConfigureAwait(false)); - - // Add any missing functions that were included but not found in the search results. - var missingFunctions = config.IncludedFunctions - .Except(result.Select(x => x.Name)) - .Join(availableFunctions, f => f, af => af.Name, (_, af) => af); - - result.AddRange(missingFunctions); - } - - return result - .OrderBy(x => x.SkillName) - .ThenBy(x => x.Name); - } - - private static async Task> GetRelevantFunctionsAsync( - SKContext context, - IEnumerable availableFunctions, - IAsyncEnumerable memories, - CancellationToken cancellationToken = default) - { - var relevantFunctions = new ConcurrentBag(); - await foreach (var memoryEntry in memories.WithCancellation(cancellationToken)) - { - var function = availableFunctions.FirstOrDefault(x => x.ToFullyQualifiedName() == memoryEntry.Metadata.Id); - if (function != null) - { - context.Logger.LogDebug("Found relevant function. Relevance Score: {0}, Function: {1}", memoryEntry.Relevance, - function.ToFullyQualifiedName()); - relevantFunctions.Add(function); - } - } - - return relevantFunctions; - } - - /// - /// Saves all available functions to memory. - /// - /// The SKContext to save the functions to. - /// The memory provide to store the functions to.. - /// The available functions to save. - /// The to monitor for cancellation requests. The default is . - internal static async Task RememberFunctionsAsync( - SKContext context, - ISemanticTextMemory memory, - List availableFunctions, - CancellationToken cancellationToken = default) - { - // Check if the functions have already been saved to memory. - if (context.Variables.ContainsKey(PlanSKFunctionsAreRemembered)) - { - return; - } - - foreach (var function in availableFunctions) - { - var functionName = function.ToFullyQualifiedName(); - var key = functionName; - var description = string.IsNullOrEmpty(function.Description) ? functionName : function.Description; - var textToEmbed = function.ToEmbeddingString(); - - // It'd be nice if there were a saveIfNotExists method on the memory interface - var memoryEntry = await memory.GetAsync(collection: PlannerMemoryCollectionName, key: key, withEmbedding: false, - cancellationToken: cancellationToken).ConfigureAwait(false); - if (memoryEntry == null) - { - // TODO It'd be nice if the minRelevanceScore could be a parameter for each item that was saved to memory - // As folks may want to tune their functions to be more or less relevant. - // Memory now supports these such strategies. - await memory.SaveInformationAsync(collection: PlannerMemoryCollectionName, text: textToEmbed, id: key, description: description, - additionalMetadata: string.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); - } - } - - // Set a flag to indicate that the functions have been saved to memory. - context.Variables.Set(PlanSKFunctionsAreRemembered, "true"); - } -} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlanParser.cs b/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlanParser.cs deleted file mode 100644 index 8e40133f27c8..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlanParser.cs +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Text.RegularExpressions; -using System.Xml; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Planning.Sequential; - -/// -/// Parse sequential plan text into a plan. -/// -internal static class SequentialPlanParser -{ - /// - /// The tag name used in the plan xml for the user's goal/ask. - /// TODO: never used - /// - internal const string GoalTag = "goal"; - - /// - /// The tag name used in the plan xml for the solution. - /// - internal const string SolutionTag = "plan"; - - /// - /// The tag name used in the plan xml for a step that calls a skill function. - /// - internal const string FunctionTag = "function."; - - /// - /// The attribute tag used in the plan xml for setting the context variable name to set the output of a function to. - /// - internal const string SetContextVariableTag = "setContextVariable"; - - /// - /// The attribute tag used in the plan xml for appending the output of a function to the final result for a plan. - /// - internal const string AppendToResultTag = "appendToResult"; - - internal static Func GetSkillFunction(SKContext context) - { - return (skillName, functionName) => - { - if (string.IsNullOrEmpty(skillName)) - { - if (context.Skills!.TryGetFunction(functionName, out var skillFunction)) - { - return skillFunction; - } - } - else if (context.Skills!.TryGetFunction(skillName, functionName, out var skillFunction)) - { - return skillFunction; - } - - return null; - }; - } - - /// - /// Convert a plan xml string to a plan. - /// - /// The plan xml string. - /// The goal for the plan. - /// The callback to get a skill function. - /// Whether to allow missing functions in the plan on creation. - /// The plan. - /// Thrown when the plan xml is invalid. - internal static Plan ToPlanFromXml(this string xmlString, string goal, Func getSkillFunction, bool allowMissingFunctions = false) - { - XmlDocument xmlDoc = new(); - try - { - xmlDoc.LoadXml("" + xmlString + ""); - } - catch (XmlException e) - { - // xmlString wasn't valid xml, let's try and parse out of it - - // ']*': Matches zero or more characters that are not the closing angle bracket (">"), effectively matching any attributes present in the opening tag. - // '>': Matches the closing angle bracket (">") to indicate the end of the opening tag. - // '(.*?)': Captures the content between the opening and closing tags using a non-greedy match. It matches any character (except newline) in a lazy manner, i.e., it captures the smallest possible match. - // '': Matches the literal string "", indicating the closing tag of the element. - Regex planRegex = new(@"]*>(.*?)", RegexOptions.Singleline); - Match match = planRegex.Match(xmlString); - if (match.Success) - { - string planXml = match.Value; - - try - { - xmlDoc.LoadXml("" + planXml + ""); - } - catch (XmlException ex) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, $"Failed to parse plan xml strings: '{xmlString}' or '{planXml}'", ex); - } - } - else - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, $"Failed to parse plan xml string: '{xmlString}'", e); - } - } - - // Get the Solution - XmlNodeList solution = xmlDoc.GetElementsByTagName(SolutionTag); - - var plan = new Plan(goal); - - // loop through solution node and add to Steps - foreach (XmlNode solutionNode in solution) - { - var parentNodeName = solutionNode.Name; - - foreach (XmlNode childNode in solutionNode.ChildNodes) - { - if (childNode.Name == "#text" || childNode.Name == "#comment") - { - // Do not add text or comments as steps. - // TODO - this could be a way to get Reasoning for a plan step. - continue; - } - - if (childNode.Name.StartsWith(FunctionTag, StringComparison.OrdinalIgnoreCase)) - { - var skillFunctionName = childNode.Name.Split(s_functionTagArray, StringSplitOptions.None)?[1] ?? string.Empty; - GetSkillFunctionNames(skillFunctionName, out var skillName, out var functionName); - - if (!string.IsNullOrEmpty(functionName)) - { - var skillFunction = getSkillFunction(skillName, functionName); - - if (skillFunction is not null) - { - var planStep = new Plan(skillFunction); - - var functionVariables = new ContextVariables(); - var functionOutputs = new List(); - var functionResults = new List(); - - var view = skillFunction.Describe(); - foreach (var p in view.Parameters) - { - functionVariables.Set(p.Name, p.DefaultValue); - } - - if (childNode.Attributes is not null) - { - foreach (XmlAttribute attr in childNode.Attributes) - { - if (attr.Name.Equals(SetContextVariableTag, StringComparison.OrdinalIgnoreCase)) - { - functionOutputs.Add(attr.InnerText); - } - else if (attr.Name.Equals(AppendToResultTag, StringComparison.OrdinalIgnoreCase)) - { - functionOutputs.Add(attr.InnerText); - functionResults.Add(attr.InnerText); - } - else - { - functionVariables.Set(attr.Name, attr.InnerText); - } - } - } - - // Plan properties - planStep.Outputs = functionOutputs; - planStep.Parameters = functionVariables; - foreach (var result in functionResults) - { - plan.Outputs.Add(result); - } - - plan.AddSteps(planStep); - } - else - { - if (allowMissingFunctions) - { - plan.AddSteps(new Plan(skillFunctionName)); - } - else - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, $"Failed to find function '{skillFunctionName}' in skill '{skillName}'."); - } - } - } - } - - // Similar to comments or text, do not add empty nodes as steps. - // TODO - This could be a way to advertise desired functions for a plan. - } - } - - return plan; - } - - private static void GetSkillFunctionNames(string skillFunctionName, out string skillName, out string functionName) - { - var skillFunctionNameParts = skillFunctionName.Split('.'); - skillName = skillFunctionNameParts?.Length > 0 ? skillFunctionNameParts[0] : string.Empty; - functionName = skillFunctionNameParts?.Length > 1 ? skillFunctionNameParts[1] : skillFunctionName; - } - - private static readonly string[] s_functionTagArray = new string[] { FunctionTag }; -} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlanner.cs b/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlanner.cs deleted file mode 100644 index 8dc137938d27..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlanner.cs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning.Sequential; -using Microsoft.SemanticKernel.SkillDefinition; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using NS of Plan -namespace Microsoft.SemanticKernel.Planning; -#pragma warning restore IDE0130 - -/// -/// A planner that uses semantic function to create a sequential plan. -/// -public sealed class SequentialPlanner : ISequentialPlanner -{ - private const string StopSequence = ""; - - /// - /// Initialize a new instance of the class. - /// - /// The semantic kernel instance. - /// The planner configuration. - /// Optional prompt override - public SequentialPlanner( - IKernel kernel, - SequentialPlannerConfig? config = null, - string? prompt = null) - { - Verify.NotNull(kernel); - this.Config = config ?? new(); - - this.Config.ExcludedSkills.Add(RestrictedSkillName); - - string promptTemplate = prompt ?? EmbeddedResource.Read("skprompt.txt"); - - this._functionFlowFunction = kernel.CreateSemanticFunction( - promptTemplate: promptTemplate, - skillName: RestrictedSkillName, - description: "Given a request or command or goal generate a step by step plan to " + - "fulfill the request using functions. This ability is also known as decision making and function flow", - maxTokens: this.Config.MaxTokens ?? 1024, - temperature: 0.0, - stopSequences: new[] { StopSequence }); - - this._context = kernel.CreateNewContext(); - } - - /// - public async Task CreatePlanAsync(string goal, CancellationToken cancellationToken = default) - { - if (string.IsNullOrEmpty(goal)) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidGoal, "The goal specified is empty"); - } - - string relevantFunctionsManual = await this._context.GetFunctionsManualAsync(goal, this.Config, cancellationToken).ConfigureAwait(false); - this._context.Variables.Set("available_functions", relevantFunctionsManual); - - this._context.Variables.Update(goal); - - var planResult = await this._functionFlowFunction.InvokeAsync(this._context, cancellationToken: cancellationToken).ConfigureAwait(false); - - if (planResult.ErrorOccurred) - { - throw new PlanningException(PlanningException.ErrorCodes.CreatePlanError, $"Error creating plan for goal: {planResult.LastErrorDescription}", planResult.LastException); - } - - string planResultString = planResult.Result.Trim(); - - try - { - var getSkillFunction = this.Config.GetSkillFunction ?? SequentialPlanParser.GetSkillFunction(this._context); - var plan = planResultString.ToPlanFromXml(goal, getSkillFunction, this.Config.AllowMissingFunctions); - - if (plan.Steps.Count == 0) - { - throw new PlanningException(PlanningException.ErrorCodes.CreatePlanError, $"Not possible to create plan for goal with available functions.\nGoal:{goal}\nFunctions:\n{relevantFunctionsManual}"); - } - - return plan; - } - catch (PlanningException planException) when (planException.ErrorCode == PlanningException.ErrorCodes.CreatePlanError) - { - throw; - } - catch (PlanningException planException) when (planException.ErrorCode == PlanningException.ErrorCodes.InvalidPlan || - planException.ErrorCode == PlanningException.ErrorCodes.InvalidGoal) - { - throw new PlanningException(PlanningException.ErrorCodes.CreatePlanError, "Unable to create plan", planException); - } - catch (Exception e) - { - throw new PlanningException(PlanningException.ErrorCodes.UnknownError, "Unknown error creating plan", e); - } - } - - private SequentialPlannerConfig Config { get; } - - private readonly SKContext _context; - - /// - /// the function flow semantic function, which takes a goal and creates an xml plan that can be executed - /// - private readonly ISKFunction _functionFlowFunction; - - /// - /// The name to use when creating semantic functions that are restricted from plan creation - /// - private const string RestrictedSkillName = "SequentialPlanner_Excluded"; -} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlannerConfig.cs b/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlannerConfig.cs deleted file mode 100644 index 6a560eb65d03..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlannerConfig.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Planning.Sequential; - -/// -/// Common configuration for planner instances. -/// -public sealed class SequentialPlannerConfig -{ - /// - /// The minimum relevancy score for a function to be considered - /// - /// - /// Depending on the embeddings engine used, the user ask, the step goal - /// and the functions available, this value may need to be adjusted. - /// For default, this is set to null to exhibit previous behavior. - /// - public double? RelevancyThreshold { get; set; } - - /// - /// The maximum number of relevant functions to include in the plan. - /// - /// - /// Limits the number of relevant functions as result of semantic - /// search included in the plan creation request. - /// will be included - /// in the plan regardless of this limit. - /// - public int MaxRelevantFunctions { get; set; } = 100; - - /// - /// A list of skills to exclude from the plan creation request. - /// - public HashSet ExcludedSkills { get; } = new(); - - /// - /// A list of functions to exclude from the plan creation request. - /// - public HashSet ExcludedFunctions { get; } = new(); - - /// - /// A list of functions to include in the plan creation request. - /// - public HashSet IncludedFunctions { get; } = new(); - - /// - /// The maximum number of tokens to allow in a plan. - /// - public int? MaxTokens { get; set; } - - /// - /// Whether to allow missing functions in the plan on creation. - /// If set to true, the plan will be created with missing functions as no-op steps. - /// If set to false (default), the plan creation will fail if any functions are missing. - /// - public bool AllowMissingFunctions { get; set; } = false; - - /// - /// Semantic memory to use for function lookup (optional). - /// - public ISemanticTextMemory Memory { get; set; } = NullMemory.Instance; - - /// - /// Optional callback to get the available functions for planning. - /// - public Func>>? GetAvailableFunctionsAsync { get; set; } - - /// - /// Optional callback to get a function by name. - /// - public Func? GetSkillFunction { get; set; } -} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlannerExtensions.cs b/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlannerExtensions.cs deleted file mode 100644 index 4b212c618c8a..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/SequentialPlannerExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Logging; - -namespace Microsoft.SemanticKernel.Planning.Sequential; - -/// -/// Extension methods for class. -/// -public static class SequentialPlannerExtensions -{ - /// - /// Returns decorated instance of with enabled instrumentation. - /// - /// Instance of to decorate. - /// Optional logger. - public static ISequentialPlanner WithInstrumentation(this ISequentialPlanner planner, ILogger? logger = null) - { - return new InstrumentedSequentialPlanner(planner, logger); - } -} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/skprompt.txt b/dotnet/src/Extensions/Planning.SequentialPlanner/skprompt.txt deleted file mode 100644 index 4e325abfe8bf..000000000000 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/skprompt.txt +++ /dev/null @@ -1,55 +0,0 @@ -Create an XML plan step by step, to satisfy the goal given, with the available functions. - -[AVAILABLE FUNCTIONS] - -{{$available_functions}} - -[END AVAILABLE FUNCTIONS] - -To create a plan, follow these steps: -0. The plan should be as short as possible. -1. From a create a as a series of . -2. A plan has 'INPUT' available in context variables by default. -3. Before using any function in a plan, check that it is present in the [AVAILABLE FUNCTIONS] list. If it is not, do not use it. -4. Only use functions that are required for the given goal. -5. Append an "END" XML comment at the end of the plan after the final closing tag. -6. Always output valid XML that can be parsed by an XML parser. -7. If a plan cannot be created with the [AVAILABLE FUNCTIONS], return . - -All plans take the form of: - - - - - - - - (... etc ...) - - - -To call a function, follow these steps: -1. A function has one or more named parameters and a single 'output' which are all strings. Parameter values should be xml escaped. -2. To save an 'output' from a , to pass into a future , use -3. To save an 'output' from a , to return as part of a plan result, use -4. Use a '$' to reference a context variable in a parameter, e.g. when `INPUT='world'` the parameter 'Hello $INPUT' will evaluate to `Hello world`. -5. Functions do not have access to the context variables of other functions. Do not attempt to use context variables as arrays or objects. Instead, use available functions to extract specific elements or properties from context variables. - -DO NOT DO THIS, THE PARAMETER VALUE IS NOT XML ESCAPED: - - -DO NOT DO THIS, THE PARAMETER VALUE IS ATTEMPTING TO USE A CONTEXT VARIABLE AS AN ARRAY/OBJECT: - - -Here is a valid example of how to call a function "_Function_.Name" with a single input and save its output: - - -Here is a valid example of how to call a function "FunctionName2" with a single input and return its output as part of the plan result: - - -Here is a valid example of how to call a function "Name3" with multiple inputs: - - -Begin! - -{{$input}} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/EmbeddedResource.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/EmbeddedResource.cs deleted file mode 100644 index 9e8a711bdb9c..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/EmbeddedResource.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.IO; -using System.Reflection; - -namespace Microsoft.SemanticKernel.Planning.Stepwise; - -internal static class EmbeddedResource -{ - private static readonly string? s_namespace = typeof(EmbeddedResource).Namespace; - - internal static string Read(string name) - { - var assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly; - if (assembly == null) { throw new PlanningException(PlanningException.ErrorCodes.InvalidConfiguration, $"[{s_namespace}] {name} assembly not found"); } - - using Stream? resource = assembly.GetManifestResourceStream($"{s_namespace}." + name); - if (resource == null) { throw new PlanningException(PlanningException.ErrorCodes.InvalidConfiguration, $"[{s_namespace}] {name} resource not found"); } - - using var reader = new StreamReader(resource); - return reader.ReadToEnd(); - } -} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/IStepwisePlanner.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/IStepwisePlanner.cs deleted file mode 100644 index f63285cf3839..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/IStepwisePlanner.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.SemanticKernel.Planning.Stepwise; - -/// -/// Interface for planner that creates a Stepwise plan using Mrkl systems. -/// -public interface IStepwisePlanner -{ - /// - /// Create a plan for a goal. - /// - /// The goal to create a plan for. - /// The plan. - /// Thrown when the plan cannot be created. - Plan CreatePlan(string goal); -} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Planning.StepwisePlanner.csproj b/dotnet/src/Extensions/Planning.StepwisePlanner/Planning.StepwisePlanner.csproj deleted file mode 100644 index accf224de931..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/Planning.StepwisePlanner.csproj +++ /dev/null @@ -1,31 +0,0 @@ - - - - - Microsoft.SemanticKernel.Planning.StepwisePlanner - Microsoft.SemanticKernel.Planning.Stepwise - netstandard2.0 - - - - - - - - Semantic Kernel - Stepwise Planner - Semantic Kernel Stepwise Planner - - - - - - - - - - - Always - - - - diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/config.json b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/config.json deleted file mode 100644 index 51ef104e4597..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/config.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "schema": 1, - "description": "Given a request or command or goal generate multi-step plan to reach the goal. After each step LLM is called to perform the reasoning for the next step.", - "type": "completion", - "completion": { - "max_tokens": 1024, - "temperature": 0, - "top_p": 0, - "presence_penalty": 0, - "frequency_penalty": 0, - "stop_sequences": ["[OBSERVATION]", "\n[THOUGHT]"] - }, - "input": { - "parameters": [ - { - "name": "question", - "description": "The question to answer", - "defaultValue": "" - }, - { - "name": "agentScratchPad", - "description": "The agent's scratch pad", - "defaultValue": "" - }, - { - "name": "functionDescriptions", - "description": "The manual of the agent's functions", - "defaultValue": "" - } - ] - } -} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/skprompt.txt b/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/skprompt.txt deleted file mode 100644 index 723b68d74c6a..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/Skills/StepwiseStep/skprompt.txt +++ /dev/null @@ -1,49 +0,0 @@ -[INSTRUCTION] -Answer the following questions as accurately as possible using the provided functions. - -[AVAILABLE FUNCTIONS] -The function definitions below are in the following format: -: - - : - - ... - -{{$functionDescriptions}} -[END AVAILABLE FUNCTIONS] - -[USAGE INSTRUCTIONS] -To use the functions, specify a JSON blob representing an action. The JSON blob should contain an "action" key with the name of the function to use, and an "action_variables" key with a JSON object of string values to use when calling the function. -Do not call functions directly; they must be invoked through an action. -The "action_variables" value should always include an "input" key, even if the input value is empty. Additional keys in the "action_variables" value should match the defined [PARAMETERS] of the named "action" in [AVAILABLE FUNCTIONS]. -Dictionary values in "action_variables" must be strings and represent the actual values to be passed to the function. -Ensure that the $JSON_BLOB contains only a SINGLE action; do NOT return multiple actions. -IMPORTANT: Use only the available functions listed in the [AVAILABLE FUNCTIONS] section. Do not attempt to use any other functions that are not specified. - -Here is an example of a valid $JSON_BLOB: -{ - "action": "FUNCTION.NAME", - "action_variables": {"INPUT": "some input", "PARAMETER_NAME": "some value", "PARAMETER_NAME_2": "42"} -} -[END USAGE INSTRUCTIONS] -[END INSTRUCTION] - -[THOUGHT PROCESS] -[QUESTION] -the input question I must answer -[THOUGHT] -To solve this problem, I should carefully analyze the given question and identify the necessary steps. Any facts I discover earlier in my thought process should be repeated here to keep them readily available. -[ACTION] -$JSON_BLOB -[OBSERVATION] -The result of the action will be provided here. -... (These Thought/Action/Observation can repeat until the final answer is reached.) -[FINAL ANSWER] -Once I have gathered all the necessary observations and performed any required actions, I can provide the final answer in a clear and human-readable format. -[END THOUGHT PROCESS] - -Let's break down the problem step by step and think about the best approach. Questions and observations should be followed by a single thought and an optional single action to take. - -Begin! - -[QUESTION] -{{$question}} -{{$agentScratchPad}} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlanner.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlanner.cs deleted file mode 100644 index 623bd853a7ec..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlanner.cs +++ /dev/null @@ -1,498 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Globalization; -using System.Linq; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning.Stepwise; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.SkillDefinition; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using NS of Plan -namespace Microsoft.SemanticKernel.Planning; -#pragma warning restore IDE0130 - -/// -/// A planner that creates a Stepwise plan using Mrkl systems. -/// -/// -/// An implementation of a Mrkl system as described in https://arxiv.org/pdf/2205.00445.pdf -/// -public class StepwisePlanner : IStepwisePlanner -{ - /// - /// Initialize a new instance of the class. - /// - /// The semantic kernel instance. - /// Optional configuration object - /// Optional prompt override - /// Optional prompt config override - public StepwisePlanner( - IKernel kernel, - StepwisePlannerConfig? config = null, - string? prompt = null, - PromptTemplateConfig? promptUserConfig = null) - { - Verify.NotNull(kernel); - this._kernel = kernel; - - this.Config = config ?? new(); - this.Config.ExcludedSkills.Add(RestrictedSkillName); - - var promptConfig = promptUserConfig ?? new PromptTemplateConfig(); - var promptTemplate = prompt ?? EmbeddedResource.Read("Skills.StepwiseStep.skprompt.txt"); - - if (promptUserConfig == null) - { - string promptConfigString = EmbeddedResource.Read("Skills.StepwiseStep.config.json"); - if (!string.IsNullOrEmpty(promptConfigString)) - { - promptConfig = PromptTemplateConfig.FromJson(promptConfigString); - } - } - - promptConfig.Completion.MaxTokens = this.Config.MaxTokens; - - this._systemStepFunction = this.ImportSemanticFunction(this._kernel, "StepwiseStep", promptTemplate, promptConfig); - this._nativeFunctions = this._kernel.ImportSkill(this, RestrictedSkillName); - - this._context = this._kernel.CreateNewContext(); - this._logger = this._kernel.Logger; - } - - /// - public Plan CreatePlan(string goal) - { - if (string.IsNullOrEmpty(goal)) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidGoal, "The goal specified is empty"); - } - - string functionDescriptions = this.GetFunctionDescriptions(); - - Plan planStep = new(this._nativeFunctions["ExecutePlan"]); - planStep.Parameters.Set("functionDescriptions", functionDescriptions); - planStep.Parameters.Set("question", goal); - - planStep.Outputs.Add("agentScratchPad"); - planStep.Outputs.Add("stepCount"); - planStep.Outputs.Add("skillCount"); - planStep.Outputs.Add("stepsTaken"); - - Plan plan = new(goal); - - plan.AddSteps(planStep); - - return plan; - } - - [SKFunction, SKName("ExecutePlan"), Description("Execute a plan")] - public async Task ExecutePlanAsync( - [Description("The question to answer")] - string question, - [Description("List of tool descriptions")] - string functionDescriptions, - SKContext context) - { - var stepsTaken = new List(); - if (!string.IsNullOrEmpty(question)) - { - for (int i = 0; i < this.Config.MaxIterations; i++) - { - var scratchPad = this.CreateScratchPad(question, stepsTaken); - - context.Variables.Set("agentScratchPad", scratchPad); - - var llmResponse = (await this._systemStepFunction.InvokeAsync(context).ConfigureAwait(false)); - - if (llmResponse.ErrorOccurred) - { - var exception = new PlanningException(PlanningException.ErrorCodes.UnknownError, $"Error occurred while executing stepwise plan: {llmResponse.LastErrorDescription}", llmResponse.LastException); - context.Fail(exception.Message, exception); - return context; - } - - string actionText = llmResponse.Result.Trim(); - this._logger?.LogTrace("Response: {ActionText}", actionText); - - var nextStep = this.ParseResult(actionText); - stepsTaken.Add(nextStep); - - if (!string.IsNullOrEmpty(nextStep.FinalAnswer)) - { - this._logger?.LogTrace("Final Answer: {FinalAnswer}", nextStep.FinalAnswer); - - context.Variables.Update(nextStep.FinalAnswer); - var updatedScratchPlan = this.CreateScratchPad(question, stepsTaken); - context.Variables.Set("agentScratchPad", updatedScratchPlan); - - // Add additional results to the context - this.AddExecutionStatsToContext(stepsTaken, context); - - return context; - } - - this._logger?.LogTrace("Thought: {Thought}", nextStep.Thought); - - if (!string.IsNullOrEmpty(nextStep!.Action!)) - { - this._logger?.LogInformation("Action: {Action}. Iteration: {Iteration}.", nextStep.Action, i + 1); - this._logger?.LogTrace("Action: {Action}({ActionVariables}). Iteration: {Iteration}.", - nextStep.Action, JsonSerializer.Serialize(nextStep.ActionVariables), i + 1); - - try - { - await Task.Delay(this.Config.MinIterationTimeMs).ConfigureAwait(false); - var result = await this.InvokeActionAsync(nextStep.Action!, nextStep!.ActionVariables!).ConfigureAwait(false); - - if (string.IsNullOrEmpty(result)) - { - nextStep.Observation = "Got no result from action"; - } - else - { - nextStep.Observation = result; - } - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - nextStep.Observation = $"Error invoking action {nextStep.Action} : {ex.Message}"; - this._logger?.LogWarning(ex, "Error invoking action {Action}", nextStep.Action); - } - - this._logger?.LogTrace("Observation: {Observation}", nextStep.Observation); - } - else - { - this._logger?.LogInformation("Action: No action to take"); - } - - // sleep 3 seconds - await Task.Delay(this.Config.MinIterationTimeMs).ConfigureAwait(false); - } - - context.Variables.Update($"Result not found, review _stepsTaken to see what happened.\n{JsonSerializer.Serialize(stepsTaken)}"); - } - else - { - context.Variables.Update("Question not found."); - } - - return context; - } - - public virtual SystemStep ParseResult(string input) - { - var result = new SystemStep - { - OriginalResponse = input - }; - - // Extract final answer - Match finalAnswerMatch = s_finalAnswerRegex.Match(input); - - if (finalAnswerMatch.Success) - { - result.FinalAnswer = finalAnswerMatch.Groups[1].Value.Trim(); - return result; - } - - // Extract thought - Match thoughtMatch = s_thoughtRegex.Match(input); - - if (thoughtMatch.Success) - { - result.Thought = thoughtMatch.Value.Trim(); - } - else if (!input.Contains(Action)) - { - result.Thought = input; - } - else - { - throw new InvalidOperationException("Unexpected input format"); - } - - result.Thought = result.Thought.Replace(Thought, string.Empty).Trim(); - - // Extract action - Match actionMatch = s_actionRegex.Match(input); - - if (actionMatch.Success) - { - var json = actionMatch.Groups[1].Value.Trim(); - - try - { - var systemStepResults = JsonSerializer.Deserialize(json); - - if (systemStepResults == null) - { - result.Observation = $"System step parsing error, empty JSON: {json}"; - } - else - { - result.Action = systemStepResults.Action; - result.ActionVariables = systemStepResults.ActionVariables; - } - } - catch (JsonException) - { - result.Observation = $"System step parsing error, invalid JSON: {json}"; - } - } - - if (string.IsNullOrEmpty(result.Thought) && string.IsNullOrEmpty(result.Action)) - { - result.Observation = "System step error, no thought or action found. Please give a valid thought and/or action."; - } - - return result; - } - - private void AddExecutionStatsToContext(List stepsTaken, SKContext context) - { - context.Variables.Set("stepCount", stepsTaken.Count.ToString(CultureInfo.InvariantCulture)); - context.Variables.Set("stepsTaken", JsonSerializer.Serialize(stepsTaken)); - - Dictionary actionCounts = new(); - foreach (var step in stepsTaken) - { - if (string.IsNullOrEmpty(step.Action)) { continue; } - - _ = actionCounts.TryGetValue(step.Action!, out int currentCount); - actionCounts[step.Action!] = ++currentCount; - } - - var skillCallListWithCounts = string.Join(", ", actionCounts.Keys.Select(skill => - $"{skill}({actionCounts[skill]})")); - - var skillCallCountStr = actionCounts.Values.Sum().ToString(CultureInfo.InvariantCulture); - - context.Variables.Set("skillCount", $"{skillCallCountStr} ({skillCallListWithCounts})"); - } - - private string CreateScratchPad(string question, List stepsTaken) - { - if (stepsTaken.Count == 0) - { - return string.Empty; - } - - var scratchPadLines = new List(); - - // Add the original first thought - scratchPadLines.Add(ScratchPadPrefix); - scratchPadLines.Add($"{Thought} {stepsTaken[0].Thought}"); - - // Keep track of where to insert the next step - var insertPoint = scratchPadLines.Count; - - // Keep the most recent steps in the scratch pad. - for (var i = stepsTaken.Count - 1; i >= 0; i--) - { - if (scratchPadLines.Count / 4.0 > (this.Config.MaxTokens * 0.75)) - { - this._logger.LogDebug("Scratchpad is too long, truncating. Skipping {CountSkipped} steps.", i + 1); - break; - } - - var s = stepsTaken[i]; - - if (!string.IsNullOrEmpty(s.Observation)) - { - scratchPadLines.Insert(insertPoint, $"{Observation} {s.Observation}"); - } - - if (!string.IsNullOrEmpty(s.Action)) - { - scratchPadLines.Insert(insertPoint, $"{Action} {{\"action\": \"{s.Action}\",\"action_variables\": {JsonSerializer.Serialize(s.ActionVariables)}}}"); - } - - if (i != 0) - { - scratchPadLines.Insert(insertPoint, $"{Thought} {s.Thought}"); - } - } - - var scratchPad = string.Join("\n", scratchPadLines).Trim(); - - if (!string.IsNullOrWhiteSpace(scratchPad)) - { - this._logger.LogTrace("Scratchpad: {ScratchPad}", scratchPad); - } - - return scratchPad; - } - - private async Task InvokeActionAsync(string actionName, Dictionary actionVariables) - { - var availableFunctions = this.GetAvailableFunctions(); - var targetFunction = availableFunctions.FirstOrDefault(f => ToFullyQualifiedName(f) == actionName); - if (targetFunction == null) - { - throw new PlanningException(PlanningException.ErrorCodes.UnknownError, $"The function '{actionName}' was not found."); - } - - try - { - var function = this._kernel.Func(targetFunction.SkillName, targetFunction.Name); - var actionContext = this.CreateActionContext(actionVariables); - - var result = await function.InvokeAsync(actionContext).ConfigureAwait(false); - - if (result.ErrorOccurred) - { - this._logger?.LogError("Error occurred: {Error}", result.LastException); - return $"Error occurred: {result.LastException}"; - } - - this._logger?.LogTrace("Invoked {FunctionName}. Result: {Result}", targetFunction.Name, result.Result); - - return result.Result; - } - catch (Exception e) when (!e.IsCriticalException()) - { - this._logger?.LogError(e, "Something went wrong in system step: {0}.{1}. Error: {2}", targetFunction.SkillName, targetFunction.Name, e.Message); - return $"Something went wrong in system step: {targetFunction.SkillName}.{targetFunction.Name}. Error: {e.Message} {e.InnerException.Message}"; - } - } - - private SKContext CreateActionContext(Dictionary actionVariables) - { - var actionContext = this._kernel.CreateNewContext(); - if (actionVariables != null) - { - foreach (var kvp in actionVariables) - { - actionContext.Variables.Set(kvp.Key, kvp.Value); - } - } - - return actionContext; - } - - private IEnumerable GetAvailableFunctions() - { - FunctionsView functionsView = this._context.Skills!.GetFunctionsView(); - - var excludedSkills = this.Config.ExcludedSkills ?? new(); - var excludedFunctions = this.Config.ExcludedFunctions ?? new(); - - var availableFunctions = - functionsView.NativeFunctions - .Concat(functionsView.SemanticFunctions) - .SelectMany(x => x.Value) - .Where(s => !excludedSkills.Contains(s.SkillName) && !excludedFunctions.Contains(s.Name)) - .OrderBy(x => x.SkillName) - .ThenBy(x => x.Name); - return availableFunctions; - } - - private string GetFunctionDescriptions() - { - var availableFunctions = this.GetAvailableFunctions(); - - string functionDescriptions = string.Join("\n", availableFunctions.Select(x => ToManualString(x))); - return functionDescriptions; - } - - private ISKFunction ImportSemanticFunction(IKernel kernel, string functionName, string promptTemplate, PromptTemplateConfig config) - { - var template = new PromptTemplate(promptTemplate, config, kernel.PromptTemplateEngine); - var functionConfig = new SemanticFunctionConfig(config, template); - - return kernel.RegisterSemanticFunction(RestrictedSkillName, functionName, functionConfig); - } - - private static string ToManualString(FunctionView function) - { - var inputs = string.Join("\n", function.Parameters.Select(parameter => - { - var defaultValueString = string.IsNullOrEmpty(parameter.DefaultValue) ? string.Empty : $"(default='{parameter.DefaultValue}')"; - return $" - {parameter.Name}: {parameter.Description} {defaultValueString}"; - })); - - var functionDescription = function.Description.Trim(); - - if (string.IsNullOrEmpty(inputs)) - { - return $"{ToFullyQualifiedName(function)}: {functionDescription}\n"; - } - - return $"{ToFullyQualifiedName(function)}: {functionDescription}\n{inputs}\n"; - } - - private static string ToFullyQualifiedName(FunctionView function) - { - return $"{function.SkillName}.{function.Name}"; - } - - /// - /// The configuration for the StepwisePlanner - /// - private StepwisePlannerConfig Config { get; } - - // Context used to access the list of functions in the kernel - private readonly SKContext _context; - private readonly IKernel _kernel; - private readonly ILogger _logger; - - /// - /// Planner native functions - /// - private IDictionary _nativeFunctions = new Dictionary(); - - /// - /// System step function for Plan execution - /// - private ISKFunction _systemStepFunction; - - /// - /// The name to use when creating semantic functions that are restricted from plan creation - /// - private const string RestrictedSkillName = "StepwisePlanner_Excluded"; - - /// - /// The Action tag - /// - private const string Action = "[ACTION]"; - - /// - /// The Thought tag - /// - private const string Thought = "[THOUGHT]"; - - /// - /// The Observation tag - /// - private const string Observation = "[OBSERVATION]"; - - /// - /// The prefix used for the scratch pad - /// - private const string ScratchPadPrefix = "This was my previous work (but they haven't seen any of it! They only see what I return as final answer):"; - - /// - /// The regex for parsing the action response - /// - private static readonly Regex s_actionRegex = new(@"\[ACTION\][^{}]*({(?:[^{}]*{[^{}]*})*[^{}]*})", RegexOptions.Singleline); - - /// - /// The regex for parsing the thought response - /// - private static readonly Regex s_thoughtRegex = new(@"(\[THOUGHT\])?(?.+?)(?=\[ACTION\]|$)", RegexOptions.Singleline); - - /// - /// The regex for parsing the final answer response - /// - private static readonly Regex s_finalAnswerRegex = new(@"\[FINAL ANSWER\](?.+)", RegexOptions.Singleline); -} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerConfig.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerConfig.cs deleted file mode 100644 index c50a1dbec2e1..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerConfig.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; - -namespace Microsoft.SemanticKernel.Planning.Stepwise; - -/// -/// Configuration for Stepwise planner instances. -/// -public sealed class StepwisePlannerConfig -{ - /// - /// The minimum relevancy score for a function to be considered - /// - /// - /// Depending on the embeddings engine used, the user ask, the step goal - /// and the functions available, this value may need to be adjusted. - /// For default, this is set to null to exhibit previous behavior. - /// - public double? RelevancyThreshold { get; set; } - - /// - /// The maximum number of relevant functions to include in the plan. - /// - /// - /// Limits the number of relevant functions as result of semantic - /// search included in the plan creation request. - /// will be included - /// in the plan regardless of this limit. - /// - public int MaxRelevantFunctions { get; set; } = 100; - - /// - /// A list of skills to exclude from the plan creation request. - /// - public HashSet ExcludedSkills { get; } = new(); - - /// - /// A list of functions to exclude from the plan creation request. - /// - public HashSet ExcludedFunctions { get; } = new(); - - /// - /// A list of functions to include in the plan creation request. - /// - public HashSet IncludedFunctions { get; } = new(); - - /// - /// The maximum number of tokens to allow in a plan. - /// - public int MaxTokens { get; set; } = 1024; - - /// - /// The maximum number of iterations to allow in a plan. - /// - public int MaxIterations { get; set; } = 100; - - /// - /// The minimum time to wait between iterations in milliseconds. - /// - public int MinIterationTimeMs { get; set; } = 0; -} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerExtensions.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerExtensions.cs deleted file mode 100644 index c982cb054b62..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/StepwisePlannerExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Logging; - -namespace Microsoft.SemanticKernel.Planning.Stepwise; - -/// -/// Extension methods for class. -/// -public static class StepwisePlannerExtensions -{ - /// - /// Returns decorated instance of with enabled instrumentation. - /// - /// Instance of to decorate. - /// Optional logger. - public static IStepwisePlanner WithInstrumentation(this IStepwisePlanner planner, ILogger? logger = null) - { - return new InstrumentedStepwisePlanner(planner, logger); - } -} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/SystemStep.cs b/dotnet/src/Extensions/Planning.StepwisePlanner/SystemStep.cs deleted file mode 100644 index 3fc8ed2dffd3..000000000000 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/SystemStep.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace Microsoft.SemanticKernel.Planning.Stepwise; - -/// -/// A step in a Stepwise plan. -/// -public class SystemStep -{ - /// - /// Gets or sets the step number. - /// - [JsonPropertyName("thought")] - public string? Thought { get; set; } - - /// - /// Gets or sets the action of the step - /// - [JsonPropertyName("action")] - public string? Action { get; set; } - - /// - /// Gets or sets the variables for the action - /// - [JsonPropertyName("action_variables")] - public Dictionary? ActionVariables { get; set; } - - /// - /// Gets or sets the output of the action - /// - [JsonPropertyName("observation")] - public string? Observation { get; set; } - - /// - /// Gets or sets the output of the system - /// - [JsonPropertyName("final_answer")] - public string? FinalAnswer { get; set; } - - /// - /// The raw response from the action - /// - [JsonPropertyName("original_response")] - public string? OriginalResponse { get; set; } -} diff --git a/dotnet/src/Extensions/Reliability.Basic/BasicHttpRetryHandler.cs b/dotnet/src/Extensions/Reliability.Basic/BasicHttpRetryHandler.cs new file mode 100644 index 000000000000..69d1073c02fa --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Basic/BasicHttpRetryHandler.cs @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Microsoft.SemanticKernel.Reliability.Basic; + +/// +/// Handler that retries HTTP requests based on a . +/// +public sealed class BasicHttpRetryHandler : DelegatingHandler +{ + /// + /// Initializes a new instance of the class. + /// + /// The retry configuration. + /// The to use for logging. If null, no logging will be performed. + internal BasicHttpRetryHandler(BasicRetryConfig? config = null, ILoggerFactory? loggerFactory = null) + : this(config ?? new(), loggerFactory, null, null) + { + } + + internal BasicHttpRetryHandler( + BasicRetryConfig config, + ILoggerFactory? loggerFactory = null, + IDelayProvider? delayProvider = null, + ITimeProvider? timeProvider = null) + { + this._config = config; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger() : NullLogger.Instance; + this._delayProvider = delayProvider ?? new TaskDelayProvider(); + this._timeProvider = timeProvider ?? new DefaultTimeProvider(); + } + + /// + /// Executes the action with retry logic + /// + /// + /// The request is retried if it throws an exception that is a retryable exception. + /// If the request throws an exception that is not a retryable exception, it is not retried. + /// If the request returns a response with a retryable error code, it is retried. + /// If the request returns a response with a non-retryable error code, it is not retried. + /// If the exception contains a RetryAfter header, the request is retried after the specified delay. + /// If configured to use exponential backoff, the delay is doubled for each retry. + /// + /// The request. + /// The cancellation token. + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + int retryCount = 0; + + var start = this._timeProvider.GetCurrentTime(); + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + TimeSpan waitFor; + string reason; + HttpResponseMessage? response = null; + try + { + response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + + // If the request does not require a retry then we're done + if (!this.ShouldRetry(response.StatusCode)) + { + return response; + } + + reason = response.StatusCode.ToString(); + + // If the retry count is greater than the max retry count then we'll + // just return + if (retryCount >= this._config.MaxRetryCount) + { + this._logger.LogError( + "Error executing request, max retry count reached. Reason: {0}", reason); + return response; + } + + // If the retry delay is longer than the total timeout, then we'll + // just return + if (!this.HasTimeForRetry(start, retryCount, response, out waitFor)) + { + var timeTaken = this._timeProvider.GetCurrentTime() - start; + this._logger.LogError( + "Error executing request, max total retry time reached. Reason: {0}. Time spent: {1}ms", reason, + timeTaken.TotalMilliseconds); + return response; + } + } + catch (Exception e) when (this.ShouldRetry(e) || this.ShouldRetry(e.InnerException)) + { + reason = e.GetType().ToString(); + if (retryCount >= this._config.MaxRetryCount) + { + this._logger.LogError(e, + "Error executing request, max retry count reached. Reason: {0}", reason); + throw; + } + else if (!this.HasTimeForRetry(start, retryCount, response, out waitFor)) + { + var timeTaken = this._timeProvider.GetCurrentTime() - start; + this._logger.LogError( + "Error executing request, max total retry time reached. Reason: {0}. Time spent: {1}ms", reason, + timeTaken.TotalMilliseconds); + throw; + } + } + + // If the request requires a retry then we'll retry + this._logger.LogWarning( + "Error executing action [attempt {0} of {1}]. Reason: {2}. Will retry after {3}ms", + retryCount + 1, + this._config.MaxRetryCount, + reason, + waitFor.TotalMilliseconds); + + // Increase retryCount + retryCount++; + + response?.Dispose(); + + // Delay + await this._delayProvider.DelayAsync(waitFor, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Interface for a delay provider, primarily to enable unit testing. + /// + internal interface IDelayProvider + { + Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken); + } + + internal sealed class TaskDelayProvider : IDelayProvider + { + public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) + { + return Task.Delay(delay, cancellationToken); + } + } + + /// + /// Interface for a time provider, primarily to enable unit testing. + /// + internal interface ITimeProvider + { + DateTimeOffset GetCurrentTime(); + } + + internal sealed class DefaultTimeProvider : ITimeProvider + { + public DateTimeOffset GetCurrentTime() + { + return DateTimeOffset.UtcNow; + } + } + + private readonly BasicRetryConfig _config; + private readonly ILogger _logger; + private readonly IDelayProvider _delayProvider; + private readonly ITimeProvider _timeProvider; + + /// + /// Get the wait time for the next retry. + /// + /// Current retry count + /// The response message that potentially contains RetryAfter header. + private TimeSpan GetWaitTime(int retryCount, HttpResponseMessage? response) + { + // If the response contains a RetryAfter header, use that value + // Otherwise, use the configured min retry delay + var retryAfter = response?.Headers.RetryAfter?.Date.HasValue == true + ? response?.Headers.RetryAfter?.Date - this._timeProvider.GetCurrentTime() + : (response?.Headers.RetryAfter?.Delta) ?? this._config.MinRetryDelay; + retryAfter ??= this._config.MinRetryDelay; + + // If the retry delay is longer than the max retry delay, use the max retry delay + var timeToWait = retryAfter > this._config.MaxRetryDelay + ? this._config.MaxRetryDelay + : retryAfter < this._config.MinRetryDelay + ? this._config.MinRetryDelay + : retryAfter ?? default; + + // If exponential backoff is enabled, and the server didn't provide a RetryAfter header, double the delay for each retry + if (this._config.UseExponentialBackoff + && response?.Headers.RetryAfter?.Date is null + && response?.Headers.RetryAfter?.Delta is null) + { + for (var backoffRetryCount = 1; backoffRetryCount < retryCount + 1; backoffRetryCount++) + { + timeToWait = timeToWait.Add(timeToWait); + } + } + + return timeToWait; + } + + /// + /// Determines if there is time left for a retry. + /// + /// The start time of the original request. + /// The current retry count. + /// The response message that potentially contains RetryAfter header. + /// The wait time for the next retry. + /// True if there is time left for a retry, false otherwise. + private bool HasTimeForRetry(DateTimeOffset start, int retryCount, HttpResponseMessage? response, out TimeSpan waitFor) + { + waitFor = this.GetWaitTime(retryCount, response); + var currentTIme = this._timeProvider.GetCurrentTime(); + var result = currentTIme - start + waitFor; + + return result < this._config.MaxTotalRetryTime; + } + + private bool ShouldRetry(HttpStatusCode statusCode) + { + return this._config.RetryableStatusCodes.Contains(statusCode); + } + + private bool ShouldRetry(Exception? exception) + { + return exception != null && this._config.RetryableExceptionTypes.Contains(exception.GetType()); + } +} diff --git a/dotnet/src/Extensions/Reliability.Basic/BasicHttpRetryHandlerFactory.cs b/dotnet/src/Extensions/Reliability.Basic/BasicHttpRetryHandlerFactory.cs new file mode 100644 index 000000000000..dd87f9913265 --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Basic/BasicHttpRetryHandlerFactory.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Http; + +namespace Microsoft.SemanticKernel.Reliability.Basic; + +/// +/// Internal factory for creating instances. +/// +public sealed class BasicHttpRetryHandlerFactory : HttpHandlerFactory +{ + /// + /// Gets the singleton instance of . + /// + public static BasicHttpRetryHandlerFactory Instance { get; } = new BasicHttpRetryHandlerFactory(); + + /// + /// Creates a new instance of with the provided configuration. + /// + /// Http retry configuration + internal BasicHttpRetryHandlerFactory(BasicRetryConfig? config = null) + { + this.Config = config ?? new(); + } + + /// + /// Creates a new instance of with the default configuration. + /// + /// Logger factory + /// Returns the created handler + public override DelegatingHandler Create(ILoggerFactory? loggerFactory = null) + { + return new BasicHttpRetryHandler(this.Config, loggerFactory); + } + + /// + /// Creates a new instance of with a specified configuration. + /// + /// Specific configuration + /// Logger factory + /// Returns the created handler + public DelegatingHandler Create(BasicRetryConfig config, ILoggerFactory? loggerFactory = null) + { + Verify.NotNull(config, nameof(config)); + + return new BasicHttpRetryHandler(config, loggerFactory); + } + + /// + /// Default retry configuration used when creating a new instance of . + /// + internal BasicRetryConfig Config { get; } +} diff --git a/dotnet/src/Extensions/Reliability.Basic/BasicRetryConfig.cs b/dotnet/src/Extensions/Reliability.Basic/BasicRetryConfig.cs new file mode 100644 index 000000000000..a20d5a0f9c82 --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Basic/BasicRetryConfig.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using Microsoft.SemanticKernel.Diagnostics; + +namespace Microsoft.SemanticKernel.Reliability.Basic; + +/// +/// Retry configuration for DefaultKernelRetryHandler that uses RetryAfter header when present. +/// +public sealed record BasicRetryConfig +{ + /// + /// Maximum number of retries. + /// + /// Thrown when value is negative. + public int MaxRetryCount + { + get => this._maxRetryCount; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(this.MaxRetryCount), "Max retry count cannot be negative."); + } + + this._maxRetryCount = value; + } + } + + /// + /// Minimum delay between retries. + /// + public TimeSpan MinRetryDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Maximum delay between retries. + /// + public TimeSpan MaxRetryDelay { get; set; } = TimeSpan.FromSeconds(60); + + /// + /// Maximum total time spent retrying. + /// + public TimeSpan MaxTotalRetryTime { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Whether to use exponential backoff or not. + /// + public bool UseExponentialBackoff { get; set; } + + /// + /// List of status codes that should be retried. + /// + public List RetryableStatusCodes { get; set; } = new() + { + (HttpStatusCode)HttpStatusCodeType.RequestTimeout, + (HttpStatusCode)HttpStatusCodeType.ServiceUnavailable, + (HttpStatusCode)HttpStatusCodeType.GatewayTimeout, + (HttpStatusCode)HttpStatusCodeType.TooManyRequests, + (HttpStatusCode)HttpStatusCodeType.BadGateway, + }; + + /// + /// List of exception types that should be retried. + /// + public List RetryableExceptionTypes { get; set; } = new() + { + typeof(HttpRequestException) + }; + + private int _maxRetryCount = 1; +} diff --git a/dotnet/src/Extensions/Reliability.Basic/Reliability.Basic.csproj b/dotnet/src/Extensions/Reliability.Basic/Reliability.Basic.csproj new file mode 100644 index 000000000000..ccc1d232283c --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Basic/Reliability.Basic.csproj @@ -0,0 +1,32 @@ + + + + + Microsoft.SemanticKernel.Reliability.Basic + Microsoft.SemanticKernel.Reliability.Basic + netstandard2.0 + + + + + + + + + + Semantic Kernel - Basic Reliability Extension + Semantic Kernel Basic Reliability Extension + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/Extensions/Reliability.Basic/ReliabilityBasicKernelBuilderExtensions.cs b/dotnet/src/Extensions/Reliability.Basic/ReliabilityBasicKernelBuilderExtensions.cs new file mode 100644 index 000000000000..e1c7dfabe436 --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Basic/ReliabilityBasicKernelBuilderExtensions.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Reliability.Basic; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of KernelConfig +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Provides extension methods for the . +/// +public static class ReliabilityBasicKernelBuilderExtensions +{ + /// + /// Sets the default retry configuration for any kernel http request. + /// + /// Target instance + /// Retry configuration + /// Self instance + public static KernelBuilder WithRetryBasic(this KernelBuilder builder, BasicRetryConfig? retryConfig = null) + { + return builder.WithHttpHandlerFactory(new BasicHttpRetryHandlerFactory(retryConfig)); + } +} diff --git a/dotnet/src/Extensions/Reliability.Polly/PollyHttpRetryHandler.cs b/dotnet/src/Extensions/Reliability.Polly/PollyHttpRetryHandler.cs new file mode 100644 index 000000000000..d36aa22c9533 --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Polly/PollyHttpRetryHandler.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Polly; + +namespace Microsoft.SemanticKernel.Reliability.Polly; + +/// +/// Customizable PollyHttpHandler that will follow the provided policy. +/// +public class PollyHttpRetryHandler : DelegatingHandler +{ + private readonly AsyncPolicy? _typedAsyncPolicy; + private readonly AsyncPolicy? _asyncPolicy; + + /// + /// Creates a new instance of . + /// + /// HttpResponseMessage typed AsyncPolicy dedicated for typed policies. + public PollyHttpRetryHandler(AsyncPolicy typedAsyncPolicy) + { + Verify.NotNull(typedAsyncPolicy); + + this._typedAsyncPolicy = typedAsyncPolicy; + } + + /// + /// Creates a new instance of dedicated for non-typed policies. + /// + /// A non-typed AsyncPolicy + public PollyHttpRetryHandler(AsyncPolicy asyncPolicy) + { + Verify.NotNull(asyncPolicy); + + this._asyncPolicy = asyncPolicy; + } + + /// + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (this._typedAsyncPolicy is not null) + { + return await this._typedAsyncPolicy.ExecuteAsync(async (cancelToken) => + { + var response = await base.SendAsync(request, cancelToken).ConfigureAwait(false); + return response; + }, cancellationToken).ConfigureAwait(false); + } + + return await this._asyncPolicy!.ExecuteAsync(async (cancelToken) => + { + var response = await base.SendAsync(request, cancelToken).ConfigureAwait(false); + return response; + }, cancellationToken).ConfigureAwait(false); + } +} diff --git a/dotnet/src/Extensions/Reliability.Polly/PollyHttpRetryHandlerFactory.cs b/dotnet/src/Extensions/Reliability.Polly/PollyHttpRetryHandlerFactory.cs new file mode 100644 index 000000000000..4e77774c164f --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Polly/PollyHttpRetryHandlerFactory.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Http; +using Polly; + +namespace Microsoft.SemanticKernel.Reliability.Polly; + +/// +/// Customizable PollyHttpHandlerFactory that will create handlers with the provided policy. +/// +public class PollyHttpRetryHandlerFactory : HttpHandlerFactory +{ + private readonly AsyncPolicy? _typedAsyncPolicy; + private readonly AsyncPolicy? _asyncPolicy; + + /// + /// Creates a new instance of . + /// + /// HttpResponseMessage typed AsyncPolicy dedicated for typed policies. + public PollyHttpRetryHandlerFactory(AsyncPolicy typedAsyncPolicy) + { + Verify.NotNull(typedAsyncPolicy); + + this._typedAsyncPolicy = typedAsyncPolicy; + } + + /// + /// Creates a new instance of dedicated for non-typed policies. + /// + /// A non-typed AsyncPolicy + public PollyHttpRetryHandlerFactory(AsyncPolicy asyncPolicy) + { + Verify.NotNull(asyncPolicy); + + this._asyncPolicy = asyncPolicy; + } + + /// + /// Creates a new instance of with the default configuration. + /// + /// Logger factory + /// Returns the created handler + public override DelegatingHandler Create(ILoggerFactory? loggerFactory = null) + { + if (this._typedAsyncPolicy is not null) + { + return new PollyHttpRetryHandler(this._typedAsyncPolicy); + } + + return new PollyHttpRetryHandler(this._asyncPolicy!); + } +} diff --git a/dotnet/src/Extensions/Reliability.Polly/Reliability.Polly.csproj b/dotnet/src/Extensions/Reliability.Polly/Reliability.Polly.csproj new file mode 100644 index 000000000000..aac4803037bc --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Polly/Reliability.Polly.csproj @@ -0,0 +1,36 @@ + + + + + Microsoft.SemanticKernel.Reliability.Polly + Microsoft.SemanticKernel.Reliability.Polly + netstandard2.0 + + + + + + + + + + Semantic Kernel - Polly Reliability Extension + Semantic Kernel Polly Reliability Extension + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/Extensions/Reliability.Polly/ReliabilityPollyKernelBuilderExtensions.cs b/dotnet/src/Extensions/Reliability.Polly/ReliabilityPollyKernelBuilderExtensions.cs new file mode 100644 index 000000000000..0236e2e81d8e --- /dev/null +++ b/dotnet/src/Extensions/Reliability.Polly/ReliabilityPollyKernelBuilderExtensions.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.SemanticKernel.Reliability.Polly; +using Polly; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of KernelConfig +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Provides extension methods for the . +/// +public static class ReliabilityPollyKernelBuilderExtensions +{ + /// + /// Sets the default retry configuration for any kernel http request. + /// + /// Target instance + /// Provided AsyncPolicy + /// Returns target instance for fluent compatibility + public static KernelBuilder WithRetryPolly(this KernelBuilder kernelConfig, AsyncPolicy retryPolicy) + { + var pollyHandler = new PollyHttpRetryHandlerFactory(retryPolicy); + return kernelConfig.WithHttpHandlerFactory(pollyHandler); + } + + /// + /// Sets the default retry configuration for any kernel http request. + /// + /// Target instance + /// Provided HttpResponseMessage AsyncPolicy + /// Returns target instance for fluent compatibility + public static KernelBuilder WithRetryPolly(this KernelBuilder kernelConfig, AsyncPolicy retryPolicy) + { + var pollyHandler = new PollyHttpRetryHandlerFactory(retryPolicy); + return kernelConfig.WithHttpHandlerFactory(pollyHandler); + } +} diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/BasicPromptTemplateEngine.cs b/dotnet/src/Extensions/TemplateEngine.Basic/BasicPromptTemplateEngine.cs new file mode 100644 index 000000000000..2b6721f89bd7 --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/BasicPromptTemplateEngine.cs @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; + +namespace Microsoft.SemanticKernel.TemplateEngine.Basic; + +/// +/// Given a prompt, that might contain references to variables and functions: +/// - Get the list of references +/// - Resolve each reference +/// - Variable references are resolved using the context variables +/// - Function references are resolved invoking those functions +/// - Functions can be invoked passing in variables +/// - Functions do not receive the context variables, unless specified using a special variable +/// - Functions can be invoked in order and in parallel so the context variables must be immutable when invoked within the template +/// +public class BasicPromptTemplateEngine : IPromptTemplateEngine +{ + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _logger; + private readonly TemplateTokenizer _tokenizer; + + /// + /// Initializes a new instance of the class. + /// + /// The to use for logging. If null, no logging will be performed. + public BasicPromptTemplateEngine(ILoggerFactory? loggerFactory = null) + { + this._loggerFactory = loggerFactory ?? NullLoggerFactory.Instance; + this._logger = this._loggerFactory.CreateLogger(typeof(BasicPromptTemplateEngine)); + this._tokenizer = new TemplateTokenizer(loggerFactory); + } + + /// + public async Task RenderAsync(string templateText, SKContext context, CancellationToken cancellationToken = default) + { + this._logger.LogTrace("Rendering string template: {0}", templateText); + var blocks = this.ExtractBlocks(templateText); + return await this.RenderAsync(blocks, context, cancellationToken).ConfigureAwait(false); + } + + /// + /// Given a prompt template string, extract all the blocks (text, variables, function calls) + /// + /// Prompt template (see skprompt.txt files) + /// Whether to validate the blocks syntax, or just return the blocks found, which could contain invalid code + /// A list of all the blocks, ie the template tokenized in text, variables and function calls + internal IList ExtractBlocks(string? templateText, bool validate = true) + { + this._logger.LogTrace("Extracting blocks from template: {0}", templateText); + var blocks = this._tokenizer.Tokenize(templateText); + + if (validate) + { + foreach (var block in blocks) + { + if (!block.IsValid(out var error)) + { + throw new SKException(error); + } + } + } + + return blocks; + } + + /// + /// Given a list of blocks render each block and compose the final result. + /// + /// Template blocks generated by ExtractBlocks. + /// Access into the current kernel execution context. + /// The to monitor for cancellation requests. The default is . + /// The prompt template ready to be used for an AI request. + internal async Task RenderAsync(IList blocks, SKContext context, CancellationToken cancellationToken = default) + { + this._logger.LogTrace("Rendering list of {0} blocks", blocks.Count); + var tasks = new List>(blocks.Count); + foreach (var block in blocks) + { + switch (block) + { + case ITextRendering staticBlock: + tasks.Add(Task.FromResult(staticBlock.Render(context.Variables))); + break; + + case ICodeRendering dynamicBlock: + tasks.Add(dynamicBlock.RenderCodeAsync(context, cancellationToken)); + break; + + default: + const string Error = "Unexpected block type, the block doesn't have a rendering method"; + this._logger.LogError(Error); + throw new SKException(Error); + } + } + + var result = new StringBuilder(); + foreach (Task t in tasks) + { + result.Append(await t.ConfigureAwait(false)); + } + + // Sensitive data, logging as trace, disabled by default + this._logger.LogTrace("Rendered prompt: {0}", result); + + return result.ToString(); + } + + /// + /// Given a list of blocks, render the Variable Blocks, replacing placeholders with the actual value in memory. + /// + /// List of blocks, typically all the blocks found in a template. + /// Container of all the temporary variables known to the kernel. + /// An updated list of blocks where Variable Blocks have rendered to Text Blocks. + internal IList RenderVariables(IList blocks, ContextVariables? variables) + { + this._logger.LogTrace("Rendering variables"); + return blocks.Select(block => block.Type != BlockTypes.Variable + ? block + : new TextBlock(((ITextRendering)block).Render(variables), this._loggerFactory)).ToList(); + } +} diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/Block.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/Block.cs new file mode 100644 index 000000000000..3e5205281926 --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/Block.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; + +/// +/// Base class for blocks parsed from a prompt template +/// +public abstract class Block +{ + internal virtual BlockTypes Type => BlockTypes.Undefined; + + // internal virtual bool? SynchronousRendering => null; + + /// + /// The block content + /// + internal string Content { get; } + + /// + /// App logger + /// + private protected ILogger Logger { get; } + + /// + /// Base constructor. Prevent external instantiation. + /// + /// Block content + /// The to use for logging. If null, no logging will be performed. + private protected Block(string? content, ILoggerFactory? loggerFactory) + { + this.Content = content ?? string.Empty; + this.Logger = loggerFactory is not null ? loggerFactory.CreateLogger(this.GetType()) : NullLogger.Instance; + } + + /// + /// Check if the block content is valid. + /// + /// Error message in case the content is not valid + /// True if the block content is valid + public abstract bool IsValid(out string errorMsg); +} diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/BlockTypes.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/BlockTypes.cs new file mode 100644 index 000000000000..74e5833a4ad9 --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/BlockTypes.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; + +internal enum BlockTypes +{ + Undefined = 0, + Text = 1, + Code = 2, + Variable = 3, + Value = 4, + FunctionId = 5, + NamedArg = 6, +} diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/CodeBlock.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/CodeBlock.cs new file mode 100644 index 000000000000..280d34b1d25c --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/CodeBlock.cs @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; + +#pragma warning disable CA2254 // error strings are used also internally, not just for logging +#pragma warning disable CA1031 // IsCriticalException is an internal utility and should not be used by extensions + +// ReSharper disable TemplateIsNotCompileTimeConstantProblem +internal sealed class CodeBlock : Block, ICodeRendering +{ + internal override BlockTypes Type => BlockTypes.Code; + + /// + /// Initializes a new instance of the class. + /// + /// Block content + /// The to use for logging. If null, no logging will be performed. + public CodeBlock(string? content, ILoggerFactory? loggerFactory) + : this(new CodeTokenizer(loggerFactory).Tokenize(content), content?.Trim(), loggerFactory) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// A list of blocks + /// Block content + /// The to use for logging. If null, no logging will be performed. + public CodeBlock(List tokens, string? content, ILoggerFactory? loggerFactory) + : base(content?.Trim(), loggerFactory) + { + this._tokens = tokens; + } + + /// + public override bool IsValid(out string errorMsg) + { + errorMsg = ""; + + foreach (Block token in this._tokens) + { + if (!token.IsValid(out errorMsg)) + { + this.Logger.LogError(errorMsg); + return false; + } + } + + if (this._tokens.Count > 0 && this._tokens[0].Type == BlockTypes.NamedArg) + { + errorMsg = "Unexpected named argument found. Expected function name first."; + this.Logger.LogError(errorMsg); + return false; + } + + if (this._tokens.Count > 1 && !this.IsValidFunctionCall(out errorMsg)) + { + return false; + } + + this._validated = true; + + return true; + } + + /// + public async Task RenderCodeAsync(SKContext context, CancellationToken cancellationToken = default) + { + if (!this._validated && !this.IsValid(out var error)) + { + throw new SKException(error); + } + + this.Logger.LogTrace("Rendering code: `{Content}`", this.Content); + + switch (this._tokens[0].Type) + { + case BlockTypes.Value: + case BlockTypes.Variable: + return ((ITextRendering)this._tokens[0]).Render(context.Variables); + + case BlockTypes.FunctionId: + return await this.RenderFunctionCallAsync((FunctionIdBlock)this._tokens[0], context).ConfigureAwait(false); + } + + throw new SKException($"Unexpected first token type: {this._tokens[0].Type:G}"); + } + + #region private ================================================================================ + + private bool _validated; + private readonly List _tokens; + + private async Task RenderFunctionCallAsync(FunctionIdBlock fBlock, SKContext context) + { + // Clone the context to avoid unexpected variable mutations from the inner function execution + ContextVariables inputVariables = context.Variables.Clone(); + + // If the code syntax is {{functionName $varName}} use $varName instead of $input + // If the code syntax is {{functionName 'value'}} use "value" instead of $input + if (this._tokens.Count > 1) + { + inputVariables = this.PopulateContextWithFunctionArguments(inputVariables); + } + try + { + await context.Runner.RunAsync(fBlock.PluginName, fBlock.FunctionName, inputVariables).ConfigureAwait(false); + } + catch (Exception ex) + { + this.Logger.LogError(ex, "Function {Plugin}.{Function} execution failed with error {Error}", fBlock.PluginName, fBlock.FunctionName, ex.Message); + throw; + } + + return inputVariables.ToString(); + } + + private bool IsValidFunctionCall(out string errorMsg) + { + errorMsg = ""; + if (this._tokens[0].Type != BlockTypes.FunctionId) + { + errorMsg = $"Unexpected second token found: {this._tokens[1].Content}"; + this.Logger.LogError(errorMsg); + return false; + } + + if (this._tokens[1].Type is not BlockTypes.Value and not BlockTypes.Variable and not BlockTypes.NamedArg) + { + errorMsg = "The first arg of a function must be a quoted string, variable or named argument"; + this.Logger.LogError(errorMsg); + return false; + } + + for (int i = 2; i < this._tokens.Count; i++) + { + if (this._tokens[i].Type is not BlockTypes.NamedArg) + { + errorMsg = $"Functions only support named arguments after the first argument. Argument {i} is not named."; + this.Logger.LogError(errorMsg); + return false; + } + } + + return true; + } + + private ContextVariables PopulateContextWithFunctionArguments(ContextVariables variables) + { + // Clone the context to avoid unexpected and hard to test input mutation + var variablesClone = variables.Clone(); + var firstArg = this._tokens[1]; + + // Sensitive data, logging as trace, disabled by default + this.Logger.LogTrace("Passing variable/value: `{Content}`", firstArg.Content); + + var namedArgsStartIndex = 1; + if (firstArg.Type is not BlockTypes.NamedArg) + { + string input = ((ITextRendering)this._tokens[1]).Render(variablesClone); + // Keep previous trust information when updating the input + variablesClone.Update(input); + namedArgsStartIndex++; + } + + for (int i = namedArgsStartIndex; i < this._tokens.Count; i++) + { + var arg = this._tokens[i] as NamedArgBlock; + + // When casting fails because the block isn't a NamedArg, arg is null + if (arg == null) + { + var errorMsg = "Functions support up to one positional argument"; + this.Logger.LogError(errorMsg); + throw new SKException($"Unexpected first token type: {this._tokens[i].Type:G}"); + } + + // Sensitive data, logging as trace, disabled by default + this.Logger.LogTrace("Passing variable/value: `{Content}`", arg.Content); + + variablesClone.Set(arg.Name, arg.GetValue(variables)); + } + + return variablesClone; + } + #endregion +} +// ReSharper restore TemplateIsNotCompileTimeConstantProblem +#pragma warning restore CA2254 diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/FunctionIdBlock.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/FunctionIdBlock.cs new file mode 100644 index 000000000000..216da326c688 --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/FunctionIdBlock.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; + +internal sealed class FunctionIdBlock : Block, ITextRendering +{ + internal override BlockTypes Type => BlockTypes.FunctionId; + + internal string PluginName { get; } = string.Empty; + + internal string FunctionName { get; } = string.Empty; + + public FunctionIdBlock(string? text, ILoggerFactory? loggerFactory = null) + : base(text?.Trim(), loggerFactory) + { + var functionNameParts = this.Content.Split('.'); + if (functionNameParts.Length > 2) + { + this.Logger.LogError("Invalid function name `{FunctionName}`.", this.Content); + throw new SKException($"Invalid function name `{this.Content}`. A function name can contain at most one dot separating the plugin name from the function name"); + } + + if (functionNameParts.Length == 2) + { + this.PluginName = functionNameParts[0]; + this.FunctionName = functionNameParts[1]; + return; + } + + this.FunctionName = this.Content; + } + + public override bool IsValid(out string errorMsg) + { + if (!s_validContentRegex.IsMatch(this.Content)) + { + errorMsg = "The function identifier is empty"; + return false; + } + + if (HasMoreThanOneDot(this.Content)) + { + errorMsg = "The function identifier can contain max one '.' char separating plugin name from function name"; + return false; + } + + errorMsg = ""; + return true; + } + + public string Render(ContextVariables? variables) + { + return this.Content; + } + + private static bool HasMoreThanOneDot(string? value) + { + if (value == null || value.Length < 2) { return false; } + + int count = 0; + return value.Any(t => t == '.' && ++count > 1); + } + + private static readonly Regex s_validContentRegex = new("^[a-zA-Z0-9_.]*$"); +} diff --git a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/ICodeRendering.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ICodeRendering.cs similarity index 92% rename from dotnet/src/SemanticKernel/TemplateEngine/Blocks/ICodeRendering.cs rename to dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ICodeRendering.cs index 93ec182b8052..cbdf9ef9c577 100644 --- a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/ICodeRendering.cs +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ICodeRendering.cs @@ -4,7 +4,7 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel.Orchestration; -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; /// /// Interface of dynamic blocks that need async IO to be rendered. diff --git a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/ITextRendering.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ITextRendering.cs similarity index 88% rename from dotnet/src/SemanticKernel/TemplateEngine/Blocks/ITextRendering.cs rename to dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ITextRendering.cs index 8f67259caa78..1fa75694c03f 100644 --- a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/ITextRendering.cs +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ITextRendering.cs @@ -2,7 +2,7 @@ using Microsoft.SemanticKernel.Orchestration; -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; /// /// Interface of static blocks that don't need async IO to be rendered. diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/NamedArgBlock.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/NamedArgBlock.cs new file mode 100644 index 000000000000..0847c8df48bb --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/NamedArgBlock.cs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; + +/// +/// A that represents a named argument for a function call. +/// For example, in the template {{ MyPlugin.MyFunction var1="foo" }}, var1="foo" is a named arg block. +/// +internal sealed class NamedArgBlock : Block, ITextRendering +{ + /// + /// Returns the . + /// + internal override BlockTypes Type => BlockTypes.NamedArg; + + /// + /// Gets the name of the function argument. + /// + internal string Name { get; } = string.Empty; + + /// + /// Initializes a new instance of the class. + /// + /// Raw text parsed from the prompt template. + /// The to use for logging. If null, no logging will be performed. + /// + public NamedArgBlock(string? text, ILoggerFactory? logger = null) + : base(NamedArgBlock.TrimWhitespace(text), logger) + { + var argParts = this.Content.Split(Symbols.NamedArgBlockSeparator); + if (argParts.Length != 2) + { + this.Logger.LogError("Invalid named argument `{Text}`", text); + throw new SKException($"A function named argument must contain a name and value separated by a '{Symbols.NamedArgBlockSeparator}' character."); + } + + this.Name = argParts[0]; + this._argNameAsVarBlock = new VarBlock($"{Symbols.VarPrefix}{argParts[0]}"); + var argValue = argParts[1]; + if (argValue.Length == 0) + { + this.Logger.LogError("Invalid named argument `{Text}`", text); + throw new SKException($"A function named argument must contain a quoted value or variable after the '{Symbols.NamedArgBlockSeparator}' character."); + } + + if (argValue[0] == Symbols.VarPrefix) + { + this._argValueAsVarBlock = new VarBlock(argValue); + } + else + { + this._valBlock = new ValBlock(argValue); + } + } + + /// + /// Gets the rendered value of the function argument. If the value is a , the value stays the same. + /// If the value is a , the value of the variable is determined by the context variables passed in. + /// + /// Variables to use for rendering the named argument value when the value is a . + /// + internal string GetValue(ContextVariables? variables) + { + var valueIsValidValBlock = this._valBlock != null && this._valBlock.IsValid(out var errorMessage); + if (valueIsValidValBlock) + { + return this._valBlock!.Render(variables); + } + + var valueIsValidVarBlock = this._argValueAsVarBlock != null && this._argValueAsVarBlock.IsValid(out var errorMessage2); + if (valueIsValidVarBlock) + { + return this._argValueAsVarBlock!.Render(variables); + } + + return string.Empty; + } + + /// + /// Renders the named arg block. + /// + /// + /// + public string Render(ContextVariables? variables) + { + return this.Content; + } + + /// + /// Returns whether the named arg block has valid syntax. + /// + /// An error message that gets set when the named arg block is not valid. + /// +#pragma warning disable CA2254 // error strings are used also internally, not just for logging + public override bool IsValid(out string errorMsg) + { + errorMsg = string.Empty; + if (string.IsNullOrEmpty(this.Name)) + { + errorMsg = "A named argument must have a name"; + this.Logger.LogError(errorMsg); + return false; + } + + if (this._valBlock != null && !this._valBlock.IsValid(out var valErrorMsg)) + { + errorMsg = $"There was an issue with the named argument value for '{this.Name}': {valErrorMsg}"; + this.Logger.LogError(errorMsg); + return false; + } + else if (this._argValueAsVarBlock != null && !this._argValueAsVarBlock.IsValid(out var variableErrorMsg)) + { + errorMsg = $"There was an issue with the named argument value for '{this.Name}': {variableErrorMsg}"; + this.Logger.LogError(errorMsg); + return false; + } + else if (this._valBlock == null && this._argValueAsVarBlock == null) + { + errorMsg = "A named argument must have a value"; + this.Logger.LogError(errorMsg); + return false; + } + + // Argument names share the same validation as variables + if (!this._argNameAsVarBlock.IsValid(out var argNameErrorMsg)) + { + errorMsg = Regex.Replace(argNameErrorMsg, "a variable", "An argument", RegexOptions.IgnoreCase); + errorMsg = Regex.Replace(errorMsg, "the variable", "The argument", RegexOptions.IgnoreCase); + return false; + } + + return true; + } +#pragma warning restore CA2254 + + #region private ================================================================================ + + private readonly VarBlock _argNameAsVarBlock; + private readonly ValBlock? _valBlock; + private readonly VarBlock? _argValueAsVarBlock; + + private static string? TrimWhitespace(string? text) + { + if (text == null) + { + return text; + } + + string[] trimmedParts = NamedArgBlock.GetTrimmedParts(text); + switch (trimmedParts?.Length) + { + case (2): + return $"{trimmedParts[0]}{Symbols.NamedArgBlockSeparator}{trimmedParts[1]}"; + case (1): + return trimmedParts[0]; + default: + return null; + } + } + + private static string[] GetTrimmedParts(string? text) + { + if (text == null) + { + return System.Array.Empty(); + } + + string[] parts = text.Split(new char[] { Symbols.NamedArgBlockSeparator }, 2); + string[] result = new string[parts.Length]; + if (parts.Length > 0) + { + result[0] = parts[0].Trim(); + } + + if (parts.Length > 1) + { + result[1] = parts[1].Trim(); + } + + return result; + } + + #endregion +} diff --git a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/Symbols.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/Symbols.cs similarity index 80% rename from dotnet/src/SemanticKernel/TemplateEngine/Blocks/Symbols.cs rename to dotnet/src/Extensions/TemplateEngine.Basic/Blocks/Symbols.cs index 7fafa502302c..c0beefb7ba69 100644 --- a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/Symbols.cs +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/Symbols.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; internal static class Symbols { @@ -8,6 +8,7 @@ internal static class Symbols internal const char BlockEnder = '}'; internal const char VarPrefix = '$'; + internal const char NamedArgBlockSeparator = '='; internal const char DblQuote = '"'; internal const char SglQuote = '\''; diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/TextBlock.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/TextBlock.cs new file mode 100644 index 000000000000..99d7c2e5174a --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/TextBlock.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; + +internal sealed class TextBlock : Block, ITextRendering +{ + internal override BlockTypes Type => BlockTypes.Text; + + public TextBlock(string? text, ILoggerFactory? loggerFactory = null) + : base(text, loggerFactory) + { + } + + public TextBlock(string text, int startIndex, int stopIndex, ILoggerFactory? loggerFactory) + : base(text.Substring(startIndex, stopIndex - startIndex), loggerFactory) + { + } + + public override bool IsValid(out string errorMsg) + { + errorMsg = ""; + return true; + } + + public string Render(ContextVariables? variables) + { + return this.Content; + } +} diff --git a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/ValBlock.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ValBlock.cs similarity index 83% rename from dotnet/src/SemanticKernel/TemplateEngine/Blocks/ValBlock.cs rename to dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ValBlock.cs index f15ad4667aab..1ae186faa6f5 100644 --- a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/ValBlock.cs +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/ValBlock.cs @@ -2,9 +2,8 @@ using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Text; -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; internal sealed class ValBlock : Block, ITextRendering { @@ -21,9 +20,9 @@ internal sealed class ValBlock : Block, ITextRendering /// Create an instance /// /// Block content, including the delimiting chars - /// Optional logger - public ValBlock(string? quotedValue, ILogger? logger = null) - : base(quotedValue?.Trim(), logger) + /// The to use for logging. If null, no logging will be performed. + public ValBlock(string? quotedValue, ILoggerFactory? loggerFactory = null) + : base(quotedValue?.Trim(), loggerFactory) { if (this.Content.Length < 2) { @@ -69,8 +68,8 @@ public string Render(ContextVariables? variables) public static bool HasValPrefix(string? text) { - return !text.IsNullOrEmpty() - && text.Length > 0 + return !string.IsNullOrEmpty(text) + && text!.Length > 0 && (text[0] is Symbols.DblQuote or Symbols.SglQuote); } } diff --git a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/VarBlock.cs b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/VarBlock.cs similarity index 90% rename from dotnet/src/SemanticKernel/TemplateEngine/Blocks/VarBlock.cs rename to dotnet/src/Extensions/TemplateEngine.Basic/Blocks/VarBlock.cs index bf09819a3c47..d348d63f6276 100644 --- a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/VarBlock.cs +++ b/dotnet/src/Extensions/TemplateEngine.Basic/Blocks/VarBlock.cs @@ -2,9 +2,10 @@ using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Orchestration; -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; +namespace Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; internal sealed class VarBlock : Block, ITextRendering { @@ -12,7 +13,7 @@ internal sealed class VarBlock : Block, ITextRendering internal string Name { get; } = string.Empty; - public VarBlock(string? content, ILogger? logger = null) : base(content?.Trim(), logger) + public VarBlock(string? content, ILoggerFactory? loggerFactory = null) : base(content?.Trim(), loggerFactory) { if (this.Content.Length < 2) { @@ -70,7 +71,7 @@ public string Render(ContextVariables? variables) { const string ErrMsg = "Variable rendering failed, the variable name is empty"; this.Logger.LogError(ErrMsg); - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, ErrMsg); + throw new SKException(ErrMsg); } if (variables.TryGetValue(this.Name, out string? value)) diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/CodeTokenizer.cs b/dotnet/src/Extensions/TemplateEngine.Basic/CodeTokenizer.cs new file mode 100644 index 000000000000..954fd0083237 --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/CodeTokenizer.cs @@ -0,0 +1,346 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; + +namespace Microsoft.SemanticKernel.TemplateEngine.Basic; + +/// +/// Simple tokenizer used for default SK template code language. +/// +/// BNF parsed by TemplateTokenizer: +/// [template] ::= "" | [block] | [block] [template] +/// [block] ::= [sk-block] | [text-block] +/// [sk-block] ::= "{{" [variable] "}}" | "{{" [value] "}}" | "{{" [function-call] "}}" +/// [text-block] ::= [any-char] | [any-char] [text-block] +/// [any-char] ::= any char +/// +/// BNF parsed by CodeTokenizer: +/// [template] ::= "" | [variable] " " [template] | [value] " " [template] | [function-call] " " [template] +/// [variable] ::= "$" [valid-name] +/// [value] ::= "'" [text] "'" | '"' [text] '"' +/// [function-call] ::= [function-id] | [function-id] [parameter] +/// [parameter] ::= [variable] | [value] +/// +/// BNF parsed by dedicated blocks +/// [function-id] ::= [valid-name] | [valid-name] "." [valid-name] +/// [valid-name] ::= [valid-symbol] | [valid-symbol] [valid-name] +/// [valid-symbol] ::= [letter] | [digit] | "_" +/// [letter] ::= "a" | "b" ... | "z" | "A" | "B" ... | "Z" +/// [digit] ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" +/// +internal sealed class CodeTokenizer +{ + private enum TokenTypes + { + None = 0, + Value = 1, + Variable = 2, + FunctionId = 3, + NamedArg = 4, + } + + private readonly ILoggerFactory _loggerFactory; + + public CodeTokenizer(ILoggerFactory? loggerFactory = null) + { + this._loggerFactory = loggerFactory ?? NullLoggerFactory.Instance; + } + + /// + /// Tokenize a code block, without checking for syntax errors + /// + /// Text to parse + /// A list of blocks + public List Tokenize(string? text) + { + // Remove spaces, which are ignored anyway + text = text?.Trim(); + + // Render NULL to "" + if (string.IsNullOrEmpty(text)) { return new List(); } + + // Track what type of token we're reading + TokenTypes currentTokenType = TokenTypes.None; + + // Track the content of the current token + var currentTokenContent = new StringBuilder(); + + char textValueDelimiter = '\0'; + + var blocks = new List(); + char nextChar = text![0]; + + // Tokens must be separated by spaces, track their presence + bool spaceSeparatorFound = false; + + // Named args may contain string values that contain spaces. These are used + // to determine when a space occurs between quotes. + bool namedArgSeparatorFound = false; + char namedArgValuePrefix = '\0'; + + // 1 char only edge case + if (text.Length == 1) + { + switch (nextChar) + { + case Symbols.VarPrefix: + blocks.Add(new VarBlock(text, this._loggerFactory)); + break; + + case Symbols.DblQuote: + case Symbols.SglQuote: + blocks.Add(new ValBlock(text, this._loggerFactory)); + break; + + default: + blocks.Add(new FunctionIdBlock(text, this._loggerFactory)); + break; + } + + return blocks; + } + + bool skipNextChar = false; + for (int nextCharCursor = 1; nextCharCursor < text.Length; nextCharCursor++) + { + char currentChar = nextChar; + nextChar = text[nextCharCursor]; + + if (skipNextChar) + { + skipNextChar = false; + continue; + } + + // First char is easy + if (nextCharCursor == 1) + { + if (IsVarPrefix(currentChar)) + { + currentTokenType = TokenTypes.Variable; + } + else if (IsQuote(currentChar)) + { + currentTokenType = TokenTypes.Value; + textValueDelimiter = currentChar; + } + else + { + currentTokenType = TokenTypes.FunctionId; + } + + currentTokenContent.Append(currentChar); + continue; + } + + // While reading a values between quotes + if (currentTokenType == TokenTypes.Value || (currentTokenType == TokenTypes.NamedArg && IsQuote(namedArgValuePrefix))) + { + // If the current char is escaping the next special char: + // - skip the current char (escape char) + // - add the next (special char) + // - jump to the one after (to handle "\\" properly) + if (currentChar == Symbols.EscapeChar && CanBeEscaped(nextChar)) + { + currentTokenContent.Append(nextChar); + skipNextChar = true; + continue; + } + + currentTokenContent.Append(currentChar); + + // When we reach the end of the value + if (currentChar == textValueDelimiter && currentTokenType == TokenTypes.Value) + { + blocks.Add(new ValBlock(currentTokenContent.ToString(), this._loggerFactory)); + currentTokenContent.Clear(); + currentTokenType = TokenTypes.None; + spaceSeparatorFound = false; + } + else if (currentChar == namedArgValuePrefix && currentTokenType == TokenTypes.NamedArg) + { + blocks.Add(new NamedArgBlock(currentTokenContent.ToString(), this._loggerFactory)); + currentTokenContent.Clear(); + currentTokenType = TokenTypes.None; + spaceSeparatorFound = false; + namedArgSeparatorFound = false; + namedArgValuePrefix = '\0'; + } + + continue; + } + + // If we're not between quotes, a space signals the end of the current token + // Note: there might be multiple consecutive spaces + if (IsBlankSpace(currentChar)) + { + if (currentTokenType == TokenTypes.Variable) + { + blocks.Add(new VarBlock(currentTokenContent.ToString(), this._loggerFactory)); + currentTokenContent.Clear(); + currentTokenType = TokenTypes.None; + } + else if (currentTokenType == TokenTypes.FunctionId) + { + var tokenContent = currentTokenContent.ToString(); + // This isn't an expected block at this point but the TemplateTokenizer should throw an error when + // a named arg is used without a function call + if (CodeTokenizer.IsValidNamedArg(tokenContent)) + { + blocks.Add(new NamedArgBlock(tokenContent, this._loggerFactory)); + } + else + { + blocks.Add(new FunctionIdBlock(tokenContent, this._loggerFactory)); + } + currentTokenContent.Clear(); + currentTokenType = TokenTypes.None; + } + else if (currentTokenType == TokenTypes.NamedArg && namedArgSeparatorFound && namedArgValuePrefix != 0) + { + blocks.Add(new NamedArgBlock(currentTokenContent.ToString(), this._loggerFactory)); + currentTokenContent.Clear(); + namedArgSeparatorFound = false; + namedArgValuePrefix = '\0'; + currentTokenType = TokenTypes.None; + } + + spaceSeparatorFound = true; + + continue; + } + + // If reading a named argument and either the '=' or the value prefix ($, ', or ") haven't been found + if (currentTokenType == TokenTypes.NamedArg && (!namedArgSeparatorFound || namedArgValuePrefix == 0)) + { + if (!namedArgSeparatorFound) + { + if (currentChar == Symbols.NamedArgBlockSeparator) + { + namedArgSeparatorFound = true; + } + } + else + { + namedArgValuePrefix = currentChar; + if (!IsQuote((char)namedArgValuePrefix) && namedArgValuePrefix != Symbols.VarPrefix) + { + throw new SKException($"Named argument values need to be prefixed with a quote or {Symbols.VarPrefix}."); + } + } + currentTokenContent.Append(currentChar); + continue; + } + + // If we're not inside a quoted value and we're not processing a space + currentTokenContent.Append(currentChar); + + if (currentTokenType == TokenTypes.None) + { + if (!spaceSeparatorFound) + { + throw new SKException("Tokens must be separated by one space least"); + } + + if (IsQuote(currentChar)) + { + // A quoted value starts here + currentTokenType = TokenTypes.Value; + textValueDelimiter = currentChar; + } + else if (IsVarPrefix(currentChar)) + { + // A variable starts here + currentTokenType = TokenTypes.Variable; + } + else if (blocks.Count == 0) + { + // A function Id starts here + currentTokenType = TokenTypes.FunctionId; + } + else + { + // A named arg starts here + currentTokenType = TokenTypes.NamedArg; + } + } + } + + // Capture last token + currentTokenContent.Append(nextChar); + switch (currentTokenType) + { + case TokenTypes.Value: + blocks.Add(new ValBlock(currentTokenContent.ToString(), this._loggerFactory)); + break; + + case TokenTypes.Variable: + blocks.Add(new VarBlock(currentTokenContent.ToString(), this._loggerFactory)); + break; + + case TokenTypes.FunctionId: + var tokenContent = currentTokenContent.ToString(); + // This isn't an expected block at this point but the TemplateTokenizer should throw an error when + // a named arg is used without a function call + if (CodeTokenizer.IsValidNamedArg(tokenContent)) + { + blocks.Add(new NamedArgBlock(tokenContent, this._loggerFactory)); + } + else + { + blocks.Add(new FunctionIdBlock(currentTokenContent.ToString(), this._loggerFactory)); + } + break; + + case TokenTypes.NamedArg: + blocks.Add(new NamedArgBlock(currentTokenContent.ToString(), this._loggerFactory)); + break; + + case TokenTypes.None: + throw new SKException("Tokens must be separated by one space least"); + } + + return blocks; + } + + private static bool IsVarPrefix(char c) + { + return (c == Symbols.VarPrefix); + } + + private static bool IsBlankSpace(char c) + { + return c is Symbols.Space or Symbols.NewLine or Symbols.CarriageReturn or Symbols.Tab; + } + + private static bool IsQuote(char c) + { + return c is Symbols.DblQuote or Symbols.SglQuote; + } + + private static bool CanBeEscaped(char c) + { + return c is Symbols.DblQuote or Symbols.SglQuote or Symbols.EscapeChar; + } + + [SuppressMessage("Design", "CA1031:Modify to catch a more specific allowed exception type, or rethrow exception", + Justification = "Does not throw an exception by design.")] + private static bool IsValidNamedArg(string tokenContent) + { + try + { + var tokenContentAsNamedArg = new NamedArgBlock(tokenContent); + return tokenContentAsNamedArg.IsValid(out var error); + } + catch + { + return false; + } + } +} diff --git a/dotnet/src/Extensions/TemplateEngine.Basic/TemplateEngine.Basic.csproj b/dotnet/src/Extensions/TemplateEngine.Basic/TemplateEngine.Basic.csproj new file mode 100644 index 000000000000..defc4bf9a7e7 --- /dev/null +++ b/dotnet/src/Extensions/TemplateEngine.Basic/TemplateEngine.Basic.csproj @@ -0,0 +1,25 @@ + + + + + Microsoft.SemanticKernel.TemplateEngine.Basic + Microsoft.SemanticKernel.TemplateEngine.Basic + netstandard2.0 + + + + + + + Semantic Kernel - Basic Prompt Template Engine + Semantic Kernel Basic Prompt Template Engine + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/SemanticKernel/TemplateEngine/TemplateTokenizer.cs b/dotnet/src/Extensions/TemplateEngine.Basic/TemplateTokenizer.cs similarity index 81% rename from dotnet/src/SemanticKernel/TemplateEngine/TemplateTokenizer.cs rename to dotnet/src/Extensions/TemplateEngine.Basic/TemplateTokenizer.cs index 52782c6dee95..10d00b1e2f51 100644 --- a/dotnet/src/SemanticKernel/TemplateEngine/TemplateTokenizer.cs +++ b/dotnet/src/Extensions/TemplateEngine.Basic/TemplateTokenizer.cs @@ -3,10 +3,10 @@ using System.Collections.Generic; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; -using Microsoft.SemanticKernel.Text; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.TemplateEngine.Basic.Blocks; -namespace Microsoft.SemanticKernel.TemplateEngine; +namespace Microsoft.SemanticKernel.TemplateEngine.Basic; /// /// Simple tokenizer used for default SK template language. @@ -37,11 +37,11 @@ internal sealed class TemplateTokenizer /// /// Create a new instance of SK tokenizer /// - /// - public TemplateTokenizer(ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + public TemplateTokenizer(ILoggerFactory? loggerFactory = null) { - this._logger = logger ?? NullLogger.Instance; - this._codeTokenizer = new CodeTokenizer(this._logger); + this._loggerFactory = loggerFactory ?? NullLoggerFactory.Instance; + this._codeTokenizer = new CodeTokenizer(loggerFactory); } /// @@ -57,15 +57,15 @@ public IList Tokenize(string? text) const int MinCodeBlockLength = EmptyCodeBlockLength + 1; // Render NULL to "" - if (text.IsNullOrEmpty()) + if (string.IsNullOrEmpty(text)) { - return new List { new TextBlock(string.Empty, this._logger) }; + return new List { new TextBlock(string.Empty, this._loggerFactory) }; } // If the template is "empty" return the content as a text block - if (text.Length < MinCodeBlockLength) + if (text!.Length < MinCodeBlockLength) { - return new List { new TextBlock(text, this._logger) }; + return new List { new TextBlock(text, this._loggerFactory) }; } var blocks = new List(); @@ -133,7 +133,7 @@ public IList Tokenize(string? text) // If there is plain text between the current var/val/code block and the previous one, capture that as a TextBlock if (blockStartPos > endOfLastBlock) { - blocks.Add(new TextBlock(text, endOfLastBlock, blockStartPos, this._logger)); + blocks.Add(new TextBlock(text, endOfLastBlock, blockStartPos, this._loggerFactory)); } // Extract raw block @@ -147,7 +147,7 @@ public IList Tokenize(string? text) if (contentWithoutDelimiters.Length == 0) { // If what is left is empty, consider the raw block a Text Block - blocks.Add(new TextBlock(contentWithDelimiters, this._logger)); + blocks.Add(new TextBlock(contentWithDelimiters, this._loggerFactory)); } else { @@ -158,8 +158,7 @@ public IList Tokenize(string? text) case BlockTypes.Variable: if (codeBlocks.Count > 1) { - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, - $"Invalid token detected after the variable: {contentWithoutDelimiters}"); + throw new SKException($"Invalid token detected after the variable: {contentWithoutDelimiters}"); } blocks.Add(codeBlocks[0]); @@ -168,29 +167,22 @@ public IList Tokenize(string? text) case BlockTypes.Value: if (codeBlocks.Count > 1) { - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, - $"Invalid token detected after the value: {contentWithoutDelimiters}"); + throw new SKException($"Invalid token detected after the value: {contentWithoutDelimiters}"); } blocks.Add(codeBlocks[0]); break; case BlockTypes.FunctionId: - if (codeBlocks.Count > 2) - { - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, - $"Functions support only one parameter: {contentWithoutDelimiters}"); - } - - blocks.Add(new CodeBlock(codeBlocks, contentWithoutDelimiters, this._logger)); + blocks.Add(new CodeBlock(codeBlocks, contentWithoutDelimiters, this._loggerFactory)); break; case BlockTypes.Code: case BlockTypes.Text: case BlockTypes.Undefined: + case BlockTypes.NamedArg: default: - throw new TemplateException(TemplateException.ErrorCodes.UnexpectedBlockType, - $"Code tokenizer returned an incorrect first token type {codeBlocks[0].Type:G}"); + throw new SKException($"Code tokenizer returned an incorrect first token type {codeBlocks[0].Type:G}"); } } @@ -204,7 +196,7 @@ public IList Tokenize(string? text) // If there is something left after the last block, capture it as a TextBlock if (endOfLastBlock < text.Length) { - blocks.Add(new TextBlock(text, endOfLastBlock, text.Length, this._logger)); + blocks.Add(new TextBlock(text, endOfLastBlock, text.Length, this._loggerFactory)); } return blocks; @@ -212,7 +204,7 @@ public IList Tokenize(string? text) #region private ================================================================================ - private readonly ILogger _logger; + private readonly ILoggerFactory _loggerFactory; private readonly CodeTokenizer _codeTokenizer; private static string SubStr(string text, int startIndex, int stopIndex) diff --git a/dotnet/src/Skills/Skills.Grpc/Extensions/GrpcOperationExtensions.cs b/dotnet/src/Functions/Functions.Grpc/Extensions/GrpcOperationExtensions.cs similarity index 85% rename from dotnet/src/Skills/Skills.Grpc/Extensions/GrpcOperationExtensions.cs rename to dotnet/src/Functions/Functions.Grpc/Extensions/GrpcOperationExtensions.cs index 21d158b2cfb0..dc5f033792d0 100644 --- a/dotnet/src/Skills/Skills.Grpc/Extensions/GrpcOperationExtensions.cs +++ b/dotnet/src/Functions/Functions.Grpc/Extensions/GrpcOperationExtensions.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. using System.Collections.Generic; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.Grpc.Model; +using Microsoft.SemanticKernel.Functions.Grpc.Model; // ReSharper disable once CheckNamespace -namespace Microsoft.SemanticKernel.Skills.Grpc.Extensions; +namespace Microsoft.SemanticKernel.Functions.Grpc.Extensions; #pragma warning disable RCS1175 // Unused 'this' parameter 'operation'. @@ -21,7 +20,7 @@ internal static class GrpcOperationExtensions /// The list of parameters. public static IReadOnlyList GetParameters(this GrpcOperation operation) { - var parameters = new List + var parameters = new ParameterView[] { // Register the "address" parameter so that it's possible to override it if needed. new ParameterView(GrpcOperation.AddressArgumentName, diff --git a/dotnet/src/Functions/Functions.Grpc/Extensions/KernelGrpcExtensions.cs b/dotnet/src/Functions/Functions.Grpc/Extensions/KernelGrpcExtensions.cs new file mode 100644 index 000000000000..399098b69bf3 --- /dev/null +++ b/dotnet/src/Functions/Functions.Grpc/Extensions/KernelGrpcExtensions.cs @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.Grpc.Model; +using Microsoft.SemanticKernel.Functions.Grpc.Protobuf; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Functions.Grpc.Extensions; + +/// +/// extensions methods for gRPC functionality. +/// +public static class KernelGrpcExtensions +{ + /// + /// Imports gRPC document from a directory. + /// + /// Semantic Kernel instance. + /// Directory containing the plugin directory. + /// Name of the directory containing the selected plugin. + /// HttpClient to use for sending requests. + /// A list of all the semantic functions representing the plugin. + public static IDictionary ImportGrpcFunctionsFromDirectory( + this IKernel kernel, + string parentDirectory, + string pluginDirectoryName, + HttpClient? httpClient = null) + { + const string ProtoFile = "grpc.proto"; + + Verify.ValidPluginName(pluginDirectoryName); + + var pluginDir = Path.Combine(parentDirectory, pluginDirectoryName); + Verify.DirectoryExists(pluginDir); + + var filePath = Path.Combine(pluginDir, ProtoFile); + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"No .proto document for the specified path - {filePath} is found."); + } + + kernel.LoggerFactory.CreateLogger(typeof(KernelGrpcExtensions)).LogTrace("Registering gRPC functions from {0} .proto document", filePath); + + using var stream = File.OpenRead(filePath); + + return kernel.RegisterGrpcFunctions(stream, pluginDirectoryName, httpClient); + } + + /// + /// Imports gRPC document from a file. + /// + /// Semantic Kernel instance. + /// Name of the plugin to register. + /// File path to .proto document. + /// HttpClient to use for sending requests. + /// A list of all the semantic functions representing the plugin. + public static IDictionary ImportGrpcFunctionsFromFile( + this IKernel kernel, + string pluginName, + string filePath, + HttpClient? httpClient = null) + { + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"No .proto document for the specified path - {filePath} is found."); + } + + kernel.LoggerFactory.CreateLogger(typeof(KernelGrpcExtensions)).LogTrace("Registering gRPC functions from {0} .proto document", filePath); + + using var stream = File.OpenRead(filePath); + + return kernel.RegisterGrpcFunctions(stream, pluginName, httpClient); + } + + /// + /// Registers an gRPC plugin. + /// + /// Semantic Kernel instance. + /// .proto document stream. + /// Plugin name. + /// HttpClient to use for sending requests. + /// A list of all the semantic functions representing the plugin. + public static IDictionary RegisterGrpcFunctions( + this IKernel kernel, + Stream documentStream, + string pluginName, + HttpClient? httpClient = null) + { + Verify.NotNull(kernel); + Verify.ValidPluginName(pluginName); + + // Parse + var parser = new ProtoDocumentParser(); + + var operations = parser.Parse(documentStream, pluginName); + + var plugin = new Dictionary(); + + var client = HttpClientProvider.GetHttpClient(kernel.HttpHandlerFactory, httpClient, kernel.LoggerFactory); + + var runner = new GrpcOperationRunner(client); + + ILogger logger = kernel.LoggerFactory.CreateLogger(typeof(KernelGrpcExtensions)); + foreach (var operation in operations) + { + try + { + logger.LogTrace("Registering gRPC function {0}.{1}", pluginName, operation.Name); + var function = kernel.RegisterGrpcFunction(runner, pluginName, operation); + plugin[function.Name] = function; + } + catch (Exception ex) when (!ex.IsCriticalException()) + { + //Logging the exception and keep registering other gRPC functions + logger.LogWarning(ex, "Something went wrong while rendering the gRPC function. Function: {0}.{1}. Error: {2}", + pluginName, operation.Name, ex.Message); + } + } + + return plugin; + } + + #region private + + /// + /// Registers SKFunction for a gRPC operation. + /// + /// Semantic Kernel instance. + /// gRPC operation runner. + /// Plugin name. + /// The gRPC operation. + /// An instance of class. + private static ISKFunction RegisterGrpcFunction( + this IKernel kernel, + GrpcOperationRunner runner, + string pluginName, + GrpcOperation operation) + { + var operationParameters = operation.GetParameters(); + + async Task ExecuteAsync(SKContext context) + { + try + { + var arguments = new Dictionary(); + + //Extract function arguments from context + foreach (var parameter in operationParameters) + { + //A try to resolve argument parameter name. + if (context.Variables.TryGetValue(parameter.Name, out string? value)) + { + arguments.Add(parameter.Name, value); + continue; + } + + throw new KeyNotFoundException($"No variable found in context to use as an argument for the '{parameter.Name}' parameter of the '{pluginName}.{operation.Name}' gRPC function."); + } + + //SKFunction should be extended to pass cancellation token for delegateFunction calls. + var result = await runner.RunAsync(operation, arguments, CancellationToken.None).ConfigureAwait(false); + + if (result != null) + { + context.Variables.Update(result.ToString()); + } + } + catch (Exception ex) when (!ex.IsCriticalException()) + { + kernel.LoggerFactory.CreateLogger(typeof(KernelGrpcExtensions)).LogWarning(ex, "Something went wrong while rendering the gRPC function. Function: {0}.{1}. Error: {2}", pluginName, operation.Name, + ex.Message); + throw; + } + + return context; + } + + var function = SKFunction.FromNativeFunction( + nativeFunction: ExecuteAsync, + parameters: operationParameters.ToList(), + description: operation.Name, + pluginName: pluginName, + functionName: operation.Name, + loggerFactory: kernel.LoggerFactory); + + return kernel.RegisterCustomFunction(function); + } + + #endregion + + #region obsolete + + [Obsolete("Methods and classes which includes Skill in the name have been renamed to use Plugin. Use Kernel.ImportGrpcFunctionsFromDirectory instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + public static IDictionary ImportGrpcSkillFromDirectory( + this IKernel kernel, + string parentDirectory, + string skillDirectoryName, + HttpClient? httpClient = null) + { + return kernel.ImportGrpcFunctionsFromDirectory(parentDirectory, skillDirectoryName, httpClient); + } +#pragma warning restore CS1591 + + [Obsolete("Methods and classes which includes Skill in the name have been renamed to use Plugin. Use Kernel.ImportGrpcFunctionsFromFile instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + public static IDictionary ImportGrpcSkillFromFile( + this IKernel kernel, + string skillName, + string filePath, + HttpClient? httpClient = null) + { + return kernel.ImportGrpcFunctionsFromFile(skillName, filePath, httpClient); + } +#pragma warning restore CS1591 + + [Obsolete("Methods and classes which includes Skill in the name have been renamed to use Plugin. Use Kernel.RegisterGrpcFunctions instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + public static IDictionary RegisterGrpcSkill( + this IKernel kernel, + Stream documentStream, + string skillName, + HttpClient? httpClient = null) + { + return kernel.RegisterGrpcFunctions(documentStream, skillName, httpClient); + } +#pragma warning restore CS1591 + + #endregion +} diff --git a/dotnet/src/Functions/Functions.Grpc/Functions.Grpc.csproj b/dotnet/src/Functions/Functions.Grpc/Functions.Grpc.csproj new file mode 100644 index 000000000000..4c09330e9237 --- /dev/null +++ b/dotnet/src/Functions/Functions.Grpc/Functions.Grpc.csproj @@ -0,0 +1,39 @@ + + + + + Microsoft.SemanticKernel.Functions.Grpc + $(AssemblyName) + netstandard2.0 + + + + + + + + Semantic Kernel - gRPC Functions + Semantic Kernel gRPC Functions + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/src/Skills/Skills.Grpc/GrpcOperationRunner.cs b/dotnet/src/Functions/Functions.Grpc/GrpcOperationRunner.cs similarity index 93% rename from dotnet/src/Skills/Skills.Grpc/GrpcOperationRunner.cs rename to dotnet/src/Functions/Functions.Grpc/GrpcOperationRunner.cs index ebb5a1a1ffe7..ca3fb3bb3fb2 100644 --- a/dotnet/src/Skills/Skills.Grpc/GrpcOperationRunner.cs +++ b/dotnet/src/Functions/Functions.Grpc/GrpcOperationRunner.cs @@ -14,15 +14,15 @@ using Grpc.Core; using Grpc.Net.Client; using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Skills.Grpc.Model; +using Microsoft.SemanticKernel.Functions.Grpc.Model; using ProtoBuf; -namespace Microsoft.SemanticKernel.Skills.Grpc; +namespace Microsoft.SemanticKernel.Functions.Grpc; /// /// Runs gRPC operation runner. /// -internal class GrpcOperationRunner +internal sealed class GrpcOperationRunner { /// /// An instance of the HttpClient class. @@ -111,7 +111,7 @@ private string GetAddress(GrpcOperation operation, IDictionary a if (string.IsNullOrEmpty(address)) { - throw new GrpcOperationException($"No address provided for the '{operation.Name}' gRPC operation."); + throw new SKException($"No address provided for the '{operation.Name}' gRPC operation."); } return address!; @@ -155,14 +155,14 @@ private object GenerateOperationRequest(GrpcOperation operation, Type type, IDic //Getting 'payload' argument to by used as gRPC request message if (!arguments.TryGetValue(GrpcOperation.PayloadArgumentName, out var payload)) { - throw new GrpcOperationException($"No '{GrpcOperation.PayloadArgumentName}' argument representing gRPC request message is found for the '{operation.Name}' gRPC operation."); + throw new SKException($"No '{GrpcOperation.PayloadArgumentName}' argument representing gRPC request message is found for the '{operation.Name}' gRPC operation."); } //Deserializing JSON payload to gRPC request message var instance = JsonSerializer.Deserialize(payload, type, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); if (instance == null) { - throw new GrpcOperationException($"Impossible to create gRPC request message for the '{operation.Name}' gRPC operation."); + throw new SKException($"Impossible to create gRPC request message for the '{operation.Name}' gRPC operation."); } return instance; @@ -226,7 +226,7 @@ private static TypeInfo BuildGrpcOperationDataContractType(GrpcOperationDataCont var type = typeBuilder.CreateTypeInfo(); if (type == null) { - throw new GrpcOperationException($"Impossible to create type for '{dataContractMetadata.Name}' data contract."); + throw new SKException($"Impossible to create type for '{dataContractMetadata.Name}' data contract."); } return type; diff --git a/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperation.cs b/dotnet/src/Functions/Functions.Grpc/Model/GrpcOperation.cs similarity index 96% rename from dotnet/src/Skills/Skills.Grpc/Model/GrpcOperation.cs rename to dotnet/src/Functions/Functions.Grpc/Model/GrpcOperation.cs index b2d9c77e2728..81e1ee8a8400 100644 --- a/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperation.cs +++ b/dotnet/src/Functions/Functions.Grpc/Model/GrpcOperation.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.Grpc.Model; +namespace Microsoft.SemanticKernel.Functions.Grpc.Model; /// /// The gRPC operation. /// -internal class GrpcOperation +internal sealed class GrpcOperation { /// /// Name of 'address' argument used as override for the address provided by gRPC operation. diff --git a/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationDataContractType.cs b/dotnet/src/Functions/Functions.Grpc/Model/GrpcOperationDataContractType.cs similarity index 87% rename from dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationDataContractType.cs rename to dotnet/src/Functions/Functions.Grpc/Model/GrpcOperationDataContractType.cs index 848212b6b990..b859ed5aecfa 100644 --- a/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationDataContractType.cs +++ b/dotnet/src/Functions/Functions.Grpc/Model/GrpcOperationDataContractType.cs @@ -2,12 +2,12 @@ using System.Collections.Generic; -namespace Microsoft.SemanticKernel.Skills.Grpc.Model; +namespace Microsoft.SemanticKernel.Functions.Grpc.Model; /// /// The gRPC operation data contract. /// -internal class GrpcOperationDataContractType +internal sealed class GrpcOperationDataContractType { /// /// Creates an instance of a class. diff --git a/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationDataContractTypeFiled.cs b/dotnet/src/Functions/Functions.Grpc/Model/GrpcOperationDataContractTypeFiled.cs similarity index 87% rename from dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationDataContractTypeFiled.cs rename to dotnet/src/Functions/Functions.Grpc/Model/GrpcOperationDataContractTypeFiled.cs index acc6d2fe920a..a98db15d77a4 100644 --- a/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationDataContractTypeFiled.cs +++ b/dotnet/src/Functions/Functions.Grpc/Model/GrpcOperationDataContractTypeFiled.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.Grpc.Model; +namespace Microsoft.SemanticKernel.Functions.Grpc.Model; /// /// The gRPC operation data contract field. /// -internal class GrpcOperationDataContractTypeFiled +internal sealed class GrpcOperationDataContractTypeFiled { /// /// Creates an instance of a class. diff --git a/dotnet/src/Skills/Skills.Grpc/Protobuf/ProtoDocumentParser.cs b/dotnet/src/Functions/Functions.Grpc/Protobuf/ProtoDocumentParser.cs similarity index 88% rename from dotnet/src/Skills/Skills.Grpc/Protobuf/ProtoDocumentParser.cs rename to dotnet/src/Functions/Functions.Grpc/Protobuf/ProtoDocumentParser.cs index cb7992c3e54b..077feed9f473 100644 --- a/dotnet/src/Skills/Skills.Grpc/Protobuf/ProtoDocumentParser.cs +++ b/dotnet/src/Functions/Functions.Grpc/Protobuf/ProtoDocumentParser.cs @@ -6,15 +6,15 @@ using System.Linq; using Google.Protobuf.Reflection; using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Skills.Grpc.Model; +using Microsoft.SemanticKernel.Functions.Grpc.Model; using ProtoBuf; -namespace Microsoft.SemanticKernel.Skills.Grpc.Protobuf; +namespace Microsoft.SemanticKernel.Functions.Grpc.Protobuf; /// /// Parser for .proto definition documents. /// -internal class ProtoDocumentParser +internal sealed class ProtoDocumentParser { /// /// Parses .proto document. @@ -36,7 +36,7 @@ public IList Parse(Stream protoDocument, string protoFileName) var errors = descriptor.GetErrors(); if (errors != null && errors.Length != 0) { - throw new ProtobufParsingException($"Parsing of '{protoFileName}' .proto document has failed. Details: {string.Join(";", errors.AsEnumerable())}"); + throw new SKException($"Parsing of '{protoFileName}' .proto document has failed. Details: {string.Join(";", errors.AsEnumerable())}"); } return this.GetGrpcOperations(descriptor.Files.Single()); @@ -91,7 +91,7 @@ private GrpcOperationDataContractType CreateDataContract(IList var messageType = allMessageTypes.SingleOrDefault(mt => mt.Name == fullTypeName || mt.Name == typeName); if (messageType == null) { - throw new ProtobufParsingException($"No '{fullTypeName}' message type is found while resolving data contracts for the '{methodName}' method."); + throw new SKException($"No '{fullTypeName}' message type is found while resolving data contracts for the '{methodName}' method."); } var fields = this.GetDataContractFields(messageType.Fields); @@ -132,7 +132,7 @@ private static string GetProtobufDataTypeName(FieldDescriptorProto.Type type) if (attribute == null) { - throw new ProtobufParsingException($"Impossible to find protobuf type name corresponding to '{type}' type."); + throw new SKException($"Impossible to find protobuf type name corresponding to '{type}' type."); } return attribute.Name; diff --git a/dotnet/src/Functions/Functions.OpenAPI/Authentication/AuthenticateRequestAsyncCallback.cs b/dotnet/src/Functions/Functions.OpenAPI/Authentication/AuthenticateRequestAsyncCallback.cs new file mode 100644 index 000000000000..dda578e223ab --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Authentication/AuthenticateRequestAsyncCallback.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using System.Threading.Tasks; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; + +/// +/// Represents a delegate that defines the method signature for asynchronously authenticating an HTTP request. +/// +/// The to authenticate. +/// A representing the asynchronous operation. +public delegate Task AuthenticateRequestAsyncCallback(HttpRequestMessage request); diff --git a/dotnet/src/Skills/Skills.OpenAPI/Authentication/BasicAuthenticationProvider.cs b/dotnet/src/Functions/Functions.OpenAPI/Authentication/BasicAuthenticationProvider.cs similarity index 95% rename from dotnet/src/Skills/Skills.OpenAPI/Authentication/BasicAuthenticationProvider.cs rename to dotnet/src/Functions/Functions.OpenAPI/Authentication/BasicAuthenticationProvider.cs index ab1e13e42586..640cc5d18cd4 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/Authentication/BasicAuthenticationProvider.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Authentication/BasicAuthenticationProvider.cs @@ -6,7 +6,7 @@ using System.Text; using System.Threading.Tasks; -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; /// /// Retrieves authentication content (e.g. username/password, API key) via the provided delegate and diff --git a/dotnet/src/Skills/Skills.OpenAPI/Authentication/BearerAuthenticationProvider.cs b/dotnet/src/Functions/Functions.OpenAPI/Authentication/BearerAuthenticationProvider.cs similarity index 94% rename from dotnet/src/Skills/Skills.OpenAPI/Authentication/BearerAuthenticationProvider.cs rename to dotnet/src/Functions/Functions.OpenAPI/Authentication/BearerAuthenticationProvider.cs index 1179b5d695f3..68591f32f5dc 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/Authentication/BearerAuthenticationProvider.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Authentication/BearerAuthenticationProvider.cs @@ -5,7 +5,7 @@ using System.Net.Http.Headers; using System.Threading.Tasks; -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; /// /// Retrieves a token via the provided delegate and applies it to HTTP requests using the diff --git a/dotnet/src/Functions/Functions.OpenAPI/Authentication/CustomAuthenticationProvider.cs b/dotnet/src/Functions/Functions.OpenAPI/Authentication/CustomAuthenticationProvider.cs new file mode 100644 index 000000000000..3769b4877ba8 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Authentication/CustomAuthenticationProvider.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; + +/// +/// Retrieves authentication content (scheme and value) via the provided delegate and applies it to HTTP requests. +/// +public sealed class CustomAuthenticationProvider +{ + private readonly Func> _header; + private readonly Func> _value; + + /// + /// Creates an instance of the class. + /// + /// Delegate for retrieving the header name. + /// Delegate for retrieving the value. + public CustomAuthenticationProvider(Func> header, Func> value) + { + this._header = header; + this._value = value; + } + + /// + /// Applies the header and value to the provided HTTP request message. + /// + /// The HTTP request message. + /// + public async Task AuthenticateRequestAsync(HttpRequestMessage request) + { + var header = await this._header().ConfigureAwait(false); + var value = await this._value().ConfigureAwait(false); + request.Headers.Add(header, value); + } +} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Authentication/InteractiveMsalAuthenticationProvider.cs b/dotnet/src/Functions/Functions.OpenAPI/Authentication/InteractiveMsalAuthenticationProvider.cs similarity index 97% rename from dotnet/src/Skills/Skills.OpenAPI/Authentication/InteractiveMsalAuthenticationProvider.cs rename to dotnet/src/Functions/Functions.OpenAPI/Authentication/InteractiveMsalAuthenticationProvider.cs index 4d8e0238ea36..ea49380224f8 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/Authentication/InteractiveMsalAuthenticationProvider.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Authentication/InteractiveMsalAuthenticationProvider.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Microsoft.Identity.Client; -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; /// /// Uses the Microsoft Authentication Library (MSAL) to authenticate HTTP requests. diff --git a/dotnet/src/Functions/Functions.OpenAPI/Authentication/README.md b/dotnet/src/Functions/Functions.OpenAPI/Authentication/README.md new file mode 100644 index 000000000000..7df11ca60cdf --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Authentication/README.md @@ -0,0 +1,66 @@ +# Authentication for the OpenAPI Functions + +The Semantic Kernel OpenAPI Function enables developers to take any REST API that follows the OpenAPI specification and import it as a plugin to the Semantic Kernel. However, the Kernel needs to be able to authenticate outgoing requests per the requirements of the target API. This document outlines the authentication model for the OpenAPI plugin as well as the reference implementations provided by the Semantic Kernel. + +## The `AuthenticateRequestAsyncCallback` delegate + +[`AuthenticateRequestAsyncCallback`](AuthenticateRequestAsyncCallback.cs) is a delegate type that serves as a callback function for adding authentication information to HTTP requests sent by the OpenAPI plugin. + +```csharp +public delegate Task AuthenticateRequestAsyncCallback(HttpRequestMessage request); +``` + +Developers may optionally provide an implementation of this delegate when importing an OpenAPI plugin to the Kernel. The delegate is then passed through to the `RestApiOperationRunner`, which is responsible for building the HTTP payload and sending the request for each REST API operation. Before the API request is sent, the delegate is executed with the HTTP request message as the parameter, allowing the request message to be updated with any necessary authentication information. + +This pattern was designed to be flexible enough to support a wide variety of authentication frameworks. Developers can provide the delegate function directly or define a class or interface that exposes one or more implementations. They have the option of writing their own custom implementation or using one of the Semantic Kernel's reference authentication providers as a starting point. + +## Reference Authentication Providers + +### [`BasicAuthenticationProvider`](./BasicAuthenticationProvider.cs) +This class implements the HTTP "basic" authentication scheme. The constructor accepts a `Func` which defines how to retrieve the user's credentials. When the `AuthenticateRequestAsync` method is called, it retrieves the credentials, encodes them as a UTF-8 encoded Base64 string, and adds them to the `HttpRequestMessage`'s authorization header. + +The following code demonstrates how to use this provider: +```csharp +var basicAuthProvider = new BasicAuthenticationProvider(() => +{ + // JIRA API expects credentials in the format "email:apikey" + return Task.FromResult( + Env.Var("MY_EMAIL_ADDRESS") + ":" + Env.Var("JIRA_API_KEY") + ); +}); +var plugin = kernel.ImportOpenApiPluginFromResource(PluginResourceNames.Jira, new OpenApiPluginExecutionParameters { AuthCallback = basicAuthProvider.AuthenticateRequestAsync } ); +``` + +### [`BearerAuthenticationProvider`](./BearerAuthenticationProvider.cs) +This class implements the HTTP "bearer" authentication scheme. The constructor accepts a `Func` which defines how to retrieve the bearer token. When the `AuthenticateRequestAsync` method is called, it retrieves the token and adds it to the `HttpRequestMessage`'s authorization header. + +The following code demonstrates how to use this provider: +```csharp +var bearerAuthProvider = new BearerAuthenticationProvider(() => +{ + return Task.FromResult(Env.Var("AZURE_KEYVAULT_TOKEN")); +}); +var plugin = kernel.ImportOpenApiPluginFromResource(PluginResourceNames.AzureKeyVault, new OpenApiPluginExecutionParameters { AuthCallback = bearerAuthProvider.AuthenticateRequestAsync } ) +``` + +### [`InteractiveMsalAuthenticationProvider`](./InteractiveMsalAuthenticationProvider.cs) + +This class uses the [Microsoft Authentication Library (MSAL)](https://learn.microsoft.com/en-us/azure/active-directory/develop/msal-overview)'s .NET library to authenticate the user and acquire an OAuth token. It follows the interactive [authorization code flow](https://learn.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-auth-code-flow), requiring the user to sign in with a Microsoft or Azure identity. This is particularly useful for authenticating requests to the Microsoft Graph or Azure APIs. + +Once the token is acquired, it is added to the HTTP authentication header via the `AuthenticateRequestAsync` method, which is inherited from `BearerAuthenticationProvider`. + +To construct this provider, the caller must specify: +- *Client ID* – identifier of the calling application. This is acquired by [registering your application with the Microsoft Identity platform](https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app). +- *Tenant ID* – identifier of the target service tenant, or “common” +- *Scopes* – permissions being requested +- *Redirect URI* – for redirecting the user back to the application. (When running locally, this is typically http://localhost.) + +```csharp +var msalAuthProvider = new InteractiveMsalAuthenticationProvider( + Env.Var("AZURE_KEYVAULT_CLIENTID"), // clientId + Env.Var("AZURE_KEYVAULT_TENANTID"), // tenantId + new string[] { ".default" }, // scopes + new Uri("http://localhost") // redirectUri +); +var plugin = kernel.ImportOpenApiPluginFromResource(PluginResourceNames.AzureKeyVault, new OpenApiPluginExecutionParameters { AuthCallback = msalAuthProvider.AuthenticateRequestAsync } ) +``` \ No newline at end of file diff --git a/dotnet/src/Functions/Functions.OpenAPI/Builders/QueryStringBuilder.cs b/dotnet/src/Functions/Functions.OpenAPI/Builders/QueryStringBuilder.cs new file mode 100644 index 000000000000..9bb2885a446f --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Builders/QueryStringBuilder.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Builders; + +/// +/// Represents a query string builder for REST API operations. +/// +internal static class QueryStringBuilder +{ + /// + /// Query string parameter serializers. + /// + private static readonly Dictionary> s_queryStringParameterSerializers = new() + { + { RestApiOperationParameterStyle.Form, FormStyleParameterSerializer.Serialize }, + { RestApiOperationParameterStyle.SpaceDelimited, SpaceDelimitedStyleParameterSerializer.Serialize }, + { RestApiOperationParameterStyle.PipeDelimited, PipeDelimitedStyleParameterSerializer.Serialize } + }; + + /// + public static string BuildQueryString(this RestApiOperation operation, IDictionary arguments) + { + var segments = new List(); + + var parameters = operation.Parameters.Where(p => p.Location == RestApiOperationParameterLocation.Query); + + foreach (var parameter in parameters) + { + if (!arguments.TryGetValue(parameter.Name, out var argument)) + { + //Throw an exception if the parameter is a required one but no value is provided. + if (parameter.IsRequired) + { + throw new SKException($"No argument found for the `{parameter.Name}` required parameter"); + } + + //Skipping not required parameter if no argument provided for it. + continue; + } + + var parameterStyle = parameter.Style ?? RestApiOperationParameterStyle.Form; + + if (!s_queryStringParameterSerializers.TryGetValue(parameterStyle, out var serializer)) + { + throw new SKException($"The query string parameter `{parameterStyle}` serialization style is not supported."); + } + + //Serializing the parameter and adding it to the query string if there's an argument for it. + segments.Add(serializer.Invoke(parameter, argument)); + } + + return string.Join("&", segments); + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/ArrayParameterValueSerializer.cs b/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/ArrayParameterValueSerializer.cs new file mode 100644 index 000000000000..943f2a3ac150 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/ArrayParameterValueSerializer.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Nodes; +using System.Web; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; + +/// +/// This class provides methods for serializing values of array parameters. +/// +internal static class ArrayParameterValueSerializer +{ + /// + /// Serializes the items of an array as separate parameters with the same name. + /// + /// The name of the parameter. + /// The array containing the items to be serialized. + /// The delimiter used to separate parameters. + /// A string containing the serialized parameters. + public static string SerializeArrayAsSeparateParameters(string name, JsonArray array, string delimiter) + { + var segments = new List(); + + foreach (var item in array) + { + segments.Add($"{name}={HttpUtility.UrlEncode(item?.ToString())}"); + } + + return string.Join(delimiter, segments); //id=1&id=2&id=3 + } + + /// + /// Serializes the items of an array as one parameter with delimited values. + /// + /// The array containing the items to be serialized. + /// The delimiter used to separate items. + /// A string containing the serialized parameter. + public static string SerializeArrayAsDelimitedValues(JsonArray array, string delimiter) + { + var values = new List(); + + foreach (var item in array) + { + values.Add(HttpUtility.UrlEncode(item?.ToString())); + } + + return string.Join(delimiter, values); + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/FormStyleParameterSerializer.cs b/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/FormStyleParameterSerializer.cs new file mode 100644 index 000000000000..12a97a494df7 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/FormStyleParameterSerializer.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Nodes; +using System.Web; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; + +/// +/// Serializes REST API operation parameter of the 'Form' style. +/// +internal static class FormStyleParameterSerializer +{ + /// + /// Serializes a REST API operation `Form` style parameter. + /// + /// The REST API operation parameter to serialize. + /// The parameter argument. + /// The serialized parameter. + public static string Serialize(RestApiOperationParameter parameter, string argument) + { + const string ArrayType = "array"; + + if (parameter is null) + { + throw new ArgumentNullException(nameof(parameter)); + } + + if (parameter.Style != RestApiOperationParameterStyle.Form) + { + throw new SKException($"Unexpected Rest Api operation parameter style - `{parameter.Style}`"); + } + + // Handling parameters of array type. + if (parameter.Type == ArrayType) + { + return SerializeArrayParameter(parameter, argument); + } + + // Handling parameters of primitive - integer, string, etc type. + return $"{parameter.Name}={HttpUtility.UrlEncode(argument)}"; + } + + /// + /// Serializes an array-type parameter. + /// + /// The REST API operation parameter to serialize. + /// The argument value. + /// The serialized parameter string. + private static string SerializeArrayParameter(RestApiOperationParameter parameter, string argument) + { + if (JsonNode.Parse(argument) is not JsonArray array) + { + throw new SKException($"Can't deserialize parameter name `{parameter.Name}` argument `{argument}` to JSON array"); + } + + if (parameter.Expand) + { + return ArrayParameterValueSerializer.SerializeArrayAsSeparateParameters(parameter.Name, array, delimiter: "&"); //id=1&id=2&id=3 + } + + return $"{parameter.Name}={ArrayParameterValueSerializer.SerializeArrayAsDelimitedValues(array, delimiter: ",")}"; //id=1,2,3 + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/PipeDelimitedStyleParameterSerializer.cs b/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/PipeDelimitedStyleParameterSerializer.cs new file mode 100644 index 000000000000..8d2a148d68f7 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/PipeDelimitedStyleParameterSerializer.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Nodes; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; + +/// +/// Serializes REST API operation parameter of the 'PipeDelimited' style. +/// +internal static class PipeDelimitedStyleParameterSerializer +{ + /// + /// Serializes a REST API operation `PipeDelimited` style parameter. + /// + /// The REST API operation parameter to serialize. + /// The parameter argument. + /// The serialized parameter. + public static string Serialize(RestApiOperationParameter parameter, string argument) + { + const string ArrayType = "array"; + + if (parameter is null) + { + throw new ArgumentNullException(nameof(parameter)); + } + + if (parameter.Style != RestApiOperationParameterStyle.PipeDelimited) + { + throw new SKException($"Unexpected Rest Api operation parameter style `{parameter.Style}`. Parameter name `{parameter.Name}`."); + } + + if (parameter.Type != ArrayType) + { + throw new SKException($"Serialization of Rest API operation parameters of type `{parameter.Type}` is not supported for the `{RestApiOperationParameterStyle.PipeDelimited}` style parameters. Parameter name `{parameter.Name}`."); + } + + return SerializeArrayParameter(parameter, argument); + } + + /// + /// Serializes an array-type parameter. + /// + /// The REST API operation parameter to serialize. + /// The argument value. + /// The serialized parameter string. + private static string SerializeArrayParameter(RestApiOperationParameter parameter, string argument) + { + if (JsonNode.Parse(argument) is not JsonArray array) + { + throw new SKException($"Can't deserialize parameter name `{parameter.Name}` argument `{argument}` to JSON array."); + } + + if (parameter.Expand) + { + return ArrayParameterValueSerializer.SerializeArrayAsSeparateParameters(parameter.Name, array, delimiter: "&"); //id=1&id=2&id=3 + } + + return $"{parameter.Name}={ArrayParameterValueSerializer.SerializeArrayAsDelimitedValues(array, delimiter: "|")}"; //id=1|2|3 + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/SpaceDelimitedStyleParameterSerializer.cs b/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/SpaceDelimitedStyleParameterSerializer.cs new file mode 100644 index 000000000000..43afd123ad21 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Builders/Serialization/SpaceDelimitedStyleParameterSerializer.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Nodes; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; + +/// +/// Serializes REST API operation parameter of the 'SpaceDelimited' style. +/// +internal static class SpaceDelimitedStyleParameterSerializer +{ + /// + /// Serializes a REST API operation `SpaceDelimited` style parameter. + /// + /// The REST API operation parameter to serialize. + /// The parameter argument. + /// The serialized parameter. + public static string Serialize(RestApiOperationParameter parameter, string argument) + { + const string ArrayType = "array"; + + if (parameter is null) + { + throw new ArgumentNullException(nameof(parameter)); + } + + if (parameter.Style != RestApiOperationParameterStyle.SpaceDelimited) + { + throw new SKException($"Unexpected Rest Api operation parameter style `{parameter.Style}`. Parameter name `{parameter.Name}`."); + } + + if (parameter.Type != ArrayType) + { + throw new SKException($"Serialization of Rest API operation parameters of type `{parameter.Type}` is not supported for the `{RestApiOperationParameterStyle.SpaceDelimited}` style parameters. Parameter name `{parameter.Name}`."); + } + + return SerializeArrayParameter(parameter, argument); + } + + /// + /// Serializes an array-type parameter. + /// + /// The REST API operation parameter to serialize. + /// The argument value. + /// The serialized parameter string. + private static string SerializeArrayParameter(RestApiOperationParameter parameter, string argument) + { + if (JsonNode.Parse(argument) is not JsonArray array) + { + throw new SKException($"Can't deserialize parameter name `{parameter.Name}` argument `{argument}` to JSON array."); + } + + if (parameter.Expand) + { + return ArrayParameterValueSerializer.SerializeArrayAsSeparateParameters(parameter.Name, array, delimiter: "&"); //id=1&id=2&id=3 + } + + return $"{parameter.Name}={ArrayParameterValueSerializer.SerializeArrayAsDelimitedValues(array, delimiter: "%20")}"; //id=1%202%203 + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Extensions/KernelAIPluginExtensions.cs b/dotnet/src/Functions/Functions.OpenAPI/Extensions/KernelAIPluginExtensions.cs new file mode 100644 index 000000000000..ff0ba2a22f55 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Extensions/KernelAIPluginExtensions.cs @@ -0,0 +1,459 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Functions.OpenAPI.OpenApi; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; + +/// +/// Provides extension methods for importing AI plugins exposed as OpenAPI v3 endpoints or through OpenAI's ChatGPT format. +/// +public static class KernelAIPluginExtensions +{ + [Obsolete("Methods and classes which includes Skill in the name have been renamed to use Plugin. Use Kernel.ImportPluginFunctionsAsync instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + public static async Task> ImportAIPluginAsync( + this IKernel kernel, + string pluginName, + string filePath, + OpenApiFunctionExecutionParameters? executionParameters = null, + CancellationToken cancellationToken = default) + { + return await kernel.ImportPluginFunctionsAsync(pluginName, filePath, executionParameters, cancellationToken).ConfigureAwait(false); + } +#pragma warning restore CS1591 + + /// + /// Imports an AI plugin that is exposed as an OpenAPI v3 endpoint or through OpenAI's ChatGPT format. + /// + /// Semantic Kernel instance. + /// Plugin name. + /// The file path to the AI Plugin + /// Plugin execution parameters. + /// The cancellation token. + /// A collection of invocable functions + public static async Task> ImportPluginFunctionsAsync( + this IKernel kernel, + string pluginName, + string filePath, + OpenApiFunctionExecutionParameters? executionParameters = null, + CancellationToken cancellationToken = default) + { + Verify.NotNull(kernel); + Verify.ValidPluginName(pluginName); + +#pragma warning disable CA2000 // Dispose objects before losing scope. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. + var httpClient = HttpClientProvider.GetHttpClient(kernel.HttpHandlerFactory, executionParameters?.HttpClient, kernel.LoggerFactory); +#pragma warning restore CA2000 + + var pluginContents = await LoadDocumentFromFilePathAsync( + kernel, + filePath, + executionParameters, + httpClient, + cancellationToken).ConfigureAwait(false); + + return await CompleteImportAsync( + kernel, + pluginContents, + pluginName, + httpClient, + executionParameters, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + [Obsolete("Methods and classes which includes Skill in the name have been renamed to use Plugin. Use Kernel.ImportPluginFunctionsAsync instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + public static async Task> ImportAIPluginAsync( + this IKernel kernel, + string pluginName, + Uri uri, + OpenApiFunctionExecutionParameters? executionParameters = null, + CancellationToken cancellationToken = default) + { + return await kernel.ImportPluginFunctionsAsync(pluginName, uri, executionParameters, cancellationToken).ConfigureAwait(false); + } +#pragma warning restore CS1591 + + /// + /// Imports an AI plugin that is exposed as an OpenAPI v3 endpoint or through OpenAI's ChatGPT format. + /// + /// Semantic Kernel instance. + /// Plugin name. + /// A local or remote URI referencing the AI Plugin + /// Plugin execution parameters. + /// The cancellation token. + /// A collection of invocable functions + public static async Task> ImportPluginFunctionsAsync( + this IKernel kernel, + string pluginName, + Uri uri, + OpenApiFunctionExecutionParameters? executionParameters = null, + CancellationToken cancellationToken = default) + { + Verify.NotNull(kernel); + Verify.ValidPluginName(pluginName); + +#pragma warning disable CA2000 // Dispose objects before losing scope. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. + var httpClient = HttpClientProvider.GetHttpClient(kernel.HttpHandlerFactory, executionParameters?.HttpClient, kernel.LoggerFactory); +#pragma warning restore CA2000 + + var pluginContents = await LoadDocumentFromUriAsync( + kernel, + uri, + executionParameters, + httpClient, + cancellationToken).ConfigureAwait(false); + + return await CompleteImportAsync( + kernel, + pluginContents, + pluginName, + httpClient, + executionParameters, + uri, + cancellationToken).ConfigureAwait(false); + } + + /// + /// Imports an AI plugin that is exposed as an OpenAPI v3 endpoint or through OpenAI's ChatGPT format. + /// + /// Semantic Kernel instance. + /// Plugin name. + /// A stream representing the AI Plugin + /// Plugin execution parameters. + /// The cancellation token. + /// A collection of invocable functions + public static async Task> ImportPluginFunctionsAsync( + this IKernel kernel, + string pluginName, + Stream stream, + OpenApiFunctionExecutionParameters? executionParameters = null, + CancellationToken cancellationToken = default) + { + Verify.NotNull(kernel); + Verify.ValidPluginName(pluginName); + +#pragma warning disable CA2000 // Dispose objects before losing scope. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. + var httpClient = HttpClientProvider.GetHttpClient(kernel.HttpHandlerFactory, executionParameters?.HttpClient, kernel.LoggerFactory); +#pragma warning restore CA2000 + + var pluginContents = await LoadDocumentFromStreamAsync(kernel, stream).ConfigureAwait(false); + + return await CompleteImportAsync( + kernel, + pluginContents, + pluginName, + httpClient, + executionParameters, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + #region private + + private static async Task> CompleteImportAsync( + IKernel kernel, + string pluginContents, + string pluginName, + HttpClient httpClient, + OpenApiFunctionExecutionParameters? executionParameters, + Uri? documentUri = null, + CancellationToken cancellationToken = default) + { + if (TryParseAIPluginForUrl(pluginContents, out var openApiUrl)) + { + return await kernel + .ImportPluginFunctionsAsync( + pluginName, + new Uri(openApiUrl), + executionParameters, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + return await LoadPluginAsync( + kernel, + pluginName, + executionParameters, + httpClient, + pluginContents, + documentUri, + cancellationToken).ConfigureAwait(false); + } + + private static async Task> LoadPluginAsync( + IKernel kernel, + string pluginName, + OpenApiFunctionExecutionParameters? executionParameters, + HttpClient httpClient, + string pluginJson, + Uri? documentUri = null, + CancellationToken cancellationToken = default) + { + var parser = new OpenApiDocumentParser(kernel.LoggerFactory); + + using (var documentStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(pluginJson))) + { + var operations = await parser.ParseAsync(documentStream, executionParameters?.IgnoreNonCompliantErrors ?? false, cancellationToken).ConfigureAwait(false); + + var runner = new RestApiOperationRunner( + httpClient, + executionParameters?.AuthCallback, + executionParameters?.UserAgent, + executionParameters?.EnableDynamicPayload ?? false, + executionParameters?.EnablePayloadNamespacing ?? false); + + var plugin = new Dictionary(); + + ILogger logger = kernel.LoggerFactory.CreateLogger(typeof(KernelAIPluginExtensions)); + foreach (var operation in operations) + { + try + { + logger.LogTrace("Registering Rest function {0}.{1}", pluginName, operation.Id); + var function = kernel.RegisterRestApiFunction(pluginName, runner, operation, executionParameters, documentUri, cancellationToken); + plugin[function.Name] = function; + } + catch (Exception ex) when (!ex.IsCriticalException()) + { + //Logging the exception and keep registering other Rest functions + logger.LogWarning(ex, "Something went wrong while rendering the Rest function. Function: {0}.{1}. Error: {2}", + pluginName, operation.Id, ex.Message); + } + } + + return plugin; + } + } + + private static async Task LoadDocumentFromUriAsync( + IKernel kernel, + Uri uri, + OpenApiFunctionExecutionParameters? executionParameters, + HttpClient httpClient, + CancellationToken cancellationToken) + { + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, uri.ToString()); + + requestMessage.Headers.UserAgent.Add(ProductInfoHeaderValue.Parse(executionParameters?.UserAgent ?? Telemetry.HttpUserAgent)); + + using var response = await httpClient.SendWithSuccessCheckAsync(requestMessage, cancellationToken).ConfigureAwait(false); + + return await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); + } + + private static async Task LoadDocumentFromFilePathAsync( + IKernel kernel, + string filePath, + OpenApiFunctionExecutionParameters? executionParameters, + HttpClient httpClient, + CancellationToken cancellationToken) + { + var pluginJson = string.Empty; + + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"Invalid URI. The specified path '{filePath}' does not exist."); + } + + kernel.LoggerFactory.CreateLogger(typeof(KernelAIPluginExtensions)).LogTrace("Importing AI Plugin from {0}", filePath); + + using (var sr = File.OpenText(filePath)) + { + return await sr.ReadToEndAsync().ConfigureAwait(false); //must await here to avoid stream reader being disposed before the string is read + } + } + + private static async Task LoadDocumentFromStreamAsync( + IKernel kernel, + Stream stream) + { + using StreamReader reader = new(stream); + return await reader.ReadToEndAsync().ConfigureAwait(false); + } + + private static bool TryParseAIPluginForUrl(string gptPluginJson, out string? openApiUrl) + { + try + { + JsonNode? gptPlugin = JsonNode.Parse(gptPluginJson); + + string? apiType = gptPlugin?["api"]?["type"]?.ToString(); + + if (string.IsNullOrWhiteSpace(apiType) || apiType != "openapi") + { + openApiUrl = null; + + return false; + } + + openApiUrl = gptPlugin?["api"]?["url"]?.ToString(); + + if (string.IsNullOrWhiteSpace(openApiUrl)) + { + return false; + } + + return true; + } + catch (System.Text.Json.JsonException) + { + openApiUrl = null; + + return false; + } + } + + /// + /// Registers SKFunction for a REST API operation. + /// + /// Semantic Kernel instance. + /// Plugin name. + /// The REST API operation runner. + /// The REST API operation. + /// Function execution parameters. + /// The URI of OpenApi document. + /// The cancellation token. + /// An instance of class. + private static ISKFunction RegisterRestApiFunction( + this IKernel kernel, + string pluginName, + RestApiOperationRunner runner, + RestApiOperation operation, + OpenApiFunctionExecutionParameters? executionParameters, + Uri? documentUri = null, + CancellationToken cancellationToken = default) + { + var restOperationParameters = operation.GetParameters( + executionParameters?.ServerUrlOverride, + executionParameters?.EnableDynamicPayload ?? false, + executionParameters?.EnablePayloadNamespacing ?? false, + documentUri + ); + + var logger = kernel.LoggerFactory is not null ? kernel.LoggerFactory.CreateLogger(typeof(KernelAIPluginExtensions)) : NullLogger.Instance; + + async Task ExecuteAsync(SKContext context) + { + try + { + // Extract function arguments from context + var arguments = new Dictionary(); + foreach (var parameter in restOperationParameters) + { + // A try to resolve argument by alternative parameter name + if (!string.IsNullOrEmpty(parameter.AlternativeName) && context.Variables.TryGetValue(parameter.AlternativeName!, out string? value)) + { + arguments.Add(parameter.Name, value); + continue; + } + + // A try to resolve argument by original parameter name + if (context.Variables.TryGetValue(parameter.Name, out value)) + { + arguments.Add(parameter.Name, value); + continue; + } + + if (parameter.IsRequired) + { + throw new KeyNotFoundException( + $"No variable found in context to use as an argument for the '{parameter.Name}' parameter of the '{pluginName}.{operation.Id}' Rest function."); + } + } + + var options = new RestApiOperationRunOptions + { + ServerUrlOverride = executionParameters?.ServerUrlOverride, + ApiHostUrl = documentUri is not null ? new Uri(documentUri.GetLeftPart(UriPartial.Authority)) : null + }; + + return await runner.RunAsync(operation, arguments, options, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (!ex.IsCriticalException()) + { + logger.LogError(ex, "RestAPI function {Plugin}.{Name} execution failed with error {Error}", pluginName, operation.Id, ex.Message); + throw; + } + } + + var parameters = restOperationParameters + .Select(p => new ParameterView(p.AlternativeName ?? p.Name) + { + Description = $"{p.Description ?? p.Name}{(p.IsRequired ? " (required)" : string.Empty)}", + DefaultValue = p.DefaultValue ?? string.Empty, + Type = string.IsNullOrEmpty(p.Type) ? null : new ParameterViewType(p.Type), + IsRequired = p.IsRequired, + }) + .ToList(); + + var function = SKFunction.FromNativeFunction( + nativeFunction: ExecuteAsync, + parameters: parameters, + description: operation.Description, + pluginName: pluginName, + functionName: ConvertOperationIdToValidFunctionName(operation.Id, logger), + loggerFactory: kernel.LoggerFactory); + + return kernel.RegisterCustomFunction(function); + } + + /// + /// Converts operation id to valid SK Function name. + /// A function name can contain only ASCII letters, digits, and underscores. + /// + /// The operation id. + /// The logger. + /// Valid SK Function name. + private static string ConvertOperationIdToValidFunctionName(string operationId, ILogger logger) + { + try + { + Verify.ValidFunctionName(operationId); + return operationId; + } + catch (SKException) + { + } + + // Tokenize operation id on forward and back slashes + string[] tokens = operationId.Split('/', '\\'); + string result = string.Empty; + + foreach (string token in tokens) + { + // Removes all characters that are not ASCII letters, digits, and underscores. + string formattedToken = s_removeInvalidCharsRegex.Replace(token, ""); + result += CultureInfo.CurrentCulture.TextInfo.ToTitleCase(formattedToken.ToLower(CultureInfo.CurrentCulture)); + } + + logger.LogInformation("Operation name \"{0}\" converted to \"{1}\" to comply with SK Function name requirements. Use \"{2}\" when invoking function.", operationId, result, result); + + return result; + } + + /// + /// Used to convert operationId to SK function names. + /// + private static readonly Regex s_removeInvalidCharsRegex = new("[^0-9A-Za-z_]"); + + #endregion +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Extensions/OpenApiFunctionExecutionParameters.cs b/dotnet/src/Functions/Functions.OpenAPI/Extensions/OpenApiFunctionExecutionParameters.cs new file mode 100644 index 000000000000..5d7a045b1998 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Extensions/OpenApiFunctionExecutionParameters.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; + +/// +/// OpenAPI function execution parameters. +/// +public class OpenApiFunctionExecutionParameters +{ + /// + /// HttpClient to use for sending HTTP requests. + /// + public HttpClient? HttpClient { get; set; } + + /// + /// Callback for adding authentication data to HTTP requests. + /// + public AuthenticateRequestAsyncCallback? AuthCallback { get; set; } + + /// + /// Override for REST API operation server url. + /// + public Uri? ServerUrlOverride { get; set; } + + /// + /// Flag indicating whether to ignore non-compliant errors or not. + /// If set to true, the operation execution will not throw exceptions for non-compliant documents. + /// Please note that enabling this option may result in incomplete or inaccurate execution results. + /// + public bool IgnoreNonCompliantErrors { get; set; } + + /// + /// Optional user agent header value. + /// + public string UserAgent { get; set; } + + /// + /// Determines whether the operation payload is constructed dynamically based on operation payload metadata. + /// If false, the operation payload must be provided via the 'payload' context variable. + /// + public bool EnableDynamicPayload { get; set; } + + /// + /// Determines whether payload parameter names are augmented with namespaces. + /// Namespaces prevent naming conflicts by adding the parent parameter name as a prefix, separated by dots. + /// For instance, without namespaces, the 'email' parameter for both the 'sender' and 'receiver' parent parameters + /// would be resolved from the same 'email' argument, which is incorrect. However, by employing namespaces, + /// the parameters 'sender.email' and 'sender.receiver' will be correctly resolved from arguments with the same names. + /// + public bool EnablePayloadNamespacing { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// The HttpClient to use for sending HTTP requests. + /// The callback for adding authentication data to HTTP requests. + /// The override for the REST API operation server URL. + /// Optional user agent header value. + /// A flag indicating whether to ignore non-compliant errors or not + /// If set to true, the operation execution will not throw exceptions for non-compliant documents. + /// Please note that enabling this option may result in incomplete or inaccurate execution results. + /// Determines whether the operation payload is constructed dynamically based on operation payload metadata. + /// If false, the operation payload must be provided via the 'payload' context variable. + /// Determines whether payload parameter names are augmented with namespaces. + /// Namespaces prevent naming conflicts by adding the parent parameter name as a prefix, separated by dots. + public OpenApiFunctionExecutionParameters( + HttpClient? httpClient = null, + AuthenticateRequestAsyncCallback? authCallback = null, + Uri? serverUrlOverride = null, + string userAgent = Telemetry.HttpUserAgent, + bool ignoreNonCompliantErrors = false, + bool enableDynamicOperationPayload = false, + bool enablePayloadNamespacing = false) + { + this.HttpClient = httpClient; + this.AuthCallback = authCallback; + this.ServerUrlOverride = serverUrlOverride; + this.UserAgent = userAgent; + this.IgnoreNonCompliantErrors = ignoreNonCompliantErrors; + this.EnableDynamicPayload = enableDynamicOperationPayload; + this.EnablePayloadNamespacing = enablePayloadNamespacing; + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Extensions/RestApiOperationExtensions.cs b/dotnet/src/Functions/Functions.OpenAPI/Extensions/RestApiOperationExtensions.cs new file mode 100644 index 000000000000..b69dc88d8b00 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Extensions/RestApiOperationExtensions.cs @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using Microsoft.SemanticKernel.Diagnostics; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; +#pragma warning restore IDE0130 + +/// +/// Class for extensions methods for the class. +/// +internal static class RestApiOperationExtensions +{ + /// + /// Returns list of REST API operation parameters. + /// + /// The REST API operation. + /// The server URL override. + /// Determines whether to include the operation payload parameters from payload metadata. + /// If false, the 'payload' and 'content-type' artificial parameters are added instead. + /// + /// Determines whether parameter names are augmented with namespaces. + /// Namespaces are created by prefixing parameter names with their root parameter names. + /// For instance, without namespaces, the 'email' parameter for both the 'sender' and 'receiver' parent parameters + /// would be resolved from the same 'email' argument, which is incorrect. However, by employing namespaces, + /// the parameters 'sender.email' and 'receiver.mail' will be correctly resolved from arguments with the same names. + /// + /// The URI of OpenApi document. + /// The list of parameters. + public static IReadOnlyList GetParameters( + this RestApiOperation operation, + Uri? serverUrlOverride = null, + bool addPayloadParamsFromMetadata = false, + bool enablePayloadNamespacing = false, + Uri? documentUri = null) + { + string? serverUrlString = null; + Uri? serverUrl = serverUrlOverride ?? operation.ServerUrl ?? documentUri; + + if (serverUrl is not null) + { + serverUrlString = $"{serverUrl.GetLeftPart(UriPartial.Authority)}/"; + } + + var parameters = new List(operation.Parameters) + { + // Register the "server-url" parameter if override is provided + new RestApiOperationParameter( + name: RestApiOperation.ServerUrlArgumentName, + type: "string", + isRequired: false, + expand: false, + RestApiOperationParameterLocation.Path, + RestApiOperationParameterStyle.Simple, + defaultValue: serverUrlString) + }; + + //Add payload parameters + if (operation.Method == HttpMethod.Put || operation.Method == HttpMethod.Post) + { + parameters.AddRange(GetPayloadParameters(operation, addPayloadParamsFromMetadata, enablePayloadNamespacing)); + } + + // Create a property alternative name without special symbols that are not supported by SK template language. + foreach (var parameter in parameters) + { + parameter.AlternativeName = s_invalidSymbolsRegex.Replace(parameter.Name, "_"); + } + + return parameters; + } + + /// + /// Retrieves the payload parameters for a given REST API operation. + /// + /// The REST API operation to retrieve parameters for. + /// Flag indicating whether to include parameters from metadata. + /// If false or not specified, the 'payload' and 'content-type' parameters are added instead. + /// Flag indicating whether to namespace payload parameter names. + /// A list of representing the payload parameters. + private static List GetPayloadParameters(RestApiOperation operation, bool useParametersFromMetadata, bool enableNamespacing) + { + if (useParametersFromMetadata) + { + if (operation.Payload is null) + { + throw new SKException($"Payload parameters cannot be retrieved from the '{operation.Id}' operation payload metadata because it is missing."); + } + + // The 'text/plain' content type payload metadata does not contain parameter names. + // So, returning artificial 'payload' parameter instead. + if (operation.Payload.MediaType == MediaTypeTextPlain) + { + return new List { CreatePayloadArtificialParameter(operation) }; + } + + return GetParametersFromPayloadMetadata(operation.Payload.Properties, enableNamespacing); + } + + // Adding artificial 'payload' and 'content-type' in case parameters from payload metadata are not required. + return new List { + CreatePayloadArtificialParameter(operation), + CreateContentTypeArtificialParameter(operation) + }; + } + + /// + /// Creates the 'content-type' artificial parameter for a REST API operation. + /// + /// The REST API operation. + /// The 'content-type' artificial parameter. + private static RestApiOperationParameter CreateContentTypeArtificialParameter(RestApiOperation operation) + { + return new RestApiOperationParameter( + RestApiOperation.ContentTypeArgumentName, + "string", + isRequired: false, + expand: false, + RestApiOperationParameterLocation.Body, + RestApiOperationParameterStyle.Simple, + description: "Content type of REST API request body."); + } + + /// + /// Creates the 'payload' artificial parameter for a REST API operation. + /// + /// The REST API operation. + /// The 'payload' artificial parameter. + private static RestApiOperationParameter CreatePayloadArtificialParameter(RestApiOperation operation) + { + return new RestApiOperationParameter( + RestApiOperation.PayloadArgumentName, + operation.Payload?.MediaType == MediaTypeTextPlain ? "string" : "object", + isRequired: true, + expand: false, + RestApiOperationParameterLocation.Body, + RestApiOperationParameterStyle.Simple, + description: operation.Payload?.Description ?? "REST API request body."); + } + + /// + /// Retrieves parameters from REST API operation payload metadata. + /// + /// The REST API operation payload properties. + /// Determines whether property names are augmented with namespaces. + /// Namespaces are created by prefixing property names with their root property names. + /// + /// The root property name. + /// The list of payload parameters. + private static List GetParametersFromPayloadMetadata(IList properties, bool enableNamespacing = false, string? rootPropertyName = null) + { + var parameters = new List(); + + foreach (var property in properties) + { + var parameterName = GetPropertyName(property, rootPropertyName, enableNamespacing); + + if (!property.Properties.Any()) + { + parameters.Add(new RestApiOperationParameter( + parameterName, + property.Type, + property.IsRequired, + expand: false, + RestApiOperationParameterLocation.Body, + RestApiOperationParameterStyle.Simple, + description: property.Description)); + } + + parameters.AddRange(GetParametersFromPayloadMetadata(property.Properties, enableNamespacing, parameterName)); + } + + return parameters; + } + + /// + /// Gets the property name based on the provided parameters. + /// + /// The property. + /// The root property name to be used for constructing the full property name. + /// Determines whether to add namespace to property name or not. + /// The property name. + private static string GetPropertyName(RestApiOperationPayloadProperty property, string? rootPropertyName, bool enableNamespacing = false) + { + if (enableNamespacing) + { + return string.IsNullOrEmpty(rootPropertyName) ? property.Name : $"{rootPropertyName}.{property.Name}"; + } + + return property.Name; + } + + private const string MediaTypeTextPlain = "text/plain"; + private static readonly Regex s_invalidSymbolsRegex = new("[^0-9A-Za-z_]+"); +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Functions.OpenAPI.csproj b/dotnet/src/Functions/Functions.OpenAPI/Functions.OpenAPI.csproj new file mode 100644 index 000000000000..fc8b5511a03c --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Functions.OpenAPI.csproj @@ -0,0 +1,41 @@ + + + + + Microsoft.SemanticKernel.Functions.OpenAPI + $(AssemblyName) + netstandard2.0 + + + + + + + + Semantic Kernel - OpenAPI Functions + Semantic Kernel OpenAPI Functions + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/src/Functions/Functions.OpenAPI/HttpContentFactory.cs b/dotnet/src/Functions/Functions.OpenAPI/HttpContentFactory.cs new file mode 100644 index 000000000000..633477bd2743 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/HttpContentFactory.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Net.Http; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI; + +/// +/// Represents a delegate for creating HTTP content for a REST API operation. +/// +/// The operation payload metadata. +/// The operation arguments. +/// The HTTP content representing the operation payload. +internal delegate HttpContent HttpContentFactory(RestApiOperationPayload? payload, IDictionary arguments); diff --git a/dotnet/src/Functions/Functions.OpenAPI/HttpResponseContentSerializer.cs b/dotnet/src/Functions/Functions.OpenAPI/HttpResponseContentSerializer.cs new file mode 100644 index 000000000000..2ea3c5f49ecd --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/HttpResponseContentSerializer.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using System.Threading.Tasks; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI; + +/// +/// Represents a delegate for serializing REST API operation response content. +/// +/// The operation response content. +/// The serialized HTTP response content. +internal delegate Task HttpResponseContentSerializer(HttpContent content); diff --git a/dotnet/src/Functions/Functions.OpenAPI/JsonPathPlugin.cs b/dotnet/src/Functions/Functions.OpenAPI/JsonPathPlugin.cs new file mode 100644 index 000000000000..1f45e283837b --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/JsonPathPlugin.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Linq; +using Microsoft.SemanticKernel.Orchestration; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI; + +/// +/// Provides methods to retrieve JSON elements from a JSON string using JsonPath queries. +/// +public sealed class JsonPathPlugin +{ + /// + /// parameter names. + /// + public static class Parameters + { + /// + /// JSON path. + /// + public const string JsonPath = "jsonpath"; + } + + /// + /// Retrieve the value of a JSON element from a JSON string using a JsonPath query. + /// + /// The JSON string to query. + /// The JsonPath query to use. + /// The value of the JSON element as a string. + /// Thrown when the provided JSON string is null or whitespace. + [SKFunction, Description("Retrieve the value of a JSON element from a JSON string using a JsonPath query.")] + public string GetJsonElementValue( + [Description("JSON string")] string json, + [Description("JSON path query.")] string jsonPath) + { + if (string.IsNullOrWhiteSpace(json)) + { + throw new ArgumentException("Variable was null or whitespace", nameof(json)); + } + + JObject jsonObject = JObject.Parse(json); + + JToken? token = jsonObject.SelectToken(jsonPath); + + return token?.Value() ?? string.Empty; + } + + /// + /// Retrieve a collection of JSON elements from a JSON string using a JsonPath query. + /// + /// The JSON string to query. + /// The JsonPath query to use. + /// A JSON string representing the collection of JSON elements. + /// Thrown when the provided JSON string is null or whitespace. + [SKFunction, Description("Retrieve a collection of JSON elements from a JSON string using a JsonPath query.")] + public string GetJsonElements( + [Description("JSON string")] string json, + [Description("JSON path query.")] string jsonPath) + { + if (string.IsNullOrWhiteSpace(json)) + { + throw new ArgumentException("Variable was null or whitespace", nameof(json)); + } + + JObject jsonObject = JObject.Parse(json); + + JToken[] tokens = jsonObject.SelectTokens(jsonPath).ToArray(); + + return JsonConvert.SerializeObject(tokens, Formatting.None); + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperation.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperation.cs new file mode 100644 index 000000000000..7c7d50b03bab --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperation.cs @@ -0,0 +1,243 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using Microsoft.SemanticKernel.Diagnostics; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +/// +/// The REST API operation. +/// +public sealed class RestApiOperation +{ + /// + /// An artificial parameter that is added to be able to override REST API operation server url. + /// + public const string ServerUrlArgumentName = "server-url"; + + /// + /// An artificial parameter to be used for operation having "text/plain" payload media type. + /// + public const string PayloadArgumentName = "payload"; + + /// + /// An artificial parameter to be used for indicate payload media-type if it's missing in payload metadata. + /// + public const string ContentTypeArgumentName = "content-type"; + + /// + /// The operation identifier. + /// + public string Id { get; } + + /// + /// The operation description. + /// + public string Description { get; } + + /// + /// The operation path. + /// + public string Path { get; } + + /// + /// The operation method - GET, POST, PUT, DELETE. + /// + public HttpMethod Method { get; } + + /// + /// The server URL. + /// + public Uri? ServerUrl { get; } + + /// + /// The operation headers. + /// + public IDictionary Headers { get; } + + /// + /// The operation parameters. + /// + public IList Parameters { get; } + + /// + /// The operation payload. + /// + public RestApiOperationPayload? Payload { get; } + + /// + /// Creates an instance of a class. + /// + /// The operation identifier. + /// The server URL. + /// The operation path. + /// The operation method. + /// The operation description. + /// The operation parameters. + /// The operation headers. + /// The operation payload. + public RestApiOperation( + string id, + Uri? serverUrl, + string path, + HttpMethod method, + string description, + IList parameters, + IDictionary headers, + RestApiOperationPayload? payload = null) + { + this.Id = id; + this.ServerUrl = serverUrl; + this.Path = path; + this.Method = method; + this.Description = description; + this.Parameters = parameters; + this.Headers = headers; + this.Payload = payload; + } + + /// + /// Builds operation Url. + /// + /// The operation arguments. + /// Override for REST API operation server url. + /// The URL of REST API host. + /// The operation Url. + public Uri BuildOperationUrl(IDictionary arguments, Uri? serverUrlOverride = null, Uri? apiHostUrl = null) + { + var serverUrl = this.GetServerUrl(arguments, serverUrlOverride, apiHostUrl); + + var path = this.ReplacePathParameters(this.Path, arguments); + + return new Uri(serverUrl, $"{path.TrimStart('/')}"); + } + + /// + /// Renders operation request headers. + /// + /// The operation arguments. + /// The rendered request headers. + public IDictionary RenderHeaders(IDictionary arguments) + { + var headers = new Dictionary(); + + foreach (var header in this.Headers) + { + var headerName = header.Key; + var headerValue = header.Value; + + //A try to resolve header value in arguments. + if (arguments.TryGetValue(headerName, out var value)) + { + headers.Add(headerName, value); + continue; + } + + //Header value is already supplied. + if (!string.IsNullOrEmpty(headerValue)) + { + headers.Add(headerName, headerValue); + continue; + } + + //Getting metadata for the header + var headerMetadata = this.Parameters.FirstOrDefault(p => p.Location == RestApiOperationParameterLocation.Header && p.Name == headerName) + ?? throw new SKException($"No value for the '{headerName} header is found.'"); + + //If parameter is required it's value should always be provided. + if (headerMetadata.IsRequired) + { + throw new SKException($"No value for the '{headerName} header is found.'"); + } + + //Parameter is not required and no default value provided. + if (string.IsNullOrEmpty(headerMetadata.DefaultValue)) + { + continue; + } + + //Using default value. + headers.Add(headerName, headerMetadata.DefaultValue!); + } + + return headers; + } + + #region private + + /// + /// Replaces path parameters by corresponding arguments. + /// + /// Operation path to replace parameters in. + /// Arguments to replace parameters by. + /// Path with replaced parameters + private string ReplacePathParameters(string path, IDictionary arguments) + { + string ReplaceParameter(Match match) + { + var parameterName = match.Groups[1].Value; + + //A try to find parameter value in arguments + if (arguments.TryGetValue(parameterName, out var value)) + { + return value; + } + + //A try to find default value for the parameter + var parameterMetadata = this.Parameters.First(p => p.Location == RestApiOperationParameterLocation.Path && p.Name == parameterName); + if (parameterMetadata?.DefaultValue == null) + { + throw new SKException($"No argument found for parameter - '{parameterName}' for operation - '{this.Id}'"); + } + + return parameterMetadata.DefaultValue; + } + + return s_urlParameterMatch.Replace(path, ReplaceParameter); + } + + /// + /// Returns operation server Url. + /// + /// The operation arguments. + /// Override for REST API operation server url. + /// The URL of REST API host. + /// The operation server url. + private Uri GetServerUrl(IDictionary arguments, Uri? serverUrlOverride, Uri? apiHostUrl) + { + string serverUrlString; + + if (serverUrlOverride is not null) + { + serverUrlString = serverUrlOverride.AbsoluteUri; + } + else if (arguments.TryGetValue(ServerUrlArgumentName, out string serverUrlFromArgument)) + { + // Override defined server url - https://api.example.com/v1 by the one from arguments. + serverUrlString = serverUrlFromArgument; + } + else + { + serverUrlString = + this.ServerUrl?.AbsoluteUri ?? + apiHostUrl?.AbsoluteUri ?? + throw new InvalidOperationException($"Server url is not defined for operation {this.Id}"); + } + + // make sure base url ends with trailing slash + if (!serverUrlString.EndsWith("/", StringComparison.OrdinalIgnoreCase)) + { + serverUrlString += "/"; + } + + return new Uri(serverUrlString); + } + + private static readonly Regex s_urlParameterMatch = new(@"\{([\w-]+)\}"); + + # endregion +} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameter.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameter.cs similarity index 85% rename from dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameter.cs rename to dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameter.cs index 1f303000c6b9..ec10244971cf 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameter.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameter.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; /// /// The REST API operation parameter. @@ -52,12 +52,18 @@ public sealed class RestApiOperationParameter /// public string? DefaultValue { get; } + /// + /// Specifies whether arrays and objects should generate separate parameters for each array item or object property. + /// + public bool Expand { get; } + /// /// Creates an instance of a class. /// /// The parameter name. /// The parameter type. /// Flag specifying if the parameter is required or not. + /// Specifies whether arrays and objects should generate separate parameters for each array item or object property. /// The parameter location. /// The parameter style - defines how multiple values are delimited. /// Type of array item for parameters of "array" type. @@ -67,6 +73,7 @@ public RestApiOperationParameter( string name, string type, bool isRequired, + bool expand, RestApiOperationParameterLocation location, RestApiOperationParameterStyle? style = null, string? arrayItemType = null, @@ -76,6 +83,7 @@ public RestApiOperationParameter( this.Name = name; this.Type = type; this.IsRequired = isRequired; + this.Expand = expand; this.Location = location; this.Style = style; this.ArrayItemType = arrayItemType; diff --git a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameterLocation.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameterLocation.cs similarity index 90% rename from dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameterLocation.cs rename to dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameterLocation.cs index a065bfd33d17..47bc58ee7035 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameterLocation.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameterLocation.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; /// /// The REST API operation parameter location. diff --git a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameterStyle.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameterStyle.cs similarity index 93% rename from dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameterStyle.cs rename to dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameterStyle.cs index 13d1b55bf1b5..fd223154c0e1 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationParameterStyle.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationParameterStyle.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; /// /// The REST API operation parameter style. diff --git a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationPayload.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationPayload.cs similarity index 94% rename from dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationPayload.cs rename to dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationPayload.cs index a55c48dd0d3b..db19565be333 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationPayload.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationPayload.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; /// /// The REST API operation payload. diff --git a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationPayloadProperty.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationPayloadProperty.cs similarity index 96% rename from dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationPayloadProperty.cs rename to dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationPayloadProperty.cs index 9e135cc6cbc9..258b9a598c3a 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationPayloadProperty.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationPayloadProperty.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; /// /// The REST API operation payload property. diff --git a/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationResponse.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationResponse.cs new file mode 100644 index 000000000000..e2c4776c2c03 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationResponse.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +/// +/// The REST API operation response. +/// +[TypeConverterAttribute(typeof(RestApiOperationResponseConverter))] +public sealed class RestApiOperationResponse +{ + /// + /// Gets the content of the response. + /// + public object Content { get; } + + /// + /// Gets the content type of the response. + /// + public string ContentType { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The content of the response. + /// The content type of the response. + public RestApiOperationResponse(object content, string contentType) + { + this.Content = content; + this.ContentType = contentType; + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationResponseConverter.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationResponseConverter.cs new file mode 100644 index 000000000000..23ef33812bc0 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationResponseConverter.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Globalization; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +/// +/// Converts a object of type to string type. +/// +public class RestApiOperationResponseConverter : TypeConverter +{ + /// + public override bool CanConvertTo(ITypeDescriptorContext context, Type destinationType) + { + return destinationType == typeof(string) || base.CanConvertTo(context, destinationType); + } + + /// + public override object ConvertTo(ITypeDescriptorContext context, CultureInfo culture, object value, Type destinationType) + { + // Convert object content to a string based on the type of the `Content` property. + // More granular conversion logic can be built based on the value of the `ContentType` property, if needed. + if (value is RestApiOperationResponse response && destinationType == typeof(string)) + { + //Case for "text/*", "application/json", "application/xml" content types. + if (response.Content is string stringContent) + { + return stringContent; + } + + //Case for "image/*" content types and others that are serialized as bytes. + if (response.Content is byte[] byteContent) + { + return Convert.ToBase64String(byteContent); + } + } + + return base.ConvertTo(context, culture, value, destinationType); + } +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationRunOptions.cs b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationRunOptions.cs new file mode 100644 index 000000000000..765f99e9ae83 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Model/RestApiOperationRunOptions.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +/// +/// Options for REST API operation run. +/// +internal class RestApiOperationRunOptions +{ + /// + /// Override for REST API operation server URL. + /// + public Uri? ServerUrlOverride { get; set; } + + /// + /// The URL of REST API host. + /// + public Uri? ApiHostUrl { get; set; } +} diff --git a/dotnet/src/Skills/Skills.OpenAPI/OpenApi/IOpenApiDocumentParser.cs b/dotnet/src/Functions/Functions.OpenAPI/OpenApi/IOpenApiDocumentParser.cs similarity index 89% rename from dotnet/src/Skills/Skills.OpenAPI/OpenApi/IOpenApiDocumentParser.cs rename to dotnet/src/Functions/Functions.OpenAPI/OpenApi/IOpenApiDocumentParser.cs index 9e182ad2c0b8..90b1fe571b48 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/OpenApi/IOpenApiDocumentParser.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/OpenApi/IOpenApiDocumentParser.cs @@ -4,9 +4,9 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; -namespace Microsoft.SemanticKernel.Skills.OpenAPI.OpenApi; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.OpenApi; /// /// Interface for OpenApi document parser classes. diff --git a/dotnet/src/Skills/Skills.OpenAPI/OpenApi/OpenApiDocumentParser.cs b/dotnet/src/Functions/Functions.OpenAPI/OpenApi/OpenApiDocumentParser.cs similarity index 88% rename from dotnet/src/Skills/Skills.OpenAPI/OpenApi/OpenApiDocumentParser.cs rename to dotnet/src/Functions/Functions.OpenAPI/OpenApi/OpenApiDocumentParser.cs index e9b33b62e1f3..455c5ace0a67 100644 --- a/dotnet/src/Skills/Skills.OpenAPI/OpenApi/OpenApiDocumentParser.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/OpenApi/OpenApiDocumentParser.cs @@ -16,10 +16,11 @@ using Microsoft.OpenApi.Any; using Microsoft.OpenApi.Models; using Microsoft.OpenApi.Readers; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; using Microsoft.SemanticKernel.Text; -namespace Microsoft.SemanticKernel.Skills.OpenAPI.OpenApi; +namespace Microsoft.SemanticKernel.Functions.OpenAPI.OpenApi; /// /// Parser for OpenAPI documents. @@ -29,10 +30,10 @@ internal sealed class OpenApiDocumentParser : IOpenApiDocumentParser /// /// Initializes a new instance of the class. /// - /// Optional logger instance. - public OpenApiDocumentParser(ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + public OpenApiDocumentParser(ILoggerFactory? loggerFactory = null) { - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(OpenApiDocumentParser)) : NullLogger.Instance; } /// @@ -40,7 +41,7 @@ public async Task> ParseAsync(Stream stream, bool ignore { var jsonObject = await this.DowngradeDocumentVersionToSupportedOneAsync(stream, cancellationToken).ConfigureAwait(false); - using var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonObject.ToJson())); + using var memoryStream = new MemoryStream(Json.SerializeToUtf8Bytes(jsonObject)); var result = await this._openApiReader.ReadAsync(memoryStream, cancellationToken).ConfigureAwait(false); @@ -94,7 +95,7 @@ private async Task DowngradeDocumentVersionToSupportedOneAsync(Strea if (jsonObject == null) { // The document is malformed. - throw new OpenApiDocumentParsingException("Parsing of OpenAPI document failed."); + throw new SKException("Parsing of OpenAPI document failed."); } if (!jsonObject.TryGetPropertyValue(OpenApiVersionPropertyName, out var propertyNode)) @@ -151,7 +152,7 @@ private static List ExtractRestApiOperations(OpenApiDocument d { var result = new List(); - var serverUrl = document.Servers.First().Url; + var serverUrl = document.Servers.FirstOrDefault()?.Url; foreach (var pathPair in document.Paths) { @@ -170,7 +171,7 @@ private static List ExtractRestApiOperations(OpenApiDocument d /// Rest resource path. /// Rest resource metadata. /// Rest operation. - private static List CreateRestApiOperations(string serverUrl, string path, OpenApiPathItem pathItem) + private static List CreateRestApiOperations(string? serverUrl, string path, OpenApiPathItem pathItem) { var operations = new List(); @@ -182,10 +183,10 @@ private static List CreateRestApiOperations(string serverUrl, var operation = new RestApiOperation( operationItem.OperationId, - new Uri(serverUrl), + string.IsNullOrEmpty(serverUrl) ? null : new Uri(serverUrl), path, new HttpMethod(method), - operationItem.Description, + string.IsNullOrEmpty(operationItem.Description) ? operationItem.Summary : operationItem.Description, CreateRestApiOperationParameters(operationItem.OperationId, operationItem.Parameters), CreateRestApiOperationHeaders(operationItem.Parameters), CreateRestApiOperationPayload(operationItem.OperationId, operationItem.RequestBody) @@ -211,18 +212,19 @@ private static List CreateRestApiOperationParameters( { if (parameter.In == null) { - throw new OpenApiDocumentParsingException($"Parameter location of {parameter.Name} parameter of {operationId} operation is undefined."); + throw new SKException($"Parameter location of {parameter.Name} parameter of {operationId} operation is undefined."); } if (parameter.Style == null) { - throw new OpenApiDocumentParsingException($"Parameter style of {parameter.Name} parameter of {operationId} operation is undefined."); + throw new SKException($"Parameter style of {parameter.Name} parameter of {operationId} operation is undefined."); } var restParameter = new RestApiOperationParameter( parameter.Name, parameter.Schema.Type, parameter.Required, + parameter.Explode, (RestApiOperationParameterLocation)Enum.Parse(typeof(RestApiOperationParameterLocation), parameter.In.ToString()), (RestApiOperationParameterStyle)Enum.Parse(typeof(RestApiOperationParameterStyle), parameter.Style.ToString()), parameter.Schema.Items?.Type, @@ -262,12 +264,12 @@ private static Dictionary CreateRestApiOperationHeaders(IList requestBody.Content.ContainsKey(smt)); if (mediaType == null) { - throw new OpenApiDocumentParsingException($"Neither of the media types of {operationId} is supported."); + throw new SKException($"Neither of the media types of {operationId} is supported."); } var mediaTypeMetadata = requestBody.Content[mediaType]; - var payloadProperties = GetPayloadProperties(operationId, mediaTypeMetadata.Schema, mediaTypeMetadata.Schema.Required); + var payloadProperties = GetPayloadProperties(operationId, mediaTypeMetadata.Schema, mediaTypeMetadata.Schema?.Required ?? new HashSet()); return new RestApiOperationPayload(mediaType, payloadProperties, requestBody.Description); } @@ -290,7 +292,7 @@ private static List GetPayloadProperties(string if (level > PayloadPropertiesHierarchyMaxDepth) { - throw new OpenApiDocumentParsingException($"Max level {PayloadPropertiesHierarchyMaxDepth} of traversing payload properties of {operationId} operation is exceeded."); + throw new SKException($"Max level {PayloadPropertiesHierarchyMaxDepth} of traversing payload properties of {operationId} operation is exceeded."); } var result = new List(); @@ -306,8 +308,7 @@ private static List GetPayloadProperties(string propertySchema.Type, requiredProperties.Contains(propertyName), GetPayloadProperties(operationId, propertySchema, requiredProperties, level + 1), - propertySchema.Description - ); + propertySchema.Description); result.Add(property); } @@ -375,7 +376,7 @@ private static List GetPayloadProperties(string return passwordValue.Value.ToString(CultureInfo.InvariantCulture); default: - throw new OpenApiDocumentParsingException($"The value type - {value.PrimitiveType} is not supported."); + throw new SKException($"The value type - {value.PrimitiveType} is not supported."); } } @@ -397,7 +398,7 @@ private void AssertReadingSuccessful(ReadResult readResult, bool ignoreNonCompli if (!ignoreNonCompliantErrors) { - throw new OpenApiDocumentParsingException(message); + throw new SKException(message); } } } diff --git a/dotnet/src/Skills/Skills.OpenAPI/Skills/AzureKeyVaultSkill/openapi.json b/dotnet/src/Functions/Functions.OpenAPI/Plugins/AzureKeyVaultPlugin/openapi.json similarity index 100% rename from dotnet/src/Skills/Skills.OpenAPI/Skills/AzureKeyVaultSkill/openapi.json rename to dotnet/src/Functions/Functions.OpenAPI/Plugins/AzureKeyVaultPlugin/openapi.json diff --git a/dotnet/src/Functions/Functions.OpenAPI/Plugins/PluginResourceNames.cs b/dotnet/src/Functions/Functions.OpenAPI/Plugins/PluginResourceNames.cs new file mode 100644 index 000000000000..d02c38675198 --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/Plugins/PluginResourceNames.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Functions.OpenAPI.Plugins; + +/// +/// Plugin resource names. +/// +public static class PluginResourceNames +{ + /// + /// Azure KeyVault plugin name. + /// + public const string AzureKeyVault = "AzureKeyVaultPlugin"; +} diff --git a/dotnet/src/Functions/Functions.OpenAPI/RestApiOperationRunner.cs b/dotnet/src/Functions/Functions.OpenAPI/RestApiOperationRunner.cs new file mode 100644 index 000000000000..71acaaa0c00a --- /dev/null +++ b/dotnet/src/Functions/Functions.OpenAPI/RestApiOperationRunner.cs @@ -0,0 +1,416 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; +using Microsoft.SemanticKernel.Functions.OpenAPI.Builders; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; + +namespace Microsoft.SemanticKernel.Functions.OpenAPI; + +/// +/// Runs REST API operation represented by RestApiOperation model class. +/// +internal sealed class RestApiOperationRunner +{ + private const string MediaTypeApplicationJson = "application/json"; + private const string MediaTypeTextPlain = "text/plain"; + + /// + /// List of payload builders/factories. + /// + private readonly Dictionary _payloadFactoryByMediaType; + + /// + /// A dictionary containing the content type as the key and the corresponding content serializer as the value. + /// + private static readonly Dictionary s_serializerByContentType = new() + { + { "image", async (content) => await content.ReadAsByteArrayAndTranslateExceptionAsync().ConfigureAwait(false) }, + { "text", async (content) => await content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false) }, + { "application/json", async (content) => await content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false)}, + { "application/xml", async (content) => await content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false)} + }; + + /// + /// An instance of the HttpClient class. + /// + private readonly HttpClient _httpClient; + + /// + /// Delegate for authorizing the HTTP request. + /// + private readonly AuthenticateRequestAsyncCallback _authCallback; + + /// + /// Request-header field containing information about the user agent originating the request + /// + private readonly string? _userAgent; + + /// + /// Determines whether the operation payload is constructed dynamically based on operation payload metadata. + /// If false, the operation payload must be provided via the 'payload' property. + /// + private readonly bool _enableDynamicPayload; + + /// + /// Determines whether payload parameters are resolved from the arguments by + /// full name (parameter name prefixed with the parent property name). + /// + private readonly bool _enablePayloadNamespacing; + + /// + /// Creates an instance of the class. + /// + /// An instance of the HttpClient class. + /// Optional callback for adding auth data to the API requests. + /// Optional request-header field containing information about the user agent originating the request. + /// Determines whether the operation payload is constructed dynamically based on operation payload metadata. + /// If false, the operation payload must be provided via the 'payload' property. + /// + /// Determines whether payload parameters are resolved from the arguments by + /// full name (parameter name prefixed with the parent property name). + public RestApiOperationRunner( + HttpClient httpClient, + AuthenticateRequestAsyncCallback? authCallback = null, + string? userAgent = null, + bool enableDynamicPayload = false, + bool enablePayloadNamespacing = false) + { + this._httpClient = httpClient; + this._userAgent = userAgent ?? Telemetry.HttpUserAgent; + this._enableDynamicPayload = enableDynamicPayload; + this._enablePayloadNamespacing = enablePayloadNamespacing; + + // If no auth callback provided, use empty function + if (authCallback is null) + { + this._authCallback = _ => Task.CompletedTask; + } + else + { + this._authCallback = authCallback; + } + + this._payloadFactoryByMediaType = new() + { + { MediaTypeApplicationJson, this.BuildJsonPayload }, + { MediaTypeTextPlain, this.BuildPlainTextPayload } + }; + } + + /// + /// Executes the specified asynchronously, using the provided . + /// + /// The REST API operation to execute. + /// The dictionary of arguments to be passed to the operation. + /// Options for REST API operation run. + /// The cancellation token. + /// The task execution result. + public Task RunAsync( + RestApiOperation operation, + IDictionary arguments, + RestApiOperationRunOptions? options = null, + CancellationToken cancellationToken = default) + { + var url = this.BuildsOperationUrl(operation, arguments, options?.ServerUrlOverride, options?.ApiHostUrl); + + var headers = operation.RenderHeaders(arguments); + + var payload = this.BuildOperationPayload(operation, arguments); + + return this.SendAsync(url, operation.Method, headers, payload, cancellationToken); + } + + #region private + + /// + /// Sends an HTTP request. + /// + /// The url to send request to. + /// The HTTP request method. + /// Headers to include into the HTTP request. + /// HTTP request payload. + /// The cancellation token. + /// Response content and content type + private async Task SendAsync( + Uri url, + HttpMethod method, + IDictionary? headers = null, + HttpContent? payload = null, + CancellationToken cancellationToken = default) + { + using var requestMessage = new HttpRequestMessage(method, url); + + await this._authCallback(requestMessage).ConfigureAwait(false); + + if (payload != null) + { + requestMessage.Content = payload; + } + + requestMessage.Headers.Add("User-Agent", !string.IsNullOrWhiteSpace(this._userAgent) + ? this._userAgent + : Telemetry.HttpUserAgent); + + if (headers != null) + { + foreach (var header in headers) + { + requestMessage.Headers.Add(header.Key, header.Value); + } + } + + using var responseMessage = await this._httpClient.SendWithSuccessCheckAsync(requestMessage, cancellationToken).ConfigureAwait(false); + + return await SerializeResponseContentAsync(responseMessage.Content).ConfigureAwait(false); + } + + /// + /// Serializes the response content of an HTTP request. + /// + /// The HttpContent object containing the response content to be serialized. + /// The serialized content. + private static async Task SerializeResponseContentAsync(HttpContent content) + { + var contentType = content.Headers.ContentType; + + var mediaType = contentType.MediaType; + + // Obtain the content serializer by media type (e.g., text/plain, application/json, image/jpg) + if (!s_serializerByContentType.TryGetValue(mediaType, out var serializer)) + { + // Split the media type into a primary-type and a sub-type + var mediaTypeParts = mediaType.Split('/'); + if (mediaTypeParts.Length != 2) + { + throw new SKException($"The string `{mediaType}` is not a valid media type."); + } + + var primaryMediaType = mediaTypeParts.First(); + + // Try to obtain the content serializer by the primary type (e.g., text, application, image) + if (!s_serializerByContentType.TryGetValue(primaryMediaType, out serializer)) + { + throw new SKException($"The content type `{mediaType}` is not supported."); + } + } + + // Serialize response content and return it + var serializedContent = await serializer.Invoke(content).ConfigureAwait(false); + + return new RestApiOperationResponse(serializedContent, contentType.ToString()); + } + + /// + /// Builds operation payload. + /// + /// The operation. + /// The payload arguments. + /// The HttpContent representing the payload. + private HttpContent? BuildOperationPayload(RestApiOperation operation, IDictionary arguments) + { + if (operation?.Method != HttpMethod.Put && operation?.Method != HttpMethod.Post) + { + return null; + } + + var mediaType = operation.Payload?.MediaType; + + // A try to resolve payload content type from the operation arguments if it's missing in the payload metadata. + if (string.IsNullOrEmpty(mediaType)) + { + if (!arguments.TryGetValue(RestApiOperation.ContentTypeArgumentName, out mediaType)) + { + throw new SKException($"No content type is provided for the {operation.Id} operation."); + } + } + + if (!this._payloadFactoryByMediaType.TryGetValue(mediaType!, out var payloadFactory)) + { + throw new SKException($"The media type {mediaType} of the {operation.Id} operation is not supported by {nameof(RestApiOperationRunner)}."); + } + + return payloadFactory.Invoke(operation.Payload, arguments); + } + + /// + /// Builds "application/json" payload. + /// + /// The payload meta-data. + /// The payload arguments. + /// The HttpContent representing the payload. + private HttpContent BuildJsonPayload(RestApiOperationPayload? payloadMetadata, IDictionary arguments) + { + //Build operation payload dynamically + if (this._enableDynamicPayload) + { + if (payloadMetadata == null) + { + throw new SKException("Payload can't be built dynamically due to the missing payload metadata."); + } + + var payload = this.BuildJsonObject(payloadMetadata.Properties, arguments); + + return new StringContent(payload.ToJsonString(), Encoding.UTF8, MediaTypeApplicationJson); + } + + //Get operation payload content from the 'payload' argument if dynamic payload building is not required. + if (!arguments.TryGetValue(RestApiOperation.PayloadArgumentName, out var content)) + { + throw new SKException($"No argument is found for the '{RestApiOperation.PayloadArgumentName}' payload content."); + } + + return new StringContent(content, Encoding.UTF8, MediaTypeApplicationJson); + } + + /// + /// Builds a JSON object from a list of RestAPI operation payload properties. + /// + /// The properties. + /// The arguments. + /// The namespace to add to the property name. + /// The JSON object. + private JsonObject BuildJsonObject(IList properties, IDictionary arguments, string? propertyNamespace = null) + { + var result = new JsonObject(); + + foreach (var propertyMetadata in properties) + { + var argumentName = this.GetArgumentNameForPayload(propertyMetadata.Name, propertyNamespace); + + if (propertyMetadata.Type == "object") + { + var node = this.BuildJsonObject(propertyMetadata.Properties, arguments, argumentName); + result.Add(propertyMetadata.Name, node); + continue; + } + + if (arguments.TryGetValue(argumentName, out var propertyValue)) + { + result.Add(propertyMetadata.Name, ConvertJsonPropertyValueType(propertyValue, propertyMetadata)); + continue; + } + + if (propertyMetadata.IsRequired) + { + throw new SKException($"No argument is found for the '{propertyMetadata.Name}' payload property."); + } + } + + return result; + } + + /// + /// Converts the JSON property value to the REST API type specified in metadata. + /// + /// The value of the property to be converted. + /// The metadata of the property. + /// A JsonNode representing the converted property value. + private static JsonNode? ConvertJsonPropertyValueType(string propertyValue, RestApiOperationPayloadProperty propertyMetadata) + { + switch (propertyMetadata.Type) + { + case "number": + { + if (long.TryParse(propertyValue, out var intValue)) + { + return JsonValue.Create(intValue); + } + + return JsonValue.Create(double.Parse(propertyValue, CultureInfo.InvariantCulture)); + } + + case "boolean": + { + return JsonValue.Create(bool.Parse(propertyValue)); + } + + case "integer": + { + return JsonValue.Create(int.Parse(propertyValue, CultureInfo.InvariantCulture)); + } + + case "array": + { + if (JsonArray.Parse(propertyValue) is JsonArray array) + { + return array; + } + + throw new SKException($"Can't convert OpenAPI property - {propertyMetadata.Name} value - {propertyValue} of 'array' type to JSON array."); + } + + case "string": + { + return JsonValue.Create(propertyValue); + } + + default: + { + throw new SKException($"Unexpected OpenAPI data type - {propertyMetadata.Type}"); + } + } + } + + /// + /// Builds "text/plain" payload. + /// + /// The payload meta-data. + /// The payload arguments. + /// The HttpContent representing the payload. + private HttpContent BuildPlainTextPayload(RestApiOperationPayload? payloadMetadata, IDictionary arguments) + { + if (!arguments.TryGetValue(RestApiOperation.PayloadArgumentName, out var propertyValue)) + { + throw new SKException($"No argument is found for the '{RestApiOperation.PayloadArgumentName}' payload content."); + } + + return new StringContent(propertyValue, Encoding.UTF8, MediaTypeTextPlain); + } + + /// + /// Retrieves the argument name for a payload property. + /// + /// The name of the property. + /// The namespace to add to the property name (optional). + /// The argument name for the payload property. + private string GetArgumentNameForPayload(string propertyName, string? propertyNamespace) + { + if (!this._enablePayloadNamespacing) + { + return propertyName; + } + + return string.IsNullOrEmpty(propertyNamespace) ? propertyName : $"{propertyNamespace}.{propertyName}"; + } + + /// + /// Builds operation Url. + /// + /// The REST API operation. + /// The operation arguments. + /// Override for REST API operation server url. + /// The URL of REST API host. + /// The operation Url. + private Uri BuildsOperationUrl(RestApiOperation operation, IDictionary arguments, Uri? serverUrlOverride = null, Uri? apiHostUrl = null) + { + var url = operation.BuildOperationUrl(arguments, serverUrlOverride, apiHostUrl); + + var urlBuilder = new UriBuilder(url); + + urlBuilder.Query = operation.BuildQueryString(arguments); + + return urlBuilder.Uri; + } + + #endregion +} diff --git a/dotnet/src/Functions/Functions.UnitTests/.editorconfig b/dotnet/src/Functions/Functions.UnitTests/.editorconfig new file mode 100644 index 000000000000..394eef685f21 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/Functions/Functions.UnitTests/Functions.UnitTests.csproj b/dotnet/src/Functions/Functions.UnitTests/Functions.UnitTests.csproj new file mode 100644 index 000000000000..cc3d8351e03c --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/Functions.UnitTests.csproj @@ -0,0 +1,50 @@ + + + + SemanticKernel.Functions.UnitTests + SemanticKernel.Functions.UnitTests + net6.0 + LatestMajor + true + enable + disable + false + CA2007,VSTHRD111 + + + + + + + + + + + + + + + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/dotnet/src/Skills/Skills.UnitTests/Grpc/Extensions/GrpcOperationExtensionsTests.cs b/dotnet/src/Functions/Functions.UnitTests/Grpc/Extensions/GrpcOperationExtensionsTests.cs similarity index 90% rename from dotnet/src/Skills/Skills.UnitTests/Grpc/Extensions/GrpcOperationExtensionsTests.cs rename to dotnet/src/Functions/Functions.UnitTests/Grpc/Extensions/GrpcOperationExtensionsTests.cs index f166f7683241..02cc8ae63edd 100644 --- a/dotnet/src/Skills/Skills.UnitTests/Grpc/Extensions/GrpcOperationExtensionsTests.cs +++ b/dotnet/src/Functions/Functions.UnitTests/Grpc/Extensions/GrpcOperationExtensionsTests.cs @@ -2,11 +2,11 @@ using System.Collections.Generic; using System.Linq; -using Microsoft.SemanticKernel.Skills.Grpc.Extensions; -using Microsoft.SemanticKernel.Skills.Grpc.Model; +using Microsoft.SemanticKernel.Functions.Grpc.Extensions; +using Microsoft.SemanticKernel.Functions.Grpc.Model; using Xunit; -namespace SemanticKernel.Skills.UnitTests.Grpc.Extensions; +namespace SemanticKernel.Functions.UnitTests.Grpc.Extensions; public class GrpcOperationExtensionsTests { diff --git a/dotnet/src/Skills/Skills.UnitTests/Grpc/GrpcRunnerTests.cs b/dotnet/src/Functions/Functions.UnitTests/Grpc/GrpcRunnerTests.cs similarity index 98% rename from dotnet/src/Skills/Skills.UnitTests/Grpc/GrpcRunnerTests.cs rename to dotnet/src/Functions/Functions.UnitTests/Grpc/GrpcRunnerTests.cs index 8b4d5c7d0347..9d8f3d80c6b7 100644 --- a/dotnet/src/Skills/Skills.UnitTests/Grpc/GrpcRunnerTests.cs +++ b/dotnet/src/Functions/Functions.UnitTests/Grpc/GrpcRunnerTests.cs @@ -10,11 +10,11 @@ using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.Grpc; -using Microsoft.SemanticKernel.Skills.Grpc.Model; +using Microsoft.SemanticKernel.Functions.Grpc; +using Microsoft.SemanticKernel.Functions.Grpc.Model; using Xunit; -namespace SemanticKernel.Skills.UnitTests.Grpc; +namespace SemanticKernel.Functions.UnitTests.Grpc; public sealed class GrpcRunnerTests : IDisposable { diff --git a/dotnet/src/Skills/Skills.UnitTests/Grpc/Protobuf/ProtoDocumentParserV30Tests.cs b/dotnet/src/Functions/Functions.UnitTests/Grpc/Protobuf/ProtoDocumentParserV30Tests.cs similarity index 92% rename from dotnet/src/Skills/Skills.UnitTests/Grpc/Protobuf/ProtoDocumentParserV30Tests.cs rename to dotnet/src/Functions/Functions.UnitTests/Grpc/Protobuf/ProtoDocumentParserV30Tests.cs index 94682f0c07bc..838fe275ed53 100644 --- a/dotnet/src/Skills/Skills.UnitTests/Grpc/Protobuf/ProtoDocumentParserV30Tests.cs +++ b/dotnet/src/Functions/Functions.UnitTests/Grpc/Protobuf/ProtoDocumentParserV30Tests.cs @@ -2,11 +2,11 @@ using System.IO; using System.Linq; -using Microsoft.SemanticKernel.Skills.Grpc.Protobuf; -using SemanticKernel.Skills.UnitTests.Grpc.Protobuf.TestSkills; +using Microsoft.SemanticKernel.Functions.Grpc.Protobuf; +using SemanticKernel.Functions.UnitTests.Grpc.Protobuf.TestPlugins; using Xunit; -namespace SemanticKernel.Skills.UnitTests.Grpc.Protobuf; +namespace SemanticKernel.Functions.UnitTests.Grpc.Protobuf; public sealed class ProtoDocumentParserV30Tests { @@ -25,7 +25,7 @@ public sealed class ProtoDocumentParserV30Tests /// public ProtoDocumentParserV30Tests() { - this._protoDocument = ResourceSkillsProvider.LoadFromResource("protoV3.proto"); + this._protoDocument = ResourcePluginsProvider.LoadFromResource("protoV3.proto"); this._sut = new ProtoDocumentParser(); } diff --git a/dotnet/src/Functions/Functions.UnitTests/Grpc/Protobuf/TestPlugins/ResourcePluginsProvider.cs b/dotnet/src/Functions/Functions.UnitTests/Grpc/Protobuf/TestPlugins/ResourcePluginsProvider.cs new file mode 100644 index 000000000000..b2405df9f7e5 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/Grpc/Protobuf/TestPlugins/ResourcePluginsProvider.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.IO; +using System.Resources; + +namespace SemanticKernel.Functions.UnitTests.Grpc.Protobuf.TestPlugins; + +internal static class ResourcePluginsProvider +{ + /// + /// Loads .proto file from assembly resource. + /// + /// The resource name. + /// The OpenApi document resource stream. + public static Stream LoadFromResource(string resourceName) + { + var type = typeof(ResourcePluginsProvider); + + var stream = type.Assembly.GetManifestResourceStream(type, resourceName); + if (stream == null) + { + throw new MissingManifestResourceException($"Unable to load gRPC plugin from assembly resource '{resourceName}'."); + } + + return stream; + } +} diff --git a/dotnet/src/Skills/Skills.UnitTests/Grpc/Protobuf/TestSkills/protoV3.proto b/dotnet/src/Functions/Functions.UnitTests/Grpc/Protobuf/TestPlugins/protoV3.proto similarity index 100% rename from dotnet/src/Skills/Skills.UnitTests/Grpc/Protobuf/TestSkills/protoV3.proto rename to dotnet/src/Functions/Functions.UnitTests/Grpc/Protobuf/TestPlugins/protoV3.proto diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/Authentication/BasicAuthenticationProviderTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/BasicAuthenticationProviderTests.cs similarity index 85% rename from dotnet/src/Skills/Skills.UnitTests/OpenAPI/Authentication/BasicAuthenticationProviderTests.cs rename to dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/BasicAuthenticationProviderTests.cs index 6a24cd3fbb3d..a06830633a77 100644 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/Authentication/BasicAuthenticationProviderTests.cs +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/BasicAuthenticationProviderTests.cs @@ -4,10 +4,10 @@ using System.Net.Http; using System.Text; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; using Xunit; -namespace SemanticKernel.Skills.UnitTests.OpenAPI.Authentication; +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Authentication; public class BasicAuthenticationProviderTests { diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/Authentication/BearerAuthenticationProviderTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/BearerAuthenticationProviderTests.cs similarity index 84% rename from dotnet/src/Skills/Skills.UnitTests/OpenAPI/Authentication/BearerAuthenticationProviderTests.cs rename to dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/BearerAuthenticationProviderTests.cs index 968f0c6f53f1..8e9fd71a0552 100644 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/Authentication/BearerAuthenticationProviderTests.cs +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/BearerAuthenticationProviderTests.cs @@ -3,10 +3,10 @@ using System; using System.Net.Http; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; using Xunit; -namespace SemanticKernel.Skills.UnitTests.OpenAPI.Authentication; +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Authentication; public class BearerAuthenticationProviderTests { diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/CustomAuthenticationProviderTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/CustomAuthenticationProviderTests.cs new file mode 100644 index 000000000000..2911f9471e57 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Authentication/CustomAuthenticationProviderTests.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Authentication; + +public class CustomAuthenticationProviderTests +{ + [Fact] + public async Task AuthenticateRequestAsyncSucceedsAsync() + { + // Arrange + var header = "X-MyHeader"; + var value = Guid.NewGuid().ToString(); + + using var request = new HttpRequestMessage(); + + var target = new CustomAuthenticationProvider(() => Task.FromResult(header), () => Task.FromResult(value)); + + // Act + await target.AuthenticateRequestAsync(request); + + // Assert + Assert.True(request.Headers.Contains(header)); + Assert.Equal(request.Headers.GetValues(header).FirstOrDefault(), value); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/QueryStringBuilderTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/QueryStringBuilderTests.cs new file mode 100644 index 000000000000..366f6f8eee54 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/QueryStringBuilderTests.cs @@ -0,0 +1,505 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Builders; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Builders; +public class QueryStringBuilderTests +{ + [Fact] + public void ShouldAddQueryStringParametersAndUseValuesFromArguments() + { + // Arrange + var firstParameterMetadata = new RestApiOperationParameter( + name: "p1", + type: "fake_type", + isRequired: true, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form); + + var secondParameterMetadata = new RestApiOperationParameter( + name: "p2", + type: "fake_type", + isRequired: true, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form); + + var operation = new RestApiOperation( + "fake_id", + new Uri("https://fake-random-test-host"), + "/", + HttpMethod.Get, + "fake_description", + new List { firstParameterMetadata, secondParameterMetadata }, + new Dictionary()); + + var arguments = new Dictionary + { + { "p1", "v1" }, + { "p2", "v2" } + }; + + // Act + var queryString = operation.BuildQueryString(arguments); + + // Assert + Assert.Equal("p1=v1&p2=v2", queryString); + } + + [Fact] + public void ShouldSkipNotRequiredQueryStringParametersIfTheirArgumentsMissing() + { + // Arrange + var firstParameterMetadata = new RestApiOperationParameter( + name: "p1", + type: "fake_type", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form); + + var secondParameterMetadata = new RestApiOperationParameter( + name: "p2", + type: "fake_type", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form); + + var operation = new RestApiOperation( + "fake_id", + new Uri("https://fake-random-test-host"), + "/", + HttpMethod.Get, + "fake_description", + new List { firstParameterMetadata, secondParameterMetadata }, + new Dictionary()); + + var arguments = new Dictionary + { + { "p2", "v2" } + }; + + // Act + var queryString = operation.BuildQueryString(arguments); + + // Assert + Assert.Equal("p2=v2", queryString); + } + + [Fact] + public void ShouldThrowExceptionIfNoValueIsProvideForRequiredQueryStringParameter() + { + // Arrange + var firstParameterMetadata = new RestApiOperationParameter( + name: "p1", + type: "fake_type", + isRequired: true, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form); + + var secondParameterMetadata = new RestApiOperationParameter( + name: "p2", + type: "fake_type", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form); + + var operation = new RestApiOperation( + "fake_id", + new Uri("https://fake-random-test-host"), + "/", + HttpMethod.Get, + "fake_description", + new List { firstParameterMetadata, secondParameterMetadata }, + new Dictionary()); + + var arguments = new Dictionary + { + { "p2", "v2" } + }; + + //Act and assert + Assert.Throws(() => operation.BuildQueryString(arguments)); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInQueryStringValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var metadata = new List + { + new RestApiOperationParameter( + name: "p1", + type: "string", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form) + }; + + var arguments = new Dictionary + { + { "p1", $"p1_value{specialSymbol}" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var queryString = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(queryString); + + Assert.EndsWith(encodedEquivalent, queryString, StringComparison.Ordinal); + } + + [Fact] + public void ItShouldCreateAmpersandSeparatedParameterPerArrayItemForFormStyleParameters() + { + // Arrange + var metadata = new List + { + new RestApiOperationParameter( + name: "p1", + type: "array", + isRequired: false, + expand: true, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form, + arrayItemType: "string"), + new RestApiOperationParameter( + name: "p2", + type: "array", + isRequired: false, + expand: true, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form, + arrayItemType: "integer") + }; + + var arguments = new Dictionary + { + { "p1", "[\"a\",\"b\",\"c\"]" }, + { "p2", "[1,2,3]" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var result = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("p1=a&p1=b&p1=c&p2=1&p2=2&p2=3", result); + } + + [Fact] + public void ItShouldCreateParameterWithCommaSeparatedValuePerArrayItemForFormStyleParameters() + { + // Arrange + var metadata = new List + { + new RestApiOperationParameter( + name: "p1", + type: "array", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form, + arrayItemType: "string"), + new RestApiOperationParameter( + name: "p2", + type: "array", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form, + arrayItemType: "integer") + }; + + var arguments = new Dictionary + { + { "p1", "[\"a\",\"b\",\"c\"]" }, + { "p2", "[1,2,3]" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var result = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("p1=a,b,c&p2=1,2,3", result); + } + + [Fact] + public void ItShouldCreateParameterForPrimitiveValuesForFormStyleParameters() + { + // Arrange + var metadata = new List + { + new RestApiOperationParameter( + name: "p1", + type: "string", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form, + arrayItemType: "string"), + new RestApiOperationParameter( + name: "p2", + type: "boolean", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form) + }; + + var arguments = new Dictionary + { + { "p1", "v1" }, + { "p2", "true" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var result = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("p1=v1&p2=true", result); + } + + [Fact] + public void ItShouldCreateAmpersandSeparatedParameterPerArrayItemForSpaceDelimitedStyleParameters() + { + // Arrange + var metadata = new List + { + new RestApiOperationParameter( + name: "p1", + type: "array", + isRequired: false, + expand: true, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.SpaceDelimited, + arrayItemType: "string"), + new RestApiOperationParameter( + name: "p2", + type: "array", + isRequired: false, + expand: true, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.SpaceDelimited, + arrayItemType: "integer") + }; + + var arguments = new Dictionary + { + { "p1", "[\"a\",\"b\",\"c\"]" }, + { "p2", "[1,2,3]" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var result = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("p1=a&p1=b&p1=c&p2=1&p2=2&p2=3", result); + } + + [Fact] + public void ItShouldCreateParameterWithSpaceSeparatedValuePerArrayItemForSpaceDelimitedStyleParameters() + { + // Arrange + var metadata = new List + { + new RestApiOperationParameter( + name: "p1", + type: "array", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.SpaceDelimited, + arrayItemType: "string"), + new RestApiOperationParameter( + name: "p2", + type: "array", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.SpaceDelimited, + arrayItemType: "integer") + }; + + var arguments = new Dictionary + { + { "p1", "[\"a\",\"b\",\"c\"]" }, + { "p2", "[1,2,3]" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var result = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("p1=a%20b%20c&p2=1%202%203", result); + } + + [Fact] + public void ItShouldCreateAmpersandSeparatedParameterPerArrayItemForPipeDelimitedStyleParameters() + { + // Arrange + var metadata = new List + { + new RestApiOperationParameter( + name: "p1", + type: "array", + isRequired: false, + expand: true, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.PipeDelimited, + arrayItemType: "string"), + new RestApiOperationParameter( + name: "p2", + type: "array", + isRequired: false, + expand: true, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.PipeDelimited, + arrayItemType: "integer") + }; + + var arguments = new Dictionary + { + { "p1", "[\"a\",\"b\",\"c\"]" }, + { "p2", "[1,2,3]" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var result = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("p1=a&p1=b&p1=c&p2=1&p2=2&p2=3", result); + } + + [Fact] + public void ItShouldCreateParameterWithPipeSeparatedValuePerArrayItemForPipeDelimitedStyleParameters() + { + // Arrange + var metadata = new List + { + new RestApiOperationParameter( + name: "p1", + type: "array", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.PipeDelimited, + arrayItemType: "string"), + new RestApiOperationParameter( + name: "p2", + type: "array", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.PipeDelimited, + arrayItemType: "integer") + }; + + var arguments = new Dictionary + { + { "p1", "[\"a\",\"b\",\"c\"]" }, + { "p2", "[1,2,3]" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var result = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("p1=a|b|c&p2=1|2|3", result); + } + + [Fact] + public void ItShouldMixAndMatchParametersOfDifferentTypesAndStyles() + { + // Arrange + var metadata = new List + { + //'Form' style array parameter with comma separated values + new RestApiOperationParameter(name: "p1", type: "array", isRequired: true, expand: false, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.Form, arrayItemType: "string"), + + //'Form' style primitive boolean parameter + new RestApiOperationParameter(name: "p2", type: "boolean", isRequired: true, expand: false, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.Form), + + //'Form' style array parameter with parameter per array item + new RestApiOperationParameter(name : "p3", type : "array", isRequired : true, expand : true, location : RestApiOperationParameterLocation.Query, style : RestApiOperationParameterStyle.Form), + + //'SpaceDelimited' style array parameter with space separated values + new RestApiOperationParameter(name : "p4", type : "array", isRequired : true, expand : false, location : RestApiOperationParameterLocation.Query, style : RestApiOperationParameterStyle.SpaceDelimited), + + //'SpaceDelimited' style array parameter with parameter per array item + new RestApiOperationParameter(name : "p5", type : "array", isRequired : true, expand : true, location : RestApiOperationParameterLocation.Query, style : RestApiOperationParameterStyle.SpaceDelimited), + + //'PipeDelimited' style array parameter with pipe separated values + new RestApiOperationParameter(name : "p6", type : "array", isRequired : true, expand : false, location : RestApiOperationParameterLocation.Query, style : RestApiOperationParameterStyle.PipeDelimited), + + //'PipeDelimited' style array parameter with parameter per array item + new RestApiOperationParameter(name : "p7", type : "array", isRequired : true, expand : true, location : RestApiOperationParameterLocation.Query, style : RestApiOperationParameterStyle.PipeDelimited), + }; + + var arguments = new Dictionary + { + { "p1", "[\"a\",\"b\"]" }, + { "p2", "false" }, + { "p3", "[1,2]" }, + { "p4", "[3,4]" }, + { "p5", "[\"c\",\"d\"]" }, + { "p6", "[5,6]" }, + { "p7", "[\"e\",\"f\"]" } + }; + + var operation = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); + + // Act + var result = operation.BuildQueryString(arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("p1=a,b&p2=false&p3=1&p3=2&p4=3%204&p5=c&p5=d&p6=5|6&p7=e&p7=f", result); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/ArrayParameterSerializerTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/ArrayParameterSerializerTests.cs new file mode 100644 index 000000000000..aa47a4648a35 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/ArrayParameterSerializerTests.cs @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Nodes; +using Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Builders.Serialization; +public class ArrayParameterSerializerTests +{ + [Fact] + public void ItShouldCreateParameterPerArrayItem() + { + // Arrange + var array = new JsonArray(1, 2, 3); + + // Act + var result = ArrayParameterValueSerializer.SerializeArrayAsSeparateParameters("id", array, delimiter: "&"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=1&id=2&id=3", result); + } + + [Fact] + public void ItShouldAllowDuplicatesWhenCreatingParameterPerArrayItem() + { + // Arrange + var array = new JsonArray(1, 2, 2, 3); + + // Act + var result = ArrayParameterValueSerializer.SerializeArrayAsSeparateParameters("id", array, delimiter: "&"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=1&id=2&id=2&id=3", result); + } + + [Fact] + public void ItShouldAllowParameterDelimiterAsValueWhenCreatingParameterPerArrayItem() + { + // Arrange + var array = new JsonArray("a", "b&", "c"); + + // Act + var result = ArrayParameterValueSerializer.SerializeArrayAsSeparateParameters("id", array, delimiter: "&"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=a&id=b%26&id=c", result); + } + + [Fact] + public void ItShouldCreateParameterWithDelimitedValuePerArrayItem() + { + // Arrange + var array = new JsonArray(1, 2, 3); + + // Act + var result = ArrayParameterValueSerializer.SerializeArrayAsDelimitedValues(array, delimiter: "%20"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("1%202%203", result); + } + + [Fact] + public void ItShouldAllowDuplicatesWhenCreatingParameterWithDelimitedValuePerArrayItem() + { + // Arrange + var array = new JsonArray(1, 2, 2, 3); + + // Act + var result = ArrayParameterValueSerializer.SerializeArrayAsDelimitedValues(array, delimiter: "%20"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("1%202%202%203", result); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInSeparateParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var array = new JsonArray($"{specialSymbol}"); + + // Act + var result = ArrayParameterValueSerializer.SerializeArrayAsSeparateParameters("id", array, delimiter: "&"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInDelimitedParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var array = new JsonArray($"{specialSymbol}"); + + // Act + var result = ArrayParameterValueSerializer.SerializeArrayAsDelimitedValues(array, delimiter: "%20"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/FormStyleParametersSerializerTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/FormStyleParametersSerializerTests.cs new file mode 100644 index 000000000000..4a5cd5120d0e --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/FormStyleParametersSerializerTests.cs @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Builders.Serialization; + +public class FormStyleParametersSerializerTests +{ + [Fact] + public void ItShouldCreateAmpersandSeparatedParameterPerArrayItem() + { + // Arrange + var parameter = new RestApiOperationParameter( + name: "id", + type: "array", + isRequired: true, + expand: true, //Specify generating a separate parameter for each array item. + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form, + arrayItemType: "integer"); + + // Act + var result = FormStyleParameterSerializer.Serialize(parameter, "[1,2,3]"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=1&id=2&id=3", result); + } + + [Fact] + public void ItShouldCreateParameterWithCommaSeparatedValuePerArrayItem() + { + // Arrange + var parameter = new RestApiOperationParameter( + name: "id", + type: "array", + isRequired: true, + expand: false, //Specify generating a parameter with comma-separated values for each array item. + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form, + arrayItemType: "integer"); + + // Act + var result = FormStyleParameterSerializer.Serialize(parameter, "[1,2,3]"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=1,2,3", result); + } + + [Fact] + public void ItShouldCreateParameterForPrimitiveValue() + { + // Arrange + var parameter = new RestApiOperationParameter( + name: "id", + type: "integer", + isRequired: true, + expand: false, + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.Form); + + // Act + var result = FormStyleParameterSerializer.Serialize(parameter, "28"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=28", result); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInPrimitiveParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var parameter = new RestApiOperationParameter("id", "string", false, false, RestApiOperationParameterLocation.Query, RestApiOperationParameterStyle.Form); + + // Act + var result = FormStyleParameterSerializer.Serialize(parameter, $"fake_query_param_value{specialSymbol}"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInAmpersandSeparatedParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var parameter = new RestApiOperationParameter("id", "array", false, true, RestApiOperationParameterLocation.Query, RestApiOperationParameterStyle.Form); + + // Act + var result = FormStyleParameterSerializer.Serialize(parameter, $"[\"{specialSymbol}\"]"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInCommaSeparatedParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var parameter = new RestApiOperationParameter("id", "array", false, false, RestApiOperationParameterLocation.Query, RestApiOperationParameterStyle.Form); + + // Act + var result = FormStyleParameterSerializer.Serialize(parameter, $"[\"{specialSymbol}\"]"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/PipeDelimitedStyleParametersSerializerTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/PipeDelimitedStyleParametersSerializerTests.cs new file mode 100644 index 000000000000..b499221a5ef5 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/PipeDelimitedStyleParametersSerializerTests.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Builders.Serialization; + +public class PipeDelimitedStyleParametersSerializerTests +{ + [Fact] + public void ItShouldThrowExceptionForUnsupportedParameterStyle() + { + //Arrange + var parameter = new RestApiOperationParameter(name: "p1", type: "string", isRequired: false, expand: false, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.Form); + + //Act & Assert + Assert.Throws(() => PipeDelimitedStyleParameterSerializer.Serialize(parameter, "fake-argument")); + } + + [Theory] + [InlineData("integer")] + [InlineData("number")] + [InlineData("string")] + [InlineData("boolean")] + [InlineData("object")] + public void ItShouldThrowExceptionIfParameterTypeIsNotArray(string parameterType) + { + //Arrange + var parameter = new RestApiOperationParameter(name: "p1", type: parameterType, isRequired: false, expand: false, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.PipeDelimited); + + //Act & Assert + Assert.Throws(() => PipeDelimitedStyleParameterSerializer.Serialize(parameter, "fake-argument")); + } + + [Fact] + public void ItShouldCreateAmpersandSeparatedParameterPerArrayItem() + { + // Arrange + var parameter = new RestApiOperationParameter( + name: "id", + type: "array", + isRequired: true, + expand: true, //Specifies to generate a separate parameter for each array item. + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.PipeDelimited, + arrayItemType: "integer"); + + // Act + var result = PipeDelimitedStyleParameterSerializer.Serialize(parameter, "[1,2,3]"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=1&id=2&id=3", result); + } + + [Fact] + public void ItShouldCreateParameterWithPipeSeparatedValuePerArrayItem() + { + // Arrange + var parameter = new RestApiOperationParameter( + name: "id", + type: "array", + isRequired: true, + expand: false, //Specify generating a parameter with pipe-separated values for each array item. + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.PipeDelimited, + arrayItemType: "integer"); + + // Act + var result = PipeDelimitedStyleParameterSerializer.Serialize(parameter, "[1,2,3]"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=1|2|3", result); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInPipeDelimitedParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var parameter = new RestApiOperationParameter(name: "id", type: "array", isRequired: false, expand: false, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.PipeDelimited); + + // Act + var result = PipeDelimitedStyleParameterSerializer.Serialize(parameter, $"[\"{specialSymbol}\"]"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInAmpersandDelimitedParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var parameter = new RestApiOperationParameter(name: "id", type: "array", isRequired: false, expand: true, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.PipeDelimited); + + // Act + var result = PipeDelimitedStyleParameterSerializer.Serialize(parameter, $"[\"{specialSymbol}\"]"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/SpaceDelimitedStyleParametersSerializerTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/SpaceDelimitedStyleParametersSerializerTests.cs new file mode 100644 index 000000000000..8bba5a4dba73 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Builders/Serialization/SpaceDelimitedStyleParametersSerializerTests.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Builders.Serialization; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Builders.Serialization; +public class SpaceDelimitedStyleParametersSerializerTests +{ + [Fact] + public void ItShouldThrowExceptionForUnsupportedParameterStyle() + { + //Arrange + var parameter = new RestApiOperationParameter(name: "p1", type: "string", isRequired: false, expand: false, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.Label); + + //Act & Assert + Assert.Throws(() => SpaceDelimitedStyleParameterSerializer.Serialize(parameter, "fake-argument")); + } + + [Theory] + [InlineData("integer")] + [InlineData("number")] + [InlineData("string")] + [InlineData("boolean")] + [InlineData("object")] + public void ItShouldThrowExceptionIfParameterTypeIsNotArray(string parameterType) + { + //Arrange + var parameter = new RestApiOperationParameter(name: "p1", type: parameterType, isRequired: false, expand: false, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.SpaceDelimited); + + //Act & Assert + Assert.Throws(() => SpaceDelimitedStyleParameterSerializer.Serialize(parameter, "fake-argument")); + } + + [Fact] + public void ItShouldCreateAmpersandSeparatedParameterPerArrayItem() + { + // Arrange + var parameter = new RestApiOperationParameter( + name: "id", + type: "array", + isRequired: true, + expand: true, //Specifies to generate a separate parameter for each array item. + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.SpaceDelimited, + arrayItemType: "integer"); + + // Act + var result = SpaceDelimitedStyleParameterSerializer.Serialize(parameter, "[1,2,3]"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=1&id=2&id=3", result); + } + + [Fact] + public void ItShouldCreateParameterWithSpaceSeparatedValuePerArrayItem() + { + // Arrange + var parameter = new RestApiOperationParameter( + name: "id", + type: "array", + isRequired: true, + expand: false, //Specify generating a parameter with space-separated values for each array item. + location: RestApiOperationParameterLocation.Query, + style: RestApiOperationParameterStyle.SpaceDelimited, + arrayItemType: "integer"); + + // Act + var result = SpaceDelimitedStyleParameterSerializer.Serialize(parameter, "[1,2,3]"); + + // Assert + Assert.NotNull(result); + + Assert.Equal("id=1%202%203", result); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInSpaceDelimitedParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var parameter = new RestApiOperationParameter(name: "id", type: "array", isRequired: false, expand: false, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.SpaceDelimited); + + // Act + var result = SpaceDelimitedStyleParameterSerializer.Serialize(parameter, $"[\"{specialSymbol}\"]"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } + + [Theory] + [InlineData(":", "%3a")] + [InlineData("/", "%2f")] + [InlineData("?", "%3f")] + [InlineData("#", "%23")] + public void ItShouldEncodeSpecialSymbolsInAmpersandDelimitedParameterValues(string specialSymbol, string encodedEquivalent) + { + // Arrange + var parameter = new RestApiOperationParameter(name: "id", type: "array", isRequired: false, expand: true, location: RestApiOperationParameterLocation.Query, style: RestApiOperationParameterStyle.SpaceDelimited); + + // Act + var result = SpaceDelimitedStyleParameterSerializer.Serialize(parameter, $"[\"{specialSymbol}\"]"); + + // Assert + Assert.NotNull(result); + + Assert.EndsWith(encodedEquivalent, result, StringComparison.Ordinal); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Extensions/KernelAIPluginExtensionsTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Extensions/KernelAIPluginExtensionsTests.cs new file mode 100644 index 000000000000..d8d34f1f1f51 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Extensions/KernelAIPluginExtensionsTests.cs @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Mime; +using System.Text; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Functions.OpenAPI.OpenApi; +using Microsoft.SemanticKernel.Orchestration; +using SemanticKernel.Functions.UnitTests.OpenAPI.TestPlugins; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Extensions; + +public sealed class KernelAIPluginExtensionsTests : IDisposable +{ + /// + /// System under test - an instance of OpenApiDocumentParser class. + /// + private readonly OpenApiDocumentParser _sut; + + /// + /// OpenAPI document stream. + /// + private readonly Stream _openApiDocument; + + /// + /// IKernel instance. + /// + private readonly IKernel _kernel; + + /// + /// Creates an instance of a class. + /// + public KernelAIPluginExtensionsTests() + { + this._kernel = KernelBuilder.Create(); + + this._openApiDocument = ResourcePluginsProvider.LoadFromResource("documentV2_0.json"); + + this._sut = new OpenApiDocumentParser(); + } + + [Fact] + public async Task ItCanIncludeOpenApiOperationParameterTypesIntoFunctionParametersViewAsync() + { + //Act + var plugin = await this._kernel.ImportPluginFunctionsAsync("fakePlugin", this._openApiDocument); + + //Assert + var setSecretFunction = plugin["SetSecret"]; + Assert.NotNull(setSecretFunction); + + var functionView = setSecretFunction.Describe(); + Assert.NotNull(functionView); + + var secretNameParameter = functionView.Parameters.First(p => p.Name == "secret_name"); + Assert.Equal(ParameterViewType.String, secretNameParameter.Type); + + var apiVersionParameter = functionView.Parameters.First(p => p.Name == "api_version"); + Assert.Equal("string", apiVersionParameter?.Type?.ToString()); + + var payloadParameter = functionView.Parameters.First(p => p.Name == "payload"); + Assert.Equal(ParameterViewType.Object, payloadParameter.Type); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ItUsesServerUrlOverrideIfProvidedAsync(bool removeServersProperty) + { + // Arrange + const string DocumentUri = "http://localhost:3001/openapi.json"; + const string ServerUrlOverride = "https://server-override.com/"; + + var openApiDocument = ResourcePluginsProvider.LoadFromResource("documentV3_0.json"); + + if (removeServersProperty) + { + openApiDocument = OpenApiTestHelper.ModifyOpenApiDocument(openApiDocument, (doc) => + { + doc.Remove("servers"); + }); + } + + using var messageHandlerStub = new HttpMessageHandlerStub(openApiDocument); + using var httpClient = new HttpClient(messageHandlerStub, false); + + var executionParameters = new OpenApiFunctionExecutionParameters { HttpClient = httpClient, ServerUrlOverride = new Uri(ServerUrlOverride) }; + var variables = this.GetFakeContextVariables(); + + // Act + var plugin = await this._kernel.ImportPluginFunctionsAsync("fakePlugin", new Uri(DocumentUri), executionParameters); + var setSecretFunction = plugin["SetSecret"]; + + messageHandlerStub.ResetResponse(); + + var result = await this._kernel.RunAsync(setSecretFunction, variables); + + // Assert + Assert.NotNull(setSecretFunction); + + var functionView = setSecretFunction.Describe(); + Assert.NotNull(functionView); + + var serverUrlParameter = functionView.Parameters.First(p => p.Name == "server_url"); + Assert.Equal(ServerUrlOverride, serverUrlParameter.DefaultValue); + + Assert.NotNull(messageHandlerStub.RequestUri); + Assert.StartsWith(ServerUrlOverride, messageHandlerStub.RequestUri.AbsoluteUri, StringComparison.Ordinal); + } + + [Theory] + [InlineData("documentV2_0.json")] + [InlineData("documentV3_0.json")] + public async Task ItUsesServerUrlFromOpenApiDocumentAsync(string documentFileName) + { + // Arrange + const string DocumentUri = "http://localhost:3001/openapi.json"; + const string ServerUrlFromDocument = "https://my-key-vault.vault.azure.net/"; + + var openApiDocument = ResourcePluginsProvider.LoadFromResource(documentFileName); + + using var messageHandlerStub = new HttpMessageHandlerStub(openApiDocument); + using var httpClient = new HttpClient(messageHandlerStub, false); + + var executionParameters = new OpenApiFunctionExecutionParameters { HttpClient = httpClient }; + var variables = this.GetFakeContextVariables(); + + // Act + var plugin = await this._kernel.ImportPluginFunctionsAsync("fakePlugin", new Uri(DocumentUri), executionParameters); + var setSecretFunction = plugin["SetSecret"]; + + messageHandlerStub.ResetResponse(); + + var result = await this._kernel.RunAsync(setSecretFunction, variables); + + // Assert + Assert.NotNull(setSecretFunction); + + var functionView = setSecretFunction.Describe(); + Assert.NotNull(functionView); + + var serverUrlParameter = functionView.Parameters.First(p => p.Name == "server_url"); + Assert.Equal(ServerUrlFromDocument, serverUrlParameter.DefaultValue); + + Assert.NotNull(messageHandlerStub.RequestUri); + Assert.StartsWith(ServerUrlFromDocument, messageHandlerStub.RequestUri.AbsoluteUri, StringComparison.Ordinal); + } + + [Theory] + [InlineData("http://localhost:3001/openapi.json", "http://localhost:3001/", "documentV2_0.json")] + [InlineData("http://localhost:3001/openapi.json", "http://localhost:3001/", "documentV3_0.json")] + [InlineData("https://api.example.com/openapi.json", "https://api.example.com/", "documentV2_0.json")] + [InlineData("https://api.example.com/openapi.json", "https://api.example.com/", "documentV3_0.json")] + [SuppressMessage("Design", "CA1054:URI-like parameters should not be strings", Justification = "Required for test data.")] + public async Task ItUsesOpenApiDocumentHostUrlWhenServerUrlIsNotProvidedAsync(string documentUri, string expectedServerUrl, string documentFileName) + { + // Arrange + var openApiDocument = ResourcePluginsProvider.LoadFromResource(documentFileName); + + using var content = OpenApiTestHelper.ModifyOpenApiDocument(openApiDocument, (doc) => + { + doc.Remove("servers"); + doc.Remove("host"); + doc.Remove("schemes"); + }); + + using var messageHandlerStub = new HttpMessageHandlerStub(content); + using var httpClient = new HttpClient(messageHandlerStub, false); + + var executionParameters = new OpenApiFunctionExecutionParameters { HttpClient = httpClient }; + var variables = this.GetFakeContextVariables(); + + // Act + var plugin = await this._kernel.ImportPluginFunctionsAsync("fakePlugin", new Uri(documentUri), executionParameters); + var setSecretFunction = plugin["SetSecret"]; + + messageHandlerStub.ResetResponse(); + + var result = await this._kernel.RunAsync(setSecretFunction, variables); + + // Assert + Assert.NotNull(setSecretFunction); + + var functionView = setSecretFunction.Describe(); + Assert.NotNull(functionView); + + var serverUrlParameter = functionView.Parameters.First(p => p.Name == "server_url"); + Assert.Equal(expectedServerUrl, serverUrlParameter.DefaultValue); + + Assert.NotNull(messageHandlerStub.RequestUri); + Assert.StartsWith(expectedServerUrl, messageHandlerStub.RequestUri.AbsoluteUri, StringComparison.Ordinal); + } + + [Fact] + public async Task ItShouldConvertPluginComplexResponseToStringToSaveItInContextAsync() + { + //Arrange + using var messageHandlerStub = new HttpMessageHandlerStub(); + messageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + using var httpClient = new HttpClient(messageHandlerStub, false); + + var executionParameters = new OpenApiFunctionExecutionParameters(); + executionParameters.HttpClient = httpClient; + + var fakePlugin = new FakePlugin(); + + var openApiPlugins = await this._kernel.ImportPluginFunctionsAsync("fakePlugin", this._openApiDocument, executionParameters); + var fakePlugins = this._kernel.ImportFunctions(fakePlugin); + + var kernel = KernelBuilder.Create(); + + var arguments = new ContextVariables(); + arguments.Add("secret-name", "fake-secret-name"); + arguments.Add("api-version", "fake-api-version"); + + //Act + var res = await kernel.RunAsync(arguments, openApiPlugins["GetSecret"], fakePlugins["DoFakeAction"]); + + //Assert + Assert.NotNull(res); + + var openApiPluginResult = res.FunctionResults.FirstOrDefault(); + Assert.NotNull(openApiPluginResult); + + var result = openApiPluginResult.GetValue(); + + //Check original response + Assert.NotNull(result); + Assert.Equal("fake-content", result.Content); + + //Check the response, converted to a string indirectly through an argument passed to a fake plugin that follows the OpenApi plugin in the pipeline since there's no direct access to the context. + Assert.Equal("fake-content", fakePlugin.ParameterValueFakeMethodCalledWith); + } + + public void Dispose() + { + this._openApiDocument.Dispose(); + } + + #region private ================================================================================ + + private ContextVariables GetFakeContextVariables() + { + var variables = new ContextVariables(); + + variables["secret-name"] = "fake-secret-name"; + variables["api-version"] = "fake-api-version"; + variables["X-API-Version"] = "fake-api-version"; + variables["payload"] = "fake-payload"; + + return variables; + } + + private sealed class FakePlugin + { + public string? ParameterValueFakeMethodCalledWith { get; private set; } + + [SKFunction] + public void DoFakeAction(string parameter) + { + this.ParameterValueFakeMethodCalledWith = parameter; + } + } + + #endregion +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Extensions/RestApiOperationExtensionsTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Extensions/RestApiOperationExtensionsTests.cs new file mode 100644 index 000000000000..f17b2c4ccfbb --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/Extensions/RestApiOperationExtensionsTests.cs @@ -0,0 +1,372 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.Extensions; +public class RestApiOperationExtensionsTests +{ + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + [InlineData("GET")] + public void ItShouldAddServerUrlParameterWithDefaultValueFromOperation(string method) + { + //Arrange + var payload = CreateTestJsonPayload(); + + var operation = CreateTestOperation(method, payload, new Uri("https://fake-random-test-host")); + + //Act + var parameters = operation.GetParameters(); + + //Assert + Assert.NotNull(parameters); + + var serverUrl = parameters.FirstOrDefault(p => p.Name == "server-url"); + Assert.NotNull(serverUrl); + Assert.Equal("string", serverUrl.Type); + Assert.False(serverUrl.IsRequired); + Assert.Equal("https://fake-random-test-host/", serverUrl.DefaultValue); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + [InlineData("GET")] + public void ItShouldAddServerUrlParameterWithDefaultValueFromOverrideParameter(string method) + { + //Arrange + var payload = CreateTestJsonPayload(); + + var operation = CreateTestOperation(method, payload); + + //Act + var parameters = operation.GetParameters(serverUrlOverride: new Uri("https://fake-random-test-host")); + + //Assert + Assert.NotNull(parameters); + + var serverUrl = parameters.FirstOrDefault(p => p.Name == "server-url"); + Assert.NotNull(serverUrl); + Assert.Equal("string", serverUrl.Type); + Assert.False(serverUrl.IsRequired); + Assert.Equal("https://fake-random-test-host/", serverUrl.DefaultValue); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + public void ItShouldAddPayloadAndContentTypeParametersByDefault(string method) + { + //Arrange + var payload = CreateTestJsonPayload(); + + var operation = CreateTestOperation(method, payload); + + //Act + var parameters = operation.GetParameters(); + + //Assert + Assert.NotNull(parameters); + + var payloadParam = parameters.FirstOrDefault(p => p.Name == "payload"); + Assert.NotNull(payloadParam); + Assert.Equal("object", payloadParam.Type); + Assert.True(payloadParam.IsRequired); + Assert.Equal("REST API request body.", payloadParam.Description); + + var contentTypeParam = parameters.FirstOrDefault(p => p.Name == "content-type"); + Assert.NotNull(contentTypeParam); + Assert.Equal("string", contentTypeParam.Type); + Assert.False(contentTypeParam.IsRequired); + Assert.Equal("Content type of REST API request body.", contentTypeParam.Description); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + public void ItShouldAddPayloadAndContentTypeParametersWhenSpecified(string method) + { + //Arrange + var payload = CreateTestJsonPayload(); + + var operation = CreateTestOperation(method, payload); + + //Act + var parameters = operation.GetParameters(addPayloadParamsFromMetadata: false); + + //Assert + Assert.NotNull(parameters); + + var payloadProp = parameters.FirstOrDefault(p => p.Name == "payload"); + Assert.NotNull(payloadProp); + Assert.Equal("object", payloadProp.Type); + Assert.True(payloadProp.IsRequired); + Assert.Equal("REST API request body.", payloadProp.Description); + + var contentTypeProp = parameters.FirstOrDefault(p => p.Name == "content-type"); + Assert.NotNull(contentTypeProp); + Assert.Equal("string", contentTypeProp.Type); + Assert.False(contentTypeProp.IsRequired); + Assert.Equal("Content type of REST API request body.", contentTypeProp.Description); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + public void ItShouldAddPayloadAndContentTypePropertiesForPlainTextContentType(string method) + { + //Arrange + var payload = CreateTestTextPayload(); + + var operation = CreateTestOperation(method, payload); + + //Act + var parameters = operation.GetParameters(); + + //Assert + Assert.NotNull(parameters); + + var payloadParam = parameters.FirstOrDefault(p => p.Name == "payload"); + Assert.NotNull(payloadParam); + Assert.Equal("string", payloadParam.Type); + Assert.True(payloadParam.IsRequired); + Assert.Equal("REST API request body.", payloadParam.Description); + + var contentTypeParam = parameters.FirstOrDefault(p => p.Name == "content-type"); + Assert.NotNull(contentTypeParam); + Assert.Equal("string", contentTypeParam.Type); + Assert.False(contentTypeParam.IsRequired); + Assert.Equal("Content type of REST API request body.", contentTypeParam.Description); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + public void ItShouldAddPayloadAndContentTypePropertiesIfParametersFromPayloadMetadataAreNotRequired(string method) + { + //Arrange + var payload = CreateTestJsonPayload(); + + var operation = CreateTestOperation(method, payload); + + //Act + var parameters = operation.GetParameters(addPayloadParamsFromMetadata: false); + + //Assert + Assert.NotNull(parameters); + + var payloadParam = parameters.FirstOrDefault(p => p.Name == "payload"); + Assert.NotNull(payloadParam); + Assert.Equal("object", payloadParam.Type); + Assert.True(payloadParam.IsRequired); + Assert.Equal("REST API request body.", payloadParam.Description); + + var contentTypeParam = parameters.FirstOrDefault(p => p.Name == "content-type"); + Assert.NotNull(contentTypeParam); + Assert.Equal("string", contentTypeParam.Type); + Assert.False(contentTypeParam.IsRequired); + Assert.Equal("Content type of REST API request body.", contentTypeParam.Description); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + public void ItShouldAddParametersDeclaredInPayloadMetadata(string method) + { + //Arrange + var payload = CreateTestJsonPayload(); + + var operation = CreateTestOperation(method, payload); + + //Act + var parameters = operation.GetParameters(addPayloadParamsFromMetadata: true); + + //Assert + Assert.NotNull(parameters); + + Assert.Equal(6, parameters.Count); //5(props from payload) + 1('server-url' property) + + var name = parameters.FirstOrDefault(p => p.Name == "name"); + Assert.NotNull(name); + Assert.Equal("string", name.Type); + Assert.True(name.IsRequired); + Assert.Equal("The name.", name.Description); + + var landmarks = parameters.FirstOrDefault(p => p.Name == "landmarks"); + Assert.NotNull(landmarks); + Assert.Equal("array", landmarks.Type); + Assert.False(landmarks.IsRequired); + Assert.Equal("The landmarks.", landmarks.Description); + + var leader = parameters.FirstOrDefault(p => p.Name == "leader"); + Assert.NotNull(leader); + Assert.Equal("string", leader.Type); + Assert.True(leader.IsRequired); + Assert.Equal("The leader.", leader.Description); + + var population = parameters.FirstOrDefault(p => p.Name == "population"); + Assert.NotNull(population); + Assert.Equal("integer", population.Type); + Assert.True(population.IsRequired); + Assert.Equal("The population.", population.Description); + + var hasMagicWards = parameters.FirstOrDefault(p => p.Name == "hasMagicWards"); + Assert.NotNull(hasMagicWards); + Assert.Equal("boolean", hasMagicWards.Type); + Assert.False(hasMagicWards.IsRequired); + Assert.Null(hasMagicWards.Description); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + public void ItShouldAddNamespaceToParametersDeclaredInPayloadMetadata(string method) + { + //Arrange + var payload = CreateTestJsonPayload(); + + var operation = CreateTestOperation(method, payload); + + //Act + var parameters = operation.GetParameters(addPayloadParamsFromMetadata: true, enablePayloadNamespacing: true); + + //Assert + Assert.NotNull(parameters); + + Assert.Equal(6, parameters.Count); //5(props from payload) + 1('server-url' property) + + var name = parameters.FirstOrDefault(p => p.Name == "name"); + Assert.NotNull(name); + Assert.Equal("string", name.Type); + Assert.True(name.IsRequired); + Assert.Equal("The name.", name.Description); + + var landmarks = parameters.FirstOrDefault(p => p.Name == "location.landmarks"); + Assert.NotNull(landmarks); + Assert.Equal("array", landmarks.Type); + Assert.False(landmarks.IsRequired); + Assert.Equal("The landmarks.", landmarks.Description); + + var leader = parameters.FirstOrDefault(p => p.Name == "rulingCouncil.leader"); + Assert.NotNull(leader); + Assert.Equal("string", leader.Type); + Assert.True(leader.IsRequired); + Assert.Equal("The leader.", leader.Description); + + var population = parameters.FirstOrDefault(p => p.Name == "population"); + Assert.NotNull(population); + Assert.Equal("integer", population.Type); + Assert.True(population.IsRequired); + Assert.Equal("The population.", population.Description); + + var hasMagicWards = parameters.FirstOrDefault(p => p.Name == "hasMagicWards"); + Assert.NotNull(hasMagicWards); + Assert.Equal("boolean", hasMagicWards.Type); + Assert.False(hasMagicWards.IsRequired); + Assert.Null(hasMagicWards.Description); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + public void ItShouldThrowExceptionIfPayloadMetadataDescribingParametersIsMissing(string method) + { + //Arrange + var operation = CreateTestOperation(method, null); + + //Act + Assert.Throws(() => operation.GetParameters(addPayloadParamsFromMetadata: true, enablePayloadNamespacing: true)); + } + + [Fact] + public void ItShouldSetAlternativeNameToParametersForGetOperation() + { + //Arrange + var operation = CreateTestOperation("GET"); + + //Act + var parameters = operation.GetParameters(addPayloadParamsFromMetadata: true); + + //Assert + Assert.NotNull(parameters); + + var serverUrlProp = parameters.FirstOrDefault(p => p.Name == "server-url"); + Assert.NotNull(serverUrlProp); + Assert.Equal("server_url", serverUrlProp.AlternativeName); + } + + [Theory] + [InlineData("PUT")] + [InlineData("POST")] + public void ItShouldSetAlternativeNameToParametersForPutAndPostOperation(string method) + { + //Arrange + var latitude = new RestApiOperationPayloadProperty("location.latitude", "number", false, new List()); + var place = new RestApiOperationPayloadProperty("place", "string", true, new List()); + + var payload = new RestApiOperationPayload("application/json", new[] { place, latitude }); + + var operation = CreateTestOperation(method, payload); + + //Act + var parameters = operation.GetParameters(addPayloadParamsFromMetadata: true); + + //Assert + Assert.NotNull(parameters); + + var serverUrlProp = parameters.FirstOrDefault(p => p.Name == "server-url"); + Assert.NotNull(serverUrlProp); + Assert.Equal("server_url", serverUrlProp.AlternativeName); + + var placeProp = parameters.FirstOrDefault(p => p.Name == "place"); + Assert.NotNull(placeProp); + Assert.Equal("place", placeProp.AlternativeName); + + var personNameProp = parameters.FirstOrDefault(p => p.Name == "location.latitude"); + Assert.NotNull(personNameProp); + Assert.Equal("location_latitude", personNameProp.AlternativeName); + } + + private static RestApiOperation CreateTestOperation(string method, RestApiOperationPayload? payload = null, Uri? url = null) + { + return new RestApiOperation( + id: "fake-id", + serverUrl: url, + path: "fake-path", + method: new HttpMethod(method), + description: "fake-description", + parameters: new List(), + headers: new Dictionary(), + payload: payload); + } + + private static RestApiOperationPayload CreateTestJsonPayload() + { + var name = new RestApiOperationPayloadProperty("name", "string", true, new List(), "The name."); + + var leader = new RestApiOperationPayloadProperty("leader", "string", true, new List(), "The leader."); + + var landmarks = new RestApiOperationPayloadProperty("landmarks", "array", false, new List(), "The landmarks."); + var location = new RestApiOperationPayloadProperty("location", "object", true, new[] { landmarks }, "The location."); + + var rulingCouncil = new RestApiOperationPayloadProperty("rulingCouncil", "object", true, new[] { leader }, "The ruling council."); + + var population = new RestApiOperationPayloadProperty("population", "integer", true, new List(), "The population."); + + var hasMagicWards = new RestApiOperationPayloadProperty("hasMagicWards", "boolean", false, new List()); + + return new RestApiOperationPayload("application/json", new[] { name, location, rulingCouncil, population, hasMagicWards }); + } + + private static RestApiOperationPayload CreateTestTextPayload() + { + return new RestApiOperationPayload("text/plain", new List()); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/HttpMessageHandlerStub.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/HttpMessageHandlerStub.cs new file mode 100644 index 000000000000..542aa33a7c91 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/HttpMessageHandlerStub.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Mime; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI; + +internal sealed class HttpMessageHandlerStub : DelegatingHandler +{ + public HttpRequestHeaders? RequestHeaders { get; private set; } + + public HttpContentHeaders? ContentHeaders { get; private set; } + + public byte[]? RequestContent { get; private set; } + + public Uri? RequestUri { get; private set; } + + public HttpMethod? Method { get; private set; } + + public HttpResponseMessage ResponseToReturn { get; set; } + + public HttpMessageHandlerStub() + { + this.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK); + this.ResponseToReturn.Content = new StringContent("{}", Encoding.UTF8, MediaTypeNames.Application.Json); + } + + public HttpMessageHandlerStub(Stream responseToReturn) + { + this.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK); + this.ResponseToReturn.Content = new StreamContent(responseToReturn); + } + + public void ResetResponse() + { + this.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK); + this.ResponseToReturn.Content = new StringContent("{}", Encoding.UTF8, MediaTypeNames.Application.Json); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + this.Method = request.Method; + this.RequestUri = request.RequestUri; + this.RequestHeaders = request.Headers; + this.RequestContent = request.Content == null ? null : await request.Content.ReadAsByteArrayAsync(cancellationToken); + this.ContentHeaders = request.Content?.Headers; + + return await Task.FromResult(this.ResponseToReturn); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/JsonPathPluginTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/JsonPathPluginTests.cs new file mode 100644 index 000000000000..6e8993c6fb20 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/JsonPathPluginTests.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.SemanticKernel.Functions.OpenAPI; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI; + +public class JsonPathPluginTests +{ + private const string Json = @"{ + 'Stores': [ + 'Lambton Quay', + 'Willis Street' + ], + 'Manufacturers': [ + { + 'Name': 'Acme Co', + 'Products': [ + { + 'Name': 'Anvil', + 'Price': 50 + } + ] + }, + { + 'Name': 'Contoso', + 'Products': [ + { + 'Name': 'Elbow Grease', + 'Price': 99.95 + }, + { + 'Name': 'Headlight Fluid', + 'Price': 4 + } + ] + } + ] +}"; + + [Theory] + [InlineData("$.Manufacturers[0].Products[0].Name", "Anvil")] // single value + [InlineData("$.Manufacturers[0].Products[0].Foo", "")] // no value + public void GetJsonElementValueSucceeds(string jsonPath, string expected) + { + var target = new JsonPathPlugin(); + + string actual = target.GetJsonElementValue(Json, jsonPath); + + Assert.Equal(expected, actual, StringComparer.OrdinalIgnoreCase); + } + + [Theory] + [InlineData("$..Products[?(@.Price >= 50)].Name", "[\"Anvil\",\"Elbow Grease\"]")] // multiple values + [InlineData("$.Manufacturers", + "[[{\"Name\":\"Acme Co\",\"Products\":[{\"Name\":\"Anvil\",\"Price\":50}]},{\"Name\":\"Contoso\",\"Products\":[{\"Name\":\"Elbow Grease\",\"Price\":99.95},{\"Name\":\"Headlight Fluid\",\"Price\":4}]}]]")] // complex value + public void GetJsonPropertyValueSucceeds(string jsonPath, string expected) + { + var target = new JsonPathPlugin(); + + string actual = target.GetJsonElements(Json, jsonPath); + + Assert.Equal(expected, actual, StringComparer.OrdinalIgnoreCase); + } +} diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV20Tests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV20Tests.cs similarity index 85% rename from dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV20Tests.cs rename to dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV20Tests.cs index 9eb72abdd342..47864d773a18 100644 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV20Tests.cs +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV20Tests.cs @@ -6,12 +6,12 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; -using Microsoft.SemanticKernel.Skills.OpenAPI.OpenApi; -using SemanticKernel.Skills.UnitTests.OpenAPI.TestSkills; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Functions.OpenAPI.OpenApi; +using SemanticKernel.Functions.UnitTests.OpenAPI.TestPlugins; using Xunit; -namespace SemanticKernel.Skills.UnitTests.OpenAPI; +namespace SemanticKernel.Functions.UnitTests.OpenAPI; public sealed class OpenApiDocumentParserV20Tests : IDisposable { @@ -30,7 +30,7 @@ public sealed class OpenApiDocumentParserV20Tests : IDisposable /// public OpenApiDocumentParserV20Tests() { - this._openApiDocument = ResourceSkillsProvider.LoadFromResource("documentV2_0.json"); + this._openApiDocument = ResourcePluginsProvider.LoadFromResource("documentV2_0.json"); this._sut = new OpenApiDocumentParser(); } @@ -60,6 +60,7 @@ public async Task ItCanParsePutOperationBodySuccessfullyAsync() Assert.NotNull(valueProperty); Assert.True(valueProperty.IsRequired); Assert.Equal("The value of the secret.", valueProperty.Description); + Assert.Equal("string", valueProperty.Type); Assert.NotNull(valueProperty.Properties); Assert.False(valueProperty.Properties.Any()); @@ -67,6 +68,7 @@ public async Task ItCanParsePutOperationBodySuccessfullyAsync() Assert.NotNull(attributesProperty); Assert.False(attributesProperty.IsRequired); Assert.Equal("attributes", attributesProperty.Description); + Assert.Equal("object", attributesProperty.Type); Assert.NotNull(attributesProperty.Properties); Assert.True(attributesProperty.Properties.Any()); @@ -74,8 +76,8 @@ public async Task ItCanParsePutOperationBodySuccessfullyAsync() Assert.NotNull(enabledProperty); Assert.False(enabledProperty.IsRequired); Assert.Equal("Determines whether the object is enabled.", enabledProperty.Description); - Assert.NotNull(enabledProperty.Properties); - Assert.False(enabledProperty.Properties.Any()); + Assert.Equal("boolean", enabledProperty.Type); + Assert.False(enabledProperty.Properties?.Any()); } [Fact] @@ -127,6 +129,21 @@ public async Task ItCanParsePutOperationMetadataSuccessfullyAsync() Assert.Equal("Content type of REST API request body.", contentTypeParameter.Description); } + [Fact] + public async Task ItCanUseOperationSummaryAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "Excuses"); + Assert.NotNull(operation); + Assert.Equal("Turn a scenario into a creative or humorous excuse to send your boss", operation.Description); + } + [Fact] public async Task ItCanExtractSimpleTypeHeaderParameterMetadataSuccessfullyAsync() { @@ -215,7 +232,24 @@ public async Task ItCanParseOperationHavingTextPlainBodySuccessfullyAsync() var properties = payload.Properties; Assert.NotNull(properties); - Assert.Equal(0, properties.Count); + Assert.Empty(properties); + } + + [Fact] + public async Task ItCanWorkWithDocumentsWithoutHostAndSchemaAttributesAsync() + { + //Arrange + using var stream = OpenApiTestHelper.ModifyOpenApiDocument(this._openApiDocument, (doc) => + { + doc.Remove("host"); + doc.Remove("schemes"); + }); + + //Act + var operations = await this._sut.ParseAsync(stream); + + //Assert + Assert.All(operations, (op) => Assert.Null(op.ServerUrl)); } private static RestApiOperationParameter GetParameterMetadata(IList operations, string operationId, diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV30Tests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV30Tests.cs new file mode 100644 index 000000000000..cfeda81fc71f --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV30Tests.cs @@ -0,0 +1,366 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Functions.OpenAPI.OpenApi; +using SemanticKernel.Functions.UnitTests.OpenAPI.TestPlugins; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI; + +public sealed class OpenApiDocumentParserV30Tests : IDisposable +{ + /// + /// System under test - an instance of OpenApiDocumentParser class. + /// + private readonly OpenApiDocumentParser _sut; + + /// + /// OpenAPI document stream. + /// + private readonly Stream _openApiDocument; + + /// + /// Creates an instance of a class. + /// + public OpenApiDocumentParserV30Tests() + { + this._openApiDocument = ResourcePluginsProvider.LoadFromResource("documentV3_0.json"); + + this._sut = new OpenApiDocumentParser(); + } + + [Fact] + public async Task ItCanParsePutOperationBodySuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var putOperation = operations.Single(o => o.Id == "SetSecret"); + Assert.NotNull(putOperation); + + var payload = putOperation.Payload; + Assert.NotNull(payload); + Assert.Equal("application/json", payload.MediaType); + + var properties = payload.Properties; + Assert.NotNull(properties); + Assert.Equal(2, properties.Count); + + var valueProperty = properties.FirstOrDefault(p => p.Name == "value"); + Assert.NotNull(valueProperty); + Assert.True(valueProperty.IsRequired); + Assert.Equal("The value of the secret.", valueProperty.Description); + Assert.Equal("string", valueProperty.Type); + Assert.NotNull(valueProperty.Properties); + Assert.False(valueProperty.Properties.Any()); + + var attributesProperty = properties.FirstOrDefault(p => p.Name == "attributes"); + Assert.NotNull(attributesProperty); + Assert.False(attributesProperty.IsRequired); + Assert.Equal("attributes", attributesProperty.Description); + Assert.Equal("object", attributesProperty.Type); + Assert.NotNull(attributesProperty.Properties); + Assert.True(attributesProperty.Properties.Any()); + + var enabledProperty = attributesProperty.Properties.FirstOrDefault(p => p.Name == "enabled"); + Assert.NotNull(enabledProperty); + Assert.False(enabledProperty.IsRequired); + Assert.Equal("Determines whether the object is enabled.", enabledProperty.Description); + Assert.Equal("boolean", enabledProperty.Type); + Assert.False(enabledProperty.Properties?.Any()); + } + + [Fact] + public async Task ItCanParsePutOperationMetadataSuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var putOperation = operations.Single(o => o.Id == "SetSecret"); + Assert.NotNull(putOperation); + Assert.Equal("Sets a secret in a specified key vault.", putOperation.Description); + Assert.Equal("https://my-key-vault.vault.azure.net/", putOperation.ServerUrl?.AbsoluteUri); + Assert.Equal(HttpMethod.Put, putOperation.Method); + Assert.Equal("/secrets/{secret-name}", putOperation.Path); + + var parameters = putOperation.GetParameters(); + Assert.NotNull(parameters); + Assert.True(parameters.Count >= 5); + + var pathParameter = parameters.Single(p => p.Name == "secret-name"); //'secret-name' path parameter. + Assert.True(pathParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Path, pathParameter.Location); + Assert.Null(pathParameter.DefaultValue); + + var apiVersionParameter = parameters.Single(p => p.Name == "api-version"); //'api-version' query string parameter. + Assert.True(apiVersionParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Query, apiVersionParameter.Location); + Assert.Equal("7.0", apiVersionParameter.DefaultValue); + + var serverUrlParameter = parameters.Single(p => p.Name == "server-url"); //'server-url' artificial parameter. + Assert.False(serverUrlParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Path, serverUrlParameter.Location); + Assert.Equal("https://my-key-vault.vault.azure.net/", serverUrlParameter.DefaultValue); + + var payloadParameter = parameters.Single(p => p.Name == "payload"); //'payload' artificial parameter. + Assert.True(payloadParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Body, payloadParameter.Location); + Assert.Null(payloadParameter.DefaultValue); + Assert.Equal("REST API request body.", payloadParameter.Description); + + var contentTypeParameter = parameters.Single(p => p.Name == "content-type"); //'content-type' artificial parameter. + Assert.False(contentTypeParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Body, contentTypeParameter.Location); + Assert.Null(contentTypeParameter.DefaultValue); + Assert.Equal("Content type of REST API request body.", contentTypeParameter.Description); + } + + [Fact] + public async Task ItCanExtractSimpleTypeHeaderParameterMetadataSuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + //Assert string header parameter metadata + var accept = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "Accept"); + + Assert.Equal("string", accept.Type); + Assert.Equal("application/json", accept.DefaultValue); + Assert.Equal("Indicates which content types, expressed as MIME types, the client is able to understand.", accept.Description); + Assert.False(accept.IsRequired); + + //Assert integer header parameter metadata + var apiVersion = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "X-API-Version"); + + Assert.Equal("integer", apiVersion.Type); + Assert.Equal("10", apiVersion.DefaultValue); + Assert.Equal("Requested API version.", apiVersion.Description); + Assert.True(apiVersion.IsRequired); + } + + [Fact] + public async Task ItCanUseOperationSummaryAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "Excuses"); + Assert.NotNull(operation); + Assert.Equal("Turn a scenario into a creative or humorous excuse to send your boss", operation.Description); + } + + [Fact] + public async Task ItCanExtractCsvStyleHeaderParameterMetadataSuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + //Assert header parameters metadata + var acceptParameter = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "X-Operation-Csv-Ids"); + + Assert.Null(acceptParameter.DefaultValue); + Assert.False(acceptParameter.IsRequired); + Assert.Equal("array", acceptParameter.Type); + Assert.Equal(RestApiOperationParameterStyle.Simple, acceptParameter.Style); + Assert.Equal("The comma separated list of operation ids.", acceptParameter.Description); + Assert.Equal("string", acceptParameter.ArrayItemType); + } + + [Fact] + public async Task ItCanExtractHeadersSuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "SetSecret"); + Assert.NotNull(operation.Headers); + Assert.Equal(3, operation.Headers.Count); + + Assert.True(operation.Headers.ContainsKey("Accept")); + Assert.True(operation.Headers.ContainsKey("X-API-Version")); + Assert.True(operation.Headers.ContainsKey("X-Operation-Csv-Ids")); + } + + [Fact] + public async Task ItCanExtractAllPathsAsOperationsAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.Equal(3, operations.Count); + } + + [Fact] + public async Task ItCanParseOperationHavingTextPlainBodySuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "Excuses"); + Assert.NotNull(operation); + + var payload = operation.Payload; + Assert.NotNull(payload); + Assert.Equal("text/plain", payload.MediaType); + Assert.Equal("excuse event", payload.Description); + + var properties = payload.Properties; + Assert.NotNull(properties); + Assert.Empty(properties); + } + + [Fact] + public async Task ItShouldThrowExceptionForNonCompliantDocumentAsync() + { + // Arrange + var nonComplaintOpenApiDocument = ResourcePluginsProvider.LoadFromResource("nonCompliant_documentV3_0.json"); + + // Act and Assert + await Assert.ThrowsAsync(async () => await this._sut.ParseAsync(nonComplaintOpenApiDocument)); + } + + [Fact] + public async Task ItShouldWorkWithNonCompliantDocumentIfAllowedAsync() + { + // Arrange + var nonComplaintOpenApiDocument = ResourcePluginsProvider.LoadFromResource("nonCompliant_documentV3_0.json"); + + // Act + await this._sut.ParseAsync(nonComplaintOpenApiDocument, ignoreNonCompliantErrors: true); + + // Assert + // The absence of any thrown exceptions serves as evidence of the functionality's success. + } + + [Fact] + public async Task ItCanWorkWithDocumentsWithoutServersAttributeAsync() + { + //Arrange + using var stream = ModifyOpenApiDocument(this._openApiDocument, (doc) => + { + doc.Remove("servers"); + }); + + //Act + var operations = await this._sut.ParseAsync(stream); + + //Assert + Assert.All(operations, (op) => Assert.Null(op.ServerUrl)); + } + + [Fact] + public async Task ItCanWorkWithDocumentsWithEmptyServersAttributeAsync() + { + //Arrange + using var stream = ModifyOpenApiDocument(this._openApiDocument, (doc) => + { + doc["servers"] = new JsonArray(); + }); + + //Act + var operations = await this._sut.ParseAsync(stream); + + //Assert + Assert.All(operations, (op) => Assert.Null(op.ServerUrl)); + } + + [Theory] + [InlineData("explodeFormParam")] + [InlineData("anotherExplodeFormParam")] + public async Task ItShouldSupportsAmpersandSeparatedParametersForFormStyleArrayQueryStringParametersAsync(string parameterName) + { + //Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + //Assert + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "GetSecret"); + + var explodeFormParam = operation.Parameters.Single(p => p.Name == parameterName); + + Assert.True(explodeFormParam.Expand); + } + + [Fact] + public async Task ItShouldSupportsCommaSeparatedValuesForFormStyleArrayQueryStringParametersAsync() + { + //Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + //Assert + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "GetSecret"); + + var explodeFormParam = operation.Parameters.Single(p => p.Name == "nonExplodeFormParam"); + + Assert.False(explodeFormParam.Expand); + } + + private static MemoryStream ModifyOpenApiDocument(Stream openApiDocument, Action transformer) + { + var json = JsonSerializer.Deserialize(openApiDocument); + + transformer(json!); + + var stream = new MemoryStream(); + + JsonSerializer.Serialize(stream, json); + + stream.Seek(0, SeekOrigin.Begin); + + return stream; + } + + private static RestApiOperationParameter GetParameterMetadata(IList operations, string operationId, + RestApiOperationParameterLocation location, string name) + { + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == operationId); + Assert.NotNull(operation.Parameters); + Assert.True(operation.Parameters.Any()); + + var parameters = operation.Parameters.Where(p => p.Location == location); + + var parameter = parameters.Single(p => p.Name == name); + Assert.NotNull(parameter); + + return parameter; + } + + public void Dispose() + { + this._openApiDocument.Dispose(); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV31Tests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV31Tests.cs new file mode 100644 index 000000000000..4bb447a46e80 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiDocumentParserV31Tests.cs @@ -0,0 +1,345 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Dynamic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Functions.OpenAPI.OpenApi; +using SemanticKernel.Functions.UnitTests.OpenAPI.TestPlugins; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI; + +public sealed class OpenApiDocumentParserV31Tests : IDisposable +{ + /// + /// System under test - an instance of OpenApiDocumentParser class. + /// + private readonly OpenApiDocumentParser _sut; + + /// + /// OpenAPI document stream. + /// + private readonly Stream _openApiDocument; + + /// + /// Creates an instance of a class. + /// + public OpenApiDocumentParserV31Tests() + { + this._openApiDocument = ResourcePluginsProvider.LoadFromResource("documentV3_1.yaml"); + + this._sut = new OpenApiDocumentParser(); + } + + [Fact] + public async Task ItCanParsePutOperationBodySuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var putOperation = operations.Single(o => o.Id == "SetSecret"); + Assert.NotNull(putOperation); + + var payload = putOperation.Payload; + Assert.NotNull(payload); + Assert.Equal("application/json", payload.MediaType); + + var properties = payload.Properties; + Assert.NotNull(properties); + Assert.Equal(2, properties.Count); + + var valueProperty = properties.FirstOrDefault(p => p.Name == "value"); + Assert.NotNull(valueProperty); + Assert.True(valueProperty.IsRequired); + Assert.Equal("The value of the secret.", valueProperty.Description); + Assert.Equal("string", valueProperty.Type); + Assert.NotNull(valueProperty.Properties); + Assert.False(valueProperty.Properties.Any()); + + var attributesProperty = properties.FirstOrDefault(p => p.Name == "attributes"); + Assert.NotNull(attributesProperty); + Assert.False(attributesProperty.IsRequired); + Assert.Equal("attributes", attributesProperty.Description); + Assert.Equal("object", attributesProperty.Type); + Assert.NotNull(attributesProperty.Properties); + Assert.True(attributesProperty.Properties.Any()); + + var enabledProperty = attributesProperty.Properties.FirstOrDefault(p => p.Name == "enabled"); + Assert.NotNull(enabledProperty); + Assert.False(enabledProperty.IsRequired); + Assert.Equal("Determines whether the object is enabled.", enabledProperty.Description); + Assert.Equal("boolean", enabledProperty.Type); + Assert.False(enabledProperty.Properties?.Any()); + } + + [Fact] + public async Task ItCanParsePutOperationMetadataSuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var putOperation = operations.Single(o => o.Id == "SetSecret"); + Assert.NotNull(putOperation); + Assert.Equal("Sets a secret in a specified key vault.", putOperation.Description); + Assert.Equal("https://my-key-vault.vault.azure.net/", putOperation.ServerUrl?.AbsoluteUri); + Assert.Equal(HttpMethod.Put, putOperation.Method); + Assert.Equal("/secrets/{secret-name}", putOperation.Path); + + var parameters = putOperation.GetParameters(); + Assert.NotNull(parameters); + Assert.True(parameters.Count >= 5); + + var pathParameter = parameters.Single(p => p.Name == "secret-name"); //'secret-name' path parameter. + Assert.True(pathParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Path, pathParameter.Location); + Assert.Null(pathParameter.DefaultValue); + + var apiVersionParameter = parameters.Single(p => p.Name == "api-version"); //'api-version' query string parameter. + Assert.True(apiVersionParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Query, apiVersionParameter.Location); + Assert.Equal("7.0", apiVersionParameter.DefaultValue); + + var serverUrlParameter = parameters.Single(p => p.Name == "server-url"); //'server-url' artificial parameter. + Assert.False(serverUrlParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Path, serverUrlParameter.Location); + Assert.Equal("https://my-key-vault.vault.azure.net/", serverUrlParameter.DefaultValue); + + var payloadParameter = parameters.Single(p => p.Name == "payload"); //'payload' artificial parameter. + Assert.True(payloadParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Body, payloadParameter.Location); + Assert.Null(payloadParameter.DefaultValue); + Assert.Equal("REST API request body.", payloadParameter.Description); + + var contentTypeParameter = parameters.Single(p => p.Name == "content-type"); //'content-type' artificial parameter. + Assert.False(contentTypeParameter.IsRequired); + Assert.Equal(RestApiOperationParameterLocation.Body, contentTypeParameter.Location); + Assert.Null(contentTypeParameter.DefaultValue); + Assert.Equal("Content type of REST API request body.", contentTypeParameter.Description); + } + + [Fact] + public async Task ItCanUseOperationSummaryAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "Excuses"); + Assert.NotNull(operation); + Assert.Equal("Turn a scenario into a creative or humorous excuse to send your boss", operation.Description); + } + + [Fact] + public async Task ItCanExtractSimpleTypeHeaderParameterMetadataSuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + //Assert string header parameter metadata + var accept = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "Accept"); + + Assert.Equal("string", accept.Type); + Assert.Equal("application/json", accept.DefaultValue); + Assert.Equal("Indicates which content types, expressed as MIME types, the client is able to understand.", accept.Description); + Assert.False(accept.IsRequired); + + //Assert integer header parameter metadata + var apiVersion = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "X-API-Version"); + + Assert.Equal("integer", apiVersion.Type); + Assert.Equal("10", apiVersion.DefaultValue); + Assert.Equal("Requested API version.", apiVersion.Description); + Assert.True(apiVersion.IsRequired); + } + + [Fact] + public async Task ItCanExtractCsvStyleHeaderParameterMetadataSuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + //Assert header parameters metadata + var acceptParameter = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "X-Operation-Csv-Ids"); + + Assert.Null(acceptParameter.DefaultValue); + Assert.False(acceptParameter.IsRequired); + Assert.Equal("array", acceptParameter.Type); + Assert.Equal(RestApiOperationParameterStyle.Simple, acceptParameter.Style); + Assert.Equal("The comma separated list of operation ids.", acceptParameter.Description); + Assert.Equal("string", acceptParameter.ArrayItemType); + } + + [Fact] + public async Task ItCanExtractHeadersSuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "SetSecret"); + Assert.NotNull(operation.Headers); + Assert.Equal(3, operation.Headers.Count); + + Assert.True(operation.Headers.ContainsKey("Accept")); + Assert.True(operation.Headers.ContainsKey("X-API-Version")); + Assert.True(operation.Headers.ContainsKey("X-Operation-Csv-Ids")); + } + + [Fact] + public async Task ItCanExtractAllPathsAsOperationsAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.Equal(3, operations.Count); + } + + [Fact] + public async Task ItCanParseOperationHavingTextPlainBodySuccessfullyAsync() + { + // Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + // Assert + Assert.NotNull(operations); + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "Excuses"); + Assert.NotNull(operation); + + var payload = operation.Payload; + Assert.NotNull(payload); + Assert.Equal("text/plain", payload.MediaType); + Assert.Equal("excuse event", payload.Description); + + var properties = payload.Properties; + Assert.NotNull(properties); + Assert.Empty(properties); + } + + [Fact] + public async Task ItCanWorkWithDocumentsWithoutServersAttributeAsync() + { + //Arrange + using var stream = ModifyOpenApiDocument(this._openApiDocument, (yaml) => + { + yaml.Remove("servers"); + }); + + //Act + var operations = await this._sut.ParseAsync(stream); + + //Assert + Assert.All(operations, (op) => Assert.Null(op.ServerUrl)); + } + + [Fact] + public async Task ItCanWorkWithDocumentsWithEmptyServersAttributeAsync() + { + //Arrange + using var stream = ModifyOpenApiDocument(this._openApiDocument, (yaml) => + { + yaml["servers"] = Array.Empty(); + }); + + //Act + var operations = await this._sut.ParseAsync(stream); + + //Assert + Assert.All(operations, (op) => Assert.Null(op.ServerUrl)); + } + + [Theory] + [InlineData("explodeFormParam")] + [InlineData("anotherExplodeFormParam")] + public async Task ItShouldSupportsAmpersandSeparatedParametersForFormStyleArrayQueryStringParametersAsync(string parameterName) + { + //Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + //Assert + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "GetSecret"); + + var explodeFormParam = operation.Parameters.Single(p => p.Name == parameterName); + + Assert.True(explodeFormParam.Expand); + } + + [Fact] + public async Task ItShouldSupportsCommaSeparatedValuesForFormStyleArrayQueryStringParametersAsync() + { + //Act + var operations = await this._sut.ParseAsync(this._openApiDocument); + + //Assert + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == "GetSecret"); + + var explodeFormParam = operation.Parameters.Single(p => p.Name == "nonExplodeFormParam"); + + Assert.False(explodeFormParam.Expand); + } + + private static MemoryStream ModifyOpenApiDocument(Stream openApiDocument, Action> transformer) + { + var serializer = new SharpYaml.Serialization.Serializer(); + + //Deserialize yaml + var yaml = serializer.Deserialize(openApiDocument); + + //Modify yaml + transformer(yaml!); + + //Serialize yaml + var stream = new MemoryStream(); + + serializer.Serialize(stream, yaml); + + stream.Seek(0, SeekOrigin.Begin); + + return stream; + } + + private static RestApiOperationParameter GetParameterMetadata(IList operations, string operationId, RestApiOperationParameterLocation location, string name) + { + Assert.True(operations.Any()); + + var operation = operations.Single(o => o.Id == operationId); + Assert.NotNull(operation.Parameters); + Assert.True(operation.Parameters.Any()); + + var parameters = operation.Parameters.Where(p => p.Location == location); + + var parameter = parameters.Single(p => p.Name == name); + Assert.NotNull(parameter); + + return parameter; + } + + public void Dispose() + { + this._openApiDocument.Dispose(); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiTestHelper.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiTestHelper.cs new file mode 100644 index 000000000000..3c487ecd1003 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/OpenApiTestHelper.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI; + +/// +/// Contains helper methods for OpenApi related tests. +/// +internal static class OpenApiTestHelper +{ + /// + /// Modifies OpenApi document for testing different scenarios. + /// + /// The OpenApi document content. + /// Delegate with document modifications. + internal static MemoryStream ModifyOpenApiDocument(Stream openApiDocument, Action transformer) + { + var json = JsonSerializer.Deserialize(openApiDocument); + + transformer(json!); + + var stream = new MemoryStream(); + + JsonSerializer.Serialize(stream, json); + + stream.Seek(0, SeekOrigin.Begin); + + return stream; + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationResponseConverterTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationResponseConverterTests.cs new file mode 100644 index 000000000000..783969e9d40f --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationResponseConverterTests.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI; +public class RestApiOperationResponseConverterTests +{ + private readonly RestApiOperationResponseConverter _sut; + + public RestApiOperationResponseConverterTests() + { + this._sut = new RestApiOperationResponseConverter(); + } + + [Fact] + public void ItShouldConvertStringContentToString() + { + //Arrange + var response = new RestApiOperationResponse("fake-content", "fake-content-type"); + + //Act + var result = this._sut.ConvertToString(response); + + //Assert + Assert.Equal("fake-content", result); + } + + [Fact] + public void ItShouldConvertByteContentToString() + { + //Arrange + var response = new RestApiOperationResponse(new byte[] { 00, 01, 02 }, "fake-content-type"); + + //Act + var result = this._sut.ConvertToString(response); + + //Assert + Assert.Equal("AAEC", result); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationRunnerTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationRunnerTests.cs new file mode 100644 index 000000000000..4c42db734667 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationRunnerTests.cs @@ -0,0 +1,1055 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Mime; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI; +using Microsoft.SemanticKernel.Functions.OpenAPI.Authentication; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Moq; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI; + +public sealed class RestApiOperationRunnerTests : IDisposable +{ + /// + /// A mock instance of the authentication callback. + /// + private readonly Mock _authenticationHandlerMock; + + /// + /// An instance of HttpMessageHandlerStub class used to get access to various properties of HttpRequestMessage sent by HTTP client. + /// + private readonly HttpMessageHandlerStub _httpMessageHandlerStub; + + /// + /// An instance of HttpClient class used by the tests. + /// + private readonly HttpClient _httpClient; + + /// + /// Creates an instance of a class. + /// + public RestApiOperationRunnerTests() + { + this._authenticationHandlerMock = new Mock(); + + this._httpMessageHandlerStub = new HttpMessageHandlerStub(); + + this._httpClient = new HttpClient(this._httpMessageHandlerStub); + } + + [Fact] + public async Task ItCanRunCreateAndUpdateOperationsWithJsonPayloadSuccessfullyAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload: null + ); + + var payload = new + { + value = "fake-value", + attributes = new + { + enabled = true + } + }; + + var arguments = new Dictionary + { + { "payload", JsonSerializer.Serialize(payload) }, + { "content-type", "application/json" } + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.RequestUri); + Assert.Equal("https://fake-random-test-host/fake-path", this._httpMessageHandlerStub.RequestUri.AbsoluteUri); + + Assert.Equal(HttpMethod.Post, this._httpMessageHandlerStub.Method); + + Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); + Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("application/json; charset=utf-8")); + + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + Assert.True(messageContent.Length != 0); + + var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent)); + Assert.NotNull(deserializedPayload); + + var valueProperty = deserializedPayload["value"]?.ToString(); + Assert.Equal("fake-value", valueProperty); + + var attributesProperty = deserializedPayload["attributes"]; + Assert.NotNull(attributesProperty); + + var enabledProperty = attributesProperty["enabled"]?.AsValue(); + Assert.NotNull(enabledProperty); + Assert.Equal("true", enabledProperty.ToString()); + + Assert.NotNull(result); + + Assert.Equal("fake-content", result.Content); + + Assert.Equal("application/json; charset=utf-8", result.ContentType); + + this._authenticationHandlerMock.Verify(x => x(It.IsAny()), Times.Once); + } + + [Fact] + public async Task ItCanRunCreateAndUpdateOperationsWithPlainTextPayloadSuccessfullyAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Text.Plain); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary + { + { "payload", "fake-input-value" }, + { "content-type", "text/plain"} + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.RequestUri); + Assert.Equal("https://fake-random-test-host/fake-path", this._httpMessageHandlerStub.RequestUri.AbsoluteUri); + + Assert.Equal(HttpMethod.Post, this._httpMessageHandlerStub.Method); + + Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); + Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("text/plain; charset=utf-8")); + + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + Assert.True(messageContent.Length != 0); + + var payloadText = Encoding.UTF8.GetString(messageContent, 0, messageContent.Length); + Assert.Equal("fake-input-value", payloadText); + + Assert.NotNull(result); + + Assert.Equal("fake-content", result.Content); + + Assert.Equal("text/plain; charset=utf-8", result.ContentType); + + this._authenticationHandlerMock.Verify(x => x(It.IsAny()), Times.Once); + } + + [Fact] + public async Task ItShouldAddHeadersToHttpRequestAsync() + { + // Arrange + var headers = new Dictionary + { + { "fake-header", string.Empty } + }; + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Get, + "fake-description", + new List(), + headers + ); + + var arguments = new Dictionary + { + { "fake-header", "fake-header-value" } + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act + await sut.RunAsync(operation, arguments); + + // Assert - 2 headers: 1 from the test and the useragent added internally + Assert.NotNull(this._httpMessageHandlerStub.RequestHeaders); + Assert.Equal(2, this._httpMessageHandlerStub.RequestHeaders.Count()); + + Assert.Contains(this._httpMessageHandlerStub.RequestHeaders, h => h.Key == "fake-header" && h.Value.Contains("fake-header-value")); + } + + [Fact] + public async Task ItShouldAddUserAgentHeaderToHttpRequestIfConfiguredAsync() + { + // Arrange + var headers = new Dictionary + { + { "fake-header", string.Empty } + }; + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Get, + "fake-description", + new List(), + headers + ); + + var arguments = new Dictionary + { + { "fake-header", "fake-header-value" } + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object, "fake-user-agent"); + + // Act + await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.RequestHeaders); + Assert.Equal(2, this._httpMessageHandlerStub.RequestHeaders.Count()); + + Assert.Contains(this._httpMessageHandlerStub.RequestHeaders, h => h.Key == "fake-header" && h.Value.Contains("fake-header-value")); + Assert.Contains(this._httpMessageHandlerStub.RequestHeaders, h => h.Key == "User-Agent" && h.Value.Contains("fake-user-agent")); + } + + [Fact] + public async Task ItShouldBuildJsonPayloadDynamicallyAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + List payloadProperties = new() + { + new("name", "string", true, new List()), + new("attributes", "object", false, new List() + { + new("enabled", "boolean", false, new List()), + }) + }; + + var payload = new RestApiOperationPayload(MediaTypeNames.Application.Json, payloadProperties); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload + ); + + var arguments = new Dictionary(); + arguments.Add("name", "fake-name-value"); + arguments.Add("enabled", "true"); + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object, enableDynamicPayload: true); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); + Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("application/json; charset=utf-8")); + + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + Assert.True(messageContent.Length != 0); + + var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent)); + Assert.NotNull(deserializedPayload); + + var name = deserializedPayload["name"]?.ToString(); + Assert.Equal("fake-name-value", name); + + var attributes = deserializedPayload["attributes"]; + Assert.NotNull(attributes); + + var enabled = attributes["enabled"]?.ToString(); + Assert.NotNull(enabled); + Assert.Equal("true", enabled); + } + + [Fact] + public async Task ItShouldBuildJsonPayloadDynamicallyUsingPayloadMetadataDataTypesAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + List payloadProperties = new() + { + new("name", "string", true, new List()), + new("attributes", "object", false, new List() + { + new("enabled", "boolean", false, new List()), + new("cardinality", "number", false, new List()), + new("coefficient", "number", false, new List()), + new("count", "integer", false, new List()), + new("params", "array", false, new List()), + }) + }; + + var payload = new RestApiOperationPayload(MediaTypeNames.Application.Json, payloadProperties); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload + ); + + var arguments = new Dictionary(); + arguments.Add("name", "fake-string-value"); + arguments.Add("enabled", "true"); + arguments.Add("cardinality", "8"); + arguments.Add("coefficient", "0.8"); + arguments.Add("count", "1"); + arguments.Add("params", "[1,2,3]"); + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object, enableDynamicPayload: true); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + + var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent)); + Assert.NotNull(deserializedPayload); + + var name = deserializedPayload["name"]?.GetValue(); + Assert.NotNull(name); + Assert.Equal(JsonValueKind.String, name.Value.ValueKind); + Assert.Equal("fake-string-value", name.ToString()); + + var attributes = deserializedPayload["attributes"]; + Assert.True(attributes is JsonObject); + + var enabled = attributes["enabled"]?.GetValue(); + Assert.NotNull(enabled); + Assert.Equal(JsonValueKind.True, enabled.Value.ValueKind); + + var cardinality = attributes["cardinality"]?.GetValue(); + Assert.NotNull(cardinality); + Assert.Equal(JsonValueKind.Number, cardinality.Value.ValueKind); + Assert.Equal("8", cardinality.Value.ToString()); + + var coefficient = attributes["coefficient"]?.GetValue(); + Assert.NotNull(coefficient); + Assert.Equal(JsonValueKind.Number, coefficient.Value.ValueKind); + Assert.Equal("0.8", coefficient.Value.ToString()); + + var count = attributes["count"]?.GetValue(); + Assert.NotNull(count); + Assert.Equal(JsonValueKind.Number, coefficient.Value.ValueKind); + Assert.Equal("1", count.Value.ToString()); + + var parameters = attributes["params"]; + Assert.NotNull(parameters); + Assert.True(parameters is JsonArray); + } + + [Fact] + public async Task ItShouldBuildJsonPayloadDynamicallyResolvingArgumentsByFullNamesAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + List payloadProperties = new() + { + new("upn", "string", true, new List()), + new("receiver", "object", false, new List() + { + new("upn", "string", false, new List()), + new("alternative", "object", false, new List() + { + new("upn", "string", false, new List()), + }), + }), + new("cc", "object", false, new List() + { + new("upn", "string", false, new List()), + }) + }; + + var payload = new RestApiOperationPayload(MediaTypeNames.Application.Json, payloadProperties); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload + ); + + var arguments = new Dictionary(); + arguments.Add("upn", "fake-sender-upn"); + arguments.Add("receiver.upn", "fake-receiver-upn"); + arguments.Add("receiver.alternative.upn", "fake-receiver-alternative-upn"); + arguments.Add("cc.upn", "fake-cc-upn"); + + var sut = new RestApiOperationRunner( + this._httpClient, + this._authenticationHandlerMock.Object, + enableDynamicPayload: true, + enablePayloadNamespacing: true); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); + Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("application/json; charset=utf-8")); + + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + Assert.True(messageContent.Length != 0); + + var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent)); + Assert.NotNull(deserializedPayload); + + //Sender props + var senderUpn = deserializedPayload["upn"]?.ToString(); + Assert.Equal("fake-sender-upn", senderUpn); + + //Receiver props + var receiver = deserializedPayload["receiver"]; + Assert.NotNull(receiver); + + var receiverUpn = receiver["upn"]?.AsValue(); + Assert.NotNull(receiverUpn); + Assert.Equal("fake-receiver-upn", receiverUpn.ToString()); + + var alternative = receiver["alternative"]; + Assert.NotNull(alternative); + + var alternativeUpn = alternative["upn"]?.AsValue(); + Assert.NotNull(alternativeUpn); + Assert.Equal("fake-receiver-alternative-upn", alternativeUpn.ToString()); + + //CC props + var carbonCopy = deserializedPayload["cc"]; + Assert.NotNull(carbonCopy); + + var ccUpn = carbonCopy["upn"]?.AsValue(); + Assert.NotNull(ccUpn); + Assert.Equal("fake-cc-upn", ccUpn.ToString()); + } + + [Fact] + public async Task ItShouldThrowExceptionIfPayloadMetadataDoesNotHaveContentTypeAsync() + { + // Arrange + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary(); + + var sut = new RestApiOperationRunner( + this._httpClient, + this._authenticationHandlerMock.Object, + enableDynamicPayload: true); + + // Act + var exception = await Assert.ThrowsAsync(async () => await sut.RunAsync(operation, arguments)); + + Assert.Contains("No content type is provided", exception.Message, StringComparison.InvariantCulture); + } + + [Fact] + public async Task ItShouldThrowExceptionIfContentTypeArgumentIsNotProvidedAsync() + { + // Arrange + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary(); + + var sut = new RestApiOperationRunner( + this._httpClient, + this._authenticationHandlerMock.Object, + enableDynamicPayload: false); + + // Act + var exception = await Assert.ThrowsAsync(async () => await sut.RunAsync(operation, arguments)); + + Assert.Contains("No content type is provided", exception.Message, StringComparison.InvariantCulture); + } + + [Fact] + public async Task ItShouldUsePayloadArgumentForPlainTextContentTypeWhenBuildingPayloadDynamicallyAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Text.Plain); + + var payload = new RestApiOperationPayload(MediaTypeNames.Text.Plain, new List()); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload + ); + + var arguments = new Dictionary + { + { "payload", "fake-input-value" }, + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object, enableDynamicPayload: true); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); + Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("text/plain; charset=utf-8")); + + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + Assert.True(messageContent.Length != 0); + + var payloadText = Encoding.UTF8.GetString(messageContent, 0, messageContent.Length); + Assert.Equal("fake-input-value", payloadText); + } + + [Theory] + [InlineData(MediaTypeNames.Text.Plain)] + [InlineData(MediaTypeNames.Application.Json)] + public async Task ItShouldUsePayloadAndContentTypeArgumentsIfDynamicPayloadBuildingIsNotRequiredAsync(string contentType) + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Text.Plain); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary + { + { "payload", "fake-input-value" }, + { "content-type", $"{contentType}" }, + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object, enableDynamicPayload: false); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); + Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains($"{contentType}; charset=utf-8")); + + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + Assert.True(messageContent.Length != 0); + + var payloadText = Encoding.UTF8.GetString(messageContent, 0, messageContent.Length); + Assert.Equal("fake-input-value", payloadText); + } + + [Fact] + public async Task ItShouldBuildJsonPayloadDynamicallyExcludingOptionalParametersIfTheirArgumentsNotProvidedAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + List payloadProperties = new() + { + new("upn", "string", false, new List()), + }; + + var payload = new RestApiOperationPayload(MediaTypeNames.Application.Json, payloadProperties); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload + ); + + var arguments = new Dictionary(); + + var sut = new RestApiOperationRunner( + this._httpClient, + this._authenticationHandlerMock.Object, + enableDynamicPayload: true, + enablePayloadNamespacing: true); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + Assert.True(messageContent.Length != 0); + + var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent)); + Assert.NotNull(deserializedPayload); + + var senderUpn = deserializedPayload["upn"]?.ToString(); + Assert.Null(senderUpn); + } + + [Fact] + public async Task ItShouldBuildJsonPayloadDynamicallyIncludingOptionalParametersIfTheirArgumentsProvidedAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + List payloadProperties = new() + { + new("upn", "string", false, new List()), + }; + + var payload = new RestApiOperationPayload(MediaTypeNames.Application.Json, payloadProperties); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload + ); + + var arguments = new Dictionary(); + arguments.Add("upn", "fake-sender-upn"); + + var sut = new RestApiOperationRunner( + this._httpClient, + this._authenticationHandlerMock.Object, + enableDynamicPayload: true, + enablePayloadNamespacing: true); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + var messageContent = this._httpMessageHandlerStub.RequestContent; + Assert.NotNull(messageContent); + Assert.True(messageContent.Length != 0); + + var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent)); + Assert.NotNull(deserializedPayload); + + var senderUpn = deserializedPayload["upn"]?.ToString(); + Assert.Equal("fake-sender-upn", senderUpn); + } + + [Fact] + public async Task ItShouldAddRequiredQueryStringParametersIfTheirArgumentsProvidedAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + var firstParameter = new RestApiOperationParameter( + "p1", + "string", + isRequired: true, //Marking the parameter as required + false, + RestApiOperationParameterLocation.Query, + RestApiOperationParameterStyle.Form); + + var secondParameter = new RestApiOperationParameter( + "p2", + "string", + isRequired: true, //Marking the parameter as required + false, + RestApiOperationParameterLocation.Query, + RestApiOperationParameterStyle.Form); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Get, + "fake-description", + new List() { firstParameter, secondParameter }, + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary + { + { "p1", "v1" }, + { "p2", "v2" }, + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.RequestUri); + Assert.Equal("https://fake-random-test-host/fake-path?p1=v1&p2=v2", this._httpMessageHandlerStub.RequestUri.AbsoluteUri); + } + + [Fact] + public async Task ItShouldAddNotRequiredQueryStringParametersIfTheirArgumentsProvidedAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + var firstParameter = new RestApiOperationParameter( + "p1", + "string", + isRequired: false, //Marking the parameter as not required + false, + RestApiOperationParameterLocation.Query, + RestApiOperationParameterStyle.Form); + + var secondParameter = new RestApiOperationParameter( + "p2", + "string", + isRequired: false, //Marking the parameter as not required + false, + RestApiOperationParameterLocation.Query, + RestApiOperationParameterStyle.Form); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Get, + "fake-description", + new List() { firstParameter, secondParameter }, + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary + { + { "p1", "v1" }, + { "p2", "v2" }, + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.RequestUri); + Assert.Equal("https://fake-random-test-host/fake-path?p1=v1&p2=v2", this._httpMessageHandlerStub.RequestUri.AbsoluteUri); + } + + [Fact] + public async Task ItShouldSkipNotRequiredQueryStringParametersIfNoArgumentsProvidedAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + var firstParameter = new RestApiOperationParameter( + "p1", + "string", + isRequired: false, //Marking the parameter as not required + false, + RestApiOperationParameterLocation.Query, + RestApiOperationParameterStyle.Form); + + var secondParameter = new RestApiOperationParameter( + "p2", + "string", + isRequired: true, //Marking the parameter as required + false, + RestApiOperationParameterLocation.Query, + RestApiOperationParameterStyle.Form); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Get, + "fake-description", + new List() { firstParameter, secondParameter }, + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary + { + { "p2", "v2" }, //Providing argument for the required parameter only + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(this._httpMessageHandlerStub.RequestUri); + Assert.Equal("https://fake-random-test-host/fake-path?p2=v2", this._httpMessageHandlerStub.RequestUri.AbsoluteUri); + } + + [Fact] + public async Task ItShouldThrowExceptionIfNoArgumentProvidedForRequiredQueryStringParameterAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); + + var parameter = new RestApiOperationParameter( + "p1", + "string", + isRequired: true, //Marking the parameter as required + false, + RestApiOperationParameterLocation.Query, + RestApiOperationParameterStyle.Form); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Get, + "fake-description", + new List() { parameter }, + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary(); //Providing no arguments + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act and Assert + await Assert.ThrowsAsync(() => sut.RunAsync(operation, arguments)); + } + + [Theory] + [InlineData(MediaTypeNames.Application.Json)] + [InlineData(MediaTypeNames.Application.Xml)] + [InlineData(MediaTypeNames.Text.Plain)] + [InlineData(MediaTypeNames.Text.Html)] + [InlineData(MediaTypeNames.Text.Xml)] + [InlineData("text/csv")] + [InlineData("text/markdown")] + public async Task ItShouldReadContentAsStringSuccessfullyAsync(string contentType) + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, contentType); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary + { + { "payload", JsonSerializer.Serialize(new { value = "fake-value" }) }, + { "content-type", "application/json" } + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal("fake-content", result.Content); + + Assert.Equal($"{contentType}; charset=utf-8", result.ContentType); + } + + [Theory] + [InlineData("image/jpeg")] + [InlineData("image/png")] + [InlineData("image/gif")] + [InlineData("image/svg+xml")] + [InlineData("image/bmp")] + [InlineData("image/x-icon")] + public async Task ItShouldReadContentAsBytesSuccessfullyAsync(string contentType) + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new ByteArrayContent(new byte[] { 00, 01, 02 }); + this._httpMessageHandlerStub.ResponseToReturn.Content.Headers.ContentType = new MediaTypeHeaderValue(contentType); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary + { + { "payload", JsonSerializer.Serialize(new { value = "fake-value" }) }, + { "content-type", "application/json" } + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act + var result = await sut.RunAsync(operation, arguments); + + // Assert + Assert.NotNull(result); + + Assert.Equal(new byte[] { 00, 01, 02 }, result.Content); + + Assert.Equal($"{contentType}", result.ContentType); + } + + [Fact] + public async Task ItShouldThrowExceptionForUnsupportedContentTypeAsync() + { + // Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, "fake/type"); + + var operation = new RestApiOperation( + "fake-id", + new Uri("https://fake-random-test-host"), + "fake-path", + HttpMethod.Post, + "fake-description", + new List(), + new Dictionary(), + payload: null + ); + + var arguments = new Dictionary + { + { "payload", JsonSerializer.Serialize(new { value = "fake-value" }) }, + { "content-type", "application/json" } + }; + + var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => sut.RunAsync(operation, arguments)); + } + + /// + /// Disposes resources used by this class. + /// + public void Dispose() + { + this._httpMessageHandlerStub.Dispose(); + + this._httpClient.Dispose(); + } + + private sealed class HttpMessageHandlerStub : DelegatingHandler + { + public HttpRequestHeaders? RequestHeaders { get; private set; } + + public HttpContentHeaders? ContentHeaders { get; private set; } + + public byte[]? RequestContent { get; private set; } + + public Uri? RequestUri { get; private set; } + + public HttpMethod? Method { get; private set; } + + public HttpResponseMessage ResponseToReturn { get; set; } + + public HttpMessageHandlerStub() + { + this.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent("{}", Encoding.UTF8, MediaTypeNames.Application.Json) + }; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + this.Method = request.Method; + this.RequestUri = request.RequestUri; + this.RequestHeaders = request.Headers; + this.RequestContent = request.Content == null ? null : await request.Content.ReadAsByteArrayAsync(cancellationToken); + this.ContentHeaders = request.Content?.Headers; + + return await Task.FromResult(this.ResponseToReturn); + } + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationTests.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationTests.cs new file mode 100644 index 000000000000..1a3f15c2312c --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/RestApiOperationTests.cs @@ -0,0 +1,331 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Xunit; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI; + +public class RestApiOperationTests +{ + [Fact] + public void ItShouldUseHostUrlIfNoOverrideProvided() + { + // Arrange + var sut = new RestApiOperation( + "fake_id", + new Uri("https://fake-random-test-host"), + "/", + HttpMethod.Get, + "fake_description", + new List(), + new Dictionary() + ); + + var arguments = new Dictionary(); + + // Act + var url = sut.BuildOperationUrl(arguments); + + // Assert + Assert.Equal("https://fake-random-test-host/", url.OriginalString); + } + + [Fact] + public void ItShouldUseHostUrlOverrideIfProvided() + { + // Arrange + var sut = new RestApiOperation( + "fake_id", + new Uri("https://fake-random-test-host"), + "/", + HttpMethod.Get, + "fake_description", + new List(), + new Dictionary() + ); + + var arguments = new Dictionary + { + { "server-url", "https://fake-random-test-host-override" } + }; + + // Act + var url = sut.BuildOperationUrl(arguments); + + // Assert + Assert.Equal("https://fake-random-test-host-override/", url.OriginalString); + } + + [Fact] + public void ItShouldReplacePathParametersByValuesFromArguments() + { + // Arrange + var sut = new RestApiOperation( + "fake_id", + new Uri("https://fake-random-test-host"), + "/{fake-path-parameter}/other_fake_path_section", + HttpMethod.Get, + "fake_description", + new List(), + new Dictionary() + ); + + var arguments = new Dictionary + { + { "fake-path-parameter", "fake-path-value" } + }; + + // Act + var url = sut.BuildOperationUrl(arguments); + + // Assert + Assert.Equal("https://fake-random-test-host/fake-path-value/other_fake_path_section", url.OriginalString); + } + + [Fact] + public void ItShouldReplacePathParametersByDefaultValues() + { + // Arrange + var parameterMetadata = new RestApiOperationParameter( + name: "fake-path-parameter", + type: "fake_type", + isRequired: true, + expand: false, + location: RestApiOperationParameterLocation.Path, + defaultValue: "fake-default-path"); + + var sut = new RestApiOperation( + "fake_id", + new Uri("https://fake-random-test-host"), + "/{fake-path-parameter}/other_fake_path_section", + HttpMethod.Get, + "fake_description", + new List { parameterMetadata }, + new Dictionary()); + + var arguments = new Dictionary(); + + // Act + var url = sut.BuildOperationUrl(arguments); + + // Assert + Assert.Equal("https://fake-random-test-host/fake-default-path/other_fake_path_section", url.OriginalString); + } + + [Fact] + public void ShouldBuildResourceUrlWithoutQueryString() + { + // Arrange + var firstParameterMetadata = new RestApiOperationParameter( + name: "p1", + type: "fake_type", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query, + defaultValue: "dv1"); + + var secondParameterMetadata = new RestApiOperationParameter( + name: "p2", + type: "fake_type", + isRequired: false, + expand: false, + location: RestApiOperationParameterLocation.Query); + + var sut = new RestApiOperation( + "fake_id", + new Uri("https://fake-random-test-host"), + "{fake-path}/", + HttpMethod.Get, + "fake_description", + new List { firstParameterMetadata, secondParameterMetadata }, + new Dictionary()); + + var arguments = new Dictionary + { + { "server-url", "https://fake-random-test-host-override" }, + { "fake-path", "fake-path-value" }, + }; + + // Act + var url = sut.BuildOperationUrl(arguments); + + // Assert + Assert.Equal("https://fake-random-test-host-override/fake-path-value/", url.OriginalString); + } + + [Fact] + public void ItShouldRenderHeaderValuesFromArguments() + { + // Arrange + var rawHeaders = new Dictionary + { + { "fake_header_one", string.Empty }, + { "fake_header_two", string.Empty } + }; + + var arguments = new Dictionary + { + { "fake_header_one", "fake_header_one_value" }, + { "fake_header_two", "fake_header_two_value" } + }; + + var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", new List(), rawHeaders); + + // Act + var headers = sut.RenderHeaders(arguments); + + // Assert + Assert.Equal(2, headers.Count); + + var headerOne = headers["fake_header_one"]; + Assert.Equal("fake_header_one_value", headerOne); + + var headerTwo = headers["fake_header_two"]; + Assert.Equal("fake_header_two_value", headerTwo); + } + + [Fact] + public void ItShouldUseHeaderValuesIfTheyAreAlreadyProvided() + { + // Arrange + var rawHeaders = new Dictionary + { + { "fake_header_one", "fake_header_one_value" }, + { "fake_header_two", "fake_header_two_value" } + }; + + var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", new List(), + rawHeaders); + + // Act + var headers = sut.RenderHeaders(new Dictionary()); + + // Assert + Assert.Equal(2, headers.Count); + + var headerOne = headers["fake_header_one"]; + Assert.Equal("fake_header_one_value", headerOne); + + var headerTwo = headers["fake_header_two"]; + Assert.Equal("fake_header_two_value", headerTwo); + } + + [Fact] + public void ItShouldThrowExceptionIfHeadersHaveNoValuesAndHeadersMetadataNotSupplied() + { + // Arrange + var rawHeaders = new Dictionary + { + { "fake_header_one", string.Empty }, + { "fake_header_two", string.Empty } + }; + + var metadata = new List(); + + var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", metadata, rawHeaders); + + // Act + void Act() => sut.RenderHeaders(new Dictionary()); + + // Assert + Assert.Throws(Act); + } + + [Fact] + public void ShouldThrowExceptionIfNoValueProvidedForRequiredHeader() + { + // Arrange + var rawHeaders = new Dictionary + { + { "fake_header_one", string.Empty }, + { "fake_header_two", string.Empty } + }; + + var metadata = new List + { + new(name: "fake_header_one", type: "string", isRequired: true, expand: false, location: RestApiOperationParameterLocation.Header, style: RestApiOperationParameterStyle.Simple), + new(name: "fake_header_two", type : "string", isRequired : false, expand : false, location : RestApiOperationParameterLocation.Header, style: RestApiOperationParameterStyle.Simple) + }; + + var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", metadata, rawHeaders); + + // Act + void Act() => sut.RenderHeaders(new Dictionary()); + + // Assert + Assert.Throws(Act); + } + + [Fact] + public void ItShouldSkipOptionalHeaderHavingNeitherValueNorDefaultValue() + { + // Arrange + var rawHeaders = new Dictionary + { + { "fake_header_one", string.Empty }, + { "fake_header_two", string.Empty } + }; + + var metadata = new List + { + new RestApiOperationParameter(name: "fake_header_one", type : "string", isRequired : true, expand : false, location : RestApiOperationParameterLocation.Header, style: RestApiOperationParameterStyle.Simple), + new RestApiOperationParameter(name : "fake_header_two", type : "string", isRequired : false, expand : false, location : RestApiOperationParameterLocation.Header, style : RestApiOperationParameterStyle.Simple) + }; + + var arguments = new Dictionary + { + { "fake_header_one", "fake_header_one_value" } + }; + + var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", metadata, rawHeaders); + + // Act + var headers = sut.RenderHeaders(arguments); + + // Assert + Assert.Single(headers); + + var headerOne = headers["fake_header_one"]; + Assert.Equal("fake_header_one_value", headerOne); + } + + [Fact] + public void ShouldUseDefaultValueForOptionalHeaderIfNoValueProvided() + { + // Arrange + var rawHeaders = new Dictionary + { + { "fake_header_one", string.Empty }, + { "fake_header_two", string.Empty } + }; + + var metadata = new List + { + new(name : "fake_header_one", type : "string", isRequired : true, expand : false, location : RestApiOperationParameterLocation.Header, style : RestApiOperationParameterStyle.Simple), + new(name: "fake_header_two", type : "string", isRequired : false, expand : false, location : RestApiOperationParameterLocation.Header, style : RestApiOperationParameterStyle.Simple, defaultValue: "fake_header_two_default_value") + }; + + var arguments = new Dictionary + { + { "fake_header_one", "fake_header_one_value" } //Argument is only provided for the first parameter and not for the second one + }; + + var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", metadata, rawHeaders); + + // Act + var headers = sut.RenderHeaders(arguments); + + // Assert + Assert.Equal(2, headers.Count); + + var headerOne = headers["fake_header_one"]; + Assert.Equal("fake_header_one_value", headerOne); + + var headerTwo = headers["fake_header_two"]; + Assert.Equal("fake_header_two_default_value", headerTwo); + } +} diff --git a/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/ResourcePluginsProvider.cs b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/ResourcePluginsProvider.cs new file mode 100644 index 000000000000..2df0a9421971 --- /dev/null +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/ResourcePluginsProvider.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.IO; +using System.Resources; + +namespace SemanticKernel.Functions.UnitTests.OpenAPI.TestPlugins; + +internal static class ResourcePluginsProvider +{ + /// + /// Loads OpenApi document from assembly resource. + /// + /// The resource name. + /// The OpenApi document resource stream. + public static Stream LoadFromResource(string resourceName) + { + var type = typeof(ResourcePluginsProvider); + + var stream = type.Assembly.GetManifestResourceStream(type, resourceName); + if (stream == null) + { + throw new MissingManifestResourceException($"Unable to load OpenApi plugin from assembly resource '{resourceName}'."); + } + + return stream; + } +} diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV2_0.json b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV2_0.json similarity index 98% rename from dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV2_0.json rename to dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV2_0.json index a251043213da..55709c56583c 100644 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV2_0.json +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV2_0.json @@ -198,9 +198,9 @@ "summary": "Create or update secret value" } }, - "/FunSkill/Excuses": { + "/FunPlugin/Excuses": { "post": { - "description": "Turn a scenario into a creative or humorous excuse to send your boss", + "summary": "Turn a scenario into a creative or humorous excuse to send your boss", "operationId": "Excuses", "consumes": [ "text/plain" diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV3_0.json b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV3_0.json similarity index 84% rename from dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV3_0.json rename to dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV3_0.json index 39350cf93a2f..eac3eace5ee2 100644 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV3_0.json +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV3_0.json @@ -34,6 +34,40 @@ "default": "7.0" }, "x-ms-visibility": "internal" + }, + { + "name": "nonExplodeFormParam", + "in": "query", + "style": "form", + "explode": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "explodeFormParam", + "in": "query", + "style": "form", + "explode": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "anotherExplodeFormParam", + "in": "query", + "schema": { + "type": "array", + "items": { + "type": "integer" + } + } } ], "responses": { @@ -139,9 +173,9 @@ } } }, - "/FunSkill/Excuses": { + "/FunPlugin/Excuses": { "post": { - "description": "Turn a scenario into a creative or humorous excuse to send your boss", + "summary": "Turn a scenario into a creative or humorous excuse to send your boss", "operationId": "Excuses", "requestBody": { "description": "excuse event", diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV3_1.yaml b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV3_1.yaml similarity index 85% rename from dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV3_1.yaml rename to dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV3_1.yaml index c97e72133561..2552d4d348e2 100644 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/documentV3_1.yaml +++ b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/documentV3_1.yaml @@ -24,6 +24,28 @@ paths: type: string default: '7.0' x-ms-visibility: internal + - name: nonExplodeFormParam + in: query + style: form + explode: false + schema: + type: array + items: + type: string + - name: explodeFormParam + in: query + style: form + explode: true + schema: + type: array + items: + type: string + - name: anotherExplodeFormParam + in: query + schema: + type: array + items: + type: integer responses: '200': description: default @@ -93,9 +115,9 @@ paths: responses: '200': description: default - /FunSkill/Excuses: + /FunPlugin/Excuses: post: - description: Turn a scenario into a creative or humorous excuse to send your boss + summary: Turn a scenario into a creative or humorous excuse to send your boss operationId: Excuses requestBody: description: excuse event diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/nonCompliant_documentV3_0.json b/dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/nonCompliant_documentV3_0.json similarity index 100% rename from dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/nonCompliant_documentV3_0.json rename to dotnet/src/Functions/Functions.UnitTests/OpenAPI/TestPlugins/nonCompliant_documentV3_0.json diff --git a/dotnet/src/IntegrationTests/.editorconfig b/dotnet/src/IntegrationTests/.editorconfig index 8f4c52fa9f51..394eef685f21 100644 --- a/dotnet/src/IntegrationTests/.editorconfig +++ b/dotnet/src/IntegrationTests/.editorconfig @@ -2,4 +2,5 @@ [*.cs] dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave - +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/IntegrationTests/Connectors/HuggingFace/TextCompletion/HuggingFaceTextCompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/HuggingFace/TextCompletion/HuggingFaceTextCompletionTests.cs index 0a09b2a055c1..d6f6275b562e 100644 --- a/dotnet/src/IntegrationTests/Connectors/HuggingFace/TextCompletion/HuggingFaceTextCompletionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/HuggingFace/TextCompletion/HuggingFaceTextCompletionTests.cs @@ -40,8 +40,8 @@ public async Task HuggingFaceLocalAndRemoteTextCompletionAsync() var huggingFaceRemote = new HuggingFaceTextCompletion(Model, apiKey: this.GetApiKey()); // Act - var localResponse = await huggingFaceLocal.CompleteAsync(Input, new CompleteRequestSettings()); - var remoteResponse = await huggingFaceRemote.CompleteAsync(Input, new CompleteRequestSettings()); + var localResponse = await huggingFaceLocal.CompleteAsync(Input); + var remoteResponse = await huggingFaceRemote.CompleteAsync(Input); // Assert Assert.NotNull(localResponse); @@ -63,7 +63,7 @@ public async Task RemoteHuggingFaceTextCompletionWithCustomHttpClientAsync() var huggingFaceRemote = new HuggingFaceTextCompletion(Model, apiKey: this.GetApiKey(), httpClient: httpClient); // Act - var remoteResponse = await huggingFaceRemote.CompleteAsync(Input, new CompleteRequestSettings()); + var remoteResponse = await huggingFaceRemote.CompleteAsync(Input); // Assert Assert.NotNull(remoteResponse); diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Chroma/ChromaMemoryStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Chroma/ChromaMemoryStoreTests.cs index 62b43eb1a183..3b7895ddb733 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Chroma/ChromaMemoryStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Chroma/ChromaMemoryStoreTests.cs @@ -5,8 +5,8 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Chroma; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; using Xunit; @@ -56,7 +56,6 @@ public async Task ItCanCreateCollectionsAsync() public async Task ItCanHandleDuplicateNameDuringCollectionCreationAsync() { // Arrange - const int expectedCollectionCount = 1; var collectionName = this.GetRandomCollectionName(); // Act @@ -67,7 +66,7 @@ public async Task ItCanHandleDuplicateNameDuringCollectionCreationAsync() var collections = await this._chromaMemoryStore.GetCollectionsAsync().ToListAsync(); var filteredCollections = collections.Where(collection => collection.Equals(collectionName, StringComparison.Ordinal)).ToList(); - Assert.Equal(expectedCollectionCount, filteredCollections.Count); + Assert.Single(filteredCollections); } [Theory(Skip = SkipReason)] @@ -119,7 +118,7 @@ public async Task ItThrowsExceptionOnNonExistentCollectionDeletionAsync() var exception = await Record.ExceptionAsync(() => this._chromaMemoryStore.DeleteCollectionAsync(collectionName)); // Assert - Assert.IsType(exception); + Assert.IsType(exception); Assert.Contains( $"Cannot delete non-existent collection {collectionName}", exception.Message, @@ -127,7 +126,7 @@ public async Task ItThrowsExceptionOnNonExistentCollectionDeletionAsync() } [Fact(Skip = SkipReason)] - public async Task ItReturnsNullOnNonExistentRecordRetrieval() + public async Task ItReturnsNullOnNonExistentRecordRetrievalAsync() { // Arrange var collectionName = this.GetRandomCollectionName(); @@ -251,11 +250,11 @@ public async Task ItCanGetNearestMatchAsync() // Arrange var collectionName = this.GetRandomCollectionName(); - var expectedRecord1 = this.GetRandomMemoryRecord(embedding: new Embedding(new[] { 10f, 10f, 10f })); - var expectedRecord2 = this.GetRandomMemoryRecord(embedding: new Embedding(new[] { 5f, 5f, 5f })); - var expectedRecord3 = this.GetRandomMemoryRecord(embedding: new Embedding(new[] { 1f, 1f, 1f })); + var expectedRecord1 = this.GetRandomMemoryRecord(embedding: new[] { 10f, 10f, 10f }); + var expectedRecord2 = this.GetRandomMemoryRecord(embedding: new[] { 5f, 5f, 5f }); + var expectedRecord3 = this.GetRandomMemoryRecord(embedding: new[] { 1f, 1f, 1f }); - var searchEmbedding = new Embedding(new[] { 2f, 2f, 2f }); + var searchEmbedding = new[] { 2f, 2f, 2f }; var batch = new List { expectedRecord1, expectedRecord2, expectedRecord3 }; var keys = batch.Select(l => l.Key); @@ -282,11 +281,11 @@ public async Task ItCanGetNearestMatchesAsync() // Arrange var collectionName = this.GetRandomCollectionName(); - var expectedRecord1 = this.GetRandomMemoryRecord(embedding: new Embedding(new[] { 10f, 10f, 10f })); - var expectedRecord2 = this.GetRandomMemoryRecord(embedding: new Embedding(new[] { 5f, 5f, 5f })); - var expectedRecord3 = this.GetRandomMemoryRecord(embedding: new Embedding(new[] { 1f, 1f, 1f })); + var expectedRecord1 = this.GetRandomMemoryRecord(embedding: new[] { 10f, 10f, 10f }); + var expectedRecord2 = this.GetRandomMemoryRecord(embedding: new[] { 5f, 5f, 5f }); + var expectedRecord3 = this.GetRandomMemoryRecord(embedding: new[] { 1f, 1f, 1f }); - var searchEmbedding = new Embedding(new[] { 2f, 2f, 2f }); + var searchEmbedding = new[] { 2f, 2f, 2f }; var batch = new List { expectedRecord1, expectedRecord2, expectedRecord3 }; var keys = batch.Select(l => l.Key); @@ -315,11 +314,11 @@ public async Task ItCanGetNearestMatchesAsync() } [Fact(Skip = SkipReason)] - public async Task ItReturnsNoMatchesFromEmptyCollection() + public async Task ItReturnsNoMatchesFromEmptyCollectionAsync() { // Arrange var collectionName = this.GetRandomCollectionName(); - var searchEmbedding = new Embedding(new[] { 2f, 2f, 2f }); + var searchEmbedding = new[] { 2f, 2f, 2f }; await this._chromaMemoryStore.CreateCollectionAsync(collectionName); @@ -423,7 +422,7 @@ private void Dispose(bool disposing) private void AssertMemoryRecordEqual(MemoryRecord expectedRecord, MemoryRecord actualRecord) { Assert.Equal(expectedRecord.Key, actualRecord.Key); - Assert.Equal(expectedRecord.Embedding.Vector, actualRecord.Embedding.Vector); + Assert.True(expectedRecord.Embedding.Span.SequenceEqual(actualRecord.Embedding.Span)); Assert.Equal(expectedRecord.Metadata.Id, actualRecord.Metadata.Id); Assert.Equal(expectedRecord.Metadata.Text, actualRecord.Metadata.Text); Assert.Equal(expectedRecord.Metadata.Description, actualRecord.Metadata.Description); @@ -437,10 +436,10 @@ private string GetRandomCollectionName() return "sk-test-" + Guid.NewGuid(); } - private MemoryRecord GetRandomMemoryRecord(string? key = null, Embedding? embedding = null) + private MemoryRecord GetRandomMemoryRecord(string? key = null, ReadOnlyMemory? embedding = null) { var recordKey = key ?? Guid.NewGuid().ToString(); - var recordEmbedding = embedding ?? new Embedding(new[] { 1f, 3f, 5f }); + var recordEmbedding = embedding ?? new[] { 1f, 3f, 5f }; return MemoryRecord.LocalRecord( id: recordKey, @@ -451,9 +450,9 @@ private MemoryRecord GetRandomMemoryRecord(string? key = null, Embedding? key: recordKey); } - private MemoryRecord GetRandomMemoryRecord(MemoryRecordMetadata metadata, Embedding? embedding = null) + private MemoryRecord GetRandomMemoryRecord(MemoryRecordMetadata metadata, ReadOnlyMemory? embedding = null) { - var recordEmbedding = embedding ?? new Embedding(new[] { 1f, 3f, 5f }); + var recordEmbedding = embedding ?? new[] { 1f, 3f, 5f }; return MemoryRecord.FromMetadata( metadata: metadata, diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/PostgresMemoryStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/PostgresMemoryStoreTests.cs index 7fb957ec62e6..916e7e44f0c1 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/PostgresMemoryStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/PostgresMemoryStoreTests.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Postgres; using Microsoft.SemanticKernel.Memory; using Npgsql; @@ -64,7 +63,7 @@ public async Task DisposeAsync() [Fact(Skip = SkipReason)] public void InitializeDbConnectionSucceeds() { - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); // Assert Assert.NotNull(memoryStore); } @@ -73,7 +72,7 @@ public void InitializeDbConnectionSucceeds() public async Task ItCanCreateAndGetCollectionAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); string collection = "test_collection"; // Act @@ -89,7 +88,7 @@ public async Task ItCanCreateAndGetCollectionAsync() public async Task ItCanCheckIfCollectionExistsAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); string collection = "my_collection"; // Act @@ -104,7 +103,7 @@ public async Task ItCanCheckIfCollectionExistsAsync() public async Task CollectionsCanBeDeletedAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); string collection = "test_collection"; await memoryStore.CreateCollectionAsync(collection); Assert.True(await memoryStore.DoesCollectionExistAsync(collection)); @@ -120,12 +119,12 @@ public async Task CollectionsCanBeDeletedAsync() public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection"; @@ -139,20 +138,20 @@ public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() // Assert Assert.NotNull(actualDefault); Assert.NotNull(actualWithEmbedding); - Assert.Empty(actualDefault.Embedding.Vector); - Assert.NotEmpty(actualWithEmbedding.Embedding.Vector); + Assert.True(actualDefault.Embedding.IsEmpty); + Assert.False(actualWithEmbedding.Embedding.IsEmpty); } [Fact(Skip = SkipReason)] public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 }), key: null, timestamp: null); string collection = "test_collection"; @@ -166,7 +165,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -177,12 +176,12 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: DateTimeOffset.FromUnixTimeMilliseconds(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())); string collection = "test_collection"; @@ -196,7 +195,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord.Metadata.Id, actual.Key); - Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.True(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); @@ -208,18 +207,18 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() public async Task UpsertReplacesExistingRecordWithSameIdAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); string commonId = "test"; MemoryRecord testRecord = MemoryRecord.LocalRecord( id: commonId, text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); MemoryRecord testRecord2 = MemoryRecord.LocalRecord( id: commonId, text: "text2", description: "description2", - embedding: new Embedding(new float[] { 1, 2, 4 })); + embedding: new float[] { 1, 2, 4 }); string collection = "test_collection"; // Act @@ -232,8 +231,8 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() Assert.NotNull(actual); Assert.Equal(testRecord.Metadata.Id, key); Assert.Equal(testRecord2.Metadata.Id, actual.Key); - Assert.NotEqual(testRecord.Embedding.Vector, actual.Embedding.Vector); - Assert.Equal(testRecord2.Embedding.Vector, actual.Embedding.Vector); + Assert.False(testRecord.Embedding.Span.SequenceEqual(actual.Embedding.Span)); + Assert.True(testRecord2.Embedding.Span.SequenceEqual(actual.Embedding.Span)); Assert.NotEqual(testRecord.Metadata.Text, actual.Metadata.Text); Assert.Equal(testRecord2.Metadata.Description, actual.Metadata.Description); } @@ -242,12 +241,12 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() public async Task ExistingRecordCanBeRemovedAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); MemoryRecord testRecord = MemoryRecord.LocalRecord( id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); string collection = "test_collection"; // Act @@ -266,7 +265,7 @@ public async Task ExistingRecordCanBeRemovedAsync() public async Task RemovingNonExistingRecordDoesNothingAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); string collection = "test_collection"; // Act @@ -282,7 +281,7 @@ public async Task RemovingNonExistingRecordDoesNothingAsync() public async Task ItCanListAllDatabaseCollectionsAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); string[] testCollections = { "random_collection1", "random_collection2", "random_collection3" }; await memoryStore.CreateCollectionAsync(testCollections[0]); await memoryStore.CreateCollectionAsync(testCollections[1]); @@ -312,8 +311,8 @@ public async Task ItCanListAllDatabaseCollectionsAsync() public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; string collection = "test_collection"; await memoryStore.CreateCollectionAsync(collection); @@ -322,7 +321,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -330,7 +329,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -338,7 +337,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -346,7 +345,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -354,7 +353,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await memoryStore.UpsertAsync(collection, testRecord); // Act @@ -374,8 +373,8 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection"; await memoryStore.CreateCollectionAsync(collection); int i = 0; @@ -383,7 +382,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -391,7 +390,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -399,7 +398,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -407,7 +406,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -415,7 +414,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await memoryStore.UpsertAsync(collection, testRecord); // Act @@ -426,16 +425,16 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( // Assert Assert.NotNull(topNResultDefault); Assert.NotNull(topNResultWithEmbedding); - Assert.Empty(topNResultDefault.Value.Item1.Embedding.Vector); - Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.Vector); + Assert.True(topNResultDefault.Value.Item1.Embedding.IsEmpty); + Assert.False(topNResultWithEmbedding.Value.Item1.Embedding.IsEmpty); } [Fact(Skip = SkipReason)] public async Task GetNearestMatchAsyncReturnsExpectedAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection"; await memoryStore.CreateCollectionAsync(collection); int i = 0; @@ -443,7 +442,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -451,7 +450,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -459,7 +458,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -467,7 +466,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await memoryStore.UpsertAsync(collection, testRecord); i++; @@ -475,7 +474,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await memoryStore.UpsertAsync(collection, testRecord); // Act @@ -492,8 +491,8 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; string collection = "test_collection"; await memoryStore.CreateCollectionAsync(collection); @@ -504,7 +503,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await memoryStore.UpsertAsync(collection, testRecord); } @@ -527,7 +526,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() public async Task ItCanBatchUpsertRecordsAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); int numRecords = 10; string collection = "test_collection"; IEnumerable records = this.CreateBatchRecords(numRecords); @@ -547,7 +546,7 @@ public async Task ItCanBatchUpsertRecordsAsync() public async Task ItCanBatchGetRecordsAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); int numRecords = 10; string collection = "test_collection"; IEnumerable records = this.CreateBatchRecords(numRecords); @@ -567,7 +566,7 @@ public async Task ItCanBatchGetRecordsAsync() public async Task ItCanBatchRemoveRecordsAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); int numRecords = 10; string collection = "test_collection"; IEnumerable records = this.CreateBatchRecords(numRecords); @@ -594,7 +593,7 @@ public async Task ItCanBatchRemoveRecordsAsync() public async Task DeletingNonExistentCollectionDoesNothingAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); string collection = "test_collection"; // Act @@ -605,7 +604,7 @@ public async Task DeletingNonExistentCollectionDoesNothingAsync() public async Task ItCanBatchGetRecordsAndSkipIfKeysDoNotExistAsync() { // Arrange - PostgresMemoryStore memoryStore = this.CreateMemoryStore(); + using PostgresMemoryStore memoryStore = this.CreateMemoryStore(); int numRecords = 10; string collection = "test_collection"; IEnumerable records = this.CreateBatchRecords(numRecords); @@ -682,7 +681,7 @@ private IEnumerable CreateBatchRecords(int numRecords) id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); records = records.Append(testRecord); } @@ -692,7 +691,7 @@ private IEnumerable CreateBatchRecords(int numRecords) externalId: "test" + i, sourceName: "sourceName" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); records = records.Append(testRecord); } diff --git a/dotnet/src/IntegrationTests/Connectors/Milvus/MilvusMemoryStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Milvus/MilvusMemoryStoreTests.cs new file mode 100644 index 000000000000..db4c1d121c03 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Milvus/MilvusMemoryStoreTests.cs @@ -0,0 +1,308 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.Memory.Milvus; +using Microsoft.SemanticKernel.Memory; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Milvus; + +public class MilvusMemoryStoreTests : IAsyncLifetime +{ + private const string MilvusHost = "127.0.0.1"; + private const int MilvusPort = 19530; + + // If null, all tests will be enabled + private const string SkipReason = "Requires Milvus up and running"; + + private const string CollectionName = "test"; + private MilvusMemoryStore Store { get; set; } = new(MilvusHost, vectorSize: 5, port: MilvusPort); + + [Fact(Skip = SkipReason)] + public async Task CreateCollectionAsync() + { + Assert.False(await this.Store.DoesCollectionExistAsync(CollectionName)); + + await this.Store.CreateCollectionAsync(CollectionName); + Assert.True(await this.Store.DoesCollectionExistAsync(CollectionName)); + } + + [Fact(Skip = SkipReason)] + public async Task DropCollectionAsync() + { + await this.Store.CreateCollectionAsync(CollectionName); + await this.Store.DeleteCollectionAsync(CollectionName); + Assert.False(await this.Store.DoesCollectionExistAsync(CollectionName)); + } + + [Fact(Skip = SkipReason)] + public async Task GetCollectionsAsync() + { + await this.Store.CreateCollectionAsync("collection1"); + await this.Store.CreateCollectionAsync("collection2"); + + List collections = this.Store.GetCollectionsAsync().ToEnumerable().ToList(); + Assert.Contains("collection1", collections); + Assert.Contains("collection2", collections); + } + + [Fact(Skip = SkipReason)] + public async Task UpsertAsync() + { + await this.Store.CreateCollectionAsync(CollectionName); + + string id = await this.Store.UpsertAsync(CollectionName, new MemoryRecord( + new MemoryRecordMetadata( + isReference: true, + id: "Some id", + description: "Some description", + text: "Some text", + externalSourceName: "Some external resource name", + additionalMetadata: "Some additional metadata"), + new[] { 10f, 11f, 12f, 13f, 14f }, + key: "Some key", + timestamp: new DateTimeOffset(2023, 1, 1, 12, 0, 0, TimeSpan.Zero))); + + Assert.Equal("Some id", id); + } + + [Theory(Skip = SkipReason)] + [InlineData(true)] + [InlineData(false)] + public async Task GetAsync(bool withEmbeddings) + { + await this.Store.CreateCollectionAsync(CollectionName); + await this.InsertSampleDataAsync(); + + MemoryRecord? record = await this.Store.GetAsync(CollectionName, "Some id", withEmbedding: withEmbeddings); + Assert.NotNull(record); + + Assert.True(record.Metadata.IsReference); + Assert.Equal("Some id", record.Metadata.Id); + Assert.Equal("Some description", record.Metadata.Description); + Assert.Equal("Some text", record.Metadata.Text); + Assert.Equal("Some external resource name", record.Metadata.ExternalSourceName); + Assert.Equal("Some additional metadata", record.Metadata.AdditionalMetadata); + Assert.Equal("Some key", record.Key); + Assert.Equal(new DateTimeOffset(2023, 1, 1, 12, 0, 0, TimeSpan.Zero), record.Timestamp); + + Assert.Equal( + withEmbeddings ? new[] { 10f, 11f, 12f, 13f, 14f } : Array.Empty(), + record.Embedding.ToArray()); + } + + [Fact(Skip = SkipReason)] + public async Task UpsertBatchAsync() + { + await this.Store.CreateCollectionAsync(CollectionName); + List ids = await this.InsertSampleDataAsync(); + + Assert.Collection(ids, + id => Assert.Equal("Some id", id), + id => Assert.Equal("Some other id", id)); + } + + [Theory(Skip = SkipReason)] + [InlineData(true)] + [InlineData(false)] + public async Task GetBatchAsync(bool withEmbeddings) + { + await this.Store.CreateCollectionAsync(CollectionName); + await this.InsertSampleDataAsync(); + + List records = this.Store.GetBatchAsync(CollectionName, new[] { "Some id", "Some other id" }, withEmbeddings: withEmbeddings).ToEnumerable().ToList(); + + Assert.Collection(records.OrderBy(r => r.Metadata.Id), + r => + { + Assert.True(r.Metadata.IsReference); + Assert.Equal("Some id", r.Metadata.Id); + Assert.Equal("Some description", r.Metadata.Description); + Assert.Equal("Some text", r.Metadata.Text); + Assert.Equal("Some external resource name", r.Metadata.ExternalSourceName); + Assert.Equal("Some additional metadata", r.Metadata.AdditionalMetadata); + Assert.Equal("Some key", r.Key); + Assert.Equal(new DateTimeOffset(2023, 1, 1, 12, 0, 0, TimeSpan.Zero), r.Timestamp); + + Assert.Equal( + withEmbeddings ? new[] { 10f, 11f, 12f, 13f, 14f } : Array.Empty(), + r.Embedding.ToArray()); + }, + r => + { + Assert.False(r.Metadata.IsReference); + Assert.Equal("Some other id", r.Metadata.Id); + Assert.Empty(r.Metadata.Description); + Assert.Empty(r.Metadata.Text); + Assert.Empty(r.Metadata.ExternalSourceName); + Assert.Empty(r.Metadata.AdditionalMetadata); + Assert.Empty(r.Key); + Assert.Null(r.Timestamp); + + Assert.Equal( + withEmbeddings ? new[] { 20f, 21f, 22f, 23f, 24f } : Array.Empty(), + r.Embedding.ToArray()); + }); + } + + [Fact(Skip = SkipReason)] + public async Task RemoveAsync() + { + await this.Store.CreateCollectionAsync(CollectionName); + await this.InsertSampleDataAsync(); + + Assert.NotNull(await this.Store.GetAsync(CollectionName, "Some id")); + await this.Store.RemoveAsync(CollectionName, "Some id"); + Assert.Null(await this.Store.GetAsync(CollectionName, "Some id")); + } + + [Fact(Skip = SkipReason)] + public async Task RemoveBatchAsync() + { + await this.Store.CreateCollectionAsync(CollectionName); + await this.InsertSampleDataAsync(); + + Assert.NotNull(await this.Store.GetAsync(CollectionName, "Some id")); + Assert.NotNull(await this.Store.GetAsync(CollectionName, "Some other id")); + await this.Store.RemoveBatchAsync(CollectionName, new[] { "Some id", "Some other id" }); + Assert.Null(await this.Store.GetAsync(CollectionName, "Some id")); + Assert.Null(await this.Store.GetAsync(CollectionName, "Some other id")); + } + + [Theory(Skip = SkipReason)] + [InlineData(true)] + [InlineData(false)] + public async Task GetNearestMatchesAsync(bool withEmbeddings) + { + await this.Store.CreateCollectionAsync(CollectionName); + await this.InsertSampleDataAsync(); + + // There seems to be some race condition where the upserted data above isn't taken into account in the search below and zero results are returned... + await Task.Delay(1000); + + List<(MemoryRecord Record, double SimilarityScore)> results = + this.Store.GetNearestMatchesAsync(CollectionName, new[] { 5f, 6f, 7f, 8f, 9f }, limit: 2, withEmbeddings: withEmbeddings).ToEnumerable().ToList(); + + Assert.All(results, t => Assert.True(t.SimilarityScore > 0)); + + Assert.Collection(results.OrderBy(r => r.SimilarityScore).Select(r => r.Record), + r => + { + Assert.True(r.Metadata.IsReference); + Assert.Equal("Some id", r.Metadata.Id); + Assert.Equal("Some description", r.Metadata.Description); + Assert.Equal("Some text", r.Metadata.Text); + Assert.Equal("Some external resource name", r.Metadata.ExternalSourceName); + Assert.Equal("Some additional metadata", r.Metadata.AdditionalMetadata); + Assert.Equal("Some key", r.Key); + Assert.Equal(new DateTimeOffset(2023, 1, 1, 12, 0, 0, TimeSpan.Zero), r.Timestamp); + + Assert.Equal( + withEmbeddings ? new[] { 10f, 11f, 12f, 13f, 14f } : Array.Empty(), + r.Embedding.ToArray()); + }, + r => + { + Assert.False(r.Metadata.IsReference); + Assert.Equal("Some other id", r.Metadata.Id); + Assert.Empty(r.Metadata.Description); + Assert.Empty(r.Metadata.Text); + Assert.Empty(r.Metadata.ExternalSourceName); + Assert.Empty(r.Metadata.AdditionalMetadata); + Assert.Empty(r.Key); + Assert.Null(r.Timestamp); + + Assert.Equal( + withEmbeddings ? new[] { 20f, 21f, 22f, 23f, 24f } : Array.Empty(), + r.Embedding.ToArray()); + }); + } + + [Fact(Skip = SkipReason)] + public async Task GetNearestMatchesWithMinRelevanceScoreAsync() + { + await this.Store.CreateCollectionAsync(CollectionName); + await this.InsertSampleDataAsync(); + + List<(MemoryRecord Record, double SimilarityScore)> results = + this.Store.GetNearestMatchesAsync(CollectionName, new[] { 5f, 6f, 7f, 8f, 9f }, limit: 2).ToEnumerable().ToList(); + + string firstId = results[0].Record.Metadata.Id; + double firstSimilarityScore = results[0].SimilarityScore; + + results = this.Store.GetNearestMatchesAsync(CollectionName, new[] { 5f, 6f, 7f, 8f, 9f }, limit: 2, minRelevanceScore: firstSimilarityScore + 0.0001).ToEnumerable().ToList(); + + Assert.DoesNotContain(firstId, results.Select(r => r.Record.Metadata.Id)); + } + + [Theory(Skip = SkipReason)] + [InlineData(true)] + [InlineData(false)] + public async Task GetNearestMatchAsync(bool withEmbeddings) + { + await this.Store.CreateCollectionAsync(CollectionName); + await this.InsertSampleDataAsync(); + + (MemoryRecord Record, double SimilarityScore)? result = + await this.Store.GetNearestMatchAsync(CollectionName, new[] { 20f, 21f, 22f, 23f, 24f }, withEmbedding: withEmbeddings); + + Assert.NotNull(result); + Assert.True(result.Value.SimilarityScore > 0); + MemoryRecord record = result.Value.Record; + + Assert.Equal("Some other id", record.Metadata.Id); + Assert.Equal( + withEmbeddings ? new[] { 20f, 21f, 22f, 23f, 24f } : Array.Empty(), + record.Embedding.ToArray()); + } + + private async Task> InsertSampleDataAsync() + { + IAsyncEnumerable ids = this.Store.UpsertBatchAsync(CollectionName, new[] + { + new MemoryRecord( + new MemoryRecordMetadata( + isReference: true, + id: "Some id", + description: "Some description", + text: "Some text", + externalSourceName: "Some external resource name", + additionalMetadata: "Some additional metadata"), + new[] { 10f, 11f, 12f, 13f, 14f }, + key: "Some key", + timestamp: new DateTimeOffset(2023, 1, 1, 12, 0, 0, TimeSpan.Zero)), + new MemoryRecord( + new MemoryRecordMetadata( + isReference: false, + id: "Some other id", + description: "", + text: "", + externalSourceName: "", + additionalMetadata: ""), + new[] { 20f, 21f, 22f, 23f, 24f }, + key: null, + timestamp: null), + }); + + List idList = new(); + + await foreach (string id in ids) + { + idList.Add(id); + } + + return idList; + } + + public async Task InitializeAsync() + => await this.Store.DeleteCollectionAsync(CollectionName); + + public Task DisposeAsync() + { + this.Store.Dispose(); + return Task.CompletedTask; + } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Oobabooga/OobaboogaTextCompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/Oobabooga/OobaboogaTextCompletionTests.cs index 78d98dafc1ba..b094f4e449bd 100644 --- a/dotnet/src/IntegrationTests/Connectors/Oobabooga/OobaboogaTextCompletionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Oobabooga/OobaboogaTextCompletionTests.cs @@ -16,6 +16,7 @@ namespace SemanticKernel.IntegrationTests.Connectors.Oobabooga; /// /// Integration tests for . /// +[Obsolete("This functionality is available as part of new NuGet package: https://www.nuget.org/packages/MyIA.SemanticKernel.Connectors.AI.Oobabooga/. This will be removed in a future release.")] public sealed class OobaboogaTextCompletionTests : IDisposable { private const string Endpoint = "http://localhost"; @@ -23,8 +24,8 @@ public sealed class OobaboogaTextCompletionTests : IDisposable private const int StreamingPort = 5005; private readonly IConfigurationRoot _configuration; - private List _webSockets = new(); - private Func _webSocketFactory; + private readonly List _webSockets = new(); + private readonly Func _webSocketFactory; public OobaboogaTextCompletionTests() { @@ -52,10 +53,10 @@ public async Task OobaboogaLocalTextCompletionAsync() blockingPort: BlockingPort); // Act - var localResponse = await oobaboogaLocal.CompleteAsync(Input, new CompleteRequestSettings() + var localResponse = await oobaboogaLocal.CompleteAsync(Input, requestSettings: new TextCompletionRequest() { Temperature = 0.01, - MaxTokens = 7, + MaxNewTokens = 7, TopP = 0.1, }); @@ -71,10 +72,10 @@ public async Task OobaboogaLocalTextCompletionStreamingAsync() webSocketFactory: this._webSocketFactory); // Act - var localResponse = oobaboogaLocal.CompleteStreamAsync(Input, new CompleteRequestSettings() + var localResponse = oobaboogaLocal.CompleteStreamAsync(Input, requestSettings: new TextCompletionRequest() { Temperature = 0.01, - MaxTokens = 7, + MaxNewTokens = 7, TopP = 0.1, }); diff --git a/dotnet/src/IntegrationTests/Connectors/OpenAI/AzureOpenAICompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/OpenAI/AzureOpenAICompletionTests.cs new file mode 100644 index 000000000000..21eff7a0f286 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/OpenAI/AzureOpenAICompletionTests.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Azure; +using Azure.AI.OpenAI; +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability.Basic; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Connectors.OpenAI; + +public sealed class AzureOpenAICompletionTests : IDisposable +{ + private readonly IConfigurationRoot _configuration; + private readonly XunitLogger _logger; + private readonly RedirectOutput _testOutputHelper; + + public AzureOpenAICompletionTests(ITestOutputHelper output) + { + this._logger = new XunitLogger(output); + this._testOutputHelper = new RedirectOutput(output); + Console.SetOut(this._testOutputHelper); + + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Theory] + [InlineData("Where is the most famous fish market in Seattle, Washington, USA?")] + public async Task AzureOpenAIChatNoHttpRetryPolicyTestShouldThrowAsync(string prompt) + { + // Arrange + var configuration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(configuration); + + var httpRetryConfig = new BasicRetryConfig { MaxRetryCount = 0 }; + BasicHttpRetryHandlerFactory defaultHttpRetryHandlerFactory = new(httpRetryConfig); + + var target = new KernelBuilder() + .WithLoggerFactory(this._logger) + .WithAzureChatCompletionService(configuration.ChatDeploymentName!, configuration.Endpoint, configuration.ApiKey) + .WithHttpHandlerFactory(defaultHttpRetryHandlerFactory) + .Build(); + + // Act + var func = target.CreateSemanticFunction(prompt); + + var exception = await Assert.ThrowsAsync(() => func.InvokeAsync(string.Empty, target, requestSettings: new OpenAIRequestSettings() { MaxTokens = 1000000, Temperature = 0.5, TopP = 0.5 })); + + // Assert + Assert.NotNull(exception); + } + + [Theory] + [InlineData("Where is the most famous fish market in Seattle, Washington, USA?")] + public async Task AzureOpenAIChatNoHttpRetryPolicyCustomClientShouldThrowAsync(string prompt) + { + // Arrange + var configuration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(configuration); + + var clientOptions = new OpenAIClientOptions(); + clientOptions.Retry.MaxRetries = 0; + clientOptions.Retry.NetworkTimeout = TimeSpan.FromSeconds(10); + + var openAIClient = new OpenAIClient(new Uri(configuration.Endpoint), new AzureKeyCredential(configuration.ApiKey), clientOptions); + + var target = new KernelBuilder() + .WithLoggerFactory(this._logger) + .WithAzureChatCompletionService(configuration.ChatDeploymentName!, openAIClient) + .Build(); + + // Act + var func = target.CreateSemanticFunction(prompt); + + var exception = await Assert.ThrowsAsync(() => func.InvokeAsync(string.Empty, target, requestSettings: new OpenAIRequestSettings() { MaxTokens = 1000000, Temperature = 0.5, TopP = 0.5 })); + + // Assert + Assert.NotNull(exception); + } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + ~AzureOpenAICompletionTests() + { + this.Dispose(false); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + this._logger.Dispose(); + this._testOutputHelper.Dispose(); + } + } +} diff --git a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAICompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAICompletionTests.cs index 11ffe6adaadf..f0a6f607f565 100644 --- a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAICompletionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAICompletionTests.cs @@ -7,10 +7,11 @@ using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Reliability; -using Microsoft.SemanticKernel.SkillDefinition; +using Microsoft.SemanticKernel.Reliability.Basic; +using Microsoft.SemanticKernel.TemplateEngine; using SemanticKernel.IntegrationTests.TestSettings; using Xunit; using Xunit.Abstractions; @@ -21,6 +22,7 @@ namespace SemanticKernel.IntegrationTests.Connectors.OpenAI; public sealed class OpenAICompletionTests : IDisposable { + private readonly KernelBuilder _kernelBuilder; private readonly IConfigurationRoot _configuration; public OpenAICompletionTests(ITestOutputHelper output) @@ -36,6 +38,9 @@ public OpenAICompletionTests(ITestOutputHelper output) .AddEnvironmentVariables() .AddUserSecrets() .Build(); + + this._kernelBuilder = new KernelBuilder(); + this._kernelBuilder.WithRetryBasic(); } [Theory(Skip = "OpenAI will often throttle requests. This test is for manual verification.")] @@ -46,8 +51,8 @@ public async Task OpenAITestAsync(string prompt, string expectedAnswerContains) var openAIConfiguration = this._configuration.GetSection("OpenAI").Get(); Assert.NotNull(openAIConfiguration); - IKernel target = Kernel.Builder - .WithLogger(this._logger) + IKernel target = this._kernelBuilder + .WithLoggerFactory(this._logger) .WithOpenAITextCompletionService( serviceId: openAIConfiguration.ServiceId, modelId: openAIConfiguration.ModelId, @@ -55,13 +60,13 @@ public async Task OpenAITestAsync(string prompt, string expectedAnswerContains) setAsDefault: true) .Build(); - IDictionary skill = TestHelpers.GetSkills(target, "ChatSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "ChatPlugin"); // Act - SKContext actual = await target.RunAsync(prompt, skill["Chat"]); + KernelResult actual = await target.RunAsync(prompt, plugins["Chat"]); // Assert - Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains(expectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } [Theory(Skip = "OpenAI will often throttle requests. This test is for manual verification.")] @@ -69,39 +74,39 @@ public async Task OpenAITestAsync(string prompt, string expectedAnswerContains) public async Task OpenAIChatAsTextTestAsync(string prompt, string expectedAnswerContains) { // Arrange - KernelBuilder builder = Kernel.Builder.WithLogger(this._logger); + KernelBuilder builder = this._kernelBuilder.WithLoggerFactory(this._logger); this.ConfigureChatOpenAI(builder); IKernel target = builder.Build(); - IDictionary skill = TestHelpers.GetSkills(target, "ChatSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "ChatPlugin"); // Act - SKContext actual = await target.RunAsync(prompt, skill["Chat"]); + KernelResult actual = await target.RunAsync(prompt, plugins["Chat"]); // Assert - Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains(expectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } [Fact(Skip = "Skipping while we investigate issue with GitHub actions.")] public async Task CanUseOpenAiChatForTextCompletionAsync() { // Note: we use OpenAi Chat Completion and GPT 3.5 Turbo - KernelBuilder builder = Kernel.Builder.WithLogger(this._logger); + KernelBuilder builder = this._kernelBuilder.WithLoggerFactory(this._logger); this.ConfigureChatOpenAI(builder); IKernel target = builder.Build(); var func = target.CreateSemanticFunction( - "List the two planets after '{{$input}}', excluding moons, using bullet points."); + "List the two planets after '{{$input}}', excluding moons, using bullet points.", + new OpenAIRequestSettings()); - var result = await func.InvokeAsync("Jupiter"); + var result = await func.InvokeAsync("Jupiter", target); Assert.NotNull(result); - Assert.False(result.ErrorOccurred, result.LastErrorDescription); - Assert.Contains("Saturn", result.Result, StringComparison.InvariantCultureIgnoreCase); - Assert.Contains("Uranus", result.Result, StringComparison.InvariantCultureIgnoreCase); + Assert.Contains("Saturn", result.GetValue(), StringComparison.InvariantCultureIgnoreCase); + Assert.Contains("Uranus", result.GetValue(), StringComparison.InvariantCultureIgnoreCase); } [Theory] @@ -110,11 +115,7 @@ public async Task CanUseOpenAiChatForTextCompletionAsync() public async Task AzureOpenAITestAsync(bool useChatModel, string prompt, string expectedAnswerContains) { // Arrange - - var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - var builder = Kernel.Builder.WithLogger(this._logger); + var builder = this._kernelBuilder.WithLoggerFactory(this._logger); if (useChatModel) { @@ -127,15 +128,13 @@ public async Task AzureOpenAITestAsync(bool useChatModel, string prompt, string IKernel target = builder.Build(); - IDictionary skill = TestHelpers.GetSkills(target, "ChatSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "ChatPlugin"); // Act - SKContext actual = await target.RunAsync(prompt, skill["Chat"]); + KernelResult actual = await target.RunAsync(prompt, plugins["Chat"]); // Assert - Assert.Empty(actual.LastErrorDescription); - Assert.False(actual.ErrorOccurred); - Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains(expectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } // If the test fails, please note that SK retry logic may not be fully integrated into the underlying code using Azure SDK @@ -145,25 +144,25 @@ public async Task AzureOpenAITestAsync(bool useChatModel, string prompt, string public async Task OpenAIHttpRetryPolicyTestAsync(string prompt, string expectedOutput) { // Arrange - var retryConfig = new HttpRetryConfig(); + var retryConfig = new BasicRetryConfig(); retryConfig.RetryableStatusCodes.Add(HttpStatusCode.Unauthorized); OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAI").Get(); Assert.NotNull(openAIConfiguration); - IKernel target = Kernel.Builder - .WithLogger(this._testOutputHelper) - .Configure(c => c.SetDefaultHttpRetryConfig(retryConfig)) + IKernel target = this._kernelBuilder + .WithLoggerFactory(this._testOutputHelper) + .WithRetryBasic(retryConfig) .WithOpenAITextCompletionService( serviceId: openAIConfiguration.ServiceId, modelId: openAIConfiguration.ModelId, apiKey: "INVALID_KEY") // Use an invalid API key to force a 401 Unauthorized response .Build(); - IDictionary skill = TestHelpers.GetSkills(target, "SummarizeSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); // Act - var context = await target.RunAsync(prompt, skill["Summarize"]); + await Assert.ThrowsAsync(() => target.RunAsync(prompt, plugins["Summarize"])); // Assert Assert.Contains(expectedOutput, this._testOutputHelper.GetLogs(), StringComparison.OrdinalIgnoreCase); @@ -176,11 +175,12 @@ public async Task OpenAIHttpRetryPolicyTestAsync(string prompt, string expectedO public async Task AzureOpenAIHttpRetryPolicyTestAsync(string prompt, string expectedOutput) { // Arrange - var retryConfig = new HttpRetryConfig(); + var retryConfig = new BasicRetryConfig(); retryConfig.RetryableStatusCodes.Add(HttpStatusCode.Unauthorized); - KernelBuilder builder = Kernel.Builder - .WithLogger(this._testOutputHelper) - .Configure(c => c.SetDefaultHttpRetryConfig(retryConfig)); + + KernelBuilder builder = this._kernelBuilder + .WithLoggerFactory(this._testOutputHelper) + .WithRetryBasic(retryConfig); var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); Assert.NotNull(azureOpenAIConfiguration); @@ -193,10 +193,10 @@ public async Task AzureOpenAIHttpRetryPolicyTestAsync(string prompt, string expe IKernel target = builder.Build(); - IDictionary skill = TestHelpers.GetSkills(target, "SummarizeSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); // Act - var context = await target.RunAsync(prompt, skill["Summarize"]); + await Assert.ThrowsAsync(() => target.RunAsync(prompt, plugins["Summarize"])); // Assert Assert.Contains(expectedOutput, this._testOutputHelper.GetLogs(), StringComparison.OrdinalIgnoreCase); @@ -210,23 +210,19 @@ public async Task OpenAIHttpInvalidKeyShouldReturnErrorDetailAsync() Assert.NotNull(openAIConfiguration); // Use an invalid API key to force a 401 Unauthorized response - IKernel target = Kernel.Builder + IKernel target = this._kernelBuilder .WithOpenAITextCompletionService( modelId: openAIConfiguration.ModelId, apiKey: "INVALID_KEY", serviceId: openAIConfiguration.ServiceId) .Build(); - IDictionary skill = TestHelpers.GetSkills(target, "SummarizeSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); - // Act - var context = await target.RunAsync("Any", skill["Summarize"]); + // Act and Assert + var ex = await Assert.ThrowsAsync(() => target.RunAsync("Any", plugins["Summarize"])); - // Assert - Assert.True(context.ErrorOccurred); - Assert.IsType(context.LastException); - Assert.Equal(AIException.ErrorCodes.AccessDenied, ((AIException)context.LastException).ErrorCode); - Assert.Contains("The request is not authorized, HTTP status: 401", ((AIException)context.LastException).Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(HttpStatusCode.Unauthorized, ((HttpOperationException)ex).StatusCode); } [Fact] @@ -236,8 +232,8 @@ public async Task AzureOpenAIHttpInvalidKeyShouldReturnErrorDetailAsync() var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); Assert.NotNull(azureOpenAIConfiguration); - IKernel target = Kernel.Builder - .WithLogger(this._testOutputHelper) + IKernel target = this._kernelBuilder + .WithLoggerFactory(this._testOutputHelper) .WithAzureTextCompletionService( deploymentName: azureOpenAIConfiguration.DeploymentName, endpoint: azureOpenAIConfiguration.Endpoint, @@ -245,16 +241,12 @@ public async Task AzureOpenAIHttpInvalidKeyShouldReturnErrorDetailAsync() serviceId: azureOpenAIConfiguration.ServiceId) .Build(); - IDictionary skill = TestHelpers.GetSkills(target, "SummarizeSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); - // Act - var context = await target.RunAsync("Any", skill["Summarize"]); + // Act and Assert + var ex = await Assert.ThrowsAsync(() => target.RunAsync("Any", plugins["Summarize"])); - // Assert - Assert.True(context.ErrorOccurred); - Assert.IsType(context.LastException); - Assert.Equal(AIException.ErrorCodes.AccessDenied, ((AIException)context.LastException).ErrorCode); - Assert.Contains("The request is not authorized, HTTP status: 401", ((AIException)context.LastException).Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(HttpStatusCode.Unauthorized, ((HttpOperationException)ex).StatusCode); } [Fact] @@ -264,8 +256,8 @@ public async Task AzureOpenAIHttpExceededMaxTokensShouldReturnErrorDetailAsync() Assert.NotNull(azureOpenAIConfiguration); // Arrange - IKernel target = Kernel.Builder - .WithLogger(this._testOutputHelper) + IKernel target = this._kernelBuilder + .WithLoggerFactory(this._testOutputHelper) .WithAzureTextCompletionService( deploymentName: azureOpenAIConfiguration.DeploymentName, endpoint: azureOpenAIConfiguration.Endpoint, @@ -273,17 +265,11 @@ public async Task AzureOpenAIHttpExceededMaxTokensShouldReturnErrorDetailAsync() serviceId: azureOpenAIConfiguration.ServiceId) .Build(); - IDictionary skill = TestHelpers.GetSkills(target, "SummarizeSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); // Act - var context = await skill["Summarize"].InvokeAsync(string.Join('.', Enumerable.Range(1, 40000))); - // Assert - Assert.True(context.ErrorOccurred); - Assert.IsType(context.LastException); - Assert.Equal(AIException.ErrorCodes.InvalidRequest, ((AIException)context.LastException).ErrorCode); - Assert.Contains("The request is not valid, HTTP status: 400", ((AIException)context.LastException).Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("maximum context length is", ((AIException)context.LastException).Detail, StringComparison.OrdinalIgnoreCase); // This message could change in the future, comes from Azure OpenAI + await Assert.ThrowsAsync(() => plugins["Summarize"].InvokeAsync(string.Join('.', Enumerable.Range(1, 40000)), target)); } [Theory(Skip = "This test is for manual verification.")] @@ -301,61 +287,90 @@ public async Task CompletionWithDifferentLineEndingsAsync(string lineEnding, AIS const string ExpectedAnswerContains = "John"; - IKernel target = Kernel.Builder.WithLogger(this._logger).Build(); + IKernel target = this._kernelBuilder.WithLoggerFactory(this._logger).Build(); this._serviceConfiguration[service](target); - IDictionary skill = TestHelpers.GetSkills(target, "ChatSkill"); + IDictionary plugins = TestHelpers.ImportSamplePlugins(target, "ChatPlugin"); // Act - SKContext actual = await target.RunAsync(prompt, skill["Chat"]); + KernelResult actual = await target.RunAsync(prompt, plugins["Chat"]); // Assert - Assert.Contains(ExpectedAnswerContains, actual.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains(ExpectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } [Fact] public async Task AzureOpenAIInvokePromptTestAsync() { // Arrange - var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - var builder = Kernel.Builder.WithLogger(this._logger); + var builder = this._kernelBuilder.WithLoggerFactory(this._logger); this.ConfigureAzureOpenAI(builder); IKernel target = builder.Build(); var prompt = "Where is the most famous fish market in Seattle, Washington, USA?"; // Act - SKContext actual = await target.InvokeSemanticFunctionAsync(prompt, maxTokens: 150); + KernelResult actual = await target.InvokeSemanticFunctionAsync(prompt, new OpenAIRequestSettings() { MaxTokens = 150 }); // Assert - Assert.Empty(actual.LastErrorDescription); - Assert.False(actual.ErrorOccurred); - Assert.Contains("Pike Place", actual.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Pike Place", actual.GetValue(), StringComparison.OrdinalIgnoreCase); } [Fact] public async Task AzureOpenAIDefaultValueTestAsync() { // Arrange - var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); + var builder = this._kernelBuilder.WithLoggerFactory(this._logger); + this.ConfigureAzureOpenAI(builder); + IKernel target = builder.Build(); + + IDictionary plugin = TestHelpers.ImportSamplePlugins(target, "FunPlugin"); + + // Act + KernelResult actual = await target.RunAsync(plugin["Limerick"]); - var builder = Kernel.Builder.WithLogger(this._logger); + // Assert + Assert.Contains("Bob", actual.GetValue(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task MultipleServiceLoadPromptConfigTestAsync() + { + // Arrange + var builder = this._kernelBuilder.WithLoggerFactory(this._logger); this.ConfigureAzureOpenAI(builder); + this.ConfigureInvalidAzureOpenAI(builder); + IKernel target = builder.Build(); - IDictionary skill = TestHelpers.GetSkills(target, "FunSkill"); + var prompt = "Where is the most famous fish market in Seattle, Washington, USA?"; + var defaultConfig = new PromptTemplateConfig(); + var azureConfig = PromptTemplateConfig.FromJson( + @" + { + ""completion"": { + ""max_tokens"": 256, + ""service_id"": ""azure-text-davinci-003"" + } + }"); + + var defaultFunc = target.RegisterSemanticFunction( + "WherePlugin", "FishMarket1", + defaultConfig, + new PromptTemplate(prompt, defaultConfig, target.PromptTemplateEngine)); + var azureFunc = target.RegisterSemanticFunction( + "WherePlugin", "FishMarket2", + azureConfig, + new PromptTemplate(prompt, azureConfig, target.PromptTemplateEngine)); // Act - SKContext actual = await target.RunAsync(skill["Limerick"]); + await Assert.ThrowsAsync(() => target.RunAsync(defaultFunc)); + + KernelResult azureResult = await target.RunAsync(azureFunc); // Assert - Assert.Empty(actual.LastErrorDescription); - Assert.False(actual.ErrorOccurred); - Assert.Contains("Bob", actual.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Pike Place", azureResult.GetValue(), StringComparison.OrdinalIgnoreCase); } #region internals @@ -385,22 +400,6 @@ private void Dispose(bool disposing) } } - private void ConfigureOpenAI(KernelBuilder kernelBuilder) - { - var openAIConfiguration = this._configuration.GetSection("OpenAI").Get(); - - Assert.NotNull(openAIConfiguration); - Assert.NotNull(openAIConfiguration.ModelId); - Assert.NotNull(openAIConfiguration.ApiKey); - Assert.NotNull(openAIConfiguration.ServiceId); - - kernelBuilder.WithOpenAITextCompletionService( - modelId: openAIConfiguration.ModelId, - apiKey: openAIConfiguration.ApiKey, - serviceId: openAIConfiguration.ServiceId, - setAsDefault: true); - } - private void ConfigureChatOpenAI(KernelBuilder kernelBuilder) { var openAIConfiguration = this._configuration.GetSection("OpenAI").Get(); @@ -434,6 +433,21 @@ private void ConfigureAzureOpenAI(KernelBuilder kernelBuilder) serviceId: azureOpenAIConfiguration.ServiceId, setAsDefault: true); } + private void ConfigureInvalidAzureOpenAI(KernelBuilder kernelBuilder) + { + var azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + + Assert.NotNull(azureOpenAIConfiguration); + Assert.NotNull(azureOpenAIConfiguration.DeploymentName); + Assert.NotNull(azureOpenAIConfiguration.Endpoint); + + kernelBuilder.WithAzureTextCompletionService( + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: "invalid-api-key", + serviceId: $"invalid-{azureOpenAIConfiguration.ServiceId}", + setAsDefault: true); + } private void ConfigureAzureOpenAIChatAsText(KernelBuilder kernelBuilder) { diff --git a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAITextEmbeddingTests.cs b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAITextEmbeddingTests.cs index a614db659f6d..83fb12cf4163 100644 --- a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAITextEmbeddingTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAITextEmbeddingTests.cs @@ -46,7 +46,7 @@ public async Task OpenAITestAsync(string testInputString) var batchResult = await embeddingGenerator.GenerateEmbeddingsAsync(new List { testInputString, testInputString, testInputString }); // Assert - Assert.Equal(AdaVectorLength, singleResult.Count); + Assert.Equal(AdaVectorLength, singleResult.Length); Assert.Equal(3, batchResult.Count); } @@ -67,7 +67,7 @@ public async Task AzureOpenAITestAsync(string testInputString) var batchResult = await embeddingGenerator.GenerateEmbeddingsAsync(new List { testInputString, testInputString, testInputString }); // Assert - Assert.Equal(AdaVectorLength, singleResult.Count); + Assert.Equal(AdaVectorLength, singleResult.Length); Assert.Equal(3, batchResult.Count); } diff --git a/dotnet/src/IntegrationTests/Connectors/Weaviate/WeaviateMemoryStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Weaviate/WeaviateMemoryStoreTests.cs index cac97cedeb00..1a2fb7e9d5ea 100644 --- a/dotnet/src/IntegrationTests/Connectors/Weaviate/WeaviateMemoryStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Weaviate/WeaviateMemoryStoreTests.cs @@ -4,9 +4,8 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Connectors.Memory.Weaviate; -using Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Diagnostics; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; using Xunit; @@ -20,112 +19,117 @@ namespace SemanticKernel.IntegrationTests.Connectors.Weaviate; [Collection("Sequential")] public sealed class WeaviateMemoryStoreTests : IDisposable { - private readonly HttpClient httpClient; - private readonly WeaviateMemoryStore weaviateMemoryStore; - private readonly string authToken; + // If null, all tests will be enabled + private const string SkipReason = "Requires Weaviate server up and running"; + + private readonly HttpClient _httpClient; + private readonly WeaviateMemoryStore _weaviateMemoryStore; + private readonly string _authToken; public WeaviateMemoryStoreTests() { - this.httpClient = new(); - this.httpClient.BaseAddress = new Uri("http://localhost:8080"); - this.authToken = "my-secret-key"; + this._httpClient = new() + { + BaseAddress = new Uri("http://localhost:8080") + }; + this._authToken = "my-secret-key"; - this.weaviateMemoryStore = new(this.httpClient, this.authToken); + this._weaviateMemoryStore = new(this._httpClient, this._authToken); } - [Fact(Skip = "Do not run on CI")] + [Fact(Skip = SkipReason)] public async Task EnsureConflictingCollectionNamesAreHandledForCreateAsync() { var collectionName = "SK" + Guid.NewGuid(); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName); - Assert.True(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName); + Assert.True(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); var conflictingCollectionName = $"___{collectionName}"; - await Assert.ThrowsAsync(async () => - await this.weaviateMemoryStore.CreateCollectionAsync(conflictingCollectionName)); + await Assert.ThrowsAsync(async () => + await this._weaviateMemoryStore.CreateCollectionAsync(conflictingCollectionName)); } - [Fact(Skip = "Do not run on CI")] + [Fact(Skip = SkipReason)] public async Task EnsureConflictingCollectionNamesAreHandledForDoesExistAsync() { var collectionName = "SK" + Guid.NewGuid(); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName); - Assert.True(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName); + Assert.True(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); var conflictingCollectionName = $"___{collectionName}"; - await Assert.ThrowsAsync(async () => - await this.weaviateMemoryStore.DoesCollectionExistAsync(conflictingCollectionName)); + await Assert.ThrowsAsync(async () => + await this._weaviateMemoryStore.DoesCollectionExistAsync(conflictingCollectionName)); } - [Fact(Skip = "Do not run on CI")] + [Fact(Skip = SkipReason)] public async Task EnsureConflictingCollectionNamesAreHandledForDeleteAsync() { var collectionName = "SK" + Guid.NewGuid(); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName); - Assert.True(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName); + Assert.True(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); var conflictingCollectionName = $"___{collectionName}"; - await Assert.ThrowsAsync(async () => - await this.weaviateMemoryStore.DeleteCollectionAsync(conflictingCollectionName)); + await Assert.ThrowsAsync(async () => + await this._weaviateMemoryStore.DeleteCollectionAsync(conflictingCollectionName)); } - [Fact(Skip = "Do not run on CI")] + [Fact(Skip = SkipReason)] public async Task ItCreatesNewCollectionAsync() { var collectionName = "SK" + Guid.NewGuid(); - Assert.False(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName); - Assert.True(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); + Assert.False(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName); + Assert.True(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); } - [Fact(Skip = "Do not run on CI")] + [Fact(Skip = SkipReason)] public async Task ItListsCollectionsAsync() { await this.DeleteAllClassesAsync(); - Assert.Empty(await this.weaviateMemoryStore.GetCollectionsAsync().ToListAsync()); + Assert.Empty(await this._weaviateMemoryStore.GetCollectionsAsync().ToListAsync()); var collectionName = "SK" + Guid.NewGuid(); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName); - Assert.True(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName); + Assert.True(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); - Assert.Single((await this.weaviateMemoryStore.GetCollectionsAsync().ToListAsync())); + Assert.Single((await this._weaviateMemoryStore.GetCollectionsAsync().ToListAsync())); var collectionName2 = "SK" + Guid.NewGuid(); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName2); - Assert.True(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName2)); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName2); + Assert.True(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName2)); - Assert.Equal(2, (await this.weaviateMemoryStore.GetCollectionsAsync().ToListAsync()).Count); + Assert.Equal(2, (await this._weaviateMemoryStore.GetCollectionsAsync().ToListAsync()).Count); } - [Fact(Skip = "Do not run on CI")] + [Fact(Skip = SkipReason)] public async Task ItDeletesCollectionAsync() { await this.DeleteAllClassesAsync(); - Assert.Empty((await this.weaviateMemoryStore.GetCollectionsAsync().ToListAsync())); + Assert.Empty((await this._weaviateMemoryStore.GetCollectionsAsync().ToListAsync())); var collectionName = "SK" + Guid.NewGuid(); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName); - Assert.True(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName); + Assert.True(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); - Assert.Single((await this.weaviateMemoryStore.GetCollectionsAsync().ToListAsync())); + Assert.Single((await this._weaviateMemoryStore.GetCollectionsAsync().ToListAsync())); - await this.weaviateMemoryStore.DeleteCollectionAsync(collectionName); - Assert.False(await this.weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); - Assert.Empty((await this.weaviateMemoryStore.GetCollectionsAsync().ToListAsync())); + await this._weaviateMemoryStore.DeleteCollectionAsync(collectionName); + Assert.False(await this._weaviateMemoryStore.DoesCollectionExistAsync(collectionName)); + Assert.Empty((await this._weaviateMemoryStore.GetCollectionsAsync().ToListAsync())); } - [Fact(Skip = "Do not run on CI")] + [Fact(Skip = SkipReason)] public async Task CrudOperationsAsync() { var id = Guid.NewGuid().ToString(); var collectionName = "SK" + Guid.NewGuid(); var timestamp = new DateTimeOffset(2023, 1, 1, 1, 1, 1, new(0)); - var embedding = new Embedding(new[] { 1f, 1f, 1f }); + var embedding = new[] { 1f, 1f, 1f }; var memoryRecord = MemoryRecord.LocalRecord( id: id, @@ -136,11 +140,11 @@ public async Task CrudOperationsAsync() key: "existing+" + id, timestamp: timestamp); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName); - var responseId = await this.weaviateMemoryStore.UpsertAsync(collectionName, memoryRecord); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName); + var responseId = await this._weaviateMemoryStore.UpsertAsync(collectionName, memoryRecord); Assert.Equal(id, responseId); - var memoryRecordResultNoVector = await this.weaviateMemoryStore.GetAsync(collectionName, id); + var memoryRecordResultNoVector = await this._weaviateMemoryStore.GetAsync(collectionName, id); if (memoryRecordResultNoVector == null) { Assert.Fail("Unable to retrieve record"); @@ -148,7 +152,7 @@ public async Task CrudOperationsAsync() Assert.Equal(id, memoryRecordResultNoVector.Key); Assert.Equal(timestamp, memoryRecordResultNoVector.Timestamp); - Assert.Equal(Array.Empty(), memoryRecordResultNoVector.Embedding.Vector); + Assert.True(memoryRecordResultNoVector.Embedding.IsEmpty); Assert.True(memoryRecordResultNoVector.HasTimestamp); Assert.Equal(memoryRecordResultNoVector.Metadata.Id, memoryRecordResultNoVector.Metadata.Id); Assert.Equal(memoryRecordResultNoVector.Metadata.AdditionalMetadata, memoryRecordResultNoVector.Metadata.AdditionalMetadata); @@ -157,7 +161,7 @@ public async Task CrudOperationsAsync() Assert.Equal(memoryRecordResultNoVector.Metadata.ExternalSourceName, memoryRecordResultNoVector.Metadata.ExternalSourceName); Assert.Equal(memoryRecordResultNoVector.Metadata.IsReference, memoryRecordResultNoVector.Metadata.IsReference); - var memoryRecordResultWithVector = await this.weaviateMemoryStore.GetAsync(collectionName, id, true); + var memoryRecordResultWithVector = await this._weaviateMemoryStore.GetAsync(collectionName, id, true); if (memoryRecordResultWithVector == null) { Assert.Fail("Unable to retrieve record"); @@ -165,7 +169,7 @@ public async Task CrudOperationsAsync() Assert.Equal(id, memoryRecordResultWithVector.Key); Assert.Equal(timestamp, memoryRecordResultWithVector.Timestamp); - Assert.Equal(memoryRecord.Embedding.Vector, memoryRecordResultWithVector.Embedding.Vector); + Assert.True(memoryRecord.Embedding.Span.SequenceEqual(memoryRecordResultWithVector.Embedding.Span)); Assert.True(memoryRecordResultWithVector.HasTimestamp); Assert.Equal(memoryRecordResultNoVector.Metadata.Id, memoryRecordResultWithVector.Metadata.Id); Assert.Equal(memoryRecordResultNoVector.Metadata.AdditionalMetadata, memoryRecordResultWithVector.Metadata.AdditionalMetadata); @@ -174,30 +178,30 @@ public async Task CrudOperationsAsync() Assert.Equal(memoryRecordResultNoVector.Metadata.ExternalSourceName, memoryRecordResultWithVector.Metadata.ExternalSourceName); Assert.Equal(memoryRecordResultNoVector.Metadata.IsReference, memoryRecordResultWithVector.Metadata.IsReference); - await this.weaviateMemoryStore.RemoveAsync(collectionName, id); - var memoryRecordAfterDeletion = await this.weaviateMemoryStore.GetAsync(collectionName, id); + await this._weaviateMemoryStore.RemoveAsync(collectionName, id); + var memoryRecordAfterDeletion = await this._weaviateMemoryStore.GetAsync(collectionName, id); if (memoryRecordAfterDeletion != null) { Assert.Fail("Unable to delete record"); } } - [Fact(Skip = "Do not run on CI")] + [Fact(Skip = SkipReason)] public async Task BatchCrudOperationsAsync() { var collectionName = "SK" + Guid.NewGuid(); var id1 = Guid.NewGuid().ToString(); var timestamp1 = new DateTimeOffset(2023, 1, 1, 1, 1, 1, new(0)); - var embedding1 = new Embedding(new[] { 1f, 1f, 1f }); + var embedding1 = new[] { 1f, 1f, 1f }; var id2 = Guid.NewGuid().ToString(); var timestamp2 = new DateTimeOffset(2023, 1, 1, 1, 1, 1, new(0)); - var embedding2 = new Embedding(new[] { 2f, 2f, 2f }); + var embedding2 = new[] { 2f, 2f, 2f }; var id3 = Guid.NewGuid().ToString(); var timestamp3 = new DateTimeOffset(2023, 1, 1, 1, 1, 1, new(0)); - var embedding3 = new Embedding(new[] { 3f, 3f, 3f }); + var embedding3 = new[] { 3f, 3f, 3f }; var memoryRecord1 = MemoryRecord.LocalRecord( id: id1, @@ -226,20 +230,20 @@ public async Task BatchCrudOperationsAsync() key: "existing3+" + id3, timestamp: timestamp3); - await this.weaviateMemoryStore.CreateCollectionAsync(collectionName); - var response = await this.weaviateMemoryStore.UpsertBatchAsync(collectionName, new[] { memoryRecord1, memoryRecord2, memoryRecord3 }).ToListAsync(); + await this._weaviateMemoryStore.CreateCollectionAsync(collectionName); + var response = await this._weaviateMemoryStore.UpsertBatchAsync(collectionName, new[] { memoryRecord1, memoryRecord2, memoryRecord3 }).ToListAsync(); Assert.Equal(id1, response[0]); Assert.Equal(id2, response[1]); Assert.Equal(id3, response[2]); - var results = await this.weaviateMemoryStore.GetNearestMatchesAsync(collectionName, embedding1, 100, 0.8, true).ToListAsync(); + var results = await this._weaviateMemoryStore.GetNearestMatchesAsync(collectionName, embedding1, 100, 0.8, true).ToListAsync(); (MemoryRecord, double) first = results[0]; (MemoryRecord, double) second = results[1]; Assert.Equal(id3, first.Item1.Key); Assert.Equal(memoryRecord3.Timestamp, first.Item1.Timestamp); - Assert.Equal(memoryRecord3.Embedding.Vector, first.Item1.Embedding.Vector); + Assert.True(memoryRecord3.Embedding.Span.SequenceEqual(first.Item1.Embedding.Span)); Assert.True(first.Item1.HasTimestamp); Assert.Equal(memoryRecord3.Metadata.Id, first.Item1.Metadata.Id); Assert.Equal(memoryRecord3.Metadata.AdditionalMetadata, first.Item1.Metadata.AdditionalMetadata); @@ -250,7 +254,7 @@ public async Task BatchCrudOperationsAsync() Assert.Equal(id2, second.Item1.Key); Assert.Equal(memoryRecord2.Timestamp, second.Item1.Timestamp); - Assert.Equal(memoryRecord2.Embedding.Vector, second.Item1.Embedding.Vector); + Assert.True(memoryRecord2.Embedding.Span.SequenceEqual(second.Item1.Embedding.Span)); Assert.True(second.Item1.HasTimestamp); Assert.Equal(memoryRecord2.Metadata.Id, second.Item1.Metadata.Id); Assert.Equal(memoryRecord2.Metadata.AdditionalMetadata, second.Item1.Metadata.AdditionalMetadata); @@ -259,10 +263,10 @@ public async Task BatchCrudOperationsAsync() Assert.Equal(memoryRecord2.Metadata.ExternalSourceName, second.Item1.Metadata.ExternalSourceName); Assert.Equal(memoryRecord2.Metadata.IsReference, second.Item1.Metadata.IsReference); - var closest = await this.weaviateMemoryStore.GetNearestMatchAsync(collectionName, embedding1, 0.8, true); + var closest = await this._weaviateMemoryStore.GetNearestMatchAsync(collectionName, embedding1, 0.8, true); Assert.Equal(id3, closest!.Value.Item1.Key); Assert.Equal(memoryRecord3.Timestamp, closest.Value.Item1.Timestamp); - Assert.Equal(memoryRecord3.Embedding.Vector, closest.Value.Item1.Embedding.Vector); + Assert.True(memoryRecord3.Embedding.Span.SequenceEqual(closest.Value.Item1.Embedding.Span)); Assert.True(closest.Value.Item1.HasTimestamp); Assert.Equal(memoryRecord3.Metadata.Id, closest.Value.Item1.Metadata.Id); Assert.Equal(memoryRecord3.Metadata.AdditionalMetadata, closest.Value.Item1.Metadata.AdditionalMetadata); @@ -271,25 +275,25 @@ public async Task BatchCrudOperationsAsync() Assert.Equal(memoryRecord3.Metadata.ExternalSourceName, closest.Value.Item1.Metadata.ExternalSourceName); Assert.Equal(memoryRecord3.Metadata.IsReference, closest.Value.Item1.Metadata.IsReference); - await this.weaviateMemoryStore.RemoveBatchAsync(collectionName, new[] { id1, id2, id3 }); - var memoryRecordsAfterDeletion = await this.weaviateMemoryStore.GetBatchAsync(collectionName, new[] { id1, id2, id3 }).ToListAsync(); + await this._weaviateMemoryStore.RemoveBatchAsync(collectionName, new[] { id1, id2, id3 }); + var memoryRecordsAfterDeletion = await this._weaviateMemoryStore.GetBatchAsync(collectionName, new[] { id1, id2, id3 }).ToListAsync(); Assert.Empty(memoryRecordsAfterDeletion); } private async Task DeleteAllClassesAsync() { - var classes = this.weaviateMemoryStore.GetCollectionsAsync(); + var classes = this._weaviateMemoryStore.GetCollectionsAsync(); await foreach (var @class in classes) { - using var requestMessage = new HttpRequestMessage(HttpMethod.Delete, $"schema/{@class}"); - requestMessage.Headers.Add("authorization", this.authToken); - var result = await this.httpClient.SendAsync(requestMessage); + using var requestMessage = new HttpRequestMessage(HttpMethod.Delete, $"v1/schema/{@class}"); + requestMessage.Headers.Add("authorization", this._authToken); + var result = await this._httpClient.SendAsync(requestMessage); result.EnsureSuccessStatusCode(); } } public void Dispose() { - this.httpClient.Dispose(); + this._httpClient.Dispose(); } } diff --git a/dotnet/src/IntegrationTests/Connectors/Weaviate/docker-compose.yml b/dotnet/src/IntegrationTests/Connectors/Weaviate/docker-compose.yml index 4fe819ef7070..9e14a2ba3830 100644 --- a/dotnet/src/IntegrationTests/Connectors/Weaviate/docker-compose.yml +++ b/dotnet/src/IntegrationTests/Connectors/Weaviate/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.4' services: weaviate: - image: semitechnologies/weaviate:1.18.0 + image: semitechnologies/weaviate:1.21.2 links: - "contextionary:contextionary" ports: @@ -35,4 +35,4 @@ services: EXTENSIONS_STORAGE_MODE: weaviate EXTENSIONS_STORAGE_ORIGIN: http://weaviate:8080 NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5 - ENABLE_COMPOUND_SPLITTING: 'false' \ No newline at end of file + ENABLE_COMPOUND_SPLITTING: 'false' diff --git a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs new file mode 100644 index 000000000000..5109a388c902 --- /dev/null +++ b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine.Basic; +using SemanticKernel.IntegrationTests.Fakes; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Extensions; + +public sealed class KernelSemanticFunctionExtensionsTests : IDisposable +{ + public KernelSemanticFunctionExtensionsTests(ITestOutputHelper output) + { + this._logger = new RedirectOutput(output); + this._target = new BasicPromptTemplateEngine(); + } + + [Fact] + public async Task ItSupportsFunctionCallsAsync() + { + var builder = Kernel.Builder + .WithAIService(null, new RedirectTextCompletion(), true) + .WithLoggerFactory(this._logger); + IKernel target = builder.Build(); + + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + var prompt = $"Hey {{{{{FunctionCollection.GlobalFunctionsPluginName}.GetEmailAddress}}}}"; + + // Act + KernelResult actual = await target.InvokeSemanticFunctionAsync(prompt, new OpenAIRequestSettings() { MaxTokens = 150 }); + + // Assert + Assert.Equal("Hey johndoe1234@example.com", actual.GetValue()); + } + + [Fact] + public async Task ItSupportsFunctionCallsWithInputAsync() + { + var builder = Kernel.Builder + .WithAIService(null, new RedirectTextCompletion(), true) + .WithLoggerFactory(this._logger); + IKernel target = builder.Build(); + + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + var prompt = $"Hey {{{{{FunctionCollection.GlobalFunctionsPluginName}.GetEmailAddress \"a person\"}}}}"; + + // Act + KernelResult actual = await target.InvokeSemanticFunctionAsync(prompt, new OpenAIRequestSettings() { MaxTokens = 150 }); + + // Assert + Assert.Equal("Hey a person@example.com", actual.GetValue()); + } + + private readonly RedirectOutput _logger; + private readonly BasicPromptTemplateEngine _target; + + public void Dispose() + { + this._logger.Dispose(); + } + + private sealed class RedirectTextCompletion : ITextCompletion + { + Task> ITextCompletion.GetCompletionsAsync(string text, AIRequestSettings? requestSettings, CancellationToken cancellationToken) + { + return Task.FromResult>(new List { new RedirectTextCompletionResult(text) }); + } + + IAsyncEnumerable ITextCompletion.GetStreamingCompletionsAsync(string text, AIRequestSettings? requestSettings, CancellationToken cancellationToken) + { + throw new NotImplementedException(); // TODO + } + } + + internal sealed class RedirectTextCompletionResult : ITextResult + { + private readonly string _completion; + + public RedirectTextCompletionResult(string completion) + { + this._completion = completion; + } + + public ModelResult ModelResult => new(this._completion); + + public Task GetCompletionAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult(this._completion); + } + } +} diff --git a/dotnet/src/IntegrationTests/Fakes/EmailPluginFake.cs b/dotnet/src/IntegrationTests/Fakes/EmailPluginFake.cs new file mode 100644 index 000000000000..0629097f8d22 --- /dev/null +++ b/dotnet/src/IntegrationTests/Fakes/EmailPluginFake.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; + +namespace SemanticKernel.IntegrationTests.Fakes; + +internal sealed class EmailPluginFake +{ + [SKFunction, Description("Given an email address and message body, send an email")] + public Task SendEmailAsync( + [Description("The body of the email message to send.")] string input = "", + [Description("The email address to send email to.")] string? email_address = "default@email.com") + { + email_address ??= string.Empty; + return Task.FromResult($"Sent email to: {email_address}. Body: {input}"); + } + + [SKFunction, Description("Lookup an email address for a person given a name")] + public Task GetEmailAddressAsync( + ILogger logger, + [Description("The name of the person to email.")] string? input = null) + { + if (string.IsNullOrEmpty(input)) + { + logger.LogTrace("Returning hard coded email for {0}", input); + return Task.FromResult("johndoe1234@example.com"); + } + + logger.LogTrace("Returning dynamic email for {0}", input); + return Task.FromResult($"{input}@example.com"); + } + + [SKFunction, Description("Write a short poem for an e-mail")] + public Task WritePoemAsync( + [Description("The topic of the poem.")] string input) + { + return Task.FromResult($"Roses are red, violets are blue, {input} is hard, so is this test."); + } +} diff --git a/dotnet/src/IntegrationTests/Fakes/EmailSkillFake.cs b/dotnet/src/IntegrationTests/Fakes/EmailSkillFake.cs deleted file mode 100644 index 5b54f167b9cd..000000000000 --- a/dotnet/src/IntegrationTests/Fakes/EmailSkillFake.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace SemanticKernel.IntegrationTests.Fakes; - -internal sealed class EmailSkillFake -{ - [SKFunction, Description("Given an email address and message body, send an email")] - public Task SendEmailAsync( - [Description("The body of the email message to send.")] string input, - [Description("The email address to send email to.")] string? email_address = "default@email.com") - { - email_address ??= string.Empty; - return Task.FromResult($"Sent email to: {email_address}. Body: {input}"); - } - - [SKFunction, Description("Lookup an email address for a person given a name")] - public Task GetEmailAddressAsync( - [Description("The name of the person to email.")] string input, - ILogger logger) - { - if (string.IsNullOrEmpty(input)) - { - logger.LogTrace("Returning hard coded email for {0}", input); - return Task.FromResult("johndoe1234@example.com"); - } - - logger.LogTrace("Returning dynamic email for {0}", input); - return Task.FromResult($"{input}@example.com"); - } - - [SKFunction, Description("Write a short poem for an e-mail")] - public Task WritePoemAsync( - [Description("The topic of the poem.")] string input) - { - return Task.FromResult($"Roses are red, violets are blue, {input} is hard, so is this test."); - } -} diff --git a/dotnet/src/IntegrationTests/IntegrationTests.csproj b/dotnet/src/IntegrationTests/IntegrationTests.csproj index 7443e4100df9..cce2ae451319 100644 --- a/dotnet/src/IntegrationTests/IntegrationTests.csproj +++ b/dotnet/src/IntegrationTests/IntegrationTests.csproj @@ -1,5 +1,4 @@  - IntegrationTests SemanticKernel.IntegrationTests @@ -10,7 +9,6 @@ CA2007,VSTHRD111 b7762d10-e29b-4bb1-8b74-b6d69a667dd4 - @@ -18,6 +16,7 @@ + runtime; build; native; contentfiles; analyzers; buildtransitive @@ -27,27 +26,28 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - - + + - - - - - - - + + + + + + + + + - Always @@ -55,9 +55,11 @@ Always + + Always + Always - - + \ No newline at end of file diff --git a/dotnet/src/IntegrationTests/Planners/PlanTests.cs b/dotnet/src/IntegrationTests/Planners/PlanTests.cs new file mode 100644 index 000000000000..b4954fb91e0b --- /dev/null +++ b/dotnet/src/IntegrationTests/Planners/PlanTests.cs @@ -0,0 +1,574 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning; +using SemanticKernel.IntegrationTests.Fakes; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Planners; + +public sealed class PlanTests : IDisposable +{ + public PlanTests(ITestOutputHelper output) + { + this._loggerFactory = NullLoggerFactory.Instance; + this._testOutputHelper = new RedirectOutput(output); + + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Theory] + [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] + public void CreatePlan(string prompt) + { + // Arrange + + // Act + var plan = new Plan(prompt); + + // Assert + Assert.Equal(prompt, plan.Description); + Assert.NotEmpty(plan.Name); + Assert.Equal(nameof(Plan), plan.PluginName); + Assert.Empty(plan.Steps); + } + + [Theory] + [InlineData("This is a story about a dog.", "kai@email.com")] + public async Task CanExecuteRunSimpleAsync(string inputToEmail, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + var expectedBody = $"Sent email to: {expectedEmail}. Body: {inputToEmail}".Trim(); + + var plan = new Plan(emailFunctions["SendEmail"]); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToEmail); + cv.Set("email_address", expectedEmail); + var result = await target.RunAsync(cv, plan); + + // Assert + Assert.Equal(expectedBody, result.GetValue()); + } + + [Theory] + [InlineData("This is a story about a dog.", "kai@email.com")] + public async Task CanExecuteAsChatAsync(string inputToEmail, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(false, true); + + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + var expectedBody = $"Sent email to: {expectedEmail}. Body: {inputToEmail}".Trim(); + + var plan = new Plan(emailFunctions["SendEmail"]); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToEmail); + cv.Set("email_address", expectedEmail); + var result = await target.RunAsync(cv, plan); + + // Assert + Assert.Equal(expectedBody, result.GetValue()); + } + + [Theory] + [InlineData("Send a story to kai.", "This is a story about a dog.", "French", "kai@email.com")] + public async Task CanExecuteRunSimpleStepsAsync(string goal, string inputToTranslate, string language, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + var writerPlugin = TestHelpers.ImportSamplePlugins(target, "WriterPlugin"); + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var plan = new Plan(goal); + plan.AddSteps(writerPlugin["Translate"], emailFunctions["SendEmail"]); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToTranslate); + cv.Set("email_address", expectedEmail); + cv.Set("language", language); + var result = (await target.RunAsync(cv, plan)).GetValue(); + + // Assert + Assert.NotNull(result); + Assert.Contains(expectedBody, result, StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < result.Length); + } + + [Fact] + public async Task CanExecutePlanWithTreeStepsAsync() + { + // Arrange + IKernel target = this.InitializeKernel(); + var goal = "Write a poem or joke and send it in an e-mail to Kai."; + var plan = new Plan(goal); + var subPlan = new Plan("Write a poem or joke"); + + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + // Arrange + var returnContext = target.CreateNewContext(); + + subPlan.AddSteps(emailFunctions["WritePoem"], emailFunctions["WritePoem"], emailFunctions["WritePoem"]); + plan.AddSteps(subPlan, emailFunctions["SendEmail"]); + plan.State.Set("email_address", "something@email.com"); + + // Act + var result = await target.RunAsync("PlanInput", plan); + + // Assert + Assert.NotNull(result); + Assert.Equal( + "Sent email to: something@email.com. Body: Roses are red, violets are blue, Roses are red, violets are blue, Roses are red, violets are blue, PlanInput is hard, so is this test. is hard, so is this test. is hard, so is this test.", + result.GetValue()); + } + + [Theory] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanSimpleManualStateAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + var sendEmailPlan = new Plan(emailFunctions["SendEmail"]) + { + Parameters = cv, + }; + + var plan = new Plan(goal); + plan.AddSteps(sendEmailPlan); + plan.State.Set("TheEmailFromState", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + Assert.Single(result.Steps); + Assert.Equal(1, result.NextStepIndex); + Assert.False(result.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanSimpleManualStateNoVariableAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", string.Empty); + var sendEmailPlan = new Plan(emailFunctions["SendEmail"]) + { + Parameters = cv, + }; + + var plan = new Plan(goal); + plan.AddSteps(sendEmailPlan); + plan.State.Set("email_address", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + Assert.Single(result.Steps); + Assert.Equal(1, result.NextStepIndex); + Assert.False(result.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanManualStateAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + var sendEmailPlan = new Plan(emailFunctions["SendEmail"]) + { + Parameters = cv + }; + + var plan = new Plan(goal); + plan.AddSteps(sendEmailPlan); + plan.State.Set("TheEmailFromState", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + Assert.False(plan.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunPlanAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var summarizePlugin = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); + var writerPlugin = TestHelpers.ImportSamplePlugins(target, "WriterPlugin"); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizePlugin["Summarize"]); + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var outputs = new List + { + "TRANSLATED_SUMMARY" + }; + var translatePlan = new Plan(writerPlugin["Translate"]) + { + Parameters = cv, + Outputs = outputs, + }; + + cv = new ContextVariables(); + cv.Update(inputName); + outputs = new List + { + "TheEmailFromState" + }; + var getEmailPlan = new Plan(emailFunctions["GetEmailAddress"]) + { + Parameters = cv, + Outputs = outputs, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + var sendEmailPlan = new Plan(emailFunctions["SendEmail"]) + { + Parameters = cv + }; + + var plan = new Plan(goal); + plan.AddSteps(summarizePlan, translatePlan, getEmailPlan, sendEmailPlan); + + // Act + var result = await target.StepAsync(inputToSummarize, plan); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(1, result.NextStepIndex); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(2, result.NextStepIndex); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(3, result.NextStepIndex); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + + // Assert + Assert.Equal(4, result.Steps.Count); + Assert.Equal(4, result.NextStepIndex); + Assert.False(result.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Contains(expectedBody, plan.State.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < plan.State.ToString().Length); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunSequentialAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + var summarizePlugin = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); + var writerPlugin = TestHelpers.ImportSamplePlugins(target, "WriterPlugin"); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizePlugin["Summarize"]); + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var outputs = new List + { + "TRANSLATED_SUMMARY" + }; + + var translatePlan = new Plan(writerPlugin["Translate"]) + { + Parameters = cv, + Outputs = outputs, + }; + + cv = new ContextVariables(); + cv.Update(inputName); + outputs = new List + { + "TheEmailFromState" + }; + var getEmailPlan = new Plan(emailFunctions["GetEmailAddress"]) + { + Parameters = cv, + Outputs = outputs, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + var sendEmailPlan = new Plan(emailFunctions["SendEmail"]) + { + Parameters = cv + }; + + var plan = new Plan(goal); + plan.AddSteps(summarizePlan, translatePlan, getEmailPlan, sendEmailPlan); + + // Act + var result = (await target.RunAsync(inputToSummarize, plan)).GetValue(); + + // Assert + Assert.NotNull(result); + Assert.Contains(expectedBody, result, StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < result.Length); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunSequentialOnDeserializedPlanAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, + string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + var summarizePlugin = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); + var writerPlugin = TestHelpers.ImportSamplePlugins(target, "WriterPlugin"); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizePlugin["Summarize"]); + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var outputs = new List + { + "TRANSLATED_SUMMARY" + }; + + var translatePlan = new Plan(writerPlugin["Translate"]) + { + Parameters = cv, + Outputs = outputs, + }; + + cv = new ContextVariables(); + cv.Update(inputName); + outputs = new List + { + "TheEmailFromState" + }; + var getEmailPlan = new Plan(emailFunctions["GetEmailAddress"]) + { + Parameters = cv, + Outputs = outputs, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + var sendEmailPlan = new Plan(emailFunctions["SendEmail"]) + { + Parameters = cv + }; + + var plan = new Plan(goal); + plan.AddSteps(summarizePlan, translatePlan, getEmailPlan, sendEmailPlan); + + // Act + var serializedPlan = plan.ToJson(); + var deserializedPlan = Plan.FromJson(serializedPlan, target.Functions); + var result = (await target.RunAsync(inputToSummarize, deserializedPlan)).GetValue(); + + // Assert + Assert.NotNull(result); + Assert.Contains(expectedBody, result, StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < result.Length); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "kai@email.com")] + public async Task CanExecuteRunSequentialFunctionsAsync(string goal, string inputToSummarize, string inputLanguage, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var summarizePlugin = TestHelpers.ImportSamplePlugins(target, "SummarizePlugin"); + var writerPlugin = TestHelpers.ImportSamplePlugins(target, "WriterPlugin"); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizePlugin["Summarize"]); + var translatePlan = new Plan(writerPlugin["Translate"]); + var sendEmailPlan = new Plan(emailFunctions["SendEmail"]); + + var plan = new Plan(goal); + plan.AddSteps(summarizePlan, translatePlan, sendEmailPlan); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToSummarize); + cv.Set("email_address", expectedEmail); + cv.Set("language", inputLanguage); + var result = await target.RunAsync(cv, plan); + + // Assert + Assert.Contains(expectedBody, result.GetValue(), StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData("computers")] + public async Task CanImportAndRunPlanAsync(string input) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailFunctions = target.ImportFunctions(new EmailPluginFake()); + + var plan = new Plan("Write a poem about a topic and send in an email."); + + var writePoem = new Plan(emailFunctions["WritePoem"]); + // fileStep.Parameters["input"] = "$INPUT"; + writePoem.Outputs.Add("POEM"); + + var sendEmail = new Plan(emailFunctions["SendEmail"]); + sendEmail.Parameters["input"] = "$POEM"; + sendEmail.Outputs.Add("EMAIL_RESULT"); + + plan.AddSteps(writePoem, sendEmail); + plan.Outputs.Add("EMAIL_RESULT"); + + //Act + var t = target.ImportPlan(plan); + + var result = await t.InvokeAsync(input, target); + + // Assert + Assert.NotNull(result); + Assert.Equal($"Sent email to: default@email.com. Body: Roses are red, violets are blue, {input} is hard, so is this test.", result.GetValue()); + } + + private IKernel InitializeKernel(bool useEmbeddings = false, bool useChatModel = false) + { + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); + Assert.NotNull(azureOpenAIEmbeddingsConfiguration); + + var builder = Kernel.Builder + .WithLoggerFactory(this._loggerFactory) + .WithRetryBasic(); + + if (useChatModel) + { + builder.WithAzureChatCompletionService( + deploymentName: azureOpenAIConfiguration.ChatDeploymentName!, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + } + else + { + builder.WithAzureTextCompletionService( + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + } + + if (useEmbeddings) + { + builder + .WithAzureTextEmbeddingGenerationService( + deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + } + + var kernel = builder.Build(); + + // Import all sample plugins available for demonstration purposes. + TestHelpers.ImportAllSamplePlugins(kernel); + + kernel.ImportFunctions(new EmailPluginFake()); + return kernel; + } + + private readonly ILoggerFactory _loggerFactory; + private readonly RedirectOutput _testOutputHelper; + private readonly IConfigurationRoot _configuration; + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + ~PlanTests() + { + this.Dispose(false); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + if (this._loggerFactory is IDisposable ld) + { + ld.Dispose(); + } + + this._testOutputHelper.Dispose(); + } + } +} diff --git a/dotnet/src/IntegrationTests/Planners/SequentialPlanner/SequentialPlanParserTests.cs b/dotnet/src/IntegrationTests/Planners/SequentialPlanner/SequentialPlanParserTests.cs new file mode 100644 index 000000000000..b894666e9481 --- /dev/null +++ b/dotnet/src/IntegrationTests/Planners/SequentialPlanner/SequentialPlanParserTests.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Planners; +using Microsoft.SemanticKernel.Planners.Sequential; +using Microsoft.SemanticKernel.Planning; +using SemanticKernel.IntegrationTests.Fakes; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Planners.SequentialPlanner; + +public class SequentialPlanParserTests +{ + public SequentialPlanParserTests(ITestOutputHelper output) + { + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Fact] + public void CanCallToPlanFromXml() + { + // Arrange + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + IKernel kernel = Kernel.Builder + .WithRetryBasic() + .WithAzureTextCompletionService( + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey, + serviceId: azureOpenAIConfiguration.ServiceId, + setAsDefault: true) + .Build(); + kernel.ImportFunctions(new EmailPluginFake(), "email"); + TestHelpers.ImportSamplePlugins(kernel, "SummarizePlugin", "WriterPlugin"); + + var planString = + @" + + + + +"; + var goal = "Summarize an input, translate to french, and e-mail to John Doe"; + + // Act + var plan = planString.ToPlanFromXml(goal, kernel.Functions.GetFunctionCallback()); + + // Assert + Assert.NotNull(plan); + Assert.Equal((string?)"Summarize an input, translate to french, and e-mail to John Doe", (string?)plan.Description); + + Assert.Equal(4, plan.Steps.Count); + Assert.Collection(plan.Steps, + step => + { + Assert.Equal("SummarizePlugin", step.PluginName); + Assert.Equal("Summarize", step.Name); + }, + step => + { + Assert.Equal("WriterPlugin", step.PluginName); + Assert.Equal("Translate", step.Name); + Assert.Equal("French", step.Parameters["language"]); + Assert.True(step.Outputs.Contains("TRANSLATED_SUMMARY")); + }, + step => + { + Assert.Equal("email", step.PluginName); + Assert.Equal("GetEmailAddress", step.Name); + Assert.Equal("John Doe", step.Parameters["input"]); + Assert.True(step.Outputs.Contains("EMAIL_ADDRESS")); + }, + step => + { + Assert.Equal("email", step.PluginName); + Assert.Equal("SendEmail", step.Name); + Assert.Equal("$TRANSLATED_SUMMARY", step.Parameters["input"]); + Assert.Equal("$EMAIL_ADDRESS", step.Parameters["email_address"]); + } + ); + } + + private readonly IConfigurationRoot _configuration; +} diff --git a/dotnet/src/IntegrationTests/Planners/SequentialPlanner/SequentialPlannerTests.cs b/dotnet/src/IntegrationTests/Planners/SequentialPlanner/SequentialPlannerTests.cs new file mode 100644 index 000000000000..6236f64e4c12 --- /dev/null +++ b/dotnet/src/IntegrationTests/Planners/SequentialPlanner/SequentialPlannerTests.cs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Planners; +using Microsoft.SemanticKernel.Plugins.Memory; +using SemanticKernel.IntegrationTests.Fakes; +using SemanticKernel.IntegrationTests.TestSettings; +using xRetry; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Planners.SequentialPlanner; + +public sealed class SequentialPlannerTests : IDisposable +{ + public SequentialPlannerTests(ITestOutputHelper output) + { + this._logger = NullLoggerFactory.Instance; + this._testOutputHelper = new RedirectOutput(output); + + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Theory] + [InlineData(false, "Write a joke and send it in an e-mail to Kai.", "SendEmail", FunctionCollection.GlobalFunctionsPluginName)] + [InlineData(true, "Write a joke and send it in an e-mail to Kai.", "SendEmail", FunctionCollection.GlobalFunctionsPluginName)] + public async Task CreatePlanFunctionFlowAsync(bool useChatModel, string prompt, string expectedFunction, string expectedPlugin) + { + // Arrange + bool useEmbeddings = false; + IKernel kernel = this.InitializeKernel(useEmbeddings, useChatModel); + kernel.ImportFunctions(new EmailPluginFake()); + TestHelpers.ImportSamplePlugins(kernel, "FunPlugin"); + + var planner = new Microsoft.SemanticKernel.Planners.SequentialPlanner(kernel); + + // Act + var plan = await planner.CreatePlanAsync(prompt); + + // Assert + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.PluginName.Equals(expectedPlugin, StringComparison.OrdinalIgnoreCase)); + } + + [RetryTheory] + [InlineData("Write a novel about software development that is 3 chapters long.", "NovelOutline", "WriterPlugin", "")] + public async Task CreatePlanWithDefaultsAsync(string prompt, string expectedFunction, string expectedPlugin, string expectedDefault) + { + // Arrange + IKernel kernel = this.InitializeKernel(); + TestHelpers.ImportSamplePlugins(kernel, "WriterPlugin", "MiscPlugin"); + + var planner = new Microsoft.SemanticKernel.Planners.SequentialPlanner(kernel); + + // Act + var plan = await planner.CreatePlanAsync(prompt); + + // Assert + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.PluginName.Equals(expectedPlugin, StringComparison.OrdinalIgnoreCase) && + step.Parameters["endMarker"].Equals(expectedDefault, StringComparison.OrdinalIgnoreCase)); + } + + [RetryTheory] + [InlineData("Write a poem and a joke and send it in an e-mail to Kai.", "SendEmail", FunctionCollection.GlobalFunctionsPluginName)] + public async Task CreatePlanGoalRelevantAsync(string prompt, string expectedFunction, string expectedPlugin) + { + // Arrange + bool useEmbeddings = true; + + IKernel kernel = this.InitializeKernel(useEmbeddings); + ISemanticTextMemory memory = this.InitializeMemory(kernel.GetService()); + + kernel.ImportFunctions(new EmailPluginFake()); + + // Import all sample plugins available for demonstration purposes. + TestHelpers.ImportAllSamplePlugins(kernel); + + var planner = new Microsoft.SemanticKernel.Planners.SequentialPlanner(kernel, + new SequentialPlannerConfig { SemanticMemoryConfig = new() { RelevancyThreshold = 0.65, MaxRelevantFunctions = 30, Memory = memory } }); + + // Act + var plan = await planner.CreatePlanAsync(prompt); + + // Assert + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.PluginName.Equals(expectedPlugin, StringComparison.OrdinalIgnoreCase)); + } + + private IKernel InitializeKernel(bool useEmbeddings = false, bool useChatModel = false) + { + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); + Assert.NotNull(azureOpenAIEmbeddingsConfiguration); + + var builder = Kernel.Builder.WithLoggerFactory(this._logger); + builder.WithRetryBasic(); + + if (useChatModel) + { + builder.WithAzureChatCompletionService( + deploymentName: azureOpenAIConfiguration.ChatDeploymentName!, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + } + else + { + builder.WithAzureTextCompletionService( + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + } + + if (useEmbeddings) + { + builder.WithAzureTextEmbeddingGenerationService( + deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + } + + var kernel = builder.Build(); + + return kernel; + } + + private ISemanticTextMemory InitializeMemory(ITextEmbeddingGeneration textEmbeddingGeneration) + { + var builder = new MemoryBuilder(); + + builder.WithLoggerFactory(this._logger); + builder.WithMemoryStore(new VolatileMemoryStore()); + builder.WithTextEmbeddingGeneration(textEmbeddingGeneration); + + return builder.Build(); + } + + private readonly ILoggerFactory _logger; + private readonly RedirectOutput _testOutputHelper; + private readonly IConfigurationRoot _configuration; + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + ~SequentialPlannerTests() + { + this.Dispose(false); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + if (this._logger is IDisposable ld) + { + ld.Dispose(); + } + + this._testOutputHelper.Dispose(); + } + } +} diff --git a/dotnet/src/IntegrationTests/Planners/StepwisePlanner/StepwisePlannerTests.cs b/dotnet/src/IntegrationTests/Planners/StepwisePlanner/StepwisePlannerTests.cs new file mode 100644 index 000000000000..c99ab62ec729 --- /dev/null +++ b/dotnet/src/IntegrationTests/Planners/StepwisePlanner/StepwisePlannerTests.cs @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; +using Microsoft.SemanticKernel.Planners; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.Plugins.Web; +using Microsoft.SemanticKernel.Plugins.Web.Bing; +using SemanticKernel.IntegrationTests.TestSettings; +using xRetry; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Planners.StepwisePlanner; + +public sealed class StepwisePlannerTests : IDisposable +{ + private readonly string _bingApiKey; + + public StepwisePlannerTests(ITestOutputHelper output) + { + this._loggerFactory = NullLoggerFactory.Instance; + this._testOutputHelper = new RedirectOutput(output); + + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + string? bingApiKeyCandidate = this._configuration["Bing:ApiKey"]; + Assert.NotNull(bingApiKeyCandidate); + this._bingApiKey = bingApiKeyCandidate; + } + + [Theory] + [InlineData(false, "Who is the current president of the United States? What is his current age divided by 2", "ExecutePlan", "StepwisePlanner")] + [InlineData(true, "Who is the current president of the United States? What is his current age divided by 2", "ExecutePlan", "StepwisePlanner")] + public void CanCreateStepwisePlan(bool useChatModel, string prompt, string expectedFunction, string expectedPlugin) + { + // Arrange + bool useEmbeddings = false; + IKernel kernel = this.InitializeKernel(useEmbeddings, useChatModel); + var bingConnector = new BingConnector(this._bingApiKey); + var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector); + kernel.ImportFunctions(webSearchEnginePlugin, "WebSearch"); + kernel.ImportFunctions(new TimePlugin(), "time"); + + var planner = new Microsoft.SemanticKernel.Planners.StepwisePlanner(kernel, new StepwisePlannerConfig() { MaxIterations = 10 }); + + // Act + var plan = planner.CreatePlan(prompt); + + // Assert + Assert.Empty(plan.Steps); + Assert.Equal(expectedFunction, plan.Name); + Assert.Contains(expectedPlugin, plan.PluginName, StringComparison.OrdinalIgnoreCase); + } + + [RetryTheory(maxRetries: 3)] + [InlineData(false, "What is the tallest mountain on Earth? How tall is it divided by 2", "Everest")] + [InlineData(true, "What is the tallest mountain on Earth? How tall is it divided by 2", "Everest")] + [InlineData(false, "What is the weather in Seattle?", "Seattle")] + [InlineData(true, "What is the weather in Seattle?", "Seattle")] + public async Task CanExecuteStepwisePlanAsync(bool useChatModel, string prompt, string partialExpectedAnswer) + { + // Arrange + bool useEmbeddings = false; + IKernel kernel = this.InitializeKernel(useEmbeddings, useChatModel); + var bingConnector = new BingConnector(this._bingApiKey); + var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector); + kernel.ImportFunctions(webSearchEnginePlugin, "WebSearch"); + kernel.ImportFunctions(new TimePlugin(), "time"); + + var planner = new Microsoft.SemanticKernel.Planners.StepwisePlanner(kernel, new StepwisePlannerConfig() { MaxIterations = 10 }); + + // Act + var plan = planner.CreatePlan(prompt); + var planResult = await plan.InvokeAsync(kernel); + var result = planResult.GetValue(); + + // Assert - should contain the expected answer + Assert.NotNull(result); + Assert.Contains(partialExpectedAnswer, result, StringComparison.InvariantCultureIgnoreCase); + Assert.True(planResult.TryGetMetadataValue("iterations", out string iterations)); + Assert.True(int.Parse(iterations, System.Globalization.CultureInfo.InvariantCulture) > 0); + Assert.True(int.Parse(iterations, System.Globalization.CultureInfo.InvariantCulture) <= 10); + } + + [Fact] + public async Task ExecutePlanFailsWithTooManyFunctionsAsync() + { + // Arrange + IKernel kernel = this.InitializeKernel(); + var bingConnector = new BingConnector(this._bingApiKey); + var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector); + kernel.ImportFunctions(webSearchEnginePlugin, "WebSearch"); + kernel.ImportFunctions(new TextPlugin(), "text"); + kernel.ImportFunctions(new ConversationSummaryPlugin(kernel), "ConversationSummary"); + kernel.ImportFunctions(new MathPlugin(), "Math"); + kernel.ImportFunctions(new FileIOPlugin(), "FileIO"); + kernel.ImportFunctions(new HttpPlugin(), "Http"); + + var planner = new Microsoft.SemanticKernel.Planners.StepwisePlanner(kernel, new() { MaxTokens = 1000 }); + + // Act + var plan = planner.CreatePlan("I need to buy a new brush for my cat. Can you show me options?"); + + // Assert + var ex = await Assert.ThrowsAsync(async () => await kernel.RunAsync(plan)); + Assert.Equal("ChatHistory is too long to get a completion. Try reducing the available functions.", ex.Message); + } + + [Fact] + public async Task ExecutePlanSucceedsWithAlmostTooManyFunctionsAsync() + { + // Arrange + IKernel kernel = this.InitializeKernel(); + + _ = await kernel.ImportPluginFunctionsAsync("Klarna", new Uri("https://www.klarna.com/.well-known/ai-plugin.json"), new OpenApiFunctionExecutionParameters(enableDynamicOperationPayload: true)); + + var planner = new Microsoft.SemanticKernel.Planners.StepwisePlanner(kernel); + + // Act + var plan = planner.CreatePlan("I need to buy a new brush for my cat. Can you show me options?"); + var kernelResult = await kernel.RunAsync(plan); + var result = kernelResult.GetValue(); + + // Assert - should contain results, for now just verify it didn't fail + Assert.NotNull(result); + Assert.DoesNotContain("Result not found, review 'stepsTaken' to see what happened", result, StringComparison.OrdinalIgnoreCase); + } + + private IKernel InitializeKernel(bool useEmbeddings = false, bool useChatModel = false) + { + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); + Assert.NotNull(azureOpenAIEmbeddingsConfiguration); + + var builder = Kernel.Builder + .WithLoggerFactory(this._loggerFactory) + .WithRetryBasic(); + + if (useChatModel) + { + builder.WithAzureChatCompletionService( + deploymentName: azureOpenAIConfiguration.ChatDeploymentName!, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + } + else + { + builder.WithAzureTextCompletionService( + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + } + + if (useEmbeddings) + { + builder.WithAzureTextEmbeddingGenerationService( + deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + } + + var kernel = builder.Build(); + + return kernel; + } + + private readonly ILoggerFactory _loggerFactory; + private readonly RedirectOutput _testOutputHelper; + private readonly IConfigurationRoot _configuration; + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + ~StepwisePlannerTests() + { + this.Dispose(false); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + if (this._loggerFactory is IDisposable ld) + { + ld.Dispose(); + } + + this._testOutputHelper.Dispose(); + } + } +} diff --git a/dotnet/src/IntegrationTests/Planning/PlanTests.cs b/dotnet/src/IntegrationTests/Planning/PlanTests.cs deleted file mode 100644 index ff885ffea801..000000000000 --- a/dotnet/src/IntegrationTests/Planning/PlanTests.cs +++ /dev/null @@ -1,540 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using SemanticKernel.IntegrationTests.Fakes; -using SemanticKernel.IntegrationTests.TestSettings; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.IntegrationTests.Planning; - -public sealed class PlanTests : IDisposable -{ - public PlanTests(ITestOutputHelper output) - { - this._logger = NullLogger.Instance; //new XunitLogger(output); - this._testOutputHelper = new RedirectOutput(output); - - // Load configuration - this._configuration = new ConfigurationBuilder() - .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables() - .AddUserSecrets() - .Build(); - } - - [Theory] - [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] - public void CreatePlan(string prompt) - { - // Arrange - - // Act - var plan = new Plan(prompt); - - // Assert - Assert.Equal(prompt, plan.Description); - Assert.Equal(string.Empty, plan.Name); - Assert.Equal(typeof(Plan).FullName, plan.SkillName); - Assert.Empty(plan.Steps); - } - - [Theory] - [InlineData("This is a story about a dog.", "kai@email.com")] - public async Task CanExecuteRunSimpleAsync(string inputToEmail, string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - var expectedBody = $"Sent email to: {expectedEmail}. Body: {inputToEmail}".Trim(); - - var plan = new Plan(emailSkill["SendEmail"]); - - // Act - var cv = new ContextVariables(); - cv.Update(inputToEmail); - cv.Set("email_address", expectedEmail); - var result = await target.RunAsync(cv, plan); - - // Assert - Assert.Equal(expectedBody, result.Result); - } - - [Theory] - [InlineData("This is a story about a dog.", "kai@email.com")] - public async Task CanExecuteAsChatAsync(string inputToEmail, string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(false, true); - - var emailSkill = target.ImportSkill(new EmailSkillFake()); - var expectedBody = $"Sent email to: {expectedEmail}. Body: {inputToEmail}".Trim(); - - var plan = new Plan(emailSkill["SendEmail"]); - - // Act - var cv = new ContextVariables(); - cv.Update(inputToEmail); - cv.Set("email_address", expectedEmail); - var result = await target.RunAsync(cv, plan); - - // Assert - Assert.Equal(expectedBody, result.Result); - } - - [Theory] - [InlineData("Send a story to kai.", "This is a story about a dog.", "French", "kai@email.com")] - public async Task CanExecuteRunSimpleStepsAsync(string goal, string inputToTranslate, string language, string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - var writerSkill = TestHelpers.GetSkills(target, "WriterSkill"); - var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); - - var plan = new Plan(goal); - plan.AddSteps(writerSkill["Translate"], emailSkill["SendEmail"]); - - // Act - var cv = new ContextVariables(); - cv.Update(inputToTranslate); - cv.Set("email_address", expectedEmail); - cv.Set("language", language); - var result = await target.RunAsync(cv, plan); - - // Assert - Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); - Assert.True(expectedBody.Length < result.Result.Length); - } - - [Fact] - public async Task CanExecutePanWithTreeStepsAsync() - { - // Arrange - IKernel target = this.InitializeKernel(); - var goal = "Write a poem or joke and send it in an e-mail to Kai."; - var plan = new Plan(goal); - var subPlan = new Plan("Write a poem or joke"); - - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - // Arrange - var returnContext = target.CreateNewContext(); - - subPlan.AddSteps(emailSkill["WritePoem"], emailSkill["WritePoem"], emailSkill["WritePoem"]); - plan.AddSteps(subPlan, emailSkill["SendEmail"]); - plan.State.Set("email_address", "something@email.com"); - - // Act - var result = await target.RunAsync("PlanInput", plan); - - // Assert - Assert.NotNull(result); - Assert.Equal( - "Sent email to: something@email.com. Body: Roses are red, violets are blue, Roses are red, violets are blue, Roses are red, violets are blue, PlanInput is hard, so is this test. is hard, so is this test. is hard, so is this test.", - result.Result); - } - - [Theory] - [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] - [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] - public async Task CanExecuteRunPlanSimpleManualStateAsync(string input, string goal, string email) - { - // Arrange - IKernel target = this.InitializeKernel(); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. - var cv = new ContextVariables(); - cv.Set("email_address", "$TheEmailFromState"); - var sendEmailPlan = new Plan(emailSkill["SendEmail"]) - { - Parameters = cv, - }; - - var plan = new Plan(goal); - plan.AddSteps(sendEmailPlan); - plan.State.Set("TheEmailFromState", email); // manually prepare the state - - // Act - var result = await target.StepAsync(input, plan); - - // Assert - var expectedBody = string.IsNullOrEmpty(input) ? goal : input; - Assert.Single(result.Steps); - Assert.Equal(1, result.NextStepIndex); - Assert.False(result.HasNextStep); - Assert.Equal(goal, plan.Description); - Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); - } - - [Theory] - [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] - [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] - public async Task CanExecuteRunPlanSimpleManualStateNoVariableAsync(string input, string goal, string email) - { - // Arrange - IKernel target = this.InitializeKernel(); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. - var cv = new ContextVariables(); - cv.Set("email_address", string.Empty); - var sendEmailPlan = new Plan(emailSkill["SendEmail"]) - { - Parameters = cv, - }; - - var plan = new Plan(goal); - plan.AddSteps(sendEmailPlan); - plan.State.Set("email_address", email); // manually prepare the state - - // Act - var result = await target.StepAsync(input, plan); - - // Assert - var expectedBody = string.IsNullOrEmpty(input) ? goal : input; - Assert.Single(result.Steps); - Assert.Equal(1, result.NextStepIndex); - Assert.False(result.HasNextStep); - Assert.Equal(goal, plan.Description); - Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); - } - - [Theory] - [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] - [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] - public async Task CanExecuteRunPlanManualStateAsync(string input, string goal, string email) - { - // Arrange - IKernel target = this.InitializeKernel(); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. - var cv = new ContextVariables(); - cv.Set("email_address", "$TheEmailFromState"); - var sendEmailPlan = new Plan(emailSkill["SendEmail"]) - { - Parameters = cv - }; - - var plan = new Plan(goal); - plan.AddSteps(sendEmailPlan); - plan.State.Set("TheEmailFromState", email); // manually prepare the state - - // Act - var result = await target.StepAsync(input, plan); - - // Assert - var expectedBody = string.IsNullOrEmpty(input) ? goal : input; - Assert.False(plan.HasNextStep); - Assert.Equal(goal, plan.Description); - Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); - } - - [Theory] - [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] - public async Task CanExecuteRunPlanAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(); - - var summarizeSkill = TestHelpers.GetSkills(target, "SummarizeSkill"); - var writerSkill = TestHelpers.GetSkills(target, "WriterSkill"); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); - - var summarizePlan = new Plan(summarizeSkill["Summarize"]); - - var cv = new ContextVariables(); - cv.Set("language", inputLanguage); - var outputs = new List - { - "TRANSLATED_SUMMARY" - }; - var translatePlan = new Plan(writerSkill["Translate"]) - { - Parameters = cv, - Outputs = outputs, - }; - - cv = new ContextVariables(); - cv.Update(inputName); - outputs = new List - { - "TheEmailFromState" - }; - var getEmailPlan = new Plan(emailSkill["GetEmailAddress"]) - { - Parameters = cv, - Outputs = outputs, - }; - - cv = new ContextVariables(); - cv.Set("email_address", "$TheEmailFromState"); - cv.Set("input", "$TRANSLATED_SUMMARY"); - var sendEmailPlan = new Plan(emailSkill["SendEmail"]) - { - Parameters = cv - }; - - var plan = new Plan(goal); - plan.AddSteps(summarizePlan, translatePlan, getEmailPlan, sendEmailPlan); - - // Act - var result = await target.StepAsync(inputToSummarize, plan); - Assert.Equal(4, result.Steps.Count); - Assert.Equal(1, result.NextStepIndex); - Assert.True(result.HasNextStep); - result = await target.StepAsync(result); - Assert.Equal(4, result.Steps.Count); - Assert.Equal(2, result.NextStepIndex); - Assert.True(result.HasNextStep); - result = await target.StepAsync(result); - Assert.Equal(4, result.Steps.Count); - Assert.Equal(3, result.NextStepIndex); - Assert.True(result.HasNextStep); - result = await target.StepAsync(result); - - // Assert - Assert.Equal(4, result.Steps.Count); - Assert.Equal(4, result.NextStepIndex); - Assert.False(result.HasNextStep); - Assert.Equal(goal, plan.Description); - Assert.Contains(expectedBody, plan.State.ToString(), StringComparison.OrdinalIgnoreCase); - Assert.True(expectedBody.Length < plan.State.ToString().Length); - } - - [Theory] - [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] - public async Task CanExecuteRunSequentialAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(); - var summarizeSkill = TestHelpers.GetSkills(target, "SummarizeSkill"); - var writerSkill = TestHelpers.GetSkills(target, "WriterSkill"); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); - - var summarizePlan = new Plan(summarizeSkill["Summarize"]); - - var cv = new ContextVariables(); - cv.Set("language", inputLanguage); - var outputs = new List - { - "TRANSLATED_SUMMARY" - }; - - var translatePlan = new Plan(writerSkill["Translate"]) - { - Parameters = cv, - Outputs = outputs, - }; - - cv = new ContextVariables(); - cv.Update(inputName); - outputs = new List - { - "TheEmailFromState" - }; - var getEmailPlan = new Plan(emailSkill["GetEmailAddress"]) - { - Parameters = cv, - Outputs = outputs, - }; - - cv = new ContextVariables(); - cv.Set("email_address", "$TheEmailFromState"); - cv.Set("input", "$TRANSLATED_SUMMARY"); - var sendEmailPlan = new Plan(emailSkill["SendEmail"]) - { - Parameters = cv - }; - - var plan = new Plan(goal); - plan.AddSteps(summarizePlan, translatePlan, getEmailPlan, sendEmailPlan); - - // Act - var result = await target.RunAsync(inputToSummarize, plan); - - // Assert - Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); - Assert.True(expectedBody.Length < result.Result.Length); - } - - [Theory] - [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] - public async Task CanExecuteRunSequentialOnDeserializedPlanAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, - string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(); - var summarizeSkill = TestHelpers.GetSkills(target, "SummarizeSkill"); - var writerSkill = TestHelpers.GetSkills(target, "WriterSkill"); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); - - var summarizePlan = new Plan(summarizeSkill["Summarize"]); - - var cv = new ContextVariables(); - cv.Set("language", inputLanguage); - var outputs = new List - { - "TRANSLATED_SUMMARY" - }; - - var translatePlan = new Plan(writerSkill["Translate"]) - { - Parameters = cv, - Outputs = outputs, - }; - - cv = new ContextVariables(); - cv.Update(inputName); - outputs = new List - { - "TheEmailFromState" - }; - var getEmailPlan = new Plan(emailSkill["GetEmailAddress"]) - { - Parameters = cv, - Outputs = outputs, - }; - - cv = new ContextVariables(); - cv.Set("email_address", "$TheEmailFromState"); - cv.Set("input", "$TRANSLATED_SUMMARY"); - var sendEmailPlan = new Plan(emailSkill["SendEmail"]) - { - Parameters = cv - }; - - var plan = new Plan(goal); - plan.AddSteps(summarizePlan, translatePlan, getEmailPlan, sendEmailPlan); - - // Act - var serializedPlan = plan.ToJson(); - var deserializedPlan = Plan.FromJson(serializedPlan, target.CreateNewContext()); - var result = await target.RunAsync(inputToSummarize, deserializedPlan); - - // Assert - Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); - Assert.True(expectedBody.Length < result.Result.Length); - } - - [Theory] - [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "kai@email.com")] - public async Task CanExecuteRunSequentialFunctionsAsync(string goal, string inputToSummarize, string inputLanguage, string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(); - - var summarizeSkill = TestHelpers.GetSkills(target, "SummarizeSkill"); - var writerSkill = TestHelpers.GetSkills(target, "WriterSkill"); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); - - var summarizePlan = new Plan(summarizeSkill["Summarize"]); - var translatePlan = new Plan(writerSkill["Translate"]); - var sendEmailPlan = new Plan(emailSkill["SendEmail"]); - - var plan = new Plan(goal); - plan.AddSteps(summarizePlan, translatePlan, sendEmailPlan); - - // Act - var cv = new ContextVariables(); - cv.Update(inputToSummarize); - cv.Set("email_address", expectedEmail); - cv.Set("language", inputLanguage); - var result = await target.RunAsync(cv, plan); - - // Assert - Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); - } - - private IKernel InitializeKernel(bool useEmbeddings = false, bool useChatModel = false) - { - AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); - Assert.NotNull(azureOpenAIEmbeddingsConfiguration); - - var builder = Kernel.Builder.WithLogger(this._logger); - - if (useChatModel) - { - builder.WithAzureChatCompletionService( - deploymentName: azureOpenAIConfiguration.ChatDeploymentName!, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - } - else - { - builder.WithAzureTextCompletionService( - deploymentName: azureOpenAIConfiguration.DeploymentName, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - } - - if (useEmbeddings) - { - builder - .WithAzureTextEmbeddingGenerationService( - deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, - endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, - apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey) - .WithMemoryStorage(new VolatileMemoryStore()); - } - - var kernel = builder.Build(); - - // Import all sample skills available for demonstration purposes. - TestHelpers.ImportSampleSkills(kernel); - - _ = kernel.ImportSkill(new EmailSkillFake()); - return kernel; - } - - private readonly ILogger _logger; - private readonly RedirectOutput _testOutputHelper; - private readonly IConfigurationRoot _configuration; - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - ~PlanTests() - { - this.Dispose(false); - } - - private void Dispose(bool disposing) - { - if (disposing) - { - if (this._logger is IDisposable ld) - { - ld.Dispose(); - } - - this._testOutputHelper.Dispose(); - } - } -} diff --git a/dotnet/src/IntegrationTests/Planning/SequentialPlanner/SequentialPlanParserTests.cs b/dotnet/src/IntegrationTests/Planning/SequentialPlanner/SequentialPlanParserTests.cs deleted file mode 100644 index d594add4ad01..000000000000 --- a/dotnet/src/IntegrationTests/Planning/SequentialPlanner/SequentialPlanParserTests.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Configuration; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.Planning.Sequential; -using SemanticKernel.IntegrationTests.Fakes; -using SemanticKernel.IntegrationTests.TestSettings; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.IntegrationTests.Planning.SequentialPlanner; - -public class SequentialPlanParserTests -{ - public SequentialPlanParserTests(ITestOutputHelper output) - { - // Load configuration - this._configuration = new ConfigurationBuilder() - .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables() - .AddUserSecrets() - .Build(); - } - - [Fact] - public void CanCallToPlanFromXml() - { - // Arrange - AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - IKernel kernel = Kernel.Builder - .WithAzureTextCompletionService( - deploymentName: azureOpenAIConfiguration.DeploymentName, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey, - serviceId: azureOpenAIConfiguration.ServiceId, - setAsDefault: true) - .Build(); - kernel.ImportSkill(new EmailSkillFake(), "email"); - TestHelpers.GetSkills(kernel, "SummarizeSkill", "WriterSkill"); - - var planString = - @" - - - - -"; - var goal = "Summarize an input, translate to french, and e-mail to John Doe"; - - // Act - var plan = planString.ToPlanFromXml(goal, SequentialPlanParser.GetSkillFunction(kernel.CreateNewContext())); - - // Assert - Assert.NotNull(plan); - Assert.Equal((string?)"Summarize an input, translate to french, and e-mail to John Doe", (string?)plan.Description); - - Assert.Equal(4, plan.Steps.Count); - Assert.Collection(plan.Steps, - step => - { - Assert.Equal("SummarizeSkill", step.SkillName); - Assert.Equal("Summarize", step.Name); - }, - step => - { - Assert.Equal("WriterSkill", step.SkillName); - Assert.Equal("Translate", step.Name); - Assert.Equal("French", step.Parameters["language"]); - Assert.True(step.Outputs.Contains("TRANSLATED_SUMMARY")); - }, - step => - { - Assert.Equal("email", step.SkillName); - Assert.Equal("GetEmailAddress", step.Name); - Assert.Equal("John Doe", step.Parameters["input"]); - Assert.True(step.Outputs.Contains("EMAIL_ADDRESS")); - }, - step => - { - Assert.Equal("email", step.SkillName); - Assert.Equal("SendEmail", step.Name); - Assert.Equal("$TRANSLATED_SUMMARY", step.Parameters["input"]); - Assert.Equal("$EMAIL_ADDRESS", step.Parameters["email_address"]); - } - ); - } - - private readonly IConfigurationRoot _configuration; -} diff --git a/dotnet/src/IntegrationTests/Planning/SequentialPlanner/SequentialPlannerTests.cs b/dotnet/src/IntegrationTests/Planning/SequentialPlanner/SequentialPlannerTests.cs deleted file mode 100644 index 5f968f0ad88c..000000000000 --- a/dotnet/src/IntegrationTests/Planning/SequentialPlanner/SequentialPlannerTests.cs +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Planning.Sequential; -using SemanticKernel.IntegrationTests.Fakes; -using SemanticKernel.IntegrationTests.TestSettings; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.IntegrationTests.Planning.SequentialPlanner; - -public sealed class SequentialPlannerTests : IDisposable -{ - public SequentialPlannerTests(ITestOutputHelper output) - { - this._logger = NullLogger.Instance; //new XunitLogger(output); - this._testOutputHelper = new RedirectOutput(output); - - // Load configuration - this._configuration = new ConfigurationBuilder() - .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables() - .AddUserSecrets() - .Build(); - } - - [Theory] - [InlineData(false, "Write a joke and send it in an e-mail to Kai.", "SendEmail", "_GLOBAL_FUNCTIONS_")] - [InlineData(true, "Write a joke and send it in an e-mail to Kai.", "SendEmail", "_GLOBAL_FUNCTIONS_")] - public async Task CreatePlanFunctionFlowAsync(bool useChatModel, string prompt, string expectedFunction, string expectedSkill) - { - // Arrange - bool useEmbeddings = false; - IKernel kernel = this.InitializeKernel(useEmbeddings, useChatModel); - _ = kernel.ImportSkill(new EmailSkillFake()); - TestHelpers.GetSkills(kernel, "FunSkill"); - - var planner = new Microsoft.SemanticKernel.Planning.SequentialPlanner(kernel); - - // Act - var plan = await planner.CreatePlanAsync(prompt); - - // Assert - Assert.Contains( - plan.Steps, - step => - step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && - step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); - } - - [Theory] - [InlineData("Write a novel about software development that is 3 chapters long.", "NovelOutline", "WriterSkill", "")] - public async Task CreatePlanWithDefaultsAsync(string prompt, string expectedFunction, string expectedSkill, string expectedDefault) - { - // Arrange - IKernel kernel = this.InitializeKernel(); - TestHelpers.GetSkills(kernel, "WriterSkill"); - - var planner = new Microsoft.SemanticKernel.Planning.SequentialPlanner(kernel); - - // Act - var plan = await planner.CreatePlanAsync(prompt); - - // Assert - Assert.Contains( - plan.Steps, - step => - step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && - step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase) && - step.Parameters["endMarker"].Equals(expectedDefault, StringComparison.OrdinalIgnoreCase)); - } - - [Theory] - [InlineData("Write a poem or joke and send it in an e-mail to Kai.", "SendEmail", "_GLOBAL_FUNCTIONS_")] - public async Task CreatePlanGoalRelevantAsync(string prompt, string expectedFunction, string expectedSkill) - { - // Arrange - bool useEmbeddings = true; - IKernel kernel = this.InitializeKernel(useEmbeddings); - _ = kernel.ImportSkill(new EmailSkillFake()); - - // Import all sample skills available for demonstration purposes. - TestHelpers.ImportSampleSkills(kernel); - - var planner = new Microsoft.SemanticKernel.Planning.SequentialPlanner(kernel, - new SequentialPlannerConfig { RelevancyThreshold = 0.65, MaxRelevantFunctions = 30, Memory = kernel.Memory }); - - // Act - var plan = await planner.CreatePlanAsync(prompt); - - // Assert - Assert.Contains( - plan.Steps, - step => - step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && - step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); - } - - private IKernel InitializeKernel(bool useEmbeddings = false, bool useChatModel = false) - { - AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); - Assert.NotNull(azureOpenAIEmbeddingsConfiguration); - - var builder = Kernel.Builder.WithLogger(this._logger); - - if (useChatModel) - { - builder.WithAzureChatCompletionService( - deploymentName: azureOpenAIConfiguration.ChatDeploymentName!, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - } - else - { - builder.WithAzureTextCompletionService( - deploymentName: azureOpenAIConfiguration.DeploymentName, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - } - - if (useEmbeddings) - { - builder.WithAzureTextEmbeddingGenerationService( - deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, - endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, - apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey) - .WithMemoryStorage(new VolatileMemoryStore()); - } - - var kernel = builder.Build(); - - return kernel; - } - - private readonly ILogger _logger; - private readonly RedirectOutput _testOutputHelper; - private readonly IConfigurationRoot _configuration; - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - ~SequentialPlannerTests() - { - this.Dispose(false); - } - - private void Dispose(bool disposing) - { - if (disposing) - { - if (this._logger is IDisposable ld) - { - ld.Dispose(); - } - - this._testOutputHelper.Dispose(); - } - } -} diff --git a/dotnet/src/IntegrationTests/Planning/StepwisePlanner/StepwisePlannerTests.cs b/dotnet/src/IntegrationTests/Planning/StepwisePlanner/StepwisePlannerTests.cs deleted file mode 100644 index 7982ca2a42b1..000000000000 --- a/dotnet/src/IntegrationTests/Planning/StepwisePlanner/StepwisePlannerTests.cs +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Text.Json; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Planning.Stepwise; -using Microsoft.SemanticKernel.Skills.Core; -using Microsoft.SemanticKernel.Skills.Web; -using Microsoft.SemanticKernel.Skills.Web.Bing; -using SemanticKernel.IntegrationTests.TestSettings; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.IntegrationTests.Planning.StepwisePlanner; - -public sealed class StepwisePlannerTests : IDisposable -{ - private readonly string _bingApiKey; - - public StepwisePlannerTests(ITestOutputHelper output) - { - this._logger = NullLogger.Instance; //new XunitLogger(output); - this._testOutputHelper = new RedirectOutput(output); - - // Load configuration - this._configuration = new ConfigurationBuilder() - .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables() - .AddUserSecrets() - .Build(); - - string? bingApiKeyCandidate = this._configuration["Bing:ApiKey"]; - Assert.NotNull(bingApiKeyCandidate); - this._bingApiKey = bingApiKeyCandidate; - } - - [Theory] - [InlineData(false, "Who is the current president of the United States? What is his current age divided by 2", "ExecutePlan", "StepwisePlanner")] - [InlineData(true, "Who is the current president of the United States? What is his current age divided by 2", "ExecutePlan", "StepwisePlanner")] - public void CanCreateStepwisePlan(bool useChatModel, string prompt, string expectedFunction, string expectedSkill) - { - // Arrange - bool useEmbeddings = false; - IKernel kernel = this.InitializeKernel(useEmbeddings, useChatModel); - var bingConnector = new BingConnector(this._bingApiKey); - var webSearchEngineSkill = new WebSearchEngineSkill(bingConnector); - kernel.ImportSkill(webSearchEngineSkill, "WebSearch"); - kernel.ImportSkill(new TimeSkill(), "time"); - - var planner = new Microsoft.SemanticKernel.Planning.StepwisePlanner(kernel, new StepwisePlannerConfig() { MaxIterations = 10 }); - - // Act - var plan = planner.CreatePlan(prompt); - - // Assert - Assert.Contains( - plan.Steps, - step => - step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && - step.SkillName.Contains(expectedSkill, StringComparison.OrdinalIgnoreCase)); - } - - [Theory] - [InlineData(false, "Who is the current president of the United States? What is his current age divided by 2")] - // [InlineData(true, "Who is the current president of the United States? What is his current age divided by 2")] // Chat tests take long - public async void CanExecuteStepwisePlan(bool useChatModel, string prompt) - { - // Arrange - bool useEmbeddings = false; - IKernel kernel = this.InitializeKernel(useEmbeddings, useChatModel); - var bingConnector = new BingConnector(this._bingApiKey); - var webSearchEngineSkill = new WebSearchEngineSkill(bingConnector); - kernel.ImportSkill(webSearchEngineSkill, "WebSearch"); - kernel.ImportSkill(new TimeSkill(), "time"); - - var planner = new Microsoft.SemanticKernel.Planning.StepwisePlanner(kernel, new StepwisePlannerConfig() { MaxIterations = 10 }); - - // Act - var plan = planner.CreatePlan(prompt); - var result = await plan.InvokeAsync(); - - // Assert - // Loose assertion -- we just want to make sure that the plan was executed and that the result contains the name of the current president. - // Calculations often wrong. - Assert.Contains("Biden", result.Result, StringComparison.InvariantCultureIgnoreCase); - - Assert.True(result.Variables.TryGetValue("stepsTaken", out string? stepsTakenString)); - var stepsTaken = JsonSerializer.Deserialize>(stepsTakenString!); - Assert.NotNull(stepsTaken); - Assert.True(stepsTaken.Count >= 3 && stepsTaken.Count <= 10, $"Actual: {stepsTaken.Count}. Expected at least 3 steps and at most 10 steps to be taken."); - } - - private IKernel InitializeKernel(bool useEmbeddings = false, bool useChatModel = false) - { - AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); - Assert.NotNull(azureOpenAIEmbeddingsConfiguration); - - var builder = Kernel.Builder.WithLogger(this._logger); - - if (useChatModel) - { - builder.WithAzureChatCompletionService( - deploymentName: azureOpenAIConfiguration.ChatDeploymentName!, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - } - else - { - builder.WithAzureTextCompletionService( - deploymentName: azureOpenAIConfiguration.DeploymentName, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - } - - if (useEmbeddings) - { - builder.WithAzureTextEmbeddingGenerationService( - deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, - endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, - apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey) - .WithMemoryStorage(new VolatileMemoryStore()); - } - - var kernel = builder.Build(); - - return kernel; - } - - private readonly ILogger _logger; - private readonly RedirectOutput _testOutputHelper; - private readonly IConfigurationRoot _configuration; - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - ~StepwisePlannerTests() - { - this.Dispose(false); - } - - private void Dispose(bool disposing) - { - if (disposing) - { - if (this._logger is IDisposable ld) - { - ld.Dispose(); - } - - this._testOutputHelper.Dispose(); - } - } -} diff --git a/dotnet/src/IntegrationTests/Plugins/PluginTests.cs b/dotnet/src/IntegrationTests/Plugins/PluginTests.cs new file mode 100644 index 000000000000..bde065e44624 --- /dev/null +++ b/dotnet/src/IntegrationTests/Plugins/PluginTests.cs @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; +using Microsoft.SemanticKernel.Orchestration; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Plugins; +public class PluginTests +{ + [Theory] + [InlineData("https://www.klarna.com/.well-known/ai-plugin.json", "Klarna", "productsUsingGET", "Laptop", 3, 200, "US")] + [InlineData("https://www.klarna.com/us/shopping/public/openai/v0/api-docs/", "Klarna", "productsUsingGET", "Laptop", 3, 200, "US")] + public async Task QueryKlarnaPluginAsync( + string pluginEndpoint, + string name, + string functionName, + string query, + int size, + int budget, + string countryCode) + { + // Arrange + var kernel = new KernelBuilder().Build(); + using HttpClient httpClient = new(); + + var plugin = await kernel.ImportPluginFunctionsAsync( + name, + new Uri(pluginEndpoint), + new OpenApiFunctionExecutionParameters(httpClient)); + + var contextVariables = new ContextVariables(); + contextVariables["q"] = query; + contextVariables["size"] = size.ToString(System.Globalization.CultureInfo.InvariantCulture); + contextVariables["budget"] = budget.ToString(System.Globalization.CultureInfo.InvariantCulture); + contextVariables["countryCode"] = countryCode; + + // Act + await plugin[functionName].InvokeAsync(kernel.CreateNewContext(contextVariables)); + } + + [Theory] + [InlineData("https://raw.githubusercontent.com/sisbell/chatgpt-plugin-store/main/manifests/instacart.com.json", + "Instacart", + "create", + "{\"title\":\"Shopping List\", \"ingredients\": [\"Flour\"], \"question\": \"what ingredients do I need to make chocolate cookies?\", \"partnerName\": \"OpenAI\" }" + )] + public async Task QueryInstacartPluginAsync( + string pluginEndpoint, + string name, + string functionName, + string payload) + { + // Arrange + var kernel = new KernelBuilder().Build(); + using HttpClient httpClient = new(); + + //note that this plugin is not compliant according to the underlying validator in SK + var plugin = await kernel.ImportPluginFunctionsAsync( + name, + new Uri(pluginEndpoint), + new OpenApiFunctionExecutionParameters(httpClient) { IgnoreNonCompliantErrors = true }); + + var contextVariables = new ContextVariables(); + contextVariables["payload"] = payload; + + // Act + await plugin[functionName].InvokeAsync(kernel.CreateNewContext(contextVariables)); + } + + [Theory] + [InlineData("Plugins/instacart-ai-plugin.json", + "Instacart", + "create", + "{\"title\":\"Shopping List\", \"ingredients\": [\"Flour\"], \"question\": \"what ingredients do I need to make chocolate cookies?\", \"partnerName\": \"OpenAI\" }" + )] + public async Task QueryInstacartPluginFromStreamAsync( + string pluginFilePath, + string name, + string functionName, + string payload) + { + // Arrange + using (var stream = System.IO.File.OpenRead(pluginFilePath)) + { + var kernel = new KernelBuilder().Build(); + using HttpClient httpClient = new(); + + //note that this plugin is not compliant according to the underlying validator in SK + var plugin = await kernel.ImportPluginFunctionsAsync( + name, + stream, + new OpenApiFunctionExecutionParameters(httpClient) { IgnoreNonCompliantErrors = true }); + + var contextVariables = new ContextVariables(); + contextVariables["payload"] = payload; + + // Act + await plugin[functionName].InvokeAsync(kernel.CreateNewContext(contextVariables)); + } + } + + [Theory] + [InlineData("Plugins/instacart-ai-plugin.json", + "Instacart", + "create", + "{\"title\":\"Shopping List\", \"ingredients\": [\"Flour\"], \"question\": \"what ingredients do I need to make chocolate cookies?\", \"partnerName\": \"OpenAI\" }" + )] + public async Task QueryInstacartPluginUsingRelativeFilePathAsync( + string pluginFilePath, + string name, + string functionName, + string payload) + { + // Arrange + var kernel = new KernelBuilder().Build(); + using HttpClient httpClient = new(); + + //note that this plugin is not compliant according to the underlying validator in SK + var plugin = await kernel.ImportPluginFunctionsAsync( + name, + pluginFilePath, + new OpenApiFunctionExecutionParameters(httpClient) { IgnoreNonCompliantErrors = true }); + + var contextVariables = new ContextVariables(); + contextVariables["payload"] = payload; + + // Act + await plugin[functionName].InvokeAsync(kernel.CreateNewContext(contextVariables)); + } +} diff --git a/dotnet/src/IntegrationTests/Plugins/SamplePluginsTests.cs b/dotnet/src/IntegrationTests/Plugins/SamplePluginsTests.cs new file mode 100644 index 000000000000..d4817e2d6173 --- /dev/null +++ b/dotnet/src/IntegrationTests/Plugins/SamplePluginsTests.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; +using Microsoft.SemanticKernel; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Plugins; +public class SamplePluginsTests +{ + [Fact] + public void CanLoadSamplePluginsRequestSettings() + { + // Arrange + var kernel = new KernelBuilder().Build(); + + // Act + TestHelpers.ImportAllSamplePlugins(kernel); + + // Assert + Assert.NotNull(kernel.Functions); + var functionViews = kernel.Functions.GetFunctionViews(); + Assert.NotNull(functionViews); + Assert.Equal(48, functionViews.Count); // currently we have 48 sample plugin functions + functionViews.ToList().ForEach(view => + { + var function = kernel.Functions.GetFunction(view.PluginName, view.Name); + Assert.NotNull(function); + Assert.NotNull(function.RequestSettings); + Assert.True(function.RequestSettings.ExtensionData.ContainsKey("max_tokens")); + }); + } + + [Fact] + // Including this to ensure backward compatibility as tools like Prompt Factory still use the old format + public void CanLoadSampleSkillsCompletions() + { + // Arrange + var kernel = new KernelBuilder().Build(); + + // Act + TestHelpers.ImportAllSampleSkills(kernel); + + // Assert + Assert.NotNull(kernel.Functions); + var functionViews = kernel.Functions.GetFunctionViews(); + Assert.NotNull(functionViews); + Assert.Equal(48, functionViews.Count); // currently we have 48 sample plugin functions + functionViews.ToList().ForEach(view => + { + var function = kernel.Functions.GetFunction(view.PluginName, view.Name); + Assert.NotNull(function); + Assert.NotNull(function.RequestSettings); + Assert.True(function.RequestSettings.ExtensionData.ContainsKey("max_tokens")); + }); + } +} diff --git a/dotnet/src/IntegrationTests/Plugins/instacart-ai-plugin.json b/dotnet/src/IntegrationTests/Plugins/instacart-ai-plugin.json new file mode 100644 index 000000000000..3dc0bcb3925e --- /dev/null +++ b/dotnet/src/IntegrationTests/Plugins/instacart-ai-plugin.json @@ -0,0 +1,17 @@ +{ + "schema_version": "v1", + "name_for_model": "Instacart", + "name_for_human": "Instacart", + "description_for_model": "The Instacart plugin can be used to create a shopping list when a recipe or meal plan is mentioned. Exclude items the user already has from the shopping list. If multiple options for ingredients are provided, separate them into individual ingredients in the shopping list. Always save the user's question in the question field. Always populate the 'partner_name' field with value 'OpenAI'. Here are some examples of food or grocery related queries where the Instacart plugin can be used: chocolate cake recipe, dinner plan for a vegetarian family of 4, instructions to make orange chicken, healthy snacks for kids to bring to school, vegetarian dinner party dish ideas, lunch ideas for a picnic in the park, how to make lasagna, dinner ideas, meal plan for a family of 4, breakfast suggestions, snack ideas for a movie night with friends, squash and pasta dish with additional ingredient suggestions. Parse all ingredients referenced, including optional ingredients, as separate ingredients so that they can be correctly interpreted as unique items by the plugin. Do include optional ingredients and separate the items in the shopping list. Always keep the ingredient quantity. Ask the users if they want to purchase the ingredients and related items on Instacart, and if users agree, generate a shopping list using the Instacart plugin and response with: Here's your Instacart shopping list for [Recipe Name]. \n\nAdditional guidelines and examples: \n\n**Guideline**\n- For ingredient containing bundled items such as 'butter or margarine', 'chicken broth or wine', 'mixed nuts (e.g. ...)', 'vegetables (...)', and 'Optional toppings (...)' you need to separate the ingredient into individual items in the shopping list, e.g. 'butter or margarine' becomes ['butter', 'margarine'] in the shopping list, 'Raw vegetables (e.g., carrot sticks, cucumber slices, bell pepper strips)' becomes ['carrot sticks', 'cucumber slices', 'bell pepper strips']\n- If users say they have something, mark it as \"you already have\" in the list and don't add it to the shopping list\n\nExample 1: \nuser: garlic butter shrimp recipe \nassistant: Here's a delicious garlic butter shrimp recipe: Ingredients: 1 pound large shrimp ... 1/4 cup chicken broth or white wine (optional) Salt and pepper to taste ... \n**Note that the shopping list should contain ['1/4 cup chicken broth', '1/4 cup white wine', 'Salt', 'pepper', ...] instead of ['1/4 cup chicken broth or white wine (optional)', 'Salt and pepper to taste', ...]\n\nExample 2: \nuser: I have squash and pasta. what can I make and what other ingredients do I need? \nassistant: You can make a delicious squash and pasta dish with just a few additional ingredients. Here's a simple recipe: Ingredients: Squash (you already have) Pasta (you already have) Olive oil onion garlic Salt and pepper, ... \n**Note that the shopping list should contain ['Olive oil', 'onion', 'garlic', 'salt', 'pepper', ...] but without 'Squash' or 'Pasta' in it since user has them already.", + "description_for_human": "What’s cookin'? Ask about recipes, meal plans, & more -- and get ingredients delivered from 40,000+ stores!", + "auth": { + "type": "none" + }, + "api": { + "type": "openapi", + "url": "https://www.instacart.com/rest/llm_integration/config/openapi.yaml" + }, + "logo_url": "https://www.instacart.com/assets/beetstrap/brand/2022/carrotlogo-1286c257354036d178c09e815906198eb7f012b8cdc4f6f8ec86d3e64d799a5b.png", + "contact_email": "help@instacart.com", + "legal_info_url": "https://www.instacart.com/terms" +} \ No newline at end of file diff --git a/dotnet/src/IntegrationTests/README.md b/dotnet/src/IntegrationTests/README.md index 9edb16e85896..8f5d0aa2252a 100644 --- a/dotnet/src/IntegrationTests/README.md +++ b/dotnet/src/IntegrationTests/README.md @@ -1,15 +1,16 @@ -# Azure/OpenAI Skill Integration Tests +# Integration Tests ## Requirements 1. **Azure OpenAI**: go to the [Azure OpenAI Quickstart](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/quickstart) and deploy an instance of Azure OpenAI, deploy a model like "text-davinci-003" find your Endpoint and API key. -2. **OpenAI**: go to [OpenAI](https://openai.com/api/) to register and procure your API key. +2. **OpenAI**: go to [OpenAI](https://openai.com/product/) to register and procure your API key. 3. **HuggingFace API key**: see https://huggingface.co/docs/huggingface_hub/guides/inference for details. 4. **Azure Bing Web Search API**: go to [Bing Web Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api) and select `Try Now` to get started. 5. **Oobabooga Text generation web UI**: Follow the [installation instructions](https://github.com/oobabooga/text-generation-webui#installation) to get a local Oobabooga instance running. Follow the [download instructions](https://github.com/oobabooga/text-generation-webui#downloading-models) to install a test model e.g. `python download-model.py gpt2`. Follow the [starting instructions](https://github.com/oobabooga/text-generation-webui#starting-the-web-ui) to start your local instance, enabling API, e.g. `python server.py --model gpt2 --listen --api --api-blocking-port "5000" --api-streaming-port "5005"`. Note that `--model` parameter is optional and models can be downloaded and hot swapped using exclusively the web UI, making it easy to test various models. -5. **Postgres**: start a postgres with the [pgvector](https://github.com/pgvector/pgvector) extension installed. You can easily do it using the docker image [ankane/pgvector](https://hub.docker.com/r/ankane/pgvector). +6. **Postgres**: start a postgres with the [pgvector](https://github.com/pgvector/pgvector) extension installed. You can easily do it using the docker image [ankane/pgvector](https://hub.docker.com/r/ankane/pgvector). +7. **Weaviate**: go to `IntegrationTests/Connectors/Weaviate` where `docker-compose.yml` is located and run `docker-compose up --build`. ## Setup diff --git a/dotnet/src/IntegrationTests/RedirectOutput.cs b/dotnet/src/IntegrationTests/RedirectOutput.cs index 64c374c57c78..1905b5849ea6 100644 --- a/dotnet/src/IntegrationTests/RedirectOutput.cs +++ b/dotnet/src/IntegrationTests/RedirectOutput.cs @@ -8,7 +8,7 @@ namespace SemanticKernel.IntegrationTests; -public class RedirectOutput : TextWriter, ILogger +public class RedirectOutput : TextWriter, ILogger, ILoggerFactory { private readonly ITestOutputHelper _output; private readonly StringBuilder _logs; @@ -27,7 +27,7 @@ public override void WriteLine(string? value) this._logs.AppendLine(value); } - public IDisposable BeginScope(TState state) + public IDisposable BeginScope(TState state) where TState : notnull { return null!; } @@ -48,4 +48,8 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except this._output?.WriteLine(message); this._logs.AppendLine(message); } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) => throw new NotSupportedException(); } diff --git a/dotnet/src/IntegrationTests/TemplateLanguage/PromptTemplateEngineTests.cs b/dotnet/src/IntegrationTests/TemplateLanguage/PromptTemplateEngineTests.cs index 00d7e55c57fc..2bbd8220e38d 100644 --- a/dotnet/src/IntegrationTests/TemplateLanguage/PromptTemplateEngineTests.cs +++ b/dotnet/src/IntegrationTests/TemplateLanguage/PromptTemplateEngineTests.cs @@ -6,8 +6,8 @@ using System.IO; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.TemplateEngine; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.TemplateEngine.Basic; using Xunit; using Xunit.Abstractions; @@ -21,7 +21,7 @@ public sealed class PromptTemplateEngineTests : IDisposable public PromptTemplateEngineTests(ITestOutputHelper output) { this._logger = new RedirectOutput(output); - this._target = new PromptTemplateEngine(); + this._target = new BasicPromptTemplateEngine(); } [Fact] @@ -70,7 +70,7 @@ public async Task ItAllowsToPassVariablesToFunctionsAsync() // Arrange const string Template = "== {{my.check123 $call}} =="; var kernel = Kernel.Builder.Build(); - kernel.ImportSkill(new MySkill(), "my"); + kernel.ImportFunctions(new MyPlugin(), "my"); var context = kernel.CreateNewContext(); context.Variables["call"] = "123"; @@ -87,7 +87,7 @@ public async Task ItAllowsToPassValuesToFunctionsAsync() // Arrange const string Template = "== {{my.check123 '234'}} =="; var kernel = Kernel.Builder.Build(); - kernel.ImportSkill(new MySkill(), "my"); + kernel.ImportFunctions(new MyPlugin(), "my"); var context = kernel.CreateNewContext(); // Act @@ -104,7 +104,7 @@ public async Task ItAllowsToPassEscapedValues1ToFunctionsAsync() const char Esc = '\\'; string template = "== {{my.check123 'a" + Esc + "'b'}} =="; var kernel = Kernel.Builder.Build(); - kernel.ImportSkill(new MySkill(), "my"); + kernel.ImportFunctions(new MyPlugin(), "my"); var context = kernel.CreateNewContext(); // Act @@ -121,7 +121,7 @@ public async Task ItAllowsToPassEscapedValues2ToFunctionsAsync() const char Esc = '\\'; string template = "== {{my.check123 \"a" + Esc + "\"b\"}} =="; var kernel = Kernel.Builder.Build(); - kernel.ImportSkill(new MySkill(), "my"); + kernel.ImportFunctions(new MyPlugin(), "my"); var context = kernel.CreateNewContext(); // Act @@ -131,20 +131,37 @@ public async Task ItAllowsToPassEscapedValues2ToFunctionsAsync() Assert.Equal("== a\"b != 123 ==", result); } + [Fact] + public async Task ItHandlesNamedArgsAsync() + { + // Arrange + string template = "Output: {{my.sayAge name=\"Mario\" birthdate=$birthdate exclamation='Wow, that\\'s surprising'}}"; + var kernel = Kernel.Builder.Build(); + kernel.ImportFunctions(new MyPlugin(), "my"); + var context = kernel.CreateNewContext(); + context.Variables["birthdate"] = "1981-08-20T00:00:00"; + + // Act + var result = await this._target.RenderAsync(template, context); + + // Assert + Assert.Equal("Output: Mario is 42 today. Wow, that's surprising!", result); + } + [Theory] [MemberData(nameof(GetTemplateLanguageTests))] public async Task ItHandleEdgeCasesAsync(string template, string expectedResult) { // Arrange var kernel = Kernel.Builder.Build(); - kernel.ImportSkill(new MySkill()); + kernel.ImportFunctions(new MyPlugin()); // Act this._logger.WriteLine("template: " + template); this._logger.WriteLine("expected: " + expectedResult); if (expectedResult.StartsWith("ERROR", StringComparison.OrdinalIgnoreCase)) { - await Assert.ThrowsAsync( + await Assert.ThrowsAsync( async () => await this._target.RenderAsync(template, kernel.CreateNewContext())); } else @@ -162,7 +179,7 @@ public static IEnumerable GetTemplateLanguageTests() return GetTestData("TemplateLanguage/tests.txt"); } - public class MySkill + public class MyPlugin { [SKFunction, Description("This is a test"), SKName("check123")] public string MyFunction(string input) @@ -171,16 +188,25 @@ public string MyFunction(string input) } [SKFunction, Description("This is a test"), SKName("asis")] - public string MyFunction2(string input) + public string? MyFunction2(string? input = null) { return input; } + + [SKFunction, Description("This is a test"), SKName("sayAge")] + public string MyFunction3(string name, DateTime birthdate, string exclamation) + { + var today = new DateTime(2023, 8, 25); + TimeSpan timespan = today - birthdate; + int age = (int)(timespan.TotalDays / 365.25); + return $"{name} is {age} today. {exclamation}!"; + } } #region internals private readonly RedirectOutput _logger; - private readonly PromptTemplateEngine _target; + private readonly BasicPromptTemplateEngine _target; private static IEnumerable GetTestData(string file) { diff --git a/dotnet/src/IntegrationTests/TestHelpers.cs b/dotnet/src/IntegrationTests/TestHelpers.cs index d97f77bbb307..e48a925913e0 100644 --- a/dotnet/src/IntegrationTests/TestHelpers.cs +++ b/dotnet/src/IntegrationTests/TestHelpers.cs @@ -5,15 +5,30 @@ using System.IO; using System.Reflection; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.SkillDefinition; namespace SemanticKernel.IntegrationTests; internal static class TestHelpers { - internal static void ImportSampleSkills(IKernel target) + internal static void ImportAllSamplePlugins(IKernel kernel) { - var chatSkill = GetSkills(target, + ImportSampleSemanticFunctions(kernel, "../../../../../../samples/plugins", + "ChatPlugin", + "SummarizePlugin", + "WriterPlugin", + "CalendarPlugin", + "ChildrensBookPlugin", + "ClassificationPlugin", + "CodingPlugin", + "FunPlugin", + "IntentDetectionPlugin", + "MiscPlugin", + "QAPlugin"); + } + + internal static void ImportAllSampleSkills(IKernel kernel) + { + ImportSampleSemanticFunctions(kernel, "../../../../../../samples/skills", "ChatSkill", "SummarizeSkill", "WriterSkill", @@ -27,7 +42,12 @@ internal static void ImportSampleSkills(IKernel target) "QASkill"); } - internal static IDictionary GetSkills(IKernel target, params string[] skillNames) + internal static IDictionary ImportSamplePlugins(IKernel kernel, params string[] pluginNames) + { + return ImportSampleSemanticFunctions(kernel, "../../../../../../samples/plugins", pluginNames); + } + + internal static IDictionary ImportSampleSemanticFunctions(IKernel kernel, string path, params string[] pluginNames) { string? currentAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); if (string.IsNullOrWhiteSpace(currentAssemblyDirectory)) @@ -35,8 +55,8 @@ internal static IDictionary GetSkills(IKernel target, param throw new InvalidOperationException("Unable to determine current assembly directory."); } - string skillParentDirectory = Path.GetFullPath(Path.Combine(currentAssemblyDirectory, "../../../../../../samples/skills")); + string parentDirectory = Path.GetFullPath(Path.Combine(currentAssemblyDirectory, path)); - return target.ImportSemanticSkillFromDirectory(skillParentDirectory, skillNames); + return kernel.ImportSemanticFunctionsFromDirectory(parentDirectory, pluginNames); } } diff --git a/dotnet/src/IntegrationTests/WebPlugin/WebPluginTests.cs b/dotnet/src/IntegrationTests/WebPlugin/WebPluginTests.cs new file mode 100644 index 000000000000..8f35c973c61c --- /dev/null +++ b/dotnet/src/IntegrationTests/WebPlugin/WebPluginTests.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Plugins.Web; +using Microsoft.SemanticKernel.Plugins.Web.Bing; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.WebPlugin; + +public sealed class WebPluginTests : IDisposable +{ + private readonly string _bingApiKey; + + public WebPluginTests(ITestOutputHelper output) + { + this._logger = new XunitLogger(output); + this._output = output; + + this._testOutputHelper = new RedirectOutput(output); + Console.SetOut(this._testOutputHelper); + + // Load configuration + IConfigurationRoot configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + string? bingApiKeyCandidate = configuration["Bing:ApiKey"]; + Assert.NotNull(bingApiKeyCandidate); + this._bingApiKey = bingApiKeyCandidate; + } + + [Theory(Skip = "Bing search results not consistent enough for testing.")] + [InlineData("What is generally recognized as the tallest building in Seattle, Washington, USA?", "Columbia Center")] + public async Task BingPluginTestAsync(string prompt, string expectedAnswerContains) + { + // Arrange + IKernel kernel = Kernel.Builder.WithLoggerFactory(this._logger).Build(); + + using XunitLogger connectorLogger = new(this._output); + BingConnector connector = new(this._bingApiKey, connectorLogger); + Assert.NotEmpty(this._bingApiKey); + + WebSearchEnginePlugin plugin = new(connector); + var searchFunctions = kernel.ImportFunctions(plugin, "WebSearchEngine"); + + // Act + KernelResult result = await kernel.RunAsync( + prompt, + searchFunctions["Search"] + ); + + // Assert + Assert.Contains(expectedAnswerContains, result.GetValue(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task WebFileDownloadPluginFileTestAsync() + { + // Arrange + IKernel kernel = Kernel.Builder.WithLoggerFactory(this._logger).Build(); + using XunitLogger pluginLogger = new(this._output); + var plugin = new WebFileDownloadPlugin(pluginLogger); + var downloadFunctions = kernel.ImportFunctions(plugin, "WebFileDownload"); + string fileWhereToSaveWebPage = Path.GetTempFileName(); + var contextVariables = new ContextVariables("https://www.microsoft.com"); + contextVariables.Set(WebFileDownloadPlugin.FilePathParamName, fileWhereToSaveWebPage); + + // Act + await kernel.RunAsync(contextVariables, downloadFunctions["DownloadToFile"]); + + // Assert + var fileInfo = new FileInfo(fileWhereToSaveWebPage); + Assert.True(fileInfo.Length > 0); + + File.Delete(fileWhereToSaveWebPage); + } + + #region internals + + private readonly ITestOutputHelper _output; + private readonly XunitLogger _logger; + private readonly RedirectOutput _testOutputHelper; + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + ~WebPluginTests() + { + this.Dispose(false); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + this._logger.Dispose(); + this._testOutputHelper.Dispose(); + } + } + + #endregion +} diff --git a/dotnet/src/IntegrationTests/WebSkill/WebSkillTests.cs b/dotnet/src/IntegrationTests/WebSkill/WebSkillTests.cs deleted file mode 100644 index 99e7094f16d9..000000000000 --- a/dotnet/src/IntegrationTests/WebSkill/WebSkillTests.cs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.IO; -using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Skills.Web; -using Microsoft.SemanticKernel.Skills.Web.Bing; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.IntegrationTests.WebSkill; - -public sealed class WebSkillTests : IDisposable -{ - private readonly string _bingApiKey; - - public WebSkillTests(ITestOutputHelper output) - { - this._logger = new XunitLogger(output); - this._output = output; - - this._testOutputHelper = new RedirectOutput(output); - Console.SetOut(this._testOutputHelper); - - // Load configuration - IConfigurationRoot configuration = new ConfigurationBuilder() - .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables() - .AddUserSecrets() - .Build(); - - string? bingApiKeyCandidate = configuration["Bing:ApiKey"]; - Assert.NotNull(bingApiKeyCandidate); - this._bingApiKey = bingApiKeyCandidate; - } - - [Theory(Skip = "Bing search results not consistent enough for testing.")] - [InlineData("What is generally recognized as the tallest building in Seattle, Washington, USA?", "Columbia Center")] - public async Task BingSkillTestAsync(string prompt, string expectedAnswerContains) - { - // Arrange - IKernel kernel = Kernel.Builder.WithLogger(this._logger).Build(); - - using XunitLogger connectorLogger = new(this._output); - BingConnector connector = new(this._bingApiKey, connectorLogger); - Assert.NotEmpty(this._bingApiKey); - - WebSearchEngineSkill skill = new(connector); - var search = kernel.ImportSkill(skill, "WebSearchEngine"); - - // Act - SKContext result = await kernel.RunAsync( - prompt, - search["Search"] - ); - - // Assert - Assert.Contains(expectedAnswerContains, result.Result, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public async Task WebFileDownloadSkillFileTestAsync() - { - // Arrange - IKernel kernel = Kernel.Builder.WithLogger(this._logger).Build(); - using XunitLogger skillLogger = new(this._output); - var skill = new WebFileDownloadSkill(skillLogger); - var download = kernel.ImportSkill(skill, "WebFileDownload"); - string fileWhereToSaveWebPage = Path.GetTempFileName(); - var contextVariables = new ContextVariables("https://www.microsoft.com"); - contextVariables.Set(WebFileDownloadSkill.FilePathParamName, fileWhereToSaveWebPage); - - // Act - await kernel.RunAsync(contextVariables, download["DownloadToFile"]); - - // Assert - var fileInfo = new FileInfo(fileWhereToSaveWebPage); - Assert.True(fileInfo.Length > 0); - - File.Delete(fileWhereToSaveWebPage); - } - - #region internals - - private readonly ITestOutputHelper _output; - private readonly XunitLogger _logger; - private readonly RedirectOutput _testOutputHelper; - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - ~WebSkillTests() - { - this.Dispose(false); - } - - private void Dispose(bool disposing) - { - if (disposing) - { - this._logger.Dispose(); - this._testOutputHelper.Dispose(); - } - } - - #endregion -} diff --git a/dotnet/src/IntegrationTests/XunitLogger.cs b/dotnet/src/IntegrationTests/XunitLogger.cs index 9118d31676a8..b1f97444ba86 100644 --- a/dotnet/src/IntegrationTests/XunitLogger.cs +++ b/dotnet/src/IntegrationTests/XunitLogger.cs @@ -9,7 +9,7 @@ namespace SemanticKernel.IntegrationTests; /// /// A logger that writes to the Xunit test output /// -internal sealed class XunitLogger : ILogger, IDisposable +internal sealed class XunitLogger : ILoggerFactory, ILogger, IDisposable { private readonly ITestOutputHelper _output; @@ -28,7 +28,7 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except public bool IsEnabled(LogLevel logLevel) => true; /// - public IDisposable BeginScope(TState state) + public IDisposable BeginScope(TState state) where TState : notnull => this; /// @@ -37,4 +37,8 @@ public void Dispose() // This class is marked as disposable to support the BeginScope method. // However, there is no need to dispose anything. } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) => throw new NotSupportedException(); } diff --git a/dotnet/src/InternalUtilities/src/Diagnostics/ExceptionExtensions.cs b/dotnet/src/InternalUtilities/src/Diagnostics/ExceptionExtensions.cs index 4581dc753120..dca7ebadaacc 100644 --- a/dotnet/src/InternalUtilities/src/Diagnostics/ExceptionExtensions.cs +++ b/dotnet/src/InternalUtilities/src/Diagnostics/ExceptionExtensions.cs @@ -18,12 +18,10 @@ internal static class ExceptionExtensions /// Exception. /// True if is a critical exception and should not be caught. internal static bool IsCriticalException(this Exception ex) - => ex is OutOfMemoryException - or ThreadAbortException + => ex is ThreadAbortException or AccessViolationException or AppDomainUnloadedException or BadImageFormatException or CannotUnloadAppDomainException - or InvalidProgramException - or StackOverflowException; + or InvalidProgramException; } diff --git a/dotnet/src/InternalUtilities/src/Diagnostics/IsExternalInit.cs b/dotnet/src/InternalUtilities/src/Diagnostics/IsExternalInit.cs new file mode 100644 index 000000000000..43f0c25312d8 --- /dev/null +++ b/dotnet/src/InternalUtilities/src/Diagnostics/IsExternalInit.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; + +namespace System.Runtime.CompilerServices; + +/// +/// Reserved to be used by the compiler for tracking metadata. +/// This class should not be used by developers in source code. +/// +[EditorBrowsable(EditorBrowsableState.Never)] +internal static class IsExternalInit +{ +} diff --git a/dotnet/src/InternalUtilities/src/Diagnostics/Verify.cs b/dotnet/src/InternalUtilities/src/Diagnostics/Verify.cs index 79ad183bd094..c19bfa19c7a0 100644 --- a/dotnet/src/InternalUtilities/src/Diagnostics/Verify.cs +++ b/dotnet/src/InternalUtilities/src/Diagnostics/Verify.cs @@ -7,7 +7,6 @@ using System.IO; using System.Runtime.CompilerServices; using System.Text.RegularExpressions; -using Microsoft.SemanticKernel.SkillDefinition; namespace Microsoft.SemanticKernel.Diagnostics; @@ -37,12 +36,12 @@ internal static void NotNullOrWhiteSpace([NotNull] string? str, [CallerArgumentE } } - internal static void ValidSkillName([NotNull] string? skillName) + internal static void ValidPluginName([NotNull] string? pluginName) { - NotNullOrWhiteSpace(skillName); - if (!s_asciiLettersDigitsUnderscoresRegex.IsMatch(skillName)) + NotNullOrWhiteSpace(pluginName); + if (!s_asciiLettersDigitsUnderscoresRegex.IsMatch(pluginName)) { - ThrowInvalidName("skill name", skillName); + ThrowInvalidName("plugin name", pluginName); } } @@ -84,7 +83,7 @@ internal static void DirectoryExists(string path) /// Make sure every function parameter name is unique /// /// List of parameters - internal static void ParametersUniqueness(IList parameters) + internal static void ParametersUniqueness(IReadOnlyList parameters) { int count = parameters.Count; if (count > 0) @@ -108,9 +107,7 @@ internal static void ParametersUniqueness(IList parameters) if (!seen.Add(p.Name)) { - throw new KernelException( - KernelException.ErrorCodes.InvalidFunctionDescription, - $"The function has two or more parameters with the same name '{p.Name}'"); + throw new SKException($"The function has two or more parameters with the same name '{p.Name}'"); } } } @@ -118,9 +115,7 @@ internal static void ParametersUniqueness(IList parameters) [DoesNotReturn] private static void ThrowInvalidName(string kind, string name) => - throw new KernelException( - KernelException.ErrorCodes.InvalidFunctionDescription, - $"A {kind} can contain only ASCII letters, digits, and underscores: '{name}' is not a valid name."); + throw new SKException($"A {kind} can contain only ASCII letters, digits, and underscores: '{name}' is not a valid name."); [DoesNotReturn] internal static void ThrowArgumentNullException(string? paramName) => diff --git a/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs b/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs new file mode 100644 index 000000000000..964bef838399 --- /dev/null +++ b/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; + +internal static class HttpClientExtensions +{ + /// + /// Sends an HTTP request using the provided instance and checks for a successful response. + /// If the response is not successful, it logs an error and throws an . + /// + /// The instance to use for sending the request. + /// The to send. + /// Indicates if HttpClient operations should be considered completed either as soon as a response is available, + /// or after reading the entire response message including the content. + /// A for canceling the request. + /// The representing the response. + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "By design. See comment below.")] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2016:Forward the 'CancellationToken' parameter to methods", Justification = "The `ReadAsStringAsync` method in the NetStandard 2.0 version does not have an overload that accepts the cancellation token.")] + internal static async Task SendWithSuccessCheckAsync(this HttpClient client, HttpRequestMessage request, HttpCompletionOption completionOption, CancellationToken cancellationToken) + { + HttpResponseMessage? response = null; + + try + { + response = await client.SendAsync(request, completionOption, cancellationToken).ConfigureAwait(false); + + response.EnsureSuccessStatusCode(); + + return response; + } + catch (HttpRequestException e) + { + string? responseContent = null; + + try + { + responseContent = await response!.Content.ReadAsStringAsync().ConfigureAwait(false); + } + catch { } // We want to suppress any exceptions that occur while reading the content, ensuring that an HttpOperationException is thrown instead. + + throw new HttpOperationException(response?.StatusCode ?? HttpStatusCode.BadRequest, responseContent, e.Message, e); + } + } + + /// + /// Sends an HTTP request using the provided instance and checks for a successful response. + /// If the response is not successful, it logs an error and throws an . + /// + /// The instance to use for sending the request. + /// The to send. + /// A for canceling the request. + /// The representing the response. + internal static async Task SendWithSuccessCheckAsync(this HttpClient client, HttpRequestMessage request, CancellationToken cancellationToken) + { + return await client.SendWithSuccessCheckAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); + } +} diff --git a/dotnet/src/InternalUtilities/src/Http/HttpClientProvider.cs b/dotnet/src/InternalUtilities/src/Http/HttpClientProvider.cs index c62d22ee607c..8acff7db5e04 100644 --- a/dotnet/src/InternalUtilities/src/Http/HttpClientProvider.cs +++ b/dotnet/src/InternalUtilities/src/Http/HttpClientProvider.cs @@ -2,7 +2,7 @@ using System.Net.Http; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Http; /// /// Provides functionality for retrieving instances of HttpClient. @@ -12,17 +12,17 @@ internal static class HttpClientProvider /// /// Retrieves an instance of HttpClient. /// - /// The kernel configuration. + /// The to be used when the HttpClient is not provided already /// An optional pre-existing instance of HttpClient. - /// An optional logger. + /// The to use for logging. If null, no logging will be performed. /// An instance of HttpClient. - public static HttpClient GetHttpClient(KernelConfig config, HttpClient? httpClient, ILogger? logger) + public static HttpClient GetHttpClient(IDelegatingHandlerFactory httpHandlerFactory, HttpClient? httpClient, ILoggerFactory? loggerFactory) { - if (httpClient == null) + if (httpClient is null) { - var retryHandler = config.HttpHandlerFactory.Create(logger); - retryHandler.InnerHandler = NonDisposableHttpClientHandler.Instance; - return new HttpClient(retryHandler, false); // We should refrain from disposing the underlying SK default HttpClient handler as it would impact other HTTP clients that utilize the same handler. + var providedHttpHandler = httpHandlerFactory.Create(loggerFactory); + providedHttpHandler.InnerHandler = NonDisposableHttpClientHandler.Instance; + return new HttpClient(providedHttpHandler, false); // We should refrain from disposing the underlying SK default HttpClient handler as it would impact other HTTP clients that utilize the same handler. } return httpClient; diff --git a/dotnet/src/InternalUtilities/src/Http/HttpContentExtensions.cs b/dotnet/src/InternalUtilities/src/Http/HttpContentExtensions.cs new file mode 100644 index 000000000000..c6f8ea5aa39f --- /dev/null +++ b/dotnet/src/InternalUtilities/src/Http/HttpContentExtensions.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; + +/// +/// Provides extension methods for working with HTTP content in a way that translates HttpRequestExceptions into HttpOperationExceptions. +/// +internal static class HttpContentExtensions +{ + /// + /// Reads the content of the HTTP response as a string and translates any HttpRequestException into an HttpOperationException. + /// + /// The HTTP content to read. + /// A string representation of the HTTP content. + public static async Task ReadAsStringWithExceptionMappingAsync(this HttpContent httpContent) + { + try + { + return await httpContent.ReadAsStringAsync().ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + throw new HttpOperationException(message: ex.Message, innerException: ex); + } + } + + /// + /// Reads the content of the HTTP response as a stream and translates any HttpRequestException into an HttpOperationException. + /// + /// The HTTP content to read. + /// A stream representing the HTTP content. + public static async Task ReadAsStreamAndTranslateExceptionAsync(this HttpContent httpContent) + { + try + { + return await httpContent.ReadAsStreamAsync().ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + throw new HttpOperationException(message: ex.Message, innerException: ex); + } + } + + /// + /// Reads the content of the HTTP response as a byte array and translates any HttpRequestException into an HttpOperationException. + /// + /// The HTTP content to read. + /// A byte array representing the HTTP content. + public static async Task ReadAsByteArrayAndTranslateExceptionAsync(this HttpContent httpContent) + { + try + { + return await httpContent.ReadAsByteArrayAsync().ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + throw new HttpOperationException(message: ex.Message, innerException: ex); + } + } +} diff --git a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs index 164baac1ffa6..206c6e18ae77 100644 --- a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs +++ b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs @@ -5,6 +5,7 @@ using System.Net.Http.Headers; using System.Text; using System.Text.Json; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel; @@ -43,7 +44,7 @@ private static HttpRequestMessage CreateRequest(HttpMethod method, Uri url, obje { byte[] utf8Bytes = payload is string s ? Encoding.UTF8.GetBytes(s) : - JsonSerializer.SerializeToUtf8Bytes(payload); + JsonSerializer.SerializeToUtf8Bytes(payload, s_jsonSerializerOptions); content = new ByteArrayContent(utf8Bytes); content.Headers.ContentType = new MediaTypeHeaderValue("application/json") { CharSet = "utf-8" }; @@ -51,4 +52,13 @@ private static HttpRequestMessage CreateRequest(HttpMethod method, Uri url, obje return content; } + + private static readonly JsonSerializerOptions s_jsonSerializerOptions = CreateSerializerOptions(); + + private static JsonSerializerOptions CreateSerializerOptions() + { + var jso = new JsonSerializerOptions(); + jso.Converters.Add(new ReadOnlyMemoryConverter()); + return jso; + } } diff --git a/dotnet/src/InternalUtilities/src/Linq/AsyncEnumerable.cs b/dotnet/src/InternalUtilities/src/Linq/AsyncEnumerable.cs index fc14a9e7d0be..8c6b081f7d03 100644 --- a/dotnet/src/InternalUtilities/src/Linq/AsyncEnumerable.cs +++ b/dotnet/src/InternalUtilities/src/Linq/AsyncEnumerable.cs @@ -4,9 +4,6 @@ using System.Threading; using System.Threading.Tasks; -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously -#pragma warning disable CA1510 // Use 'ArgumentNullException.ThrowIfNull' (.NET 8) - // Used for compatibility with System.Linq.Async Nuget pkg namespace System.Linq; @@ -14,6 +11,7 @@ internal static class AsyncEnumerable { public static IAsyncEnumerable Empty() => EmptyAsyncEnumerable.Instance; +#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits public static IEnumerable ToEnumerable(this IAsyncEnumerable source, CancellationToken cancellationToken = default) { var enumerator = source.GetAsyncEnumerator(cancellationToken); @@ -29,7 +27,10 @@ public static IEnumerable ToEnumerable(this IAsyncEnumerable source, Ca enumerator.DisposeAsync().AsTask().GetAwaiter().GetResult(); } } +#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits +#pragma warning disable IDE1006 // Naming rule violation: Missing suffix: 'Async' +#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously public static async IAsyncEnumerable ToAsyncEnumerable(this IEnumerable source) { foreach (var item in source) @@ -37,6 +38,8 @@ public static async IAsyncEnumerable ToAsyncEnumerable(this IEnumerable yield return item; } } +#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously +#pragma warning restore IDE1006 // Naming rule violation: Missing suffix: 'Async' public static async ValueTask FirstOrDefaultAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/InternalUtilities/src/System/EnvExtensions.cs b/dotnet/src/InternalUtilities/src/System/EnvExtensions.cs new file mode 100644 index 000000000000..7edc707bdd22 --- /dev/null +++ b/dotnet/src/InternalUtilities/src/System/EnvExtensions.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable IDE0130 // Namespace does not match folder structure +// ReSharper disable once CheckNamespace +namespace System; +#pragma warning restore IDE0130 + +internal static class EnvExtensions +{ + /// + /// Source: https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/src/DiagnosticsOptions.cs + /// Values: https://learn.microsoft.com/en-us/dotnet/api/azure.core.diagnosticsoptions.istelemetryenabled?view=azure-dotnet + /// + internal static bool? GetBoolEnvVar(string name) + { + string? value = Environment.GetEnvironmentVariable(name); + + if (string.Equals(bool.TrueString, value, StringComparison.OrdinalIgnoreCase) || + string.Equals("1", value, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(bool.FalseString, value, StringComparison.OrdinalIgnoreCase) || + string.Equals("0", value, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return null; + } +} diff --git a/dotnet/src/InternalUtilities/src/Text/Json.cs b/dotnet/src/InternalUtilities/src/Text/Json.cs index 73ef89835f07..d85597ba3e1b 100644 --- a/dotnet/src/InternalUtilities/src/Text/Json.cs +++ b/dotnet/src/InternalUtilities/src/Text/Json.cs @@ -6,31 +6,31 @@ namespace Microsoft.SemanticKernel.Text; internal static class Json { - internal static string Serialize(object? o) - { - return JsonSerializer.Serialize(o, s_options); - } + internal static string Serialize(object? o) => JsonSerializer.Serialize(o, s_options); - internal static T? Deserialize(string json) - { - return JsonSerializer.Deserialize(json, s_options); - } + internal static byte[] SerializeToUtf8Bytes(object? o) => JsonSerializer.SerializeToUtf8Bytes(o, s_options); - internal static string ToJson(this object o) - { - return JsonSerializer.Serialize(o, s_options); - } + internal static T? Deserialize(string json) => JsonSerializer.Deserialize(json, s_options); #region private ================================================================================ - private static readonly JsonSerializerOptions s_options = new() + private static readonly JsonSerializerOptions s_options = CreateOptions(); + + private static JsonSerializerOptions CreateOptions() { - WriteIndented = true, - MaxDepth = 20, - AllowTrailingCommas = true, - PropertyNameCaseInsensitive = true, - ReadCommentHandling = JsonCommentHandling.Skip, - }; + JsonSerializerOptions options = new() + { + WriteIndented = true, + MaxDepth = 20, + AllowTrailingCommas = true, + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + + options.Converters.Add(new ReadOnlyMemoryConverter()); + + return options; + } #endregion } diff --git a/dotnet/src/InternalUtilities/src/Text/ReadOnlyMemoryConverter.cs b/dotnet/src/InternalUtilities/src/Text/ReadOnlyMemoryConverter.cs new file mode 100644 index 000000000000..ecce02508e23 --- /dev/null +++ b/dotnet/src/InternalUtilities/src/Text/ReadOnlyMemoryConverter.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Text; + +// .NET 8 and the System.Text.Json v8.0.0 nuget package include built-in support for ReadOnlyMemory. +// This is a temporary workaround for .NET 6 and the System.Text.Json v6.0.0 nuget package. +// It should be removed once SK projects upgrade to System.Text.Json v8.0.0. + +/// Provides a converter for . +internal sealed class ReadOnlyMemoryConverter : JsonConverter> +{ + /// An instance of a converter for float[] that all operations delegate to. + private static readonly JsonConverter s_arrayConverter = (JsonConverter)new JsonSerializerOptions().GetConverter(typeof(float[])); + + public override ReadOnlyMemory Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + s_arrayConverter.Read(ref reader, typeof(float[]), options).AsMemory(); + + public override void Write(Utf8JsonWriter writer, ReadOnlyMemory value, JsonSerializerOptions options) => + // This provides an efficient implementation when the ReadOnlyMemory represents the full length of an array. + // This is the common case for these projects, and thus the implementation doesn't spend more code on a complex + // implementation to efficiently handle slices or instances backed by MemoryManagers. + s_arrayConverter.Write( + writer, + MemoryMarshal.TryGetArray(value, out ArraySegment array) && array.Count == value.Length ? array.Array! : value.ToArray(), + options); +} diff --git a/dotnet/src/InternalUtilities/test/FunctionHelpers.cs b/dotnet/src/InternalUtilities/test/FunctionHelpers.cs index 4aaae63a4c8a..0057c0f027e6 100644 --- a/dotnet/src/InternalUtilities/test/FunctionHelpers.cs +++ b/dotnet/src/InternalUtilities/test/FunctionHelpers.cs @@ -4,7 +4,6 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; namespace SemanticKernel.UnitTests; @@ -12,23 +11,23 @@ namespace SemanticKernel.UnitTests; internal static class FunctionHelpers { /// - /// Invokes a function on a skill instance via the kernel. + /// Invokes a function on a plugin instance via the kernel. /// - public static Task CallViaKernel( - object skillInstance, + public static Task CallViaKernelAsync( + object pluginInstance, string methodName, - params (string Name, string Value)[] variables) + params (string Name, object Value)[] variables) { var kernel = Kernel.Builder.Build(); - IDictionary funcs = kernel.ImportSkill(skillInstance); + IDictionary functions = kernel.ImportFunctions(pluginInstance); SKContext context = kernel.CreateNewContext(); - foreach ((string Name, string Value) pair in variables) + foreach ((string Name, object Value) pair in variables) { - context.Variables.Set(pair.Name, pair.Value); + context.Variables.Set(pair.Name, pair.Value.ToString()); } - return funcs[methodName].InvokeAsync(context); + return kernel.RunAsync(context.Variables, functions[methodName]); } } diff --git a/dotnet/src/InternalUtilities/test/Linq/AsyncEnumerable.cs b/dotnet/src/InternalUtilities/test/Linq/AsyncEnumerable.cs new file mode 100644 index 000000000000..8c6b081f7d03 --- /dev/null +++ b/dotnet/src/InternalUtilities/test/Linq/AsyncEnumerable.cs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +// Used for compatibility with System.Linq.Async Nuget pkg +namespace System.Linq; + +internal static class AsyncEnumerable +{ + public static IAsyncEnumerable Empty() => EmptyAsyncEnumerable.Instance; + +#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits + public static IEnumerable ToEnumerable(this IAsyncEnumerable source, CancellationToken cancellationToken = default) + { + var enumerator = source.GetAsyncEnumerator(cancellationToken); + try + { + while (enumerator.MoveNextAsync().AsTask().GetAwaiter().GetResult()) + { + yield return enumerator.Current; + } + } + finally + { + enumerator.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } + } +#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits + +#pragma warning disable IDE1006 // Naming rule violation: Missing suffix: 'Async' +#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + public static async IAsyncEnumerable ToAsyncEnumerable(this IEnumerable source) + { + foreach (var item in source) + { + yield return item; + } + } +#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously +#pragma warning restore IDE1006 // Naming rule violation: Missing suffix: 'Async' + + public static async ValueTask FirstOrDefaultAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) + { + await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + return item; + } + + return default; + } + + public static async ValueTask LastOrDefaultAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) + { + var last = default(T)!; // NB: Only matters when hasLast is set to true. + var hasLast = false; + + await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + hasLast = true; + last = item; + } + + return hasLast ? last! : default; + } + + public static async ValueTask> ToListAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) + { + var result = new List(); + + await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + result.Add(item); + } + + return result; + } + + public static async ValueTask ContainsAsync(this IAsyncEnumerable source, T value) + { + await foreach (var item in source.ConfigureAwait(false)) + { + if (EqualityComparer.Default.Equals(item, value)) + { + return true; + } + } + + return false; + } + + public static async ValueTask CountAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) + { + int count = 0; + await foreach (var _ in source.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + checked { count++; } + } + + return count; + } + + /// + /// Determines whether any element of an async-enumerable sequence satisfies a condition. + /// + /// The type of the elements in the source sequence. + /// An async-enumerable sequence whose elements to apply the predicate to. + /// A function to test each element for a condition. + /// The optional cancellation token to be used for cancelling the sequence at any time. + /// An async-enumerable sequence containing a single element determining whether any elements in the source sequence pass the test in the specified predicate. + /// or is null. + /// The return type of this operator differs from the corresponding operator on IEnumerable in order to retain asynchronous behavior. + public static ValueTask AnyAsync(this IAsyncEnumerable source, Func predicate, CancellationToken cancellationToken = default) + { + if (source == null) + { + throw new ArgumentNullException(nameof(source)); + } + + if (predicate == null) + { + throw new ArgumentNullException(nameof(predicate)); + } + + return Core(source, predicate, cancellationToken); + + static async ValueTask Core(IAsyncEnumerable source, Func predicate, CancellationToken cancellationToken) + { + await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + if (predicate(item)) + { + return true; + } + } + + return false; + } + } + + private sealed class EmptyAsyncEnumerable : IAsyncEnumerable, IAsyncEnumerator + { + public static readonly EmptyAsyncEnumerable Instance = new(); + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) => this; + public ValueTask MoveNextAsync() => new(false); + public T Current => default!; + public ValueTask DisposeAsync() => default; + } +} diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/.editorconfig b/dotnet/src/Planners/Planners.Core.UnitTests/.editorconfig new file mode 100644 index 000000000000..394eef685f21 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/Action/ActionPlannerTests.cs b/dotnet/src/Planners/Planners.Core.UnitTests/Action/ActionPlannerTests.cs new file mode 100644 index 000000000000..3c42aab05303 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/Action/ActionPlannerTests.cs @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Globalization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Moq; +using Xunit; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel.Planners.Action.UnitTests; +#pragma warning restore IDE0130 // Namespace does not match folder structure + +public sealed class ActionPlannerTests +{ + [Fact] + public async Task ExtractsAndDeserializesWellFormedJsonFromPlannerResultAsync() + { + // Arrange + var plugins = this.CreateMockFunctionCollection(); + + var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(ValidPlanString, plugins); + + var planner = new ActionPlanner(kernel.Object); + + // Act + var plan = await planner.CreatePlanAsync("goal"); + + // Assert + Assert.Equal("goal", plan.Description); + + Assert.Single(plan.Steps); + Assert.Equal("GitHubPlugin", plan.Steps[0].PluginName); + Assert.Equal("PullsList", plan.Steps[0].Name); + } + + [Fact] + public async Task InvalidJsonThrowsAsync() + { + // Arrange + string invalidJsonString = "<>"; + + var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(invalidJsonString); + + var planner = new ActionPlanner(kernel.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => planner.CreatePlanAsync("goal")); + } + + [Fact] + public void UsesPromptDelegateWhenProvided() + { + // Arrange + var kernel = new Mock(); + kernel.Setup(x => x.LoggerFactory).Returns(NullLoggerFactory.Instance); + var getPromptTemplateMock = new Mock>(); + var config = new ActionPlannerConfig() + { + GetPromptTemplate = getPromptTemplateMock.Object + }; + + // Act + var planner = new ActionPlanner(kernel.Object, config); + + // Assert + getPromptTemplateMock.Verify(x => x(), Times.Once()); + } + + [Fact] + public async Task MalformedJsonThrowsAsync() + { + // Arrange + + // Extra opening brace before rationale + string invalidJsonString = @"Here is a possible plan to accomplish the user intent: + +{ + ""plan"": { { + ""rationale"": ""the list contains a function that allows to list pull requests"", + ""function"": ""GitHubPlugin.PullsList"", + ""parameters"": { + ""owner"": ""microsoft"", + ""repo"": ""semantic-kernel"", + ""state"": ""open"" + } + } +} + +This plan uses the `GitHubPlugin.PullsList` function to list the open pull requests for the `semantic-kernel` repository owned by `microsoft`. The `state` parameter is set to `""open""` to filter the results to only show open pull requests. +"; + + var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(invalidJsonString); + + var planner = new ActionPlanner(kernel.Object); + + // Act & Assert + await Assert.ThrowsAsync(async () => await planner.CreatePlanAsync("goal")); + } + + [Fact] + public async Task ListOfFunctionsIncludesNativeAndSemanticFunctionsAsync() + { + // Arrange + var plugins = this.CreateMockFunctionCollection(); + var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(ValidPlanString, plugins); + var planner = new ActionPlanner(kernel.Object); + var context = kernel.Object.CreateNewContext(); + + // Act + var result = await planner.ListOfFunctionsAsync("goal", context); + + // Assert + var expected = $"// Send an e-mail.{Environment.NewLine}email.SendEmail{Environment.NewLine}// List pull requests.{Environment.NewLine}GitHubPlugin.PullsList{Environment.NewLine}// List repositories.{Environment.NewLine}GitHubPlugin.RepoList{Environment.NewLine}"; + Assert.Equal(expected, result); + } + + [Fact] + public async Task ListOfFunctionsExcludesExcludedPluginsAsync() + { + // Arrange + var plugins = this.CreateMockFunctionCollection(); + var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(ValidPlanString, plugins); + var config = new ActionPlannerConfig(); + config.ExcludedPlugins.Add("GitHubPlugin"); + var planner = new ActionPlanner(kernel.Object, config: config); + var context = kernel.Object.CreateNewContext(); + + // Act + var result = await planner.ListOfFunctionsAsync("goal", context); + + // Assert + var expected = $"// Send an e-mail.{Environment.NewLine}email.SendEmail{Environment.NewLine}"; + Assert.Equal(expected, result); + } + + [Fact] + public async Task ListOfFunctionsExcludesExcludedFunctionsAsync() + { + // Arrange + var plugins = this.CreateMockFunctionCollection(); + var kernel = this.CreateMockKernelAndFunctionFlowWithTestString(ValidPlanString, plugins); + var config = new ActionPlannerConfig(); + config.ExcludedFunctions.Add("PullsList"); + var planner = new ActionPlanner(kernel.Object, config: config); + var context = kernel.Object.CreateNewContext(); + + // Act + var result = await planner.ListOfFunctionsAsync("goal", context); + + // Assert + var expected = $"// Send an e-mail.{Environment.NewLine}email.SendEmail{Environment.NewLine}// List repositories.{Environment.NewLine}GitHubPlugin.RepoList{Environment.NewLine}"; + Assert.Equal(expected, result); + } + + private Mock CreateMockKernelAndFunctionFlowWithTestString(string testPlanString, Mock? functions = null) + { + if (functions is null) + { + functions = new Mock(); + functions.Setup(x => x.GetFunctionViews()).Returns(new List()); + } + var functionRunner = new Mock(); + var kernel = new Mock(); + + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(testPlanString), functions.Object); + + var context = new SKContext(functionRunner.Object, functions: functions.Object); + + var mockFunctionFlowFunction = new Mock(); + mockFunctionFlowFunction.Setup(x => x.InvokeAsync( + It.IsAny(), + null, + default + )).Callback( + (c, s, ct) => c.Variables.Update("Hello world!") + ).Returns(() => Task.FromResult(new FunctionResult("FunctionName", "PluginName", returnContext, testPlanString))); + + kernel.Setup(x => x.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(context); + kernel.Setup(x => x.Functions).Returns(functions.Object); + kernel.Setup(x => x.LoggerFactory).Returns(NullLoggerFactory.Instance); + + kernel.Setup(x => x.RegisterCustomFunction(It.IsAny())) + .Returns(mockFunctionFlowFunction.Object); + + return kernel; + } + + // Method to create Mock objects + private static Mock CreateMockFunction(FunctionView functionView) + { + var mockFunction = new Mock(); + mockFunction.Setup(x => x.Describe()).Returns(functionView); + mockFunction.Setup(x => x.Name).Returns(functionView.Name); + mockFunction.Setup(x => x.PluginName).Returns(functionView.PluginName); + return mockFunction; + } + + private Mock CreateMockFunctionCollection() + { + var functions = new List<(string name, string pluginName, string description, bool isSemantic)>() + { + ("SendEmail", "email", "Send an e-mail", false), + ("PullsList", "GitHubPlugin", "List pull requests", true), + ("RepoList", "GitHubPlugin", "List repositories", true), + }; + + var functionsView = new List(); + var plugins = new Mock(); + foreach (var (name, pluginName, description, isSemantic) in functions) + { + var functionView = new FunctionView(name, pluginName, description); + var mockFunction = CreateMockFunction(functionView); + functionsView.Add(functionView); + + mockFunction.Setup(x => + x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((context, settings, CancellationToken) => + { + context.Variables.Update("MOCK FUNCTION CALLED"); + return Task.FromResult(new FunctionResult(name, pluginName, context)); + }); + plugins.Setup(x => x.GetFunction(pluginName, name)) + .Returns(mockFunction.Object); + ISKFunction? outFunc = mockFunction.Object; + plugins.Setup(x => x.TryGetFunction(pluginName, name, out outFunc)).Returns(true); + } + + plugins.Setup(x => x.GetFunctionViews()).Returns(functionsView); + return plugins; + } + + private const string ValidPlanString = @"Here is a possible plan to accomplish the user intent: +{ + ""plan"":{ + ""rationale"": ""the list contains a function that allows to list pull requests"", + ""function"": ""GitHubPlugin.PullsList"", + ""parameters"": { + ""owner"": ""microsoft"", + ""repo"": ""semantic-kernel"", + ""state"": ""open"" + } + } +} + +This plan uses the `GitHubPlugin.PullsList` function to list the open pull requests for the `semantic-kernel` repository owned by `microsoft`. The `state` parameter is set to `""open""` to filter the results to only show open pull requests."; +} diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/Extensions/ReadOnlyFunctionCollectionExtensionsTests.cs b/dotnet/src/Planners/Planners.Core.UnitTests/Extensions/ReadOnlyFunctionCollectionExtensionsTests.cs new file mode 100644 index 000000000000..c4079d1b2114 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/Extensions/ReadOnlyFunctionCollectionExtensionsTests.cs @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Moq; +using Xunit; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel.Planners.UnitTests; +#pragma warning restore IDE0130 // Namespace does not match folder structure + +public class ReadOnlyFunctionCollectionExtensionsTests +{ + private static PlannerConfigBase InitializeConfig(Type t) + { + PlannerConfigBase? config = Activator.CreateInstance(t) as PlannerConfigBase; + Assert.NotNull(config); + return config; + } + + private async IAsyncEnumerable GetAsyncEnumerableAsync(IEnumerable results) + { + foreach (T result in results) + { + yield return await Task.FromResult(result); + } + } + + [Theory] + [InlineData(typeof(ActionPlannerConfig))] + [InlineData(typeof(SequentialPlannerConfig))] + [InlineData(typeof(StepwisePlannerConfig))] + public async Task CanCallGetAvailableFunctionsWithNoFunctionsAsync(Type t) + { + // Arrange + var kernel = new Mock(); + + var variables = new ContextVariables(); + var functions = new FunctionCollection(); + var cancellationToken = default(CancellationToken); + + // Arrange Mock Memory and Result + var memory = new Mock(); + var memoryQueryResult = new MemoryQueryResult( + new MemoryRecordMetadata( + isReference: false, + id: "id", + text: "text", + description: "description", + externalSourceName: "sourceName", + additionalMetadata: "value"), + relevance: 0.8, + embedding: null); + IAsyncEnumerable asyncEnumerable = this.GetAsyncEnumerableAsync(new[] { memoryQueryResult }); + memory.Setup(x => + x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(asyncEnumerable); + + var functionRunner = new Mock(); + + // Arrange GetAvailableFunctionsAsync parameters + var context = new SKContext(functionRunner.Object, variables); + var config = InitializeConfig(t); + var semanticQuery = "test"; + + // Act + var result = await context.Functions.GetAvailableFunctionsAsync(config, semanticQuery, null, cancellationToken); + + // Assert + Assert.NotNull(result); + memory.Verify( + x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + + config.SemanticMemoryConfig = new(); + + // Act + result = await context.Functions.GetAvailableFunctionsAsync(config, semanticQuery, null, cancellationToken); + + // Assert + Assert.NotNull(result); + memory.Verify( + x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + + config.SemanticMemoryConfig = new() { Memory = memory.Object }; + + // Act + result = await context.Functions.GetAvailableFunctionsAsync(config, semanticQuery, null, cancellationToken); + + // Assert + Assert.NotNull(result); + memory.Verify( + x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } + + [Theory] + [InlineData(typeof(ActionPlannerConfig))] + [InlineData(typeof(SequentialPlannerConfig))] + [InlineData(typeof(StepwisePlannerConfig))] + public async Task CanCallGetAvailableFunctionsWithFunctionsAsync(Type t) + { + // Arrange + var kernel = new Mock(); + var variables = new ContextVariables(); + var cancellationToken = default(CancellationToken); + + // Arrange FunctionView + var functionMock = new Mock(); + var functionView = new FunctionView("functionName", "pluginName", "description"); + var nativeFunctionView = new FunctionView("nativeFunctionName", "pluginName", "description"); + var functionsView = new List() { functionView, nativeFunctionView }; + + // Arrange Mock Memory and Result + var functions = new Mock(); + var memoryQueryResult = + new MemoryQueryResult( + new MemoryRecordMetadata( + isReference: false, + id: functionView.ToFullyQualifiedName(), + text: "text", + description: "description", + externalSourceName: "sourceName", + additionalMetadata: "value"), + relevance: 0.8, + embedding: null); + var asyncEnumerable = this.GetAsyncEnumerableAsync(new[] { memoryQueryResult }); + var memory = new Mock(); + memory.Setup(x => + x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(asyncEnumerable); + + functions.Setup(x => x.TryGetFunction(It.IsAny(), It.IsAny(), out It.Ref.IsAny)).Returns(true); + functions.Setup(x => x.GetFunction(It.IsAny(), It.IsAny())).Returns(functionMock.Object); + functions.Setup(x => x.GetFunctionViews()).Returns(functionsView); + + var functionRunner = new Mock(); + + // Arrange GetAvailableFunctionsAsync parameters + var context = new SKContext(functionRunner.Object, variables, functions.Object); + var config = InitializeConfig(t); + var semanticQuery = "test"; + + // Act + var result = (await context.Functions.GetAvailableFunctionsAsync(config, semanticQuery, null, cancellationToken)).ToList(); + + // Assert + Assert.NotNull(result); + Assert.Equal(2, result.Count); + Assert.Equal(functionView, result[0]); + + // Arrange update IncludedFunctions + config.SemanticMemoryConfig = new() { Memory = memory.Object }; + config.SemanticMemoryConfig.IncludedFunctions.UnionWith(new List<(string, string)> { ("pluginName", "nativeFunctionName") }); + + // Act + result = (await context.Functions.GetAvailableFunctionsAsync(config, semanticQuery)).ToList(); + + // Assert + Assert.NotNull(result); + Assert.Equal(2, result.Count); // IncludedFunctions should be added to the result + Assert.Equal(functionView, result[0]); + Assert.Equal(nativeFunctionView, result[1]); + } + + [Theory] + [InlineData(typeof(ActionPlannerConfig))] + [InlineData(typeof(SequentialPlannerConfig))] + [InlineData(typeof(StepwisePlannerConfig))] + public async Task CanCallGetAvailableFunctionsWithFunctionsWithRelevancyAsync(Type t) + { + // Arrange + var kernel = new Mock(); + + var variables = new ContextVariables(); + var cancellationToken = default(CancellationToken); + + // Arrange FunctionView + var functionMock = new Mock(); + var functionView = new FunctionView("functionName", "pluginName", "description"); + var nativeFunctionView = new FunctionView("nativeFunctionName", "pluginName", "description"); + var functionsView = new List() { functionView, nativeFunctionView }; + + // Arrange Mock Memory and Result + var functions = new Mock(); + var memoryQueryResult = + new MemoryQueryResult( + new MemoryRecordMetadata( + isReference: false, + id: functionView.ToFullyQualifiedName(), + text: "text", + description: "description", + externalSourceName: "sourceName", + additionalMetadata: "value"), + relevance: 0.8, + embedding: null); + var asyncEnumerable = this.GetAsyncEnumerableAsync(new[] { memoryQueryResult }); + var memory = new Mock(); + memory.Setup(x => + x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(asyncEnumerable); + + functions.Setup(x => x.TryGetFunction(It.IsAny(), It.IsAny(), out It.Ref.IsAny)).Returns(true); + functions.Setup(x => x.GetFunction(It.IsAny(), It.IsAny())).Returns(functionMock.Object); + functions.Setup(x => x.GetFunctionViews()).Returns(functionsView); + + var functionRunner = new Mock(); + + // Arrange GetAvailableFunctionsAsync parameters + var context = new SKContext(functionRunner.Object, variables, functions.Object); + var config = InitializeConfig(t); + config.SemanticMemoryConfig = new() { RelevancyThreshold = 0.78, Memory = memory.Object }; + var semanticQuery = "test"; + + // Act + var result = (await context.Functions.GetAvailableFunctionsAsync(config, semanticQuery, null, cancellationToken)).ToList(); + + // Assert + Assert.NotNull(result); + Assert.Single(result); + Assert.Equal(functionView, result[0]); + + // Arrange update IncludedFunctions + config.SemanticMemoryConfig.IncludedFunctions.UnionWith(new List<(string, string)> { ("pluginName", "nativeFunctionName") }); + + // Act + result = (await context.Functions.GetAvailableFunctionsAsync(config, semanticQuery)).ToList(); + + // Assert + Assert.NotNull(result); + Assert.Equal(2, result.Count); // IncludedFunctions should be added to the result + Assert.Equal(functionView, result[0]); + Assert.Equal(nativeFunctionView, result[1]); + } + + [Theory] + [InlineData(typeof(ActionPlannerConfig))] + [InlineData(typeof(SequentialPlannerConfig))] + [InlineData(typeof(StepwisePlannerConfig))] + public async Task CanCallGetAvailableFunctionsAsyncWithDefaultRelevancyAsync(Type t) + { + // Arrange + var kernel = new Mock(); + var functionRunner = new Mock(); + + var variables = new ContextVariables(); + var functions = new FunctionCollection(); + var cancellationToken = default(CancellationToken); + + // Arrange Mock Memory and Result + var memory = new Mock(); + var memoryQueryResult = + new MemoryQueryResult( + new MemoryRecordMetadata( + isReference: false, + id: "id", + text: "text", + description: "description", + externalSourceName: "sourceName", + additionalMetadata: "value"), + relevance: 0.8, + embedding: null); + var asyncEnumerable = this.GetAsyncEnumerableAsync(new[] { memoryQueryResult }); + memory.Setup(x => + x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(asyncEnumerable); + + // Arrange GetAvailableFunctionsAsync parameters + var context = new SKContext(functionRunner.Object, variables); + var config = InitializeConfig(t); + config.SemanticMemoryConfig = new() { RelevancyThreshold = 0.78, Memory = memory.Object }; + var semanticQuery = "test"; + + // Act + var result = await context.Functions.GetAvailableFunctionsAsync(config, semanticQuery, null, cancellationToken); + + // Assert + Assert.NotNull(result); + memory.Verify( + x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } +} diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/Planners.Core.UnitTests.csproj b/dotnet/src/Planners/Planners.Core.UnitTests/Planners.Core.UnitTests.csproj new file mode 100644 index 000000000000..0b772375dd48 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/Planners.Core.UnitTests.csproj @@ -0,0 +1,34 @@ + + + + Microsoft.SemanticKernel.Planners.Core.UnitTests + Microsoft.SemanticKernel.Planners.UnitTests + net6.0 + LatestMajor + true + enable + enable + false + CA2007,VSTHRD111 + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/Sequential/SequentialPlanParserTests.cs b/dotnet/src/Planners/Planners.Core.UnitTests/Sequential/SequentialPlanParserTests.cs new file mode 100644 index 000000000000..141ac5f572ff --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/Sequential/SequentialPlanParserTests.cs @@ -0,0 +1,394 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Globalization; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine; +using Moq; +using Xunit; +using Xunit.Abstractions; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel.Planners.Sequential.UnitTests; +#pragma warning restore IDE0130 // Namespace does not match folder structure + +public class SequentialPlanParserTests +{ + private readonly ITestOutputHelper _testOutputHelper; + + public SequentialPlanParserTests(ITestOutputHelper testOutputHelper) + { + this._testOutputHelper = testOutputHelper; + } + + private Mock CreateKernelMock( + out Mock mockFunctionCollection, + out Mock mockLogger) + { + mockFunctionCollection = new Mock(); + mockLogger = new Mock(); + + var kernelMock = new Mock(); + kernelMock.SetupGet(k => k.Functions).Returns(mockFunctionCollection.Object); + kernelMock.SetupGet(k => k.LoggerFactory).Returns(new Mock().Object); + + return kernelMock; + } + + private SKContext CreateSKContext( + IFunctionRunner functionRunner, + ContextVariables? variables = null) + { + return new SKContext(functionRunner, variables); + } + + private static Mock CreateMockFunction(FunctionView functionView, string result = "") + { + var mockFunction = new Mock(); + mockFunction.Setup(x => x.Describe()).Returns(functionView); + mockFunction.Setup(x => x.Name).Returns(functionView.Name); + mockFunction.Setup(x => x.PluginName).Returns(functionView.PluginName); + return mockFunction; + } + + private void CreateKernelAndFunctionCreateMocks(List<(string name, string pluginName, string description, bool isSemantic, string result)> functions, + out IKernel kernel) + { + var kernelMock = this.CreateKernelMock(out var functionCollection, out _); + kernel = kernelMock.Object; + + var functionRunnerMock = new Mock(); + + // For Create + kernelMock.Setup(k => k.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((contextVariables, skills, loggerFactory, culture) => + { + return this.CreateSKContext(functionRunnerMock.Object, contextVariables); + }); + + var functionsView = new List(); + foreach (var (name, pluginName, description, isSemantic, resultString) in functions) + { + var functionView = new FunctionView(name, pluginName, description) + { + Parameters = new ParameterView[] { new("param", "description") } + }; + var mockFunction = CreateMockFunction(functionView); + functionsView.Add(functionView); + + var result = this.CreateSKContext(functionRunnerMock.Object); + result.Variables.Update(resultString); + mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) + .ReturnsAsync(new FunctionResult(name, pluginName, result)); + + if (string.IsNullOrEmpty(name)) + { + kernelMock.Setup(x => x.RegisterSemanticFunction( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny() + )).Returns(mockFunction.Object); + } + else + { + functionCollection.Setup(x => x.GetFunction(It.Is(s => s == pluginName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + ISKFunction? outFunc = mockFunction.Object; + functionCollection.Setup(x => x.TryGetFunction(It.Is(s => s == name), out outFunc)).Returns(true); + functionCollection.Setup(x => x.TryGetFunction(It.Is(s => s == pluginName), It.Is(s => s == name), out outFunc)).Returns(true); + } + } + + functionCollection.Setup(x => x.GetFunctionViews()).Returns(functionsView); + } + + [Fact] + public void CanCallToPlanFromXml() + { + // Arrange + var functions = new List<(string name, string pluginName, string description, bool isSemantic, string result)>() + { + ("Summarize", "SummarizePlugin", "Summarize an input", true, "This is the summary."), + ("Translate", "WriterPlugin", "Translate to french", true, "Bonjour!"), + ("GetEmailAddressAsync", "email", "Get email address", false, "johndoe@email.com"), + ("SendEmailAsync", "email", "Send email", false, "Email sent."), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + var planString = + @" + + + + + +"; + var goal = "Summarize an input, translate to french, and e-mail to John Doe"; + + // Act + var plan = planString.ToPlanFromXml(goal, kernel.Functions.GetFunctionCallback()); + + // Assert + Assert.NotNull(plan); + Assert.Equal("Summarize an input, translate to french, and e-mail to John Doe", plan.Description); + + Assert.Equal(4, plan.Steps.Count); + Assert.Collection(plan.Steps, + step => + { + Assert.Equal("SummarizePlugin", step.PluginName); + Assert.Equal("Summarize", step.Name); + }, + step => + { + Assert.Equal("WriterPlugin", step.PluginName); + Assert.Equal("Translate", step.Name); + Assert.Equal("French", step.Parameters["language"]); + Assert.True(step.Outputs.Contains("TRANSLATED_SUMMARY")); + }, + step => + { + Assert.Equal("email", step.PluginName); + Assert.Equal("GetEmailAddressAsync", step.Name); + Assert.Equal("John Doe", step.Parameters["input"]); + Assert.True(step.Outputs.Contains("EMAIL_ADDRESS")); + }, + step => + { + Assert.Equal("email", step.PluginName); + Assert.Equal("SendEmailAsync", step.Name); + Assert.Equal("$TRANSLATED_SUMMARY", step.Parameters["input"]); + Assert.Equal("$EMAIL_ADDRESS", step.Parameters["email_address"]); + } + ); + } + + private const string GoalText = "Solve the equation x^2 = 2."; + + [Fact] + public void InvalidPlanExecutePlanReturnsInvalidResult() + { + // Arrange + this.CreateKernelAndFunctionCreateMocks(new(), out var kernel); + var planString = ""; + + // Act + Assert.Throws(() => planString.ToPlanFromXml(GoalText, kernel.Functions.GetFunctionCallback())); + } + + // Test that contains a #text node in the plan + [Theory] + [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner + + + This is some text + ")] + public void CanCreatePlanWithTextNodes(string goalText, string planText) + { + // Arrange + var functions = new List<(string name, string pluginName, string description, bool isSemantic, string result)>() + { + ("Echo", "MockPlugin", "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + var plan = planText.ToPlanFromXml(goalText, kernel.Functions.GetFunctionCallback()); + + // Assert + Assert.NotNull(plan); + Assert.Equal(goalText, plan.Description); + Assert.Single(plan.Steps); + Assert.Equal("MockPlugin", plan.Steps[0].PluginName); + Assert.Equal("Echo", plan.Steps[0].Name); + } + + [Theory] + [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner + + ")] + public void CanCreatePlanWithPartialXml(string goalText, string planText) + { + // Arrange + var functions = new List<(string name, string pluginName, string description, bool isSemantic, string result)>() + { + ("Echo", "MockPlugin", "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + var plan = planText.ToPlanFromXml(goalText, kernel.Functions.GetFunctionCallback()); + + // Assert + Assert.NotNull(plan); + Assert.Equal(goalText, plan.Description); + Assert.Single(plan.Steps); + Assert.Equal("MockPlugin", plan.Steps[0].PluginName); + Assert.Equal("Echo", plan.Steps[0].Name); + } + + [Theory] + [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner + + + ")] + public void CanCreatePlanWithFunctionName(string goalText, string planText) + { + // Arrange + var functions = new List<(string name, string pluginName, string description, bool isSemantic, string result)>() + { + ("Echo", FunctionCollection.GlobalFunctionsPluginName, "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + var plan = planText.ToPlanFromXml(goalText, kernel.Functions.GetFunctionCallback()); + + // Assert + Assert.NotNull(plan); + Assert.Equal(goalText, plan.Description); + Assert.Single(plan.Steps); + Assert.Equal(FunctionCollection.GlobalFunctionsPluginName, plan.Steps[0].PluginName); + Assert.Equal("Echo", plan.Steps[0].Name); + } + + // Test that contains a #text node in the plan + [Theory] + [InlineData(@" + + + + ", true)] + [InlineData(@" + + + + ", false)] + public void CanCreatePlanWithInvalidFunctionNodes(string planText, bool allowMissingFunctions) + { + // Arrange + var functions = new List<(string name, string pluginName, string description, bool isSemantic, string result)>() + { + ("Echo", "MockPlugin", "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + if (allowMissingFunctions) + { + // it should not throw + var plan = planText.ToPlanFromXml(string.Empty, kernel.Functions.GetFunctionCallback(), allowMissingFunctions); + + // Assert + Assert.NotNull(plan); + Assert.Equal(2, plan.Steps.Count); + + Assert.Equal("MockPlugin", plan.Steps[0].PluginName); + Assert.Equal("Echo", plan.Steps[0].Name); + Assert.Null(plan.Steps[0].Description); + + Assert.Equal(plan.GetType().Name, plan.Steps[1].PluginName); + Assert.NotEmpty(plan.Steps[1].Name); + Assert.Equal("MockPlugin.DoesNotExist", plan.Steps[1].Description); + } + else + { + Assert.Throws(() => planText.ToPlanFromXml(string.Empty, kernel.Functions.GetFunctionCallback(), allowMissingFunctions)); + } + } + + [Theory] + [InlineData("Test the functionFlowRunner", @"Possible result: Test the functionFlowRunner + + + This is some text + ")] + [InlineData("Test the functionFlowRunner", @" + + + This is some text + + + plan end")] + [InlineData("Test the functionFlowRunner", @" + + + This is some text + + + plan end")] + public void CanCreatePlanWithOtherText(string goalText, string planText) + { + // Arrange + var functions = new List<(string name, string pluginName, string description, bool isSemantic, string result)>() + { + ("Echo", "MockPlugin", "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + var plan = planText.ToPlanFromXml(goalText, kernel.Functions.GetFunctionCallback()); + + // Assert + Assert.NotNull(plan); + Assert.Equal(goalText, plan.Description); + Assert.Single(plan.Steps); + Assert.Equal("MockPlugin", plan.Steps[0].PluginName); + Assert.Equal("Echo", plan.Steps[0].Name); + } + + [Theory] + [InlineData(@" ")] + [InlineData("\n \n")] + [InlineData("\n \n")] + public void CanCreatePlanWithOpenApiPlugin(string planText) + { + // Arrange + var functions = new List<(string name, string pluginName, string description, bool isSemantic, string result)>() + { + ("codesearchresults_post", "CodeSearch", "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + var plan = planText.ToPlanFromXml(string.Empty, kernel.Functions.GetFunctionCallback()); + + // Assert + Assert.NotNull(plan); + Assert.Single(plan.Steps); + Assert.Equal("CodeSearch", plan.Steps[0].PluginName); + Assert.Equal("codesearchresults_post", plan.Steps[0].Name); + } + + // test that a that is not will just get skipped + [Theory] + [InlineData("Test the functionFlowRunner", @" + + Some other tag + + ")] + public void CanCreatePlanWithIgnoredNodes(string goalText, string planText) + { + // Arrange + var functions = new List<(string name, string pluginName, string description, bool isSemantic, string result)>() + { + ("Echo", "MockPlugin", "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + var plan = planText.ToPlanFromXml(goalText, kernel.Functions.GetFunctionCallback()); + + // Assert + Assert.NotNull(plan); + Assert.Equal(goalText, plan.Description); + Assert.Equal(2, plan.Steps.Count); + Assert.Equal("MockPlugin", plan.Steps[0].PluginName); + Assert.Equal("Echo", plan.Steps[0].Name); + Assert.Empty(plan.Steps[1].Steps); + Assert.Equal("MockPlugin", plan.Steps[1].PluginName); + Assert.Equal("Echo", plan.Steps[1].Name); + } +} diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/Sequential/SequentialPlannerTests.cs b/dotnet/src/Planners/Planners.Core.UnitTests/Sequential/SequentialPlannerTests.cs new file mode 100644 index 000000000000..785eed220189 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/Sequential/SequentialPlannerTests.cs @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Globalization; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Moq; +using Xunit; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel.Planners.Sequential.UnitTests; +#pragma warning restore IDE0130 // Namespace does not match folder structure + +public sealed class SequentialPlannerTests +{ + [Theory] + [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] + public async Task ItCanCreatePlanAsync(string goal) + { + // Arrange + var kernel = new Mock(); + kernel.Setup(x => x.LoggerFactory).Returns(new Mock().Object); + kernel.Setup(x => x.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (vars, cancellationToken, functions) => + { + var functionResult = await functions[0].InvokeAsync(kernel.Object, vars, cancellationToken: cancellationToken); + return KernelResult.FromFunctionResults(functionResult.GetValue(), new List { functionResult }); + }); + + var input = new List<(string name, string pluginName, string description, bool isSemantic)>() + { + ("SendEmail", "email", "Send an e-mail", false), + ("GetEmailAddress", "email", "Get an e-mail address", false), + ("Translate", "WriterPlugin", "Translate something", true), + ("Summarize", "SummarizePlugin", "Summarize something", true) + }; + + var functionsView = new List(); + var functions = new Mock(); + foreach (var (name, pluginName, description, isSemantic) in input) + { + var functionView = new FunctionView(name, pluginName, description); + var mockFunction = CreateMockFunction(functionView); + functionsView.Add(functionView); + + mockFunction.Setup(x => + x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((context, settings, cancellationToken) => + { + context.Variables.Update("MOCK FUNCTION CALLED"); + return Task.FromResult(new FunctionResult(name, pluginName, context)); + }); + + functions.Setup(x => x.GetFunction(It.Is(s => s == pluginName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + ISKFunction? outFunc = mockFunction.Object; + functions.Setup(x => x.TryGetFunction(It.Is(s => s == pluginName), It.Is(s => s == name), out outFunc)).Returns(true); + } + + functions.Setup(x => x.GetFunctionViews()).Returns(functionsView); + var functionRunner = new Mock(); + kernel.Setup(x => x.LoggerFactory).Returns(new Mock().Object); + + var expectedFunctions = input.Select(x => x.name).ToList(); + var expectedPlugins = input.Select(x => x.pluginName).ToList(); + + var context = new SKContext( + functionRunner.Object, + new ContextVariables()); + + var returnContext = new SKContext( + functionRunner.Object, + new ContextVariables()); + + var planString = + @" + + + + + +"; + + returnContext.Variables.Update(planString); + + var mockFunctionFlowFunction = new Mock(); + mockFunctionFlowFunction.Setup(x => x.InvokeAsync( + It.IsAny(), + null, + default + )).Callback( + (c, s, ct) => c.Variables.Update("Hello world!") + ).Returns(() => Task.FromResult(new FunctionResult("FunctionName", "PluginName", returnContext, planString))); + + // Mock Plugins + kernel.Setup(x => x.Functions).Returns(functions.Object); + kernel.Setup(x => x.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(context); + + kernel.Setup(x => x.RegisterCustomFunction(It.IsAny())) + .Returns(mockFunctionFlowFunction.Object); + + var planner = new SequentialPlanner(kernel.Object); + + // Act + var plan = await planner.CreatePlanAsync(goal, default); + + // Assert + Assert.Equal(goal, plan.Description); + + Assert.Contains( + plan.Steps, + step => + expectedFunctions.Contains(step.Name) && + expectedPlugins.Contains(step.PluginName)); + + foreach (var expectedFunction in expectedFunctions) + { + Assert.Contains( + plan.Steps, + step => step.Name == expectedFunction); + } + + foreach (var expectedPlugin in expectedPlugins) + { + Assert.Contains( + plan.Steps, + step => step.PluginName == expectedPlugin); + } + } + + [Fact] + public async Task EmptyGoalThrowsAsync() + { + // Arrange + var kernel = new Mock(); + + var planner = new SequentialPlanner(kernel.Object); + + // Act + await Assert.ThrowsAsync(async () => await planner.CreatePlanAsync("")); + } + + [Fact] + public async Task InvalidXMLThrowsAsync() + { + // Arrange + var functionRunner = new Mock(); + var kernel = new Mock(); + var functions = new Mock(); + + functions.Setup(x => x.GetFunctionViews()).Returns(new List()); + + var planString = "notvalid<"; + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(planString)); + + var context = new SKContext(functionRunner.Object, new ContextVariables()); + + var mockFunctionFlowFunction = new Mock(); + mockFunctionFlowFunction.Setup(x => x.InvokeAsync( + It.IsAny(), + null, + default + )).Callback( + (c, s, ct) => c.Variables.Update("Hello world!") + ).Returns(() => Task.FromResult(new FunctionResult("FunctionName", "PluginName", returnContext, planString))); + + // Mock Plugins + kernel.Setup(x => x.Functions).Returns(functions.Object); + kernel.Setup(x => x.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (vars, cancellationToken, functions) => + { + var functionResult = await functions[0].InvokeAsync(kernel.Object, vars, cancellationToken: cancellationToken); + return KernelResult.FromFunctionResults(functionResult.GetValue(), new List { functionResult }); + }); + + kernel.Setup(x => x.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(context); + + kernel.Setup(x => x.RegisterCustomFunction(It.IsAny())) + .Returns(mockFunctionFlowFunction.Object); + + var planner = new SequentialPlanner(kernel.Object); + + // Act + await Assert.ThrowsAsync(async () => await planner.CreatePlanAsync("goal")); + } + + [Fact] + public void UsesPromptDelegateWhenProvided() + { + // Arrange + var kernel = new Mock(); + var getPromptTemplateMock = new Mock>(); + var config = new SequentialPlannerConfig() + { + GetPromptTemplate = getPromptTemplateMock.Object + }; + + // Act + var planner = new SequentialPlanner(kernel.Object, config); + + // Assert + getPromptTemplateMock.Verify(x => x(), Times.Once()); + } + + // Method to create Mock objects + private static Mock CreateMockFunction(FunctionView functionView) + { + var mockFunction = new Mock(); + mockFunction.Setup(x => x.Describe()).Returns(functionView); + mockFunction.Setup(x => x.Name).Returns(functionView.Name); + mockFunction.Setup(x => x.PluginName).Returns(functionView.PluginName); + return mockFunction; + } +} diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/Stepwise/ParseResultTests.cs b/dotnet/src/Planners/Planners.Core.UnitTests/Stepwise/ParseResultTests.cs new file mode 100644 index 000000000000..566c7982905a --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/Stepwise/ParseResultTests.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel.Planners.Stepwise.UnitTests; +#pragma warning restore IDE0130 // Namespace does not match folder structure + +public sealed class ParseResultTests +{ + [Theory] + [InlineData("[FINAL ANSWER] 42", "42")] + [InlineData("[FINAL ANSWER]42", "42")] + [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42", "42")] + [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42\n", "42")] + [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42\n\n", "42")] + [InlineData("I think I have everything I need.\n[FINAL ANSWER]42\n\n\n", "42")] + [InlineData("I think I have everything I need.\n[FINAL ANSWER]\n 42\n\n\n", "42")] + [InlineData("I think I have everything I need.\n\n[FINALANSWER]\n 42\n\n\n", "42")] + [InlineData("I think I have everything I need.\n[FINAL_ANSWER]\n 42\n\n\n", "42")] + [InlineData("I think I have everything I need.\n[FINAL-ANSWER]\n 42\n\n\n", "42")] + public void WhenInputIsFinalAnswerReturnsFinalAnswer(string input, string expected) + { + // Arrange + var kernel = new Mock(); + kernel.Setup(x => x.LoggerFactory).Returns(NullLoggerFactory.Instance); + + var planner = new StepwisePlanner(kernel.Object); + + // Act + var result = planner.ParseResult(input); + + // Assert + Assert.Equal(expected, result.FinalAnswer); + } + + [Theory] + [InlineData("To answer the first part of the question, I need to search.\n[ACTION]\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"something to search\"}\n}", "To answer the first part of the question, I need to search.", "Search", "input", "something to search")] + [InlineData("To answer the first part of the question, I need to search.\n[ACTION]\n```\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"something to search\"}\n}\n```", "To answer the first part of the question, I need to search.", "Search", "input", "something to search")] + [InlineData("The web search result is a snippet from a Wikipedia article that says something.\n\n[ACTION] {\n \"action\": \"WebSearch.Search\",\n \"action_variables\": {\"input\": \"another search\", \"count\": \"1\"}\n}", "The web search result is a snippet from a Wikipedia article that says something.", "WebSearch.Search", "input", + "another search", "count", "1")] + [InlineData("[ACTION] {\"action\": \"time.Year\", \"action_variables\": {\"input\": \"\"}}", null, "time.Year", "input", "")] + [InlineData(@"[ACTION]{ + ""action"": ""RepositoryPlugin.PushChangesToBranch"", + ""action_variables"": { + ""branchName"": ""myBranchName"", + ""comment"": ""{MyComment"" + } +} +", null, "RepositoryPlugin.PushChangesToBranch", "branchName", "myBranchName", "comment", "{MyComment")] + [InlineData(@"[ACTION]{ + ""action"": ""RepositoryPlugin.PushChangesToBranch"", + ""action_variables"": { + ""branchName"": ""myBranchName"", + ""comment"": ""}MyComment"" + } +} +", null, "RepositoryPlugin.PushChangesToBranch", "branchName", "myBranchName", "comment", "}MyComment")] + [InlineData(@"[ACTION]{ + ""action"": ""RepositoryPlugin.PushChangesToBranch"", + ""action_variables"": { + ""branchName"": ""myBranchName"", + ""comment"": ""{MyComment}"" + } +} +", null, "RepositoryPlugin.PushChangesToBranch", "branchName", "myBranchName", "comment", "{MyComment}")] + public void ParseActionReturnsAction(string input, string expectedThought, string expectedAction, params string[] expectedVariables) + { + Dictionary? expectedDictionary = null; + for (int i = 0; i < expectedVariables.Length; i += 2) + { + expectedDictionary ??= new Dictionary(); + expectedDictionary.Add(expectedVariables[i], expectedVariables[i + 1]); + } + + // Arrange + var kernel = new Mock(); + kernel.Setup(x => x.LoggerFactory).Returns(NullLoggerFactory.Instance); + + var planner = new StepwisePlanner(kernel.Object); + + // Act + var result = planner.ParseResult(input); + + // Assert + Assert.Equal(expectedAction ?? string.Empty, result.Action); + Assert.Equal(expectedDictionary, result.ActionVariables); + Assert.Equal(expectedThought ?? string.Empty, result.Thought); + } + + // Method to create Mock objects + private static Mock CreateMockFunction(FunctionView functionView) + { + var mockFunction = new Mock(); + mockFunction.Setup(x => x.Describe()).Returns(functionView); + mockFunction.Setup(x => x.Name).Returns(functionView.Name); + mockFunction.Setup(x => x.PluginName).Returns(functionView.PluginName); + return mockFunction; + } +} diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/Stepwise/StepwisePlannerTests.cs b/dotnet/src/Planners/Planners.Core.UnitTests/Stepwise/StepwisePlannerTests.cs new file mode 100644 index 000000000000..42fba5e88830 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/Stepwise/StepwisePlannerTests.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel.Planners.Stepwise.UnitTests; +#pragma warning restore IDE0130 // Namespace does not match folder structure + +public sealed class StepwisePlannerTests +{ + [Fact] + public void UsesPromptDelegateWhenProvided() + { + // Arrange + var kernel = new Mock(); + kernel.Setup(x => x.LoggerFactory).Returns(NullLoggerFactory.Instance); + var getPromptTemplateMock = new Mock>(); + var config = new StepwisePlannerConfig() + { + GetPromptTemplate = getPromptTemplateMock.Object + }; + + // Act + var planner = new StepwisePlanner(kernel.Object, config); + + // Assert + getPromptTemplateMock.Verify(x => x(), Times.Once()); + } +} diff --git a/dotnet/src/Planners/Planners.Core.UnitTests/XunitHelpers/TestConsoleLogger.cs b/dotnet/src/Planners/Planners.Core.UnitTests/XunitHelpers/TestConsoleLogger.cs new file mode 100644 index 000000000000..d71996dccd49 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core.UnitTests/XunitHelpers/TestConsoleLogger.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Planners.UnitTests.XunitHelpers; + +/// +/// Basic logger printing to console +/// +internal static class TestConsoleLogger +{ + internal static ILogger Log => LoggerFactory.CreateLogger(); + + internal static ILoggerFactory LoggerFactory => s_loggerFactory.Value; + private static readonly Lazy s_loggerFactory = new(LogBuilder); + + private static ILoggerFactory LogBuilder() + { + return Extensions.Logging.LoggerFactory.Create(builder => + { + builder.SetMinimumLevel(LogLevel.Trace); + // builder.AddFilter("Microsoft", LogLevel.Trace); + // builder.AddFilter("Microsoft", LogLevel.Debug); + // builder.AddFilter("Microsoft", LogLevel.Information); + // builder.AddFilter("Microsoft", LogLevel.Warning); + // builder.AddFilter("Microsoft", LogLevel.Error); + builder.AddConsole(); + }); + } +} diff --git a/dotnet/src/Extensions/Planning.ActionPlanner/ActionPlanResponse.cs b/dotnet/src/Planners/Planners.Core/Action/ActionPlanResponse.cs similarity index 94% rename from dotnet/src/Extensions/Planning.ActionPlanner/ActionPlanResponse.cs rename to dotnet/src/Planners/Planners.Core/Action/ActionPlanResponse.cs index bd4634128ad0..d902659ccc7c 100644 --- a/dotnet/src/Extensions/Planning.ActionPlanner/ActionPlanResponse.cs +++ b/dotnet/src/Planners/Planners.Core/Action/ActionPlanResponse.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; -namespace Microsoft.SemanticKernel.Planning.Action; +namespace Microsoft.SemanticKernel.Planners.Action; /// /// Plan data structure returned by the basic planner semantic function diff --git a/dotnet/src/Planners/Planners.Core/Action/ActionPlanner.cs b/dotnet/src/Planners/Planners.Core/Action/ActionPlanner.cs new file mode 100644 index 000000000000..39a6c0408bf2 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Action/ActionPlanner.cs @@ -0,0 +1,317 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planners.Action; +using Microsoft.SemanticKernel.Planning; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Action Planner allows to select one function out of many, to achieve a given goal. +/// The planner implement the Intent Detection pattern, uses the functions registered +/// in the kernel to see if there's a relevant one, providing instructions to call the +/// function and the rationale used to select it. The planner can also return +/// "no function" is nothing relevant is available. +/// The rationale is currently available only in the prompt, we might include it in +/// the Plan object in future. +/// +public sealed class ActionPlanner : IActionPlanner +{ + private const string StopSequence = "#END-OF-PLAN"; + private const string PluginName = "this"; + + /// + /// The regular expression for extracting serialized plan. + /// + private static readonly Regex s_planRegex = new("^[^{}]*(((?'Open'{)[^{}]*)+((?'Close-Open'})[^{}]*)+)*(?(Open)(?!))", RegexOptions.Singleline | RegexOptions.Compiled); + + // Planner semantic function + private readonly ISKFunction _plannerFunction; + + // Context used to access the list of functions in the kernel + private readonly SKContext _context; + private readonly IKernel _kernel; + private readonly ILogger _logger; + + // TODO: allow to inject plugin store + /// + /// Initialize a new instance of the class. + /// + /// The semantic kernel instance. + /// The planner configuration. + public ActionPlanner( + IKernel kernel, + ActionPlannerConfig? config = null) + { + Verify.NotNull(kernel); + this._kernel = kernel; + + // Set up Config with default values and excluded plugins + this.Config = config ?? new(); + this.Config.ExcludedPlugins.Add(PluginName); + + string promptTemplate = this.Config.GetPromptTemplate?.Invoke() ?? EmbeddedResource.Read("Action.skprompt.txt"); + + this._plannerFunction = kernel.CreateSemanticFunction( + pluginName: PluginName, + promptTemplate: promptTemplate, + requestSettings: new AIRequestSettings() + { + ExtensionData = new Dictionary() + { + { "StopSequences", new[] { StopSequence } }, + { "MaxTokens", this.Config.MaxTokens }, + } + }); + + kernel.ImportFunctions(this, pluginName: PluginName); + + // Create context and logger + this._context = kernel.CreateNewContext(); + this._logger = this._kernel.LoggerFactory.CreateLogger(this.GetType()); + } + + /// + public async Task CreatePlanAsync(string goal, CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(goal)) + { + throw new SKException("The goal specified is empty"); + } + + this._context.Variables.Update(goal); + + FunctionResult result = await this._plannerFunction.InvokeAsync(this._context, cancellationToken: cancellationToken).ConfigureAwait(false); + ActionPlanResponse? planData = this.ParsePlannerResult(result); + + if (planData == null) + { + throw new SKException("The plan deserialized to a null object"); + } + + // Build and return plan + Plan? plan = null; + + FunctionUtils.SplitPluginFunctionName(planData.Plan.Function, out var pluginName, out var functionName); + if (!string.IsNullOrEmpty(functionName)) + { + var getFunctionCallback = this.Config.GetFunctionCallback ?? this._kernel.Functions.GetFunctionCallback(); + var pluginFunction = getFunctionCallback(pluginName, functionName); + if (pluginFunction != null) + { + plan = new Plan(goal, pluginFunction); + } + } + + plan ??= new(goal); + + // Populate plan parameters using the function and the parameters suggested by the planner + if (plan.Steps.Count > 0) + { + foreach (KeyValuePair p in planData.Plan.Parameters) + { + if (p.Value != null) + { + plan.Steps[0].Parameters[p.Key] = p.Value.ToString(); + } + } + } + + return plan; + } + + // TODO: use goal to find relevant functions in a plugin store + /// + /// Native function returning a list of all the functions in the current context, + /// excluding functions in the planner itself. + /// + /// Currently unused. Will be used to handle long lists of functions. + /// Function execution context + /// The token to use to request cancellation. + /// List of functions, formatted accordingly to the prompt + [SKFunction, Description("List all functions available in the kernel")] + public async Task ListOfFunctionsAsync( + [Description("The current goal processed by the planner")] string goal, + SKContext context, + CancellationToken cancellationToken = default) + { + // Prepare list using the format used by skprompt.txt + var list = new StringBuilder(); + var availableFunctions = await context.Functions.GetFunctionsAsync(this.Config, goal, this._logger, cancellationToken).ConfigureAwait(false); + this.PopulateList(list, availableFunctions); + + return list.ToString(); + } + + // TODO: generate string programmatically + // TODO: use goal to find relevant examples + /// + /// Native function that provides a list of good examples of plans to generate. + /// + /// The current goal processed by the planner. + /// Function execution context. + /// List of good examples, formatted accordingly to the prompt. + [SKFunction, Description("List a few good examples of plans to generate")] + public string GoodExamples( + [Description("The current goal processed by the planner")] string goal, + SKContext context) + { + return @" +[EXAMPLE] +- List of functions: +// Read a file. +FileIOPlugin.ReadAsync +Parameter ""path"": Source file. +// Write a file. +FileIOPlugin.WriteAsync +Parameter ""path"": Destination file. (default value: sample.txt) +Parameter ""content"": File content. +// Get the current time. +TimePlugin.Time +No parameters. +// Makes a POST request to a uri. +HttpPlugin.PostAsync +Parameter ""body"": The body of the request. +- End list of functions. +Goal: create a file called ""something.txt"". +{""plan"":{ +""rationale"": ""the list contains a function that allows to create files"", +""function"": ""FileIOPlugin.WriteAsync"", +""parameters"": { +""path"": ""something.txt"", +""content"": null +}}} +#END-OF-PLAN +"; + } + + // TODO: generate string programmatically + /// + /// Native function that provides a list of edge case examples of plans to handle. + /// + /// The current goal processed by the planner. + /// Function execution context. + /// List of edge case examples, formatted accordingly to the prompt. + [SKFunction, Description("List a few edge case examples of plans to handle")] + public string EdgeCaseExamples( + [Description("The current goal processed by the planner")] string goal, + SKContext context) + { + return @" +[EXAMPLE] +- List of functions: +// Get the current time. +TimePlugin.Time +No parameters. +// Write a file. +FileIOPlugin.WriteAsync +Parameter ""path"": Destination file. (default value: sample.txt) +Parameter ""content"": File content. +// Makes a POST request to a uri. +HttpPlugin.PostAsync +Parameter ""body"": The body of the request. +// Read a file. +FileIOPlugin.ReadAsync +Parameter ""path"": Source file. +- End list of functions. +Goal: tell me a joke. +{""plan"":{ +""rationale"": ""the list does not contain functions to tell jokes or something funny"", +""function"": """", +""parameters"": { +}}} +#END-OF-PLAN +"; + } + + #region private ================================================================================ + + /// + /// The configuration for the ActionPlanner + /// + private ActionPlannerConfig Config { get; } + + /// + /// Native function that filters out good JSON from planner result in case additional text is present + /// using a similar regex to the balancing group regex defined here: https://learn.microsoft.com/en-us/dotnet/standard/base-types/grouping-constructs-in-regular-expressions#balancing-group-definitions + /// + /// Result of planner function. + /// Instance of object deserialized from extracted JSON. + private ActionPlanResponse? ParsePlannerResult(FunctionResult plannerResult) + { + Match match = s_planRegex.Match(plannerResult.GetValue()); + + if (match.Success && match.Groups["Close"].Length > 0) + { + string planJson = $"{{{match.Groups["Close"]}}}"; + try + { + return JsonSerializer.Deserialize(planJson, new JsonSerializerOptions + { + AllowTrailingCommas = true, + DictionaryKeyPolicy = null, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + PropertyNameCaseInsensitive = true, + }); + } + catch (Exception e) + { + throw new SKException("Plan parsing error, invalid JSON", e); + } + } + else + { + throw new SKException($"Failed to extract valid json string from planner result: '{plannerResult}'"); + } + } + + private void PopulateList(StringBuilder list, IEnumerable functions) + { + foreach (FunctionView func in functions) + { + // Function description + if (func.Description != null) + { + list.AppendLine($"// {AddPeriod(func.Description)}"); + } + else + { + this._logger.LogWarning("{0}.{1} is missing a description", func.PluginName, func.Name); + list.AppendLine($"// Function {func.PluginName}.{func.Name}."); + } + + // Function name + list.AppendLine($"{func.PluginName}.{func.Name}"); + + // Function parameters + foreach (var p in func.Parameters) + { + var description = string.IsNullOrEmpty(p.Description) ? p.Name : p.Description!; + var defaultValueString = string.IsNullOrEmpty(p.DefaultValue) ? string.Empty : $" (default value: {p.DefaultValue})"; + list.AppendLine($"Parameter \"{p.Name}\": {AddPeriod(description)} {defaultValueString}"); + } + } + } + + private static string AddPeriod(string x) + { + return x.EndsWith(".", StringComparison.Ordinal) ? x : $"{x}."; + } + + #endregion +} diff --git a/dotnet/src/Planners/Planners.Core/Action/ActionPlannerConfig.cs b/dotnet/src/Planners/Planners.Core/Action/ActionPlannerConfig.cs new file mode 100644 index 000000000000..8c606fe9f3e3 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Action/ActionPlannerConfig.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Configuration for Action planner instances. +/// +public sealed class ActionPlannerConfig : PlannerConfigBase +{ + /// + /// Initializes a new instance of the class. + /// + public ActionPlannerConfig() + { + this.MaxTokens = 1024; + } +} diff --git a/dotnet/src/Planners/Planners.Core/Action/ActionPlannerExtensions.cs b/dotnet/src/Planners/Planners.Core/Action/ActionPlannerExtensions.cs new file mode 100644 index 000000000000..9fea5ad4f7de --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Action/ActionPlannerExtensions.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Extension methods for class. +/// +public static class ActionPlannerExtensions +{ + /// + /// Returns decorated instance of with enabled instrumentation. + /// + /// Instance of to decorate. + /// The to use for logging. If null, no logging will be performed. + public static IActionPlanner WithInstrumentation(this IActionPlanner planner, ILoggerFactory? loggerFactory = null) + { + return new InstrumentedActionPlanner(planner, loggerFactory); + } +} diff --git a/dotnet/src/Planners/Planners.Core/Action/IActionPlanner.cs b/dotnet/src/Planners/Planners.Core/Action/IActionPlanner.cs new file mode 100644 index 000000000000..d0cb4e2bfe9d --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Action/IActionPlanner.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Planning; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Interface for planner that uses a set of semantic functions to select one function out of many and create a plan. +/// +public interface IActionPlanner +{ + /// + /// Create a plan for a goal. + /// + /// The goal to create a plan for. + /// The to monitor for cancellation requests. The default is . + /// The plan. + /// Thrown when the plan cannot be created. + Task CreatePlanAsync(string goal, CancellationToken cancellationToken = default); +} diff --git a/dotnet/src/Extensions/Planning.ActionPlanner/InstrumentedActionPlanner.cs b/dotnet/src/Planners/Planners.Core/Action/InstrumentedActionPlanner.cs similarity index 79% rename from dotnet/src/Extensions/Planning.ActionPlanner/InstrumentedActionPlanner.cs rename to dotnet/src/Planners/Planners.Core/Action/InstrumentedActionPlanner.cs index e9fbcc264ddf..e29b49670b60 100644 --- a/dotnet/src/Extensions/Planning.ActionPlanner/InstrumentedActionPlanner.cs +++ b/dotnet/src/Planners/Planners.Core/Action/InstrumentedActionPlanner.cs @@ -7,26 +7,30 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Planning; -namespace Microsoft.SemanticKernel.Planning.Action; +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 /// /// Instrumented planner that uses set of semantic functions to select one function out of many and create a plan. /// Captures planner-related logs and metrics. /// -public class InstrumentedActionPlanner : IActionPlanner +internal sealed class InstrumentedActionPlanner : IActionPlanner { /// /// Initialize a new instance of the class. /// /// Instance of to decorate. - /// Optional logger. + /// The to use for logging. If null, no logging will be performed. public InstrumentedActionPlanner( IActionPlanner planner, - ILogger? logger = null) + ILoggerFactory? loggerFactory = null) { this._planner = planner; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(InstrumentedActionPlanner)) : NullLogger.Instance; } /// @@ -83,17 +87,17 @@ public async Task CreatePlanAsync(string goal, CancellationToken cancellat /// /// Instance of for planner-related activities. /// - private static ActivitySource s_activitySource = new(typeof(InstrumentedActionPlanner).FullName); + private static readonly ActivitySource s_activitySource = new(typeof(InstrumentedActionPlanner).FullName); /// /// Instance of for planner-related metrics. /// - private static Meter s_meter = new(typeof(InstrumentedActionPlanner).FullName); + private static readonly Meter s_meter = new(typeof(InstrumentedActionPlanner).FullName); /// /// Instance of to record plan creation execution time. /// - private static Histogram s_createPlanExecutionTime = + private static readonly Histogram s_createPlanExecutionTime = s_meter.CreateHistogram( name: $"SK.{PlannerType}.CreatePlan.ExecutionTime", unit: "ms", diff --git a/dotnet/src/Extensions/Planning.ActionPlanner/skprompt.txt b/dotnet/src/Planners/Planners.Core/Action/skprompt.txt similarity index 100% rename from dotnet/src/Extensions/Planning.ActionPlanner/skprompt.txt rename to dotnet/src/Planners/Planners.Core/Action/skprompt.txt diff --git a/dotnet/src/Planners/Planners.Core/Extensions/FunctionViewExtensions.cs b/dotnet/src/Planners/Planners.Core/Extensions/FunctionViewExtensions.cs new file mode 100644 index 000000000000..7d2c5be80baa --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Extensions/FunctionViewExtensions.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; + +#pragma warning disable IDE0130 +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Provides extension methods for the class. +/// +internal static class FunctionViewExtensions +{ + /// + /// Create a manual-friendly string for a function. + /// + /// The function to create a manual-friendly string for. + /// A manual-friendly string for a function. + internal static string ToManualString(this FunctionView function) + { + var inputs = string.Join("\n", function.Parameters.Select(parameter => + { + var defaultValueString = string.IsNullOrEmpty(parameter.DefaultValue) ? string.Empty : $" (default value: {parameter.DefaultValue})"; + return $" - {parameter.Name}: {parameter.Description}{defaultValueString}"; + })); + + // description and inputs are indented by 2 spaces + // While each parameter in inputs is indented by 4 spaces + return $@"{function.ToFullyQualifiedName()}: + description: {function.Description} + inputs: +{inputs}"; + } + + /// + /// Create a fully qualified name for a function. + /// + /// The function to create a fully qualified name for. + /// A fully qualified name for a function. + internal static string ToFullyQualifiedName(this FunctionView function) + { + return $"{function.PluginName}.{function.Name}"; + } + + /// + /// Create a string for generating an embedding for a function. + /// + /// The function to create a string for generating an embedding for. + /// A string for generating an embedding for a function. + internal static string ToEmbeddingString(this FunctionView function) + { + var inputs = string.Join("\n", function.Parameters.Select(p => $" - {p.Name}: {p.Description}")); + return $"{function.Name}:\n description: {function.Description}\n inputs:\n{inputs}"; + } +} diff --git a/dotnet/src/Planners/Planners.Core/Extensions/PromptTemplateConfigExtensions.cs b/dotnet/src/Planners/Planners.Core/Extensions/PromptTemplateConfigExtensions.cs new file mode 100644 index 000000000000..594e053d4426 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Extensions/PromptTemplateConfigExtensions.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. + +// ReSharper disable once CheckNamespace - Using the namespace of IKernel +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.TemplateEngine; + +#pragma warning disable IDE0130 +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Extension methods for PromptTemplateConfig +/// +internal static class PromptTemplateConfigExtensions +{ + /// + /// Set the max_tokens request setting to be used by OpenAI models + /// + /// PromptTemplateConfig instance + /// Value of max tokens to set + internal static void SetMaxTokens(this PromptTemplateConfig config, int maxTokens) + { + AIRequestSettings requestSettings = config.GetDefaultRequestSettings() ?? new(); + if (config.ModelSettings.Count == 0) + { + config.ModelSettings.Add(requestSettings); + } + requestSettings.ExtensionData["max_tokens"] = maxTokens; + } +} diff --git a/dotnet/src/Planners/Planners.Core/Extensions/ReadOnlyFunctionCollectionPlannerExtensions.cs b/dotnet/src/Planners/Planners.Core/Extensions/ReadOnlyFunctionCollectionPlannerExtensions.cs new file mode 100644 index 000000000000..a4e32df5fc40 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Extensions/ReadOnlyFunctionCollectionPlannerExtensions.cs @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Memory; + +#pragma warning disable IDE0130 +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Provides extension methods for the implementations for planners. +/// +public static class ReadOnlyFunctionCollectionPlannerExtensions +{ + internal const string PlannerMemoryCollectionName = "Planning.SKFunctionsManual"; + + /// + /// Returns a function callback that can be used to retrieve a function from the function provider. + /// + /// The function provider. + /// A function callback that can be used to retrieve a function from the function provider. + public static Func GetFunctionCallback(this IReadOnlyFunctionCollection functions) + { + return (pluginName, functionName) => + { + if (string.IsNullOrEmpty(pluginName)) + { + if (functions.TryGetFunction(functionName, out var pluginFunction)) + { + return pluginFunction; + } + } + else if (functions.TryGetFunction(pluginName, functionName, out var pluginFunction)) + { + return pluginFunction; + } + + return null; + }; + } + + /// + /// Returns a string containing the manual for all available functions. + /// + /// The function provider. + /// The planner config. + /// The semantic query for finding relevant registered functions + /// The logger to use for logging. + /// The to monitor for cancellation requests. The default is . + /// A string containing the manual for all available functions. + public static async Task GetFunctionsManualAsync( + this IReadOnlyFunctionCollection functions, + PlannerConfigBase config, + string? semanticQuery = null, + ILogger? logger = null, + CancellationToken cancellationToken = default) + { + IOrderedEnumerable availableFunctions = await functions.GetFunctionsAsync(config, semanticQuery, logger, cancellationToken).ConfigureAwait(false); + + return string.Join("\n\n", availableFunctions.Select(x => x.ToManualString())); + } + + /// + /// Returns a list of functions that are available to the user based on the semantic query and the excluded plugins and functions. + /// + /// The function provider. + /// The planner config. + /// The semantic query for finding relevant registered functions + /// The logger to use for logging. + /// The to monitor for cancellation requests. The default is . + /// A list of functions that are available to the user based on the semantic query and the excluded plugins and functions. + public static async Task> GetFunctionsAsync( + this IReadOnlyFunctionCollection functions, + PlannerConfigBase config, + string? semanticQuery, + ILogger? logger, + CancellationToken cancellationToken) + { + // Use configured function provider if available, otherwise use the default SKContext function provider. + return config.GetAvailableFunctionsAsync is null ? + await functions.GetAvailableFunctionsAsync(config, semanticQuery, logger, cancellationToken).ConfigureAwait(false) : + await config.GetAvailableFunctionsAsync(config, semanticQuery, cancellationToken).ConfigureAwait(false); + } + + /// + /// Returns a list of functions that are available to the user based on the semantic query and the excluded plugins and functions. + /// + /// The function provider. + /// The planner config. + /// The semantic query for finding relevant registered functions + /// The logger to use for logging. + /// The to monitor for cancellation requests. The default is . + /// A list of functions that are available to the user based on the semantic query and the excluded plugins and functions. + public static async Task> GetAvailableFunctionsAsync( + this IReadOnlyFunctionCollection functions, + PlannerConfigBase config, + string? semanticQuery = null, + ILogger? logger = null, + CancellationToken cancellationToken = default) + { + var functionsView = functions.GetFunctionViews(); + + var availableFunctions = functionsView + .Where(s => !config.ExcludedPlugins.Contains(s.PluginName, StringComparer.OrdinalIgnoreCase) + && !config.ExcludedFunctions.Contains(s.Name, StringComparer.OrdinalIgnoreCase)) + .ToList(); + + List? result = null; + var semanticMemoryConfig = config.SemanticMemoryConfig; + if (string.IsNullOrEmpty(semanticQuery) || semanticMemoryConfig.Memory is NullMemory) + { + // If no semantic query is provided, return all available functions. + // If a Memory provider has not been registered, return all available functions. + result = availableFunctions; + } + else + { + result = new List(); + + // Remember functions in memory so that they can be searched. + await RememberFunctionsAsync(semanticMemoryConfig.Memory, availableFunctions, cancellationToken).ConfigureAwait(false); + + // Search for functions that match the semantic query. + var memories = semanticMemoryConfig.Memory.SearchAsync( + PlannerMemoryCollectionName, + semanticQuery!, + semanticMemoryConfig.MaxRelevantFunctions, + semanticMemoryConfig.RelevancyThreshold ?? 0.0, + cancellationToken: cancellationToken); + + // Add functions that were found in the search results. + result.AddRange(await GetRelevantFunctionsAsync(availableFunctions, memories, logger ?? NullLogger.Instance, cancellationToken).ConfigureAwait(false)); + + // Add any missing functions that were included but not found in the search results. + var missingFunctions = semanticMemoryConfig.IncludedFunctions + .Except(result.Select(x => (x.PluginName, x.Name))) + .Join(availableFunctions, f => f, af => (af.PluginName, af.Name), (_, af) => af); + + result.AddRange(missingFunctions); + } + + return result + .OrderBy(x => x.PluginName) + .ThenBy(x => x.Name); + } + + private static async Task> GetRelevantFunctionsAsync( + IEnumerable availableFunctions, + IAsyncEnumerable memories, + ILogger logger, + CancellationToken cancellationToken = default) + { + var relevantFunctions = new ConcurrentBag(); + await foreach (var memoryEntry in memories.WithCancellation(cancellationToken)) + { + var function = availableFunctions.FirstOrDefault(x => x.ToFullyQualifiedName() == memoryEntry.Metadata.Id); + if (function != null) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug("Found relevant function. Relevance Score: {0}, Function: {1}", memoryEntry.Relevance, function.ToFullyQualifiedName()); + } + + relevantFunctions.Add(function); + } + } + + return relevantFunctions; + } + + /// + /// Saves all available functions to memory. + /// + /// The memory provided to store the functions to. + /// The available functions to save. + /// The to monitor for cancellation requests. The default is . + private static async Task RememberFunctionsAsync( + ISemanticTextMemory memory, + List availableFunctions, + CancellationToken cancellationToken = default) + { + foreach (var function in availableFunctions) + { + var functionName = function.ToFullyQualifiedName(); + var key = functionName; + var description = string.IsNullOrEmpty(function.Description) ? functionName : function.Description; + var textToEmbed = function.ToEmbeddingString(); + + // It'd be nice if there were a saveIfNotExists method on the memory interface + var memoryEntry = await memory.GetAsync(collection: PlannerMemoryCollectionName, key: key, withEmbedding: false, + cancellationToken: cancellationToken).ConfigureAwait(false); + if (memoryEntry == null) + { + // TODO It'd be nice if the minRelevanceScore could be a parameter for each item that was saved to memory + // As folks may want to tune their functions to be more or less relevant. + // Memory now supports these such strategies. + await memory.SaveInformationAsync(collection: PlannerMemoryCollectionName, text: textToEmbed, id: key, description: description, + additionalMetadata: string.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/dotnet/src/Planners/Planners.Core/PlannerConfigBase.cs b/dotnet/src/Planners/Planners.Core/PlannerConfigBase.cs new file mode 100644 index 000000000000..c0d6cf64728d --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/PlannerConfigBase.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Base class for planner configs +/// +public abstract class PlannerConfigBase +{ + /// + /// Delegate to get the prompt template string. + /// + public Func? GetPromptTemplate { get; set; } + + /// + /// A list of plugins to exclude from the plan creation request. + /// + public HashSet ExcludedPlugins { get; } = new(); + + /// + /// A list of functions to exclude from the plan creation request. + /// + public HashSet ExcludedFunctions { get; } = new(); + + /// + /// Semantic Memory configuration, used to enable function filtering during plan creation. + /// + /// + /// This configuration will be ignored if is set. + /// + public SemanticMemoryConfig SemanticMemoryConfig { get; set; } = new(); + + /// + /// Callback to get the available functions for planning (optional). + /// Use if you want to override the default function lookup behavior. + /// If set, this function takes precedence over . + /// Setting , will be used to filter the results. + /// + public Func>>? GetAvailableFunctionsAsync { get; set; } + + /// + /// Callback to get a function by name (optional). + /// Use if you want to override the default function lookup behavior. + /// + public Func? GetFunctionCallback { get; set; } + + /// + /// The maximum total number of tokens to allow in a completion request, + /// which includes the tokens from the prompt and completion + /// + public int MaxTokens { get; set; } +} diff --git a/dotnet/src/Planners/Planners.Core/Planners.Core.csproj b/dotnet/src/Planners/Planners.Core/Planners.Core.csproj new file mode 100644 index 000000000000..13899566ae3d --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Planners.Core.csproj @@ -0,0 +1,68 @@ + + + + + Microsoft.SemanticKernel.Planners.Core + Microsoft.SemanticKernel.Planners + netstandard2.0 + + + + + + + + Semantic Kernel - Planners + Semantic Kernel Core Planners which include the Action, Sequential, and Stepwise planners. + + + + + + + + + + + + + + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + + + + + + + + + + + + + diff --git a/dotnet/src/Planners/Planners.Core/SemanticMemoryConfig.cs b/dotnet/src/Planners/Planners.Core/SemanticMemoryConfig.cs new file mode 100644 index 000000000000..2a6f8e5ff8fb --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/SemanticMemoryConfig.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.SemanticKernel.Memory; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Semantic memory configuration. +/// +public class SemanticMemoryConfig +{ + /// + /// A list of functions to be included regardless of relevancy. + /// + public HashSet<(string PluginName, string FunctionName)> IncludedFunctions { get; } = new(); + + /// + /// Semantic memory to use for filtering function lookup during plan creation. + /// + public ISemanticTextMemory Memory { get; set; } = NullMemory.Instance; + + /// + /// The maximum number of relevant functions to search for. + /// + /// + /// Limits the number of relevant functions as result of semantic + /// search included in the plan creation request. + /// will be included + /// in the plan regardless of this limit. + /// + public int MaxRelevantFunctions { get; set; } = 100; + + /// + /// The minimum relevancy score for a function to be considered. + /// + /// + /// Depending on the embeddings engine used, the user ask, the step goal + /// and the functions available, this value may need to be adjusted. + /// For default, this is set to null which will return the top + /// sorted by relevancy. + /// + public double? RelevancyThreshold { get; set; } +} diff --git a/dotnet/src/Planners/Planners.Core/Sequential/ISequentialPlanner.cs b/dotnet/src/Planners/Planners.Core/Sequential/ISequentialPlanner.cs new file mode 100644 index 000000000000..da18d793d580 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Sequential/ISequentialPlanner.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Planning; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Interface for planner that uses a set of semantic functions to create a sequential plan. +/// +public interface ISequentialPlanner +{ + /// + /// Create a plan for a goal. + /// + /// The goal to create a plan for. + /// The to monitor for cancellation requests. The default is . + /// The plan. + /// Thrown when the plan cannot be created. + Task CreatePlanAsync(string goal, CancellationToken cancellationToken = default); +} diff --git a/dotnet/src/Extensions/Planning.SequentialPlanner/InstrumentedSequentialPlanner.cs b/dotnet/src/Planners/Planners.Core/Sequential/InstrumentedSequentialPlanner.cs similarity index 78% rename from dotnet/src/Extensions/Planning.SequentialPlanner/InstrumentedSequentialPlanner.cs rename to dotnet/src/Planners/Planners.Core/Sequential/InstrumentedSequentialPlanner.cs index 7a3c98f3c991..76773428ba07 100644 --- a/dotnet/src/Extensions/Planning.SequentialPlanner/InstrumentedSequentialPlanner.cs +++ b/dotnet/src/Planners/Planners.Core/Sequential/InstrumentedSequentialPlanner.cs @@ -7,26 +7,30 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Planning; -namespace Microsoft.SemanticKernel.Planning.Sequential; +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 /// /// Instrumented planner that uses semantic function to create a sequential plan. /// Captures planner-related logs and metrics. /// -public sealed class InstrumentedSequentialPlanner : ISequentialPlanner +internal sealed class InstrumentedSequentialPlanner : ISequentialPlanner { /// /// Initialize a new instance of the class. /// /// Instance of to decorate. - /// Optional logger. + /// The to use for logging. If null, no logging will be performed. public InstrumentedSequentialPlanner( ISequentialPlanner planner, - ILogger? logger = null) + ILoggerFactory? loggerFactory = null) { this._planner = planner; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(InstrumentedSequentialPlanner)) : NullLogger.Instance; } /// @@ -83,17 +87,17 @@ public async Task CreatePlanAsync(string goal, CancellationToken cancellat /// /// Instance of for planner-related activities. /// - private static ActivitySource s_activitySource = new(typeof(InstrumentedSequentialPlanner).FullName); + private static readonly ActivitySource s_activitySource = new(typeof(InstrumentedSequentialPlanner).FullName); /// /// Instance of for planner-related metrics. /// - private static Meter s_meter = new(typeof(InstrumentedSequentialPlanner).FullName); + private static readonly Meter s_meter = new(typeof(InstrumentedSequentialPlanner).FullName); /// /// Instance of to record plan creation execution time. /// - private static Histogram s_createPlanExecutionTime = + private static readonly Histogram s_createPlanExecutionTime = s_meter.CreateHistogram( name: $"SK.{PlannerType}.CreatePlan.ExecutionTime", unit: "ms", diff --git a/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlanParser.cs b/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlanParser.cs new file mode 100644 index 000000000000..5418a6bb9fbc --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlanParser.cs @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; +using System.Xml; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning; + +namespace Microsoft.SemanticKernel.Planners.Sequential; + +/// +/// Parse sequential plan text into a plan. +/// +internal static class SequentialPlanParser +{ + /// + /// The tag name used in the plan xml for the user's goal/ask. + /// TODO: never used + /// + internal const string GoalTag = "goal"; + + /// + /// The tag name used in the plan xml for the solution. + /// + internal const string SolutionTag = "plan"; + + /// + /// The tag name used in the plan xml for a step that calls a plugin function. + /// + internal const string FunctionTag = "function."; + + /// + /// The attribute tag used in the plan xml for setting the context variable name to set the output of a function to. + /// + internal const string SetContextVariableTag = "setContextVariable"; + + /// + /// The attribute tag used in the plan xml for appending the output of a function to the final result for a plan. + /// + internal const string AppendToResultTag = "appendToResult"; + + /// + /// Convert a plan xml string to a plan. + /// + /// The plan xml string. + /// The goal for the plan. + /// The callback to get a plugin function. + /// Whether to allow missing functions in the plan on creation. + /// The plan. + /// Thrown when the plan xml is invalid. + internal static Plan ToPlanFromXml(this string xmlString, string goal, Func getFunctionCallback, bool allowMissingFunctions = false) + { + XmlDocument xmlDoc = new(); + try + { + xmlDoc.LoadXml("" + xmlString + ""); + } + catch (XmlException e) + { + // xmlString wasn't valid xml, let's try and parse out of it + + // ']*': Matches zero or more characters that are not the closing angle bracket (">"), effectively matching any attributes present in the opening tag. + // '>': Matches the closing angle bracket (">") to indicate the end of the opening tag. + // '(.*?)': Captures the content between the opening and closing tags using a non-greedy match. It matches any character (except newline) in a lazy manner, i.e., it captures the smallest possible match. + // '': Matches the literal string "", indicating the closing tag of the element. + Regex planRegex = new(@"]*>(.*?)", RegexOptions.Singleline); + Match match = planRegex.Match(xmlString); + + if (!match.Success) + { + match = planRegex.Match($"{xmlString}"); // try again with a closing tag + } + + if (match.Success) + { + string planXml = match.Value; + + try + { + xmlDoc.LoadXml("" + planXml + ""); + } + catch (XmlException ex) + { + throw new SKException($"Failed to parse plan xml strings: '{xmlString}' or '{planXml}'", ex); + } + } + else + { + throw new SKException($"Failed to parse plan xml string: '{xmlString}'", e); + } + } + + // Get the Solution + XmlNodeList solution = xmlDoc.GetElementsByTagName(SolutionTag); + + var plan = new Plan(goal); + + // loop through solution node and add to Steps + foreach (XmlNode solutionNode in solution) + { + var parentNodeName = solutionNode.Name; + + foreach (XmlNode childNode in solutionNode.ChildNodes) + { + if (childNode.Name == "#text" || childNode.Name == "#comment") + { + // Do not add text or comments as steps. + // TODO - this could be a way to get Reasoning for a plan step. + continue; + } + + if (childNode.Name.StartsWith(FunctionTag, StringComparison.OrdinalIgnoreCase)) + { + var pluginFunctionName = childNode.Name.Split(s_functionTagArray, StringSplitOptions.None)?[1] ?? string.Empty; + FunctionUtils.SplitPluginFunctionName(pluginFunctionName, out var pluginName, out var functionName); + + if (!string.IsNullOrEmpty(functionName)) + { + var pluginFunction = getFunctionCallback(pluginName, functionName); + + if (pluginFunction is not null) + { + var planStep = new Plan(pluginFunction); + + var functionVariables = new ContextVariables(); + var functionOutputs = new List(); + var functionResults = new List(); + + var view = pluginFunction.Describe(); + foreach (var p in view.Parameters) + { + functionVariables.Set(p.Name, p.DefaultValue); + } + + if (childNode.Attributes is not null) + { + foreach (XmlAttribute attr in childNode.Attributes) + { + if (attr.Name.Equals(SetContextVariableTag, StringComparison.OrdinalIgnoreCase)) + { + functionOutputs.Add(attr.InnerText); + } + else if (attr.Name.Equals(AppendToResultTag, StringComparison.OrdinalIgnoreCase)) + { + functionOutputs.Add(attr.InnerText); + functionResults.Add(attr.InnerText); + } + else + { + functionVariables.Set(attr.Name, attr.InnerText); + } + } + } + + // Plan properties + planStep.Outputs = functionOutputs; + planStep.Parameters = functionVariables; + foreach (var result in functionResults) + { + plan.Outputs.Add(result); + } + + plan.AddSteps(planStep); + } + else + { + if (allowMissingFunctions) + { + plan.AddSteps(new Plan(pluginFunctionName)); + } + else + { + throw new SKException($"Failed to find function '{pluginFunctionName}' in plugin '{pluginName}'."); + } + } + } + } + + // Similar to comments or text, do not add empty nodes as steps. + // TODO - This could be a way to advertise desired functions for a plan. + } + } + + return plan; + } + + private static readonly string[] s_functionTagArray = new string[] { FunctionTag }; +} diff --git a/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlanner.cs b/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlanner.cs new file mode 100644 index 000000000000..47387db16015 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlanner.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planners.Sequential; +using Microsoft.SemanticKernel.Planning; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// A planner that uses semantic function to create a sequential plan. +/// +public sealed class SequentialPlanner : ISequentialPlanner +{ + private const string StopSequence = ""; + private const string AvailableFunctionsKey = "available_functions"; + + /// + /// Initialize a new instance of the class. + /// + /// The semantic kernel instance. + /// The planner configuration. + public SequentialPlanner( + IKernel kernel, + SequentialPlannerConfig? config = null) + { + Verify.NotNull(kernel); + + // Set up config with default value and excluded plugins + this.Config = config ?? new(); + this.Config.ExcludedPlugins.Add(RestrictedPluginName); + + // Set up prompt template + string promptTemplate = this.Config.GetPromptTemplate?.Invoke() ?? EmbeddedResource.Read("Sequential.skprompt.txt"); + + this._functionFlowFunction = kernel.CreateSemanticFunction( + promptTemplate: promptTemplate, + pluginName: RestrictedPluginName, + description: "Given a request or command or goal generate a step by step plan to " + + "fulfill the request using functions. This ability is also known as decision making and function flow", + requestSettings: new AIRequestSettings() + { + ExtensionData = new Dictionary() + { + { "Temperature", 0.0 }, + { "StopSequences", new[] { StopSequence } }, + { "MaxTokens", this.Config.MaxTokens }, + } + }); + + this._kernel = kernel; + } + + /// + public async Task CreatePlanAsync(string goal, CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(goal)) + { + throw new SKException("The goal specified is empty"); + } + + string relevantFunctionsManual = await this._kernel.Functions.GetFunctionsManualAsync(this.Config, goal, null, cancellationToken).ConfigureAwait(false); + + ContextVariables vars = new(goal) + { + [AvailableFunctionsKey] = relevantFunctionsManual + }; + + KernelResult planResult = await this._kernel.RunAsync(this._functionFlowFunction, vars, cancellationToken).ConfigureAwait(false); + + string? planResultString = planResult.GetValue()?.Trim(); + + if (string.IsNullOrWhiteSpace(planResultString)) + { + throw new SKException( + "Unable to create plan. No response from Function Flow function. " + + $"\nGoal:{goal}\nFunctions:\n{relevantFunctionsManual}"); + } + + var getFunctionCallback = this.Config.GetFunctionCallback ?? this._kernel.Functions.GetFunctionCallback(); + + Plan plan; + try + { + plan = planResultString!.ToPlanFromXml(goal, getFunctionCallback, this.Config.AllowMissingFunctions); + } + catch (SKException e) + { + throw new SKException($"Unable to create plan for goal with available functions.\nGoal:{goal}\nFunctions:\n{relevantFunctionsManual}", e); + } + + if (plan.Steps.Count == 0) + { + throw new SKException($"Not possible to create plan for goal with available functions.\nGoal:{goal}\nFunctions:\n{relevantFunctionsManual}"); + } + + return plan; + } + + private SequentialPlannerConfig Config { get; } + + private readonly IKernel _kernel; + + /// + /// the function flow semantic function, which takes a goal and creates an xml plan that can be executed + /// + private readonly ISKFunction _functionFlowFunction; + + /// + /// The name to use when creating semantic functions that are restricted from plan creation + /// + private const string RestrictedPluginName = "SequentialPlanner_Excluded"; +} diff --git a/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlannerConfig.cs b/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlannerConfig.cs new file mode 100644 index 000000000000..807b644f4224 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlannerConfig.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Common configuration for planner instances. +/// +public sealed class SequentialPlannerConfig : PlannerConfigBase +{ + /// + /// Initializes a new instance of the class. + /// + public SequentialPlannerConfig() + { + this.MaxTokens = 1024; + } + + /// + /// Whether to allow missing functions in the plan on creation. + /// If set to true, the plan will be created with missing functions as no-op steps. + /// If set to false (default), the plan creation will fail if any functions are missing. + /// + public bool AllowMissingFunctions { get; set; } = false; +} diff --git a/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlannerExtensions.cs b/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlannerExtensions.cs new file mode 100644 index 000000000000..50eea1a2666d --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Sequential/SequentialPlannerExtensions.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Extension methods for class. +/// +public static class SequentialPlannerExtensions +{ + /// + /// Returns decorated instance of with enabled instrumentation. + /// + /// Instance of to decorate. + /// The to use for logging. If null, no logging will be performed. + public static ISequentialPlanner WithInstrumentation(this ISequentialPlanner planner, ILoggerFactory? loggerFactory = null) + { + return new InstrumentedSequentialPlanner(planner, loggerFactory); + } +} diff --git a/dotnet/src/Planners/Planners.Core/Sequential/skprompt.txt b/dotnet/src/Planners/Planners.Core/Sequential/skprompt.txt new file mode 100644 index 000000000000..325beca173be --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Sequential/skprompt.txt @@ -0,0 +1,55 @@ +Create an XML plan step by step, to satisfy the goal given, with the available functions. + +[AVAILABLE FUNCTIONS] + +{{$available_functions}} + +[END AVAILABLE FUNCTIONS] + +To create a plan, follow these steps: +0. The plan should be as short as possible. +1. From a create a as a series of . +2. A plan has 'INPUT' available in context variables by default. +3. Before using any function in a plan, check that it is present in the [AVAILABLE FUNCTIONS] list. If it is not, do not use it. +4. Only use functions that are required for the given goal. +5. Append an "END" XML comment at the end of the plan after the final closing tag. +6. Always output valid XML that can be parsed by an XML parser. +7. If a plan cannot be created with the [AVAILABLE FUNCTIONS], return . + +All plans take the form of: + + + + + + + + (... etc ...) + + + +To call a function, follow these steps: +1. A function has one or more named parameters and a single 'output' which are all strings. Parameter values should be xml escaped. +2. To save an 'output' from a , to pass into a future , use +3. To save an 'output' from a , to return as part of a plan result, use +4. Use a '$' to reference a context variable in a parameter, e.g. when `INPUT='world'` the parameter 'Hello $INPUT' will evaluate to `Hello world`. +5. Functions do not have access to the context variables of other functions. Do not attempt to use context variables as arrays or objects. Instead, use available functions to extract specific elements or properties from context variables. + +DO NOT DO THIS, THE PARAMETER VALUE IS NOT XML ESCAPED: + + +DO NOT DO THIS, THE PARAMETER VALUE IS ATTEMPTING TO USE A CONTEXT VARIABLE AS AN ARRAY/OBJECT: + + +Here is a valid example of how to call a function "_Function_.Name" with a single input and save its output: + + +Here is a valid example of how to call a function "FunctionName2" with a single input and return its output as part of the plan result: + + +Here is a valid example of how to call a function "Name3" with multiple inputs: + + +Begin! + +{{$input}} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/ChatHistoryExtensions.cs b/dotnet/src/Planners/Planners.Core/Stepwise/ChatHistoryExtensions.cs new file mode 100644 index 000000000000..60e810f105b5 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/ChatHistoryExtensions.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using static Microsoft.SemanticKernel.Text.TextChunker; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Extension methods for class. +/// +public static class ChatHistoryExtensions +{ + /// + /// Returns the number of tokens in the chat history. + /// + // The chat history. + // An additional message to include in the token count. + // The index to start skipping messages. + // The number of messages to skip. + // The token counter to use. + internal static int GetTokenCount(this ChatHistory chatHistory, string? additionalMessage = null, int skipStart = 0, int skipCount = 0, TokenCounter? tokenCounter = null) + { + tokenCounter ??= DefaultTokenCounter; + + var messages = string.Join("\n", chatHistory.Where((m, i) => i < skipStart || i >= skipStart + skipCount).Select(m => m.Content)); + + if (!string.IsNullOrEmpty(additionalMessage)) + { + messages = $"{messages}\n{additionalMessage}"; + } + + var tokenCount = tokenCounter(messages); + return tokenCount; + } + + private static int DefaultTokenCounter(string input) + { + return input.Length / 4; + } +} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/IStepwisePlanner.cs b/dotnet/src/Planners/Planners.Core/Stepwise/IStepwisePlanner.cs new file mode 100644 index 000000000000..484e2ecee57e --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/IStepwisePlanner.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Planning; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Interface for planner that creates a Stepwise plan using Mrkl systems. +/// +public interface IStepwisePlanner +{ + /// + /// Create a plan for a goal. + /// + /// The goal to create a plan for. + /// The plan. + /// Thrown when the plan cannot be created. + Plan CreatePlan(string goal); +} diff --git a/dotnet/src/Extensions/Planning.StepwisePlanner/InstrumentedStepwisePlanner.cs b/dotnet/src/Planners/Planners.Core/Stepwise/InstrumentedStepwisePlanner.cs similarity index 76% rename from dotnet/src/Extensions/Planning.StepwisePlanner/InstrumentedStepwisePlanner.cs rename to dotnet/src/Planners/Planners.Core/Stepwise/InstrumentedStepwisePlanner.cs index c47d64d6f7cd..afec10f2e985 100644 --- a/dotnet/src/Extensions/Planning.StepwisePlanner/InstrumentedStepwisePlanner.cs +++ b/dotnet/src/Planners/Planners.Core/Stepwise/InstrumentedStepwisePlanner.cs @@ -5,26 +5,30 @@ using System.Diagnostics.Metrics; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Planning; -namespace Microsoft.SemanticKernel.Planning.Stepwise; +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 /// /// Instrumented planner that creates a Stepwise plan using Mrkl systems. /// Captures planner-related logs and metrics. /// -public class InstrumentedStepwisePlanner : IStepwisePlanner +internal class InstrumentedStepwisePlanner : IStepwisePlanner { /// /// Initialize a new instance of the class. /// /// Instance of to decorate. - /// Optional logger. + /// The to use for logging. If null, no logging will be performed. public InstrumentedStepwisePlanner( IStepwisePlanner planner, - ILogger? logger = null) + ILoggerFactory? loggerFactory = null) { this._planner = planner; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(InstrumentedStepwisePlanner)) : NullLogger.Instance; } /// @@ -76,17 +80,17 @@ public Plan CreatePlan(string goal) /// /// Instance of for planner-related activities. /// - private static ActivitySource s_activitySource = new(typeof(InstrumentedStepwisePlanner).FullName); + private static readonly ActivitySource s_activitySource = new(typeof(InstrumentedStepwisePlanner).FullName); /// /// Instance of for planner-related metrics. /// - private static Meter s_meter = new(typeof(InstrumentedStepwisePlanner).FullName); + private static readonly Meter s_meter = new(typeof(InstrumentedStepwisePlanner).FullName); /// /// Instance of to record plan creation execution time. /// - private static Histogram s_createPlanExecutionTime = + private static readonly Histogram s_createPlanExecutionTime = s_meter.CreateHistogram( name: $"SK.{PlannerType}.CreatePlan.ExecutionTime", unit: "ms", diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderFunctionManual/config.json b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderFunctionManual/config.json new file mode 100644 index 000000000000..a2044c431772 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderFunctionManual/config.json @@ -0,0 +1,14 @@ +{ + "schema": 1, + "description": "Render a function manual text for the agent's functions", + "type": "completion", + "input": { + "parameters": [ + { + "name": "functionDescriptions", + "description": "The manual of the agent's functions", + "defaultValue": "" + } + ] + } +} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderFunctionManual/skprompt.txt b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderFunctionManual/skprompt.txt new file mode 100644 index 000000000000..e55ce658979e --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderFunctionManual/skprompt.txt @@ -0,0 +1,8 @@ +[AVAILABLE FUNCTIONS] +The function definitions below are in the following format: +: + - : + - ... + +{{$functionDescriptions}} +[END AVAILABLE FUNCTIONS] diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderQuestion/config.json b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderQuestion/config.json new file mode 100644 index 000000000000..514b474d9515 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderQuestion/config.json @@ -0,0 +1,14 @@ +{ + "schema": 1, + "description": "Render a plan question text for the agent", + "type": "completion", + "input": { + "parameters": [ + { + "name": "question", + "description": "", + "defaultValue": "" + } + ] + } +} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderQuestion/skprompt.txt b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderQuestion/skprompt.txt new file mode 100644 index 000000000000..b48a31717560 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/RenderQuestion/skprompt.txt @@ -0,0 +1,2 @@ +[QUESTION] +{{$question}} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/StepwiseStep/config.json b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/StepwiseStep/config.json new file mode 100644 index 000000000000..64bd4ba62bb4 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/StepwiseStep/config.json @@ -0,0 +1,27 @@ +{ + "schema": 1, + "description": "Given a request or command or goal generate multi-step plan to reach the goal. After each step LLM is called to perform the reasoning for the next step.", + "type": "completion", + "completion": { + "max_tokens": 1024, + "temperature": 0, + "top_p": 0, + "presence_penalty": 0, + "frequency_penalty": 0, + "stop_sequences": ["[OBSERVATION]", "\n[THOUGHT]"] + }, + "input": { + "parameters": [ + { + "name": "functionDescriptions", + "description": "The manual of the agent's functions", + "defaultValue": "" + }, + { + "name": "suffix", + "description": "", + "defaultValue": "Let's break down the problem step by step and think about the best approach. Label steps as they are taken.\n\nContinue the thought process!" + } + ] + } +} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/StepwiseStep/skprompt.txt b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/StepwiseStep/skprompt.txt new file mode 100644 index 000000000000..5a0b5005e4c1 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/Plugin/StepwiseStep/skprompt.txt @@ -0,0 +1,43 @@ +[INSTRUCTION] +Answer the following questions as accurately as possible using the provided functions. + +{{$functionDescriptions}} +[USAGE INSTRUCTIONS] +To use the functions, specify a JSON blob representing an action. The JSON blob should contain an "action" key with the name of the function to use, and an "action_variables" key with a JSON object of string values to use when calling the function. +Do not call functions directly; they must be invoked through an action. +The keys in "action_variables" value should match the defined [PARAMETERS] of the named "action" in [AVAILABLE FUNCTIONS]. +The values in "action_variables" must be of type string and represent the actual values to be passed to the function. Do not attempt to pass a variable name or other reference to a function. +If a function has no parameters, the "action_variables" key may be omitted. +Ensure that the $JSON_BLOB contains only a SINGLE action; do NOT return multiple actions. +IMPORTANT: Use only the available functions listed in the [AVAILABLE FUNCTIONS] section. Do not attempt to use any other functions that are not specified. + +Here is an example of a valid $JSON_BLOB: +{ + "action": "FUNCTION.NAME", + "action_variables": {"PARAMETER_NAME": "some value", "PARAMETER_NAME_2": "42"} +} + +Here is an example of a valid $JSON_BLOB with no parameters: +{ + "action": "FUNCTION.NAME" +} + +[END USAGE INSTRUCTIONS] +[END INSTRUCTION] + +[VALID STEP LIST] +[QUESTION] - The input question I must answer +[THOUGHT] - A thought I have about the question and how to answer it. +[ACTION] - A single $JSON_BLOB representing a single action to be performed +[OBSERVATION] - The result of the action will be provided here +[FINAL ANSWER] - Once I have gathered all the necessary observations through producing thoughts and actions, I can provide the final answer in a clear and human-readable format. +[END VALID STEP LIST] + +Every Question should be followed by a Thought. +Every Thought should be followed by an Action or Final Answer. +Every Action should be followed by an Observation. +Every Observation should be followed by a Thought or Final Answer. +Produce Thoughts and Actions as necessary until you have a Final Answer. + + +{{$suffix}} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlanner.cs b/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlanner.cs new file mode 100644 index 000000000000..8adaf550b428 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlanner.cs @@ -0,0 +1,699 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning; +using Microsoft.SemanticKernel.Services; +using Microsoft.SemanticKernel.TemplateEngine; +using Microsoft.SemanticKernel.TemplateEngine.Basic; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// A planner that creates a Stepwise plan using Mrkl systems. +/// +/// +/// An implementation of a Mrkl system as described in https://arxiv.org/pdf/2205.00445.pdf +/// +public class StepwisePlanner : IStepwisePlanner +{ + /// + /// Initialize a new instance of the class. + /// + /// The semantic kernel instance. + /// Optional configuration object + public StepwisePlanner( + IKernel kernel, + StepwisePlannerConfig? config = null) + { + Verify.NotNull(kernel); + this._kernel = kernel; + + // Set up Config with default values and excluded plugins + this.Config = config ?? new(); + this.Config.ExcludedPlugins.Add(RestrictedPluginName); + + // Set up prompt templates + this._promptTemplate = this.Config.GetPromptTemplate?.Invoke() ?? EmbeddedResource.Read("Stepwise.Plugin.StepwiseStep.skprompt.txt"); + this._manualTemplate = EmbeddedResource.Read("Stepwise.Plugin.RenderFunctionManual.skprompt.txt"); + this._questionTemplate = EmbeddedResource.Read("Stepwise.Plugin.RenderQuestion.skprompt.txt"); + + // Load or use default PromptConfig + this._promptConfig = this.Config.PromptUserConfig ?? LoadPromptConfigFromResource(); + + // Set MaxTokens for the prompt config + this._promptConfig.SetMaxTokens(this.Config.MaxCompletionTokens); + + // Initialize prompt renderer + this._promptRenderer = new BasicPromptTemplateEngine(this._kernel.LoggerFactory); + + // Import native functions + this._nativeFunctions = this._kernel.ImportFunctions(this, RestrictedPluginName); + + // Create context and logger + this._logger = this._kernel.LoggerFactory.CreateLogger(this.GetType()); + } + + /// + public Plan CreatePlan(string goal) + { + if (string.IsNullOrEmpty(goal)) + { + throw new SKException("The goal specified is empty"); + } + + Plan plan = new(this._nativeFunctions["ExecutePlan"]); + plan.Parameters.Set("question", goal); + + plan.Outputs.Add("stepCount"); + plan.Outputs.Add("functionCount"); + plan.Outputs.Add("stepsTaken"); + plan.Outputs.Add("iterations"); + + return plan; + } + + /// + /// Execute a plan + /// + /// The question to answer + /// The context to use + /// The to monitor for cancellation requests. The default is . + /// The context with the result + /// No AIService available for getting completions. + [SKFunction, SKName("ExecutePlan"), Description("Execute a plan")] + public async Task ExecutePlanAsync( + [Description("The question to answer")] + string question, + SKContext context, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(question)) + { + context.Variables.Update("Question not found."); + return context; + } + + ChatHistory chatHistory = await this.InitializeChatHistoryAsync(this.CreateChatHistory(this._kernel, out var aiService), aiService, question, context, cancellationToken).ConfigureAwait(false); + + if (aiService is null) + { + throw new SKException("No AIService available for getting completions."); + } + + if (chatHistory is null) + { + throw new SKException("ChatHistory is null."); + } + + var startingMessageCount = chatHistory.Count; + + var stepsTaken = new List(); + SystemStep? lastStep = null; + + async Task GetNextStepAsync() + { + var actionText = await this.GetNextStepCompletionAsync(stepsTaken, chatHistory, aiService, startingMessageCount, cancellationToken).ConfigureAwait(false); + this._logger?.LogDebug("Response: {ActionText}", actionText); + return this.ParseResult(actionText); + } + + SKContext? TryGetFinalAnswer(SystemStep step, int iterations, SKContext context) + { + // If a final answer is found, update the context to be returned + if (!string.IsNullOrEmpty(step.FinalAnswer)) + { + this._logger?.LogInformation("Final Answer: {FinalAnswer}", step.FinalAnswer); + + context.Variables.Update(step.FinalAnswer); + + stepsTaken.Add(step); + + // Add additional results to the context + AddExecutionStatsToContext(stepsTaken, context, iterations); + + return context; + } + + return null; + } + + bool TryGetObservations(SystemStep step) + { + // If no Action/Thought is found, return any already available Observation from parsing the response. + // Otherwise, add a message to the chat history to guide LLM into returning the next thought|action. + if (string.IsNullOrEmpty(step.Action) && + string.IsNullOrEmpty(step.Thought)) + { + // If there is an observation, add it to the chat history + if (!string.IsNullOrEmpty(step.Observation)) + { + this._logger?.LogWarning("Invalid response from LLM, observation: {Observation}", step.Observation); + chatHistory.AddUserMessage($"{Observation} {step.Observation}"); + stepsTaken.Add(step); + lastStep = step; + return true; + } + + if (lastStep is not null && string.IsNullOrEmpty(lastStep.Action)) + { + this._logger?.LogWarning("No response from LLM, expected Action"); + chatHistory.AddUserMessage(Action); + } + else + { + this._logger?.LogWarning("No response from LLM, expected Thought"); + chatHistory.AddUserMessage(Thought); + } + + // No action or thought from LLM + return true; + } + + return false; + } + + SystemStep AddNextStep(SystemStep step) + { + // If the thought is empty and the last step had no action, copy action to last step and set as new nextStep + if (string.IsNullOrEmpty(step.Thought) && lastStep is not null && string.IsNullOrEmpty(lastStep.Action)) + { + lastStep.Action = step.Action; + lastStep.ActionVariables = step.ActionVariables; + + lastStep.OriginalResponse += step.OriginalResponse; + step = lastStep; + if (chatHistory.Count > startingMessageCount) + { + chatHistory.RemoveAt(chatHistory.Count - 1); + } + } + else + { + this._logger?.LogInformation("Thought: {Thought}", step.Thought); + stepsTaken.Add(step); + lastStep = step; + } + + return step; + } + + async Task TryGetActionObservationAsync(SystemStep step) + { + if (!string.IsNullOrEmpty(step.Action)) + { + this._logger?.LogInformation("Action: {Action}({ActionVariables}).", + step.Action, JsonSerializer.Serialize(step.ActionVariables)); + + // add [thought and] action to chat history + var actionMessage = $"{Action} {{\"action\": \"{step.Action}\",\"action_variables\": {JsonSerializer.Serialize(step.ActionVariables)}}}"; + var message = string.IsNullOrEmpty(step.Thought) ? actionMessage : $"{Thought} {step.Thought}\n{actionMessage}"; + + chatHistory.AddAssistantMessage(message); + + // Invoke the action + try + { + var result = await this.InvokeActionAsync(step.Action, step.ActionVariables, cancellationToken).ConfigureAwait(false); + + step.Observation = string.IsNullOrEmpty(result) ? "Got no result from action" : result!; + } + catch (Exception ex) when (!ex.IsCriticalException()) + { + step.Observation = $"Error invoking action {step.Action} : {ex.Message}"; + this._logger?.LogWarning(ex, "Error invoking action {Action}", step.Action); + } + + this._logger?.LogInformation("Observation: {Observation}", step.Observation); + chatHistory.AddUserMessage($"{Observation} {step.Observation}"); + + return true; + } + + return false; + } + + bool TryGetThought(SystemStep step) + { + // Add thought to chat history + if (!string.IsNullOrEmpty(step.Thought)) + { + chatHistory.AddAssistantMessage($"{Thought} {step.Thought}"); + } + + return false; + } + + for (int i = 0; i < this.Config.MaxIterations; i++) + { + // sleep for a bit to avoid rate limiting + if (i > 0) + { + await Task.Delay(this.Config.MinIterationTimeMs, cancellationToken).ConfigureAwait(false); + } + + // Get next step from LLM + var nextStep = await GetNextStepAsync().ConfigureAwait(false); + + // If final answer is available, we're done, return the context + var finalContext = TryGetFinalAnswer(nextStep, i + 1, context); + if (finalContext is not null) + { + return finalContext; + } + + // If we have an observation before running the action, continue to the next iteration + if (TryGetObservations(nextStep)) + { + continue; + } + + // Add next step to steps taken, merging with last step if necessary + // (e.g. the LLM gave Thought and Action one at a time, merge to encourage LLM to give both at once in future steps) + nextStep = AddNextStep(nextStep); + + // Execute actions and get observations + if (await TryGetActionObservationAsync(nextStep).ConfigureAwait(false)) + { + continue; + } + + this._logger?.LogInformation("Action: No action to take"); + + // If we have a thought, continue to the next iteration + if (TryGetThought(nextStep)) + { + continue; + } + } + + AddExecutionStatsToContext(stepsTaken, context, this.Config.MaxIterations); + context.Variables.Update(NoFinalAnswerFoundMessage); + + return context; + } + + #region setup helpers + + private async Task InitializeChatHistoryAsync(ChatHistory chatHistory, IAIService aiService, string question, SKContext context, CancellationToken cancellationToken) + { + string userManual = await this.GetUserManualAsync(question, context, cancellationToken).ConfigureAwait(false); + string userQuestion = await this.GetUserQuestionAsync(context, cancellationToken).ConfigureAwait(false); + + var systemContext = this._kernel.CreateNewContext(); + + systemContext.Variables.Set("suffix", this.Config.Suffix); + systemContext.Variables.Set("functionDescriptions", userManual); + string systemMessage = await this.GetSystemMessageAsync(systemContext, cancellationToken).ConfigureAwait(false); + + chatHistory.AddSystemMessage(systemMessage); + chatHistory.AddUserMessage(userQuestion); + + return chatHistory; + } + + private ChatHistory CreateChatHistory(IKernel kernel, out IAIService aiService) + { + ChatHistory chatHistory; + if (TryGetChatCompletion(this._kernel, out var chatCompletion)) + { + chatHistory = chatCompletion.CreateNewChat(); + aiService = chatCompletion; + } + else + { + var textCompletion = this._kernel.GetService(); + aiService = textCompletion; + chatHistory = new ChatHistory(); + } + + return chatHistory; + } + + private async Task GetUserManualAsync(string question, SKContext context, CancellationToken cancellationToken) + { + var descriptions = await this._kernel.Functions.GetFunctionsManualAsync(this.Config, question, this._logger, cancellationToken).ConfigureAwait(false); + context.Variables.Set("functionDescriptions", descriptions); + return await this._promptRenderer.RenderAsync(this._manualTemplate, context, cancellationToken).ConfigureAwait(false); + } + + private Task GetUserQuestionAsync(SKContext context, CancellationToken cancellationToken) + => this._promptRenderer.RenderAsync(this._questionTemplate, context, cancellationToken); + + private Task GetSystemMessageAsync(SKContext context, CancellationToken cancellationToken) + => this._promptRenderer.RenderAsync(this._promptTemplate, context, cancellationToken); + + #endregion setup helpers + + #region execution helpers + + private Task GetNextStepCompletionAsync(List stepsTaken, ChatHistory chatHistory, IAIService aiService, int startingMessageCount, CancellationToken token) + { + var skipStart = startingMessageCount; + var skipCount = 0; + var lastObservationIndex = chatHistory.FindLastIndex(m => m.Content.StartsWith(Observation, StringComparison.OrdinalIgnoreCase)); + var messagesToKeep = lastObservationIndex >= 0 ? chatHistory.Count - lastObservationIndex : 0; + + string? originalThought = null; + + var tokenCount = chatHistory.GetTokenCount(); + while (tokenCount >= this.Config.MaxPromptTokens && chatHistory.Count > (skipStart + skipCount + messagesToKeep)) + { + originalThought = $"{Thought} {stepsTaken.FirstOrDefault()?.Thought}"; + tokenCount = chatHistory.GetTokenCount($"{originalThought}\n{string.Format(CultureInfo.InvariantCulture, TrimMessageFormat, skipCount)}", skipStart, ++skipCount); + } + + if (tokenCount >= this.Config.MaxPromptTokens) + { + throw new SKException("ChatHistory is too long to get a completion. Try reducing the available functions."); + } + + var reducedChatHistory = new ChatHistory(); + reducedChatHistory.AddRange(chatHistory.Where((m, i) => i < skipStart || i >= skipStart + skipCount)); + + if (skipCount > 0 && originalThought is not null) + { + reducedChatHistory.InsertMessage(skipStart, AuthorRole.Assistant, string.Format(CultureInfo.InvariantCulture, TrimMessageFormat, skipCount)); + reducedChatHistory.InsertMessage(skipStart, AuthorRole.Assistant, originalThought); + } + + return this.GetCompletionAsync(aiService, reducedChatHistory, stepsTaken.Count == 0, token); + } + + private async Task GetCompletionAsync(IAIService aiService, ChatHistory chatHistory, bool addThought, CancellationToken token) + { + if (aiService is IChatCompletion chatCompletion) + { + var llmResponse = (await chatCompletion.GenerateMessageAsync(chatHistory, this._promptConfig.GetDefaultRequestSettings(), token).ConfigureAwait(false)); + return llmResponse; + } + else if (aiService is ITextCompletion textCompletion) + { + var thoughtProcess = string.Join("\n", chatHistory.Select(m => m.Content)); + + // Add Thought to the thought process at the start of the first iteration + if (addThought) + { + thoughtProcess = $"{thoughtProcess}\n{Thought}"; + addThought = false; + } + + thoughtProcess = $"{thoughtProcess}\n"; + IReadOnlyList results = await textCompletion.GetCompletionsAsync(thoughtProcess, this._promptConfig.GetDefaultRequestSettings(), token).ConfigureAwait(false); + + if (results.Count == 0) + { + throw new SKException("No completions returned."); + } + + return await results[0].GetCompletionAsync(token).ConfigureAwait(false); + } + + throw new SKException("No AIService available for getting completions."); + } + + /// + /// Parse LLM response into a SystemStep during execution + /// + /// The response from the LLM + /// A SystemStep + protected internal virtual SystemStep ParseResult(string input) + { + var result = new SystemStep + { + OriginalResponse = input + }; + + // Extract final answer + Match finalAnswerMatch = s_finalAnswerRegex.Match(input); + + if (finalAnswerMatch.Success) + { + result.FinalAnswer = finalAnswerMatch.Groups[1].Value.Trim(); + return result; + } + + // Extract thought + Match thoughtMatch = s_thoughtRegex.Match(input); + + if (thoughtMatch.Success) + { + // if it contains Action, it was only an action + if (!thoughtMatch.Value.Contains(Action)) + { + result.Thought = thoughtMatch.Value.Trim(); + } + } + else if (!input.Contains(Action)) + { + result.Thought = input; + } + else + { + return result; + } + + result.Thought = result.Thought.Replace(Thought, string.Empty).Trim(); + + // Extract action + // Using regex is prone to issues with complex action json, so we use a simple string search instead + // This can be less fault tolerant in some scenarios where the LLM tries to call multiple actions, for example. + // TODO -- that could possibly be improved if we allow an action to be a list of actions. + int actionIndex = input.IndexOf(Action, StringComparison.OrdinalIgnoreCase); + + if (actionIndex != -1) + { + int jsonStartIndex = input.IndexOf("{", actionIndex, StringComparison.OrdinalIgnoreCase); + if (jsonStartIndex != -1) + { + int jsonEndIndex = input.Substring(jsonStartIndex).LastIndexOf("}", StringComparison.OrdinalIgnoreCase); + if (jsonEndIndex != -1) + { + string json = input.Substring(jsonStartIndex, jsonEndIndex + 1); + + try + { + var systemStepResults = JsonSerializer.Deserialize(json); + + if (systemStepResults is not null) + { + result.Action = systemStepResults.Action; + result.ActionVariables = systemStepResults.ActionVariables; + } + } + catch (JsonException je) + { + result.Observation = $"Action parsing error: {je.Message}\nInvalid action: {json}"; + } + } + } + } + + return result; + } + + private async Task InvokeActionAsync(string actionName, Dictionary actionVariables, CancellationToken cancellationToken) + { + FunctionUtils.SplitPluginFunctionName(actionName, out var pluginName, out var functionName); + if (string.IsNullOrEmpty(functionName)) + { + this._logger?.LogDebug("Attempt to invoke action {Action} failed", actionName); + return $"Could not parse functionName from actionName: {actionName}. Please try again using one of the [AVAILABLE FUNCTIONS]."; + } + + var getFunctionCallback = this.Config.GetFunctionCallback ?? this._kernel.Functions.GetFunctionCallback(); + var targetFunction = getFunctionCallback(pluginName, functionName); + + if (targetFunction == null) + { + this._logger?.LogDebug("Attempt to invoke action {Action} failed", actionName); + return $"{actionName} is not in [AVAILABLE FUNCTIONS]. Please try again using one of the [AVAILABLE FUNCTIONS]."; + } + + try + { + string? result = null; + + var vars = this.CreateActionContextVariables(actionVariables); + var kernelResult = await this._kernel.RunAsync(targetFunction, vars, cancellationToken).ConfigureAwait(false); + var resultObject = kernelResult.GetValue(); + + var converter = TypeDescriptor.GetConverter(resultObject); + + if (converter.CanConvertTo(typeof(string))) + { + result = converter.ConvertToString(resultObject); + } + + this._logger?.LogTrace("Invoked {FunctionName}. Result: {Result}", targetFunction.Name, result); + + return result; + } + catch (Exception e) when (!e.IsCriticalException()) + { + this._logger?.LogError(e, "Something went wrong in system step: {Plugin}.{Function}. Error: {Error}", targetFunction.PluginName, targetFunction.Name, e.Message); + throw; + } + } + + private ContextVariables CreateActionContextVariables(Dictionary actionVariables) + { + ContextVariables vars = new(); + if (actionVariables != null) + { + foreach (var kvp in actionVariables) + { + vars.Set(kvp.Key, kvp.Value); + } + } + + return vars; + } + + #endregion execution helpers + + private static PromptTemplateConfig LoadPromptConfigFromResource() + { + string promptConfigString = EmbeddedResource.Read("Stepwise.Plugin.StepwiseStep.config.json"); + return !string.IsNullOrEmpty(promptConfigString) ? PromptTemplateConfig.FromJson(promptConfigString) : new PromptTemplateConfig(); + } + + private static bool TryGetChatCompletion(IKernel kernel, [NotNullWhen(true)] out IChatCompletion? chatCompletion) + { + try + { + // Client used to request answers to chat completion models + // TODO #2635 - Using TryGetService would improve cost of this method to avoid exception handling + chatCompletion = kernel.GetService(); + return true; + } + catch (SKException) + { + chatCompletion = null; + } + + return false; + } + + private static void AddExecutionStatsToContext(List stepsTaken, SKContext context, int iterations) + { + context.Variables.Set("stepCount", stepsTaken.Count.ToString(CultureInfo.InvariantCulture)); + context.Variables.Set("stepsTaken", JsonSerializer.Serialize(stepsTaken)); + context.Variables.Set("iterations", iterations.ToString(CultureInfo.InvariantCulture)); + + Dictionary actionCounts = new(); + foreach (var step in stepsTaken) + { + if (string.IsNullOrEmpty(step.Action)) { continue; } + + _ = actionCounts.TryGetValue(step.Action, out int currentCount); + actionCounts[step.Action!] = ++currentCount; + } + + var functionCallListWithCounts = string.Join(", ", actionCounts.Keys.Select(function => + $"{function}({actionCounts[function]})")); + + var functionCallCountStr = actionCounts.Values.Sum().ToString(CultureInfo.InvariantCulture); + + context.Variables.Set("functionCount", $"{functionCallCountStr} ({functionCallListWithCounts})"); + } + + #region private + + /// + /// The configuration for the StepwisePlanner + /// + private StepwisePlannerConfig Config { get; } + + // Context used to access the list of functions in the kernel + private readonly IKernel _kernel; + private readonly ILogger? _logger; + + /// + /// Planner native functions + /// + private readonly IDictionary _nativeFunctions = new Dictionary(); + + /// + /// The prompt template to use for the system step + /// + private readonly string _promptTemplate; + + /// + /// The question template to use for the system step + /// + private readonly string _questionTemplate; + + /// + /// The function manual template to use for the system step + /// + private readonly string _manualTemplate; + + /// + /// The prompt renderer to use for the system step + /// + private readonly BasicPromptTemplateEngine _promptRenderer; + + /// + /// The prompt config to use for the system step + /// + private readonly PromptTemplateConfig _promptConfig; + + /// + /// The name to use when creating semantic functions that are restricted from plan creation + /// + private const string RestrictedPluginName = "StepwisePlanner_Excluded"; + + /// + /// The Action tag + /// + private const string Action = "[ACTION]"; + + /// + /// The Thought tag + /// + private const string Thought = "[THOUGHT]"; + + /// + /// The Observation tag + /// + private const string Observation = "[OBSERVATION]"; + + /// + /// The chat message to include when trimming thought process history + /// + private const string TrimMessageFormat = "... I've removed the first {0} steps of my previous work to make room for the new stuff ..."; + + /// + /// The regex for parsing the thought response + /// + private static readonly Regex s_thoughtRegex = new(@"(\[THOUGHT\])?(?.+?)(?=\[ACTION\]|$)", RegexOptions.Singleline | RegexOptions.IgnoreCase); + + /// + /// The regex for parsing the final answer response + /// + private static readonly Regex s_finalAnswerRegex = new(@"\[FINAL[_\s\-]?ANSWER\](?.+)", RegexOptions.Singleline | RegexOptions.IgnoreCase); + + /// + /// The message to include when no final answer is found + /// + private const string NoFinalAnswerFoundMessage = "Result not found, review 'stepsTaken' to see what happened."; + + #endregion private +} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlannerConfig.cs b/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlannerConfig.cs new file mode 100644 index 000000000000..545edf191cb9 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlannerConfig.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.TemplateEngine; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Configuration for Stepwise planner instances. +/// +public sealed class StepwisePlannerConfig : PlannerConfigBase +{ + /// + /// Initializes a new instance of the + /// + public StepwisePlannerConfig() + { + this.MaxTokens = 4000; + } + + /// + /// The ratio of tokens to allocate to the completion request. (prompt / (prompt + completion)) + /// + public double MaxTokensRatio { get; set; } = 0.1; + + internal int MaxCompletionTokens { get { return (int)(this.MaxTokens * this.MaxTokensRatio); } } + + internal int MaxPromptTokens { get { return (int)(this.MaxTokens * (1 - this.MaxTokensRatio)); } } + + /// + /// The maximum number of iterations to allow in a plan. + /// + public int MaxIterations { get; set; } = 15; + + /// + /// The minimum time to wait between iterations in milliseconds. + /// + public int MinIterationTimeMs { get; set; } + + /// + /// The configuration to use for the prompt template. + /// + public PromptTemplateConfig? PromptUserConfig { get; set; } + + /// + /// A suffix to use within the default prompt template. + /// + public string Suffix { get; set; } = @"Let's break down the problem step by step and think about the best approach. Label steps as they are taken. + +Continue the thought process!"; +} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlannerExtensions.cs b/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlannerExtensions.cs new file mode 100644 index 000000000000..9a49f49680c5 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/StepwisePlannerExtensions.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Extension methods for class. +/// +public static class StepwisePlannerExtensions +{ + /// + /// Returns decorated instance of with enabled instrumentation. + /// + /// Instance of to decorate. + /// The to use for logging. If null, no logging will be performed. + public static IStepwisePlanner WithInstrumentation(this IStepwisePlanner planner, ILoggerFactory? loggerFactory = null) + { + return new InstrumentedStepwisePlanner(planner, loggerFactory); + } +} diff --git a/dotnet/src/Planners/Planners.Core/Stepwise/SystemStep.cs b/dotnet/src/Planners/Planners.Core/Stepwise/SystemStep.cs new file mode 100644 index 000000000000..812db626cca2 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Stepwise/SystemStep.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// A step in a Stepwise plan. +/// +public class SystemStep +{ + /// + /// Gets or sets the step number. + /// + [JsonPropertyName("thought")] + public string Thought { get; set; } = string.Empty; + + /// + /// Gets or sets the action of the step + /// + [JsonPropertyName("action")] + public string Action { get; set; } = string.Empty; + + /// + /// Gets or sets the variables for the action + /// + [JsonPropertyName("action_variables")] + public Dictionary ActionVariables { get; set; } = new(); + + /// + /// Gets or sets the output of the action + /// + [JsonPropertyName("observation")] + public string Observation { get; set; } = string.Empty; + + /// + /// Gets or sets the output of the system + /// + [JsonPropertyName("final_answer")] + public string FinalAnswer { get; set; } = string.Empty; + + /// + /// The raw response from the action + /// + [JsonPropertyName("original_response")] + public string OriginalResponse { get; set; } = string.Empty; +} diff --git a/dotnet/src/Planners/Planners.Core/Utils/EmbeddedResource.cs b/dotnet/src/Planners/Planners.Core/Utils/EmbeddedResource.cs new file mode 100644 index 000000000000..c759b9ef9171 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Utils/EmbeddedResource.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.IO; +using System.Reflection; +using Microsoft.SemanticKernel.Diagnostics; + +#pragma warning disable IDE0130 +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +internal static class EmbeddedResource +{ + private static readonly string? s_namespace = typeof(EmbeddedResource).Namespace; + + internal static string Read(string name) + { + var assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly; + if (assembly == null) { throw new SKException($"[{s_namespace}] {name} assembly not found"); } + + using Stream? resource = assembly.GetManifestResourceStream($"{s_namespace}." + name); + if (resource == null) { throw new SKException($"[{s_namespace}] {name} resource not found"); } + + using var reader = new StreamReader(resource); + return reader.ReadToEnd(); + } +} diff --git a/dotnet/src/Planners/Planners.Core/Utils/FunctionUtils.cs b/dotnet/src/Planners/Planners.Core/Utils/FunctionUtils.cs new file mode 100644 index 000000000000..f67dcb4af978 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Utils/FunctionUtils.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +internal static class FunctionUtils +{ + internal static void SplitPluginFunctionName(string pluginFunctionName, out string pluginName, out string functionName) + { + var pluginFunctionNameParts = pluginFunctionName.Split('.'); + pluginName = pluginFunctionNameParts?.Length > 1 ? pluginFunctionNameParts[0] : string.Empty; + functionName = pluginFunctionNameParts?.Length > 1 ? pluginFunctionNameParts[1] : pluginFunctionName; + } +} diff --git a/dotnet/src/Plugins/Plugins.Core/ConversationSummaryPlugin.cs b/dotnet/src/Plugins/Plugins.Core/ConversationSummaryPlugin.cs new file mode 100644 index 000000000000..633e50f1c4b3 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Core/ConversationSummaryPlugin.cs @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Text; + +namespace Microsoft.SemanticKernel.Plugins.Core; + +/// +/// Semantic plugin that enables conversations summarization. +/// +/// +/// +/// var kernel Kernel.Builder.Build(); +/// kernel.ImportFunctions(new ConversationSummaryPlugin(kernel)); +/// +/// +public class ConversationSummaryPlugin +{ + /// + /// The max tokens to process in a single semantic function call. + /// + private const int MaxTokens = 1024; + + private readonly ISKFunction _summarizeConversationFunction; + private readonly ISKFunction _conversationActionItemsFunction; + private readonly ISKFunction _conversationTopicsFunction; + + /// + /// Initializes a new instance of the class. + /// + /// Kernel instance + public ConversationSummaryPlugin(IKernel kernel) + { + this._summarizeConversationFunction = kernel.CreateSemanticFunction( + SemanticFunctionConstants.SummarizeConversationDefinition, + pluginName: nameof(ConversationSummaryPlugin), + description: "Given a section of a conversation transcript, summarize the part of the conversation.", + requestSettings: new AIRequestSettings() + { + ExtensionData = new Dictionary() + { + { "Temperature", 0.1 }, + { "TopP", 0.5 }, + { "MaxTokens", MaxTokens } + } + }); + + this._conversationActionItemsFunction = kernel.CreateSemanticFunction( + SemanticFunctionConstants.GetConversationActionItemsDefinition, + pluginName: nameof(ConversationSummaryPlugin), + description: "Given a section of a conversation transcript, identify action items.", + requestSettings: new AIRequestSettings() + { + ExtensionData = new Dictionary() + { + { "Temperature", 0.1 }, + { "TopP", 0.5 }, + { "MaxTokens", MaxTokens } + } + }); + + this._conversationTopicsFunction = kernel.CreateSemanticFunction( + SemanticFunctionConstants.GetConversationTopicsDefinition, + pluginName: nameof(ConversationSummaryPlugin), + description: "Analyze a conversation transcript and extract key topics worth remembering.", + requestSettings: new AIRequestSettings() + { + ExtensionData = new Dictionary() + { + { "Temperature", 0.1 }, + { "TopP", 0.5 }, + { "MaxTokens", MaxTokens } + } + }); + } + + /// + /// Given a long conversation transcript, summarize the conversation. + /// + /// A long conversation transcript. + /// The SKContext for function execution. + [SKFunction, Description("Given a long conversation transcript, summarize the conversation.")] + public Task SummarizeConversationAsync( + [Description("A long conversation transcript.")] string input, + SKContext context) + { + List lines = TextChunker.SplitPlainTextLines(input, MaxTokens); + List paragraphs = TextChunker.SplitPlainTextParagraphs(lines, MaxTokens); + + return this._summarizeConversationFunction + .AggregatePartitionedResultsAsync(paragraphs, context); + } + + /// + /// Given a long conversation transcript, identify action items. + /// + /// A long conversation transcript. + /// The SKContext for function execution. + [SKFunction, Description("Given a long conversation transcript, identify action items.")] + public Task GetConversationActionItemsAsync( + [Description("A long conversation transcript.")] string input, + SKContext context) + { + List lines = TextChunker.SplitPlainTextLines(input, MaxTokens); + List paragraphs = TextChunker.SplitPlainTextParagraphs(lines, MaxTokens); + + return this._conversationActionItemsFunction + .AggregatePartitionedResultsAsync(paragraphs, context); + } + + /// + /// Given a long conversation transcript, identify topics. + /// + /// A long conversation transcript. + /// The SKContext for function execution. + [SKFunction, Description("Given a long conversation transcript, identify topics worth remembering.")] + public Task GetConversationTopicsAsync( + [Description("A long conversation transcript.")] string input, + SKContext context) + { + List lines = TextChunker.SplitPlainTextLines(input, MaxTokens); + List paragraphs = TextChunker.SplitPlainTextParagraphs(lines, MaxTokens); + + return this._conversationTopicsFunction + .AggregatePartitionedResultsAsync(paragraphs, context); + } +} diff --git a/dotnet/src/Plugins/Plugins.Core/FileIOPlugin.cs b/dotnet/src/Plugins/Plugins.Core/FileIOPlugin.cs new file mode 100644 index 000000000000..c50ed4e9302c --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Core/FileIOPlugin.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.SemanticKernel.Plugins.Core; + +/// +/// Read and write from a file. +/// +/// +/// Usage: kernel.ImportFunctions(new FileIOPlugin(), "file"); +/// Examples: +/// {{file.readAsync $path }} => "hello world" +/// {{file.writeAsync}} +/// +public sealed class FileIOPlugin +{ + /// + /// Read a file + /// + /// + /// {{file.readAsync $path }} => "hello world" + /// + /// Source file + /// File content + [SKFunction, Description("Read a file")] + public async Task ReadAsync([Description("Source file")] string path) + { + using var reader = File.OpenText(path); + return await reader.ReadToEndAsync().ConfigureAwait(false); + } + + /// + /// Write a file + /// + /// + /// {{file.writeAsync}} + /// + /// The destination file path + /// The file content to write + /// An awaitable task + [SKFunction, Description("Write a file")] + public async Task WriteAsync( + [Description("Destination file")] string path, + [Description("File content")] string content) + { + byte[] text = Encoding.UTF8.GetBytes(content); + if (File.Exists(path) && File.GetAttributes(path).HasFlag(FileAttributes.ReadOnly)) + { + // Most environments will throw this with OpenWrite, but running inside docker on Linux will not. + throw new UnauthorizedAccessException($"File is read-only: {path}"); + } + + using var writer = File.OpenWrite(path); + await writer.WriteAsync(text, 0, text.Length).ConfigureAwait(false); + } +} diff --git a/dotnet/src/Plugins/Plugins.Core/HttpPlugin.cs b/dotnet/src/Plugins/Plugins.Core/HttpPlugin.cs new file mode 100644 index 000000000000..e8f5124c4ba7 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Core/HttpPlugin.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; + +namespace Microsoft.SemanticKernel.Plugins.Core; + +/// +/// A plugin that provides HTTP functionality. +/// +/// +/// Usage: kernel.ImportFunctions(new HttpPlugin(), "http"); +/// Examples: +/// SKContext.Variables["url"] = "https://www.bing.com" +/// {{http.getAsync $url}} +/// {{http.postAsync $url}} +/// {{http.putAsync $url}} +/// {{http.deleteAsync $url}} +/// +[System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1054:URI-like parameters should not be strings", + Justification = "Semantic Kernel operates on strings")] +public sealed class HttpPlugin +{ + private readonly HttpClient _client; + + /// + /// Initializes a new instance of the class. + /// + public HttpPlugin() : this(new HttpClient(NonDisposableHttpClientHandler.Instance, disposeHandler: false)) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The HTTP client to use. + /// + /// assumes ownership of the instance and will dispose it when the plugin is disposed. + /// + public HttpPlugin(HttpClient client) => + this._client = client; + + /// + /// Sends an HTTP GET request to the specified URI and returns the response body as a string. + /// + /// URI of the request + /// The token to use to request cancellation. + /// The response body as a string. + [SKFunction, Description("Makes a GET request to a uri")] + public Task GetAsync( + [Description("The URI of the request")] string uri, + CancellationToken cancellationToken = default) => + this.SendRequestAsync(uri, HttpMethod.Get, requestContent: null, cancellationToken); + + /// + /// Sends an HTTP POST request to the specified URI and returns the response body as a string. + /// + /// URI of the request + /// The body of the request + /// The token to use to request cancellation. + /// The response body as a string. + [SKFunction, Description("Makes a POST request to a uri")] + public Task PostAsync( + [Description("The URI of the request")] string uri, + [Description("The body of the request")] string body, + CancellationToken cancellationToken = default) => + this.SendRequestAsync(uri, HttpMethod.Post, new StringContent(body), cancellationToken); + + /// + /// Sends an HTTP PUT request to the specified URI and returns the response body as a string. + /// + /// URI of the request + /// The body of the request + /// The token to use to request cancellation. + /// The response body as a string. + [SKFunction, Description("Makes a PUT request to a uri")] + public Task PutAsync( + [Description("The URI of the request")] string uri, + [Description("The body of the request")] string body, + CancellationToken cancellationToken = default) => + this.SendRequestAsync(uri, HttpMethod.Put, new StringContent(body), cancellationToken); + + /// + /// Sends an HTTP DELETE request to the specified URI and returns the response body as a string. + /// + /// URI of the request + /// The token to use to request cancellation. + /// The response body as a string. + [SKFunction, Description("Makes a DELETE request to a uri")] + public Task DeleteAsync( + [Description("The URI of the request")] string uri, + CancellationToken cancellationToken = default) => + this.SendRequestAsync(uri, HttpMethod.Delete, requestContent: null, cancellationToken); + + /// Sends an HTTP request and returns the response content as a string. + /// The URI of the request. + /// The HTTP method for the request. + /// Optional request content. + /// The token to use to request cancellation. + private async Task SendRequestAsync(string uri, HttpMethod method, HttpContent? requestContent, CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(method, uri) { Content = requestContent }; + request.Headers.Add("User-Agent", Telemetry.HttpUserAgent); + using var response = await this._client.SendWithSuccessCheckAsync(request, cancellationToken).ConfigureAwait(false); + return await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); + } +} diff --git a/dotnet/src/Plugins/Plugins.Core/MathPlugin.cs b/dotnet/src/Plugins/Plugins.Core/MathPlugin.cs new file mode 100644 index 000000000000..c8f0a84d8978 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Core/MathPlugin.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; + +namespace Microsoft.SemanticKernel.Plugins.Core; + +/// +/// MathPlugin provides a set of functions to make Math calculations. +/// +/// +/// Usage: kernel.ImportFunctions(new MathPlugin(), "math"); +/// Examples: +/// {{math.Add}} => Returns the sum of FirstNumber and SecondNumber (provided in the SKContext) +/// +public sealed class MathPlugin +{ + /// + /// Returns the Addition result of initial and amount values provided. + /// + /// Initial value to which to add the specified amount + /// The amount to add as a string. + /// The resulting sum as a string. + [SKFunction, Description("Adds an amount to a value")] + public int Add( + [Description("The value to add")] int value, + [Description("Amount to add")] int amount) => + value + amount; + + /// + /// Returns the Sum of two SKContext numbers provided. + /// + /// Initial value from which to subtract the specified amount + /// The amount to subtract as a string. + /// The resulting subtraction as a string. + [SKFunction, Description("Subtracts an amount from a value")] + public int Subtract( + [Description("The value to subtract")] int value, + [Description("Amount to subtract")] int amount) => + value - amount; +} diff --git a/dotnet/src/Plugins/Plugins.Core/Plugins.Core.csproj b/dotnet/src/Plugins/Plugins.Core/Plugins.Core.csproj new file mode 100644 index 000000000000..04ac3a26b1d6 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Core/Plugins.Core.csproj @@ -0,0 +1,27 @@ + + + + + Microsoft.SemanticKernel.Plugins.Core + $(AssemblyName) + netstandard2.0 + + + + + + + + Semantic Kernel - Core Plugins + Semantic Kernel core plugins. + + + + + + + + + + + diff --git a/dotnet/src/Skills/Skills.Core/SemanticFunctionConstants.cs b/dotnet/src/Plugins/Plugins.Core/SemanticFunctionConstants.cs similarity index 98% rename from dotnet/src/Skills/Skills.Core/SemanticFunctionConstants.cs rename to dotnet/src/Plugins/Plugins.Core/SemanticFunctionConstants.cs index 329b21561b85..eaa18c28c93b 100644 --- a/dotnet/src/Skills/Skills.Core/SemanticFunctionConstants.cs +++ b/dotnet/src/Plugins/Plugins.Core/SemanticFunctionConstants.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.Core; +namespace Microsoft.SemanticKernel.Plugins.Core; internal static class SemanticFunctionConstants { diff --git a/dotnet/src/Plugins/Plugins.Core/TextPlugin.cs b/dotnet/src/Plugins/Plugins.Core/TextPlugin.cs new file mode 100644 index 000000000000..2ba5d1d5d2f6 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Core/TextPlugin.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Globalization; + +namespace Microsoft.SemanticKernel.Plugins.Core; + +/// +/// TextPlugin provides a set of functions to manipulate strings. +/// +/// +/// Usage: kernel.ImportFunctions(new TextPlugin(), "text"); +/// +/// Examples: +/// SKContext.Variables["input"] = " hello world " +/// {{text.trim $input}} => "hello world" +/// {{text.trimStart $input} => "hello world " +/// {{text.trimEnd $input} => " hello world" +/// SKContext.Variables["input"] = "hello world" +/// {{text.uppercase $input}} => "HELLO WORLD" +/// SKContext.Variables["input"] = "HELLO WORLD" +/// {{text.lowercase $input}} => "hello world" +/// +public sealed class TextPlugin +{ + /// + /// Trim whitespace from the start and end of a string. + /// + /// + /// SKContext.Variables["input"] = " hello world " + /// {{text.trim $input}} => "hello world" + /// + /// The string to trim. + /// The trimmed string. + [SKFunction, Description("Trim whitespace from the start and end of a string.")] + public string Trim(string input) => input.Trim(); + + /// + /// Trim whitespace from the start of a string. + /// + /// + /// SKContext.Variables["input"] = " hello world " + /// {{text.trimStart $input} => "hello world " + /// + /// The string to trim. + /// The trimmed string. + [SKFunction, Description("Trim whitespace from the start of a string.")] + public string TrimStart(string input) => input.TrimStart(); + + /// + /// Trim whitespace from the end of a string. + /// + /// + /// SKContext.Variables["input"] = " hello world " + /// {{text.trimEnd $input} => " hello world" + /// + /// The string to trim. + /// The trimmed string. + [SKFunction, Description("Trim whitespace from the end of a string.")] + public string TrimEnd(string input) => input.TrimEnd(); + + /// + /// Convert a string to uppercase. + /// + /// + /// SKContext.Variables["input"] = "hello world" + /// {{text.uppercase $input}} => "HELLO WORLD" + /// + /// The string to convert. + /// An object that supplies culture-specific casing rules. + /// The converted string. + [SKFunction, Description("Convert a string to uppercase.")] + public string Uppercase(string input, CultureInfo? cultureInfo = null) => input.ToUpper(cultureInfo); + + /// + /// Convert a string to lowercase. + /// + /// + /// SKContext.Variables["input"] = "HELLO WORLD" + /// {{text.lowercase $input}} => "hello world" + /// + /// The string to convert. + /// An object that supplies culture-specific casing rules. + /// The converted string. + [SKFunction, Description("Convert a string to lowercase.")] + public string Lowercase(string input, CultureInfo? cultureInfo = null) => input.ToLower(cultureInfo); + + /// + /// Get the length of a string. Returns 0 if null or empty + /// + /// + /// SKContext.Variables["input"] = "HELLO WORLD" + /// {{text.length $input}} => "11" + /// + /// The string to get length. + /// The length size of string (0) if null or empty. + [SKFunction, Description("Get the length of a string.")] + public int Length(string input) => input?.Length ?? 0; + + /// + /// Concatenate two strings into one + /// + /// + /// text = "HELLO " + /// SKContext.Variables["input2"] = "WORLD" + /// Result: "HELLO WORLD" + /// + /// First input to concatenate with + /// Second input to concatenate with + /// Concatenation result from both inputs. + [SKFunction, Description("Concat two strings into one.")] + public string Concat( + [Description("First input to concatenate with")] string input, + [Description("Second input to concatenate with")] string input2) => + string.Concat(input, input2); + + /// + /// Echo the input string. Useful for capturing plan input for use in multiple functions. + /// + /// Input string to echo. + /// The input string. + [SKFunction, Description("Echo the input string. Useful for capturing plan input for use in multiple functions.")] + public string Echo( + [Description("Input string to echo.")] string text) + { + return text; + } +} diff --git a/dotnet/src/Plugins/Plugins.Core/TimePlugin.cs b/dotnet/src/Plugins/Plugins.Core/TimePlugin.cs new file mode 100644 index 000000000000..4ec697ae7cee --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Core/TimePlugin.cs @@ -0,0 +1,279 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; + +namespace Microsoft.SemanticKernel.Plugins.Core; + +/// +/// TimePlugin provides a set of functions to get the current time and date. +/// +/// +/// Usage: kernel.ImportFunctions(new TimePlugin(), "time"); +/// Examples: +/// {{time.date}} => Sunday, 12 January, 2031 +/// {{time.today}} => Sunday, 12 January, 2031 +/// {{time.now}} => Sunday, January 12, 2031 9:15 PM +/// {{time.utcNow}} => Sunday, January 13, 2031 5:15 AM +/// {{time.time}} => 09:15:07 PM +/// {{time.year}} => 2031 +/// {{time.month}} => January +/// {{time.monthNumber}} => 01 +/// {{time.day}} => 12 +/// {{time.dayOfMonth}} => 12 +/// {{time.dayOfWeek}} => Sunday +/// {{time.hour}} => 9 PM +/// {{time.hourNumber}} => 21 +/// {{time.daysAgo $days}} => Sunday, January 12, 2025 9:15 PM +/// {{time.lastMatchingDay $dayName}} => Sunday, 7 May, 2023 +/// {{time.minute}} => 15 +/// {{time.minutes}} => 15 +/// {{time.second}} => 7 +/// {{time.seconds}} => 7 +/// {{time.timeZoneOffset}} => -08:00 +/// {{time.timeZoneName}} => PST +/// +/// +/// Note: the time represents the time on the hw/vm/machine where the kernel is running. +/// TODO: import and use user's timezone +/// +public sealed class TimePlugin +{ + /// + /// Get the current date + /// + /// + /// {{time.date}} => Sunday, 12 January, 2031 + /// + /// The current date + [SKFunction, Description("Get the current date")] + public string Date(IFormatProvider? formatProvider = null) => + // Example: Sunday, 12 January, 2025 + DateTimeOffset.Now.ToString("D", formatProvider); + + /// + /// Get the current date + /// + /// + /// {{time.today}} => Sunday, 12 January, 2031 + /// + /// The current date + [SKFunction, Description("Get the current date")] + public string Today(IFormatProvider? formatProvider = null) => + // Example: Sunday, 12 January, 2025 + this.Date(formatProvider); + + /// + /// Get the current date and time in the local time zone" + /// + /// + /// {{time.now}} => Sunday, January 12, 2025 9:15 PM + /// + /// The current date and time in the local time zone + [SKFunction, Description("Get the current date and time in the local time zone")] + public string Now(IFormatProvider? formatProvider = null) => + // Sunday, January 12, 2025 9:15 PM + DateTimeOffset.Now.ToString("f", formatProvider); + + /// + /// Get the current UTC date and time + /// + /// + /// {{time.utcNow}} => Sunday, January 13, 2025 5:15 AM + /// + /// The current UTC date and time + [SKFunction, Description("Get the current UTC date and time")] + public string UtcNow(IFormatProvider? formatProvider = null) => + // Sunday, January 13, 2025 5:15 AM + DateTimeOffset.UtcNow.ToString("f", formatProvider); + + /// + /// Get the current time + /// + /// + /// {{time.time}} => 09:15:07 PM + /// + /// The current time + [SKFunction, Description("Get the current time")] + public string Time(IFormatProvider? formatProvider = null) => + // Example: 09:15:07 PM + DateTimeOffset.Now.ToString("hh:mm:ss tt", formatProvider); + + /// + /// Get the current year + /// + /// + /// {{time.year}} => 2025 + /// + /// The current year + [SKFunction, Description("Get the current year")] + public string Year(IFormatProvider? formatProvider = null) => + // Example: 2025 + DateTimeOffset.Now.ToString("yyyy", formatProvider); + + /// + /// Get the current month name + /// + /// + /// {time.month}} => January + /// + /// The current month name + [SKFunction, Description("Get the current month name")] + public string Month(IFormatProvider? formatProvider = null) => + // Example: January + DateTimeOffset.Now.ToString("MMMM", formatProvider); + + /// + /// Get the current month number + /// + /// + /// {{time.monthNumber}} => 01 + /// + /// The current month number + [SKFunction, Description("Get the current month number")] + public string MonthNumber(IFormatProvider? formatProvider = null) => + // Example: 01 + DateTimeOffset.Now.ToString("MM", formatProvider); + + /// + /// Get the current day of the month + /// + /// + /// {{time.day}} => 12 + /// + /// The current day of the month + [SKFunction, Description("Get the current day of the month")] + public string Day(IFormatProvider? formatProvider = null) => + // Example: 12 + DateTimeOffset.Now.ToString("dd", formatProvider); + + /// + /// Get the date a provided number of days in the past + /// + /// + /// SKContext.Variables["input"] = "3" + /// {{time.daysAgo}} => Sunday, January 12, 2025 9:15 PM + /// + /// The date the provided number of days before today + [SKFunction] + [Description("Get the date offset by a provided number of days from today")] + public string DaysAgo([Description("The number of days to offset from today"), SKName("input")] double daysOffset, IFormatProvider? formatProvider = null) => + DateTimeOffset.Now.AddDays(-daysOffset).ToString("D", formatProvider); + + /// + /// Get the current day of the week + /// + /// + /// {{time.dayOfWeek}} => Sunday + /// + /// The current day of the week + [SKFunction, Description("Get the current day of the week")] + public string DayOfWeek(IFormatProvider? formatProvider = null) => + // Example: Sunday + DateTimeOffset.Now.ToString("dddd", formatProvider); + + /// + /// Get the current clock hour + /// + /// + /// {{time.hour}} => 9 PM + /// + /// The current clock hour + [SKFunction, Description("Get the current clock hour")] + public string Hour(IFormatProvider? formatProvider = null) => + // Example: 9 PM + DateTimeOffset.Now.ToString("h tt", formatProvider); + + /// + /// Get the current clock 24-hour number + /// + /// + /// {{time.hourNumber}} => 21 + /// + /// The current clock 24-hour number + [SKFunction, Description("Get the current clock 24-hour number")] + public string HourNumber(IFormatProvider? formatProvider = null) => + // Example: 21 + DateTimeOffset.Now.ToString("HH", formatProvider); + + /// + /// Get the date of the previous day matching the supplied day name + /// + /// + /// {{time.lastMatchingDay $dayName}} => Sunday, 7 May, 2023 + /// + /// The date of the last instance of this day name + /// dayName is not a recognized name of a day of the week + [SKFunction] + [Description("Get the date of the last day matching the supplied week day name in English. Example: Che giorno era 'Martedi' scorso -> dateMatchingLastDayName 'Tuesday' => Tuesday, 16 May, 2023")] + public string DateMatchingLastDayName( + [Description("The day name to match"), SKName("input")] DayOfWeek dayName, + IFormatProvider? formatProvider = null) + { + DateTimeOffset dateTime = DateTimeOffset.Now; + + // Walk backwards from the previous day for up to a week to find the matching day + for (int i = 1; i <= 7; ++i) + { + dateTime = dateTime.AddDays(-1); + if (dateTime.DayOfWeek == dayName) + { + break; + } + } + + return dateTime.ToString("D", formatProvider); + } + + /// + /// Get the minutes on the current hour + /// + /// + /// {{time.minute}} => 15 + /// + /// The minutes on the current hour + [SKFunction, Description("Get the minutes on the current hour")] + public string Minute(IFormatProvider? formatProvider = null) => + // Example: 15 + DateTimeOffset.Now.ToString("mm", formatProvider); + + /// + /// Get the seconds on the current minute + /// + /// + /// {{time.second}} => 7 + /// + /// The seconds on the current minute + [SKFunction, Description("Get the seconds on the current minute")] + public string Second(IFormatProvider? formatProvider = null) => + // Example: 07 + DateTimeOffset.Now.ToString("ss", formatProvider); + + /// + /// Get the local time zone offset from UTC + /// + /// + /// {{time.timeZoneOffset}} => -08:00 + /// + /// The local time zone offset from UTC + [SKFunction, Description("Get the local time zone offset from UTC")] + public string TimeZoneOffset(IFormatProvider? formatProvider = null) => + // Example: -08:00 + DateTimeOffset.Now.ToString("%K", formatProvider); + + /// + /// Get the local time zone name + /// + /// + /// {{time.timeZoneName}} => PST + /// + /// + /// Note: this is the "current" timezone and it can change over the year, e.g. from PST to PDT + /// + /// The local time zone name + [SKFunction, Description("Get the local time zone name")] + public string TimeZoneName() => + // Example: PST + // Note: this is the "current" timezone and it can change over the year, e.g. from PST to PDT + TimeZoneInfo.Local.DisplayName; +} diff --git a/dotnet/src/Plugins/Plugins.Core/WaitPlugin.cs b/dotnet/src/Plugins/Plugins.Core/WaitPlugin.cs new file mode 100644 index 000000000000..a5f10d4a19a6 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Core/WaitPlugin.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Threading.Tasks; + +namespace Microsoft.SemanticKernel.Plugins.Core; + +/// +/// WaitPlugin provides a set of functions to wait before making the rest of operations. +/// +/// +/// Usage: kernel.ImportFunctions(new WaitPlugin(), "wait"); +/// Examples: +/// {{wait.seconds 10}} => Wait 10 seconds +/// +public sealed class WaitPlugin +{ + private readonly TimeProvider _timeProvider; + + /// + /// Initializes a new instance of the class. + /// + /// An optional time provider. If not provided, a default time provider will be used. + public WaitPlugin(TimeProvider? timeProvider = null) => + this._timeProvider = timeProvider ?? TimeProvider.System; + + /// + /// Wait a given amount of seconds + /// + /// + /// {{wait.seconds 10}} (Wait 10 seconds) + /// + [SKFunction, Description("Wait a given amount of seconds")] + public Task SecondsAsync([Description("The number of seconds to wait")] decimal seconds) => + this._timeProvider.Delay(TimeSpan.FromSeconds((double)Math.Max(seconds, 0))); +} diff --git a/dotnet/src/Plugins/Plugins.Document/DocumentPlugin.cs b/dotnet/src/Plugins/Plugins.Document/DocumentPlugin.cs new file mode 100644 index 000000000000..4800c0ce67e6 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Document/DocumentPlugin.cs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Plugins.Document.FileSystem; + +namespace Microsoft.SemanticKernel.Plugins.Document; + +//********************************************************************************************************************** +// EXAMPLE USAGE +// Option #1: as a standalone C# function +// +// DocumentPlugin documentPlugin = new(new WordDocumentConnector(), new LocalDriveConnector()); +// string filePath = "PATH_TO_DOCX_FILE.docx"; +// string text = await documentPlugin.ReadTextAsync(filePath); +// Console.WriteLine(text); +// +// +// Option #2: with the Semantic Kernel +// +// DocumentPlugin documentPlugin = new(new WordDocumentConnector(), new LocalDriveConnector()); +// string filePath = "PATH_TO_DOCX_FILE.docx"; +// ISemanticKernel kernel = SemanticKernel.Build(); +// var result = await kernel.RunAsync( +// filePath, +// documentPlugin.ReadTextAsync); +// Console.WriteLine(result); +//********************************************************************************************************************** + +/// +/// Plugin for interacting with documents (e.g. Microsoft Word) +/// +public sealed class DocumentPlugin +{ + /// + /// parameter names. + /// + public static class Parameters + { + /// + /// Document file path. + /// + public const string FilePath = "filePath"; + } + + private readonly IDocumentConnector _documentConnector; + private readonly IFileSystemConnector _fileSystemConnector; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Document connector + /// File system connector + /// The to use for logging. If null, no logging will be performed. + public DocumentPlugin(IDocumentConnector documentConnector, IFileSystemConnector fileSystemConnector, ILoggerFactory? loggerFactory = null) + { + this._documentConnector = documentConnector ?? throw new ArgumentNullException(nameof(documentConnector)); + this._fileSystemConnector = fileSystemConnector ?? throw new ArgumentNullException(nameof(fileSystemConnector)); + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(DocumentPlugin)) : NullLogger.Instance; + } + + /// + /// Read all text from a document, using as the file path. + /// + [SKFunction, Description("Read all text from a document")] + public async Task ReadTextAsync( + [Description("Path to the file to read")] string filePath, + CancellationToken cancellationToken = default) + { + this._logger.LogDebug("Reading text from {0}", filePath); + using var stream = await this._fileSystemConnector.GetFileContentStreamAsync(filePath, cancellationToken).ConfigureAwait(false); + return this._documentConnector.ReadText(stream); + } + + /// + /// Append the text in to a document. If the document doesn't exist, it will be created. + /// + [SKFunction, Description("Append text to a document. If the document doesn't exist, it will be created.")] + public async Task AppendTextAsync( + [Description("Text to append")] string text, + [Description("Destination file path")] string filePath, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(filePath)) + { + throw new ArgumentException("Variable was null or whitespace", nameof(filePath)); + } + + // If the document already exists, open it. If not, create it. + if (await this._fileSystemConnector.FileExistsAsync(filePath, cancellationToken).ConfigureAwait(false)) + { + this._logger.LogDebug("Writing text to file {0}", filePath); + using Stream stream = await this._fileSystemConnector.GetWriteableFileStreamAsync(filePath, cancellationToken).ConfigureAwait(false); + this._documentConnector.AppendText(stream, text); + } + else + { + this._logger.LogDebug("File does not exist. Creating file at {0}", filePath); + using Stream stream = await this._fileSystemConnector.CreateFileAsync(filePath, cancellationToken).ConfigureAwait(false); + this._documentConnector.Initialize(stream); + + this._logger.LogDebug("Writing text to {0}", filePath); + this._documentConnector.AppendText(stream, text); + } + } +} diff --git a/dotnet/src/Skills/Skills.Document/FileSystem/IFileSystemConnector.cs b/dotnet/src/Plugins/Plugins.Document/FileSystem/IFileSystemConnector.cs similarity index 96% rename from dotnet/src/Skills/Skills.Document/FileSystem/IFileSystemConnector.cs rename to dotnet/src/Plugins/Plugins.Document/FileSystem/IFileSystemConnector.cs index c98d61878012..bcb274a23808 100644 --- a/dotnet/src/Skills/Skills.Document/FileSystem/IFileSystemConnector.cs +++ b/dotnet/src/Plugins/Plugins.Document/FileSystem/IFileSystemConnector.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.SemanticKernel.Skills.Document.FileSystem; +namespace Microsoft.SemanticKernel.Plugins.Document.FileSystem; /// /// Interface for filesystem connections. diff --git a/dotnet/src/Skills/Skills.Document/FileSystem/LocalFileSystemConnector.cs b/dotnet/src/Plugins/Plugins.Document/FileSystem/LocalFileSystemConnector.cs similarity index 98% rename from dotnet/src/Skills/Skills.Document/FileSystem/LocalFileSystemConnector.cs rename to dotnet/src/Plugins/Plugins.Document/FileSystem/LocalFileSystemConnector.cs index dbe6ac21289f..fd708eb24af1 100644 --- a/dotnet/src/Skills/Skills.Document/FileSystem/LocalFileSystemConnector.cs +++ b/dotnet/src/Plugins/Plugins.Document/FileSystem/LocalFileSystemConnector.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.SemanticKernel.Skills.Document.FileSystem; +namespace Microsoft.SemanticKernel.Plugins.Document.FileSystem; #pragma warning disable CA1031 // Exceptions are caught and returned in a task diff --git a/dotnet/src/Skills/Skills.Document/IDocumentConnector.cs b/dotnet/src/Plugins/Plugins.Document/IDocumentConnector.cs similarity index 94% rename from dotnet/src/Skills/Skills.Document/IDocumentConnector.cs rename to dotnet/src/Plugins/Plugins.Document/IDocumentConnector.cs index 893c7ae4e106..82934b86cecf 100644 --- a/dotnet/src/Skills/Skills.Document/IDocumentConnector.cs +++ b/dotnet/src/Plugins/Plugins.Document/IDocumentConnector.cs @@ -2,7 +2,7 @@ using System.IO; -namespace Microsoft.SemanticKernel.Skills.Document; +namespace Microsoft.SemanticKernel.Plugins.Document; /// /// Interface for document connections (e.g. Microsoft Word) diff --git a/dotnet/src/Skills/Skills.Document/OpenXml/Extensions/WordprocessingDocumentEx.cs b/dotnet/src/Plugins/Plugins.Document/OpenXml/Extensions/WordprocessingDocumentEx.cs similarity index 97% rename from dotnet/src/Skills/Skills.Document/OpenXml/Extensions/WordprocessingDocumentEx.cs rename to dotnet/src/Plugins/Plugins.Document/OpenXml/Extensions/WordprocessingDocumentEx.cs index 7afb1037b80c..81d8838532ea 100644 --- a/dotnet/src/Skills/Skills.Document/OpenXml/Extensions/WordprocessingDocumentEx.cs +++ b/dotnet/src/Plugins/Plugins.Document/OpenXml/Extensions/WordprocessingDocumentEx.cs @@ -5,7 +5,7 @@ using DocumentFormat.OpenXml.Packaging; using DocumentFormat.OpenXml.Wordprocessing; -namespace Microsoft.SemanticKernel.Skills.Document.OpenXml.Extensions; +namespace Microsoft.SemanticKernel.Plugins.Document.OpenXml.Extensions; /// /// Extension methods for DocumentFormat.OpenXml.Packaging.WordprocessingDocument diff --git a/dotnet/src/Skills/Skills.Document/OpenXml/WordDocumentConnector.cs b/dotnet/src/Plugins/Plugins.Document/OpenXml/WordDocumentConnector.cs similarity index 95% rename from dotnet/src/Skills/Skills.Document/OpenXml/WordDocumentConnector.cs rename to dotnet/src/Plugins/Plugins.Document/OpenXml/WordDocumentConnector.cs index 148ac5b1936e..8d745536d915 100644 --- a/dotnet/src/Skills/Skills.Document/OpenXml/WordDocumentConnector.cs +++ b/dotnet/src/Plugins/Plugins.Document/OpenXml/WordDocumentConnector.cs @@ -3,9 +3,9 @@ using System.IO; using DocumentFormat.OpenXml; using DocumentFormat.OpenXml.Packaging; -using Microsoft.SemanticKernel.Skills.Document.OpenXml.Extensions; +using Microsoft.SemanticKernel.Plugins.Document.OpenXml.Extensions; -namespace Microsoft.SemanticKernel.Skills.Document.OpenXml; +namespace Microsoft.SemanticKernel.Plugins.Document.OpenXml; /// /// Connector for Microsoft Word (.docx) files diff --git a/dotnet/src/Plugins/Plugins.Document/Plugins.Document.csproj b/dotnet/src/Plugins/Plugins.Document/Plugins.Document.csproj new file mode 100644 index 000000000000..fdc9d46c39be --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Document/Plugins.Document.csproj @@ -0,0 +1,26 @@ + + + + + Microsoft.SemanticKernel.Plugins.Document + $(AssemblyName) + netstandard2.0 + + + + + + + Semantic Kernel - Document Plugins + Semantic Kernel Document Plugins: Word processing, OpenXML, etc. + + + + + + + + + + + diff --git a/dotnet/src/Plugins/Plugins.Memory/Collections/MinHeap.cs b/dotnet/src/Plugins/Plugins.Memory/Collections/MinHeap.cs new file mode 100644 index 000000000000..3726c5b0a76e --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Memory/Collections/MinHeap.cs @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using Microsoft.SemanticKernel.Diagnostics; + +namespace Microsoft.SemanticKernel.Plugins.Memory.Collections; + +/// +/// Implements the classic 'heap' data structure. By default, the item with the lowest value is at the top of the heap. +/// +/// Data type. +internal sealed class MinHeap : IEnumerable where T : IComparable +{ + private const int DefaultCapacity = 7; + private const int MinCapacity = 0; + + private static readonly T[] s_emptyBuffer = Array.Empty(); + + private T[] _items; + private int _count; + + /// + /// Initializes a new instance of the class. + /// + /// Heap minimum value, which will be used as first item in collection. + /// Number of elements that collection can hold. + public MinHeap(T minValue, int capacity = DefaultCapacity) + { + if (capacity < MinCapacity) + { + Verify.ThrowArgumentOutOfRangeException(nameof(capacity), capacity, $"MinHeap capacity must be greater than {MinCapacity}."); + } + + this._items = new T[capacity + 1]; + // + // The 0'th item is a sentinel entry that simplifies the code + // + this._items[0] = minValue; + } + + /// + /// Initializes a new instance of the class. + /// + /// Heap minimum value, which will be used as first item in collection. + /// List of items to add. + public MinHeap(T minValue, IList items) + : this(minValue, items.Count) + { + this.Add(items); + } + + /// + /// Gets the current number of items in the collection. + /// + public int Count + { + get => this._count; + internal set + { + Debug.Assert(value <= this.Capacity); + this._count = value; + } + } + + /// + /// Gets the number of elements that collection can hold. + /// + public int Capacity => this._items.Length - 1; // 0'th item is always a sentinel to simplify code + + /// + /// Gets the element at the specified index. + /// + public T this[int index] + { + get => this._items[index + 1]; + internal set { this._items[index + 1] = value; } + } + + /// + /// Gets first item in collection. + /// + public T Top => this._items[1]; + + /// + /// Gets the boolean flag which indicates if collection is empty. + /// + public bool IsEmpty => (this._count == 0); + + /// + /// Sets collection item count to zero. + /// + public void Clear() + { + this._count = 0; + } + + /// + /// Sets collection item count to zero and removes all items in collection. + /// + public void Erase() + { + Array.Clear(this._items, 1, this._count); + this._count = 0; + } + + /// + /// Removes all items in collection and returns them. + /// + public T[] DetachBuffer() + { + T[] buf = this._items; + this._items = s_emptyBuffer; + this._count = 0; + return buf; + } + + /// + /// Adds new item to collection. + /// + /// Item to add. + public void Add(T item) + { + // + // the 0'th item is always a sentinel and not included in this._count. + // The length of the buffer is always this._count + 1 + // + this._count++; + this.EnsureCapacity(); + this._items[this._count] = item; + this.UpHeap(this._count); + } + + /// + /// Adds new items to collection. + /// + /// Items to add. + public void Add(IEnumerable items) + { + foreach (T item in items) + { + this.Add(item); + } + } + + /// + /// Adds new items starting from specified index. + /// + /// Items to add. + /// Starting point of items to add. + public void Add(IList items, int startAt = 0) + { + Verify.NotNull(items); + + int newItemCount = items.Count; + if (startAt >= newItemCount) + { + Verify.ThrowArgumentOutOfRangeException(nameof(startAt), startAt, $"{nameof(startAt)} value must be less than {nameof(items)}.{nameof(items.Count)}."); + } + + this.EnsureCapacity(this._count + (newItemCount - startAt)); + for (int i = startAt; i < newItemCount; ++i) + { + // + // the 0'th item is always a sentinel and not included in this._count. + // The length of the buffer is always this._count + 1 + // + this._count++; + this._items[this._count] = items[i]; + this.UpHeap(this._count); + } + } + + /// + /// Removes first item in collection and returns it. + /// + public T RemoveTop() + { + if (this._count == 0) + { + throw new InvalidOperationException("MinHeap is empty."); + } + + T item = this._items[1]; + this._items[1] = this._items[this._count--]; + this.DownHeap(1); + return item; + } + + /// + /// Removes all items in collection and returns them. + /// + public IEnumerable RemoveAll() + { + while (this._count > 0) + { + yield return this.RemoveTop(); + } + } + + /// + /// Resizes collection to specified capacity. + /// + /// Number of elements that collection can hold. + public void EnsureCapacity(int capacity) + { + if (capacity < MinCapacity) + { + Verify.ThrowArgumentOutOfRangeException(nameof(capacity), capacity, $"MinHeap capacity must be greater than {MinCapacity}."); + } + + // 0th item is always a sentinel + capacity++; + if (capacity > this._items.Length) + { + Array.Resize(ref this._items, capacity); + } + } + + /// + /// Doubles collection capacity. + /// + public void EnsureCapacity() + { + if (this._count == this._items.Length) + { + Array.Resize(ref this._items, (this._count * 2) + 1); + } + } + + private void UpHeap(int startAt) + { + int i = startAt; + T[] items = this._items; + T item = items[i]; + int parent = i >> 1; //i / 2; + + while (parent > 0 && items[parent].CompareTo(item) > 0) + { + // Child > parent. Exchange with parent, thus moving the child up the queue + items[i] = items[parent]; + i = parent; + parent = i >> 1; //i / 2; + } + + items[i] = item; + } + + private void DownHeap(int startAt) + { + int i = startAt; + int count = this._count; + int maxParent = count >> 1; + T[] items = this._items; + T item = items[i]; + + while (i <= maxParent) + { + int child = i + i; + // + // Exchange the item with the smaller of its two children - if one is smaller, i.e. + // + // First, find the smaller child + // + if (child < count && items[child].CompareTo(items[child + 1]) > 0) + { + child++; + } + + if (item.CompareTo(items[child]) <= 0) + { + // Heap condition is satisfied. Parent <= both its children + break; + } + + // Else, swap parent with the smallest child + items[i] = items[child]; + i = child; + } + + items[i] = item; + } + + /// + /// Returns an enumerator that iterates through the collection. + /// + public IEnumerator GetEnumerator() + { + // The 0'th item in the queue is a sentinel. i is 1 based. + for (int i = 1; i <= this._count; ++i) + { + yield return this._items[i]; + } + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + + /// + /// Heap Sort in-place. + /// This is destructive. Once you do this, the heap order is lost. + /// The advantage on in-place is that we don't need to do another allocation + /// + public void SortDescending() + { + int count = this._count; + int i = count; // remember that the 0'th item in the queue is always a sentinel. So i is 1 based + + while (this._count > 0) + { + // + // this dequeues the item with the current LOWEST relevancy + // We take that and place it at the 'back' of the array - thus inverting it + // + T item = this.RemoveTop(); + this._items[i--] = item; + } + + this._count = count; + } + + /// + /// Restores heap order + /// + internal void Restore() + { + this.Clear(); + this.Add(this._items, 1); + } + + internal void Sort(IComparer comparer) + { + Array.Sort(this._items, 1, this._count, comparer); + } +} diff --git a/dotnet/src/Plugins/Plugins.Memory/Collections/ScoredValue.cs b/dotnet/src/Plugins/Plugins.Memory/Collections/ScoredValue.cs new file mode 100644 index 000000000000..ebdb360b69f9 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Memory/Collections/ScoredValue.cs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; + +namespace Microsoft.SemanticKernel.Plugins.Memory.Collections; + +/// +/// Structure for storing data which can be scored. +/// +/// Data type. +internal readonly struct ScoredValue : IComparable>, IEquatable> +{ + /// + /// Initializes a new instance of the struct. + /// + /// The item to be scored. + /// The score of the item. + public ScoredValue(T item, double score) + { + this.Value = item; + this.Score = score; + } + + /// + /// Gets the value of the scored item. + /// + public T Value { get; } + /// + /// Gets the score of the item. + /// + public double Score { get; } + + /// + /// Compares the current instance with another instance of . + /// + /// The other instance of to compare with. + /// A value indicating the relative order of the instances. + public int CompareTo(ScoredValue other) + { + return this.Score.CompareTo(other.Score); + } + + /// + /// Returns a string representation of the current instance. + /// + /// A string representation of the current instance. + public override string ToString() + { + return $"{this.Score}, {this.Value}"; + } + + /// + /// Converts the score of the current instance to a double. + /// + /// The current instance of . + public static explicit operator double(ScoredValue src) + { + return src.Score; + } + + /// + /// Converts the value of the current instance to the specified type. + /// + /// The current instance of . + public static explicit operator T(ScoredValue src) + { + return src.Value; + } + + /// + /// Converts a to a . + /// + /// The to convert. + public static implicit operator ScoredValue(KeyValuePair src) + { + return new ScoredValue(src.Key, src.Value); + } + + /// + public override bool Equals(object obj) + { + return (obj is ScoredValue other) && this.Equals(other); + } + + /// + /// Determines whether the current instance is equal to another instance of . + /// + /// The other instance of to compare with. + /// True if the instances are equal, false otherwise. + public bool Equals(ScoredValue other) + { + return EqualityComparer.Default.Equals(this.Value, other.Value) && + this.Score.Equals(other.Score); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(this.Value, this.Score); + } + + /// + /// Determines whether two instances of are equal. + /// + public static bool operator ==(ScoredValue left, ScoredValue right) + { + return left.Equals(right); + } + + /// + /// Determines whether two instances of are not equal. + /// + public static bool operator !=(ScoredValue left, ScoredValue right) + { + return !(left == right); + } + + /// + /// Determines whether the left instance of is less than the right instance. + /// + public static bool operator <(ScoredValue left, ScoredValue right) + { + return left.CompareTo(right) < 0; + } + + /// + /// Determines whether the left instance of is less than or equal to the right instance. + /// + public static bool operator <=(ScoredValue left, ScoredValue right) + { + return left.CompareTo(right) <= 0; + } + + /// + /// Determines whether the left instance of is greater than the right instance. + /// + public static bool operator >(ScoredValue left, ScoredValue right) + { + return left.CompareTo(right) > 0; + } + + /// + /// Determines whether the left instance of is greater than or equal to the right instance. + /// + public static bool operator >=(ScoredValue left, ScoredValue right) + { + return left.CompareTo(right) >= 0; + } + + /// + /// Returns the minimum possible value of a . + /// + internal static ScoredValue Min() + { + return new ScoredValue(default!, double.MinValue); + } +} diff --git a/dotnet/src/Plugins/Plugins.Memory/Collections/TopNCollection.cs b/dotnet/src/Plugins/Plugins.Memory/Collections/TopNCollection.cs new file mode 100644 index 000000000000..9a8b874cd2c8 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Memory/Collections/TopNCollection.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections; +using System.Collections.Generic; + +namespace Microsoft.SemanticKernel.Plugins.Memory.Collections; + +/// +/// A collector for Top N matches. Keeps only the best N matches by Score. +/// Automatically flushes out any not in the top N. +/// By default, items are not sorted by score until you call . +/// +internal sealed class TopNCollection : IEnumerable> +{ + private readonly MinHeap> _heap; + private bool _sorted = false; + + /// + /// Initializes a new instance of the class. + /// + /// The maximum number of items to keep in the collection. + public TopNCollection(int maxItems) + { + this.MaxItems = maxItems; + this._heap = new MinHeap>(ScoredValue.Min(), maxItems); + } + + /// + /// Gets the maximum number of items allowed in the collection. + /// + public int MaxItems { get; } + + /// + /// Gets the current number of items in the collection. + /// + public int Count => this._heap.Count; + + internal ScoredValue this[int i] => this._heap[i]; + internal ScoredValue Top => this._heap.Top; + + /// + /// Resets the collection, allowing it to be reused. + /// + public void Reset() + { + this._heap.Clear(); + } + + /// + /// Adds a single scored value to the collection. + /// + /// The scored value to add. + public void Add(ScoredValue value) + { + if (this._sorted) + { + this._heap.Restore(); + this._sorted = false; + } + + if (this._heap.Count == this.MaxItems) + { + // Queue is full. We will need to dequeue the item with lowest weight + if (value.Score <= this.Top.Score) + { + // This score is lower than the lowest score on the queue right now. Ignore it + return; + } + + this._heap.RemoveTop(); + } + + this._heap.Add(value); + } + + /// + /// Adds a value with a specified score to the collection. + /// + /// The value to add. + /// The score associated with the value. + public void Add(T value, double score) + { + this.Add(new ScoredValue(value, score)); + } + + /// + /// Sorts the collection in descending order by score. + /// + public void SortByScore() + { + if (!this._sorted && this._heap.Count > 0) + { + this._heap.SortDescending(); + this._sorted = true; + } + } + + /// + /// Returns a list containing the scored values in the collection. + /// + /// A list of scored values. + public IList> ToList() + { + var list = new List>(this.Count); + for (int i = 0, count = this.Count; i < count; ++i) + { + list.Add(this[i]); + } + + return list; + } + + /// + /// Returns an enumerator that iterates through the collection. + /// + /// An enumerator for the collection. + public IEnumerator> GetEnumerator() + { + return this._heap.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return this._heap.GetEnumerator(); + } +} diff --git a/dotnet/src/Plugins/Plugins.Memory/MemoryBuilder.cs b/dotnet/src/Plugins/Plugins.Memory/MemoryBuilder.cs new file mode 100644 index 000000000000..646f4232cacb --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Memory/MemoryBuilder.cs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.Memory; + +namespace Microsoft.SemanticKernel.Plugins.Memory; + +/// +/// A builder for Memory plugin. +/// +public sealed class MemoryBuilder +{ + private Func? _memoryStoreFactory = null; + private Func? _embeddingGenerationFactory = null; + private IDelegatingHandlerFactory _httpHandlerFactory = NullHttpHandlerFactory.Instance; + private ILoggerFactory _loggerFactory = NullLoggerFactory.Instance; + + /// + /// Build a new instance of using the settings passed so far. + /// + /// Instance of . + public ISemanticTextMemory Build() + { + var memoryStore = this._memoryStoreFactory?.Invoke() ?? + throw new SKException($"{nameof(IMemoryStore)} dependency was not provided. Use {nameof(WithMemoryStore)} method."); + + var embeddingGeneration = this._embeddingGenerationFactory?.Invoke() ?? + throw new SKException($"{nameof(ITextEmbeddingGeneration)} dependency was not provided. Use {nameof(WithTextEmbeddingGeneration)} method."); + + return new SemanticTextMemory(memoryStore, embeddingGeneration); + } + + /// + /// Add a logger factory. + /// + /// The to use for logging. If null, no logging will be performed. + /// Updated Memory builder including the logger factory. + public MemoryBuilder WithLoggerFactory(ILoggerFactory loggerFactory) + { + Verify.NotNull(loggerFactory); + this._loggerFactory = loggerFactory; + return this; + } + + /// + /// Add a http handler factory. + /// + /// Http handler factory to add. + /// Updated Memory builder including the http handler factory. + public MemoryBuilder WithHttpHandlerFactory(IDelegatingHandlerFactory httpHandlerFactory) + { + Verify.NotNull(httpHandlerFactory); + this._httpHandlerFactory = httpHandlerFactory; + return this; + } + + /// + /// Add memory store. + /// + /// Store to add. + /// Updated Memory builder including the memory store. + public MemoryBuilder WithMemoryStore(IMemoryStore store) + { + Verify.NotNull(store); + this._memoryStoreFactory = () => store; + return this; + } + + /// + /// Add memory store factory. + /// + /// The store factory. + /// Updated Memory builder including the memory store. + public MemoryBuilder WithMemoryStore(Func factory) where TStore : IMemoryStore + { + Verify.NotNull(factory); + this._memoryStoreFactory = () => factory(this._loggerFactory); + return this; + } + + /// + /// Add memory store factory. + /// + /// The store factory. + /// Updated Memory builder including the memory store. + public MemoryBuilder WithMemoryStore(Func factory) where TStore : IMemoryStore + { + Verify.NotNull(factory); + this._memoryStoreFactory = () => factory(this._loggerFactory, this._httpHandlerFactory); + return this; + } + + /// + /// Add text embedding generation. + /// + /// The text embedding generation. + /// Updated Memory builder including the text embedding generation. + public MemoryBuilder WithTextEmbeddingGeneration(ITextEmbeddingGeneration textEmbeddingGeneration) + { + Verify.NotNull(textEmbeddingGeneration); + this._embeddingGenerationFactory = () => textEmbeddingGeneration; + return this; + } + + /// + /// Add text embedding generation. + /// + /// The text embedding generation factory. + /// Updated Memory builder including the text embedding generation. + public MemoryBuilder WithTextEmbeddingGeneration( + Func factory) where TEmbeddingGeneration : ITextEmbeddingGeneration + { + Verify.NotNull(factory); + this._embeddingGenerationFactory = () => factory(this._loggerFactory, this._httpHandlerFactory); + return this; + } +} diff --git a/dotnet/src/Plugins/Plugins.Memory/Plugins.Memory.csproj b/dotnet/src/Plugins/Plugins.Memory/Plugins.Memory.csproj new file mode 100644 index 000000000000..0634d2b185bf --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Memory/Plugins.Memory.csproj @@ -0,0 +1,29 @@ + + + + + Microsoft.SemanticKernel.Plugins.Memory + $(AssemblyName) + netstandard2.0 + + + + + + + + Semantic Kernel - Memory Plugin + Semantic Kernel Memory Plugin + + + + + + + + + + + + + diff --git a/dotnet/src/Plugins/Plugins.Memory/TextMemoryPlugin.cs b/dotnet/src/Plugins/Plugins.Memory/TextMemoryPlugin.cs new file mode 100644 index 000000000000..c7e9a6ea229a --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Memory/TextMemoryPlugin.cs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Memory; + +namespace Microsoft.SemanticKernel.Plugins.Memory; + +/// +/// TextMemoryPlugin provides a plugin to save or recall information from the long or short term memory. +/// +/// +/// Usage: kernel.ImportFunctions(new TextMemoryPlugin(), "memory"); +/// Examples: +/// SKContext.Variables["input"] = "what is the capital of France?" +/// {{memory.recall $input }} => "Paris" +/// +public sealed class TextMemoryPlugin +{ + /// + /// Name of the context variable used to specify which memory collection to use. + /// + public const string CollectionParam = "collection"; + + /// + /// Name of the context variable used to specify memory search relevance score. + /// + public const string RelevanceParam = "relevance"; + + /// + /// Name of the context variable used to specify a unique key associated with stored information. + /// + public const string KeyParam = "key"; + + /// + /// Name of the context variable used to specify the number of memories to recall + /// + public const string LimitParam = "limit"; + + private const string DefaultCollection = "generic"; + private const double DefaultRelevance = 0.0; + private const int DefaultLimit = 1; + + private readonly ISemanticTextMemory _memory; + + /// + /// Creates a new instance of the TextMemoryPlugin + /// + public TextMemoryPlugin(ISemanticTextMemory memory) + { + this._memory = memory; + } + + /// + /// Key-based lookup for a specific memory + /// + /// Memories collection associated with the memory to retrieve + /// The key associated with the memory to retrieve. + /// The to use for logging. If null, no logging will be performed. + /// The to monitor for cancellation requests. The default is . + /// + /// SKContext.Variables[TextMemoryPlugin.KeyParam] = "countryInfo1" + /// {{memory.retrieve }} + /// + [SKFunction, Description("Key-based lookup for a specific memory")] + public async Task RetrieveAsync( + [SKName(CollectionParam), Description("Memories collection associated with the memory to retrieve"), DefaultValue(DefaultCollection)] string? collection, + [SKName(KeyParam), Description("The key associated with the memory to retrieve")] string key, + ILoggerFactory? loggerFactory, + CancellationToken cancellationToken = default) + { + Verify.NotNullOrWhiteSpace(collection); + Verify.NotNullOrWhiteSpace(key); + + loggerFactory?.CreateLogger(typeof(TextMemoryPlugin)).LogDebug("Recalling memory with key '{0}' from collection '{1}'", key, collection); + + var memory = await this._memory.GetAsync(collection, key, cancellationToken: cancellationToken).ConfigureAwait(false); + + return memory?.Metadata.Text ?? string.Empty; + } + + /// + /// Semantic search and return up to N memories related to the input text + /// + /// + /// SKContext.Variables["input"] = "what is the capital of France?" + /// {{memory.recall $input }} => "Paris" + /// + /// The input text to find related memories for. + /// Memories collection to search. + /// The relevance score, from 0.0 to 1.0, where 1.0 means perfect match. + /// The maximum number of relevant memories to recall. + /// The to use for logging. If null, no logging will be performed. + /// The to monitor for cancellation requests. The default is . + [SKFunction, Description("Semantic search and return up to N memories related to the input text")] + public async Task RecallAsync( + [Description("The input text to find related memories for")] string input, + [SKName(CollectionParam), Description("Memories collection to search"), DefaultValue(DefaultCollection)] string collection, + [SKName(RelevanceParam), Description("The relevance score, from 0.0 to 1.0, where 1.0 means perfect match"), DefaultValue(DefaultRelevance)] double? relevance, + [SKName(LimitParam), Description("The maximum number of relevant memories to recall"), DefaultValue(DefaultLimit)] int? limit, + ILoggerFactory? loggerFactory, + CancellationToken cancellationToken = default) + { + Verify.NotNullOrWhiteSpace(collection); + relevance ??= DefaultRelevance; + limit ??= DefaultLimit; + + ILogger? logger = loggerFactory?.CreateLogger(typeof(TextMemoryPlugin)); + + logger?.LogDebug("Searching memories in collection '{0}', relevance '{1}'", collection, relevance); + + // Search memory + List memories = await this._memory + .SearchAsync(collection, input, limit.Value, relevance.Value, cancellationToken: cancellationToken) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + if (memories.Count == 0) + { + logger?.LogWarning("Memories not found in collection: {0}", collection); + return string.Empty; + } + + logger?.LogTrace("Done looking for memories in collection '{0}')", collection); + return limit == 1 ? memories[0].Metadata.Text : JsonSerializer.Serialize(memories.Select(x => x.Metadata.Text)); + } + + /// + /// Save information to semantic memory + /// + /// + /// SKContext.Variables["input"] = "the capital of France is Paris" + /// SKContext.Variables[TextMemoryPlugin.KeyParam] = "countryInfo1" + /// {{memory.save $input }} + /// + /// The information to save + /// Memories collection associated with the information to save + /// The key associated with the information to save + /// The to use for logging. If null, no logging will be performed. + /// The to monitor for cancellation requests. The default is . + [SKFunction, Description("Save information to semantic memory")] + public async Task SaveAsync( + [Description("The information to save")] string input, + [SKName(CollectionParam), Description("Memories collection associated with the information to save"), DefaultValue(DefaultCollection)] string collection, + [SKName(KeyParam), Description("The key associated with the information to save")] string key, + ILoggerFactory? loggerFactory, + CancellationToken cancellationToken = default) + { + Verify.NotNullOrWhiteSpace(collection); + Verify.NotNullOrWhiteSpace(key); + + loggerFactory?.CreateLogger(typeof(TextMemoryPlugin)).LogDebug("Saving memory to collection '{0}'", collection); + + await this._memory.SaveInformationAsync(collection, text: input, id: key, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + /// + /// Remove specific memory + /// + /// + /// SKContext.Variables[TextMemoryPlugin.KeyParam] = "countryInfo1" + /// {{memory.remove }} + /// + /// Memories collection associated with the information to save + /// The key associated with the information to save + /// The to use for logging. If null, no logging will be performed. + /// The to monitor for cancellation requests. The default is . + [SKFunction, Description("Remove specific memory")] + public async Task RemoveAsync( + [SKName(CollectionParam), Description("Memories collection associated with the information to save"), DefaultValue(DefaultCollection)] string collection, + [SKName(KeyParam), Description("The key associated with the information to save")] string key, + ILoggerFactory? loggerFactory, + CancellationToken cancellationToken = default) + { + Verify.NotNullOrWhiteSpace(collection); + Verify.NotNullOrWhiteSpace(key); + + loggerFactory?.CreateLogger(typeof(TextMemoryPlugin)).LogDebug("Removing memory from collection '{0}'", collection); + + await this._memory.RemoveAsync(collection, key, cancellationToken: cancellationToken).ConfigureAwait(false); + } +} diff --git a/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs b/dotnet/src/Plugins/Plugins.Memory/VolatileMemoryStore.cs similarity index 75% rename from dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs rename to dotnet/src/Plugins/Plugins.Memory/VolatileMemoryStore.cs index 3bd8db0d8640..0cb4ddba93b9 100644 --- a/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs +++ b/dotnet/src/Plugins/Plugins.Memory/VolatileMemoryStore.cs @@ -1,18 +1,19 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Numerics.Tensors; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Memory.Collections; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Plugins.Memory.Collections; -namespace Microsoft.SemanticKernel.Memory; +namespace Microsoft.SemanticKernel.Plugins.Memory; /// /// A simple volatile memory embeddings store. @@ -22,11 +23,9 @@ public class VolatileMemoryStore : IMemoryStore /// public Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default) { - if (!this._store.TryAdd(collectionName, new ConcurrentDictionary())) - { - return Task.FromException(new MemoryException(MemoryException.ErrorCodes.FailedToCreateCollection, $"Could not create collection {collectionName}")); - } + Verify.NotNullOrWhiteSpace(collectionName); + this._store.TryAdd(collectionName, new ConcurrentDictionary()); return Task.CompletedTask; } @@ -47,7 +46,7 @@ public Task DeleteCollectionAsync(string collectionName, CancellationToken cance { if (!this._store.TryRemove(collectionName, out _)) { - return Task.FromException(new MemoryException(MemoryException.ErrorCodes.FailedToDeleteCollection, $"Could not delete collection {collectionName}")); + return Task.FromException(new SKException($"Could not delete collection {collectionName}")); } return Task.CompletedTask; @@ -66,8 +65,7 @@ public Task UpsertAsync(string collectionName, MemoryRecord record, Canc } else { - return Task.FromException(new MemoryException(MemoryException.ErrorCodes.AttemptedToAccessNonexistentCollection, - $"Attempted to access a memory collection that does not exist: {collectionName}")); + return Task.FromException(new SKException($"Attempted to access a memory collection that does not exist: {collectionName}")); } return Task.FromResult(record.Key); @@ -134,9 +132,19 @@ public Task RemoveBatchAsync(string collectionName, IEnumerable keys, Ca return Task.WhenAll(keys.Select(k => this.RemoveAsync(collectionName, k, cancellationToken))); } + /// + /// Retrieves the nearest matches to the given embedding in the specified collection. + /// + /// The name of the collection to search. + /// The embedding to find the nearest matches for. + /// The maximum number of matches to return. + /// The minimum relevance score for a match to be included in the results. + /// Whether to include the embeddings in the returned memory records. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of memory records and their relevance scores. public IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0.0, bool withEmbeddings = false, @@ -164,12 +172,10 @@ public Task RemoveBatchAsync(string collectionName, IEnumerable keys, Ca { if (record != null) { - double similarity = embedding - .AsReadOnlySpan() - .CosineSimilarity(record.Embedding.AsReadOnlySpan()); + double similarity = TensorPrimitives.CosineSimilarity(embedding.Span, record.Embedding.Span); if (similarity >= minRelevanceScore) { - var entry = withEmbeddings ? record : MemoryRecord.FromMetadata(record.Metadata, Embedding.Empty, record.Key, record.Timestamp); + var entry = withEmbeddings ? record : MemoryRecord.FromMetadata(record.Metadata, ReadOnlyMemory.Empty, record.Key, record.Timestamp); embeddings.Add(new(entry, similarity)); } } @@ -177,13 +183,13 @@ public Task RemoveBatchAsync(string collectionName, IEnumerable keys, Ca embeddings.SortByScore(); - return embeddings.Select(x => (x.Value, x.Score.Value)).ToAsyncEnumerable(); + return embeddings.Select(x => (x.Value, x.Score)).ToAsyncEnumerable(); } /// public async Task<(MemoryRecord, double)?> GetNearestMatchAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, double minRelevanceScore = 0.0, bool withEmbedding = false, CancellationToken cancellationToken = default) @@ -198,7 +204,13 @@ public Task RemoveBatchAsync(string collectionName, IEnumerable keys, Ca } #region protected ================================================================================ - + /// + /// Tries to get the collection with the specified name. + /// + /// The name of the collection to get. + /// The retrieved collection, if found. + /// Whether to create the collection if it does not exist. + /// True if the collection was found or created, false otherwise. protected bool TryGetCollection( string name, [NotNullWhen(true)] out ConcurrentDictionary +/// Plugin for calendar operations. +/// +public sealed class CalendarPlugin +{ + /// + /// parameter names. + /// + public static class Parameters + { + /// + /// Event start as DateTimeOffset. + /// + public const string Start = "start"; + + /// + /// Event end as DateTimeOffset. + /// + public const string End = "end"; + + /// + /// Event's location. + /// + public const string Location = "location"; + + /// + /// Event's content. + /// + public const string Content = "content"; + + /// + /// Event's attendees, separated by ',' or ';'. + /// + public const string Attendees = "attendees"; + + /// + /// The name of the top parameter used to limit the number of results returned in the response. + /// + public const string MaxResults = "maxResults"; + + /// + /// The name of the skip parameter used to skip a certain number of results in the response. + /// + public const string Skip = "skip"; + } + + private readonly ICalendarConnector _connector; + private readonly ILogger _logger; + private static readonly JsonSerializerOptions s_options = new() + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + /// + /// Initializes a new instance of the class. + /// + /// Calendar connector. + /// The to use for logging. If null, no logging will be performed. + public CalendarPlugin(ICalendarConnector connector, ILoggerFactory? loggerFactory = null) + { + Ensure.NotNull(connector, nameof(connector)); + + this._connector = connector; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(CalendarPlugin)) : NullLogger.Instance; + } + + /// + /// Add an event to my calendar using as the subject. + /// + [SKFunction, Description("Add an event to my calendar.")] + public async Task AddEventAsync( + [Description("Event subject"), SKName("input")] string subject, + [Description("Event start date/time as DateTimeOffset")] DateTimeOffset start, + [Description("Event end date/time as DateTimeOffset")] DateTimeOffset end, + [Description("Event location (optional)")] string? location = null, + [Description("Event content/body (optional)")] string? content = null, + [Description("Event attendees, separated by ',' or ';'.")] string? attendees = null) + { + if (string.IsNullOrWhiteSpace(subject)) + { + throw new ArgumentException($"{nameof(subject)} variable was null or whitespace", nameof(subject)); + } + + CalendarEvent calendarEvent = new() + { + Subject = subject, + Start = start, + End = end, + Location = location, + Content = content, + Attendees = attendees is not null ? attendees.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries) : Enumerable.Empty(), + }; + + // Sensitive data, logging as trace, disabled by default + this._logger.LogTrace("Adding calendar event '{0}'", calendarEvent.Subject); + await this._connector.AddEventAsync(calendarEvent).ConfigureAwait(false); + } + + /// + /// Get calendar events with specified optional clauses used to query for messages. + /// + [SKFunction, Description("Get calendar events.")] + public async Task GetCalendarEventsAsync( + [Description("Optional limit of the number of events to retrieve.")] int? maxResults = 10, + [Description("Optional number of events to skip before retrieving results.")] int? skip = 0, + CancellationToken cancellationToken = default) + { + this._logger.LogDebug("Getting calendar events with query options top: '{0}', skip:'{1}'.", maxResults, skip); + + const string SelectString = "start,subject,organizer,location"; + + IEnumerable events = await this._connector.GetEventsAsync( + top: maxResults, + skip: skip, + select: SelectString, + cancellationToken + ).ConfigureAwait(false); + + return JsonSerializer.Serialize(value: events, options: s_options); + } +} diff --git a/dotnet/src/Plugins/Plugins.MsGraph/CloudDrivePlugin.cs b/dotnet/src/Plugins/Plugins.MsGraph/CloudDrivePlugin.cs new file mode 100644 index 000000000000..f38aebf8519e --- /dev/null +++ b/dotnet/src/Plugins/Plugins.MsGraph/CloudDrivePlugin.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Plugins.MsGraph.Diagnostics; + +namespace Microsoft.SemanticKernel.Plugins.MsGraph; + +/// +/// Cloud drive plugin (e.g. OneDrive). +/// +public sealed class CloudDrivePlugin +{ + /// + /// parameter names. + /// + public static class Parameters + { + /// + /// Document file path. + /// + public const string DestinationPath = "destinationPath"; + } + + private readonly ICloudDriveConnector _connector; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The cloud drive connector. + /// The to use for logging. If null, no logging will be performed. + public CloudDrivePlugin(ICloudDriveConnector connector, ILoggerFactory? loggerFactory = null) + { + Ensure.NotNull(connector, nameof(connector)); + + this._connector = connector; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(CloudDrivePlugin)) : NullLogger.Instance; + } + + /// + /// Get the contents of a file stored in a cloud drive. + /// + /// The path to the file. + /// A cancellation token to observe while waiting for the task to complete. + /// A string containing the file content. + [SKFunction, Description("Get the contents of a file in a cloud drive.")] + public async Task GetFileContentAsync( + [Description("Path to file")] string filePath, + CancellationToken cancellationToken = default) + { + this._logger.LogDebug("Getting file content for '{0}'", filePath); + Stream fileContentStream = await this._connector.GetFileContentStreamAsync(filePath, cancellationToken).ConfigureAwait(false); + + using StreamReader sr = new(fileContentStream); + string content = await sr.ReadToEndAsync().ConfigureAwait(false); + + return content; + } + + /// + /// Upload a small file to OneDrive (less than 4MB). + /// + /// The path to the file. + /// The remote path to store the file. + /// A cancellation token to observe while waiting for the task to complete. + [SKFunction, Description("Upload a small file to OneDrive (less than 4MB).")] + public async Task UploadFileAsync( + [Description("Path to file")] string filePath, + [Description("Remote path to store the file")] string destinationPath, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(destinationPath)) + { + throw new ArgumentException("Variable was null or whitespace", nameof(destinationPath)); + } + + this._logger.LogDebug("Uploading file '{0}'", filePath); + + // TODO Add support for large file uploads (i.e. upload sessions) + await this._connector.UploadSmallFileAsync(filePath, destinationPath, cancellationToken).ConfigureAwait(false); + } + + /// + /// Create a sharable link to a file stored in a cloud drive. + /// + /// The path to the file. + /// A cancellation token to observe while waiting for the task to complete. + /// A string containing the sharable link. + [SKFunction, Description("Create a sharable link to a file stored in a cloud drive.")] + public async Task CreateLinkAsync( + [Description("Path to file")] string filePath, + CancellationToken cancellationToken = default) + { + this._logger.LogDebug("Creating link for '{0}'", filePath); + const string Type = "view"; // TODO expose this as an SK variable + const string Scope = "anonymous"; // TODO expose this as an SK variable + + return await this._connector.CreateShareLinkAsync(filePath, Type, Scope, cancellationToken).ConfigureAwait(false); + } +} diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/Client/MsGraphClientLoggingHandler.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/Client/MsGraphClientLoggingHandler.cs similarity index 97% rename from dotnet/src/Skills/Skills.MsGraph/Connectors/Client/MsGraphClientLoggingHandler.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Connectors/Client/MsGraphClientLoggingHandler.cs index 5f74814ce7dc..88196b4a77ff 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/Client/MsGraphClientLoggingHandler.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/Client/MsGraphClientLoggingHandler.cs @@ -9,7 +9,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Client; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.Client; /// /// An HTTPClient logging handler for ensuring diagnostic headers for Graph API calls are available. diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/Client/MsGraphConfiguration.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/Client/MsGraphConfiguration.cs similarity index 95% rename from dotnet/src/Skills/Skills.MsGraph/Connectors/Client/MsGraphConfiguration.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Connectors/Client/MsGraphConfiguration.cs index 062bdf85f214..6a8e3e593b2a 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/Client/MsGraphConfiguration.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/Client/MsGraphConfiguration.cs @@ -5,7 +5,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Client; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.Client; /// /// Graph API connector configuration model. diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/CredentialManagers/LocalUserMSALCredentialManager.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/CredentialManagers/LocalUserMSALCredentialManager.cs similarity index 80% rename from dotnet/src/Skills/Skills.MsGraph/Connectors/CredentialManagers/LocalUserMSALCredentialManager.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Connectors/CredentialManagers/LocalUserMSALCredentialManager.cs index 932c08199269..2031ee521dc2 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/CredentialManagers/LocalUserMSALCredentialManager.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/CredentialManagers/LocalUserMSALCredentialManager.cs @@ -7,9 +7,9 @@ using System.Threading.Tasks; using Microsoft.Identity.Client; using Microsoft.Identity.Client.Extensions.Msal; -using Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Diagnostics; +using Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.Diagnostics; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors.CredentialManagers; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.CredentialManagers; /// /// Manages acquiring and caching MSAL credentials locally. @@ -46,6 +46,10 @@ private LocalUserMSALCredentialManager(StorageCreationProperties storage, MsalCa this._cacheHelper.VerifyPersistence(); } + /// + /// Creates a new instance of the class. + /// + /// A task that represents the asynchronous operation. The task result contains the created . public static async Task CreateAsync() { // Initialize persistent storage for the token cache @@ -58,7 +62,7 @@ public static async Task CreateAsync() .WithLinuxKeyring( schemaName: CacheSchemaName, collection: MsalCacheHelper.LinuxKeyRingDefaultCollection, - secretLabel: "MSAL token cache for Semantic Kernel skills.", + secretLabel: "MSAL token cache for Semantic Kernel plugins.", attribute1: new KeyValuePair("Version", "1"), attribute2: new KeyValuePair("Product", "SemanticKernel")) .Build(); @@ -71,6 +75,11 @@ public static async Task CreateAsync() /// /// Acquires an access token for the specified client ID, tenant ID, scopes, and redirect URI. /// + /// The client ID of the application. + /// The tenant ID of the application. + /// The scopes for which the access token is requested. + /// The redirect URI of the application. + /// A task that represents the asynchronous operation. The task result contains the access token. public async Task GetTokenAsync(string clientId, string tenantId, string[] scopes, Uri redirectUri) { Ensure.NotNullOrWhitespace(clientId, nameof(clientId)); @@ -79,7 +88,7 @@ public async Task GetTokenAsync(string clientId, string tenantId, string Ensure.NotNull(scopes, nameof(scopes)); IPublicClientApplication app = this._publicClientApplications.GetOrAdd( - key: this.PublicClientApplicationsKey(clientId, tenantId), + key: PublicClientApplicationsKey(clientId, tenantId), valueFactory: _ => { IPublicClientApplication newPublicApp = PublicClientApplicationBuilder.Create(clientId) @@ -113,5 +122,5 @@ public async Task GetTokenAsync(string clientId, string tenantId, string /// /// Returns a key for the public client application dictionary. /// - private string PublicClientApplicationsKey(string clientId, string tenantId) => $"{clientId}_{tenantId}"; + private static string PublicClientApplicationsKey(string clientId, string tenantId) => $"{clientId}_{tenantId}"; } diff --git a/dotnet/src/Plugins/Plugins.MsGraph/Connectors/Diagnostics/Ensure.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/Diagnostics/Ensure.cs new file mode 100644 index 000000000000..bab7c077571c --- /dev/null +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/Diagnostics/Ensure.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.Diagnostics; + +/// +/// Internal data validation class. +/// +internal static class Ensure +{ + /// + /// Ensures the given parameter is not null or does not contain only white-space characters. + /// Throws an if the parameter is invalid. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void NotNullOrWhitespace([NotNull] string parameter, [NotNull] string parameterName) + { + if (string.IsNullOrWhiteSpace(parameter)) + { + throw new ArgumentException($"Parameter '{parameterName}' cannot be null or whitespace.", parameterName); + } + } + + /// + /// Ensures the given parameter is not null. + /// Throws an if the parameter is invalid. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void NotNull([NotNull] object parameter, [NotNull] string parameterName) + { + if (parameter == null) + { + throw new ArgumentNullException($"Parameter '{parameterName}' cannot be null.", parameterName); + } + } +} diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/MicrosoftGraphModelExtensions.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/MicrosoftGraphModelExtensions.cs similarity index 95% rename from dotnet/src/Skills/Skills.MsGraph/Connectors/MicrosoftGraphModelExtensions.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Connectors/MicrosoftGraphModelExtensions.cs index 5be82741ac47..4046dd436d2f 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/MicrosoftGraphModelExtensions.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/MicrosoftGraphModelExtensions.cs @@ -4,12 +4,12 @@ using System.Linq; using Microsoft.Graph; using Microsoft.Graph.Extensions; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors; /// -/// Extensions for converting between Microsoft Graph models and skill models. +/// Extensions for converting between Microsoft Graph models and plugin models. /// internal static class MicrosoftGraphModelExtensions { diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/MicrosoftToDoConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/MicrosoftToDoConnector.cs similarity index 94% rename from dotnet/src/Skills/Skills.MsGraph/Connectors/MicrosoftToDoConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Connectors/MicrosoftToDoConnector.cs index e0491584dda4..615657388836 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/MicrosoftToDoConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/MicrosoftToDoConnector.cs @@ -7,12 +7,12 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Graph; -using Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Diagnostics; -using Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Exceptions; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.Diagnostics; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; using TaskStatus = Microsoft.Graph.TaskStatus; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors; /// /// Connector for Microsoft To-Do API @@ -50,7 +50,7 @@ public MicrosoftToDoConnector(GraphServiceClient graphServiceClient) if (result == null) { - throw new MsGraphConnectorException("Could not find default task list."); + throw new SKException("Could not find default task list."); } return new TaskManagementTaskList(result.Id, result.DisplayName); diff --git a/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OneDriveConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OneDriveConnector.cs new file mode 100644 index 000000000000..c9a086e3ca2a --- /dev/null +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OneDriveConnector.cs @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Graph; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.Diagnostics; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors; + +/// +/// Connector for OneDrive API +/// +public class OneDriveConnector : ICloudDriveConnector +{ + private readonly GraphServiceClient _graphServiceClient; + + /// + /// Initializes a new instance of the class. + /// + /// A graph service client. + public OneDriveConnector(GraphServiceClient graphServiceClient) + { + this._graphServiceClient = graphServiceClient; + } + + /// + public async Task GetFileContentStreamAsync(string filePath, CancellationToken cancellationToken = default) + { + Ensure.NotNullOrWhitespace(filePath, nameof(filePath)); + + return await this._graphServiceClient.Me + .Drive.Root + .ItemWithPath(filePath).Content + .Request().GetAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Checks if a file exists at the specified path in OneDrive. + /// + /// The path to the file in OneDrive. + /// An optional to observe while waiting for the task to complete. + /// A representing the result of the asynchronous operation. True if the file exists, false otherwise. + public async Task FileExistsAsync(string filePath, CancellationToken cancellationToken = default) + { + Ensure.NotNullOrWhitespace(filePath, nameof(filePath)); + + try + { + await this._graphServiceClient.Me + .Drive.Root + .ItemWithPath(filePath).Request().GetAsync(cancellationToken).ConfigureAwait(false); + + // If no exception is thrown, the file exists. + return true; + } + catch (ServiceException ex) + { + // If the exception is a 404 Not Found, the file does not exist. + if (ex.StatusCode == HttpStatusCode.NotFound) + { + return false; + } + + throw new HttpOperationException(ex.StatusCode, responseContent: null, ex.Message, ex); + } + } + + /// + public async Task UploadSmallFileAsync(string filePath, string destinationPath, CancellationToken cancellationToken = default) + { + Ensure.NotNullOrWhitespace(filePath, nameof(filePath)); + Ensure.NotNullOrWhitespace(destinationPath, nameof(destinationPath)); + + filePath = Environment.ExpandEnvironmentVariables(filePath); + + long fileSize = new FileInfo(filePath).Length; + if (fileSize > 4 * 1024 * 1024) + { + throw new IOException("File is too large to upload - function currently only supports files up to 4MB."); + } + + using FileStream fileContentStream = new(filePath, FileMode.Open, FileAccess.Read); + + GraphResponse? response = null; + + try + { + response = await this._graphServiceClient.Me + .Drive.Root + .ItemWithPath(destinationPath).Content + .Request().PutResponseAsync(fileContentStream, cancellationToken, HttpCompletionOption.ResponseContentRead).ConfigureAwait(false); + + response.ToHttpResponseMessage().EnsureSuccessStatusCode(); + } + catch (ServiceException ex) + { + throw new HttpOperationException(ex.StatusCode, responseContent: null, ex.Message, ex); + } + catch (HttpRequestException ex) + { + throw new HttpOperationException(response?.StatusCode, responseContent: null, ex.Message, ex); + } + } + + /// + public async Task CreateShareLinkAsync(string filePath, string type = "view", string scope = "anonymous", + CancellationToken cancellationToken = default) + { + Ensure.NotNullOrWhitespace(filePath, nameof(filePath)); + Ensure.NotNullOrWhitespace(type, nameof(type)); + Ensure.NotNullOrWhitespace(scope, nameof(scope)); + + GraphResponse? response = null; + + try + { + response = await this._graphServiceClient.Me + .Drive.Root + .ItemWithPath(filePath) + .CreateLink(type, scope) + .Request().PostResponseAsync(cancellationToken).ConfigureAwait(false); + + response.ToHttpResponseMessage().EnsureSuccessStatusCode(); + } + catch (ServiceException ex) + { + throw new HttpOperationException(ex.StatusCode, responseContent: null, ex.Message, ex); + } + catch (HttpRequestException ex) + { + throw new HttpOperationException(response?.StatusCode, responseContent: null, ex.Message, ex); + } + + string? result = (await response.GetResponseObjectAsync().ConfigureAwait(false)).Link?.WebUrl; + if (string.IsNullOrWhiteSpace(result)) + { + throw new SKException("Shareable file link was null or whitespace."); + } + + return result!; + } +} diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/OrganizationHierarchyConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OrganizationHierarchyConnector.cs similarity index 97% rename from dotnet/src/Skills/Skills.MsGraph/Connectors/OrganizationHierarchyConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Connectors/OrganizationHierarchyConnector.cs index 32ff6eb608d0..01f0df582b1c 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/OrganizationHierarchyConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OrganizationHierarchyConnector.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Microsoft.Graph; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors; /// /// Connector for Microsoft Graph API for organizational hierarchy. diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/OutlookCalendarConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OutlookCalendarConnector.cs similarity index 94% rename from dotnet/src/Skills/Skills.MsGraph/Connectors/OutlookCalendarConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Connectors/OutlookCalendarConnector.cs index 3dbe78848064..bc856c8bbd4b 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/OutlookCalendarConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OutlookCalendarConnector.cs @@ -5,9 +5,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Graph; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors; /// /// Connector for Outlook Calendar API diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/OutlookMailConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OutlookMailConnector.cs similarity index 93% rename from dotnet/src/Skills/Skills.MsGraph/Connectors/OutlookMailConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Connectors/OutlookMailConnector.cs index ec4c78efb32a..78c484910bff 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/OutlookMailConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Connectors/OutlookMailConnector.cs @@ -5,10 +5,10 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Graph; -using Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Diagnostics; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; +using Microsoft.SemanticKernel.Plugins.MsGraph.Connectors.Diagnostics; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Connectors; /// /// Connector for Outlook Mail API diff --git a/dotnet/src/Plugins/Plugins.MsGraph/Diagnostics/Ensure.cs b/dotnet/src/Plugins/Plugins.MsGraph/Diagnostics/Ensure.cs new file mode 100644 index 000000000000..97fdc0102b9c --- /dev/null +++ b/dotnet/src/Plugins/Plugins.MsGraph/Diagnostics/Ensure.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Diagnostics; + +internal static class Ensure +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void NotNullOrWhitespace([NotNull] string parameter, [NotNull] string parameterName) + { + if (string.IsNullOrWhiteSpace(parameter)) + { + throw new ArgumentException($"Parameter '{parameterName}' cannot be null or whitespace.", parameterName); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void NotNull([NotNull] object parameter, [NotNull] string parameterName) + { + if (parameter == null) + { + throw new ArgumentNullException($"Parameter '{parameterName}' cannot be null.", parameterName); + } + } +} diff --git a/dotnet/src/Skills/Skills.MsGraph/Diagnostics/NullableAttributes.cs b/dotnet/src/Plugins/Plugins.MsGraph/Diagnostics/NullableAttributes.cs similarity index 100% rename from dotnet/src/Skills/Skills.MsGraph/Diagnostics/NullableAttributes.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Diagnostics/NullableAttributes.cs diff --git a/dotnet/src/Plugins/Plugins.MsGraph/EmailPlugin.cs b/dotnet/src/Plugins/Plugins.MsGraph/EmailPlugin.cs new file mode 100644 index 000000000000..d20351690cc6 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.MsGraph/EmailPlugin.cs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Plugins.MsGraph.Diagnostics; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; + +namespace Microsoft.SemanticKernel.Plugins.MsGraph; + +/// +/// Email plugin (e.g. Outlook). +/// +public sealed class EmailPlugin +{ + /// + /// parameter names. + /// + public static class Parameters + { + /// + /// Email recipients, separated by ',' or ';'. + /// + public const string Recipients = "recipients"; + + /// + /// Email subject. + /// + public const string Subject = "subject"; + + /// + /// The name of the top parameter used to limit the number of results returned in the response. + /// + public const string MaxResults = "maxResults"; + + /// + /// The name of the skip parameter used to skip a certain number of results in the response. + /// + public const string Skip = "skip"; + } + + private readonly IEmailConnector _connector; + private readonly ILogger _logger; + private static readonly JsonSerializerOptions s_options = new() + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + /// + /// Initializes a new instance of the class. + /// + /// Email connector. + /// The to use for logging. If null, no logging will be performed. + public EmailPlugin(IEmailConnector connector, ILoggerFactory? loggerFactory = null) + { + Ensure.NotNull(connector, nameof(connector)); + + this._connector = connector; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(EmailPlugin)) : NullLogger.Instance; + } + + /// + /// Get my email address. + /// + [SKFunction, Description("Gets the email address for me.")] + public async Task GetMyEmailAddressAsync() + => await this._connector.GetMyEmailAddressAsync().ConfigureAwait(false); + + /// + /// Send an email using as the body. + /// + [SKFunction, Description("Send an email to one or more recipients.")] + public async Task SendEmailAsync( + [Description("Email content/body")] string content, + [Description("Recipients of the email, separated by ',' or ';'.")] string recipients, + [Description("Subject of the email")] string subject, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(recipients)) + { + throw new ArgumentException("Variable was null or whitespace", nameof(recipients)); + } + + if (string.IsNullOrWhiteSpace(subject)) + { + throw new ArgumentException("Variable was null or whitespace", nameof(subject)); + } + + // Sensitive data, logging as trace, disabled by default + this._logger.LogTrace("Sending email to '{0}' with subject '{1}'", recipients, subject); + string[] recipientList = recipients.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries); + await this._connector.SendEmailAsync(subject, content, recipientList, cancellationToken).ConfigureAwait(false); + } + + /// + /// Get email messages with specified optional clauses used to query for messages. + /// + [SKFunction, Description("Get email messages.")] + public async Task GetEmailMessagesAsync( + [Description("Optional limit of the number of message to retrieve.")] int? maxResults = 10, + [Description("Optional number of message to skip before retrieving results.")] int? skip = 0, + CancellationToken cancellationToken = default) + { + this._logger.LogDebug("Getting email messages with query options top: '{0}', skip:'{1}'.", maxResults, skip); + + const string SelectString = "subject,receivedDateTime,bodyPreview"; + + IEnumerable messages = await this._connector.GetMessagesAsync( + top: maxResults, + skip: skip, + select: SelectString, + cancellationToken) + .ConfigureAwait(false); + + return JsonSerializer.Serialize(value: messages, options: s_options); + } +} diff --git a/dotnet/src/Skills/Skills.MsGraph/ICalendarConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/ICalendarConnector.cs similarity index 93% rename from dotnet/src/Skills/Skills.MsGraph/ICalendarConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/ICalendarConnector.cs index 22a9ec7670be..e12f87cfda79 100644 --- a/dotnet/src/Skills/Skills.MsGraph/ICalendarConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/ICalendarConnector.cs @@ -3,9 +3,9 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; -namespace Microsoft.SemanticKernel.Skills.MsGraph; +namespace Microsoft.SemanticKernel.Plugins.MsGraph; /// /// Interface for calendar connections (e.g. Outlook). diff --git a/dotnet/src/Skills/Skills.MsGraph/ICloudDriveConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/ICloudDriveConnector.cs similarity index 97% rename from dotnet/src/Skills/Skills.MsGraph/ICloudDriveConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/ICloudDriveConnector.cs index 854ab7bdbc03..f13fa7240c57 100644 --- a/dotnet/src/Skills/Skills.MsGraph/ICloudDriveConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/ICloudDriveConnector.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.SemanticKernel.Skills.MsGraph; +namespace Microsoft.SemanticKernel.Plugins.MsGraph; /// /// Interface for cloud drive connections (e.g. OneDrive). diff --git a/dotnet/src/Skills/Skills.MsGraph/IEmailConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/IEmailConnector.cs similarity index 94% rename from dotnet/src/Skills/Skills.MsGraph/IEmailConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/IEmailConnector.cs index 3b735a214244..8faf50f973e2 100644 --- a/dotnet/src/Skills/Skills.MsGraph/IEmailConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/IEmailConnector.cs @@ -3,9 +3,9 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; -namespace Microsoft.SemanticKernel.Skills.MsGraph; +namespace Microsoft.SemanticKernel.Plugins.MsGraph; /// /// Interface for email connections (e.g. Outlook). diff --git a/dotnet/src/Skills/Skills.MsGraph/IOrganizationHierarchyConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/IOrganizationHierarchyConnector.cs similarity index 96% rename from dotnet/src/Skills/Skills.MsGraph/IOrganizationHierarchyConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/IOrganizationHierarchyConnector.cs index 79c7aaafbc96..543cbf57cad6 100644 --- a/dotnet/src/Skills/Skills.MsGraph/IOrganizationHierarchyConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/IOrganizationHierarchyConnector.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.SemanticKernel.Skills.MsGraph; +namespace Microsoft.SemanticKernel.Plugins.MsGraph; /// /// Interface for organization hierarchy connections (e.g. Azure AD). diff --git a/dotnet/src/Skills/Skills.MsGraph/ITaskManagementConnector.cs b/dotnet/src/Plugins/Plugins.MsGraph/ITaskManagementConnector.cs similarity index 96% rename from dotnet/src/Skills/Skills.MsGraph/ITaskManagementConnector.cs rename to dotnet/src/Plugins/Plugins.MsGraph/ITaskManagementConnector.cs index 30ee8fcea0f6..56c6cf7d99a3 100644 --- a/dotnet/src/Skills/Skills.MsGraph/ITaskManagementConnector.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/ITaskManagementConnector.cs @@ -3,9 +3,9 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; -namespace Microsoft.SemanticKernel.Skills.MsGraph; +namespace Microsoft.SemanticKernel.Plugins.MsGraph; /// /// Interface for task list connections (e.g. Microsoft To-Do). diff --git a/dotnet/src/Skills/Skills.MsGraph/Models/CalendarEvent.cs b/dotnet/src/Plugins/Plugins.MsGraph/Models/CalendarEvent.cs similarity index 94% rename from dotnet/src/Skills/Skills.MsGraph/Models/CalendarEvent.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Models/CalendarEvent.cs index ff1b644c621b..935aec562780 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Models/CalendarEvent.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Models/CalendarEvent.cs @@ -4,7 +4,7 @@ using System.Collections.Generic; using System.Linq; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Models; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Models; /// /// Model for a calendar event. diff --git a/dotnet/src/Skills/Skills.MsGraph/Models/EmailAddress.cs b/dotnet/src/Plugins/Plugins.MsGraph/Models/EmailAddress.cs similarity index 86% rename from dotnet/src/Skills/Skills.MsGraph/Models/EmailAddress.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Models/EmailAddress.cs index 756d6d7382f6..e328b3f0efcc 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Models/EmailAddress.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Models/EmailAddress.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.MsGraph.Models; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Models; /// /// Model for an email address. diff --git a/dotnet/src/Skills/Skills.MsGraph/Models/EmailMessage.cs b/dotnet/src/Plugins/Plugins.MsGraph/Models/EmailMessage.cs similarity index 95% rename from dotnet/src/Skills/Skills.MsGraph/Models/EmailMessage.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Models/EmailMessage.cs index 0b9a146741ca..939f14fc6f3e 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Models/EmailMessage.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Models/EmailMessage.cs @@ -3,7 +3,7 @@ using System; using System.Collections.Generic; -namespace Microsoft.SemanticKernel.Skills.MsGraph.Models; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Models; /// /// Model for an email message. diff --git a/dotnet/src/Skills/Skills.MsGraph/Models/TaskManagementTask.cs b/dotnet/src/Plugins/Plugins.MsGraph/Models/TaskManagementTask.cs similarity index 96% rename from dotnet/src/Skills/Skills.MsGraph/Models/TaskManagementTask.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Models/TaskManagementTask.cs index b8803fa869e1..1cdb5e79317b 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Models/TaskManagementTask.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Models/TaskManagementTask.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.MsGraph.Models; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Models; /// /// Model for a task in a task list. diff --git a/dotnet/src/Skills/Skills.MsGraph/Models/TaskManagementTaskList.cs b/dotnet/src/Plugins/Plugins.MsGraph/Models/TaskManagementTaskList.cs similarity index 92% rename from dotnet/src/Skills/Skills.MsGraph/Models/TaskManagementTaskList.cs rename to dotnet/src/Plugins/Plugins.MsGraph/Models/TaskManagementTaskList.cs index ff4e37c4170a..e8cea6d8ead8 100644 --- a/dotnet/src/Skills/Skills.MsGraph/Models/TaskManagementTaskList.cs +++ b/dotnet/src/Plugins/Plugins.MsGraph/Models/TaskManagementTaskList.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.SemanticKernel.Skills.MsGraph.Models; +namespace Microsoft.SemanticKernel.Plugins.MsGraph.Models; /// /// Model for a list of tasks. diff --git a/dotnet/src/Plugins/Plugins.MsGraph/OrganizationHierarchyPlugin.cs b/dotnet/src/Plugins/Plugins.MsGraph/OrganizationHierarchyPlugin.cs new file mode 100644 index 000000000000..156212199fa0 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.MsGraph/OrganizationHierarchyPlugin.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Plugins.MsGraph.Diagnostics; + +namespace Microsoft.SemanticKernel.Plugins.MsGraph; + +/// +/// Organizational Hierarchy plugin. +/// Provides methods to get information about the organization hierarchy, such as direct reports and manager details. +/// +public sealed class OrganizationHierarchyPlugin +{ + private readonly IOrganizationHierarchyConnector _connector; + + /// + /// Initializes a new instance of the class. + /// + /// The connector to be used for fetching organization hierarchy data. + public OrganizationHierarchyPlugin(IOrganizationHierarchyConnector connector) + { + Ensure.NotNull(connector, nameof(connector)); + + this._connector = connector; + } + + /// + /// Get the emails of the direct reports of the current user. + /// + /// An optional to observe while waiting for the task to complete. + /// A JSON string containing the email addresses of the direct reports of the current user. + [SKFunction, Description("Get my direct report's email addresses.")] + public async Task GetMyDirectReportsEmailAsync(CancellationToken cancellationToken = default) + => JsonSerializer.Serialize(await this._connector.GetDirectReportsEmailAsync(cancellationToken).ConfigureAwait(false)); + + /// + /// Get the email of the manager of the current user. + /// + /// An optional to observe while waiting for the task to complete. + /// A string containing the email address of the manager of the current user. + [SKFunction, Description("Get my manager's email address.")] + public async Task GetMyManagerEmailAsync(CancellationToken cancellationToken = default) + => await this._connector.GetManagerEmailAsync(cancellationToken).ConfigureAwait(false); + + /// + /// Get the name of the manager of the current user. + /// + /// An optional to observe while waiting for the task to complete. + /// A string containing the name of the manager of the current user. + [SKFunction, Description("Get my manager's name.")] + public async Task GetMyManagerNameAsync(CancellationToken cancellationToken = default) + => await this._connector.GetManagerNameAsync(cancellationToken).ConfigureAwait(false); +} diff --git a/dotnet/src/Plugins/Plugins.MsGraph/Plugins.MsGraph.csproj b/dotnet/src/Plugins/Plugins.MsGraph/Plugins.MsGraph.csproj new file mode 100644 index 000000000000..52004dbd4c09 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.MsGraph/Plugins.MsGraph.csproj @@ -0,0 +1,28 @@ + + + + + Microsoft.SemanticKernel.Plugins.MsGraph + $(AssemblyName) + netstandard2.0 + + + + + + + Semantic Kernel - Microsoft Graph Plugins + Semantic Kernel Microsoft Graph Plugins: access your tenant data, schedule meetings, send emails, etc. + + + + + + + + + + + + + diff --git a/dotnet/src/Plugins/Plugins.MsGraph/TaskListPlugin.cs b/dotnet/src/Plugins/Plugins.MsGraph/TaskListPlugin.cs new file mode 100644 index 000000000000..e530072de005 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.MsGraph/TaskListPlugin.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Plugins.MsGraph.Diagnostics; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; + +namespace Microsoft.SemanticKernel.Plugins.MsGraph; + +/// +/// Task list plugin (e.g. Microsoft To-Do) +/// +public sealed class TaskListPlugin +{ + /// + /// parameter names. + /// + public static class Parameters + { + /// + /// Task reminder as DateTimeOffset. + /// + public const string Reminder = "reminder"; + + /// + /// Whether to include completed tasks. + /// + public const string IncludeCompleted = "includeCompleted"; + } + + private readonly ITaskManagementConnector _connector; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Task list connector. + /// The to use for logging. If null, no logging will be performed. + public TaskListPlugin(ITaskManagementConnector connector, ILoggerFactory? loggerFactory = null) + { + Ensure.NotNull(connector, nameof(connector)); + + this._connector = connector; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(TaskListPlugin)) : NullLogger.Instance; + } + + /// + /// Calculates an upcoming day of week (e.g. 'next Monday'). + /// + public static DateTimeOffset GetNextDayOfWeek(DayOfWeek dayOfWeek, TimeSpan timeOfDay) + { + DateTimeOffset today = new(DateTime.Today); + int nextDayOfWeekOffset = dayOfWeek - today.DayOfWeek; + if (nextDayOfWeekOffset <= 0) + { + nextDayOfWeekOffset += 7; + } + + DateTimeOffset nextDayOfWeek = today.AddDays(nextDayOfWeekOffset); + DateTimeOffset nextDayOfWeekAtTimeOfDay = nextDayOfWeek.Add(timeOfDay); + + return nextDayOfWeekAtTimeOfDay; + } + + /// + /// Add a task to a To-Do list with an optional reminder. + /// + [SKFunction, Description("Add a task to a task list with an optional reminder.")] + public async Task AddTaskAsync( + [Description("Title of the task.")] string title, + [Description("Reminder for the task in DateTimeOffset (optional)")] string? reminder = null, + CancellationToken cancellationToken = default) + { + TaskManagementTaskList? defaultTaskList = await this._connector.GetDefaultTaskListAsync(cancellationToken).ConfigureAwait(false); + if (defaultTaskList == null) + { + throw new InvalidOperationException("No default task list found."); + } + + TaskManagementTask task = new( + id: Guid.NewGuid().ToString(), + title: title, + reminder: reminder); + + // Sensitive data, logging as trace, disabled by default + this._logger.LogTrace("Adding task '{0}' to task list '{1}'", task.Title, defaultTaskList.Name); + + await this._connector.AddTaskAsync(defaultTaskList.Id, task, cancellationToken).ConfigureAwait(false); + } + + /// + /// Get tasks from the default task list. + /// + [SKFunction, Description("Get tasks from the default task list.")] + public async Task GetDefaultTasksAsync( + [Description("Whether to include completed tasks (optional)")] string includeCompleted = "false", + CancellationToken cancellationToken = default) + { + TaskManagementTaskList? defaultTaskList = await this._connector.GetDefaultTaskListAsync(cancellationToken).ConfigureAwait(false); + if (defaultTaskList == null) + { + throw new InvalidOperationException("No default task list found."); + } + + if (!bool.TryParse(includeCompleted, out bool includeCompletedValue)) + { + this._logger.LogWarning("Invalid value for '{0}' variable: '{1}'", nameof(includeCompleted), includeCompleted); + } + + IEnumerable tasks = await this._connector.GetTasksAsync(defaultTaskList.Id, includeCompletedValue, cancellationToken).ConfigureAwait(false); + return JsonSerializer.Serialize(tasks); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/.editorconfig b/dotnet/src/Plugins/Plugins.UnitTests/.editorconfig new file mode 100644 index 000000000000..394eef685f21 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Core/FileIOPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Core/FileIOPluginTests.cs new file mode 100644 index 000000000000..3ba1750aa78c --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Core/FileIOPluginTests.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Plugins.Core; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Core; + +public class FileIOPluginTests +{ + [Fact] + public void ItCanBeInstantiated() + { + // Act - Assert no exception occurs + _ = new FileIOPlugin(); + } + + [Fact] + public void ItCanBeImported() + { + // Arrange + var kernel = Kernel.Builder.Build(); + + // Act + var functions = kernel.ImportFunctions(new FileIOPlugin(), "fileIO"); + + // Assert no exception occurs e.g. due to reflection + Assert.NotNull(functions); + } + + [Fact] + public async Task ItCanReadAsync() + { + // Arrange + var plugin = new FileIOPlugin(); + var path = Path.GetTempFileName(); + File.WriteAllText(path, "hello world"); + + // Act + var result = await plugin.ReadAsync(path); + + // Assert + Assert.Equal("hello world", result); + } + + [Fact] + public async Task ItCannotReadAsync() + { + // Arrange + var plugin = new FileIOPlugin(); + var path = Path.GetTempFileName(); + File.Delete(path); + + // Act + Task Fn() + { + return plugin.ReadAsync(path); + } + + // Assert + _ = await Assert.ThrowsAsync(Fn); + } + + [Fact] + public async Task ItCanWriteAsync() + { + // Arrange + var plugin = new FileIOPlugin(); + var path = Path.GetTempFileName(); + + // Act + await plugin.WriteAsync(path, "hello world"); + + // Assert + Assert.Equal("hello world", await File.ReadAllTextAsync(path)); + } + + [Fact] + public async Task ItCannotWriteAsync() + { + // Arrange + var plugin = new FileIOPlugin(); + var path = Path.GetTempFileName(); + File.SetAttributes(path, FileAttributes.ReadOnly); + + // Act + Task Fn() + { + return plugin.WriteAsync(path, "hello world"); + } + + // Assert + _ = await Assert.ThrowsAsync(Fn); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Core/HttpPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Core/HttpPluginTests.cs new file mode 100644 index 000000000000..6c49006dbd54 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Core/HttpPluginTests.cs @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Plugins.Core; +using Moq; +using Moq.Protected; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Core; + +public class HttpPluginTests : IDisposable +{ + private readonly string _content = "hello world"; + private readonly string _uriString = "http://www.example.com"; + + private readonly HttpResponseMessage _response = new() + { + StatusCode = HttpStatusCode.OK, + Content = new StringContent("hello world"), + }; + + [Fact] + public void ItCanBeInstantiated() + { + // Act - Assert no exception occurs + var plugin = new HttpPlugin(); + } + + [Fact] + public void ItCanBeImported() + { + // Arrange + var kernel = KernelBuilder.Create(); + var plugin = new HttpPlugin(); + + // Act - Assert no exception occurs e.g. due to reflection + kernel.ImportFunctions(plugin, "http"); + } + + [Fact] + public async Task ItCanGetAsync() + { + // Arrange + var mockHandler = this.CreateMock(); + using var client = new HttpClient(mockHandler.Object); + var plugin = new HttpPlugin(client); + + // Act + var result = await plugin.GetAsync(this._uriString); + + // Assert + Assert.Equal(this._content, result); + this.VerifyMock(mockHandler, HttpMethod.Get); + } + + [Fact] + public async Task ItCanPostAsync() + { + // Arrange + var mockHandler = this.CreateMock(); + using var client = new HttpClient(mockHandler.Object); + var plugin = new HttpPlugin(client); + + // Act + var result = await plugin.PostAsync(this._uriString, this._content); + + // Assert + Assert.Equal(this._content, result); + this.VerifyMock(mockHandler, HttpMethod.Post); + } + + [Fact] + public async Task ItCanPutAsync() + { + // Arrange + var mockHandler = this.CreateMock(); + using var client = new HttpClient(mockHandler.Object); + var plugin = new HttpPlugin(client); + + // Act + var result = await plugin.PutAsync(this._uriString, this._content); + + // Assert + Assert.Equal(this._content, result); + this.VerifyMock(mockHandler, HttpMethod.Put); + } + + [Fact] + public async Task ItCanDeleteAsync() + { + // Arrange + var mockHandler = this.CreateMock(); + using var client = new HttpClient(mockHandler.Object); + var plugin = new HttpPlugin(client); + + // Act + var result = await plugin.DeleteAsync(this._uriString); + + // Assert + Assert.Equal(this._content, result); + this.VerifyMock(mockHandler, HttpMethod.Delete); + } + + private Mock CreateMock() + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(this._response); + return mockHandler; + } + + private void VerifyMock(Mock mockHandler, HttpMethod method) + { + mockHandler.Protected().Verify( + "SendAsync", + Times.Exactly(1), // we expected a single external request + ItExpr.Is(req => + req.Method == method // we expected a POST request + && req.RequestUri == new Uri(this._uriString) // to this uri + ), + ItExpr.IsAny() + ); + } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._response.Dispose(); + } + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Core/MathPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Core/MathPluginTests.cs new file mode 100644 index 000000000000..7cd2f2334d8b --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Core/MathPluginTests.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Plugins.Core; +using SemanticKernel.UnitTests; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Core; + +public class MathPluginTests +{ + [Fact] + public void ItCanBeInstantiated() + { + // Act - Assert no exception occurs + var _ = new MathPlugin(); + } + + [Fact] + public void ItCanBeImported() + { + // Arrange + var kernel = Kernel.Builder.Build(); + + // Act - Assert no exception occurs e.g. due to reflection + kernel.ImportFunctions(new MathPlugin(), "math"); + } + + [Theory] + [InlineData(10, 10, 20)] + [InlineData(0, 10, 10)] + [InlineData(0, -10, -10)] + [InlineData(10, 0, 10)] + [InlineData(-1, 10, 9)] + [InlineData(-10, 10, 0)] + [InlineData(-192, 13, -179)] + [InlineData(-192, -13, -205)] + public async Task AddWhenValidParametersShouldSucceedAsync(int initialValue, int amount, int expectedResult) + { + // Arrange + var target = new MathPlugin(); + + // Act + var result = await FunctionHelpers.CallViaKernelAsync(target, "Add", ("input", initialValue), ("amount", amount)); + + // Assert + Assert.Equal(expectedResult, result.GetValue()); + } + + [Theory] + [InlineData(10, 10, 0)] + [InlineData(0, 10, -10)] + [InlineData(10, 0, 10)] + [InlineData(100, -10, 110)] + [InlineData(100, 102, -2)] + [InlineData(-1, 10, -11)] + [InlineData(-10, 10, -20)] + [InlineData(-192, 13, -205)] + public async Task SubtractWhenValidParametersShouldSucceedAsync(int initialValue, int amount, int expectedResult) + { + // Arrange + var target = new MathPlugin(); + + // Act + var result = await FunctionHelpers.CallViaKernelAsync(target, "Subtract", ("input", initialValue), ("amount", amount)); // Assert + + // Assert + Assert.Equal(expectedResult, result.GetValue()); + } + + [Theory] + [InlineData("$0")] + [InlineData("one hundred")] + [InlineData("20..,,2,1")] + [InlineData(".2,2.1")] + [InlineData("0.1.0")] + [InlineData("00-099")] + [InlineData("¹²¹")] + [InlineData("2²")] + [InlineData("zero")] + [InlineData("-100 units")] + [InlineData("1 banana")] + public async Task AddWhenInvalidInitialValueShouldThrowAsync(string initialValue) + { + // Arrange + var target = new MathPlugin(); + + // Act + var ex = await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(target, "Add", ("input", initialValue), ("amount", "1"))); + + // Assert + AssertExtensions.AssertIsArgumentOutOfRange(ex, "value", initialValue); + } + + [Theory] + [InlineData("$0")] + [InlineData("one hundred")] + [InlineData("20..,,2,1")] + [InlineData(".2,2.1")] + [InlineData("0.1.0")] + [InlineData("00-099")] + [InlineData("¹²¹")] + [InlineData("2²")] + [InlineData("zero")] + [InlineData("-100 units")] + [InlineData("1 banana")] + public async Task AddWhenInvalidAmountShouldThrowAsync(string amount) + { + // Arrange + var target = new MathPlugin(); + + // Act + var ex = await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(target, "Add", ("input", "1"), ("amount", amount))); + + // Assert + AssertExtensions.AssertIsArgumentOutOfRange(ex, "amount", amount); + } + + [Theory] + [InlineData("$0")] + [InlineData("one hundred")] + [InlineData("20..,,2,1")] + [InlineData(".2,2.1")] + [InlineData("0.1.0")] + [InlineData("00-099")] + [InlineData("¹²¹")] + [InlineData("2²")] + [InlineData("zero")] + [InlineData("-100 units")] + [InlineData("1 banana")] + public async Task SubtractWhenInvalidInitialValueShouldThrowAsync(string initialValue) + { + // Arrange + var target = new MathPlugin(); + + // Act + var ex = await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(target, "Subtract", ("input", initialValue), ("amount", "1"))); + + // Assert + AssertExtensions.AssertIsArgumentOutOfRange(ex, "value", initialValue); + } + + [Theory] + [InlineData("$0")] + [InlineData("one hundred")] + [InlineData("20..,,2,1")] + [InlineData(".2,2.1")] + [InlineData("0.1.0")] + [InlineData("00-099")] + [InlineData("¹²¹")] + [InlineData("2²")] + [InlineData("zero")] + [InlineData("-100 units")] + [InlineData("1 banana")] + public async Task SubtractAsyncWhenInvalidAmountShouldThrowAsync(string amount) + { + // Arrange + var target = new MathPlugin(); + + // Act + var ex = await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(target, "Subtract", ("input", "1"), ("amount", amount))); + + // Assert + AssertExtensions.AssertIsArgumentOutOfRange(ex, "amount", amount); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Core/TextPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Core/TextPluginTests.cs new file mode 100644 index 000000000000..268140be23a0 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Core/TextPluginTests.cs @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Plugins.Core; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Core; + +public class TextPluginTests +{ + [Fact] + public void ItCanBeInstantiated() + { + // Act - Assert no exception occurs + var _ = new TextPlugin(); + } + + [Fact] + public void ItCanBeImported() + { + // Arrange + var kernel = Kernel.Builder.Build(); + + // Act - Assert no exception occurs e.g. due to reflection + kernel.ImportFunctions(new TextPlugin(), "text"); + } + + [Fact] + public void ItCanTrim() + { + // Arrange + var plugin = new TextPlugin(); + + // Act + var result = plugin.Trim(" hello world "); + + // Assert + Assert.Equal("hello world", result); + } + + [Fact] + public void ItCanTrimStart() + { + // Arrange + var plugin = new TextPlugin(); + + // Act + var result = plugin.TrimStart(" hello world "); + + // Assert + Assert.Equal("hello world ", result); + } + + [Fact] + public void ItCanTrimEnd() + { + // Arrange + var plugin = new TextPlugin(); + + // Act + var result = plugin.TrimEnd(" hello world "); + + // Assert + Assert.Equal(" hello world", result); + } + + [Fact] + public void ItCanUppercase() + { + // Arrange + var plugin = new TextPlugin(); + + // Act + var result = plugin.Uppercase("hello world"); + + // Assert + Assert.Equal("HELLO WORLD", result); + } + + [Fact] + public void ItCanLowercase() + { + // Arrange + var plugin = new TextPlugin(); + + // Act + var result = plugin.Lowercase("HELLO WORLD"); + + // Assert + Assert.Equal("hello world", result); + } + + [Theory] + [InlineData("hello world ", 12)] + [InlineData("hello World", 11)] + [InlineData("HELLO", 5)] + [InlineData("World", 5)] + [InlineData("", 0)] + [InlineData(" ", 1)] + [InlineData(null, 0)] + public void ItCanLength(string textToLength, int expectedLength) + { + // Arrange + var target = new TextPlugin(); + + // Act + var result = target.Length(textToLength); + + // Assert + Assert.Equal(expectedLength, result); + } + + [Theory] + [InlineData("hello world", "hello world")] + [InlineData("hello World", "hello World")] + [InlineData("HELLO", "HELLO")] + [InlineData("World", "World")] + [InlineData("", "")] + [InlineData(" ", " ")] + [InlineData(null, "")] + public void ItCanConcat(string textToConcat, string text2ToConcat) + { + // Arrange + var target = new TextPlugin(); + var expected = string.Concat(textToConcat, text2ToConcat); + + // Act + string result = target.Concat(textToConcat, text2ToConcat); + + // Assert + Assert.Equal(expected, result); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Core/TimePluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Core/TimePluginTests.cs new file mode 100644 index 000000000000..a066612b357e --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Core/TimePluginTests.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Plugins.Core; +using SemanticKernel.UnitTests; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Core; + +// TODO: allow clock injection and test all functions +public class TimePluginTests +{ + [Fact] + public void ItCanBeInstantiated() + { + // Act - Assert no exception occurs + var _ = new TimePlugin(); + } + + [Fact] + public void ItCanBeImported() + { + // Arrange + var kernel = Kernel.Builder.Build(); + + // Act - Assert no exception occurs e.g. due to reflection + kernel.ImportFunctions(new TimePlugin(), "time"); + } + + [Fact] + public void DaysAgo() + { + double interval = 2; + DateTime expected = DateTime.Now.AddDays(-interval); + var plugin = new TimePlugin(); + string result = plugin.DaysAgo(interval, CultureInfo.CurrentCulture); + DateTime returned = DateTime.Parse(result, CultureInfo.CurrentCulture); + Assert.Equal(expected.Day, returned.Day); + Assert.Equal(expected.Month, returned.Month); + Assert.Equal(expected.Year, returned.Year); + } + + [Fact] + public void Day() + { + string expected = DateTime.Now.ToString("dd", CultureInfo.CurrentCulture); + var plugin = new TimePlugin(); + string result = plugin.Day(CultureInfo.CurrentCulture); + Assert.Equal(expected, result); + Assert.True(int.TryParse(result, out _)); + } + + [Fact] + public async Task LastMatchingDayBadInputAsync() + { + var plugin = new TimePlugin(); + + var ex = await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(plugin, "DateMatchingLastDayName", ("input", "not a day name"))); + + AssertExtensions.AssertIsArgumentOutOfRange(ex, "input", "not a day name"); + } + + [Theory] + [MemberData(nameof(DayOfWeekEnumerator))] + public void LastMatchingDay(DayOfWeek dayName) + { + int steps = 0; + DateTime date = DateTime.Now.Date.AddDays(-1); + while (date.DayOfWeek != dayName && steps <= 7) + { + date = date.AddDays(-1); + steps++; + } + bool found = date.DayOfWeek == dayName; + Assert.True(found); + + var plugin = new TimePlugin(); + string result = plugin.DateMatchingLastDayName(dayName, CultureInfo.CurrentCulture); + DateTime returned = DateTime.Parse(result, CultureInfo.CurrentCulture); + Assert.Equal(date.Day, returned.Day); + Assert.Equal(date.Month, returned.Month); + Assert.Equal(date.Year, returned.Year); + } + + public static IEnumerable DayOfWeekEnumerator() + { + foreach (var day in Enum.GetValues()) + { + yield return new object[] { day }; + } + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Core/WaitPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Core/WaitPluginTests.cs new file mode 100644 index 000000000000..4cc8a472b3d7 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Core/WaitPluginTests.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Plugins.Core; +using SemanticKernel.UnitTests; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Core; + +// TODO: allow clock injection and test all functions +public class WaitPluginTests +{ + [Fact] + public void ItCanBeInstantiated() + { + // Act - Assert no exception occurs + var _ = new WaitPlugin(); + } + + [Fact] + public void ItCanBeImported() + { + // Arrange + var kernel = Kernel.Builder.Build(); + + // Act - Assert no exception occurs e.g. due to reflection + kernel.ImportFunctions(new WaitPlugin(), "wait"); + } + + [Theory] + [InlineData("0", 0)] + [InlineData("100", 100000)] + [InlineData("20.1", 20100)] + [InlineData("0.1", 100)] + [InlineData("0.01", 10)] + [InlineData("0.001", 1)] + [InlineData("0.0001", 0)] + [InlineData("-0.0001", 0)] + [InlineData("-10000", 0)] + public async Task ItWaitSecondsWhenValidParametersSucceedAsync(string textSeconds, int expectedMilliseconds) + { + // Arrange + var timeProvider = new FakeTimeProvider(); + var target = new WaitPlugin(timeProvider); + var expectedTimeSpan = TimeSpan.FromMilliseconds(expectedMilliseconds); + + // Act and Assert + long startingTime = timeProvider.GetTimestamp(); + Task wait = FunctionHelpers.CallViaKernelAsync(target, "Seconds", ("input", textSeconds)); + + if (expectedMilliseconds > 0) + { + timeProvider.Advance(TimeSpan.FromMilliseconds(expectedMilliseconds - 1)); + Assert.False(wait.IsCompleted); + } + + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await wait; + + Assert.InRange(timeProvider.GetElapsedTime(startingTime).TotalMilliseconds, expectedMilliseconds, double.MaxValue); + } + + [Theory] + [InlineData("$0")] + [InlineData("one hundred")] + [InlineData("20..,,2,1")] + [InlineData(".2,2.1")] + [InlineData("0.1.0")] + [InlineData("00-099")] + [InlineData("¹²¹")] + [InlineData("2²")] + [InlineData("zero")] + [InlineData("-100 seconds")] + [InlineData("1 second")] + public async Task ItWaitSecondsWhenInvalidParametersFailsAsync(string textSeconds) + { + // Arrange + var target = new WaitPlugin(); + + // Act + var ex = await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(target, "Seconds", ("input", textSeconds))); + + // Assert + AssertExtensions.AssertIsArgumentOutOfRange(ex, "seconds", textSeconds); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Document/DocumentPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Document/DocumentPluginTests.cs new file mode 100644 index 000000000000..72853fad55b2 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Document/DocumentPluginTests.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Plugins.Document; +using Microsoft.SemanticKernel.Plugins.Document.FileSystem; +using Moq; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Document; + +public class DocumentPluginTests +{ + [Fact] + public async Task ReadTextAsyncSucceedsAsync() + { + // Arrange + var expectedText = Guid.NewGuid().ToString(); + var anyFilePath = Guid.NewGuid().ToString(); + + var fileSystemConnectorMock = new Mock(); + fileSystemConnectorMock + .Setup(mock => mock.GetFileContentStreamAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), + It.IsAny())) + .ReturnsAsync(Stream.Null); + + var documentConnectorMock = new Mock(); + documentConnectorMock + .Setup(mock => mock.ReadText(It.IsAny())) + .Returns(expectedText); + + var target = new DocumentPlugin(documentConnectorMock.Object, fileSystemConnectorMock.Object); + + // Act + string actual = await target.ReadTextAsync(anyFilePath); + + // Assert + Assert.Equal(expectedText, actual); + fileSystemConnectorMock.VerifyAll(); + documentConnectorMock.VerifyAll(); + } + + [Fact] + public async Task AppendTextAsyncFileExistsSucceedsAsync() + { + // Arrange + var anyText = Guid.NewGuid().ToString(); + var anyFilePath = Guid.NewGuid().ToString(); + + var fileSystemConnectorMock = new Mock(); + fileSystemConnectorMock + .Setup(mock => mock.FileExistsAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), + It.IsAny())) + .ReturnsAsync(true); + fileSystemConnectorMock + .Setup(mock => mock.GetWriteableFileStreamAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), + It.IsAny())) + .ReturnsAsync(Stream.Null); + + var documentConnectorMock = new Mock(); + documentConnectorMock + .Setup(mock => mock.AppendText(It.IsAny(), It.Is(text => text.Equals(anyText, StringComparison.Ordinal)))); + + var target = new DocumentPlugin(documentConnectorMock.Object, fileSystemConnectorMock.Object); + + // Act + await target.AppendTextAsync(anyText, anyFilePath); + + // Assert + fileSystemConnectorMock.VerifyAll(); + documentConnectorMock.VerifyAll(); + } + + [Fact] + public async Task AppendTextAsyncFileDoesNotExistSucceedsAsync() + { + // Arrange + var anyText = Guid.NewGuid().ToString(); + var anyFilePath = Guid.NewGuid().ToString(); + + var fileSystemConnectorMock = new Mock(); + fileSystemConnectorMock + .Setup(mock => mock.FileExistsAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), + It.IsAny())) + .ReturnsAsync(false); + fileSystemConnectorMock + .Setup(mock => mock.CreateFileAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), + It.IsAny())) + .ReturnsAsync(Stream.Null); + + var documentConnectorMock = new Mock(); + documentConnectorMock + .Setup(mock => mock.Initialize(It.IsAny())); + documentConnectorMock + .Setup(mock => mock.AppendText(It.IsAny(), It.Is(text => text.Equals(anyText, StringComparison.Ordinal)))); + + var target = new DocumentPlugin(documentConnectorMock.Object, fileSystemConnectorMock.Object); + + // Act + await target.AppendTextAsync(anyText, anyFilePath); + + // Assert + fileSystemConnectorMock.VerifyAll(); + documentConnectorMock.VerifyAll(); + } + + [Fact] + public async Task AppendTextAsyncNoFilePathFailsAsync() + { + // Arrange + var anyText = Guid.NewGuid().ToString(); + + var fileSystemConnectorMock = new Mock(); + var documentConnectorMock = new Mock(); + + var target = new DocumentPlugin(documentConnectorMock.Object, fileSystemConnectorMock.Object); + + // Act/Assert + await Assert.ThrowsAnyAsync(() => + target.AppendTextAsync(anyText, null!)); + + // Assert + fileSystemConnectorMock.Verify(mock => mock.GetWriteableFileStreamAsync(It.IsAny(), It.IsAny()), Times.Never()); + documentConnectorMock.Verify(mock => mock.AppendText(It.IsAny(), It.IsAny()), Times.Never()); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Memory/MemoryBuilderTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Memory/MemoryBuilderTests.cs new file mode 100644 index 000000000000..92aa2ca5dc07 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Memory/MemoryBuilderTests.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Plugins.Memory; +using Moq; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Memory; + +/// +/// Unit tests for class. +/// +public sealed class MemoryBuilderTests +{ + [Fact] + public void ItThrowsExceptionWhenMemoryStoreIsNotProvided() + { + // Arrange + var builder = new MemoryBuilder(); + + // Act + var exception = Assert.Throws(() => builder.Build()); + + // Assert + Assert.Equal("IMemoryStore dependency was not provided. Use WithMemoryStore method.", exception.Message); + } + + [Fact] + public void ItThrowsExceptionWhenEmbeddingGenerationIsNotProvided() + { + // Arrange + var builder = new MemoryBuilder() + .WithMemoryStore(Mock.Of()); + + // Act + var exception = Assert.Throws(() => builder.Build()); + + // Assert + Assert.Equal("ITextEmbeddingGeneration dependency was not provided. Use WithTextEmbeddingGeneration method.", exception.Message); + } + + [Fact] + public void ItInitializesMemoryWhenRequiredDependenciesAreProvided() + { + // Arrange + var builder = new MemoryBuilder() + .WithMemoryStore(Mock.Of()) + .WithTextEmbeddingGeneration(Mock.Of()); + + // Act + var memory = builder.Build(); + + // Assert + Assert.NotNull(memory); + } + + [Fact] + public void ItUsesProvidedLoggerFactory() + { + // Arrange + var loggerFactoryUsed = Mock.Of(); + var loggerFactoryUnused = Mock.Of(); + + // Act & Assert + var builder = new MemoryBuilder() + .WithLoggerFactory(loggerFactoryUsed) + .WithMemoryStore((loggerFactory) => + { + Assert.Same(loggerFactoryUsed, loggerFactory); + Assert.NotSame(loggerFactoryUnused, loggerFactory); + + return Mock.Of(); + }) + .WithTextEmbeddingGeneration((loggerFactory, httpHandlerFactory) => + { + Assert.Same(loggerFactoryUsed, loggerFactory); + Assert.NotSame(loggerFactoryUnused, loggerFactory); + + return Mock.Of(); + }) + .Build(); + } + + [Fact] + public void ItUsesProvidedHttpHandlerFactory() + { + // Arrange + var httpHandlerFactoryUsed = Mock.Of(); + var httpHandlerFactoryUnused = Mock.Of(); + + // Act & Assert + var builder = new MemoryBuilder() + .WithHttpHandlerFactory(httpHandlerFactoryUsed) + .WithMemoryStore((loggerFactory, httpHandlerFactory) => + { + Assert.Same(httpHandlerFactoryUsed, httpHandlerFactory); + Assert.NotSame(httpHandlerFactoryUnused, httpHandlerFactory); + + return Mock.Of(); + }) + .WithTextEmbeddingGeneration((loggerFactory, httpHandlerFactory) => + { + Assert.Same(httpHandlerFactoryUsed, httpHandlerFactory); + Assert.NotSame(httpHandlerFactoryUnused, httpHandlerFactory); + + return Mock.Of(); + }) + .Build(); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Memory/VolatileMemoryStoreTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Memory/VolatileMemoryStoreTests.cs similarity index 85% rename from dotnet/src/SemanticKernel.UnitTests/Memory/VolatileMemoryStoreTests.cs rename to dotnet/src/Plugins/Plugins.UnitTests/Memory/VolatileMemoryStoreTests.cs index 1a2f5ebc3f0e..4fb078a6db79 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Memory/VolatileMemoryStoreTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Memory/VolatileMemoryStoreTests.cs @@ -5,11 +5,12 @@ using System.Collections.Immutable; using System.Linq; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Plugins.Memory; using Xunit; -namespace SemanticKernel.UnitTests.Memory; +namespace SemanticKernel.Plugins.UnitTests.Memory; public class VolatileMemoryStoreTests { @@ -34,7 +35,7 @@ private IEnumerable CreateBatchRecords(int numRecords) id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); records = records.Append(testRecord); } @@ -44,7 +45,7 @@ private IEnumerable CreateBatchRecords(int numRecords) externalId: "test" + i, sourceName: "sourceName" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); records = records.Append(testRecord); } @@ -75,17 +76,13 @@ public async Task ItCanCreateAndGetCollectionAsync() } [Fact] - public async Task ItCannotCreateDuplicateCollectionAsync() + public async Task ItHandlesExceptionsWhenCreatingCollectionAsync() { // Arrange - string collection = "test_collection" + this._collectionNum; - this._collectionNum++; - - // Act - await this._db.CreateCollectionAsync(collection); + string? collection = null; // Assert - await Assert.ThrowsAsync(async () => await this._db.CreateCollectionAsync(collection)); + await Assert.ThrowsAsync(async () => await this._db.CreateCollectionAsync(collection!)); } [Fact] @@ -96,14 +93,14 @@ public async Task ItCannotInsertIntoNonExistentCollectionAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection" + this._collectionNum; this._collectionNum++; // Assert - await Assert.ThrowsAsync(async () => await this._db.UpsertAsync(collection, testRecord)); + await Assert.ThrowsAsync(async () => await this._db.UpsertAsync(collection, testRecord)); } [Fact] @@ -114,7 +111,7 @@ public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection" + this._collectionNum; @@ -129,8 +126,8 @@ public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() // Assert Assert.NotNull(actualDefault); Assert.NotNull(actualWithEmbedding); - Assert.Empty(actualDefault.Embedding.Vector); - Assert.NotEmpty(actualWithEmbedding.Embedding.Vector); + Assert.True(actualDefault.Embedding.IsEmpty); + Assert.False(actualWithEmbedding.Embedding.IsEmpty); Assert.NotEqual(testRecord, actualDefault); Assert.Equal(testRecord, actualWithEmbedding); } @@ -143,7 +140,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: null); string collection = "test_collection" + this._collectionNum; @@ -167,7 +164,7 @@ public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 }), + embedding: new float[] { 1, 2, 3 }, key: null, timestamp: DateTimeOffset.UtcNow); string collection = "test_collection" + this._collectionNum; @@ -192,12 +189,12 @@ public async Task UpsertReplacesExistingRecordWithSameIdAsync() id: commonId, text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); MemoryRecord testRecord2 = MemoryRecord.LocalRecord( id: commonId, text: "text2", description: "description2", - embedding: new Embedding(new float[] { 1, 2, 4 })); + embedding: new float[] { 1, 2, 4 }); string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -222,7 +219,7 @@ public async Task ExistingRecordCanBeRemovedAsync() id: "test", text: "text", description: "description", - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -282,7 +279,7 @@ public async Task ItCanListAllDatabaseCollectionsAsync() public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() { // Arrange - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -292,7 +289,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -300,7 +297,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -308,7 +305,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -316,7 +313,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -324,7 +321,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await this._db.UpsertAsync(collection, testRecord); // Act @@ -344,7 +341,7 @@ public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() { // Arrange - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection" + this._collectionNum; this._collectionNum++; await this._db.CreateCollectionAsync(collection); @@ -353,7 +350,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -361,7 +358,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -369,7 +366,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -377,7 +374,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -385,7 +382,7 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await this._db.UpsertAsync(collection, testRecord); // Act @@ -396,15 +393,15 @@ public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync( // Assert Assert.NotNull(topNResultDefault); Assert.NotNull(topNResultWithEmbedding); - Assert.Empty(topNResultDefault.Value.Item1.Embedding.Vector); - Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.Vector); + Assert.True(topNResultDefault.Value.Item1.Embedding.IsEmpty); + Assert.False(topNResultWithEmbedding.Value.Item1.Embedding.IsEmpty); } [Fact] public async Task GetNearestMatchAsyncReturnsExpectedAsync() { // Arrange - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; string collection = "test_collection" + this._collectionNum; this._collectionNum++; await this._db.CreateCollectionAsync(collection); @@ -413,7 +410,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -421,7 +418,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -1, -1 })); + embedding: new float[] { -1, -1, -1 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -429,7 +426,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 2, 3 })); + embedding: new float[] { 1, 2, 3 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -437,7 +434,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { -1, -2, -3 })); + embedding: new float[] { -1, -2, -3 }); _ = await this._db.UpsertAsync(collection, testRecord); i++; @@ -445,7 +442,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, -1, -2 })); + embedding: new float[] { 1, -1, -2 }); _ = await this._db.UpsertAsync(collection, testRecord); // Act @@ -462,7 +459,7 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() { // Arrange - var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + var compareEmbedding = new float[] { 1, 1, 1 }; int topN = 4; string collection = "test_collection" + this._collectionNum; this._collectionNum++; @@ -474,7 +471,7 @@ public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() id: "test" + i, text: "text" + i, description: "description" + i, - embedding: new Embedding(new float[] { 1, 1, 1 })); + embedding: new float[] { 1, 1, 1 }); _ = await this._db.UpsertAsync(collection, testRecord); } @@ -590,6 +587,6 @@ public async Task ItThrowsWhenDeletingNonExistentCollectionAsync() this._collectionNum++; // Act - await Assert.ThrowsAsync(() => this._db.DeleteCollectionAsync(collection)); + await Assert.ThrowsAsync(() => this._db.DeleteCollectionAsync(collection)); } } diff --git a/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/CalendarPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/CalendarPluginTests.cs new file mode 100644 index 000000000000..573e6c6c69a9 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/CalendarPluginTests.cs @@ -0,0 +1,240 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Plugins.MsGraph; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; +using Moq; +using SemanticKernel.UnitTests; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.MsGraph; + +public class CalendarPluginTests +{ + [Fact] + public async Task AddEventAsyncSucceedsAsync() + { + // Arrange + string anyContent = Guid.NewGuid().ToString(); + string anySubject = Guid.NewGuid().ToString(); + string anyLocation = Guid.NewGuid().ToString(); + DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); + DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); + string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + + CalendarEvent expected = new() + { + Subject = anySubject, + Location = anyLocation, + Attendees = anyAttendees + }; + + Mock connectorMock = new(); + connectorMock.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(expected); + + CalendarPlugin target = new(connectorMock.Object); + + // Act + var context = await FunctionHelpers.CallViaKernelAsync(target, "AddEvent", + ("input", anySubject), + ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), + ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), + ("location", anyLocation), + ("content", anyContent), + ("attendees", string.Join(";", anyAttendees))); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task AddEventAsyncWithoutLocationSucceedsAsync() + { + // Arrange + string anyContent = Guid.NewGuid().ToString(); + string anySubject = Guid.NewGuid().ToString(); + DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); + DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); + string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + + CalendarEvent expected = new() + { + Content = anyContent, + Subject = anySubject, + Attendees = anyAttendees, + Start = anyStartTime, + End = anyEndTime + }; + + Mock connectorMock = new(); + connectorMock.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(expected); + + CalendarPlugin target = new(connectorMock.Object); + + // Act + var context = await FunctionHelpers.CallViaKernelAsync(target, "AddEvent", + ("input", anySubject), + ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), + ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), + ("content", anyContent), + ("attendees", string.Join(";", anyAttendees))); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task AddEventAsyncWithoutContentSucceedsAsync() + { + // Arrange + string anySubject = Guid.NewGuid().ToString(); + string anyLocation = Guid.NewGuid().ToString(); + DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); + DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); + string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + + CalendarEvent expected = new() + { + Subject = anySubject, + Start = anyStartTime, + End = anyEndTime, + Location = anyLocation, + Attendees = anyAttendees + }; + + Mock connectorMock = new(); + connectorMock.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(expected); + + CalendarPlugin target = new(connectorMock.Object); + + // Act + var context = await FunctionHelpers.CallViaKernelAsync(target, "AddEvent", + ("input", anySubject), + ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), + ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), + ("location", anyLocation), + ("attendees", string.Join(";", anyAttendees))); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task AddEventAsyncWithoutAttendeesSucceedsAsync() + { + // Arrange + string anyContent = Guid.NewGuid().ToString(); + string anySubject = Guid.NewGuid().ToString(); + string anyLocation = Guid.NewGuid().ToString(); + DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); + DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); + + CalendarEvent expected = new() + { + Subject = anySubject, + Start = anyStartTime, + End = anyEndTime, + Content = anyContent, + Location = anyLocation + }; + + Mock connectorMock = new(); + connectorMock.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(expected); + + CalendarPlugin target = new(connectorMock.Object); + + // Act + var context = await FunctionHelpers.CallViaKernelAsync(target, "AddEvent", + ("input", anySubject), + ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), + ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), + ("location", anyLocation), + ("content", anyContent)); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task AddEventAsyncWithoutStartFailsAsync() + { + // Arrange + string anyContent = Guid.NewGuid().ToString(); + string anySubject = Guid.NewGuid().ToString(); + string anyLocation = Guid.NewGuid().ToString(); + DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); + string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + + Mock connectorMock = new(); + + CalendarPlugin target = new(connectorMock.Object); + + // Act and Assert + await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(target, "AddEvent", + ("input", anySubject), + ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), + ("location", anyLocation), + ("content", anyContent), + ("attendees", string.Join(";", anyAttendees))) + ); + } + + [Fact] + public async Task AddEventAsyncWithoutEndFailsAsync() + { + // Arrange + string anyContent = Guid.NewGuid().ToString(); + string anySubject = Guid.NewGuid().ToString(); + string anyLocation = Guid.NewGuid().ToString(); + DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); + string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + + Mock connectorMock = new(); + + CalendarPlugin target = new(connectorMock.Object); + + // Act + await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(target, "AddEvent", + ("input", anySubject), + ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), + ("location", anyLocation), + ("content", anyContent), + ("attendees", string.Join(";", anyAttendees))) + ); + } + + [Fact] + public async Task AddEventAsyncWithoutSubjectFailsAsync() + { + // Arrange + string anyContent = Guid.NewGuid().ToString(); + string anyLocation = Guid.NewGuid().ToString(); + DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); + DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); + string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + + Mock connectorMock = new(); + + CalendarPlugin target = new(connectorMock.Object); + + // Act & Assert + var ex = await Assert.ThrowsAsync(() => FunctionHelpers.CallViaKernelAsync(target, "AddEvent", + ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), + ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), + ("location", anyLocation), + ("content", anyContent), + ("attendees", string.Join(";", anyAttendees))) + ); + + Assert.True(ex.InnerException is ArgumentException); + Assert.Equal("input", ((ArgumentException)ex.InnerException).ParamName); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/CloudDrivePluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/CloudDrivePluginTests.cs new file mode 100644 index 000000000000..389c72663239 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/CloudDrivePluginTests.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Plugins.MsGraph; +using Moq; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.MsGraph; + +public class CloudDrivePluginTests +{ + [Fact] + public async Task UploadSmallFileAsyncSucceedsAsync() + { + // Arrange + string anyFilePath = Guid.NewGuid().ToString(); + + Mock connectorMock = new(); + connectorMock.Setup(c => c.UploadSmallFileAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + CloudDrivePlugin target = new(connectorMock.Object); + + // Act + await target.UploadFileAsync(anyFilePath, Guid.NewGuid().ToString()); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task CreateLinkAsyncSucceedsAsync() + { + // Arrange + string anyFilePath = Guid.NewGuid().ToString(); + string anyLink = Guid.NewGuid().ToString(); + + Mock connectorMock = new(); + connectorMock.Setup(c => c.CreateShareLinkAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(anyLink); + + CloudDrivePlugin target = new(connectorMock.Object); + + // Act + string actual = await target.CreateLinkAsync(anyFilePath); + + // Assert + Assert.Equal(anyLink, actual); + connectorMock.VerifyAll(); + } + + [Fact] + public async Task GetFileContentAsyncSucceedsAsync() + { + string anyFilePath = Guid.NewGuid().ToString(); + string expectedContent = Guid.NewGuid().ToString(); + using MemoryStream expectedStream = new(Encoding.UTF8.GetBytes(expectedContent)); + + // Arrange + Mock connectorMock = new(); + connectorMock.Setup(c => c.GetFileContentStreamAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(expectedStream); + + CloudDrivePlugin target = new(connectorMock.Object); + + // Act + string actual = await target.GetFileContentAsync(anyFilePath); + + // Assert + Assert.Equal(expectedContent, actual); + connectorMock.VerifyAll(); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/EmailPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/EmailPluginTests.cs new file mode 100644 index 000000000000..f2f27419b2ea --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/EmailPluginTests.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Plugins.MsGraph; +using Moq; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.MsGraph; + +public class EmailPluginTests +{ + [Fact] + public async Task SendEmailAsyncSucceedsAsync() + { + // Arrange + Mock connectorMock = new(); + connectorMock.Setup(c => c.SendEmailAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + EmailPlugin target = new(connectorMock.Object); + + string anyContent = Guid.NewGuid().ToString(); + string anySubject = Guid.NewGuid().ToString(); + string anyRecipient = Guid.NewGuid().ToString(); + + // Act + await target.SendEmailAsync(anyContent, anyRecipient, anySubject); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task SendEmailAsyncNoRecipientFailsAsync() + { + // Arrange + Mock connectorMock = new(); + EmailPlugin target = new(connectorMock.Object); + + string anyContent = Guid.NewGuid().ToString(); + string anySubject = Guid.NewGuid().ToString(); + + // Act/Assert + await Assert.ThrowsAnyAsync(() => + target.SendEmailAsync(anyContent, null!, anySubject)); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task SendEmailAsyncNoSubjectFailsAsync() + { + // Arrange + Mock connectorMock = new(); + EmailPlugin target = new(connectorMock.Object); + + string anyContent = Guid.NewGuid().ToString(); + string anyRecipient = Guid.NewGuid().ToString(); + + // Act/Assert + await Assert.ThrowsAnyAsync(() => + target.SendEmailAsync(anyContent, anyRecipient, null!)); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task GetMyEmailAddressAsyncSucceedsAsync() + { + // Arrange + string anyEmailAddress = Guid.NewGuid().ToString(); + Mock connectorMock = new(); + connectorMock.Setup(c => c.GetMyEmailAddressAsync(It.IsAny())) + .ReturnsAsync(anyEmailAddress); + + EmailPlugin target = new(connectorMock.Object); + + // Act + string actual = await target.GetMyEmailAddressAsync(); + + // Assert + Assert.Equal(anyEmailAddress, actual); + connectorMock.VerifyAll(); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/OrganizationHierarchyPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/OrganizationHierarchyPluginTests.cs new file mode 100644 index 000000000000..9f90a5b9079c --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/OrganizationHierarchyPluginTests.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Plugins.MsGraph; +using Moq; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.MsGraph; + +public class OrganizationHierarchyPluginTests +{ + [Fact] + public async Task GetMyDirectReportsEmailAsyncSucceedsAsync() + { + // Arrange + string[] anyDirectReportsEmail = { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; + Mock connectorMock = new(); + connectorMock.Setup(c => c.GetDirectReportsEmailAsync(It.IsAny())).ReturnsAsync(anyDirectReportsEmail); + OrganizationHierarchyPlugin target = new(connectorMock.Object); + + // Act + string actual = await target.GetMyDirectReportsEmailAsync(); + + // Assert + var emails = JsonSerializer.Deserialize>(actual); + Assert.NotNull(emails); + foreach (string directReportEmail in anyDirectReportsEmail) + { + Assert.Contains(directReportEmail, emails); + } + + connectorMock.VerifyAll(); + } + + [Fact] + public async Task GetMyManagerEmailAsyncSucceedsAsync() + { + // Arrange + string anyManagerEmail = Guid.NewGuid().ToString(); + Mock connectorMock = new(); + connectorMock.Setup(c => c.GetManagerEmailAsync(It.IsAny())).ReturnsAsync(anyManagerEmail); + OrganizationHierarchyPlugin target = new(connectorMock.Object); + + // Act + string actual = await target.GetMyManagerEmailAsync(); + + // Assert + Assert.Equal(anyManagerEmail, actual); + connectorMock.VerifyAll(); + } + + [Fact] + public async Task GetMyManagerNameAsyncSucceedsAsync() + { + // Arrange + string anyManagerName = Guid.NewGuid().ToString(); + Mock connectorMock = new(); + connectorMock.Setup(c => c.GetManagerNameAsync(It.IsAny())).ReturnsAsync(anyManagerName); + OrganizationHierarchyPlugin target = new(connectorMock.Object); + + // Act + string actual = await target.GetMyManagerNameAsync(); + + // Assert + Assert.Equal(anyManagerName, actual); + connectorMock.VerifyAll(); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/TaskListPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/TaskListPluginTests.cs new file mode 100644 index 000000000000..226410ac2768 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/MsGraph/TaskListPluginTests.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Plugins.MsGraph; +using Microsoft.SemanticKernel.Plugins.MsGraph.Models; +using Moq; +using Xunit; +using static Microsoft.SemanticKernel.Plugins.MsGraph.TaskListPlugin; + +namespace SemanticKernel.Plugins.UnitTests.MsGraph; + +public class TaskListPluginTests +{ + private readonly TaskManagementTaskList _anyTaskList = new( + id: Guid.NewGuid().ToString(), + name: Guid.NewGuid().ToString()); + + private readonly TaskManagementTask _anyTask = new( + id: Guid.NewGuid().ToString(), + title: Guid.NewGuid().ToString(), + reminder: (DateTimeOffset.Now + TimeSpan.FromDays(1)).ToString("o"), + due: DateTimeOffset.Now.ToString("o"), + isCompleted: false); + + [Fact] + public async Task AddTaskAsyncNoReminderSucceedsAsync() + { + // Arrange + string anyTitle = Guid.NewGuid().ToString(); + + Mock connectorMock = new(); + connectorMock.Setup(c => c.GetDefaultTaskListAsync(It.IsAny())) + .ReturnsAsync(this._anyTaskList); + + connectorMock.Setup(c => c.AddTaskAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(this._anyTask); + + TaskListPlugin target = new(connectorMock.Object); + + // Act + await target.AddTaskAsync(anyTitle); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task AddTaskAsyncWithReminderSucceedsAsync() + { + // Arrange + string anyTitle = Guid.NewGuid().ToString(); + + Mock connectorMock = new(); + connectorMock.Setup(c => c.GetDefaultTaskListAsync(It.IsAny())) + .ReturnsAsync(this._anyTaskList); + + connectorMock.Setup(c => c.AddTaskAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(this._anyTask); + + string anyReminder = (DateTimeOffset.Now + TimeSpan.FromHours(1)).ToString("o"); + + TaskListPlugin target = new(connectorMock.Object); + + // Act + await target.AddTaskAsync(anyTitle, anyReminder); + + // Assert + connectorMock.VerifyAll(); + } + + [Fact] + public async Task AddTaskAsyncNoDefaultTaskListFailsAsync() + { + // Arrange + string anyTitle = Guid.NewGuid().ToString(); + + Mock connectorMock = new(); +#pragma warning disable CS8600 // Converting null literal or possible null value to non-nullable type. + connectorMock.Setup(c => c.GetDefaultTaskListAsync(It.IsAny())) + .ReturnsAsync((TaskManagementTaskList)null); +#pragma warning restore CS8600 // Converting null literal or possible null value to non-nullable type. + + string anyReminder = (DateTimeOffset.Now + TimeSpan.FromHours(1)).ToString("o"); + + TaskListPlugin target = new(connectorMock.Object); + + // Act/Assert + await Assert.ThrowsAnyAsync(() => + target.AddTaskAsync(anyTitle, anyReminder)); + + // Assert + connectorMock.VerifyAll(); + } + + [Theory] + [InlineData(DayOfWeek.Sunday)] + [InlineData(DayOfWeek.Monday)] + [InlineData(DayOfWeek.Tuesday)] + [InlineData(DayOfWeek.Wednesday)] + [InlineData(DayOfWeek.Thursday)] + [InlineData(DayOfWeek.Friday)] + [InlineData(DayOfWeek.Saturday)] + public void GetNextDayOfWeekIsCorrect(DayOfWeek dayOfWeek) + { + // Arrange + DateTimeOffset today = new(DateTime.Today); + TimeSpan timeOfDay = TimeSpan.FromHours(13); + + // Act + DateTimeOffset actual = GetNextDayOfWeek(dayOfWeek, timeOfDay); + + // Assert + Assert.Equal(dayOfWeek, actual.DayOfWeek); + Assert.True(today.ToUnixTimeSeconds() < actual.ToUnixTimeSeconds()); + Assert.Equal(timeOfDay.Hours, actual.Hour); + } +} diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Plugins.UnitTests.csproj b/dotnet/src/Plugins/Plugins.UnitTests/Plugins.UnitTests.csproj new file mode 100644 index 000000000000..ef6f62b2800a --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Plugins.UnitTests.csproj @@ -0,0 +1,38 @@ + + + + SemanticKernel.Plugins.UnitTests + SemanticKernel.Plugins.UnitTests + net6.0 + LatestMajor + true + enable + disable + false + CA2007,VSTHRD111 + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Web/SearchUrlSkillTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Web/SearchUrlSkillTests.cs new file mode 100644 index 000000000000..8a47bfba536b --- /dev/null +++ b/dotnet/src/Plugins/Plugins.UnitTests/Web/SearchUrlSkillTests.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Encodings.Web; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Plugins.Web; +using Xunit; + +namespace SemanticKernel.Plugins.UnitTests.Web; + +public class SearchUrlPluginTests +{ + private const string AnyInput = ""; + private readonly string _encodedInput = UrlEncoder.Default.Encode(AnyInput); + + [Fact] + public void ItCanBeInstantiated() + { + // Act - Assert no exception occurs + var _ = new SearchUrlPlugin(); + } + + [Fact] + public void ItCanBeImported() + { + // Arrange + IKernel kernel = Kernel.Builder.Build(); + + // Act - Assert no exception occurs e.g. due to reflection + kernel.ImportFunctions(new SearchUrlPlugin(), "search"); + } + + [Fact] + public void AmazonSearchUrlSucceeds() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.AmazonSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.amazon.com/s?k={this._encodedInput}", actual); + } + + [Fact] + public void BingSearchUrlSucceeds() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.BingSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.bing.com/search?q={this._encodedInput}", actual); + } + + [Fact] + public void BingImagesSearchUrlSucceeds() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.BingImagesSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.bing.com/images/search?q={this._encodedInput}", actual); + } + + [Fact] + public void BingMapsSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.BingMapsSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.bing.com/maps?q={this._encodedInput}", actual); + } + + [Fact] + public void BingShoppingSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.BingShoppingSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.bing.com/shop?q={this._encodedInput}", actual); + } + + [Fact] + public void BingNewsSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.BingNewsSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.bing.com/news/search?q={this._encodedInput}", actual); + } + + [Fact] + public void BingTravelSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.BingTravelSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.bing.com/travel/search?q={this._encodedInput}", actual); + } + + [Fact] + public void FacebookSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.FacebookSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.facebook.com/search/top/?q={this._encodedInput}", actual); + } + + [Fact] + public void GitHubSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.GitHubSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://github.com/search?q={this._encodedInput}", actual); + } + + [Fact] + public void LinkedInSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.LinkedInSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://www.linkedin.com/search/results/index/?keywords={this._encodedInput}", actual); + } + + [Fact] + public void TwitterSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.TwitterSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://twitter.com/search?q={this._encodedInput}", actual); + } + + [Fact] + public void WikipediaSearchUrl() + { + // Arrange + var plugin = new SearchUrlPlugin(); + + // Act + string actual = plugin.WikipediaSearchUrl(AnyInput); + + // Assert + Assert.Equal($"https://wikipedia.org/w/index.php?search={this._encodedInput}", actual); + } +} diff --git a/dotnet/src/Skills/Skills.UnitTests/Web/WebSearchEngineSkillTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Web/WebSearchEngineSkillTests.cs similarity index 78% rename from dotnet/src/Skills/Skills.UnitTests/Web/WebSearchEngineSkillTests.cs rename to dotnet/src/Plugins/Plugins.UnitTests/Web/WebSearchEngineSkillTests.cs index 22ebb25d08b6..e184ec3648b6 100644 --- a/dotnet/src/Skills/Skills.UnitTests/Web/WebSearchEngineSkillTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Web/WebSearchEngineSkillTests.cs @@ -4,13 +4,13 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.Web; +using Microsoft.SemanticKernel.Plugins.Web; using Moq; using Xunit; -namespace SemanticKernel.Skills.UnitTests.Web; +namespace SemanticKernel.Plugins.UnitTests.Web; -public sealed class WebSearchEngineSkillTests +public sealed class WebSearchEnginePluginTests { [Fact] public async Task SearchAsyncSucceedsAsync() @@ -22,7 +22,7 @@ public async Task SearchAsyncSucceedsAsync() connectorMock.Setup(c => c.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(expected); - WebSearchEngineSkill target = new(connectorMock.Object); + WebSearchEnginePlugin target = new(connectorMock.Object); string anyQuery = Guid.NewGuid().ToString(); diff --git a/dotnet/src/Plugins/Plugins.Web/Bing/BingConnector.cs b/dotnet/src/Plugins/Plugins.Web/Bing/BingConnector.cs new file mode 100644 index 000000000000..fb8fddd4ecdd --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/Bing/BingConnector.cs @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; + +namespace Microsoft.SemanticKernel.Plugins.Web.Bing; + +/// +/// Bing API connector. +/// +public sealed class BingConnector : IWebSearchEngineConnector +{ + private readonly ILogger _logger; + private readonly HttpClient _httpClient; + private readonly string? _apiKey; + + /// + /// Initializes a new instance of the class. + /// + /// The API key to authenticate the connector. + /// The to use for logging. If null, no logging will be performed. + public BingConnector(string apiKey, ILoggerFactory? loggerFactory = null) : + this(apiKey, new HttpClient(NonDisposableHttpClientHandler.Instance, false), loggerFactory) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The API key to authenticate the connector. + /// The HTTP client to use for making requests. + /// The to use for logging. If null, no logging will be performed. + public BingConnector(string apiKey, HttpClient httpClient, ILoggerFactory? loggerFactory = null) + { + Verify.NotNull(httpClient); + + this._apiKey = apiKey; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(BingConnector)) : NullLogger.Instance; + this._httpClient = httpClient; + this._httpClient.DefaultRequestHeaders.Add("User-Agent", Telemetry.HttpUserAgent); + } + + /// + public async Task> SearchAsync(string query, int count = 1, int offset = 0, CancellationToken cancellationToken = default) + { + if (count is <= 0 or >= 50) + { + throw new ArgumentOutOfRangeException(nameof(count), count, $"{nameof(count)} value must be greater than 0 and less than 50."); + } + + if (offset < 0) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + Uri uri = new($"https://api.bing.microsoft.com/v7.0/search?q={Uri.EscapeDataString(query)}&count={count}&offset={offset}"); + + this._logger.LogDebug("Sending request: {Uri}", uri); + + using HttpResponseMessage response = await this.SendGetRequestAsync(uri, cancellationToken).ConfigureAwait(false); + + this._logger.LogDebug("Response received: {StatusCode}", response.StatusCode); + + string json = await response.Content.ReadAsStringWithExceptionMappingAsync().ConfigureAwait(false); + + // Sensitive data, logging as trace, disabled by default + this._logger.LogTrace("Response content received: {Data}", json); + + BingSearchResponse? data = JsonSerializer.Deserialize(json); + + WebPage[]? results = data?.WebPages?.Value; + + return results == null ? Enumerable.Empty() : results.Select(x => x.Snippet); + } + + /// + /// Sends a GET request to the specified URI. + /// + /// The URI to send the request to. + /// A cancellation token to cancel the request. + /// A representing the response from the request. + private async Task SendGetRequestAsync(Uri uri, CancellationToken cancellationToken = default) + { + using var httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, uri); + + if (!string.IsNullOrEmpty(this._apiKey)) + { + httpRequestMessage.Headers.Add("Ocp-Apim-Subscription-Key", this._apiKey); + } + + return await this._httpClient.SendWithSuccessCheckAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); + } + + [SuppressMessage("Performance", "CA1812:Internal class that is apparently never instantiated", + Justification = "Class is instantiated through deserialization.")] + private sealed class BingSearchResponse + { + [JsonPropertyName("webPages")] + public WebPages? WebPages { get; set; } + } + + [SuppressMessage("Performance", "CA1812:Internal class that is apparently never instantiated", + Justification = "Class is instantiated through deserialization.")] + private sealed class WebPages + { + [JsonPropertyName("value")] + public WebPage[]? Value { get; set; } + } + + [SuppressMessage("Performance", "CA1812:Internal class that is apparently never instantiated", + Justification = "Class is instantiated through deserialization.")] + private sealed class WebPage + { + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("url")] + public string Url { get; set; } = string.Empty; + + [JsonPropertyName("snippet")] + public string Snippet { get; set; } = string.Empty; + } +} diff --git a/dotnet/src/Plugins/Plugins.Web/Google/GoogleConnector.cs b/dotnet/src/Plugins/Plugins.Web/Google/GoogleConnector.cs new file mode 100644 index 000000000000..2720b3c51838 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/Google/GoogleConnector.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Google.Apis.CustomSearchAPI.v1; +using Google.Apis.Services; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; + +namespace Microsoft.SemanticKernel.Plugins.Web.Google; + +/// +/// Google search connector. +/// Provides methods to search using Google Custom Search API. +/// +public sealed class GoogleConnector : IWebSearchEngineConnector, IDisposable +{ + private readonly ILogger _logger; + private readonly CustomSearchAPIService _search; + private readonly string? _searchEngineId; + + /// + /// Initializes a new instance of the class. + /// + /// Google Custom Search API (looks like "ABcdEfG1...") + /// Google Search Engine ID (looks like "a12b345...") + /// The to use for logging. If null, no logging will be performed. + public GoogleConnector( + string apiKey, + string searchEngineId, + ILoggerFactory? loggerFactory = null) : this(new BaseClientService.Initializer { ApiKey = apiKey }, searchEngineId, loggerFactory) + { + Verify.NotNullOrWhiteSpace(apiKey); + } + + /// + /// Initializes a new instance of the class. + /// + /// The connector initializer + /// Google Search Engine ID (looks like "a12b345...") + /// The to use for logging. If null, no logging will be performed. + public GoogleConnector( + BaseClientService.Initializer initializer, + string searchEngineId, + ILoggerFactory? loggerFactory = null) + { + Verify.NotNull(initializer); + Verify.NotNullOrWhiteSpace(searchEngineId); + + this._search = new CustomSearchAPIService(initializer); + this._searchEngineId = searchEngineId; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(GoogleConnector)) : NullLogger.Instance; + } + + /// + public async Task> SearchAsync( + string query, + int count, + int offset, + CancellationToken cancellationToken) + { + if (count is <= 0 or > 10) + { + throw new ArgumentOutOfRangeException(nameof(count), count, $"{nameof(count)} value must be must be greater than 0 and less than or equals 10."); + } + + if (offset < 0) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + var search = this._search.Cse.List(); + search.Cx = this._searchEngineId; + search.Q = query; + search.Num = count; + search.Start = offset; + + var results = await search.ExecuteAsync(cancellationToken).ConfigureAwait(false); + + return results.Items.Select(item => item.Snippet); + } + + /// + /// Disposes the resources used by the instance. + /// + /// True to release both managed and unmanaged resources; false to release only unmanaged resources. + private void Dispose(bool disposing) + { + if (disposing) + { + this._search.Dispose(); + } + } + + /// + /// Disposes the instance. + /// + public void Dispose() + { + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } +} diff --git a/dotnet/src/Skills/Skills.Web/IWebSearchEngineConnector.cs b/dotnet/src/Plugins/Plugins.Web/IWebSearchEngineConnector.cs similarity index 94% rename from dotnet/src/Skills/Skills.Web/IWebSearchEngineConnector.cs rename to dotnet/src/Plugins/Plugins.Web/IWebSearchEngineConnector.cs index e9b084d20c5e..c027c30f4058 100644 --- a/dotnet/src/Skills/Skills.Web/IWebSearchEngineConnector.cs +++ b/dotnet/src/Plugins/Plugins.Web/IWebSearchEngineConnector.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.SemanticKernel.Skills.Web; +namespace Microsoft.SemanticKernel.Plugins.Web; /// /// Web search engine connector interface. diff --git a/dotnet/src/Plugins/Plugins.Web/Plugins.Web.csproj b/dotnet/src/Plugins/Plugins.Web/Plugins.Web.csproj new file mode 100644 index 000000000000..d8154e524997 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/Plugins.Web.csproj @@ -0,0 +1,29 @@ + + + + + Microsoft.SemanticKernel.Plugins.Web + $(AssemblyName) + netstandard2.0 + + + + + + + + Semantic Kernel - Web Plugins + Semantic Kernel web plugins: search the web, download files, etc. + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/Plugins/Plugins.Web/SearchUrlPlugin.cs b/dotnet/src/Plugins/Plugins.Web/SearchUrlPlugin.cs new file mode 100644 index 000000000000..c7e2abd64e93 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/SearchUrlPlugin.cs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Text.Encodings.Web; + +namespace Microsoft.SemanticKernel.Plugins.Web; + +/// +/// Get search URLs for various websites +/// +[SuppressMessage("Design", "CA1055:URI return values should not be strings", Justification = "Semantic Kernel operates on strings")] +public sealed class SearchUrlPlugin +{ + /** + * Amazon Search URLs + */ + /// + /// Get search URL for Amazon + /// + [SKFunction, Description("Return URL for Amazon search query")] + public string AmazonSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.amazon.com/s?k={encoded}"; + } + + /** + * Bing Search URLs + */ + /// + /// Get search URL for Bing + /// + [SKFunction, Description("Return URL for Bing search query.")] + public string BingSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.bing.com/search?q={encoded}"; + } + + /// + /// Get search URL for Bing Images + /// + [SKFunction, Description("Return URL for Bing Images search query.")] + public string BingImagesSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.bing.com/images/search?q={encoded}"; + } + + /// + /// Get search URL for Bing Maps + /// + [SKFunction, Description("Return URL for Bing Maps search query.")] + public string BingMapsSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.bing.com/maps?q={encoded}"; + } + + /// + /// Get search URL for Bing Shopping + /// + [SKFunction, Description("Return URL for Bing Shopping search query.")] + public string BingShoppingSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.bing.com/shop?q={encoded}"; + } + + /// + /// Get search URL for Bing News + /// + [SKFunction, Description("Return URL for Bing News search query.")] + public string BingNewsSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.bing.com/news/search?q={encoded}"; + } + + /// + /// Get search URL for Bing Travel + /// + [SKFunction, Description("Return URL for Bing Travel search query.")] + public string BingTravelSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.bing.com/travel/search?q={encoded}"; + } + + /** + * Facebook Search URLs + */ + /// + /// Get search URL for Facebook + /// + [SKFunction, Description("Return URL for Facebook search query.")] + public string FacebookSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.facebook.com/search/top/?q={encoded}"; + } + + /** + * GitHub Search URLs + */ + /// + /// Get search URL for GitHub + /// + [SKFunction, Description("Return URL for GitHub search query.")] + public string GitHubSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://github.com/search?q={encoded}"; + } + + /** + * LinkedIn Search URLs + */ + /// + /// Get search URL for LinkedIn + /// + [SKFunction, Description("Return URL for LinkedIn search query.")] + public string LinkedInSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://www.linkedin.com/search/results/index/?keywords={encoded}"; + } + + /** + * Twitter Search URLs + */ + /// + /// Get search URL for Twitter + /// + [SKFunction, Description("Return URL for Twitter search query.")] + public string TwitterSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://twitter.com/search?q={encoded}"; + } + + /** + * Wikipedia Search URLs + */ + /// + /// Get search URL for Wikipedia + /// + [SKFunction, Description("Return URL for Wikipedia search query.")] + public string WikipediaSearchUrl([Description("Text to search for")] string query) + { + string encoded = UrlEncoder.Default.Encode(query); + return $"https://wikipedia.org/w/index.php?search={encoded}"; + } +} diff --git a/dotnet/src/Plugins/Plugins.Web/WebFileDownloadPlugin.cs b/dotnet/src/Plugins/Plugins.Web/WebFileDownloadPlugin.cs new file mode 100644 index 000000000000..a231b9e3ec6b --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/WebFileDownloadPlugin.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Microsoft.SemanticKernel.Plugins.Web; + +/// +/// Plugin to download web files. +/// +public sealed class WebFileDownloadPlugin +{ + /// + /// Plugin parameter: where to save file. + /// + public const string FilePathParamName = "filePath"; + + private readonly ILogger _logger; + private readonly HttpClient _httpClient; + + /// + /// Initializes a new instance of the class. + /// + /// The to use for logging. If null, no logging will be performed. + public WebFileDownloadPlugin(ILoggerFactory? loggerFactory = null) : + this(new HttpClient(NonDisposableHttpClientHandler.Instance, false), loggerFactory) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The HTTP client to use for making requests. + /// The to use for logging. If null, no logging will be performed. + public WebFileDownloadPlugin(HttpClient httpClient, ILoggerFactory? loggerFactory = null) + { + this._httpClient = httpClient; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(WebFileDownloadPlugin)) : NullLogger.Instance; + } + + /// + /// Downloads a file to a local file path. + /// + /// URI of file to download + /// Path where to save file locally + /// The token to use to request cancellation. + /// Task. + /// Thrown when the location where to download the file is not provided + [SKFunction, Description("Downloads a file to local storage")] + public async Task DownloadToFileAsync( + [Description("URL of file to download")] Uri url, + [Description("Path where to save file locally")] string filePath, + CancellationToken cancellationToken = default) + { + this._logger.LogDebug($"{nameof(this.DownloadToFileAsync)} got called"); + + this._logger.LogDebug("Sending GET request for {0}", url); + + using HttpRequestMessage request = new(HttpMethod.Get, url); + + using HttpResponseMessage response = await this._httpClient.SendWithSuccessCheckAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + this._logger.LogDebug("Response received: {0}", response.StatusCode); + + using Stream webStream = await response.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false); + using FileStream outputFileStream = new(Environment.ExpandEnvironmentVariables(filePath), FileMode.Create); + + await webStream.CopyToAsync(outputFileStream, 81920 /*same value used by default*/, cancellationToken).ConfigureAwait(false); + } +} diff --git a/dotnet/src/Plugins/Plugins.Web/WebSearchEnginePlugin.cs b/dotnet/src/Plugins/Plugins.Web/WebSearchEnginePlugin.cs new file mode 100644 index 000000000000..ce480ed58651 --- /dev/null +++ b/dotnet/src/Plugins/Plugins.Web/WebSearchEnginePlugin.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.SemanticKernel.Plugins.Web; + +/// +/// Web search engine plugin (e.g. Bing). +/// +public sealed class WebSearchEnginePlugin +{ + /// + /// The count parameter name. + /// + public const string CountParam = "count"; + + /// + /// The offset parameter name. + /// + public const string OffsetParam = "offset"; + + private readonly IWebSearchEngineConnector _connector; + + /// + /// Initializes a new instance of the class. + /// + /// The web search engine connector. + public WebSearchEnginePlugin(IWebSearchEngineConnector connector) + { + this._connector = connector; + } + + /// + /// Performs a web search using the provided query, count, and offset. + /// + /// The text to search for. + /// The number of results to return. Default is 1. + /// The number of results to skip. Default is 0. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that represents the asynchronous operation. The value of the TResult parameter contains the search results as a string. + [SKFunction, Description("Perform a web search.")] + public async Task SearchAsync( + [Description("Search query")] string query, + [Description("Number of results")] int count = 10, + [Description("Number of results to skip")] int offset = 0, + CancellationToken cancellationToken = default) + { + var results = await this._connector.SearchAsync(query, count, offset, cancellationToken).ConfigureAwait(false); + if (!results.Any()) + { + throw new InvalidOperationException("Failed to get a response from the web search engine."); + } + + return count == 1 + ? results.FirstOrDefault() ?? string.Empty + : JsonSerializer.Serialize(results); + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/AIException.cs b/dotnet/src/SemanticKernel.Abstractions/AI/AIException.cs deleted file mode 100644 index 2239a83f9591..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/AI/AIException.cs +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.AI; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown for errors related to AI logic. -/// -public class AIException : SKException -{ - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public AIException(ErrorCodes errorCode, string? message) - : this(errorCode, message, detail: null, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public AIException(ErrorCodes errorCode, string? message, Exception? innerException) - : this(errorCode, message, detail: null, innerException) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// A string that provides additional details about the error. - public AIException(ErrorCodes errorCode, string? message, string? detail) - : this(errorCode, message, detail, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, additional details, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// A string that provides additional details about the error. - /// The exception that is the cause of the current exception. - public AIException(ErrorCodes errorCode, string? message = null, string? detail = null, Exception? innerException = null) - : base(GetDefaultMessage(errorCode, message), innerException) - { - this.ErrorCode = errorCode; - this.Detail = detail; - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// - /// Gets the extended details for this exception. - /// - public string? Detail { get; } - - /// Translate the error code into a default message. - /// The error code. - /// Default error message if nothing available. - private static string GetDefaultMessage(ErrorCodes errorCode, string? defaultMessage) - { - string description = errorCode switch - { - ErrorCodes.NoResponse => "No response", - ErrorCodes.AccessDenied => "Access denied", - ErrorCodes.InvalidRequest => "Invalid request", - ErrorCodes.InvalidResponseContent => "Invalid response content", - ErrorCodes.Throttling => "Throttling", - ErrorCodes.RequestTimeout => "Request timeout", - ErrorCodes.ServiceError => "Service error", - ErrorCodes.ModelNotAvailable => "Model not available", - ErrorCodes.InvalidConfiguration => "Invalid configuration", - ErrorCodes.FunctionTypeNotSupported => "Function type not supported", - _ => $"Unknown error ({errorCode:G})", - }; - - return defaultMessage is not null ? $"{description}: {defaultMessage}" : description; - } - - /// - /// Possible error codes for exceptions - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError = -1, - - /// - /// No response. - /// - NoResponse, - - /// - /// Access is denied. - /// - AccessDenied, - - /// - /// The request was invalid. - /// - InvalidRequest, - - /// - /// The content of the response was invalid. - /// - InvalidResponseContent, - - /// - /// The request was throttled. - /// - Throttling, - - /// - /// The request timed out. - /// - RequestTimeout, - - /// - /// There was an error in the service. - /// - ServiceError, - - /// - /// The requested model is not available. - /// - ModelNotAvailable, - - /// - /// The supplied configuration was invalid. - /// - InvalidConfiguration, - - /// - /// The function is not supported. - /// - FunctionTypeNotSupported, - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/AIFunctionResultExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/AIFunctionResultExtensions.cs new file mode 100644 index 000000000000..a443b25ccd7e --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/AI/AIFunctionResultExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.AI; + +/// +/// Class with extension methods related to AI logic for class. +/// +public static class AIFunctionResultExtensions +{ + /// + /// Function result metadata key for records. + /// + public const string ModelResultsMetadataKey = "ModelResults"; + + /// + /// Returns collection of records from metadata. + /// + /// Instance of class. + public static IReadOnlyCollection? GetModelResults(this FunctionResult result) + { + if (result.TryGetMetadataValue(ModelResultsMetadataKey, out IReadOnlyCollection? modelResults)) + { + return modelResults; + } + + return null; + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/AIRequestSettings.cs b/dotnet/src/SemanticKernel.Abstractions/AI/AIRequestSettings.cs new file mode 100644 index 000000000000..4856b2adc563 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/AI/AIRequestSettings.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.AI.TextCompletion; + +namespace Microsoft.SemanticKernel.AI; + +/// +/// Request settings for an AI request. +/// Implementors of or can extend this +/// if the service they are calling supports additional properties. For an example please reference +/// the Microsoft.SemanticKernel.Connectors.AI.OpenAI.OpenAIRequestSettings implementation. +/// +public class AIRequestSettings +{ + private Dictionary? _extensionData; + + /// + /// Service identifier. + /// This identifies a service and is set when the AI service is registered. + /// + [JsonPropertyName("service_id")] + public string? ServiceId { get; set; } = null; + + /// + /// Model identifier. + /// This identifies the AI model these settings are configured for e.g., gpt-4, gpt-3.5-turbo + /// + [JsonPropertyName("model_id")] + public string? ModelId { get; set; } = null; + + /// + /// Extra properties + /// + [JsonExtensionData] + public Dictionary ExtensionData + { + get => this._extensionData ??= new(); + set => this._extensionData = value; + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/AuthorRole.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/AuthorRole.cs index af25d35fe857..cf29fb2dd63b 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/AuthorRole.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/AuthorRole.cs @@ -15,15 +15,22 @@ namespace Microsoft.SemanticKernel.AI.ChatCompletion; /// The role that instructs or sets the behavior of the assistant. /// public static readonly AuthorRole System = new("system"); + /// /// The role that provides responses to system-instructed, user-prompted input. /// public static readonly AuthorRole Assistant = new("assistant"); + /// /// The role that provides input for chat completions. /// public static readonly AuthorRole User = new("user"); + /// + /// The role that provides additional information and references for chat completions. + /// + public static readonly AuthorRole Tool = new("tool"); + /// /// Gets the label associated with this AuthorRole. /// @@ -51,16 +58,6 @@ public AuthorRole(string label) /// true if left and right are both null or have equivalent labels; false otherwise public static bool operator ==(AuthorRole left, AuthorRole right) { - if (Object.ReferenceEquals(left, right)) - { - return true; - } - - if (Object.ReferenceEquals(left, null) || Object.ReferenceEquals(right, null)) - { - return false; - } - return left.Equals(right); } @@ -86,8 +83,7 @@ public override int GetHashCode() /// public bool Equals(AuthorRole other) - => !Object.ReferenceEquals(other, null) - && string.Equals(this.Label, other.Label, StringComparison.OrdinalIgnoreCase); + => string.Equals(this.Label, other.Label, StringComparison.OrdinalIgnoreCase); /// public override string ToString() => this.Label; diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs index 66757c8c4a57..94dcdd9c39d9 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs @@ -6,52 +6,57 @@ using System.Threading.Tasks; namespace Microsoft.SemanticKernel.AI.ChatCompletion; - +/// +/// Provides extension methods for the IChatCompletion interface. +/// public static class ChatCompletionExtensions { /// - /// Generate a new chat message + /// Generates a new chat message as an asynchronous stream. /// - /// Target interface to extend - /// Chat history - /// AI request settings - /// Async cancellation token - /// This extension does not support multiple prompt results (Only the first will be returned) - /// Stream the generated chat message in string format + /// The target IChatCompletion interface to extend. + /// The chat history. + /// The AI request settings (optional). + /// The asynchronous cancellation token (optional). + /// This extension does not support multiple prompt results (only the first will be returned). + /// An asynchronous stream of the generated chat message in string format. public static async IAsyncEnumerable GenerateMessageStreamAsync( this IChatCompletion chatCompletion, ChatHistory chat, - ChatRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var chatCompletionResult in chatCompletion.GetStreamingChatCompletionsAsync(chat, requestSettings, cancellationToken).ConfigureAwait(false)) + // Using var below results in Microsoft.CSharp.RuntimeBinder.RuntimeBinderException : Cannot apply indexing with [] to an expression of type 'object' + IAsyncEnumerable chatCompletionResults = chatCompletion.GetStreamingChatCompletionsAsync(chat, requestSettings, cancellationToken); + await foreach (var chatCompletionResult in chatCompletionResults) { await foreach (var chatMessageStream in chatCompletionResult.GetStreamingChatMessageAsync(cancellationToken).ConfigureAwait(false)) { yield return chatMessageStream.Content; } + yield break; } } /// - /// Generate a new chat message + /// Generates a new chat message asynchronously. /// - /// Target interface to extend - /// Chat history - /// AI request settings - /// Async cancellation token - /// This extension does not support multiple prompt results (Only the first will be returned) - /// Generated chat message in string format + /// The target IChatCompletion interface to extend. + /// The chat history. + /// The AI request settings (optional). + /// The asynchronous cancellation token (optional). + /// This extension does not support multiple prompt results (only the first will be returned). + /// A task representing the generated chat message in string format. public static async Task GenerateMessageAsync( this IChatCompletion chatCompletion, ChatHistory chat, - ChatRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - var chatResults = await chatCompletion.GetChatCompletionsAsync(chat, requestSettings, cancellationToken).ConfigureAwait(false); + // Using var below results in Microsoft.CSharp.RuntimeBinder.RuntimeBinderException : Cannot apply indexing with [] to an expression of type 'object' + IReadOnlyList chatResults = await chatCompletion.GetChatCompletionsAsync(chat, requestSettings, cancellationToken).ConfigureAwait(false); var firstChatMessage = await chatResults[0].GetChatMessageAsync(cancellationToken).ConfigureAwait(false); - return firstChatMessage.Content; } } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionServiceExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionServiceExtensions.cs index f6edaf5a2ee0..589ca69aecdd 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionServiceExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionServiceExtensions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Services; // Use base namespace for better discoverability and to avoid conflicts with other extensions. @@ -8,6 +9,9 @@ namespace Microsoft.SemanticKernel; #pragma warning restore IDE0130 // Namespace does not match folder structure +/// +/// Provides extension methods for working with chat completion services. +/// public static class ChatCompletionServiceExtensions { /// @@ -17,11 +21,11 @@ public static class ChatCompletionServiceExtensions /// The service provider. /// Optional identifier of the desired service. /// The completion service id matching the given id or the default. - /// Thrown when no suitable service is found. + /// Thrown when no suitable service is found. public static IChatCompletion GetChatCompletionService( this IAIServiceProvider services, string? serviceId = null) => services.GetService(serviceId) - ?? throw new KernelException(KernelException.ErrorCodes.ServiceNotFound, "Chat completion service not found"); + ?? throw new SKException("Chat completion service not found"); /// /// Returns true if a exist with the specified ID. diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatHistory.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatHistory.cs index bb020a279288..1ec25161e418 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatHistory.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatHistory.cs @@ -33,6 +33,17 @@ public void AddMessage(AuthorRole authorRole, string content) this.Add(new ChatMessage(authorRole, content)); } + /// + /// Insert a message into the chat history + /// + /// Index of the message to insert + /// Role of the message author + /// Message content + public void InsertMessage(int index, AuthorRole authorRole, string content) + { + this.Insert(index, new ChatMessage(authorRole, content)); + } + /// /// Add a user message to the chat history /// diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs deleted file mode 100644 index c74373e2f332..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; - -namespace Microsoft.SemanticKernel.AI.ChatCompletion; - -/// -/// Settings for a chat completion request. -/// For OpenAI see https://platform.openai.com/docs/api-reference/chat -/// -public class ChatRequestSettings -{ - /// - /// Temperature controls the randomness of the completion. - /// The higher the temperature, the more random the completion. - /// - public double Temperature { get; set; } = 0; - - /// - /// TopP controls the diversity of the completion. - /// The higher the TopP, the more diverse the completion. - /// - public double TopP { get; set; } = 0; - - /// - /// Number between -2.0 and 2.0. Positive values penalize new tokens - /// based on whether they appear in the text so far, increasing the - /// model's likelihood to talk about new topics. - /// - public double PresencePenalty { get; set; } = 0; - - /// - /// Number between -2.0 and 2.0. Positive values penalize new tokens - /// based on their existing frequency in the text so far, decreasing - /// the model's likelihood to repeat the same line verbatim. - /// - public double FrequencyPenalty { get; set; } = 0; - - /// - /// Sequences where the completion will stop generating further tokens. - /// - public IList StopSequences { get; set; } = Array.Empty(); - - /// - /// How many completions to generate for each prompt. Default is 1. - /// Note: Because this parameter generates many completions, it can quickly consume your token quota. - /// Use carefully and ensure that you have reasonable settings for max_tokens and stop. - /// - public int ResultsPerPrompt { get; set; } = 1; - - /// - /// The maximum number of tokens to generate in the completion. - /// - public int? MaxTokens { get; set; } - - /// - /// Modify the likelihood of specified tokens appearing in the completion. - /// - public IDictionary TokenSelectionBiases { get; set; } = new Dictionary(); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs index 354cb24b954b..6d713703c93d 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs @@ -28,7 +28,7 @@ public interface IChatCompletion : IAIService /// List of different chat results generated by the remote model Task> GetChatCompletionsAsync( ChatHistory chat, - ChatRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); /// @@ -40,6 +40,6 @@ Task> GetChatCompletionsAsync( /// AsyncEnumerable list of different streaming chat results generated by the remote model IAsyncEnumerable GetStreamingChatCompletionsAsync( ChatHistory chat, - ChatRequestSettings? requestSettings = null, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatResult.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatResult.cs index adbadcaa53e3..f02027449744 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatResult.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatResult.cs @@ -8,7 +8,7 @@ namespace Microsoft.SemanticKernel.AI.ChatCompletion; /// /// Interface for chat completion results /// -public interface IChatResult +public interface IChatResult : IResultBase { /// /// Get the chat message from the result. diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatStreamingResult.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatStreamingResult.cs index 5c99240967b5..5105c64ebb7d 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatStreamingResult.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatStreamingResult.cs @@ -8,7 +8,7 @@ namespace Microsoft.SemanticKernel.AI.ChatCompletion; /// /// Interface for chat completion streaming results /// -public interface IChatStreamingResult : IChatResult +public interface IChatStreamingResult : IResultBase { /// /// Get the chat message from the streaming result. diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/Embedding.cs b/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/Embedding.cs deleted file mode 100644 index 325e41e98bdf..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/Embedding.cs +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Text.Json.Serialization; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.AI.Embeddings; - -/// -/// Represents a strongly typed vector of numeric data. -/// -/// -public readonly struct Embedding : IEquatable> - where TEmbedding : unmanaged -{ - /// - /// An empty instance. - /// - [SuppressMessage("Design", "CA1000:Do not declare static members on generic types", Justification = "Static empty struct instance.")] - public static Embedding Empty - { - get - { - if (!Embedding.IsSupported()) - { - ThrowNotSupportedEmbedding(); - } - - return default; - } - } - - /// - /// Initializes a new instance of the class that contains numeric elements copied from the specified collection. - /// - /// The source data. - /// An unsupported type is used as TEmbedding. - /// A null vector is passed in. - [JsonConstructor] - public Embedding(IEnumerable vector) : this(vector, transferOwnership: false) - { - } - - /// - /// Initializes a new instance of the class that contains either a copy of or the reference to the specified collection. - /// - /// The source data. - /// - /// to transfer logical ownership of to this instance; after doing so, - /// the caller should no longer mutate the original array. to instead make a copy of . - /// - /// An unsupported type is used as TEmbedding. - /// A null vector is passed in. - public Embedding(IEnumerable vector, bool transferOwnership) - { - Verify.NotNull(vector); - - if (!Embedding.IsSupported()) - { - ThrowNotSupportedEmbedding(); - } - - // Create a local, protected copy if transferOwnership is false or if the vector is not an array. - // If the vector is an array and transferOwnership is true, then we can use the array directly. - this._vector = - transferOwnership && vector is TEmbedding[] array ? array : vector.ToArray(); - } - - private static void ThrowNotSupportedEmbedding() => - throw new NotSupportedException($"Embeddings do not support type '{typeof(TEmbedding).Name}'. Supported types include: [ Single, Double ]"); - - /// - /// Gets the vector as an - /// - [JsonPropertyName("vector")] - public IEnumerable Vector => this._vector ?? Array.Empty(); - - /// - /// true if the vector is empty. - /// - [JsonIgnore] - public bool IsEmpty - { - get - { - TEmbedding[]? vector = this._vector; - return vector is null || vector.Length == 0; - } - } - - /// - /// The number of elements in the vector. - /// - [JsonIgnore] - public int Count => this._vector?.Length ?? 0; - - /// - /// Gets the vector as a read-only span. - /// - public ReadOnlySpan AsReadOnlySpan() - { - return new(this._vector); - } - - /// - /// Serves as the default hash function. - /// - /// A hash code for the current object. - public override int GetHashCode() - { - return this._vector?.GetHashCode() ?? 0; - } - - /// - /// Determines whether two object instances are equal. - /// - /// The object to compare with the current object. - /// true if the specified object is equal to the current object; otherwise, false. - public override bool Equals(object obj) - { - return obj is Embedding other && this.Equals(other); - } - - /// - /// Compares two embeddings for equality. - /// - /// The to compare with the current object. - /// >true if the specified object is equal to the current object; otherwise, false. - public bool Equals(Embedding other) - { - TEmbedding[]? vector = this._vector; - return vector is null ? other._vector is null : vector.Equals(other._vector); - } - - /// - /// Compares two embeddings for equality. - /// - /// The left . - /// The right . - /// true if the embeddings contain identical data; false otherwise - public static bool operator ==(Embedding left, Embedding right) - { - return left.Equals(right); - } - - /// - /// Compares two embeddings for inequality. - /// - /// The left . - /// The right . - /// true if the embeddings do not contain identical data; false otherwise - public static bool operator !=(Embedding left, Embedding right) - { - return !(left == right); - } - - /// - /// Implicit creation of an object from an array of data. - /// - /// An array of data. - public static explicit operator Embedding(TEmbedding[] vector) - { - return new Embedding(vector, transferOwnership: false); - } - - /// - /// Implicit creation of an array of type from a . - /// - /// Source . - /// A clone of the underlying data. - public static explicit operator TEmbedding[](Embedding embedding) - { - return embedding._vector is null ? Array.Empty() : (TEmbedding[])embedding._vector.Clone(); - } - - /// - /// Implicit creation of an from a . - /// - /// Source . - /// A clone of the underlying data. - public static explicit operator ReadOnlySpan(Embedding embedding) - { - return embedding.AsReadOnlySpan(); - } - - #region private ================================================================================ - - private readonly TEmbedding[]? _vector; - - #endregion -} - -/// -/// Provides functionality related to . -/// -public static class Embedding -{ - /// - /// Gets whether the specified is supported for use with . - /// - /// The type to be checked. - /// - /// if the type is supported; otherwise, . - /// Currently only and are supported. - /// - public static bool IsSupported() => typeof(TEmbedding) == typeof(float) || typeof(TEmbedding) == typeof(double); - - /// - /// Gets whether the specified is supported for use with . - /// - /// The type to be checked. - /// - /// if the type is supported; otherwise, . - /// Currently only and are supported. - /// - public static bool IsSupported(Type type) => type == typeof(float) || type == typeof(double); - - /// - /// Gets an enumerable of the types supported by the struct. - /// - public static IEnumerable SupportedTypes { get; } = Array.AsReadOnly(new Type[] { typeof(float), typeof(double) }); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/EmbeddingGenerationServiceExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/EmbeddingGenerationServiceExtensions.cs index a15e7678904b..a28d3cb9167f 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/EmbeddingGenerationServiceExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/EmbeddingGenerationServiceExtensions.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -18,10 +19,10 @@ public static class EmbeddingGenerationExtensions /// The type from which embeddings will be generated. /// The numeric type of the embedding data. /// The embedding generator. - /// A value from which an will be generated. + /// A value from which an embedding will be generated. /// Cancellation token - /// A list of structs representing the input . - public static async Task> GenerateEmbeddingAsync + /// A list of embedding structs representing the input . + public static async Task> GenerateEmbeddingAsync (this IEmbeddingGeneration generator, TValue value, CancellationToken cancellationToken = default) where TEmbedding : unmanaged { diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/IEmbeddingGeneration.cs b/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/IEmbeddingGeneration.cs index d28a25b9d6a5..a2fc94d18f38 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/IEmbeddingGeneration.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/IEmbeddingGeneration.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -21,5 +22,5 @@ public interface IEmbeddingGeneration : IAIService /// List of strings to generate embeddings for /// The to monitor for cancellation requests. The default is . /// List of embeddings - Task>> GenerateEmbeddingsAsync(IList data, CancellationToken cancellationToken = default); + Task>> GenerateEmbeddingsAsync(IList data, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/TextEmbeddingServiceExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/TextEmbeddingServiceExtensions.cs index 613161f68002..2f453ba308d5 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/TextEmbeddingServiceExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/Embeddings/TextEmbeddingServiceExtensions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Services; // Use base namespace for better discoverability and to avoid conflicts with other extensions. @@ -8,6 +9,9 @@ namespace Microsoft.SemanticKernel; #pragma warning restore IDE0130 // Namespace does not match folder structure +/// +/// Provides extension methods for working with text embedding services. +/// public static class TextEmbeddingServiceExtensions { /// @@ -17,12 +21,12 @@ public static class TextEmbeddingServiceExtensions /// The service provider. /// Optional identifier of the desired service. /// The embedding service matching the given id or the default service. - /// Thrown when no suitable service is found. + /// Thrown when no suitable service is found. public static ITextEmbeddingGeneration GetTextEmbeddingService( this IAIServiceProvider services, string? serviceId = null) => services.GetService(serviceId) - ?? throw new KernelException(KernelException.ErrorCodes.ServiceNotFound, "Text embedding service not available"); + ?? throw new SKException("Text embedding service not found"); /// /// Returns true if a exist with the specified ID. diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/IResultBase.cs b/dotnet/src/SemanticKernel.Abstractions/AI/IResultBase.cs new file mode 100644 index 000000000000..0c631cc3fb6e --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/AI/IResultBase.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.AI; + +/// +/// Interface for model results +/// +public interface IResultBase +{ + /// + /// Gets the model result data. + /// + ModelResult ModelResult { get; } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ImageGeneration/ImageGenerationServiceExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ImageGeneration/ImageGenerationServiceExtensions.cs index 17541b3e3aca..d458ff3fa7f4 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ImageGeneration/ImageGenerationServiceExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ImageGeneration/ImageGenerationServiceExtensions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using Microsoft.SemanticKernel.AI.ImageGeneration; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Services; // Use base namespace for better discoverability and to avoid conflicts with other extensions. @@ -8,6 +9,9 @@ namespace Microsoft.SemanticKernel; #pragma warning restore IDE0130 // Namespace does not match folder structure +/// +/// Provides extension methods for working with services. +/// public static class ImageGenerationServiceExtensions { /// @@ -17,11 +21,11 @@ public static class ImageGenerationServiceExtensions /// The service provider. /// Optional identifier of the desired service. /// The id matching the given id or the default. - /// Thrown when no suitable service is found. + /// Thrown when no suitable service is found. public static IImageGeneration GetImageGenerationService( this IAIServiceProvider services, string? serviceId = null) => services.GetService(serviceId) - ?? throw new KernelException(KernelException.ErrorCodes.ServiceNotFound, "Image generation service not found"); + ?? throw new SKException("Image generation service not found"); /// /// Returns true if a exist with the specified ID. diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/CompleteRequestSettings.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/CompleteRequestSettings.cs deleted file mode 100644 index 5999e25cfc16..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/CompleteRequestSettings.cs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using Microsoft.SemanticKernel.SemanticFunctions; - -namespace Microsoft.SemanticKernel.AI.TextCompletion; - -/// -/// Settings for a text completion request. -/// -public class CompleteRequestSettings -{ - /// - /// Temperature controls the randomness of the completion. - /// The higher the temperature, the more random the completion. - /// - public double Temperature { get; set; } = 0; - - /// - /// TopP controls the diversity of the completion. - /// The higher the TopP, the more diverse the completion. - /// - public double TopP { get; set; } = 0; - - /// - /// Number between -2.0 and 2.0. Positive values penalize new tokens - /// based on whether they appear in the text so far, increasing the - /// model's likelihood to talk about new topics. - /// - public double PresencePenalty { get; set; } = 0; - - /// - /// Number between -2.0 and 2.0. Positive values penalize new tokens - /// based on their existing frequency in the text so far, decreasing - /// the model's likelihood to repeat the same line verbatim. - /// - public double FrequencyPenalty { get; set; } = 0; - - /// - /// The maximum number of tokens to generate in the completion. - /// - public int? MaxTokens { get; set; } - - /// - /// Sequences where the completion will stop generating further tokens. - /// - public IList StopSequences { get; set; } = Array.Empty(); - - /// - /// How many completions to generate for each prompt. Default is 1. - /// Note: Because this parameter generates many completions, it can quickly consume your token quota. - /// Use carefully and ensure that you have reasonable settings for max_tokens and stop. - /// - public int ResultsPerPrompt { get; set; } = 1; - - /// - /// The system prompt to use when generating text completions using a chat model. - /// Defaults to "Assistant is a large language model." - /// - public string ChatSystemPrompt { get; set; } = "Assistant is a large language model."; - - /// - /// Modify the likelihood of specified tokens appearing in the completion. - /// - public IDictionary TokenSelectionBiases { get; set; } = new Dictionary(); - - /// - /// Create a new settings object with the values from another settings object. - /// - /// - /// An instance of - public static CompleteRequestSettings FromCompletionConfig(PromptTemplateConfig.CompletionConfig config) - { - return new CompleteRequestSettings - { - Temperature = config.Temperature, - TopP = config.TopP, - PresencePenalty = config.PresencePenalty, - FrequencyPenalty = config.FrequencyPenalty, - MaxTokens = config.MaxTokens, - StopSequences = config.StopSequences, - }; - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs index 71ef848584f0..42d8f295ef65 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs @@ -21,7 +21,7 @@ public interface ITextCompletion : IAIService /// List of different completions results generated by the remote model Task> GetCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); /// @@ -33,6 +33,6 @@ Task> GetCompletionsAsync( /// List of different completion streaming results generated by the remote model IAsyncEnumerable GetStreamingCompletionsAsync( string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextResult.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextResult.cs index c652b5d4a162..f108f3d00104 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextResult.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextResult.cs @@ -2,19 +2,18 @@ using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.Orchestration; namespace Microsoft.SemanticKernel.AI.TextCompletion; /// -/// Interface for text completion results +/// Interface for text completion results. /// -public interface ITextResult +public interface ITextResult : IResultBase { /// - /// Gets the model result data. + /// Asynchronously retrieves the text completion result. /// - ModelResult ModelResult { get; } - + /// An optional to observe while waiting for the task to complete. + /// A representing the asynchronous operation, with the result being the completed text. Task GetCompletionAsync(CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextStreamingResult.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextStreamingResult.cs index 7b4c8a8bf9aa..b4c4f51d6fcf 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextStreamingResult.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextStreamingResult.cs @@ -6,9 +6,15 @@ namespace Microsoft.SemanticKernel.AI.TextCompletion; /// -/// Interface for text completion streaming results +/// Interface for text completion streaming results. +/// Provides an asynchronous enumerable of text completion results. /// -public interface ITextStreamingResult : ITextResult +public interface ITextStreamingResult : IResultBase { + /// + /// Gets an asynchronous enumerable of text completion results. + /// + /// An optional to observe while waiting for the task to complete. + /// An of representing the text completion results. IAsyncEnumerable GetCompletionStreamingAsync(CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionExtensions.cs index 31d468bfe647..4350f709d287 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionExtensions.cs @@ -23,7 +23,7 @@ public static class TextCompletionExtensions /// Text generated by the remote model public static async Task CompleteAsync(this ITextCompletion textCompletion, string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { var completions = await textCompletion.GetCompletionsAsync(text, requestSettings, cancellationToken).ConfigureAwait(false); @@ -43,10 +43,10 @@ public static async Task CompleteAsync(this ITextCompletion textCompleti /// Streaming content of the text generated by the remote model public static async IAsyncEnumerable CompleteStreamAsync(this ITextCompletion textCompletion, string text, - CompleteRequestSettings requestSettings, + AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var completionResults = textCompletion.GetStreamingCompletionsAsync(text, requestSettings, cancellationToken); + IAsyncEnumerable completionResults = textCompletion.GetStreamingCompletionsAsync(text, requestSettings, cancellationToken); await foreach (var completionResult in completionResults) { @@ -58,4 +58,28 @@ public static async IAsyncEnumerable CompleteStreamAsync(this ITextCompl yield break; } } + + /// + /// Creates a completion for the prompt and settings. + /// + /// Target interface to extend. + /// The prompt to complete. + /// Request settings for the completion API. + /// The to monitor for cancellation requests. The default is . + /// Streaming content of the text generated by the remote model. + public static async IAsyncEnumerable CompleteStreamsAsync(this ITextCompletion textCompletion, + string text, + AIRequestSettings? requestSettings = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + IAsyncEnumerable completionResults = textCompletion.GetStreamingCompletionsAsync(text, requestSettings, cancellationToken); + + await foreach (var completionResult in completionResults) + { + await foreach (var word in completionResult.GetCompletionStreamingAsync(cancellationToken).ConfigureAwait(false)) + { + yield return word; + } + } + } } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionServiceExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionServiceExtensions.cs index 5a0a9308af28..dc69b42f4e99 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionServiceExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionServiceExtensions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Services; // Use base namespace for better discoverability and to avoid conflicts with other extensions. @@ -8,6 +9,9 @@ namespace Microsoft.SemanticKernel; #pragma warning restore IDE0130 // Namespace does not match folder structure +/// +/// Provides extension methods for working with services. +/// public static class TextCompletionServiceExtensions { /// @@ -17,11 +21,11 @@ public static class TextCompletionServiceExtensions /// The service provider. /// Optional identifier of the desired service. /// The text completion service id matching the given ID or the default. - /// Thrown when no suitable service is found. + /// Thrown when no suitable service is found. public static ITextCompletion GetTextCompletionServiceOrDefault( this IAIServiceProvider services, string? serviceId = null) => services.GetService(serviceId) - ?? throw new KernelException(KernelException.ErrorCodes.ServiceNotFound, "Text completion service not found"); + ?? throw new SKException("Text completion service not found"); /// /// Returns true if a exist with the specified ID. diff --git a/dotnet/src/SemanticKernel.Abstractions/Diagnostics/HttpOperationException.cs b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/HttpOperationException.cs new file mode 100644 index 000000000000..58581fedf3cb --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/HttpOperationException.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; + +namespace Microsoft.SemanticKernel.Diagnostics; + +/// +/// Represents an exception specific to HTTP operations. +/// +public class HttpOperationException : Exception +{ + /// + /// Initializes a new instance of the class. + /// + public HttpOperationException() : base() + { + } + + /// + /// Initializes a new instance of the class with its message set to . + /// + /// A string that describes the error. + public HttpOperationException(string? message) : base(message) + { + } + + /// + /// Initializes a new instance of the class with its message set to . + /// + /// A string that describes the error. + /// The exception that is the cause of the current exception. + public HttpOperationException(string? message, Exception? innerException) : base(message, innerException) + { + } + + /// + /// Initializes a new instance of the class with its message + /// and additional properties for the HTTP status code and response content. + /// + /// The HTTP status code. + /// The content of the HTTP response. + /// A string that describes the error. + /// The exception that is the cause of the current exception. + public HttpOperationException(HttpStatusCode? statusCode, string? responseContent, string? message, Exception? innerException) + : base(message, innerException) + { + this.StatusCode = statusCode; + this.ResponseContent = responseContent; + } + + /// + /// Gets or sets the HTTP status code. If the property is null, it indicates that no response was received. + /// + public HttpStatusCode? StatusCode { get; set; } + + /// + /// Gets or sets the content of the HTTP response. + /// + public string? ResponseContent { get; set; } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Diagnostics/SKException.cs b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/SKException.cs index bc21d3db998e..c8c83ddfd33f 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Diagnostics/SKException.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/SKException.cs @@ -5,31 +5,31 @@ namespace Microsoft.SemanticKernel.Diagnostics; /// -/// Provides the base exception from which all Semantic Kernel exceptions derive. +/// Represents the base exception from which all Semantic Kernel exceptions derive. /// -public abstract class SKException : Exception +public class SKException : Exception { /// - /// Initializes a new instance of the class with a default message. + /// Initializes a new instance of the class. /// - protected SKException() + public SKException() { } /// - /// Initializes a new instance of the class with its message set to . + /// Initializes a new instance of the class with a specified error message. /// - /// A string that describes the error. - protected SKException(string? message) : base(message) + /// The error message that explains the reason for the exception. + public SKException(string? message) : base(message) { } /// - /// Initializes a new instance of the class with its message set to . + /// Initializes a new instance of the class with a specified error message and a reference to the inner exception that is the cause of this exception. /// - /// A string that describes the error. - /// The exception that is the cause of the current exception. - protected SKException(string? message, Exception? innerException) : base(message, innerException) + /// The error message that explains the reason for the exception. + /// The exception that is the cause of the current exception, or a null reference if no inner exception is specified. + public SKException(string? message, Exception? innerException) : base(message, innerException) { } } diff --git a/dotnet/src/SemanticKernel.Abstractions/Diagnostics/Telemetry.cs b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/Telemetry.cs new file mode 100644 index 000000000000..27bfa2006400 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Diagnostics/Telemetry.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.SemanticKernel.Diagnostics; +/// +/// Provides functionality to manage telemetry settings. +/// +public static class Telemetry +{ + /// + /// Environment variable used in Azure to enable/disable telemetry. + /// See: https://learn.microsoft.com/en-us/dotnet/api/azure.core.diagnosticsoptions.istelemetryenabled?view=azure-dotnet + /// + private const string TelemetryDisabledEnvVar = "AZURE_TELEMETRY_DISABLED"; + + /// + /// HTTP User Agent. + /// Note: Azure max length 24 chars. + /// + public const string HttpUserAgent = "Semantic-Kernel"; + + /// + /// Gets a value indicating whether telemetry is enabled or not. + /// Source: https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/src/DiagnosticsOptions.cs + /// Azure customers setting AZURE_TELEMETRY_DISABLED=1 expect telemetry to be disabled. + /// + public static bool IsTelemetryEnabled => !EnvExtensions.GetBoolEnvVar(TelemetryDisabledEnvVar) ?? true; +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Events/FunctionInvokedEventArgs.cs b/dotnet/src/SemanticKernel.Abstractions/Events/FunctionInvokedEventArgs.cs new file mode 100644 index 000000000000..b36674e28a15 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Events/FunctionInvokedEventArgs.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Events; + +/// +/// Event arguments available to the Kernel.FunctionInvoked event. +/// +public class FunctionInvokedEventArgs : SKCancelEventArgs +{ + private Dictionary? _metadata; + + /// + /// Indicates if the function execution should repeat. + /// + public bool IsRepeatRequested => this._repeatRequested; + + /// + /// Metadata for storing additional information about function execution result. + /// + public Dictionary Metadata => this._metadata ??= new(); + + /// + /// Initializes a new instance of the class. + /// + /// Function view details + /// Function result + public FunctionInvokedEventArgs(FunctionView functionView, FunctionResult result) : base(functionView, result.Context) + { + this._metadata = result._metadata; + } + + /// + /// Repeat the current function invocation. + /// + public void Repeat() + { + this._repeatRequested = true; + } + + private bool _repeatRequested; +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Events/FunctionInvokingEventArgs.cs b/dotnet/src/SemanticKernel.Abstractions/Events/FunctionInvokingEventArgs.cs new file mode 100644 index 000000000000..bf8e707b44ab --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Events/FunctionInvokingEventArgs.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Events; + +/// +/// Event arguments available to the Kernel.FunctionInvoking event. +/// +public class FunctionInvokingEventArgs : SKCancelEventArgs +{ + /// + /// Indicates if the function execution should be skipped. + /// + public bool IsSkipRequested => this._skipRequested; + + /// + /// Initializes a new instance of the class. + /// + /// Function view details + /// Context related to the event + public FunctionInvokingEventArgs(FunctionView functionView, SKContext context) : base(functionView, context) + { + } + + /// + /// Skip the current function invoking attempt. + /// + public void Skip() + { + this._skipRequested = true; + } + + private bool _skipRequested; +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Events/SKCancelEventArgs.cs b/dotnet/src/SemanticKernel.Abstractions/Events/SKCancelEventArgs.cs new file mode 100644 index 000000000000..dfcaae8c149d --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Events/SKCancelEventArgs.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Events; +#pragma warning disable CA1001 // Types that own disposable fields should be disposable + +/// +/// Base arguments for cancellable events. +/// +public abstract class SKCancelEventArgs : SKEventArgs +{ + private readonly CancellationTokenSource _cancelTokenSource = new(); + + internal SKCancelEventArgs(FunctionView functionView, SKContext context) : base(functionView, context) + { + } + + /// + /// Cancellation token to be used to cancel further execution. + /// + public CancellationToken CancelToken => this._cancelTokenSource.Token; + + /// + /// Cancel all further execution. + /// + public void Cancel() + { + this._cancelTokenSource.Cancel(); + } + + /// + /// Dispose resources. + /// + ~SKCancelEventArgs() + { + this._cancelTokenSource.Dispose(); + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Events/SKEventArgs.cs b/dotnet/src/SemanticKernel.Abstractions/Events/SKEventArgs.cs new file mode 100644 index 000000000000..e04f0d7bbc03 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Events/SKEventArgs.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Events; + +/// +/// Base arguments for events. +/// +public abstract class SKEventArgs : EventArgs +{ + /// + /// Initializes a new instance of the class. + /// + /// Function view details + /// Context related to the event + internal SKEventArgs(FunctionView functionView, SKContext context) + { + Verify.NotNull(context); + Verify.NotNull(functionView); + + this.FunctionView = functionView; + this.SKContext = context; + } + + /// + /// Function view details. + /// + public FunctionView FunctionView { get; } + + /// + /// Context related to the event. + /// + public SKContext SKContext { get; } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/FunctionView.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/FunctionView.cs new file mode 100644 index 000000000000..a6d0818c9f1c --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/FunctionView.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// A function view is a read-only representation of a function. +/// +/// Name of the function. The name is used by the function collection and in prompt templates e.g. {{pluginName.functionName}} +/// Name of the plugin containing the function. The name is used by the function collection and in prompt templates e.g. {{pluginName.functionName}} +/// Function description. The description is used in combination with embeddings when searching relevant functions. +/// Optional list of function parameters +public sealed record FunctionView( + string Name, + string PluginName, + string Description = "", + IReadOnlyList? Parameters = null) +{ + /// + /// List of function parameters + /// + public IReadOnlyList Parameters { get; init; } = Parameters ?? Array.Empty(); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/IFunctionCollection.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/IFunctionCollection.cs new file mode 100644 index 000000000000..dc6fad98e3cf --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/IFunctionCollection.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Function collection interface. +/// +[SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix")] +public interface IFunctionCollection : IReadOnlyFunctionCollection +{ + /// + /// Add a function to the collection + /// + /// Function delegate + /// Self instance + IFunctionCollection AddFunction(ISKFunction functionInstance); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/IReadOnlyFunctionCollection.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/IReadOnlyFunctionCollection.cs new file mode 100644 index 000000000000..67360a0cd9c0 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/IReadOnlyFunctionCollection.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel.Diagnostics; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Read-only function collection interface. +/// +[SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix")] +public interface IReadOnlyFunctionCollection +{ + /// + /// Gets the function stored in the collection. + /// + /// The name of the function to retrieve. + /// The function retrieved from the collection. + /// The specified function could not be found in the collection. + ISKFunction GetFunction(string functionName); + + /// + /// Gets the function stored in the collection. + /// + /// The name of the plugin with which the function is associated. + /// The name of the function to retrieve. + /// The function retrieved from the collection. + /// The specified function could not be found in the collection. + ISKFunction GetFunction(string pluginName, string functionName); + + /// + /// Check if a function is available in the current context, and return it. + /// + /// The name of the function to retrieve. + /// When this method returns, the function that was retrieved if one with the specified name was found; otherwise, . + /// if the function was found; otherwise, . + bool TryGetFunction(string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction); + + /// + /// Check if a function is available in the current context, and return it. + /// + /// The name of the plugin with which the function is associated. + /// The name of the function to retrieve. + /// When this method returns, the function that was retrieved if one with the specified name was found; otherwise, . + /// if the function was found; otherwise, . + bool TryGetFunction(string pluginName, string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction); + + /// + /// Get a snapshot all registered functions details, minus the delegates + /// + /// An object containing all the functions details + IReadOnlyList GetFunctionViews(); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs new file mode 100644 index 000000000000..717639dab76f --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Orchestration; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Semantic Kernel callable function interface +/// +public interface ISKFunction +{ + /// + /// Name of the function. The name is used by the function collection and in prompt templates e.g. {{pluginName.functionName}} + /// + string Name { get; } + + /// + /// Name of the plugin containing the function. The name is used by the function collection and in prompt templates e.g. {{pluginName.functionName}} + /// + string PluginName { get; } + + /// + /// Function description. The description is used in combination with embeddings when searching relevant functions. + /// + string Description { get; } + + /// + /// AI service settings + /// + AIRequestSettings? RequestSettings { get; } + + /// + /// Returns a description of the function, including parameters. + /// + /// An instance of describing the function + FunctionView Describe(); + + /// + /// Invoke the . + /// + /// SK context + /// LLM completion settings (for semantic functions only) + /// The updated context, potentially a new one if context switching is implemented. + /// The to monitor for cancellation requests. The default is . + Task InvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); + + /// + /// Set the AI service used by the semantic function, passing a factory method. + /// The factory allows to lazily instantiate the client and to properly handle its disposal. + /// + /// AI service factory + /// Self instance + ISKFunction SetAIService(Func serviceFactory); + + /// + /// Set the AI completion settings used with LLM requests + /// + /// LLM completion settings + /// Self instance + ISKFunction SetAIConfiguration(AIRequestSettings? requestSettings); + + #region Obsolete + + /// + /// Name of the plugin containing the function. The name is used by the function collection and in prompt templates e.g. {{skillName.functionName}} + /// + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use ISKFunction.SkillName instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + string SkillName { get; } + + /// + /// Set the default function collection to use when the function is invoked + /// without a context or with a context that doesn't have a collection. + /// + /// Kernel's function collection + /// Self instance + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + ISKFunction SetDefaultSkillCollection(IReadOnlyFunctionCollection skills); + + /// + /// Set the default function collection to use when the function is invoked + /// without a context or with a context that doesn't have a collection. + /// + /// Kernel's function collection + /// Self instance + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + ISKFunction SetDefaultFunctionCollection(IReadOnlyFunctionCollection functions); + + /// + /// Whether the function is defined using a prompt template. + /// IMPORTANT: native functions might use semantic functions internally, + /// so when this property is False, executing the function might still involve AI calls. + /// + [Obsolete("Kernel no longer differentiates between Semantic and Native functions. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + bool IsSemantic { get; } + + #endregion +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/NullReadOnlyFunctionCollection.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/NullReadOnlyFunctionCollection.cs new file mode 100644 index 000000000000..f89db9e471fa --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/NullReadOnlyFunctionCollection.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel.Diagnostics; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +[DebuggerDisplay("Count = 0")] +internal sealed class NullReadOnlyFunctionCollection : IReadOnlyFunctionCollection +{ + public static readonly NullReadOnlyFunctionCollection Instance = new(); + + /// + public ISKFunction GetFunction(string functionName) + { + throw new SKException($"Function not available: {functionName}"); + } + + /// + public ISKFunction GetFunction(string pluginName, string functionName) + { + throw new SKException($"Function not available: {pluginName}.{functionName}"); + } + + /// + public bool TryGetFunction(string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction) + { + availableFunction = null; + return false; + } + + /// + public bool TryGetFunction(string pluginName, string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction) + { + availableFunction = null; + return false; + } + + /// + public IReadOnlyList GetFunctionViews() + { + return Array.Empty(); + } + + private NullReadOnlyFunctionCollection() + { + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/ParameterView.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/ParameterView.cs new file mode 100644 index 000000000000..909f3321b99f --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/ParameterView.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Class used to copy and export data about parameters for planner and related scenarios. +/// +/// Parameter name. The name must be alphanumeric (underscore is the only special char allowed). +/// Parameter description +/// Default parameter value, if not provided +/// Parameter type. +/// Whether the parameter is required. +public sealed record ParameterView( + string Name, + string? Description = null, + string? DefaultValue = null, + ParameterViewType? Type = null, + bool? IsRequired = null); diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/ParameterViewType.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/ParameterViewType.cs new file mode 100644 index 000000000000..ff89b8d69060 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/ParameterViewType.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +#pragma warning disable CA1720 // Identifier contains type name + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Represents the type for the parameter view. +/// +public readonly record struct ParameterViewType(string Name) +{ + /// + /// Represents the "string" parameter view type. + /// + public static readonly ParameterViewType String = new("string"); + + /// + /// Represents the "number" parameter view type. + /// + public static readonly ParameterViewType Number = new("number"); + + /// + /// Represents the "object" parameter view type. + /// + public static readonly ParameterViewType Object = new("object"); + + /// + /// Represents the "array" parameter view type. + /// + public static readonly ParameterViewType Array = new("array"); + + /// + /// Represents the "boolean" parameter view type. + /// + public static readonly ParameterViewType Boolean = new("boolean"); + + /// + /// Gets the name of the parameter view type. + /// + public string Name { get; init; } = !string.IsNullOrEmpty(Name) ? Name : throw new ArgumentNullException(nameof(Name)); + + /// + /// Returns a string representation of the parameter view type. + /// + /// A string representing the parameter view type. + public override string ToString() => this.Name; + + /// + /// Returns the hash code for this instance. + /// + /// A hash code for the current instance. + public override int GetHashCode() + { + return this.Name.GetHashCode(); + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKFunctionAttribute.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/SKFunctionAttribute.cs similarity index 91% rename from dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKFunctionAttribute.cs rename to dotnet/src/SemanticKernel.Abstractions/Functions/SKFunctionAttribute.cs index 94fca1020f36..adad4b5d357a 100644 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKFunctionAttribute.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/SKFunctionAttribute.cs @@ -8,14 +8,17 @@ using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.Orchestration; -namespace Microsoft.SemanticKernel.SkillDefinition; +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 /// /// Specifies that a method is a native function available to Semantic Kernel. /// /// /// -/// When the kernel imports a skill, it searches all public methods tagged with this attribute. +/// When the kernel imports native functions, it searches all public methods tagged with this attribute. /// If a method is not tagged with this attribute, it may still be imported directly via a /// or referencing the method directly. /// diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKNameAttribute.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/SKNameAttribute.cs similarity index 83% rename from dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKNameAttribute.cs rename to dotnet/src/SemanticKernel.Abstractions/Functions/SKNameAttribute.cs index 236faf7d95bf..99f8ef585683 100644 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKNameAttribute.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/SKNameAttribute.cs @@ -2,7 +2,10 @@ using System; -namespace Microsoft.SemanticKernel.SkillDefinition; +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 /// Overrides the default name used by a Semantic Kernel native function name or parameter. /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Http/HttpHandlerFactory{THandler}.cs b/dotnet/src/SemanticKernel.Abstractions/Http/HttpHandlerFactory{THandler}.cs new file mode 100644 index 000000000000..c8c9836e6022 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Http/HttpHandlerFactory{THandler}.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Http; + +/// +/// A factory for creating instances of . +/// +/// +public abstract class HttpHandlerFactory : IDelegatingHandlerFactory where THandler : DelegatingHandler +{ + /// + /// Creates a new instance of . + /// + /// + /// + public virtual DelegatingHandler Create(ILoggerFactory? loggerFactory = null) + { + return (DelegatingHandler)Activator.CreateInstance(typeof(THandler), loggerFactory); + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Http/IDelegatingHandlerFactory.cs b/dotnet/src/SemanticKernel.Abstractions/Http/IDelegatingHandlerFactory.cs new file mode 100644 index 000000000000..fbb19a834015 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Http/IDelegatingHandlerFactory.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Http; + +/// +/// Factory for creating instances. +/// +public interface IDelegatingHandlerFactory +{ + /// + /// Creates a new instance with the specified logger. + /// + /// The to use for logging. If null, no logging will be performed. + /// A new instance. + DelegatingHandler Create(ILoggerFactory? loggerFactory); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Http/NullHttpHandler.cs b/dotnet/src/SemanticKernel.Abstractions/Http/NullHttpHandler.cs new file mode 100644 index 000000000000..3ed5113f26ae --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Http/NullHttpHandler.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; + +namespace Microsoft.SemanticKernel.Http; + +/// +/// A http retry handler that does nothing. +/// +public sealed class NullHttpHandler : DelegatingHandler +{ +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Http/NullHttpHandlerFactory.cs b/dotnet/src/SemanticKernel.Abstractions/Http/NullHttpHandlerFactory.cs new file mode 100644 index 000000000000..07c5d5ccd73a --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Http/NullHttpHandlerFactory.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Http; + +/// +/// Implementation of that creates instances. +/// +public sealed class NullHttpHandlerFactory : IDelegatingHandlerFactory +{ + /// + /// Gets the singleton instance of . + /// + public static NullHttpHandlerFactory Instance => new(); + + /// + /// Creates a new instance. + /// + /// The logger factory to use. + /// A new instance. + public DelegatingHandler Create(ILoggerFactory? loggerFactory) + { + return new NullHttpHandler(); + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/IKernel.cs b/dotnet/src/SemanticKernel.Abstractions/IKernel.cs index cc4c3a96ebfe..29b10e9e1f65 100644 --- a/dotnet/src/SemanticKernel.Abstractions/IKernel.cs +++ b/dotnet/src/SemanticKernel.Abstractions/IKernel.cs @@ -2,14 +2,16 @@ using System; using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Events; +using Microsoft.SemanticKernel.Http; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SemanticFunctions; using Microsoft.SemanticKernel.Services; -using Microsoft.SemanticKernel.SkillDefinition; using Microsoft.SemanticKernel.TemplateEngine; namespace Microsoft.SemanticKernel; @@ -20,25 +22,9 @@ namespace Microsoft.SemanticKernel; public interface IKernel { /// - /// Settings required to execute functions, including details about AI dependencies, e.g. endpoints and API keys. + /// The ILoggerFactory used to create a logger for logging. /// - KernelConfig Config { get; } - - /// - /// App logger - /// - [Obsolete("Use Logger instead. This will be removed in a future release.")] - ILogger Log { get; } - - /// - /// App logger - /// - ILogger Logger { get; } - - /// - /// Semantic memory instance - /// - ISemanticTextMemory Memory { get; } + ILoggerFactory LoggerFactory { get; } /// /// Reference to the engine rendering prompt templates @@ -46,154 +32,108 @@ public interface IKernel IPromptTemplateEngine PromptTemplateEngine { get; } /// - /// Reference to the read-only skill collection containing all the imported functions + /// Reference to the read-only function collection containing all the imported functions /// - IReadOnlySkillCollection Skills { get; } + IReadOnlyFunctionCollection Functions { get; } /// - /// Build and register a function in the internal skill collection, in a global generic skill. + /// Reference to Http handler factory /// - /// Name of the semantic function. The name can contain only alphanumeric chars + underscore. - /// Function configuration, e.g. I/O params, AI settings, localization details, etc. - /// A C# function wrapping AI logic, usually defined with natural language - ISKFunction RegisterSemanticFunction( - string functionName, - SemanticFunctionConfig functionConfig); + IDelegatingHandlerFactory HttpHandlerFactory { get; } /// - /// Build and register a function in the internal skill collection. - /// - /// Name of the skill containing the function. The name can contain only alphanumeric chars + underscore. - /// Name of the semantic function. The name can contain only alphanumeric chars + underscore. - /// Function configuration, e.g. I/O params, AI settings, localization details, etc. - /// A C# function wrapping AI logic, usually defined with natural language - ISKFunction RegisterSemanticFunction( - string skillName, - string functionName, - SemanticFunctionConfig functionConfig); - - /// - /// Registers a custom function in the internal skill collection. + /// Registers a custom function in the internal function collection. /// /// The custom function to register. /// A C# function wrapping the function execution logic. ISKFunction RegisterCustomFunction(ISKFunction customFunction); - /// - /// Import a set of functions from the given skill. The functions must have the `SKFunction` attribute. - /// Once these functions are imported, the prompt templates can use functions to import content at runtime. - /// - /// Instance of a class containing functions - /// Name of the skill for skill collection and prompt templates. If the value is empty functions are registered in the global namespace. - /// A list of all the semantic functions found in the directory, indexed by function name. - IDictionary ImportSkill(object skillInstance, string? skillName = null); - - /// - /// Set the semantic memory to use - /// - /// Semantic memory instance - void RegisterMemory(ISemanticTextMemory memory); - - /// - /// Run a single synchronous or asynchronous . - /// - /// A Semantic Kernel function to run - /// The to monitor for cancellation requests. The default is . - /// Result of the function composition - Task RunAsync( - ISKFunction skFunction, - CancellationToken cancellationToken = default); - /// /// Run a pipeline composed of synchronous and asynchronous functions. /// + /// Input to process + /// The to monitor for cancellation requests. The default is . /// List of functions /// Result of the function composition - Task RunAsync( + Task RunAsync( + ContextVariables variables, + CancellationToken cancellationToken, params ISKFunction[] pipeline); /// - /// Run a pipeline composed of synchronous and asynchronous functions. + /// Create a new instance of a context, linked to the kernel internal state. /// - /// Input to process - /// List of functions - /// Result of the function composition - Task RunAsync( - string input, - params ISKFunction[] pipeline); + /// Initializes the context with the provided variables + /// Provide specific scoped functions. Defaults to all existing in the kernel + /// Logged factory used within the context + /// Optional culture info related to the context + /// SK context + SKContext CreateNewContext( + ContextVariables? variables = null, + IReadOnlyFunctionCollection? functions = null, + ILoggerFactory? loggerFactory = null, + CultureInfo? culture = null); /// - /// Run a pipeline composed of synchronous and asynchronous functions. + /// Get one of the configured services. Currently limited to AI services. /// - /// Input to process - /// List of functions - /// Result of the function composition - Task RunAsync( - ContextVariables variables, - params ISKFunction[] pipeline); + /// Optional name. If the name is not provided, returns the default T available + /// Service type + /// Instance of T + T GetService(string? name = null) where T : IAIService; /// - /// Run a pipeline composed of synchronous and asynchronous functions. + /// Used for registering a function invoking event handler. + /// Triggers before each function invocation. /// - /// The to monitor for cancellation requests. The default is . - /// List of functions - /// Result of the function composition - Task RunAsync( - CancellationToken cancellationToken, - params ISKFunction[] pipeline); + event EventHandler? FunctionInvoking; /// - /// Run a pipeline composed of synchronous and asynchronous functions. + /// Used for registering a function invoked event handler. + /// Triggers after each function invocation. /// - /// Input to process - /// The to monitor for cancellation requests. The default is . - /// List of functions - /// Result of the function composition - Task RunAsync( - string input, - CancellationToken cancellationToken, - params ISKFunction[] pipeline); + event EventHandler? FunctionInvoked; + + #region Obsolete /// - /// Run a pipeline composed of synchronous and asynchronous functions. + /// Semantic memory instance /// - /// Input to process - /// The to monitor for cancellation requests. The default is . - /// List of functions - /// Result of the function composition - Task RunAsync( - ContextVariables variables, - CancellationToken cancellationToken, - params ISKFunction[] pipeline); + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + ISemanticTextMemory Memory { get; } + + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use Kernel.Functions instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + IReadOnlyFunctionCollection Skills { get; } +#pragma warning restore CS1591 /// - /// Access registered functions by skill + name. Not case sensitive. + /// Access registered functions by plugin name and function name. Not case sensitive. /// The function might be native or semantic, it's up to the caller handling it. /// - /// Skill name + /// Plugin name /// Function name /// Delegate to execute the function - ISKFunction Func(string skillName, string functionName); + [Obsolete("Func shorthand no longer no longer supported. Use Kernel.Plugins collection instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + ISKFunction Func(string pluginName, string functionName); - /// - /// Create a new instance of a context, linked to the kernel internal state. - /// - /// SK context - SKContext CreateNewContext(); + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use Kernel.ImportFunctions instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + IDictionary ImportSkill(object functionsInstance, string? pluginName = null); +#pragma warning restore CS1591 /// - /// Create a new instance of a context, linked to the kernel internal state. + /// Set the semantic memory to use /// - /// Optional cancellation token for operations in context. - /// SK context - [Obsolete("SKContext no longer contains the CancellationToken. Use CreateNewContext().")] - SKContext CreateNewContext(CancellationToken cancellationToken); + /// Semantic memory instance + /// + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + void RegisterMemory(ISemanticTextMemory memory); - /// - /// Get one of the configured services. Currently limited to AI services. - /// - /// Optional name. If the name is not provided, returns the default T available - /// Service type - /// Instance of T - T GetService(string? name = null) where T : IAIService; + #endregion } diff --git a/dotnet/src/SemanticKernel.Abstractions/KernelConfig.cs b/dotnet/src/SemanticKernel.Abstractions/KernelConfig.cs deleted file mode 100644 index 1594fe926113..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/KernelConfig.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel.Reliability; - -namespace Microsoft.SemanticKernel; - -/// -/// Semantic kernel configuration. -/// TODO: use .NET ServiceCollection (will require a lot of changes) -/// -public sealed class KernelConfig -{ - /// - /// Factory for creating HTTP handlers. - /// - public IDelegatingHandlerFactory HttpHandlerFactory { get; private set; } = new DefaultHttpRetryHandlerFactory(new HttpRetryConfig()); - - /// - /// Default HTTP retry configuration for built-in HTTP handler factory. - /// - public HttpRetryConfig DefaultHttpRetryConfig { get; private set; } = new(); - - /// - /// Set the http retry handler factory to use for the kernel. - /// - /// Http retry handler factory to use. - /// The updated kernel configuration. - public KernelConfig SetHttpRetryHandlerFactory(IDelegatingHandlerFactory? httpHandlerFactory = null) - { - if (httpHandlerFactory != null) - { - this.HttpHandlerFactory = httpHandlerFactory; - } - - return this; - } - - public KernelConfig SetDefaultHttpRetryConfig(HttpRetryConfig? httpRetryConfig) - { - if (httpRetryConfig != null) - { - this.DefaultHttpRetryConfig = httpRetryConfig; - this.SetHttpRetryHandlerFactory(new DefaultHttpRetryHandlerFactory(httpRetryConfig)); - } - - return this; - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/KernelException.cs b/dotnet/src/SemanticKernel.Abstractions/KernelException.cs deleted file mode 100644 index 681fc4d81c71..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/KernelException.cs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown for errors related to kernel logic. -/// -public class KernelException : SKException -{ - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public KernelException(ErrorCodes errorCode, string? message) - : this(errorCode, message, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public KernelException(ErrorCodes errorCode, string? message = null, Exception? innerException = null) - : base(GetDefaultMessage(errorCode, message), innerException) - { - this.ErrorCode = errorCode; - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// Translate the error code into a default message. - /// The error code. - /// Default error message if nothing available. - private static string GetDefaultMessage(ErrorCodes errorCode, string? defaultMessage) - { - string description = errorCode switch - { - ErrorCodes.InvalidFunctionDescription => "Invalid function description", - ErrorCodes.FunctionOverloadNotSupported => "Function overload not supported", - ErrorCodes.FunctionNotAvailable => "Function not available", - ErrorCodes.FunctionTypeNotSupported => "Function type not supported", - ErrorCodes.InvalidFunctionType => "Invalid function type", - ErrorCodes.InvalidServiceConfiguration => "Invalid service configuration", - ErrorCodes.ServiceNotFound => "Service not found", - ErrorCodes.SkillCollectionNotSet => "Skill collection not set", - ErrorCodes.FunctionInvokeError => "Function invoke error", - _ => $"Unknown error ({errorCode:G})", - }; - - return defaultMessage is not null ? $"{description}: {defaultMessage}" : description; - } - - /// - /// Semantic kernel error codes. - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError = -1, - - /// - /// Invalid function description. - /// - InvalidFunctionDescription, - - /// - /// Function overload not supported. - /// - FunctionOverloadNotSupported, - - /// - /// Function not available. - /// - FunctionNotAvailable, - - /// - /// Function type not supported. - /// - FunctionTypeNotSupported, - - /// - /// Invalid function type. - /// - InvalidFunctionType, - - /// - /// Invalid service configuration. - /// - InvalidServiceConfiguration, - - /// - /// Service not found. - /// - ServiceNotFound, - - /// - /// Skill collection not set. - /// - SkillCollectionNotSet, - - /// - /// Represents an error that occurs when invoking a function. - /// - FunctionInvokeError, - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/Memory/IMemoryStore.cs b/dotnet/src/SemanticKernel.Abstractions/Memory/IMemoryStore.cs index 997dba6a9d17..72825192c64e 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Memory/IMemoryStore.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Memory/IMemoryStore.cs @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI.Embeddings; namespace Microsoft.SemanticKernel.Memory; @@ -100,35 +100,35 @@ public interface IMemoryStore Task RemoveBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default); /// - /// Gets the nearest matches to the of type . Does not guarantee that the collection exists. + /// Gets the nearest matches to an embedding of type . Does not guarantee that the collection exists. /// /// The name associated with a collection of embeddings. - /// The to compare the collection's embeddings with. + /// The embedding to compare the collection's embeddings with. /// The maximum number of similarity results to return. - /// The minimum relevance threshold for returned results. + /// The minimum cosine similarity threshold for returned results. /// If true, the embeddings will be returned in the memory records. /// The to monitor for cancellation requests. The default is . /// A group of tuples where item1 is a and item2 is its similarity score as a . IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0.0, bool withEmbeddings = false, CancellationToken cancellationToken = default); /// - /// Gets the nearest match to the of type . Does not guarantee that the collection exists. + /// Gets the nearest match to an embedding of type . Does not guarantee that the collection exists. /// /// The name associated with a collection of embeddings. - /// The to compare the collection's embeddings with. + /// The embedding to compare the collection's embeddings with. /// The minimum relevance threshold for returned results. /// If true, the embedding will be returned in the memory record. /// The to monitor for cancellation requests. The default is . /// A tuple consisting of the and the similarity score as a . Null if no nearest match found. Task<(MemoryRecord, double)?> GetNearestMatchAsync( string collectionName, - Embedding embedding, + ReadOnlyMemory embedding, double minRelevanceScore = 0.0, bool withEmbedding = false, CancellationToken cancellationToken = default); diff --git a/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryException.cs b/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryException.cs deleted file mode 100644 index 53c8dcd21076..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryException.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Memory; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown for errors related to memory logic. -/// -public class MemoryException : SKException -{ - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public MemoryException(ErrorCodes errorCode, string? message) - : this(errorCode, message, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public MemoryException(ErrorCodes errorCode, string? message = null, Exception? innerException = null) - : base(GetDefaultMessage(errorCode, message), innerException) - { - this.ErrorCode = errorCode; - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// Translate the error code into a default message. - private static string GetDefaultMessage(ErrorCodes errorCode, string? message) - { - string description = errorCode switch - { - ErrorCodes.FailedToCreateCollection => "Failed to create collection", - ErrorCodes.FailedToDeleteCollection => "Failed to delete collection", - ErrorCodes.UnableToDeserializeMetadata => "Unable to deserialize metadata", - ErrorCodes.AttemptedToAccessNonexistentCollection => "Attempted to access non-existent collection", - _ => $"Unknown error ({errorCode:G})", - }; - - return message is not null ? $"{description}: {message}" : description; - } - - /// - /// Semantic kernel memory error codes. - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError = -1, - - /// - /// Failed to create collection. - /// - FailedToCreateCollection, - - /// - /// Failed to delete collection. - /// - FailedToDeleteCollection, - - /// - /// Unable to construct memory from serialized metadata. - /// - UnableToDeserializeMetadata, - - /// - /// Attempted to access a memory collection that does not exist. - /// - AttemptedToAccessNonexistentCollection, - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryQueryResult.cs b/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryQueryResult.cs index a68fac697df0..76efda1317ad 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryQueryResult.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryQueryResult.cs @@ -1,7 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Text.Json.Serialization; -using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Memory; @@ -24,7 +25,8 @@ public class MemoryQueryResult /// /// Nullable embedding associated with the metadata returned for by a query. /// - public Embedding? Embedding { get; } + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory? Embedding { get; } /// /// Create a new instance of MemoryQueryResult @@ -39,20 +41,25 @@ public class MemoryQueryResult public MemoryQueryResult( MemoryRecordMetadata metadata, double relevance, - Embedding? embedding) + ReadOnlyMemory? embedding) { this.Metadata = metadata; this.Relevance = relevance; this.Embedding = embedding; } - internal static MemoryQueryResult FromMemoryRecord( - MemoryRecord rec, + /// + /// Creates instance of based on and search relevance. + /// + /// Instance of . + /// Search relevance, from 0 to 1, where 1 means perfect match. + public static MemoryQueryResult FromMemoryRecord( + MemoryRecord record, double relevance) { return new MemoryQueryResult( - (MemoryRecordMetadata)rec.Metadata.Clone(), + (MemoryRecordMetadata)record.Metadata.Clone(), relevance, - rec.Embedding.IsEmpty ? null : rec.Embedding); + record.Embedding.IsEmpty ? null : record.Embedding); } } diff --git a/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryRecord.cs b/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryRecord.cs index fa4249e66f0e..d87c7a876ed3 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryRecord.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Memory/MemoryRecord.cs @@ -3,7 +3,8 @@ using System; using System.Text.Json; using System.Text.Json.Serialization; -using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.Memory; @@ -16,7 +17,8 @@ public class MemoryRecord : DataEntryBase /// Source content embeddings. /// [JsonPropertyName("embedding")] - public Embedding Embedding { get; } + [JsonConverter(typeof(ReadOnlyMemoryConverter))] + public ReadOnlyMemory Embedding { get; } /// /// Metadata associated with a Semantic Kernel memory. @@ -30,7 +32,7 @@ public class MemoryRecord : DataEntryBase [JsonConstructor] public MemoryRecord( MemoryRecordMetadata metadata, - Embedding embedding, + ReadOnlyMemory embedding, string? key, DateTimeOffset? timestamp = null) : base(key, timestamp) { @@ -54,7 +56,7 @@ public static MemoryRecord ReferenceRecord( string externalId, string sourceName, string? description, - Embedding embedding, + ReadOnlyMemory embedding, string? additionalMetadata = null, string? key = null, DateTimeOffset? timestamp = null) @@ -90,7 +92,7 @@ public static MemoryRecord LocalRecord( string id, string text, string? description, - Embedding embedding, + ReadOnlyMemory embedding, string? additionalMetadata = null, string? key = null, DateTimeOffset? timestamp = null) @@ -120,19 +122,17 @@ public static MemoryRecord LocalRecord( /// Optional existing database key. /// optional timestamp. /// Memory record - /// + /// public static MemoryRecord FromJsonMetadata( string json, - Embedding? embedding, + ReadOnlyMemory embedding, string? key = null, DateTimeOffset? timestamp = null) { var metadata = JsonSerializer.Deserialize(json); return metadata != null - ? new MemoryRecord(metadata, embedding ?? Embedding.Empty, key, timestamp) - : throw new MemoryException( - MemoryException.ErrorCodes.UnableToDeserializeMetadata, - "Unable to create memory record from serialized metadata"); + ? new MemoryRecord(metadata, embedding, key, timestamp) + : throw new SKException("Unable to create memory record from serialized metadata"); } /// @@ -145,11 +145,11 @@ public static MemoryRecord FromJsonMetadata( /// Memory record public static MemoryRecord FromMetadata( MemoryRecordMetadata metadata, - Embedding? embedding, + ReadOnlyMemory embedding, string? key = null, DateTimeOffset? timestamp = null) { - return new MemoryRecord(metadata, embedding ?? Embedding.Empty, key, timestamp); + return new MemoryRecord(metadata, embedding, key, timestamp); } /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/ContextVariables.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/ContextVariables.cs index 4ed2404ba844..42369f39c7d1 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Orchestration/ContextVariables.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/ContextVariables.cs @@ -1,18 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; +using System.Linq; using Microsoft.SemanticKernel.Diagnostics; -#pragma warning disable CA1710 // ContextVariables doesn't end in Dictionary or Collection -#pragma warning disable CA1725, RCS1168 // Uses "name" instead of "key" for some public APIs -#pragma warning disable CS8767 // Reference type nullability doesn't match because netstandard2.0 surface area isn't nullable reference type annotated -// TODO: support more complex data types, and plan for rendering these values into prompt templates. - namespace Microsoft.SemanticKernel.Orchestration; /// @@ -21,15 +14,16 @@ namespace Microsoft.SemanticKernel.Orchestration; /// [DebuggerDisplay("{DebuggerDisplay,nq}")] [DebuggerTypeProxy(typeof(ContextVariables.TypeProxy))] -public sealed class ContextVariables : IDictionary +public sealed class ContextVariables : Dictionary { /// /// Constructor for context variables. /// /// Optional value for the main variable of the context including trust information. public ContextVariables(string? value = null) + : base(StringComparer.OrdinalIgnoreCase) { - this._variables[MainKey] = value ?? string.Empty; + this.Set(MainKey, value); } /// @@ -39,7 +33,7 @@ public ContextVariables(string? value = null) public ContextVariables Clone() { var clone = new ContextVariables(); - foreach (KeyValuePair x in this._variables) + foreach (KeyValuePair x in this) { clone.Set(x.Key, x.Value); } @@ -49,41 +43,17 @@ public ContextVariables Clone() /// Gets the main input string. /// If the main input string was removed from the collection, an empty string will be returned. - public string Input => this._variables.TryGetValue(MainKey, out string? value) ? value : string.Empty; + public string Input => this.TryGetValue(MainKey, out string? value) ? value : string.Empty; /// /// Updates the main input text with the new value after a function is complete. - /// The string includes trust information and will overwrite the trust state of the input. /// /// The new input value, for the next function in the pipeline, or as a result for the user /// if the pipeline reached the end. /// The current instance public ContextVariables Update(string? value) { - this._variables[MainKey] = value ?? string.Empty; - return this; - } - - /// - /// Updates all the local data with new data, merging the two datasets. - /// Do not discard old data - /// - /// New data to be merged - /// Whether to merge and keep old data, or replace. False == discard old data. - /// The current instance - public ContextVariables Update(ContextVariables newData, bool merge = true) - { - if (!object.ReferenceEquals(this, newData)) - { - // If requested, discard old data and keep only the new one. - if (!merge) { this._variables.Clear(); } - - foreach (KeyValuePair varData in newData._variables) - { - this._variables[varData.Key] = varData.Value; - } - } - + this.Set(MainKey, value); return this; } @@ -91,7 +61,6 @@ public ContextVariables Update(ContextVariables newData, bool merge = true) /// This method allows to store additional data in the context variables, e.g. variables needed by functions in the /// pipeline. These "variables" are visible also to semantic functions using the "{{varName}}" syntax, allowing /// to inject more information into prompt templates. - /// The string value includes trust information and will overwrite the trust information already stored for the variable. /// /// Variable name /// Value to store. If the value is NULL the variable is deleted. @@ -100,103 +69,30 @@ public void Set(string name, string? value) Verify.NotNullOrWhiteSpace(name); if (value != null) { - this._variables[name] = value; + this[name] = value; } else { - this._variables.TryRemove(name, out _); - } - } - - /// - /// Gets the variable value associated with the specified name. - /// - /// The name of the variable to get. - /// - /// When this method returns, contains the variable value associated with the specified name, if the variable is found; - /// otherwise, null. - /// - /// true if the contains a variable with the specified name; otherwise, false. - public bool TryGetValue(string name, [NotNullWhen(true)] out string? value) - { - if (this._variables.TryGetValue(name, out value)) - { - return true; - } - - value = null; - return false; - } - - /// - /// Array of all variables in the context variables. - /// - /// The name of the variable. - /// The value of the variable. - public string this[string name] - { - get => this._variables[name]; - set - { - this._variables[name] = value; + this.Remove(name); } } - /// - /// Determines whether the contains the specified variable. - /// - /// The name of the variable to locate. - /// true if the contains a variable with the specified name; otherwise, false. - public bool ContainsKey(string name) - { - return this._variables.ContainsKey(name); - } - /// /// Print the processed input, aka the current data after any processing occurred. /// /// Processed input, aka result public override string ToString() => this.Input; - /// - /// Get an enumerator that iterates through the context variables. - /// - /// An enumerator that iterates through the context variables - public IEnumerator> GetEnumerator() => this._variables.GetEnumerator(); - - IEnumerator IEnumerable.GetEnumerator() => this._variables.GetEnumerator(); - void IDictionary.Add(string key, string value) => ((IDictionary)this._variables).Add(key, value); - bool IDictionary.Remove(string key) => ((IDictionary)this._variables).Remove(key); - void ICollection>.Add(KeyValuePair item) => ((ICollection>)this._variables).Add(item); - void ICollection>.Clear() => ((ICollection>)this._variables).Clear(); - bool ICollection>.Contains(KeyValuePair item) => ((ICollection>)this._variables).Contains(item); - void ICollection>.CopyTo(KeyValuePair[] array, int arrayIndex) => ((ICollection>)this._variables).CopyTo(array, arrayIndex); - bool ICollection>.Remove(KeyValuePair item) => ((ICollection>)this._variables).Remove(item); - ICollection IDictionary.Keys => ((IDictionary)this._variables).Keys; - ICollection IDictionary.Values => ((IDictionary)this._variables).Values; - int ICollection>.Count => ((ICollection>)this._variables).Count; - bool ICollection>.IsReadOnly => ((ICollection>)this._variables).IsReadOnly; - string IDictionary.this[string key] - { - get => ((IDictionary)this._variables)[key]; - set => ((IDictionary)this._variables)[key] = value; - } - internal const string MainKey = "INPUT"; [DebuggerBrowsable(DebuggerBrowsableState.Never)] internal string DebuggerDisplay => this.TryGetValue(MainKey, out string? input) && !string.IsNullOrEmpty(input) - ? $"Variables = {this._variables.Count}, Input = {input}" - : $"Variables = {this._variables.Count}"; + ? $"Variables = {this.Count}, Input = {input}" + : $"Variables = {this.Count}"; #region private ================================================================================ - /// - /// Important: names are case insensitive - /// - private readonly ConcurrentDictionary _variables = new(StringComparer.OrdinalIgnoreCase); - private sealed class TypeProxy { private readonly ContextVariables _variables; @@ -204,7 +100,7 @@ private sealed class TypeProxy public TypeProxy(ContextVariables variables) => this._variables = variables; [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] - public KeyValuePair[] Items => this._variables._variables.ToArray(); + public KeyValuePair[] Items => this._variables.ToArray(); } #endregion diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/FunctionResult.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/FunctionResult.cs new file mode 100644 index 000000000000..eaac4d7ae3e1 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/FunctionResult.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; + +namespace Microsoft.SemanticKernel.Orchestration; + +/// +/// Function result after execution. +/// +public sealed class FunctionResult +{ + internal Dictionary? _metadata; + + /// + /// Name of executed function. + /// + public string FunctionName { get; internal set; } + + /// + /// Name of the plugin containing the function. + /// + public string PluginName { get; internal set; } + + /// + /// Metadata for storing additional information about function execution result. + /// + public Dictionary Metadata + { + get => this._metadata ??= new(); + internal set => this._metadata = value; + } + + /// + /// Function result object. + /// + internal object? Value { get; private set; } = null; + + /// + /// Instance of to pass in function pipeline. + /// + internal SKContext Context { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// Name of executed function. + /// Name of the plugin containing the function. + /// Instance of to pass in function pipeline. + public FunctionResult(string functionName, string pluginName, SKContext context) + { + this.FunctionName = functionName; + this.PluginName = pluginName; + this.Context = context; + } + + /// + /// Initializes a new instance of the class. + /// + /// Name of executed function. + /// Name of the plugin containing the function. + /// Instance of to pass in function pipeline. + /// Function result object. + public FunctionResult(string functionName, string pluginName, SKContext context, object? value) + : this(functionName, pluginName, context) + { + this.Value = value; + } + + /// + /// Returns function result value. + /// + /// Target type for result value casting. + /// Thrown when it's not possible to cast result value to . + public T? GetValue() + { + if (this.Value is null) + { + return default; + } + + if (this.Value is T typedResult) + { + return typedResult; + } + + throw new InvalidCastException($"Cannot cast {this.Value.GetType()} to {typeof(T)}"); + } + + /// + /// Get typed value from metadata. + /// + public bool TryGetMetadataValue(string key, out T value) + { + if (this._metadata is { } metadata && + metadata.TryGetValue(key, out object? valueObject) && + valueObject is T typedValue) + { + value = typedValue; + return true; + } + + value = default!; + return false; + } + + /// + public override string ToString() => this.Value?.ToString() ?? base.ToString(); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs new file mode 100644 index 000000000000..71357d2f304a --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.SemanticKernel.Orchestration; + +/// +/// Function runner interface. +/// +public interface IFunctionRunner +{ + /// + /// Execute a function using the resources loaded in the context. + /// + /// Target function to run + /// Input to process + /// The to monitor for cancellation requests. The default is . + /// Result of the function composition + Task RunAsync( + ISKFunction skFunction, + ContextVariables? variables = null, + CancellationToken cancellationToken = default); + + /// + /// Execute a function using the resources loaded in the context. + /// + /// The name of the plugin containing the function to run + /// The name of the function to run + /// Input to process + /// The to monitor for cancellation requests. The default is . + /// Result of the function composition + Task RunAsync( + string pluginName, + string functionName, + ContextVariables? variables = null, + CancellationToken cancellationToken = default); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/KernelResult.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/KernelResult.cs new file mode 100644 index 000000000000..d4537579e286 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/KernelResult.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; + +namespace Microsoft.SemanticKernel.Orchestration; + +/// +/// Kernel result after execution. +/// +public sealed class KernelResult +{ + /// + /// Results from all functions in pipeline. + /// + public IReadOnlyCollection FunctionResults { get; internal set; } = Array.Empty(); + + /// + /// Kernel result object. + /// + internal object? Value { get; private set; } = null; + + /// + /// Returns kernel result value. + /// + /// Target type for result value casting. + /// Thrown when it's not possible to cast result value to . + public T? GetValue() + { + if (this.Value is null) + { + return default; + } + + if (this.Value is T typedResult) + { + return typedResult; + } + + throw new InvalidCastException($"Cannot cast {this.Value.GetType()} to {typeof(T)}"); + } + + /// + /// Creates instance of based on function results. + /// + /// Kernel result object. + /// Results from all functions in pipeline. + public static KernelResult FromFunctionResults(object? value, IReadOnlyCollection functionResults) + { + return new KernelResult + { + Value = value, + FunctionResults = functionResults + }; + } + + /// + public override string ToString() => this.Value?.ToString() ?? base.ToString(); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/ModelResult.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/ModelResult.cs index c653496cbc2e..0f97d4f4ee40 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Orchestration/ModelResult.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/ModelResult.cs @@ -8,31 +8,53 @@ #pragma warning disable CA1024 namespace Microsoft.SemanticKernel.Orchestration; + +/// +/// Represents a result from a model execution. +/// public sealed class ModelResult { - private readonly object result; + private readonly object _result; + /// + /// Initializes a new instance of the class with the specified result object. + /// + /// The result object to be stored in the ModelResult instance. public ModelResult(object result) { Verify.NotNull(result); - this.result = result; + this._result = result; } - public object GetRawResult() => this.result; - + /// + /// Gets the raw result object stored in the instance. + /// + /// The raw result object. + public object GetRawResult() => this._result; + + /// + /// Gets the result object stored in the instance, cast to the specified type. + /// + /// The type to cast the result object to. + /// The result object cast to the specified type. + /// Thrown when the result object cannot be cast to the specified type. public T GetResult() { - if (this.result is T typedResult) + if (this._result is T typedResult) { return typedResult; } - throw new InvalidCastException($"Cannot cast {this.result.GetType()} to {typeof(T)}"); + throw new InvalidCastException($"Cannot cast {this._result.GetType()} to {typeof(T)}"); } + /// + /// Gets the result object stored in the ModelResult instance as a JSON element. + /// + /// The result object as a JSON element. public JsonElement GetJsonResult() { - return Json.Deserialize(this.result.ToJson()); + return Json.Deserialize(Json.Serialize(this._result)); } } diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/SKContext.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/SKContext.cs index 9ea6c27abf1b..cb220a3ee648 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Orchestration/SKContext.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/SKContext.cs @@ -4,11 +4,9 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; -using System.Threading; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.SkillDefinition; +using Microsoft.SemanticKernel.Diagnostics; namespace Microsoft.SemanticKernel.Orchestration; @@ -18,11 +16,6 @@ namespace Microsoft.SemanticKernel.Orchestration; [DebuggerDisplay("{DebuggerDisplay,nq}")] public sealed class SKContext { - /// - /// The culture currently associated with this context. - /// - private CultureInfo _culture; - /// /// Print the processed input, aka the current data after any processing occurred. /// @@ -33,7 +26,8 @@ public sealed class SKContext /// When a prompt is processed, aka the current data after any model results processing occurred. /// (One prompt can have multiple results). /// - public IReadOnlyCollection ModelResults { get; set; } = Array.Empty(); + [Obsolete($"ModelResults are now part of {nameof(FunctionResult.Metadata)} property. Use 'ModelResults' key or available extension methods to get model results.")] + public IReadOnlyCollection ModelResults => Array.Empty(); /// /// The culture currently associated with this context. @@ -50,95 +44,94 @@ public CultureInfo Culture public ContextVariables Variables { get; } /// - /// Read only skills collection + /// Read only functions collection /// - public IReadOnlySkillCollection Skills { get; } + public IReadOnlyFunctionCollection Functions { get; } /// - /// Access registered functions by skill + name. Not case sensitive. - /// The function might be native or semantic, it's up to the caller handling it. - /// - /// Skill name - /// Function name - /// Delegate to execute the function - public ISKFunction Func(string skillName, string functionName) - { - return this.Skills.GetFunction(skillName, functionName); - } - - /// - /// App logger (obsolete - use 'Logger' instead). + /// App logger /// - [Obsolete("Use SKContext.Logger instead. This will be removed in a future release.")] - public ILogger Log => this.Logger; + public ILoggerFactory LoggerFactory { get; } /// - /// App logger + /// Executes functions using the current resources loaded in the context /// - public ILogger Logger { get; } + public IFunctionRunner Runner { get; } /// /// Constructor for the context. /// + /// Function runner reference /// Context variables to include in context. - /// Skills to include in context. - /// Logger for operations in context. - public SKContext( + /// Functions to include in context. + /// Logger factory to be used in context + /// Culture related to the context + internal SKContext( + IFunctionRunner functionRunner, ContextVariables? variables = null, - IReadOnlySkillCollection? skills = null, - ILogger? logger = null) + IReadOnlyFunctionCollection? functions = null, + ILoggerFactory? loggerFactory = null, + CultureInfo? culture = null) { + Verify.NotNull(functionRunner, nameof(functionRunner)); + + this.Runner = functionRunner; this.Variables = variables ?? new(); - this.Skills = skills ?? NullReadOnlySkillCollection.Instance; - this.Logger = logger ?? NullLogger.Instance; - this._culture = CultureInfo.CurrentCulture; + this.Functions = functions ?? NullReadOnlyFunctionCollection.Instance; + this.LoggerFactory = loggerFactory ?? NullLoggerFactory.Instance; + this._culture = culture ?? CultureInfo.CurrentCulture; } /// /// Print the processed input, aka the current data after any processing occurred. - /// If an error occurred, prints the last exception message instead. /// - /// Processed input, aka result, or last exception message if any + /// Processed input, aka result. public override string ToString() { - return this.ErrorOccurred ? $"Error: {this.LastErrorDescription}" : this.Result; + return this.Result; } /// - /// Create a clone of the current context, using the same kernel references (memory, skills, logger) + /// Create a clone of the current context, using the same kernel references (memory, functions, logger) /// and a new set variables, so that variables can be modified without affecting the original context. /// - /// A new context copied from the current one + /// A new context cloned from the current one public SKContext Clone() + => this.Clone(null, null); + + /// + /// Create a clone of the current context, using the same kernel references (memory, functions, logger) + /// and optionally allows overriding the variables and functions. + /// + /// Override the variables with the provided ones + /// Override the functions with the provided ones + /// A new context cloned from the current one + public SKContext Clone(ContextVariables? variables, IReadOnlyFunctionCollection? functions) { return new SKContext( - variables: this.Variables.Clone(), - skills: this.Skills, - logger: this.Logger) - { - Culture = this.Culture, - ErrorOccurred = this.ErrorOccurred, - LastErrorDescription = this.LastErrorDescription, - LastException = this.LastException, - }; + this.Runner, + variables ?? this.Variables.Clone(), + functions ?? this.Functions, + this.LoggerFactory, + this.Culture); } + /// + /// The culture currently associated with this context. + /// + private CultureInfo _culture; + [DebuggerBrowsable(DebuggerBrowsableState.Never)] private string DebuggerDisplay { get { - if (this.ErrorOccurred) - { - return $"Error: {this.LastErrorDescription}"; - } - string display = this.Variables.DebuggerDisplay; - if (this.Skills is IReadOnlySkillCollection skills) + if (this.Functions is IReadOnlyFunctionCollection functions) { - var view = skills.GetFunctionsView(); - display += $", Skills = {view.NativeFunctions.Count + view.SemanticFunctions.Count}"; + var view = functions.GetFunctionViews(); + display += $", Functions = {view.Count}"; } display += $", Culture = {this.Culture.EnglishName}"; @@ -146,65 +139,4 @@ private string DebuggerDisplay return display; } } - - #region Error handling - /// - /// Whether an error occurred while executing functions in the pipeline. - /// - public bool ErrorOccurred { get; private set; } - - /// - /// Error details. - /// - public string LastErrorDescription { get; private set; } = string.Empty; - - /// - /// When an error occurs, this is the most recent exception. - /// - public Exception? LastException { get; private set; } - - /// - /// Call this method to signal when an error occurs. - /// In the usual scenarios this is also how execution is stopped, e.g. to inform the user or take necessary steps. - /// - /// Error description - /// If available, the exception occurred - /// The current instance - public SKContext Fail(string errorDescription, Exception? exception = null) - { - this.ErrorOccurred = true; - this.LastErrorDescription = errorDescription; - this.LastException = exception; - return this; - } - #endregion - - #region Obsolete - /// - /// Shortcut into user data, access variables by name - /// - /// Variable name - [Obsolete("Use SKContext.Variables instead. The SKContext[...] indexer will be removed in a future release.")] - public string this[string name] - { - get => this.Variables[name]; - set => this.Variables[name] = value; - } - - /// - /// The token to monitor for cancellation requests. - /// - [Obsolete("Add a CancellationToken param to SKFunction method signatures instead of retrieving it from SKContext.")] - public CancellationToken CancellationToken { get; } = default; - - /// - /// Semantic memory - /// - [Obsolete("Memory no longer passed through SKContext. Instead, initialize your skill class with the memory provider it needs.")] - public ISemanticTextMemory Memory - { - get => throw new InvalidOperationException( - "Memory no longer passed through SKContext. Instead, initialize your skill class with the memory provider it needs."); - } - #endregion } diff --git a/dotnet/src/SemanticKernel.Abstractions/Planning/PlanningException.cs b/dotnet/src/SemanticKernel.Abstractions/Planning/PlanningException.cs deleted file mode 100644 index d3c60f5332e7..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/Planning/PlanningException.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Planning; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown for errors related to planning. -/// -public class PlanningException : SKException -{ - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public PlanningException(ErrorCodes errorCode, string? message) - : this(errorCode, message, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public PlanningException(ErrorCodes errorCode, string? message = null, Exception? innerException = null) - : base(GetDefaultMessage(errorCode, message), innerException) - { - this.ErrorCode = errorCode; - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// Translate the error code into a default message. - private static string GetDefaultMessage(ErrorCodes errorCode, string? message) - { - string description = errorCode switch - { - ErrorCodes.InvalidGoal => "Invalid goal", - ErrorCodes.InvalidPlan => "Invalid plan", - ErrorCodes.InvalidConfiguration => "Invalid configuration", - ErrorCodes.CreatePlanError => "Create plan error", - _ => $"Unknown error ({errorCode:G})", - }; - - return message is not null ? $"{description}: {message}" : description; - } - - /// - /// Error codes for . - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError = -1, - - /// - /// Invalid goal. - /// - InvalidGoal, - - /// - /// Invalid plan. - /// - InvalidPlan, - - /// - /// Invalid configuration. - /// - InvalidConfiguration, - - /// - /// Create plan error. - /// - CreatePlanError, - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/Reliability/DefaultHttpRetryHandler.cs b/dotnet/src/SemanticKernel.Abstractions/Reliability/DefaultHttpRetryHandler.cs index 109ec8c2839b..c9997c6c2a1d 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Reliability/DefaultHttpRetryHandler.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Reliability/DefaultHttpRetryHandler.cs @@ -9,27 +9,30 @@ using Microsoft.Extensions.Logging.Abstractions; namespace Microsoft.SemanticKernel.Reliability; - +/// +/// A delegating handler that provides retry logic for HTTP requests. +/// +[Obsolete("Usage of Semantic Kernel internal retry abstractions is deprecated.\nCheck KernelSyntaxExamples.Example42_KernelBuilder.cs for alternatives")] public sealed class DefaultHttpRetryHandler : DelegatingHandler { /// /// Initializes a new instance of the class. /// /// The retry configuration. - /// The logger. - public DefaultHttpRetryHandler(HttpRetryConfig? config = null, ILogger? logger = null) - : this(config ?? new HttpRetryConfig(), logger, null, null) + /// The to use for logging. If null, no logging will be performed. + public DefaultHttpRetryHandler(HttpRetryConfig? config = null, ILoggerFactory? loggerFactory = null) + : this(config ?? new HttpRetryConfig(), loggerFactory, null, null) { } internal DefaultHttpRetryHandler( HttpRetryConfig config, - ILogger? logger = null, + ILoggerFactory? loggerFactory = null, IDelayProvider? delayProvider = null, ITimeProvider? timeProvider = null) { this._config = config; - this._logger = logger ?? NullLogger.Instance; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(DefaultHttpRetryHandler)) : NullLogger.Instance; this._delayProvider = delayProvider ?? new TaskDelayProvider(); this._timeProvider = timeProvider ?? new DefaultTimeProvider(); } diff --git a/dotnet/src/SemanticKernel.Abstractions/Reliability/DefaultHttpRetryHandlerFactory.cs b/dotnet/src/SemanticKernel.Abstractions/Reliability/DefaultHttpRetryHandlerFactory.cs index fb40453d8ebe..5dd07072837f 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Reliability/DefaultHttpRetryHandlerFactory.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Reliability/DefaultHttpRetryHandlerFactory.cs @@ -1,21 +1,40 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Net.Http; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Http; namespace Microsoft.SemanticKernel.Reliability; +/// +/// Deprecated A factory class for creating instances of . +/// Implements the interface. +/// +[Obsolete("Usage of Semantic Kernel internal retry abstractions is deprecated.\nCheck KernelSyntaxExamples.Example42_KernelBuilder.cs for alternatives")] public class DefaultHttpRetryHandlerFactory : IDelegatingHandlerFactory { + /// + /// Initializes a new instance of the class. + /// + /// An optional instance to configure the retry behavior. If not provided, default configuration will be used. public DefaultHttpRetryHandlerFactory(HttpRetryConfig? config = null) { - this._config = config; + this.Config = config; } - public DelegatingHandler Create(ILogger? logger) + /// + /// Creates a new instance of with the specified logger. + /// + /// The to use for logging. If null, no logging will be performed. + /// A new instance of . + public DelegatingHandler Create(ILoggerFactory? loggerFactory) { - return new DefaultHttpRetryHandler(this._config, logger); + return new DefaultHttpRetryHandler(this.Config, loggerFactory); } - private readonly HttpRetryConfig? _config; + /// + /// Gets the instance used to configure the retry behavior. + /// + public HttpRetryConfig? Config { get; } } diff --git a/dotnet/src/SemanticKernel.Abstractions/Reliability/HttpRetryConfig.cs b/dotnet/src/SemanticKernel.Abstractions/Reliability/HttpRetryConfig.cs index d2f840d04c1d..e18677abd853 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Reliability/HttpRetryConfig.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Reliability/HttpRetryConfig.cs @@ -11,6 +11,7 @@ namespace Microsoft.SemanticKernel.Reliability; /// /// Retry configuration for IHttpRetryPolicy that uses RetryAfter header when present. /// +[Obsolete("Usage of Semantic Kernel internal retry abstractions is deprecated.\nCheck KernelSyntaxExamples.Example42_KernelBuilder.cs for alternatives")] public sealed class HttpRetryConfig { /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Reliability/IDelegatingHandlerFactory.cs b/dotnet/src/SemanticKernel.Abstractions/Reliability/IDelegatingHandlerFactory.cs deleted file mode 100644 index 68f09e6fdc2d..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/Reliability/IDelegatingHandlerFactory.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Net.Http; -using Microsoft.Extensions.Logging; - -namespace Microsoft.SemanticKernel.Reliability; - -/// -/// Factory for creating instances. -/// -public interface IDelegatingHandlerFactory -{ - DelegatingHandler Create(ILogger? logger); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/IPromptTemplate.cs b/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/IPromptTemplate.cs deleted file mode 100644 index e38a2182f7a2..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/IPromptTemplate.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.SemanticFunctions; - -/// -/// Interface for prompt template. -/// -public interface IPromptTemplate -{ - /// - /// Get the list of parameters required by the template, using configuration and template info. - /// - /// List of parameters - IList GetParameters(); - - /// - /// Render the template using the information in the context - /// - /// Kernel execution context helpers - /// The to monitor for cancellation requests. The default is . - /// Prompt rendered to string - public Task RenderAsync(SKContext executionContext, CancellationToken cancellationToken = default); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/PromptTemplateConfig.cs b/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/PromptTemplateConfig.cs deleted file mode 100644 index 1e3e34e43602..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/PromptTemplateConfig.cs +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; -using Microsoft.SemanticKernel.Text; - -namespace Microsoft.SemanticKernel.SemanticFunctions; - -/// -/// Prompt template configuration. -/// -public class PromptTemplateConfig -{ - /// - /// Completion configuration parameters. - /// - public class CompletionConfig - { - /// - /// Sampling temperature to use, between 0 and 2. Higher values will make the output more random. - /// Lower values will make it more focused and deterministic. - /// - [JsonPropertyName("temperature")] - [JsonPropertyOrder(1)] - public double Temperature { get; set; } = 0.0f; - - /// - /// Cut-off of top_p probability mass of tokens to consider. - /// For example, 0.1 means only the tokens comprising the top 10% probability mass are considered. - /// - [JsonPropertyName("top_p")] - [JsonPropertyOrder(2)] - public double TopP { get; set; } = 0.0f; - - /// - /// Lowers the probability of a word appearing if it already appeared in the predicted text. - /// Unlike the frequency penalty, the presence penalty does not depend on the frequency at which words - /// appear in past predictions. - /// - [JsonPropertyName("presence_penalty")] - [JsonPropertyOrder(3)] - public double PresencePenalty { get; set; } = 0.0f; - - /// - /// Controls the model’s tendency to repeat predictions. The frequency penalty reduces the probability - /// of words that have already been generated. The penalty depends on how many times a word has already - /// occurred in the prediction. - /// - [JsonPropertyName("frequency_penalty")] - [JsonPropertyOrder(4)] - public double FrequencyPenalty { get; set; } = 0.0f; - - /// - /// Maximum number of tokens that can be generated. - /// - [JsonPropertyName("max_tokens")] - [JsonPropertyOrder(5)] - public int? MaxTokens { get; set; } - - /// - /// Stop sequences are optional sequences that tells the AI model when to stop generating tokens. - /// - [JsonPropertyName("stop_sequences")] - [JsonPropertyOrder(6)] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List StopSequences { get; set; } = new(); - } - - /// - /// Input parameter for semantic functions. - /// - public class InputParameter - { - /// - /// Name of the parameter to pass to the function. - /// e.g. when using "{{$input}}" the name is "input", when using "{{$style}}" the name is "style", etc. - /// - [JsonPropertyName("name")] - [JsonPropertyOrder(1)] - public string Name { get; set; } = string.Empty; - - /// - /// Parameter description for UI apps and planner. Localization is not supported here. - /// - [JsonPropertyName("description")] - [JsonPropertyOrder(2)] - public string Description { get; set; } = string.Empty; - - /// - /// Default value when nothing is provided. - /// - [JsonPropertyName("defaultValue")] - [JsonPropertyOrder(3)] - public string DefaultValue { get; set; } = string.Empty; - } - - /// - /// Input configuration (list of all input parameters for a semantic function). - /// - public class InputConfig - { - [JsonPropertyName("parameters")] - [JsonPropertyOrder(1)] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List Parameters { get; set; } = new(); - } - - /// - /// Schema - Not currently used. - /// - [JsonPropertyName("schema")] - [JsonPropertyOrder(1)] - public int Schema { get; set; } = 1; - - /// - /// Type, such as "completion", "embeddings", etc. - /// - /// TODO: use enum - [JsonPropertyName("type")] - [JsonPropertyOrder(2)] - public string Type { get; set; } = "completion"; - - /// - /// Description - /// - [JsonPropertyName("description")] - [JsonPropertyOrder(3)] - public string Description { get; set; } = string.Empty; - - /// - /// Completion configuration parameters. - /// - [JsonPropertyName("completion")] - [JsonPropertyOrder(4)] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public CompletionConfig Completion { get; set; } = new(); - - /// - /// Default AI services to use. - /// - [JsonPropertyName("default_services")] - [JsonPropertyOrder(5)] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List DefaultServices { get; set; } = new(); - - /// - /// Input configuration (that is, list of all input parameters). - /// - [JsonPropertyName("input")] - [JsonPropertyOrder(6)] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public InputConfig Input { get; set; } = new(); - - /// - /// Remove some default properties to reduce the JSON complexity. - /// - /// Compacted prompt template configuration. - public PromptTemplateConfig Compact() - { - if (this.Completion.StopSequences.Count == 0) - { - this.Completion.StopSequences = null!; - } - - if (this.DefaultServices.Count == 0) - { - this.DefaultServices = null!; - } - - return this; - } - - /// - /// Creates a prompt template configuration from JSON. - /// - /// JSON of the prompt template configuration. - /// Prompt template configuration. - public static PromptTemplateConfig FromJson(string json) - { - var result = Json.Deserialize(json); - return result ?? throw new ArgumentException("Unable to deserialize prompt template config from argument. The deserialization returned null.", nameof(json)); - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/SemanticFunctionConfig.cs b/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/SemanticFunctionConfig.cs deleted file mode 100644 index 20af5f86e7c5..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/SemanticFunctionConfig.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.SemanticKernel.SemanticFunctions; - -/// -/// Semantic function configuration. -/// -public sealed class SemanticFunctionConfig -{ - /// - /// Prompt template configuration. - /// - public PromptTemplateConfig PromptTemplateConfig { get; } - - /// - /// Prompt template. - /// - public IPromptTemplate PromptTemplate { get; } - - /// - /// Constructor for SemanticFunctionConfig. - /// - /// Prompt template configuration. - /// Prompt template. - public SemanticFunctionConfig( - PromptTemplateConfig config, - IPromptTemplate template) - { - this.PromptTemplateConfig = config; - this.PromptTemplate = template; - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj b/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj index 8611f93d350f..c11cf51e2215 100644 --- a/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj +++ b/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj @@ -25,7 +25,8 @@ - + + diff --git a/dotnet/src/SemanticKernel.Abstractions/Services/IAIService.cs b/dotnet/src/SemanticKernel.Abstractions/Services/IAIService.cs index 3d2ab690bfb5..54085c6f5b4b 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Services/IAIService.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Services/IAIService.cs @@ -4,6 +4,9 @@ namespace Microsoft.SemanticKernel.Services; +/// +/// Represents an empty interface for AI services. +/// [SuppressMessage("Design", "CA1040:Avoid empty interfaces")] public interface IAIService { diff --git a/dotnet/src/SemanticKernel.Abstractions/Services/IAIServiceProvider.cs b/dotnet/src/SemanticKernel.Abstractions/Services/IAIServiceProvider.cs index 5f4ff9a507cf..293973f6ed14 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Services/IAIServiceProvider.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Services/IAIServiceProvider.cs @@ -2,6 +2,9 @@ namespace Microsoft.SemanticKernel.Services; +/// +/// Represents an interface for AI service providers that implements the INamedServiceProvider interface. +/// public interface IAIServiceProvider : INamedServiceProvider { } diff --git a/dotnet/src/SemanticKernel.Abstractions/Services/INamedServiceProvider.cs b/dotnet/src/SemanticKernel.Abstractions/Services/INamedServiceProvider.cs index a0dc4c201ec8..2cb9263f8b86 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Services/INamedServiceProvider.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Services/INamedServiceProvider.cs @@ -2,6 +2,10 @@ namespace Microsoft.SemanticKernel.Services; +/// +/// Represents a named service provider that can retrieve services by type and name. +/// +/// The base type of the services provided by this provider. public interface INamedServiceProvider { /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Services/IServiceConfig.cs b/dotnet/src/SemanticKernel.Abstractions/Services/IServiceConfig.cs deleted file mode 100644 index e89553231597..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/Services/IServiceConfig.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.SemanticKernel.Services; - -/// -/// Service configuration. -/// -public interface IServiceConfig -{ - /// - /// An identifier used to map semantic functions to AI connectors, - /// decoupling prompts configurations from the actual model and AI provider used. - /// - public string ServiceId { get; } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/FunctionView.cs b/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/FunctionView.cs deleted file mode 100644 index 8cbea2fa8d7f..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/FunctionView.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Diagnostics; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Class used to copy and export data from the skill collection. -/// The data is mutable, but changes do not affect the skill collection. -/// -[DebuggerDisplay("{DebuggerDisplay,nq}")] -public sealed class FunctionView -{ - /// - /// Name of the function. The name is used by the skill collection and in prompt templates e.g. {{skillName.functionName}} - /// - public string Name { get; set; } = string.Empty; - - /// - /// Name of the skill containing the function. The name is used by the skill collection and in prompt templates e.g. {{skillName.functionName}} - /// - public string SkillName { get; set; } = string.Empty; - - /// - /// Function description. The description is used in combination with embeddings when searching relevant functions. - /// - public string Description { get; set; } = string.Empty; - - /// - /// Whether the delegate points to a semantic function - /// - public bool IsSemantic { get; set; } - - /// - /// Whether the delegate is an asynchronous function - /// - public bool IsAsynchronous { get; set; } - - /// - /// List of function parameters - /// - public IList Parameters { get; set; } = new List(); - - /// - /// Constructor - /// - public FunctionView() - { - } - - /// - /// Create a function view. - /// - /// Function name - /// Skill name, e.g. the function namespace - /// Function description - /// List of function parameters provided by the skill developer - /// Whether the function is a semantic one (or native is False) - /// Whether the function is async. Note: all semantic functions are async. - public FunctionView( - string name, - string skillName, - string description, - IList parameters, - bool isSemantic, - bool isAsynchronous = true) - { - this.Name = name; - this.SkillName = skillName; - this.Description = description; - this.Parameters = parameters; - this.IsSemantic = isSemantic; - this.IsAsynchronous = isAsynchronous; - } - - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - private string DebuggerDisplay => $"{this.Name} ({this.Description})"; -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/FunctionsView.cs b/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/FunctionsView.cs deleted file mode 100644 index 8b79585b1331..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/FunctionsView.cs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Reflection; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Class used to copy and export data from the skill collection. -/// The data is mutable, but changes do not affect the skill collection. -/// The class can be used to create custom lists in case your scenario needs to. -/// -[DebuggerDisplay("{DebuggerDisplay,nq}")] -public sealed class FunctionsView -{ - /// - /// Collection of semantic skill names and function names, including function parameters. - /// Functions are grouped by skill name. - /// - public ConcurrentDictionary> SemanticFunctions { get; set; } - = new(StringComparer.OrdinalIgnoreCase); - - /// - /// Collection of native skill names and function views, including function parameters. - /// Functions are grouped by skill name. - /// - public ConcurrentDictionary> NativeFunctions { get; set; } - = new(StringComparer.OrdinalIgnoreCase); - - /// - /// Add a function to the list - /// - /// Function details - /// Current instance - public FunctionsView AddFunction(FunctionView view) - { - if (view.IsSemantic) - { - if (!this.SemanticFunctions.ContainsKey(view.SkillName)) - { - this.SemanticFunctions[view.SkillName] = new(); - } - - this.SemanticFunctions[view.SkillName].Add(view); - } - else - { - if (!this.NativeFunctions.ContainsKey(view.SkillName)) - { - this.NativeFunctions[view.SkillName] = new(); - } - - this.NativeFunctions[view.SkillName].Add(view); - } - - return this; - } - - /// - /// Returns true if the function specified is unique and semantic - /// - /// Skill name - /// Function name - /// True if unique and semantic - /// - public bool IsSemantic(string skillName, string functionName) - { - var sf = this.SemanticFunctions.ContainsKey(skillName) - && this.SemanticFunctions[skillName] - .Any(x => string.Equals(x.Name, functionName, StringComparison.OrdinalIgnoreCase)); - - var nf = this.NativeFunctions.ContainsKey(skillName) - && this.NativeFunctions[skillName] - .Any(x => string.Equals(x.Name, functionName, StringComparison.OrdinalIgnoreCase)); - - if (sf && nf) - { - throw new AmbiguousMatchException("There are 2 functions with the same name, one native and one semantic"); - } - - return sf; - } - - /// - /// Returns true if the function specified is unique and native - /// - /// Skill name - /// Function name - /// True if unique and native - /// - public bool IsNative(string skillName, string functionName) - { - var sf = this.SemanticFunctions.ContainsKey(skillName) - && this.SemanticFunctions[skillName] - .Any(x => string.Equals(x.Name, functionName, StringComparison.OrdinalIgnoreCase)); - - var nf = this.NativeFunctions.ContainsKey(skillName) - && this.NativeFunctions[skillName] - .Any(x => string.Equals(x.Name, functionName, StringComparison.OrdinalIgnoreCase)); - - if (sf && nf) - { - throw new AmbiguousMatchException("There are 2 functions with the same name, one native and one semantic"); - } - - return nf; - } - - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - private string DebuggerDisplay => $"Native = {this.NativeFunctions.Count}, Semantic = {this.SemanticFunctions.Count}"; -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/IReadOnlySkillCollection.cs b/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/IReadOnlySkillCollection.cs deleted file mode 100644 index abcebb7bf762..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/IReadOnlySkillCollection.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Read-only skill collection interface. -/// -[SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix")] -public interface IReadOnlySkillCollection -{ - /// - /// Gets the function stored in the collection. - /// - /// The name of the function to retrieve. - /// The function retrieved from the collection. - /// The specified function could not be found in the collection. - ISKFunction GetFunction(string functionName); - - /// - /// Gets the function stored in the collection. - /// - /// The name of the skill with which the function is associated. - /// The name of the function to retrieve. - /// The function retrieved from the collection. - /// The specified function could not be found in the collection. - ISKFunction GetFunction(string skillName, string functionName); - - /// - /// Check if a function is available in the current context, and return it. - /// - /// The name of the function to retrieve. - /// When this method returns, the function that was retrieved if one with the specified name was found; otherwise, . - /// if the function was found; otherwise, . - bool TryGetFunction(string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction); - - /// - /// Check if a function is available in the current context, and return it. - /// - /// The name of the skill with which the function is associated. - /// The name of the function to retrieve. - /// When this method returns, the function that was retrieved if one with the specified name was found; otherwise, . - /// if the function was found; otherwise, . - bool TryGetFunction(string skillName, string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction); - - /// - /// Get all registered functions details, minus the delegates - /// - /// Whether to include semantic functions in the list - /// Whether to include native functions in the list - /// An object containing all the functions details - FunctionsView GetFunctionsView(bool includeSemantic = true, bool includeNative = true); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ISKFunction.cs b/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ISKFunction.cs deleted file mode 100644 index 883b79b6226a..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ISKFunction.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Semantic Kernel callable function interface -/// -public interface ISKFunction -{ - /// - /// Name of the function. The name is used by the skill collection and in prompt templates e.g. {{skillName.functionName}} - /// - string Name { get; } - - /// - /// Name of the skill containing the function. The name is used by the skill collection and in prompt templates e.g. {{skillName.functionName}} - /// - string SkillName { get; } - - /// - /// Function description. The description is used in combination with embeddings when searching relevant functions. - /// - string Description { get; } - - /// - /// Whether the function is defined using a prompt template. - /// IMPORTANT: native functions might use semantic functions internally, - /// so when this property is False, executing the function might still involve AI calls. - /// - bool IsSemantic { get; } - - /// - /// AI service settings - /// - CompleteRequestSettings RequestSettings { get; } - - /// - /// Returns a description of the function, including parameters. - /// - /// An instance of describing the function - FunctionView Describe(); - - /// - /// Invoke the . - /// - /// SK context - /// LLM completion settings (for semantic functions only) - /// The updated context, potentially a new one if context switching is implemented. - /// The to monitor for cancellation requests. The default is . - Task InvokeAsync( - SKContext context, - CompleteRequestSettings? settings = null, - CancellationToken cancellationToken = default); - - /// - /// Invoke the . - /// - /// String input - /// LLM completion settings (for semantic functions only) - /// Application logger - /// The to monitor for cancellation requests. The default is . - /// The updated context, potentially a new one if context switching is implemented. - Task InvokeAsync( - string? input = null, - CompleteRequestSettings? settings = null, - ILogger? logger = null, - CancellationToken cancellationToken = default); - - /// - /// Set the default skill collection to use when the function is invoked - /// without a context or with a context that doesn't have a collection. - /// - /// Kernel's skill collection - /// Self instance - ISKFunction SetDefaultSkillCollection(IReadOnlySkillCollection skills); - - /// - /// Set the AI service used by the semantic function, passing a factory method. - /// The factory allows to lazily instantiate the client and to properly handle its disposal. - /// - /// AI service factory - /// Self instance - ISKFunction SetAIService(Func serviceFactory); - - /// - /// Set the AI completion settings used with LLM requests - /// - /// LLM completion settings - /// Self instance - ISKFunction SetAIConfiguration(CompleteRequestSettings settings); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ISkillCollection.cs b/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ISkillCollection.cs deleted file mode 100644 index 402454a4f24f..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ISkillCollection.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Skill collection interface. -/// -[SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix")] -public interface ISkillCollection : IReadOnlySkillCollection -{ - /// - /// Add a function to the collection - /// - /// Function delegate - /// Self instance - ISkillCollection AddFunction(ISKFunction functionInstance); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/NullReadOnlySkillCollection.cs b/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/NullReadOnlySkillCollection.cs deleted file mode 100644 index 2b833e9b4151..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/NullReadOnlySkillCollection.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -[DebuggerDisplay("Count = 0")] -internal sealed class NullReadOnlySkillCollection : IReadOnlySkillCollection -{ - public static readonly NullReadOnlySkillCollection Instance = new(); - - public ISKFunction GetFunction(string functionName) - { - return ThrowFunctionNotAvailable(functionName); - } - - public ISKFunction GetFunction(string skillName, string functionName) - { - return ThrowFunctionNotAvailable(skillName, functionName); - } - - public bool TryGetFunction(string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction) - { - availableFunction = null; - return false; - } - - public bool TryGetFunction(string skillName, string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction) - { - availableFunction = null; - return false; - } - - public FunctionsView GetFunctionsView(bool includeSemantic = true, bool includeNative = true) - { - return new(); - } - - private NullReadOnlySkillCollection() - { - } - - [DoesNotReturn] - private static ISKFunction ThrowFunctionNotAvailable(string skillName, string functionName) - { - throw new KernelException( - KernelException.ErrorCodes.FunctionNotAvailable, - $"Function not available: {skillName}.{functionName}"); - } - - [DoesNotReturn] - private static ISKFunction ThrowFunctionNotAvailable(string functionName) - { - throw new KernelException( - KernelException.ErrorCodes.FunctionNotAvailable, - $"Function not available: {functionName}"); - } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ParameterView.cs b/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ParameterView.cs deleted file mode 100644 index 4537a6244622..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/ParameterView.cs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Class used to copy and export data about parameters -/// for planner and related scenarios. -/// -[DebuggerDisplay("{DebuggerDisplay,nq}")] -public sealed class ParameterView -{ - private string _name = string.Empty; - - /// - /// Parameter name. Alphanumeric chars + "_" only. - /// - public string Name - { - get => this._name; - set - { - Verify.ValidFunctionParamName(value); - this._name = value; - } - } - - /// - /// Parameter description. - /// - public string? Description { get; set; } - - /// - /// Default value when the value is not provided. - /// - public string? DefaultValue { get; set; } - - /// - /// Constructor - /// - public ParameterView() - { - } - - /// - /// Create a function parameter view, using information provided by the skill developer. - /// - /// Parameter name. The name must be alphanumeric (underscore is the only special char allowed). - /// Parameter description - /// Default parameter value, if not provided - public ParameterView( - string name, - string? description = null, - string? defaultValue = null) - { - this.Name = name; - this.Description = description; - this.DefaultValue = defaultValue; - } - - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - private string DebuggerDisplay => string.IsNullOrEmpty(this.Description) - ? this.Name - : $"{this.Name} ({this.Description})"; -} diff --git a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKParameterAttribute.cs b/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKParameterAttribute.cs deleted file mode 100644 index 4bba1c2f9c40..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/SkillDefinition/SKParameterAttribute.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Attribute to describe additional parameters used by a native function that aren't part of its method signature. -/// -[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public sealed class SKParameterAttribute : Attribute -{ - public SKParameterAttribute(string name, string description) - { - this.Name = name; - this.Description = description; - } - - /// - /// Gets or sets the name of the parameter. - /// - public string Name { get; } - - /// - /// Gets the context parameter description. - /// - public string Description { get; } - - /// - /// Gets or sets the default value of the parameter to use if no context variable is supplied matching the parameter name. - /// - /// - /// There are two ways to supply a default value to a parameter. A default value can be supplied for the parameter in - /// the method signature itself, or a default value can be specified using this property. If both are specified, the - /// value in the attribute is used. The attribute is most useful when the target parameter is followed by a non-optional - /// parameter (such that this parameter isn't permitted to be optional) or when the attribute is applied to a method - /// to indicate a context parameter that is not specified as a method parameter but that's still used by the method body. - /// - public string? DefaultValue { get; set; } -} diff --git a/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/Blocks/Block.cs b/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/Blocks/Block.cs deleted file mode 100644 index 47b4649d4683..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/Blocks/Block.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; - -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; - -/// -/// Base class for blocks parsed from a prompt template -/// -public abstract class Block -{ - internal virtual BlockTypes Type => BlockTypes.Undefined; - - // internal virtual bool? SynchronousRendering => null; - - /// - /// The block content - /// - internal string Content { get; } - - /// - /// App logger - /// - protected ILogger Logger { get; } = NullLogger.Instance; - - /// - /// Base constructor - /// - /// Block content - /// App logger - protected Block(string? content, ILogger? logger = null) - { - if (logger != null) { this.Logger = logger; } - - this.Content = content ?? string.Empty; - } - - /// - /// Check if the block content is valid. - /// - /// Error message in case the content is not valid - /// True if the block content is valid - public abstract bool IsValid(out string errorMsg); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/Blocks/BlockTypes.cs b/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/Blocks/BlockTypes.cs deleted file mode 100644 index 9ede01cfd0e1..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/Blocks/BlockTypes.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; - -internal enum BlockTypes -{ - Undefined = 0, - Text = 1, - Code = 2, - Variable = 3, - Value = 4, - FunctionId = 5, -} diff --git a/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/IPromptTemplate.cs b/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/IPromptTemplate.cs new file mode 100644 index 000000000000..e0cc43e31814 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/IPromptTemplate.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.TemplateEngine; + +/// +/// Interface for prompt template. +/// +public interface IPromptTemplate +{ + /// + /// The list of parameters required by the template, using configuration and template info. + /// + IReadOnlyList Parameters { get; } + + /// + /// Render the template using the information in the context + /// + /// Kernel execution context helpers + /// The to monitor for cancellation requests. The default is . + /// Prompt rendered to string + public Task RenderAsync(SKContext executionContext, CancellationToken cancellationToken = default); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/IPromptTemplateEngine.cs b/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/IPromptTemplateEngine.cs index 0774fdaadd8b..2ae1f7b2c566 100644 --- a/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/IPromptTemplateEngine.cs +++ b/dotnet/src/SemanticKernel.Abstractions/TemplateEngine/IPromptTemplateEngine.cs @@ -1,10 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; namespace Microsoft.SemanticKernel.TemplateEngine; @@ -13,16 +11,6 @@ namespace Microsoft.SemanticKernel.TemplateEngine; /// public interface IPromptTemplateEngine { - /// - /// Given a prompt template string, extract all the blocks (text, variables, function calls) - /// - /// Prompt template (see skprompt.txt files) - /// Whether to validate the blocks syntax, or just return the blocks found, which could contain invalid code - /// A list of all the blocks, ie the template tokenized in text, variables and function calls - IList ExtractBlocks( - string? templateText, - bool validate = true); - /// /// Given a prompt template, replace the variables with their values and execute the functions replacing their /// reference with the function result. diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/CosineSimilarityOperation.cs b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/CosineSimilarityOperation.cs similarity index 96% rename from dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/CosineSimilarityOperation.cs rename to dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/CosineSimilarityOperation.cs index 99bcd57d332b..dd07a01594e1 100644 --- a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/CosineSimilarityOperation.cs +++ b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/CosineSimilarityOperation.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.ComponentModel; using System.Numerics; using System.Runtime.InteropServices; @@ -12,6 +13,8 @@ namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; /// /// https://en.wikipedia.org/wiki/Cosine_similarity /// +[Obsolete("Numerical operations will be removed in a future release. Use System.Numerics.Tensors.TensorPrimitives instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class CosineSimilarityOperation { /// @@ -36,8 +39,7 @@ public static double CosineSimilarity(this ReadOnlySpan x, Rea return CosineSimilarityImplementation(doubleSpanX, doubleSpanY); } - EmbeddingSpan.ThrowTEmbeddingNotSupported(); - return default; + throw new NotSupportedException(); } /// diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/DivideOperation.cs b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/DivideOperation.cs similarity index 93% rename from dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/DivideOperation.cs rename to dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/DivideOperation.cs index 85f2a7cceb13..420728402bd4 100644 --- a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/DivideOperation.cs +++ b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/DivideOperation.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.ComponentModel; using System.Numerics; using System.Runtime.InteropServices; @@ -9,6 +10,8 @@ namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; /// /// Extension methods for vector division. /// +[Obsolete("Numerical operations will be removed in a future release. Use System.Numerics.Tensors.TensorPrimitives instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class DivideOperation { /// @@ -32,7 +35,7 @@ public static void DivideByInPlace(this Span span, double divi } else { - EmbeddingSpan.ThrowTEmbeddingNotSupported(); + throw new NotSupportedException(); } } diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/DotProductOperation.cs b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/DotProductOperation.cs similarity index 94% rename from dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/DotProductOperation.cs rename to dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/DotProductOperation.cs index 6498e6b525a5..6264a9ed47c6 100644 --- a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/DotProductOperation.cs +++ b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/DotProductOperation.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.ComponentModel; using System.Numerics; using System.Runtime.InteropServices; @@ -12,6 +13,8 @@ namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; /// /// https://en.wikipedia.org/wiki/Dot_product /// +[Obsolete("Numerical operations will be removed in a future release. Use System.Numerics.Tensors.TensorPrimitives instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class DotProductOperation { /// @@ -36,9 +39,10 @@ public static double DotProduct(this ReadOnlySpan x, ReadOnlyS ReadOnlySpan doubleSpanY = MemoryMarshal.Cast(y); return DotProductImplementation(doubleSpanX, doubleSpanY); } - - EmbeddingSpan.ThrowTEmbeddingNotSupported(); - return default; + else + { + throw new NotSupportedException(); + } } /// diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/EuclideanLengthOperation.cs b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/EuclideanLengthOperation.cs similarity index 90% rename from dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/EuclideanLengthOperation.cs rename to dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/EuclideanLengthOperation.cs index 431a56a392e5..416569ebe0cf 100644 --- a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/EuclideanLengthOperation.cs +++ b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/EuclideanLengthOperation.cs @@ -1,12 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.ComponentModel; namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; /// /// Extension methods to calculate the Euclidean length of a vector. /// +[Obsolete("Numerical operations will be removed in a future release. Use System.Numerics.Tensors.TensorPrimitives instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class EuclideanLengthOperation { /// diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/MultiplyOperation.cs b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/MultiplyOperation.cs similarity index 93% rename from dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/MultiplyOperation.cs rename to dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/MultiplyOperation.cs index 06e09800512f..cb1596a334de 100644 --- a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/MultiplyOperation.cs +++ b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/MultiplyOperation.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.ComponentModel; using System.Numerics; using System.Runtime.InteropServices; @@ -9,6 +10,8 @@ namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; /// /// Extension methods to multiply a vector by a scalar. /// +[Obsolete("Numerical operations will be removed in a future release. Use System.Numerics.Tensors.TensorPrimitives instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class MultiplyOperation { /// @@ -33,7 +36,7 @@ public static void MultiplyByInPlace(this Span vector, double } else { - EmbeddingSpan.ThrowTEmbeddingNotSupported(); + throw new NotSupportedException(); } } diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/NormalizeOperation.cs b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/NormalizeOperation.cs similarity index 87% rename from dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/NormalizeOperation.cs rename to dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/NormalizeOperation.cs index 3fb2a9fa448d..e17d319b5216 100644 --- a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/NormalizeOperation.cs +++ b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/NormalizeOperation.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.ComponentModel; namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; @@ -10,6 +11,8 @@ namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; /// /// https://en.wikipedia.org/wiki/Unit_vector /// +[Obsolete("Numerical operations will be removed in a future release. Use System.Numerics.Tensors.TensorPrimitives instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] public static class NormalizeOperation { /// diff --git a/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/SpanExtensions.cs b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/SpanExtensions.cs new file mode 100644 index 000000000000..7182853aaf2e --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/AI/Embeddings/VectorOperations/SpanExtensions.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; + +namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; + +/// +/// Extension methods to convert from array and to . +/// +[Obsolete("Numerical operations will be removed in a future release. Use System.Numerics.Tensors.TensorPrimitives instead.")] +[EditorBrowsable(EditorBrowsableState.Never)] +internal static class SpanExtensions +{ + internal static ReadOnlySpan AsReadOnlySpan(this TNumber[] vector) + { + return new ReadOnlySpan(vector); + } + + internal static ReadOnlySpan AsReadOnlySpan(this Span span) + { + return span; + } +} diff --git a/dotnet/src/SemanticKernel.Core/Extensions/KernelSemanticFunctionExtensions.cs b/dotnet/src/SemanticKernel.Core/Extensions/KernelSemanticFunctionExtensions.cs new file mode 100644 index 000000000000..ef8e7e06e518 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Extensions/KernelSemanticFunctionExtensions.cs @@ -0,0 +1,296 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine; +using Microsoft.SemanticKernel.Text; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the namespace of IKernel +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Class for extensions methods to define semantic functions. +/// +public static class KernelSemanticFunctionExtensions +{ + /// + /// Build and register a function in the internal function collection, in a global generic plugin. + /// + /// Semantic Kernel instance + /// Name of the semantic function. The name can contain only alphanumeric chars + underscore. + /// Prompt template configuration. + /// Prompt template. + /// A C# function wrapping AI logic, usually defined with natural language + public static ISKFunction RegisterSemanticFunction( + this IKernel kernel, + string functionName, + PromptTemplateConfig promptTemplateConfig, + IPromptTemplate promptTemplate) + { + return kernel.RegisterSemanticFunction(FunctionCollection.GlobalFunctionsPluginName, functionName, promptTemplateConfig, promptTemplate); + } + + /// + /// Build and register a function in the internal function collection. + /// + /// Semantic Kernel instance + /// Name of the plugin containing the function. The name can contain only alphanumeric chars + underscore. + /// Name of the semantic function. The name can contain only alphanumeric chars + underscore. + /// Prompt template configuration. + /// Prompt template. + /// A C# function wrapping AI logic, usually defined with natural language + public static ISKFunction RegisterSemanticFunction( + this IKernel kernel, + string pluginName, + string functionName, + PromptTemplateConfig promptTemplateConfig, + IPromptTemplate promptTemplate) + { + // Future-proofing the name not to contain special chars + Verify.ValidFunctionName(functionName); + + ISKFunction function = kernel.CreateSemanticFunction(pluginName, functionName, promptTemplateConfig, promptTemplate); + return kernel.RegisterCustomFunction(function); + } + + /// + /// Define a string-to-string semantic function, with no direct support for input context. + /// The function can be referenced in templates and will receive the context, but when invoked programmatically you + /// can only pass in a string in input and receive a string in output. + /// + /// Semantic Kernel instance + /// Plain language definition of the semantic function, using SK template language + /// A name for the given function. The name can be referenced in templates and used by the pipeline planner. + /// Optional plugin name, for namespacing and avoid collisions + /// Optional description, useful for the planner + /// Optional LLM request settings + /// A function ready to use + public static ISKFunction CreateSemanticFunction( + this IKernel kernel, + string promptTemplate, + string? functionName = null, + string? pluginName = null, + string? description = null, + AIRequestSettings? requestSettings = null) + { + functionName ??= RandomFunctionName(); + + var promptTemplateConfig = new PromptTemplateConfig + { + Description = description ?? "Generic function, unknown purpose", + }; + + if (requestSettings is not null) + { + promptTemplateConfig.ModelSettings.Add(requestSettings); + } + + return kernel.CreateSemanticFunction( + promptTemplate: promptTemplate, + promptTemplateConfig: promptTemplateConfig, + functionName: functionName, + pluginName: pluginName); + } + + /// + /// Allow to define a semantic function passing in the definition in natural language, i.e. the prompt template. + /// + /// Semantic Kernel instance + /// Plain language definition of the semantic function, using SK template language + /// Prompt template configuration. + /// A name for the given function. The name can be referenced in templates and used by the pipeline planner. + /// An optional plugin name, e.g. to namespace functions with the same name. When empty, + /// the function is added to the global namespace, overwriting functions with the same name + /// A function ready to use + public static ISKFunction CreateSemanticFunction( + this IKernel kernel, + string promptTemplate, + PromptTemplateConfig promptTemplateConfig, + string? functionName = null, + string? pluginName = null) + { + functionName ??= RandomFunctionName(); + Verify.ValidFunctionName(functionName); + if (!string.IsNullOrEmpty(pluginName)) { Verify.ValidPluginName(pluginName); } + + var template = new PromptTemplate(promptTemplate, promptTemplateConfig, kernel.PromptTemplateEngine); + + // TODO: manage overwrites, potentially error out + return string.IsNullOrEmpty(pluginName) + ? kernel.RegisterSemanticFunction(functionName, promptTemplateConfig, template) + : kernel.RegisterSemanticFunction(pluginName!, functionName, promptTemplateConfig, template); + } + + /// + /// Invoke a semantic function using the provided prompt template. + /// + /// Semantic Kernel instance + /// Plain language definition of the semantic function, using SK template language + /// A name for the given function. The name can be referenced in templates and used by the pipeline planner. + /// Optional plugin name, for namespacing and avoid collisions + /// Optional description, useful for the planner + /// Optional LLM request settings + /// Kernel execution result + public static Task InvokeSemanticFunctionAsync( + this IKernel kernel, + string template, + string? functionName = null, + string? pluginName = null, + string? description = null, + AIRequestSettings? requestSettings = null) + { + var skFunction = kernel.CreateSemanticFunction( + template, + functionName, + pluginName, + description, + requestSettings); + + return kernel.RunAsync(skFunction); + } + + [Obsolete("Methods and classes which includes Skill in the name have been renamed to use Plugin. Use Kernel.ImportSemanticFunctionsFromDirectory instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + public static IDictionary ImportSemanticSkillFromDirectory( + this IKernel kernel, string parentDirectory, params string[] pluginDirectoryNames) + { + return kernel.ImportSemanticFunctionsFromDirectory(parentDirectory, pluginDirectoryNames); + } +#pragma warning restore CS1591 + + /// + /// Imports semantic functions, defined by prompt templates stored in the filesystem. + /// + /// + /// + /// A plugin directory contains a set of subdirectories, one for each semantic function. + /// + /// + /// This method accepts the path of the parent directory (e.g. "d:\plugins") and the name of the plugin directory + /// (e.g. "OfficePlugin"), which is used also as the "plugin name" in the internal function collection (note that + /// plugin and function names can contain only alphanumeric chars and underscore). + /// + /// + /// Example: + /// D:\plugins\ # parentDirectory = "D:\plugins" + /// + /// |__ OfficePlugin\ # pluginDirectoryName = "SummarizeEmailThread" + /// + /// |__ ScheduleMeeting # semantic function + /// |__ skprompt.txt # prompt template + /// |__ config.json # settings (optional file) + /// + /// |__ SummarizeEmailThread # semantic function + /// |__ skprompt.txt # prompt template + /// |__ config.json # settings (optional file) + /// + /// |__ MergeWordAndExcelDocs # semantic function + /// |__ skprompt.txt # prompt template + /// |__ config.json # settings (optional file) + /// + /// |__ XboxPlugin\ # another plugin, etc. + /// + /// |__ MessageFriend + /// |__ skprompt.txt + /// |__ config.json + /// |__ LaunchGame + /// |__ skprompt.txt + /// |__ config.json + /// + /// + /// See https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins for examples in the Semantic Kernel repository. + /// + /// + /// Semantic Kernel instance + /// Directory containing the plugin directory, e.g. "d:\myAppPlugins" + /// Name of the directories containing the selected plugins, e.g. "StrategyPlugin" + /// A list of all the semantic functions found in the directory, indexed by plugin name. + public static IDictionary ImportSemanticFunctionsFromDirectory( + this IKernel kernel, string parentDirectory, params string[] pluginDirectoryNames) + { + const string ConfigFile = "config.json"; + const string PromptFile = "skprompt.txt"; + + var functions = new Dictionary(); + + ILogger? logger = null; + foreach (string pluginDirectoryName in pluginDirectoryNames) + { + Verify.ValidPluginName(pluginDirectoryName); + var pluginDirectory = Path.Combine(parentDirectory, pluginDirectoryName); + Verify.DirectoryExists(pluginDirectory); + + string[] directories = Directory.GetDirectories(pluginDirectory); + foreach (string dir in directories) + { + var functionName = Path.GetFileName(dir); + + // Continue only if prompt template exists + var promptPath = Path.Combine(dir, PromptFile); + if (!File.Exists(promptPath)) { continue; } + + // Load prompt configuration. Note: the configuration is optional. + var config = new PromptTemplateConfig(); + var configPath = Path.Combine(dir, ConfigFile); + if (File.Exists(configPath)) + { + config = PromptTemplateConfig.FromJson(File.ReadAllText(configPath)); + } + + logger ??= kernel.LoggerFactory.CreateLogger(typeof(IKernel)); + if (logger.IsEnabled(LogLevel.Trace)) + { + logger.LogTrace("Config {0}: {1}", functionName, Json.Serialize(config)); + } + + // Load prompt template + var template = new PromptTemplate(File.ReadAllText(promptPath), config, kernel.PromptTemplateEngine); + + if (logger.IsEnabled(LogLevel.Trace)) + { + logger.LogTrace("Registering function {0}.{1} loaded from {2}", pluginDirectoryName, functionName, dir); + } + + functions[functionName] = kernel.RegisterSemanticFunction(pluginDirectoryName, functionName, config, template); + } + } + + return functions; + } + + private static string RandomFunctionName() => "func" + Guid.NewGuid().ToString("N"); + + private static ISKFunction CreateSemanticFunction( + this IKernel kernel, + string pluginName, + string functionName, + PromptTemplateConfig promptTemplateConfig, + IPromptTemplate promptTemplate) + { + ISKFunction func = SemanticFunction.FromSemanticConfig( + pluginName, + functionName, + promptTemplateConfig, + promptTemplate, + kernel.LoggerFactory + ); + + func.SetAIConfiguration(promptTemplateConfig.GetDefaultRequestSettings()); + + // Note: the service is instantiated using the kernel configuration state when the function is invoked + func.SetAIService(() => kernel.GetService(promptTemplateConfig.GetDefaultRequestSettings()?.ServiceId ?? null)); + + return func; + } +} diff --git a/dotnet/src/SemanticKernel.Core/Extensions/SKFunctionExtensions.cs b/dotnet/src/SemanticKernel.Core/Extensions/SKFunctionExtensions.cs new file mode 100644 index 000000000000..b855aa9b5d58 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Extensions/SKFunctionExtensions.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Orchestration; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 // Namespace does not match folder structure + +/// +/// Class that holds extension methods for objects implementing ISKFunction. +/// +public static class SKFunctionExtensions +{ + /// + /// Configure the LLM settings used by semantic function. + /// + /// Semantic function + /// Request settings + /// Self instance + public static ISKFunction UseCompletionSettings(this ISKFunction skFunction, AIRequestSettings requestSettings) + { + return skFunction.SetAIConfiguration(requestSettings); + } + + /// + /// Execute a function allowing to pass the main input separately from the rest of the context. + /// + /// Function to execute + /// Kernel + /// Input variables for the function + /// Collection of functions that this function can access + /// Culture to use for the function execution + /// LLM completion settings (for semantic functions only) + /// The to use for logging. If null, no logging will be performed. + /// The to monitor for cancellation requests. The default is . + /// The result of the function execution + public static Task InvokeAsync(this ISKFunction function, + IKernel kernel, + ContextVariables? variables = null, + IReadOnlyFunctionCollection? functions = null, + CultureInfo? culture = null, + AIRequestSettings? requestSettings = null, + ILoggerFactory? loggerFactory = null, + CancellationToken cancellationToken = default) + { + var context = kernel.CreateNewContext(variables, functions, loggerFactory, culture); + return function.InvokeAsync(context, requestSettings ?? function.RequestSettings, cancellationToken); + } + + /// + /// Execute a function allowing to pass the main input separately from the rest of the context. + /// + /// Function to execute + /// Input string for the function + /// Kernel + /// Collection of functions that this function can access + /// Culture to use for the function execution + /// LLM completion settings (for semantic functions only) + /// The to use for logging. If null, no logging will be performed. + /// The to monitor for cancellation requests. The default is . + /// The result of the function execution + public static Task InvokeAsync(this ISKFunction function, + string input, + IKernel kernel, + IReadOnlyFunctionCollection? functions = null, + CultureInfo? culture = null, + AIRequestSettings? requestSettings = null, + ILoggerFactory? loggerFactory = null, + CancellationToken cancellationToken = default) + => function.InvokeAsync(kernel, new ContextVariables(input), functions, culture, requestSettings, loggerFactory, cancellationToken); + + /// + /// Returns decorated instance of with enabled instrumentation. + /// + /// Instance of to decorate. + /// The to use for logging. If null, no logging will be performed. + public static ISKFunction WithInstrumentation(this ISKFunction function, ILoggerFactory? loggerFactory = null) + { + return new InstrumentedSKFunction(function, loggerFactory); + } +} diff --git a/dotnet/src/SemanticKernel.Core/Functions/FunctionCollection.cs b/dotnet/src/SemanticKernel.Core/Functions/FunctionCollection.cs new file mode 100644 index 000000000000..37927178df05 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/FunctionCollection.cs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Semantic Kernel default function collection class. +/// The class holds a list of all the functions, native and semantic, known to the kernel instance. +/// The list is used by the planner and when executing pipelines of function compositions. +/// +[SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix")] +[DebuggerTypeProxy(typeof(IReadOnlyFunctionCollectionTypeProxy))] +[DebuggerDisplay("{DebuggerDisplay,nq}")] +public class FunctionCollection : IFunctionCollection +{ + /// + /// Plugin name used when storing global functions. + /// + public const string GlobalFunctionsPluginName = "_GLOBAL_FUNCTIONS_"; + + /// + /// Initializes a new instance of the class. + /// + public FunctionCollection() : this((IReadOnlyFunctionCollection?)null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Collection of functions with which to populate this instance. + public FunctionCollection(IReadOnlyFunctionCollection? readOnlyFunctionCollection) + { + // Important: names are case insensitive + this._functionCollection = new(StringComparer.OrdinalIgnoreCase); + + if (readOnlyFunctionCollection is not null) + { + foreach (var functionView in readOnlyFunctionCollection.GetFunctionViews()) + { + this.AddFunction(readOnlyFunctionCollection.GetFunction(functionView.PluginName, functionView.Name)); + } + } + } + + /// + /// Adds a function to the function collection. + /// + /// The function instance to add. + /// The updated function collection. + public IFunctionCollection AddFunction(ISKFunction functionInstance) + { + Verify.NotNull(functionInstance); + + ConcurrentDictionary functions = this._functionCollection.GetOrAdd(functionInstance.PluginName, static _ => new(StringComparer.OrdinalIgnoreCase)); + functions[functionInstance.Name] = functionInstance; + + return this; + } + + /// + public ISKFunction GetFunction(string functionName) => + this.GetFunction(GlobalFunctionsPluginName, functionName); + + /// + public ISKFunction GetFunction(string pluginName, string functionName) + { + pluginName = !string.IsNullOrWhiteSpace(pluginName) ? pluginName : GlobalFunctionsPluginName; + + if (!this.TryGetFunction(pluginName, functionName, out ISKFunction? functionInstance)) + { + throw new SKException($"Function not available {pluginName}.{functionName}"); + } + + return functionInstance; + } + + /// + public bool TryGetFunction(string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction) => + this.TryGetFunction(GlobalFunctionsPluginName, functionName, out availableFunction); + + /// + public bool TryGetFunction(string pluginName, string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction) + { + Verify.NotNull(pluginName); + Verify.NotNull(functionName); + + if (this._functionCollection.TryGetValue(pluginName, out ConcurrentDictionary? functions)) + { + return functions.TryGetValue(functionName, out availableFunction); + } + + availableFunction = null; + return false; + } + + /// + public IReadOnlyList GetFunctionViews() + { + var result = new List(); + + foreach (var functions in this._functionCollection.Values) + { + foreach (ISKFunction f in functions.Values) + { + result.Add(f.Describe()); + } + } + + return result; + } + + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + internal string DebuggerDisplay => $"Count = {this._functionCollection.Count}"; + + #region Obsolete to be removed + /// + /// Initializes a new instance of the class. + /// + /// Optional skill collection to copy from + /// The logger factory. + [Obsolete("Use a constructor that doesn't accept an ILoggerFactory. This constructor will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public FunctionCollection(IReadOnlyFunctionCollection? readOnlyFunctionCollection = null, ILoggerFactory? loggerFactory = null) : this(readOnlyFunctionCollection) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The logger factory. + [Obsolete("Use a constructor that doesn't accept an ILoggerFactory. This constructor will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public FunctionCollection(ILoggerFactory? loggerFactory = null) : this() + { + } + #endregion + + #region private ================================================================================ + + private readonly ConcurrentDictionary> _functionCollection; + + #endregion +} diff --git a/dotnet/src/SemanticKernel.Core/Functions/IReadOnlyFunctionCollectionTypeProxy.cs b/dotnet/src/SemanticKernel.Core/Functions/IReadOnlyFunctionCollectionTypeProxy.cs new file mode 100644 index 000000000000..0d41bc2ff4df --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/IReadOnlyFunctionCollectionTypeProxy.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Debugger type proxy for . +/// +// ReSharper disable once InconsistentNaming +internal sealed class IReadOnlyFunctionCollectionTypeProxy +{ + private readonly IReadOnlyFunctionCollection _collection; + + public IReadOnlyFunctionCollectionTypeProxy(IReadOnlyFunctionCollection collection) => this._collection = collection; + + [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] + public FunctionsProxy[] Functions + { + get + { + return this._collection.GetFunctionViews() + .GroupBy(f => f.PluginName) + .Select(g => new FunctionsProxy(g) { Name = g.Key }) + .ToArray(); + } + } + + [DebuggerDisplay("{Name}")] + public sealed class FunctionsProxy : List + { + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public string? Name; + + public FunctionsProxy(IEnumerable functions) : base(functions) { } + } +} diff --git a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs new file mode 100644 index 000000000000..302f98aa18f9 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Orchestration; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Standard Semantic Kernel callable function with instrumentation. +/// +internal sealed class InstrumentedSKFunction : ISKFunction +{ + /// + public string Name => this._function.Name; + + /// + public string PluginName => this._function.PluginName; + + /// + public string Description => this._function.Description; + + /// + public AIRequestSettings? RequestSettings => this._function.RequestSettings; + + /// + /// Initialize a new instance of the class. + /// + /// Instance of to decorate. + /// The to use for logging. If null, no logging will be performed. + public InstrumentedSKFunction( + ISKFunction function, + ILoggerFactory? loggerFactory = null) + { + this._function = function; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(InstrumentedSKFunction)) : NullLogger.Instance; + + this._executionTimeHistogram = s_meter.CreateHistogram( + name: $"SK.{this.PluginName}.{this.Name}.ExecutionTime", + unit: "ms", + description: "Duration of function execution"); + + this._executionTotalCounter = s_meter.CreateCounter( + name: $"SK.{this.PluginName}.{this.Name}.ExecutionTotal", + description: "Total number of function executions"); + + this._executionSuccessCounter = s_meter.CreateCounter( + name: $"SK.{this.PluginName}.{this.Name}.ExecutionSuccess", + description: "Number of successful function executions"); + + this._executionFailureCounter = s_meter.CreateCounter( + name: $"SK.{this.PluginName}.{this.Name}.ExecutionFailure", + description: "Number of failed function executions"); + } + + /// + public FunctionView Describe() => + this._function.Describe(); + + /// + public async Task InvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + return await this.InvokeWithInstrumentationAsync(() => + this._function.InvokeAsync(context, requestSettings, cancellationToken)).ConfigureAwait(false); + } + + /// + public ISKFunction SetAIConfiguration(AIRequestSettings? requestSettings) => + this._function.SetAIConfiguration(requestSettings); + + /// + public ISKFunction SetAIService(Func serviceFactory) => + this._function.SetAIService(serviceFactory); + + #region private ================================================================================ + + private readonly ISKFunction _function; + private readonly ILogger _logger; + + /// + /// Instance of for function-related activities. + /// + private static readonly ActivitySource s_activitySource = new(typeof(SKFunction).FullName); + + /// + /// Instance of for function-related metrics. + /// + private static readonly Meter s_meter = new(typeof(SKFunction).FullName); + + /// + /// Instance of to measure and track the time of function execution. + /// + private readonly Histogram _executionTimeHistogram; + + /// + /// Instance of to keep track of the total number of function executions. + /// + private readonly Counter _executionTotalCounter; + + /// + /// Instance of to keep track of the number of successful function executions. + /// + private readonly Counter _executionSuccessCounter; + + /// + /// Instance of to keep track of the number of failed function executions. + /// + private readonly Counter _executionFailureCounter; + + /// + /// Wrapper for instrumentation to be used in multiple invocation places. + /// + /// Delegate to instrument. + private async Task InvokeWithInstrumentationAsync(Func> func) + { + using var activity = s_activitySource.StartActivity($"{this.PluginName}.{this.Name}"); + + this._logger.LogInformation("{PluginName}.{FunctionName}: Function execution started.", this.PluginName, this.Name); + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + FunctionResult result; + + try + { + result = await func().ConfigureAwait(false); + } + catch (Exception ex) + { + this._logger.LogWarning("{PluginName}.{FunctionName}: Function execution status: {Status}", + this.PluginName, this.Name, "Failed"); + + this._logger.LogError(ex, "{PluginName}.{FunctionName}: Function execution exception details: {Message}", + this.PluginName, this.Name, ex.Message); + + this._executionFailureCounter.Add(1); + + throw; + } + finally + { + stopwatch.Stop(); + this._executionTotalCounter.Add(1); + this._executionTimeHistogram.Record(stopwatch.ElapsedMilliseconds); + } + + this._logger.LogInformation("{PluginName}.{FunctionName}: Function execution status: {Status}", + this.PluginName, this.Name, "Success"); + + this._logger.LogInformation("{PluginName}.{FunctionName}: Function execution finished in {ExecutionTime}ms", + this.PluginName, this.Name, stopwatch.ElapsedMilliseconds); + + this._executionSuccessCounter.Add(1); + + return result; + } + + #endregion + + #region Obsolete ======================================================================= + + /// + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use ISKFunction.PluginName instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public string SkillName => this._function.PluginName; + + /// + [Obsolete("Kernel no longer differentiates between Semantic and Native functions. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public bool IsSemantic => this._function.IsSemantic; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultSkillCollection(IReadOnlyFunctionCollection skills) => this; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultFunctionCollection(IReadOnlyFunctionCollection functions) => this; + + #endregion +} diff --git a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs new file mode 100644 index 000000000000..aa52bd7f8147 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs @@ -0,0 +1,897 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +#pragma warning disable format + +/// +/// Standard Semantic Kernel callable function. +/// SKFunction is used to extend one C# , , , +/// with additional methods required by the kernel. +/// +[DebuggerDisplay("{DebuggerDisplay,nq}")] +internal sealed class NativeFunction : ISKFunction, IDisposable +{ + /// + public string Name { get; } + + /// + public string PluginName { get; } + + /// + public string Description { get; } + + /// + public AIRequestSettings? RequestSettings { get; } + + /// + /// List of function parameters + /// + public IReadOnlyList Parameters { get; } + + /// + /// Create a native function instance, wrapping a native object method + /// + /// Signature of the method to invoke + /// Object containing the method to invoke + /// SK plugin name + /// The to use for logging. If null, no logging will be performed. + /// SK function instance + public static ISKFunction FromNativeMethod( + MethodInfo method, + object? target = null, + string? pluginName = null, + ILoggerFactory? loggerFactory = null) + { + if (!method.IsStatic && target is null) + { + throw new ArgumentNullException(nameof(target), "Argument cannot be null for non-static methods"); + } + + if (string.IsNullOrWhiteSpace(pluginName)) + { + pluginName = FunctionCollection.GlobalFunctionsPluginName; + } + + ILogger logger = loggerFactory?.CreateLogger(method.DeclaringType ?? typeof(SKFunction)) ?? NullLogger.Instance; + + MethodDetails methodDetails = GetMethodDetails(method, target, pluginName!, logger); + + return new NativeFunction( + delegateFunction: methodDetails.Function, + parameters: methodDetails.Parameters, + pluginName: pluginName!, + functionName: methodDetails.Name, + description: methodDetails.Description, + logger: logger); + } + + /// + /// Create a native function instance, wrapping a delegate function + /// + /// Function to invoke + /// SK plugin name + /// SK function name + /// SK function description + /// SK function parameters + /// The to use for logging. If null, no logging will be performed. + /// SK function instance + public static ISKFunction FromNativeFunction( + Delegate nativeFunction, + string? pluginName = null, + string? functionName = null, + string? description = null, + IEnumerable? parameters = null, + ILoggerFactory? loggerFactory = null) + { + ILogger logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(ISKFunction)) : NullLogger.Instance; + + if (string.IsNullOrWhiteSpace(pluginName)) + { + pluginName = FunctionCollection.GlobalFunctionsPluginName; + } + + MethodDetails methodDetails = GetMethodDetails(nativeFunction.Method, nativeFunction.Target, pluginName!, logger); + + functionName ??= methodDetails.Name; + parameters ??= methodDetails.Parameters; + description ??= methodDetails.Description; + + return new NativeFunction( + delegateFunction: methodDetails.Function, + parameters: parameters.ToList(), + description: description, + pluginName: pluginName!, + functionName: functionName, + logger: logger); + } + + /// + public FunctionView Describe() + => this._view.Value; + + /// + public async Task InvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + try + { + return await this._function(null, requestSettings, context, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) when (!e.IsCriticalException()) + { + this._logger.LogError(e, "Native function {Plugin}.{Name} execution failed with error {Error}", this.PluginName, this.Name, e.Message); + throw; + } + } + + /// + public ISKFunction SetAIService(Func serviceFactory) + { + this.ThrowNotSemantic(); + return this; + } + + /// + public ISKFunction SetAIConfiguration(AIRequestSettings? requestSettings) + { + this.ThrowNotSemantic(); + return this; + } + + /// + /// Dispose of resources. + /// + public void Dispose() + { + } + + /// + /// JSON serialized string representation of the function. + /// + public override string ToString() + => this.ToString(false); + + /// + /// JSON serialized string representation of the function. + /// + public string ToString(bool writeIndented) + => JsonSerializer.Serialize(this, options: writeIndented ? s_toStringIndentedSerialization : s_toStringStandardSerialization); + + #region private + + private static readonly JsonSerializerOptions s_toStringStandardSerialization = new(); + private static readonly JsonSerializerOptions s_toStringIndentedSerialization = new() { WriteIndented = true }; + private readonly NativeFunctionDelegate _function; + private readonly ILogger _logger; + + private struct MethodDetails + { + public NativeFunctionDelegate Function { get; set; } + public List Parameters { get; set; } + public string Name { get; set; } + public string Description { get; set; } + } + + internal NativeFunction( + NativeFunctionDelegate delegateFunction, + IReadOnlyList parameters, + string pluginName, + string functionName, + string description, + ILogger logger) + { + Verify.NotNull(delegateFunction); + Verify.ValidPluginName(pluginName); + Verify.ValidFunctionName(functionName); + + this._logger = logger; + + this._function = delegateFunction; + this.Parameters = parameters.ToArray(); + Verify.ParametersUniqueness(this.Parameters); + + this.Name = functionName; + this.PluginName = pluginName; + this.Description = description; + + this._view = new(() => new (functionName, pluginName, description) { Parameters = this.Parameters }); + } + + /// + /// Throw an exception if the function is not semantic, use this method when some logic makes sense only for semantic functions. + /// + /// + [DoesNotReturn] + private void ThrowNotSemantic() + { + this._logger.LogError("The function is not semantic"); + throw new SKException("Invalid operation, the method requires a semantic function"); + } + + private static MethodDetails GetMethodDetails( + MethodInfo method, + object? target, + string pluginName, + ILogger? logger = null) + { + Verify.NotNull(method); + + // Get the name to use for the function. If the function has an SKName attribute, we use that. + // Otherwise, we use the name of the method, but strip off any "Async" suffix if it's {Value}Task-returning. + // We don't apply any heuristics to the value supplied by SKName so that it can always be used + // as a definitive override. + string? functionName = method.GetCustomAttribute(inherit: true)?.Name?.Trim(); + if (string.IsNullOrEmpty(functionName)) + { + functionName = SanitizeMetadataName(method.Name!); + Verify.ValidFunctionName(functionName); + + if (IsAsyncMethod(method) && + functionName.EndsWith("Async", StringComparison.Ordinal) && + functionName.Length > "Async".Length) + { + functionName = functionName.Substring(0, functionName.Length - "Async".Length); + } + } + + string? description = method.GetCustomAttribute(inherit: true)?.Description; + + var result = new MethodDetails + { + Name = functionName!, + Description = description ?? string.Empty, + }; + + (result.Function, result.Parameters) = GetDelegateInfo(functionName!, pluginName, target, method); + + logger?.LogTrace("Method '{0}' found", result.Name); + + return result; + } + + /// Gets whether a method has a known async return type. + private static bool IsAsyncMethod(MethodInfo method) + { + Type t = method.ReturnType; + + if (t == typeof(Task) || t == typeof(ValueTask)) + { + return true; + } + + if (t.IsGenericType) + { + t = t.GetGenericTypeDefinition(); + if (t == typeof(Task<>) || t == typeof(ValueTask<>)) + { + return true; + } + } + + return false; + } + + // Inspect a method and returns the corresponding delegate and related info + private static (NativeFunctionDelegate function, List) GetDelegateInfo( + string functionName, + string pluginName, + object? instance, + MethodInfo method) + { + ThrowForInvalidSignatureIf(method.IsGenericMethodDefinition, method, "Generic methods are not supported"); + + var stringParameterViews = new List(); + var parameters = method.GetParameters(); + + // Get marshaling funcs for parameters and build up the parameter views. + var parameterFuncs = new Func[parameters.Length]; + bool sawFirstParameter = false, hasSKContextParam = false, hasCancellationTokenParam = false, hasLoggerParam = false, hasMemoryParam = false, hasCultureParam = false; + for (int i = 0; i < parameters.Length; i++) + { + (parameterFuncs[i], ParameterView? parameterView) = GetParameterMarshalerDelegate( + method, parameters[i], + ref sawFirstParameter, ref hasSKContextParam, ref hasCancellationTokenParam, ref hasLoggerParam, ref hasMemoryParam, ref hasCultureParam); + if (parameterView is not null) + { + stringParameterViews.Add(parameterView); + } + } + + // Get marshaling func for the return value. + Func> returnFunc = GetReturnValueMarshalerDelegate(method); + + // Create the func + Task Function(ITextCompletion? text, AIRequestSettings? requestSettings, SKContext context, CancellationToken cancellationToken) + { + // Create the arguments. + object?[] args = parameterFuncs.Length != 0 ? new object?[parameterFuncs.Length] : Array.Empty(); + for (int i = 0; i < args.Length; i++) + { + args[i] = parameterFuncs[i](context, cancellationToken); + } + + // Invoke the method. + object? result = method.Invoke(instance, args); + + // Extract and return the result. + return returnFunc(functionName, pluginName, result, context); + } + + // Check for param names conflict + Verify.ParametersUniqueness(stringParameterViews); + + // Return the function and its parameter views. + return (Function, stringParameterViews); + } + + /// + /// Gets a delegate for handling the marshaling of a parameter. + /// + private static (Func, ParameterView?) GetParameterMarshalerDelegate( + MethodInfo method, ParameterInfo parameter, + ref bool sawFirstParameter, ref bool hasSKContextParam, ref bool hasCancellationTokenParam, ref bool hasLoggerParam, ref bool hasMemoryParam, ref bool hasCultureParam) + { + Type type = parameter.ParameterType; + + // Handle special types based on SKContext data. These can each show up at most once in the method signature, + // with the SKContext itself or the primary data from it mapped directly into the method's parameter. + // They do not get parameter views as they're not supplied from context variables. + + if (type == typeof(SKContext)) + { + TrackUniqueParameterType(ref hasSKContextParam, method, $"At most one {nameof(SKContext)} parameter is permitted."); + return (static (SKContext context, CancellationToken _) => context, null); + } + + if (type == typeof(ILogger) || type == typeof(ILoggerFactory)) + { + TrackUniqueParameterType(ref hasLoggerParam, method, $"At most one {nameof(ILogger)}/{nameof(ILoggerFactory)} parameter is permitted."); + return type == typeof(ILogger) ? + ((SKContext context, CancellationToken _) => context.LoggerFactory.CreateLogger(method?.DeclaringType ?? typeof(SKFunction)), null) : + ((SKContext context, CancellationToken _) => context.LoggerFactory, null); + } + + if (type == typeof(CultureInfo) || type == typeof(IFormatProvider)) + { + TrackUniqueParameterType(ref hasCultureParam, method, $"At most one {nameof(CultureInfo)}/{nameof(IFormatProvider)} parameter is permitted."); + return (static (SKContext context, CancellationToken _) => context.Culture, null); + } + + if (type == typeof(CancellationToken)) + { + TrackUniqueParameterType(ref hasCancellationTokenParam, method, $"At most one {nameof(CancellationToken)} parameter is permitted."); + return (static (SKContext _, CancellationToken cancellationToken) => cancellationToken, null); + } + + // Handle context variables. These are supplied from the SKContext's Variables dictionary. + + if (!type.IsByRef && GetParser(type) is Func parser) + { + // Use either the parameter's name or an override from an applied SKName attribute. + SKNameAttribute? nameAttr = parameter.GetCustomAttribute(inherit: true); + string name = nameAttr?.Name?.Trim() ?? SanitizeMetadataName(parameter.Name); + bool nameIsInput = name.Equals("input", StringComparison.OrdinalIgnoreCase); + ThrowForInvalidSignatureIf(name.Length == 0, method, $"Parameter {parameter.Name}'s context attribute defines an invalid name."); + ThrowForInvalidSignatureIf(sawFirstParameter && nameIsInput, method, "Only the first parameter may be named 'input'"); + + // Use either the parameter's optional default value as contained in parameter metadata (e.g. `string s = "hello"`) + // or an override from an applied SKParameter attribute. Note that a default value may be null. + DefaultValueAttribute defaultValueAttribute = parameter.GetCustomAttribute(inherit: true); + bool hasDefaultValue = defaultValueAttribute is not null; + object? defaultValue = defaultValueAttribute?.Value; + if (!hasDefaultValue && parameter.HasDefaultValue) + { + hasDefaultValue = true; + defaultValue = parameter.DefaultValue; + } + + if (hasDefaultValue) + { + // If we got a default value, make sure it's of the right type. This currently supports + // null values if the target type is a reference type or a Nullable, strings, + // anything that can be parsed from a string via a registered TypeConverter, + // and a value that's already the same type as the parameter. + if (defaultValue is string defaultStringValue && defaultValue.GetType() != typeof(string)) + { + // Invariant culture is used here as this value comes from the C# source + // and it should be deterministic across cultures. + defaultValue = parser(defaultStringValue, CultureInfo.InvariantCulture); + } + else + { + ThrowForInvalidSignatureIf( + defaultValue is null && type.IsValueType && Nullable.GetUnderlyingType(type) is null, + method, + $"Type {type} is a non-nullable value type but a null default value was specified."); + ThrowForInvalidSignatureIf( + defaultValue is not null && !type.IsAssignableFrom(defaultValue.GetType()), + method, + $"Default value {defaultValue} for parameter {name} is not assignable to type {type}."); + } + } + + bool fallBackToInput = !sawFirstParameter && !nameIsInput; + object? parameterFunc(SKContext context, CancellationToken _) + { + // 1. Use the value of the variable if it exists. + if (context.Variables.TryGetValue(name, out string? value)) + { + return Process(value); + } + + // 2. Otherwise, use the default value if there is one, sourced either from an attribute or the parameter's default. + if (hasDefaultValue) + { + return defaultValue; + } + + // 3. Otherwise, use "input" if this is the first (or only) parameter. + if (fallBackToInput) + { + return Process(context.Variables.Input); + } + + // 4. Otherwise, fail. + throw new SKException($"Missing value for parameter '{name}'", + new ArgumentException("Missing value function parameter", name)); + + object ? Process(string value) + { + if (type == typeof(string)) + { + return value; + } + + try + { + return parser(value, context.Culture); + } + catch (Exception e) when (!e.IsCriticalException()) + { + throw new ArgumentOutOfRangeException(name, value, e.Message); + } + } + } + + sawFirstParameter = true; + + var parameterView = new ParameterView( + name, + parameter.GetCustomAttribute(inherit: true)?.Description ?? string.Empty, + defaultValue?.ToString() ?? string.Empty, + IsRequired: !parameter.IsOptional); + + return (parameterFunc, parameterView); + } + + // Fail for unknown parameter types. + throw GetExceptionForInvalidSignature(method, $"Unknown parameter type {parameter.ParameterType}"); + } + + /// + /// Gets a delegate for handling the result value of a method, converting it into the to return from the invocation. + /// + private static Func> GetReturnValueMarshalerDelegate(MethodInfo method) + { + // Handle each known return type for the method + Type returnType = method.ReturnType; + + // No return value, either synchronous (void) or asynchronous (Task / ValueTask). + + if (returnType == typeof(void)) + { + return static (functionName, pluginName, result, context) => + Task.FromResult(new FunctionResult(functionName, pluginName, context)); + } + + if (returnType == typeof(Task)) + { + return async static (functionName, pluginName, result, context) => + { + await ((Task)ThrowIfNullResult(result)).ConfigureAwait(false); + return new FunctionResult(functionName, pluginName, context); + }; + } + + if (returnType == typeof(ValueTask)) + { + return async static (functionName, pluginName, result, context) => + { + await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(false); + return new FunctionResult(functionName, pluginName, context); + }; + } + + // SKContext, either synchronous (SKContext) or asynchronous (Task / ValueTask). + + if (returnType == typeof(SKContext)) + { + return static (functionName, pluginName, result, _) => + { + var context = (SKContext)ThrowIfNullResult(result); + return Task.FromResult(new FunctionResult(functionName, pluginName, context, context.Result)); + }; + } + + if (returnType == typeof(Task)) + { + return static async (functionName, pluginName, result, _) => + { + var context = await ((Task)ThrowIfNullResult(result)).ConfigureAwait(false); + return new FunctionResult(functionName, pluginName, context, context.Result); + }; + } + + if (returnType == typeof(ValueTask)) + { + return static async (functionName, pluginName, result, _) => + { + var context = await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(false); + return new FunctionResult(functionName, pluginName, context, context); + }; + } + + // string (which is special as no marshaling is required), either synchronous (string) or asynchronous (Task / ValueTask) + + if (returnType == typeof(string)) + { + return static (functionName, pluginName, result, context) => + { + var resultString = (string?)result; + context.Variables.Update(resultString); + return Task.FromResult(new FunctionResult(functionName, pluginName, context, resultString)); + }; + } + + if (returnType == typeof(Task)) + { + return async static (functionName, pluginName, result, context) => + { + var resultString = await ((Task)ThrowIfNullResult(result)).ConfigureAwait(false); + context.Variables.Update(resultString); + return new FunctionResult(functionName, pluginName, context, resultString); + }; + } + + if (returnType == typeof(ValueTask)) + { + return async static (functionName, pluginName, result, context) => + { + var resultString = await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(false); + context.Variables.Update(resultString); + return new FunctionResult(functionName, pluginName, context, resultString); + }; + } + + // All other synchronous return types T. + + if (!returnType.IsGenericType || returnType.GetGenericTypeDefinition() == typeof(Nullable<>)) + { + if (GetFormatter(returnType) is not Func formatter) + { + throw GetExceptionForInvalidSignature(method, $"Unknown return type {returnType}"); + } + + return (functionName, pluginName, result, context) => + { + context.Variables.Update(formatter(result, context.Culture)); + return Task.FromResult(new FunctionResult(functionName, pluginName, context, result)); + }; + } + + // All other asynchronous return types + + // Task + if (returnType.GetGenericTypeDefinition() is Type genericTask && + genericTask == typeof(Task<>) && + returnType.GetProperty("Result", BindingFlags.Public | BindingFlags.Instance)?.GetGetMethod() is MethodInfo taskResultGetter && + GetFormatter(taskResultGetter.ReturnType) is Func taskResultFormatter) + { + return async (functionName, pluginName, result, context) => + { + await ((Task)ThrowIfNullResult(result)).ConfigureAwait(false); + + var taskResult = taskResultGetter.Invoke(result!, Array.Empty()); + + context.Variables.Update(taskResultFormatter(taskResult, context.Culture)); + return new FunctionResult(functionName, pluginName, context, taskResult); + }; + } + + // ValueTask + if (returnType.GetGenericTypeDefinition() is Type genericValueTask && + genericValueTask == typeof(ValueTask<>) && + returnType.GetMethod("AsTask", BindingFlags.Public | BindingFlags.Instance) is MethodInfo valueTaskAsTask && + valueTaskAsTask.ReturnType.GetProperty("Result", BindingFlags.Public | BindingFlags.Instance)?.GetGetMethod() is MethodInfo asTaskResultGetter && + GetFormatter(asTaskResultGetter.ReturnType) is Func asTaskResultFormatter) + { + return async (functionName, pluginName, result, context) => + { + Task task = (Task)valueTaskAsTask.Invoke(ThrowIfNullResult(result), Array.Empty()); + await task.ConfigureAwait(false); + + var taskResult = asTaskResultGetter.Invoke(task!, Array.Empty()); + + context.Variables.Update(asTaskResultFormatter(taskResult, context.Culture)); + return new FunctionResult(functionName, pluginName, context, taskResult); + }; + } + + // IAsyncEnumerable + if (returnType.GetGenericTypeDefinition() is Type genericAsyncEnumerable && genericAsyncEnumerable == typeof(IAsyncEnumerable<>)) + { + Type elementType = returnType.GetGenericArguments()[0]; + + MethodInfo getAsyncEnumeratorMethod = typeof(IAsyncEnumerable<>) + .MakeGenericType(elementType) + .GetMethod("GetAsyncEnumerator"); + + if (getAsyncEnumeratorMethod is not null) + { + return (functionName, pluginName, result, context) => + { + var asyncEnumerator = getAsyncEnumeratorMethod.Invoke(result, new object[] { default(CancellationToken) }); + + if (asyncEnumerator is not null) + { + return Task.FromResult(new FunctionResult(functionName, pluginName, context, asyncEnumerator)); + } + + return Task.FromResult(new FunctionResult(functionName, pluginName, context)); + }; + } + } + + // Unrecognized return type. + throw GetExceptionForInvalidSignature(method, $"Unknown return type {returnType}"); + + // Throws an exception if a result is found to be null unexpectedly + static object ThrowIfNullResult(object? result) => + result ?? + throw new SKException("Function returned null unexpectedly."); + } + + /// Gets an exception that can be thrown indicating an invalid signature. + [DoesNotReturn] + private static Exception GetExceptionForInvalidSignature(MethodInfo method, string reason) => + throw new SKException($"Function '{method.Name}' is not supported by the kernel. {reason}"); + + /// Throws an exception indicating an invalid SKFunction signature if the specified condition is not met. + private static void ThrowForInvalidSignatureIf([DoesNotReturnIf(true)] bool condition, MethodInfo method, string reason) + { + if (condition) + { + throw GetExceptionForInvalidSignature(method, reason); + } + } + + /// Tracks whether a particular kind of parameter has been seen, throwing an exception if it has, and marking it as seen if it hasn't + private static void TrackUniqueParameterType(ref bool hasParameterType, MethodInfo method, string failureMessage) + { + ThrowForInvalidSignatureIf(hasParameterType, method, failureMessage); + hasParameterType = true; + } + + /// + /// Gets a TypeConverter-based parser for parsing a string as the target type. + /// + /// Specifies the target type into which a string should be parsed. + /// The parsing function if the target type is supported; otherwise, null. + /// + /// The parsing function uses whatever TypeConverter is registered for the target type. + /// Parsing is first attempted using the current culture, and if that fails, it tries again + /// with the invariant culture. If both fail, an exception is thrown. + /// + private static Func? GetParser(Type targetType) => + s_parsers.GetOrAdd(targetType, static targetType => + { + // Strings just parse to themselves. + if (targetType == typeof(string)) + { + return (input, cultureInfo) => input; + } + + // For nullables, parse as the inner type. We then just need to be careful to treat null as null, + // as the underlying parser might not be expecting null. + bool wasNullable = false; + if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) + { + wasNullable = true; + targetType = Nullable.GetUnderlyingType(targetType); + } + + // For enums, delegate to Enum.Parse, special-casing null if it was actually Nullable. + if (targetType.IsEnum) + { + return (input, cultureInfo) => + { + if (wasNullable && input is null) + { + return null!; + } + + return Enum.Parse(targetType, input, ignoreCase: true); + }; + } + + // Finally, look up and use a type converter. Again, special-case null if it was actually Nullable. + if (GetTypeConverter(targetType) is TypeConverter converter && converter.CanConvertFrom(typeof(string))) + { + return (input, cultureInfo) => + { + if (wasNullable && input is null) + { + return null!; + } + + // First try to parse using the supplied culture (or current if none was supplied). + // If that fails, try with the invariant culture and allow any exception to propagate. + try + { + return converter.ConvertFromString(context: null, cultureInfo, input); + } + catch (Exception e) when (!e.IsCriticalException() && cultureInfo != CultureInfo.InvariantCulture) + { + return converter.ConvertFromInvariantString(input); + } + }; + } + + // Unsupported type. + return null; + }); + + /// + /// Gets a TypeConverter-based formatter for formatting an object as a string. + /// + /// + /// Formatting is performed in the invariant culture whenever possible. + /// + private static Func? GetFormatter(Type targetType) => + s_formatters.GetOrAdd(targetType, static targetType => + { + // For nullables, render as the underlying type. + bool wasNullable = false; + if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) + { + wasNullable = true; + targetType = Nullable.GetUnderlyingType(targetType); + } + + // For enums, just ToString() and allow the object override to do the right thing. + if (targetType.IsEnum) + { + return (input, cultureInfo) => input?.ToString()!; + } + + // Strings just render as themselves. + if (targetType == typeof(string)) + { + return (input, cultureInfo) => (string)input!; + } + + // Finally, look up and use a type converter. + if (GetTypeConverter(targetType) is TypeConverter converter && converter.CanConvertTo(typeof(string))) + { + return (input, cultureInfo) => + { + if (wasNullable && input is null) + { + return null!; + } + + return converter.ConvertToString(context: null, cultureInfo, input); + }; + } + + return null; + }); + + private static TypeConverter? GetTypeConverter(Type targetType) + { + // In an ideal world, this would use TypeDescriptor.GetConverter. However, that is not friendly to + // any form of ahead-of-time compilation, as it could end up requiring functionality that was trimmed. + // Instead, we just use a hard-coded set of converters for the types we know about and then also support + // types that are explicitly attributed with TypeConverterAttribute. + + if (targetType == typeof(byte)) { return new ByteConverter(); } + if (targetType == typeof(sbyte)) { return new SByteConverter(); } + if (targetType == typeof(bool)) { return new BooleanConverter(); } + if (targetType == typeof(ushort)) { return new UInt16Converter(); } + if (targetType == typeof(short)) { return new Int16Converter(); } + if (targetType == typeof(char)) { return new CharConverter(); } + if (targetType == typeof(uint)) { return new UInt32Converter(); } + if (targetType == typeof(int)) { return new Int32Converter(); } + if (targetType == typeof(ulong)) { return new UInt64Converter(); } + if (targetType == typeof(long)) { return new Int64Converter(); } + if (targetType == typeof(float)) { return new SingleConverter(); } + if (targetType == typeof(double)) { return new DoubleConverter(); } + if (targetType == typeof(decimal)) { return new DecimalConverter(); } + if (targetType == typeof(TimeSpan)) { return new TimeSpanConverter(); } + if (targetType == typeof(DateTime)) { return new DateTimeConverter(); } + if (targetType == typeof(DateTimeOffset)) { return new DateTimeOffsetConverter(); } + if (targetType == typeof(Uri)) { return new UriTypeConverter(); } + if (targetType == typeof(Guid)) { return new GuidConverter(); } + + if (targetType.GetCustomAttribute() is TypeConverterAttribute tca && + Type.GetType(tca.ConverterTypeName, throwOnError: false) is Type converterType && + Activator.CreateInstance(converterType) is TypeConverter converter) + { + return converter; + } + + return null; + } + + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + private string DebuggerDisplay => $"{this.Name} ({this.Description})"; + + /// + /// Remove characters from method name that are valid in metadata but invalid for SK. + /// + private static string SanitizeMetadataName(string methodName) => + s_invalidNameCharsRegex.Replace(methodName, "_"); + + /// Regex that flags any character other than ASCII digits or letters or the underscore. + private static readonly Regex s_invalidNameCharsRegex = new("[^0-9A-Za-z_]"); + + /// Parser functions for converting strings to parameter types. + private static readonly ConcurrentDictionary?> s_parsers = new(); + + /// Formatter functions for converting parameter types to strings. + private static readonly ConcurrentDictionary?> s_formatters = new(); + + private readonly Lazy _view; + + #endregion + + #region Obsolete + + /// + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use ISKFunction.PluginName instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public string SkillName => this.PluginName; + + /// + [Obsolete("Kernel no longer differentiates between Semantic and Native functions. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public bool IsSemantic => true; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultSkillCollection(IReadOnlyFunctionCollection skills) => this; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultFunctionCollection(IReadOnlyFunctionCollection functions) => this; + + #endregion +} diff --git a/dotnet/src/SemanticKernel.Core/Functions/NativeFunctionDelegate.cs b/dotnet/src/SemanticKernel.Core/Functions/NativeFunctionDelegate.cs new file mode 100644 index 000000000000..99cc51ada273 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/NativeFunctionDelegate.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Orchestration; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +internal delegate Task NativeFunctionDelegate( + ITextCompletion? textCompletion, + AIRequestSettings? requestSettings, + SKContext context, + CancellationToken cancellationToken); diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunction.cs new file mode 100644 index 000000000000..3b92a52135b8 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunction.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Reflection; +using Microsoft.Extensions.Logging; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +#pragma warning disable format + +/// +/// Static helpers to create instances. +/// +public static class SKFunction +{ + /// + /// Create a native function instance, wrapping a native object method + /// + /// Signature of the method to invoke + /// Object containing the method to invoke + /// SK plugin name + /// The to use for logging. If null, no logging will be performed. + /// SK function instance + public static ISKFunction FromNativeMethod( + MethodInfo method, + object? target = null, + string? pluginName = null, + ILoggerFactory? loggerFactory = null) + => NativeFunction.FromNativeMethod(method, target, pluginName, loggerFactory); + + /// + /// Create a native function instance, wrapping a delegate function + /// + /// Function to invoke + /// SK plugin name + /// SK function name + /// SK function description + /// SK function parameters + /// The to use for logging. If null, no logging will be performed. + /// SK function instance + public static ISKFunction FromNativeFunction( + Delegate nativeFunction, + string? pluginName = null, + string? functionName = null, + string? description = null, + IEnumerable? parameters = null, + ILoggerFactory? loggerFactory = null) + => NativeFunction.FromNativeFunction(nativeFunction, pluginName, functionName, description, parameters, loggerFactory); +} diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionTextExtensions.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionTextExtensions.cs new file mode 100644 index 000000000000..4c537f8606d6 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionTextExtensions.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Orchestration; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Class with extension methods for semantic functions. +/// +public static class SKFunctionTextExtensions +{ + /// + /// Extension method to aggregate partitioned results of a semantic function. + /// + /// Semantic Kernel function + /// Input to aggregate. + /// Semantic Kernel context. + /// Separator to use between results. + /// Optional request settings. + /// The to monitor for cancellation requests. The default is . + /// Aggregated results. + public static async Task AggregatePartitionedResultsAsync( + this ISKFunction func, + List partitionedInput, + SKContext context, + string resultsSeparator = "\n", + AIRequestSettings? settings = null, + CancellationToken cancellationToken = default) + { + var results = new List(); + foreach (var partition in partitionedInput) + { + context.Variables.Update(partition); + + var result = await func.InvokeAsync(context, settings, cancellationToken).ConfigureAwait(false); + + context = result.Context; + + results.Add(context.Variables.ToString()); + } + + context.Variables.Update(string.Join(resultsSeparator, results)); + return context; + } +} diff --git a/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs new file mode 100644 index 000000000000..bb5add248e3e --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs @@ -0,0 +1,249 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +#pragma warning disable format + +/// +/// A Semantic Kernel "Semantic" prompt function. +/// +[DebuggerDisplay("{DebuggerDisplay,nq}")] +internal sealed class SemanticFunction : ISKFunction, IDisposable +{ + /// + public string Name { get; } + + /// + public string PluginName { get; } + + /// + public string Description { get; } + + /// + public AIRequestSettings? RequestSettings { get; private set; } + + /// + /// List of function parameters + /// + public IReadOnlyList Parameters => this._promptTemplate.Parameters; + + /// + /// Create a semantic function instance, given a semantic function configuration. + /// + /// Name of the plugin to which the function being created belongs. + /// Name of the function to create. + /// Prompt template configuration. + /// Prompt template. + /// The to use for logging. If null, no logging will be performed. + /// The to monitor for cancellation requests. The default is . + /// SK function instance. + public static ISKFunction FromSemanticConfig( + string pluginName, + string functionName, + PromptTemplateConfig promptTemplateConfig, + IPromptTemplate promptTemplate, + ILoggerFactory? loggerFactory = null, + CancellationToken cancellationToken = default) + { + Verify.NotNull(promptTemplateConfig); + Verify.NotNull(promptTemplate); + + var func = new SemanticFunction( + template: promptTemplate, + description: promptTemplateConfig.Description, + pluginName: pluginName, + functionName: functionName, + loggerFactory: loggerFactory + ); + func.SetAIConfiguration(promptTemplateConfig.GetDefaultRequestSettings()); + + return func; + } + + /// + public FunctionView Describe() + { + return new FunctionView(this.Name, this.PluginName, this.Description) { Parameters = this.Parameters }; + } + + /// + public async Task InvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + this.AddDefaultValues(context.Variables); + + return await this.RunPromptAsync(this._aiService?.Value, requestSettings ?? this.RequestSettings, context, cancellationToken).ConfigureAwait(false); + } + + /// + public ISKFunction SetAIService(Func serviceFactory) + { + Verify.NotNull(serviceFactory); + this._aiService = new Lazy(serviceFactory); + return this; + } + + /// + public ISKFunction SetAIConfiguration(AIRequestSettings? requestSettings) + { + this.RequestSettings = requestSettings; + return this; + } + + /// + /// Dispose of resources. + /// + public void Dispose() + { + if (this._aiService is { IsValueCreated: true } aiService) + { + (aiService.Value as IDisposable)?.Dispose(); + } + } + + /// + /// JSON serialized string representation of the function. + /// + public override string ToString() + => this.ToString(false); + + /// + /// JSON serialized string representation of the function. + /// + public string ToString(bool writeIndented) + => JsonSerializer.Serialize(this, options: writeIndented ? s_toStringIndentedSerialization : s_toStringStandardSerialization); + + internal SemanticFunction( + IPromptTemplate template, + string pluginName, + string functionName, + string description, + ILoggerFactory? loggerFactory = null) + { + Verify.NotNull(template); + Verify.ValidPluginName(pluginName); + Verify.ValidFunctionName(functionName); + + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(SemanticFunction)) : NullLogger.Instance; + + this._promptTemplate = template; + Verify.ParametersUniqueness(this.Parameters); + + this.Name = functionName; + this.PluginName = pluginName; + this.Description = description; + + this._view = new(() => new(functionName, pluginName, description, this.Parameters)); + } + + #region private + + private static readonly JsonSerializerOptions s_toStringStandardSerialization = new(); + private static readonly JsonSerializerOptions s_toStringIndentedSerialization = new() { WriteIndented = true }; + private readonly ILogger _logger; + private Lazy? _aiService; + private readonly Lazy _view; + public IPromptTemplate _promptTemplate { get; } + + private static async Task GetCompletionsResultContentAsync(IReadOnlyList completions, CancellationToken cancellationToken = default) + { + // To avoid any unexpected behavior we only take the first completion result (when running from the Kernel) + return await completions[0].GetCompletionAsync(cancellationToken).ConfigureAwait(false); + } + + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + private string DebuggerDisplay => $"{this.Name} ({this.Description})"; + + /// Add default values to the context variables if the variable is not defined + private void AddDefaultValues(ContextVariables variables) + { + foreach (var parameter in this.Parameters) + { + if (!variables.ContainsKey(parameter.Name) && parameter.DefaultValue != null) + { + variables[parameter.Name] = parameter.DefaultValue; + } + } + } + + private async Task RunPromptAsync( + ITextCompletion? client, + AIRequestSettings? requestSettings, + SKContext context, + CancellationToken cancellationToken) + { + Verify.NotNull(client); + + FunctionResult result; + + try + { + string renderedPrompt = await this._promptTemplate.RenderAsync(context, cancellationToken).ConfigureAwait(false); + IReadOnlyList completionResults = await client.GetCompletionsAsync(renderedPrompt, requestSettings, cancellationToken).ConfigureAwait(false); + string completion = await GetCompletionsResultContentAsync(completionResults, cancellationToken).ConfigureAwait(false); + + // Update the result with the completion + context.Variables.Update(completion); + + var modelResults = completionResults.Select(c => c.ModelResult).ToArray(); + + result = new FunctionResult(this.Name, this.PluginName, context, completion); + + result.Metadata.Add(AIFunctionResultExtensions.ModelResultsMetadataKey, modelResults); + } + catch (Exception ex) when (!ex.IsCriticalException()) + { + this._logger?.LogError(ex, "Semantic function {Plugin}.{Name} execution failed with error {Error}", this.PluginName, this.Name, ex.Message); + throw; + } + + return result; + } + + #endregion + + #region Obsolete + + /// + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use ISKFunction.PluginName instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public string SkillName => this.PluginName; + + /// + [Obsolete("Kernel no longer differentiates between Semantic and Native functions. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public bool IsSemantic => true; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultSkillCollection(IReadOnlyFunctionCollection skills) => this; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultFunctionCollection(IReadOnlyFunctionCollection functions) => this; + + #endregion +} diff --git a/dotnet/src/SemanticKernel.Core/Kernel.cs b/dotnet/src/SemanticKernel.Core/Kernel.cs new file mode 100644 index 000000000000..92c582e97507 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Kernel.cs @@ -0,0 +1,294 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Events; +using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Services; +using Microsoft.SemanticKernel.TemplateEngine; + +namespace Microsoft.SemanticKernel; + +/// +/// Semantic kernel class. +/// The kernel provides a function collection to define native and semantic functions, an orchestrator to execute a list of functions. +/// Semantic functions are automatically rendered and executed using an internal prompt template rendering engine. +/// Future versions will allow to: +/// * customize the rendering engine +/// * include branching logic in the functions pipeline +/// * persist execution state for long running pipelines +/// * distribute pipelines over a network +/// * RPC functions and secure environments, e.g. sandboxing and credentials management +/// * auto-generate pipelines given a higher level goal +/// +public sealed class Kernel : IKernel, IDisposable +{ + /// + public ILoggerFactory LoggerFactory { get; } + + /// + public IReadOnlyFunctionCollection Functions => this._functionCollection; + + /// + public IPromptTemplateEngine PromptTemplateEngine { get; } + + /// + /// Return a new instance of the kernel builder, used to build and configure kernel instances. + /// + public static KernelBuilder Builder => new(); + + /// + public IDelegatingHandlerFactory HttpHandlerFactory { get; } + + /// + public event EventHandler? FunctionInvoking; + + /// + public event EventHandler? FunctionInvoked; + + /// + /// Kernel constructor. See KernelBuilder for an easier and less error prone approach to create kernel instances. + /// + /// function collection + /// AI Service Provider + /// Prompt template engine + /// Semantic text Memory + /// + /// The to use for logging. If null, no logging will be performed. + public Kernel( + IFunctionCollection functionCollection, + IAIServiceProvider aiServiceProvider, + IPromptTemplateEngine promptTemplateEngine, + ISemanticTextMemory memory, + IDelegatingHandlerFactory httpHandlerFactory, + ILoggerFactory? loggerFactory) + { + loggerFactory ??= NullLoggerFactory.Instance; + + this.LoggerFactory = loggerFactory; + this.HttpHandlerFactory = httpHandlerFactory; + this.PromptTemplateEngine = promptTemplateEngine; + this._memory = memory; + this._aiServiceProvider = aiServiceProvider; + this._promptTemplateEngine = promptTemplateEngine; + this._functionCollection = functionCollection; + + this._logger = loggerFactory.CreateLogger(typeof(Kernel)); + } + + /// + public ISKFunction RegisterCustomFunction(ISKFunction customFunction) + { + Verify.NotNull(customFunction); + + this._functionCollection.AddFunction(customFunction); + + return customFunction; + } + + /// + public async Task RunAsync(ContextVariables variables, CancellationToken cancellationToken, params ISKFunction[] pipeline) + { + var context = this.CreateNewContext(variables); + + FunctionResult? functionResult = null; + + int pipelineStepCount = 0; + var allFunctionResults = new List(); + + foreach (ISKFunction skFunction in pipeline) + { +repeat: + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var functionDetails = skFunction.Describe(); + + var functionInvokingArgs = this.OnFunctionInvoking(functionDetails, context); + if (functionInvokingArgs?.CancelToken.IsCancellationRequested ?? false) + { + this._logger.LogInformation("Execution was cancelled on function invoking event of pipeline step {StepCount}: {PluginName}.{FunctionName}.", pipelineStepCount, skFunction.PluginName, skFunction.Name); + break; + } + + if (functionInvokingArgs?.IsSkipRequested ?? false) + { + this._logger.LogInformation("Execution was skipped on function invoking event of pipeline step {StepCount}: {PluginName}.{FunctionName}.", pipelineStepCount, skFunction.PluginName, skFunction.Name); + continue; + } + + functionResult = await skFunction.InvokeAsync(context, cancellationToken: cancellationToken).ConfigureAwait(false); + + context = functionResult.Context; + + var functionInvokedArgs = this.OnFunctionInvoked(functionDetails, functionResult); + + if (functionInvokedArgs is not null) + { + // All changes to the SKContext by invoked handlers may reflect in the original function result + functionResult = new FunctionResult(functionDetails.Name, functionDetails.PluginName, functionInvokedArgs.SKContext, functionInvokedArgs.SKContext.Result); + } + + allFunctionResults.Add(functionResult); + + if (functionInvokedArgs?.CancelToken.IsCancellationRequested ?? false) + { + this._logger.LogInformation("Execution was cancelled on function invoked event of pipeline step {StepCount}: {PluginName}.{FunctionName}.", pipelineStepCount, skFunction.PluginName, skFunction.Name); + break; + } + + if (functionInvokedArgs?.IsRepeatRequested ?? false) + { + this._logger.LogInformation("Execution repeat request on function invoked event of pipeline step {StepCount}: {PluginName}.{FunctionName}.", pipelineStepCount, skFunction.PluginName, skFunction.Name); + goto repeat; + } + } + catch (Exception ex) + { + this._logger.LogError("Plugin {Plugin} function {Function} call fail during pipeline step {Step} with error {Error}:", skFunction.PluginName, skFunction.Name, pipelineStepCount, ex.Message); + throw; + } + + pipelineStepCount++; + } + + return KernelResult.FromFunctionResults(functionResult?.Value, allFunctionResults); + } + + /// + public SKContext CreateNewContext( + ContextVariables? variables = null, + IReadOnlyFunctionCollection? functions = null, + ILoggerFactory? loggerFactory = null, + CultureInfo? culture = null) + { + return new SKContext( + new FunctionRunner(this), + variables, + functions ?? this.Functions, + loggerFactory ?? this.LoggerFactory, + culture); + } + + /// + public T GetService(string? name = null) where T : IAIService + { + var service = this._aiServiceProvider.GetService(name); + if (service != null) + { + return service; + } + + throw new SKException($"Service of type {typeof(T)} and name {name ?? ""} not registered."); + } + + /// + /// Dispose of resources. + /// + public void Dispose() + { + // ReSharper disable once SuspiciousTypeConversion.Global + if (this._memory is IDisposable mem) { mem.Dispose(); } + + // ReSharper disable once SuspiciousTypeConversion.Global + if (this._functionCollection is IDisposable reg) { reg.Dispose(); } + } + + #region private ================================================================================ + + private readonly IFunctionCollection _functionCollection; + private ISemanticTextMemory _memory; + private readonly IPromptTemplateEngine _promptTemplateEngine; + private readonly IAIServiceProvider _aiServiceProvider; + private readonly ILogger _logger; + + /// + /// Execute the OnFunctionInvoking event handlers. + /// + /// Function view details + /// SKContext before function invocation + /// FunctionInvokingEventArgs if the event was handled, null otherwise + private FunctionInvokingEventArgs? OnFunctionInvoking(FunctionView functionView, SKContext context) + { + if (this.FunctionInvoking is not null) + { + var args = new FunctionInvokingEventArgs(functionView, context); + this.FunctionInvoking.Invoke(this, args); + + return args; + } + + return null; + } + + /// + /// Execute the OnFunctionInvoked event handlers. + /// + /// Function view details + /// Function result after invocation + /// FunctionInvokedEventArgs if the event was handled, null otherwise + private FunctionInvokedEventArgs? OnFunctionInvoked(FunctionView functionView, FunctionResult result) + { + if (this.FunctionInvoked is not null) + { + var args = new FunctionInvokedEventArgs(functionView, result); + this.FunctionInvoked.Invoke(this, args); + + return args; + } + + return null; + } + + #endregion + + #region Obsolete =============================================================================== + + /// + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISemanticTextMemory Memory => this._memory; + + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use Kernel.Functions instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + public IReadOnlyFunctionCollection Skills => this._functionCollection; +#pragma warning restore CS1591 + + /// + [Obsolete("Func shorthand no longer no longer supported. Use Kernel.Functions collection instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction Func(string pluginName, string functionName) + { + return this.Functions.GetFunction(pluginName, functionName); + } + + /// + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public void RegisterMemory(ISemanticTextMemory memory) + { + this._memory = memory; + } + + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use Kernel.ImportFunctions instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable CS1591 + public IDictionary ImportSkill(object functionsInstance, string? pluginName = null) + { + return this.ImportFunctions(functionsInstance, pluginName); + } +#pragma warning restore CS1591 + + #endregion +} diff --git a/dotnet/src/SemanticKernel.Core/KernelBuilder.cs b/dotnet/src/SemanticKernel.Core/KernelBuilder.cs new file mode 100644 index 000000000000..67609299edfb --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/KernelBuilder.cs @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Services; +using Microsoft.SemanticKernel.TemplateEngine; + +namespace Microsoft.SemanticKernel; + +/// +/// A builder for Semantic Kernel. +/// +public sealed class KernelBuilder +{ + private Func _memoryFactory = () => NullMemory.Instance; + private ILoggerFactory _loggerFactory = NullLoggerFactory.Instance; + private Func? _memoryStorageFactory = null; + private IDelegatingHandlerFactory _httpHandlerFactory = NullHttpHandlerFactory.Instance; + private IPromptTemplateEngine? _promptTemplateEngine; + private readonly AIServiceCollection _aiServices = new(); + + private static bool s_promptTemplateEngineInitialized = false; + private static Type? s_promptTemplateEngineType = null; + + /// + /// Create a new kernel instance + /// + /// New kernel instance + public static IKernel Create() + { + var builder = new KernelBuilder(); + return builder.Build(); + } + + /// + /// Build a new kernel instance using the settings passed so far. + /// + /// Kernel instance + public IKernel Build() + { + var instance = new Kernel( + new FunctionCollection(), + this._aiServices.Build(), + this._promptTemplateEngine ?? this.CreateDefaultPromptTemplateEngine(this._loggerFactory), + this._memoryFactory.Invoke(), + this._httpHandlerFactory, + this._loggerFactory + ); + + // TODO: decouple this from 'UseMemory' kernel extension + if (this._memoryStorageFactory != null) + { +#pragma warning disable CS0618 // This will be removed in a future release. + instance.UseMemory(this._memoryStorageFactory.Invoke()); +#pragma warning restore CS0618 // This will be removed in a future release. + } + + return instance; + } + + /// + /// Add a logger to the kernel to be built. + /// + /// The to use for logging. If null, no logging will be performed. + /// Updated kernel builder including the logger. + public KernelBuilder WithLoggerFactory(ILoggerFactory loggerFactory) + { + Verify.NotNull(loggerFactory); + this._loggerFactory = loggerFactory; + return this; + } + + /// + /// Add a semantic text memory entity to the kernel to be built. + /// + /// Semantic text memory entity to add. + /// Updated kernel builder including the semantic text memory entity. + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public KernelBuilder WithMemory(ISemanticTextMemory memory) + { + Verify.NotNull(memory); + this._memoryFactory = () => memory; + return this; + } + + /// + /// Add a semantic text memory store factory. + /// + /// The store factory. + /// Updated kernel builder including the semantic text memory entity. + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public KernelBuilder WithMemory(Func factory) where TStore : ISemanticTextMemory + { + Verify.NotNull(factory); + this._memoryFactory = () => factory(this._loggerFactory); + return this; + } + + /// + /// Add memory storage to the kernel to be built. + /// + /// Storage to add. + /// Updated kernel builder including the memory storage. + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public KernelBuilder WithMemoryStorage(IMemoryStore storage) + { + Verify.NotNull(storage); + this._memoryStorageFactory = () => storage; + return this; + } + + /// + /// Add memory storage factory to the kernel. + /// + /// The storage factory. + /// Updated kernel builder including the memory storage. + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public KernelBuilder WithMemoryStorage(Func factory) where TStore : IMemoryStore + { + Verify.NotNull(factory); + this._memoryStorageFactory = () => factory(this._loggerFactory); + return this; + } + + /// + /// Add memory storage factory to the kernel. + /// + /// The storage factory. + /// Updated kernel builder including the memory storage. + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public KernelBuilder WithMemoryStorage(Func factory) where TStore : IMemoryStore + { + Verify.NotNull(factory); + this._memoryStorageFactory = () => factory(this._loggerFactory, this._httpHandlerFactory); + return this; + } + + /// + /// Add prompt template engine to the kernel to be built. + /// + /// Prompt template engine to add. + /// Updated kernel builder including the prompt template engine. + public KernelBuilder WithPromptTemplateEngine(IPromptTemplateEngine promptTemplateEngine) + { + Verify.NotNull(promptTemplateEngine); + this._promptTemplateEngine = promptTemplateEngine; + return this; + } + + /// + /// Add a http handler factory to the kernel to be built. + /// + /// Http handler factory to add. + /// Updated kernel builder including the http handler factory. + public KernelBuilder WithHttpHandlerFactory(IDelegatingHandlerFactory httpHandlerFactory) + { + Verify.NotNull(httpHandlerFactory); + this._httpHandlerFactory = httpHandlerFactory; + return this; + } + + /// + /// Add a retry handler factory to the kernel to be built. + /// + /// Retry handler factory to add. + /// Updated kernel builder including the retry handler factory. + [Obsolete("This method is deprecated, use WithHttpHandlerFactory instead")] + public KernelBuilder WithRetryHandlerFactory(IDelegatingHandlerFactory httpHandlerFactory) + { + return this.WithHttpHandlerFactory(httpHandlerFactory); + } + + /// + /// Adds a instance to the services collection + /// + /// The instance. + public KernelBuilder WithDefaultAIService(TService instance) where TService : IAIService + { + this._aiServices.SetService(instance); + return this; + } + + /// + /// Adds a factory method to the services collection + /// + /// The factory method that creates the AI service instances of type . + public KernelBuilder WithDefaultAIService(Func factory) where TService : IAIService + { + this._aiServices.SetService(() => factory(this._loggerFactory)); + return this; + } + + /// + /// Adds a instance to the services collection + /// + /// The service ID + /// The instance. + /// Optional: set as the default AI service for type + public KernelBuilder WithAIService( + string? serviceId, + TService instance, + bool setAsDefault = false) where TService : IAIService + { + this._aiServices.SetService(serviceId, instance, setAsDefault); + return this; + } + + /// + /// Adds a factory method to the services collection + /// + /// The service ID + /// The factory method that creates the AI service instances of type . + /// Optional: set as the default AI service for type + public KernelBuilder WithAIService( + string? serviceId, + Func factory, + bool setAsDefault = false) where TService : IAIService + { + this._aiServices.SetService(serviceId, () => factory(this._loggerFactory), setAsDefault); + return this; + } + + /// + /// Adds a factory method to the services collection + /// + /// The service ID + /// The factory method that creates the AI service instances of type . + /// Optional: set as the default AI service for type + public KernelBuilder WithAIService( + string? serviceId, + Func factory, + bool setAsDefault = false) where TService : IAIService + { + this._aiServices.SetService(serviceId, () => factory(this._loggerFactory, this._httpHandlerFactory), setAsDefault); + return this; + } + + /// + /// Create a default prompt template engine. + /// + /// This is a temporary solution to avoid breaking existing clients. + /// There will be a separate task to add support for registering instances of IPromptTemplateEngine and obsoleting the current approach. + /// + /// + /// Logger factory to be used by the template engine + /// Instance of . + private IPromptTemplateEngine CreateDefaultPromptTemplateEngine(ILoggerFactory? loggerFactory = null) + { + if (!s_promptTemplateEngineInitialized) + { + s_promptTemplateEngineType = this.GetPromptTemplateEngineType(); + s_promptTemplateEngineInitialized = true; + } + + if (s_promptTemplateEngineType is not null) + { + var constructor = s_promptTemplateEngineType.GetConstructor(new Type[] { typeof(ILoggerFactory) }); + if (constructor is not null) + { +#pragma warning disable CS8601 // Null logger factory is OK + return (IPromptTemplateEngine)constructor.Invoke(new object[] { loggerFactory }); +#pragma warning restore CS8601 + } + } + + return new NullPromptTemplateEngine(); + } + + /// + /// Get the prompt template engine type if available + /// + /// The type for the prompt template engine if available + private Type? GetPromptTemplateEngineType() + { + try + { + var assembly = Assembly.Load("Microsoft.SemanticKernel.TemplateEngine.Basic"); + + return assembly.ExportedTypes.Single(type => + type.Name.Equals("BasicPromptTemplateEngine", StringComparison.Ordinal) && + type.GetInterface(nameof(IPromptTemplateEngine)) is not null); + } + catch (Exception ex) when (!ex.IsCriticalException()) + { + return null; + } + } +} + +/// +/// No-operation IPromptTemplateEngine which performs no rendering of the template. +/// +/// This is a temporary solution to avoid breaking existing clients. +/// +internal sealed class NullPromptTemplateEngine : IPromptTemplateEngine +{ + public Task RenderAsync(string templateText, SKContext context, CancellationToken cancellationToken = default) + { + return Task.FromResult(templateText); + } +} diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs new file mode 100644 index 000000000000..41545d15a716 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel; + +/// Extension methods for interacting with . +public static class KernelExtensions +{ + /// + /// Import a set of functions as a plugin from the given object instance. Only the functions that have the `SKFunction` attribute will be included in the plugin. + /// Once these functions are imported, the prompt templates can use functions to import content at runtime. + /// + /// The kernel. + /// Instance of a class containing functions + /// Name of the plugin for function collection and prompt templates. If the value is empty functions are registered in the global namespace. + /// A list of all the semantic functions found in the directory, indexed by function name. + public static IDictionary ImportFunctions( + this IKernel kernel, + object functionsInstance, + string? pluginName = null) + { + Verify.NotNull(kernel); + Verify.NotNull(functionsInstance); + + ILogger logger = kernel.LoggerFactory.CreateLogger(kernel.GetType()); + if (string.IsNullOrWhiteSpace(pluginName)) + { + pluginName = FunctionCollection.GlobalFunctionsPluginName; + logger.LogTrace("Importing functions from {0} to the global plugin namespace", functionsInstance.GetType().FullName); + } + else + { + logger.LogTrace("Importing functions from {0} to the {1} namespace", functionsInstance.GetType().FullName, pluginName); + } + + MethodInfo[] methods = functionsInstance.GetType().GetMethods(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public); + logger.LogTrace("Importing plugin name: {0}. Potential methods found: {1}", pluginName, methods.Length); + + // Filter out non-SKFunctions and fail if two functions have the same name + Dictionary result = new(StringComparer.OrdinalIgnoreCase); + foreach (MethodInfo method in methods) + { + if (method.GetCustomAttribute() is not null) + { + ISKFunction function = SKFunction.FromNativeMethod(method, functionsInstance, pluginName, kernel.LoggerFactory); + if (result.ContainsKey(function.Name)) + { + throw new SKException("Function overloads are not supported, please differentiate function names"); + } + + result.Add(function.Name, function); + } + } + + logger.LogTrace("Methods imported {0}", result.Count); + + foreach (KeyValuePair f in result) + { + kernel.RegisterCustomFunction(f.Value); + } + + return result; + } + + /// + /// Run a single synchronous or asynchronous . + /// + /// The kernel. + /// A Semantic Kernel function to run + /// Input to process + /// The to monitor for cancellation requests. The default is . + /// Result of the function + public static Task RunAsync( + this IKernel kernel, + ISKFunction skFunction, + ContextVariables? variables = null, + CancellationToken cancellationToken = default) + { + Verify.NotNull(kernel); + return kernel.RunAsync(variables ?? new(), cancellationToken, skFunction); + } + + /// + /// Run a pipeline composed of synchronous and asynchronous functions. + /// + /// The kernel. + /// List of functions + /// Result of the function composition + public static Task RunAsync( + this IKernel kernel, + params ISKFunction[] pipeline) + { + Verify.NotNull(kernel); + return kernel.RunAsync(new ContextVariables(), pipeline); + } + + /// + /// Run a pipeline composed of synchronous and asynchronous functions. + /// + /// The kernel. + /// Input to process + /// List of functions + /// Result of the function composition + public static Task RunAsync( + this IKernel kernel, + string input, + params ISKFunction[] pipeline) + { + Verify.NotNull(kernel); + return kernel.RunAsync(new ContextVariables(input), pipeline); + } + + /// + /// Run a pipeline composed of synchronous and asynchronous functions. + /// + /// The kernel. + /// Input to process + /// List of functions + /// Result of the function composition + public static Task RunAsync( + this IKernel kernel, + ContextVariables variables, + params ISKFunction[] pipeline) + { + Verify.NotNull(kernel); + return kernel.RunAsync(variables, CancellationToken.None, pipeline); + } + + /// + /// Run a pipeline composed of synchronous and asynchronous functions. + /// + /// The kernel. + /// The to monitor for cancellation requests. The default is . + /// List of functions + /// Result of the function composition + public static Task RunAsync( + this IKernel kernel, + CancellationToken cancellationToken, + params ISKFunction[] pipeline) + { + Verify.NotNull(kernel); + return kernel.RunAsync(new ContextVariables(), cancellationToken, pipeline); + } + + /// + /// Run a pipeline composed of synchronous and asynchronous functions. + /// + /// The kernel. + /// Input to process + /// The to monitor for cancellation requests. The default is . + /// List of functions + /// Result of the function composition + public static Task RunAsync( + this IKernel kernel, + string input, + CancellationToken cancellationToken, + params ISKFunction[] pipeline) + { + Verify.NotNull(kernel); + return kernel.RunAsync(new ContextVariables(input), cancellationToken, pipeline); + } +} diff --git a/dotnet/src/SemanticKernel.Core/Memory/MemoryConfiguration.cs b/dotnet/src/SemanticKernel.Core/Memory/MemoryConfiguration.cs new file mode 100644 index 000000000000..fb08aeb3cb93 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Memory/MemoryConfiguration.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Memory; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of IKernel +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Kernel extension to configure the semantic memory with custom settings +/// +public static class MemoryConfiguration +{ + /// + /// Set the semantic memory to use the given memory storage and embeddings service. + /// + /// Kernel instance + /// Memory storage + /// Kernel service id for embedding generation + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public static void UseMemory(this IKernel kernel, IMemoryStore storage, string? embeddingsServiceId = null) + { + var embeddingGenerator = kernel.GetService(embeddingsServiceId); + + UseMemory(kernel, embeddingGenerator, storage); + } + + /// + /// Set the semantic memory to use the given memory storage and embedding generator. + /// + /// Kernel instance + /// Embedding generator + /// Memory storage + [SuppressMessage("Reliability", "CA2000:Dispose objects before losing scope", Justification = "The embeddingGenerator object is disposed by the kernel")] + [Obsolete("Memory functionality will be placed in separate Microsoft.SemanticKernel.Plugins.Memory package. This will be removed in a future release. See sample dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs in the semantic-kernel repository.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public static void UseMemory(this IKernel kernel, ITextEmbeddingGeneration embeddingGenerator, IMemoryStore storage) + { + Verify.NotNull(storage); + Verify.NotNull(embeddingGenerator); + + kernel.RegisterMemory(new SemanticTextMemory(storage, embeddingGenerator)); + } +} diff --git a/dotnet/src/SemanticKernel/Memory/SemanticTextMemory.cs b/dotnet/src/SemanticKernel.Core/Memory/SemanticTextMemory.cs similarity index 86% rename from dotnet/src/SemanticKernel/Memory/SemanticTextMemory.cs rename to dotnet/src/SemanticKernel.Core/Memory/SemanticTextMemory.cs index 57b94fc8262d..32fae7dba086 100644 --- a/dotnet/src/SemanticKernel/Memory/SemanticTextMemory.cs +++ b/dotnet/src/SemanticKernel.Core/Memory/SemanticTextMemory.cs @@ -11,13 +11,19 @@ namespace Microsoft.SemanticKernel.Memory; /// -/// Implementation of ./>. +/// Implementation of . Provides methods to save, retrieve, and search for text information +/// in a semantic memory store. /// -public sealed class SemanticTextMemory : ISemanticTextMemory, IDisposable +public sealed class SemanticTextMemory : ISemanticTextMemory { private readonly ITextEmbeddingGeneration _embeddingGenerator; private readonly IMemoryStore _storage; + /// + /// Initializes a new instance of the class. + /// + /// The memory store to use for storing and retrieving data. + /// The text embedding generator to use for generating embeddings. public SemanticTextMemory( IMemoryStore storage, ITextEmbeddingGeneration embeddingGenerator) @@ -101,7 +107,7 @@ public async IAsyncEnumerable SearchAsync( bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - Embedding queryEmbedding = await this._embeddingGenerator.GenerateEmbeddingAsync(query, cancellationToken).ConfigureAwait(false); + ReadOnlyMemory queryEmbedding = await this._embeddingGenerator.GenerateEmbeddingAsync(query, cancellationToken).ConfigureAwait(false); IAsyncEnumerable<(MemoryRecord, double)> results = this._storage.GetNearestMatchesAsync( collectionName: collection, @@ -122,13 +128,4 @@ public async Task> GetCollectionsAsync(CancellationToken cancellat { return await this._storage.GetCollectionsAsync(cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false); } - - public void Dispose() - { - // ReSharper disable once SuspiciousTypeConversion.Global - if (this._embeddingGenerator is IDisposable emb) { emb.Dispose(); } - - // ReSharper disable once SuspiciousTypeConversion.Global - if (this._storage is IDisposable storage) { storage.Dispose(); } - } } diff --git a/dotnet/src/SemanticKernel/Orchestration/ContextVariablesConverter.cs b/dotnet/src/SemanticKernel.Core/Orchestration/ContextVariablesConverter.cs similarity index 85% rename from dotnet/src/SemanticKernel/Orchestration/ContextVariablesConverter.cs rename to dotnet/src/SemanticKernel.Core/Orchestration/ContextVariablesConverter.cs index 26496054466a..6dfbdaffd5f5 100644 --- a/dotnet/src/SemanticKernel/Orchestration/ContextVariablesConverter.cs +++ b/dotnet/src/SemanticKernel.Core/Orchestration/ContextVariablesConverter.cs @@ -13,7 +13,7 @@ namespace Microsoft.SemanticKernel.Orchestration; public class ContextVariablesConverter : JsonConverter { /// - /// Read the JSON and convert to ContextVariables + /// Read the JSON and convert to ContextVariables. /// /// The JSON reader. /// The type to convert. @@ -39,6 +39,12 @@ public override ContextVariables Read(ref Utf8JsonReader reader, Type typeToConv return context; } + /// + /// Write the ContextVariables to JSON. + /// + /// The JSON writer. + /// The to write. + /// The JSON serializer options. public override void Write(Utf8JsonWriter writer, ContextVariables value, JsonSerializerOptions options) { writer.WriteStartArray(); diff --git a/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs b/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs new file mode 100644 index 000000000000..bb89b166f867 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.SemanticKernel.Orchestration; + +/// +/// Function runner implementation. +/// +internal class FunctionRunner : IFunctionRunner +{ + private readonly IKernel _kernel; + + /// + /// Initializes a new instance of the class. + /// + /// The kernel instance. + public FunctionRunner(IKernel kernel) + { + this._kernel = kernel; + } + + /// + public async Task RunAsync(ISKFunction skFunction, ContextVariables? variables = null, CancellationToken cancellationToken = default) + { + return (await this._kernel.RunAsync(skFunction, variables, cancellationToken).ConfigureAwait(false)) + .FunctionResults.First(); + } + + /// + public Task RunAsync(string pluginName, string functionName, ContextVariables? variables = null, CancellationToken cancellationToken = default) + { + var function = this._kernel.Functions.GetFunction(pluginName, functionName); + return this.RunAsync(function, variables, cancellationToken); + } +} diff --git a/dotnet/src/SemanticKernel/Planning/IPlan.cs b/dotnet/src/SemanticKernel.Core/Planning/IPlan.cs similarity index 82% rename from dotnet/src/SemanticKernel/Planning/IPlan.cs rename to dotnet/src/SemanticKernel.Core/Planning/IPlan.cs index da1005c25b16..f38b88bd19ee 100644 --- a/dotnet/src/SemanticKernel/Planning/IPlan.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/IPlan.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using Microsoft.SemanticKernel.SkillDefinition; - namespace Microsoft.SemanticKernel.Planning; /// diff --git a/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs b/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs new file mode 100644 index 000000000000..b6808c6130a5 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Planning; + +/// +/// Standard Semantic Kernel callable plan with instrumentation. +/// +internal sealed class InstrumentedPlan : IPlan +{ + /// + public string Name => this._plan.Name; + + /// + public string PluginName => this._plan.PluginName; + + /// + public string Description => this._plan.Description; + + /// + public AIRequestSettings? RequestSettings => this._plan.RequestSettings; + + /// + /// Initialize a new instance of the class. + /// + /// Instance of to decorate. + /// The to use for logging. If null, no logging will be performed. + public InstrumentedPlan( + IPlan plan, + ILoggerFactory? loggerFactory = null) + { + this._plan = plan; + this._logger = loggerFactory is not null ? loggerFactory.CreateLogger(typeof(InstrumentedPlan)) : NullLogger.Instance; + } + + /// + public FunctionView Describe() + { + return this._plan.Describe(); + } + + /// + public async Task InvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + return await this.InvokeWithInstrumentationAsync(() => + this._plan.InvokeAsync(context, requestSettings, cancellationToken)).ConfigureAwait(false); + } + + /// + public ISKFunction SetAIConfiguration(AIRequestSettings? requestSettings) => + this._plan.SetAIConfiguration(requestSettings); + + /// + public ISKFunction SetAIService(Func serviceFactory) => + this._plan.SetAIService(serviceFactory); + + #region private ================================================================================ + + private readonly IPlan _plan; + private readonly ILogger _logger; + + /// + /// Instance of for plan-related metrics. + /// + private static readonly Meter s_meter = new(typeof(Plan).FullName); + + /// + /// Instance of to measure and track the time of plan execution. + /// + private static readonly Histogram s_executionTimeHistogram = + s_meter.CreateHistogram( + name: "SK.Plan.Execution.ExecutionTime", + unit: "ms", + description: "Duration of plan execution"); + + /// + /// Instance of to keep track of the total number of plan executions. + /// + private static readonly Counter s_executionTotalCounter = + s_meter.CreateCounter( + name: "SK.Plan.Execution.ExecutionTotal", + description: "Total number of plan executions"); + + /// + /// Instance of to keep track of the number of successful plan executions. + /// + private static readonly Counter s_executionSuccessCounter = + s_meter.CreateCounter( + name: "SK.Plan.Execution.ExecutionSuccess", + description: "Number of successful plan executions"); + + /// + /// Instance of to keep track of the number of failed plan executions. + /// + private static readonly Counter s_executionFailureCounter = + s_meter.CreateCounter( + name: "SK.Plan.Execution.ExecutionFailure", + description: "Number of failed plan executions"); + + /// + /// Wrapper for instrumentation to be used in multiple invocation places. + /// + /// Delegate to instrument. + private async Task InvokeWithInstrumentationAsync(Func> func) + { + this._logger.LogInformation("Plan execution started."); + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + FunctionResult result; + + try + { + result = await func().ConfigureAwait(false); + } + catch (Exception ex) + { + this._logger.LogWarning("Plan execution status: {Status}", "Failed"); + this._logger.LogError(ex, "Plan execution exception details: {Message}", ex.Message); + + s_executionFailureCounter.Add(1); + throw; + } + finally + { + stopwatch.Stop(); + s_executionTotalCounter.Add(1); + s_executionTimeHistogram.Record(stopwatch.ElapsedMilliseconds); + } + + this._logger.LogInformation("Plan execution status: {Status}", "Success"); + this._logger.LogInformation("Plan execution finished in {ExecutionTime}ms", stopwatch.ElapsedMilliseconds); + + s_executionSuccessCounter.Add(1); + + return result; + } + + #endregion + + #region Obsolete ======================================================================= + + /// + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use ISKFunction.PluginName instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public string SkillName => this._plan.PluginName; + + /// + [Obsolete("Kernel no longer differentiates between Semantic and Native functions. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public bool IsSemantic => this._plan.IsSemantic; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultSkillCollection(IReadOnlyFunctionCollection skills) => this; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultFunctionCollection(IReadOnlyFunctionCollection functions) => this; + + #endregion +} diff --git a/dotnet/src/SemanticKernel/Planning/KernelPlanExtensions.cs b/dotnet/src/SemanticKernel.Core/Planning/KernelPlanExtensions.cs similarity index 93% rename from dotnet/src/SemanticKernel/Planning/KernelPlanExtensions.cs rename to dotnet/src/SemanticKernel.Core/Planning/KernelPlanExtensions.cs index 73bb71f44681..6918130cfed9 100644 --- a/dotnet/src/SemanticKernel/Planning/KernelPlanExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/KernelPlanExtensions.cs @@ -21,7 +21,7 @@ public static class KernelPlanExtensions /// Kernel instance to use /// Plan to import /// Function definition for the plan - public static SkillDefinition.ISKFunction ImportPlan(this IKernel kernel, Plan plan) + public static ISKFunction ImportPlan(this IKernel kernel, Plan plan) { return kernel.RegisterCustomFunction(plan); } @@ -32,9 +32,9 @@ public static SkillDefinition.ISKFunction ImportPlan(this IKernel kernel, Plan p /// Kernel instance to use /// Json representation of the plan /// Function definition for the plan - public static SkillDefinition.ISKFunction ImportPlanFromJson(this IKernel kernel, string json) + public static ISKFunction ImportPlanFromJson(this IKernel kernel, string json) { - return kernel.RegisterCustomFunction(Plan.FromJson(json, kernel.CreateNewContext())); + return kernel.RegisterCustomFunction(Plan.FromJson(json, kernel.Functions)); } /// diff --git a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs new file mode 100644 index 000000000000..d68b47a78374 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs @@ -0,0 +1,663 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Planning; + +/// +/// Standard Semantic Kernel callable plan. +/// Plan is used to create trees of s. +/// +[DebuggerDisplay("{DebuggerDisplay,nq}")] +public sealed class Plan : IPlan +{ + /// + /// State of the plan + /// + [JsonPropertyName("state")] + [JsonConverter(typeof(ContextVariablesConverter))] + public ContextVariables State { get; } = new(); + + /// + /// Steps of the plan + /// + [JsonPropertyName("steps")] + public IReadOnlyList Steps => this._steps.AsReadOnly(); + + /// + /// Parameters for the plan, used to pass information to the next step + /// + [JsonPropertyName("parameters")] + [JsonConverter(typeof(ContextVariablesConverter))] + public ContextVariables Parameters { get; set; } = new(); + + /// + /// Outputs for the plan, used to pass information to the caller + /// + [JsonPropertyName("outputs")] + public IList Outputs { get; set; } = new List(); + + /// + /// Gets whether the plan has a next step. + /// + [JsonIgnore] + public bool HasNextStep => this.NextStepIndex < this.Steps.Count; + + /// + /// Gets the next step index. + /// + [JsonPropertyName("next_step_index")] + public int NextStepIndex { get; private set; } + + #region ISKFunction implementation + + /// + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + /// + [JsonPropertyName("plugin_name")] + public string PluginName { get; set; } = string.Empty; + + /// + [JsonPropertyName("description")] + public string Description { get; set; } = string.Empty; + + /// + [JsonIgnore] + public AIRequestSettings? RequestSettings { get; private set; } + + #endregion ISKFunction implementation + + /// + /// Initializes a new instance of the class with a goal description. + /// + /// The goal of the plan used as description. + public Plan(string goal) + { + this.Name = GetRandomPlanName(); + this.Description = goal; + this.PluginName = nameof(Plan); + } + + /// + /// Initializes a new instance of the class with a goal description and steps. + /// + /// The goal of the plan used as description. + /// The steps to add. + public Plan(string goal, params ISKFunction[] steps) : this(goal) + { + this.AddSteps(steps); + } + + /// + /// Initializes a new instance of the class with a goal description and steps. + /// + /// The goal of the plan used as description. + /// The steps to add. + public Plan(string goal, params Plan[] steps) : this(goal) + { + this.AddSteps(steps); + } + + /// + /// Initializes a new instance of the class with a function. + /// + /// The function to execute. + public Plan(ISKFunction function) + { + this.SetFunction(function); + } + + /// + /// Initializes a new instance of the class with a function and steps. + /// + /// The name of the plan. + /// The name of the plugin. + /// The description of the plan. + /// The index of the next step. + /// The state of the plan. + /// The parameters of the plan. + /// The outputs of the plan. + /// The steps of the plan. + [JsonConstructor] + public Plan( + string name, + string pluginName, + string description, + int nextStepIndex, + ContextVariables state, + ContextVariables parameters, + IList outputs, + IReadOnlyList steps) + { + this.Name = name; + this.PluginName = pluginName; + this.Description = description; + this.NextStepIndex = nextStepIndex; + this.State = state; + this.Parameters = parameters; + this.Outputs = outputs; + this._steps.Clear(); + this.AddSteps(steps.ToArray()); + } + + /// + /// Deserialize a JSON string into a Plan object. + /// TODO: the context should never be null, it's required internally + /// + /// JSON string representation of a Plan + /// The collection of available functions.. + /// Whether to require functions to be registered. Only used when context is not null. + /// An instance of a Plan object. + /// If Context is not supplied, plan will not be able to execute. + public static Plan FromJson(string json, IReadOnlyFunctionCollection? functions = null, bool requireFunctions = true) + { + var plan = JsonSerializer.Deserialize(json, new JsonSerializerOptions { IncludeFields = true }) ?? new Plan(string.Empty); + + if (functions != null) + { + plan = SetAvailableFunctions(plan, functions, requireFunctions); + } + + return plan; + } + + /// + /// Get JSON representation of the plan. + /// + /// Whether to emit indented JSON + /// Plan serialized using JSON format + public string ToJson(bool indented = false) + { + return JsonSerializer.Serialize(this, new JsonSerializerOptions { WriteIndented = indented }); + } + + /// + /// Adds one or more existing plans to the end of the current plan as steps. + /// + /// The plans to add as steps to the current plan. + /// + /// When you add a plan as a step to the current plan, the steps of the added plan are executed after the steps of the current plan have completed. + /// + public void AddSteps(params Plan[] steps) + { + this._steps.AddRange(steps); + } + + /// + /// Adds one or more new steps to the end of the current plan. + /// + /// The steps to add to the current plan. + /// + /// When you add a new step to the current plan, it is executed after the previous step in the plan has completed. Each step can be a function call or another plan. + /// + public void AddSteps(params ISKFunction[] steps) + { + this._steps.AddRange(steps.Select(step => step is Plan plan ? plan : new Plan(step))); + } + + /// + /// Runs the next step in the plan using the provided kernel instance and variables. + /// + /// The kernel instance to use for executing the plan. + /// The variables to use for the execution of the plan. + /// The to monitor for cancellation requests. The default is . + /// A task representing the asynchronous execution of the plan's next step. + /// + /// This method executes the next step in the plan using the specified kernel instance and context variables. + /// The context variables contain the necessary information for executing the plan, such as the functions and logger. + /// The method returns a task representing the asynchronous execution of the plan's next step. + /// + public Task RunNextStepAsync(IKernel kernel, ContextVariables variables, CancellationToken cancellationToken = default) + { + var context = kernel.CreateNewContext(variables); + + return this.InvokeNextStepAsync(context, cancellationToken); + } + + /// + /// Invoke the next step of the plan + /// + /// Context to use + /// The to monitor for cancellation requests. The default is . + /// The updated plan + /// If an error occurs while running the plan + public async Task InvokeNextStepAsync(SKContext context, CancellationToken cancellationToken = default) + { + if (this.HasNextStep) + { + var step = this.Steps[this.NextStepIndex]; + + // Merge the state with the current context variables for step execution + var functionVariables = this.GetNextStepVariables(context.Variables, step); + + // Execute the step + var result = await context.Runner.RunAsync(step, functionVariables, cancellationToken).ConfigureAwait(false); + + var resultValue = result.Context.Result.Trim(); + + #region Update State + + // Update state with result + this.State.Update(resultValue); + + // Update Plan Result in State with matching outputs (if any) + if (this.Outputs.Intersect(step.Outputs).Any()) + { + if (this.State.TryGetValue(DefaultResultKey, out string? currentPlanResult)) + { + this.State.Set(DefaultResultKey, $"{currentPlanResult}\n{resultValue}"); + } + else + { + this.State.Set(DefaultResultKey, resultValue); + } + } + + // Update state with outputs (if any) + foreach (var item in step.Outputs) + { + if (result.Context.Variables.TryGetValue(item, out string? val)) + { + this.State.Set(item, val); + } + else + { + this.State.Set(item, resultValue); + } + } + + #endregion Update State + + this.NextStepIndex++; + } + + return this; + } + + #region ISKFunction implementation + + /// + public FunctionView Describe() + { + if (this.Function is not null) + { + return this.Function.Describe(); + } + + // The parameter mapping definitions from Plan -> Function + var stepParameters = this.Steps.SelectMany(s => s.Parameters); + + // The parameter descriptions from the Function + var stepDescriptions = this.Steps.SelectMany(s => s.Describe().Parameters); + + // The parameters for the Plan + var parameters = this.Parameters.Select(p => + { + var matchingParameter = stepParameters.FirstOrDefault(sp => sp.Value.Equals($"${p.Key}", StringComparison.OrdinalIgnoreCase)); + var stepDescription = stepDescriptions.FirstOrDefault(sd => sd.Name.Equals(matchingParameter.Key, StringComparison.OrdinalIgnoreCase)); + + return new ParameterView(p.Key, stepDescription?.Description, stepDescription?.DefaultValue, stepDescription?.Type, stepDescription?.IsRequired); + } + ).ToList(); + + return new(this.Name, this.PluginName, this.Description, parameters); + } + + /// + public async Task InvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + var result = new FunctionResult(this.Name, this.PluginName, context); + + if (this.Function is not null) + { + // Merge state with the current context variables. + // Then filter the variables to only those needed for the next step. + // This is done to prevent the function from having access to variables that it shouldn't. + AddVariablesToContext(this.State, context); + var functionVariables = this.GetNextStepVariables(context.Variables, this); + var functionContext = context.Clone(functionVariables, context.Functions); + + // Execute the step + result = await this.Function + .WithInstrumentation(context.LoggerFactory) + .InvokeAsync(functionContext, requestSettings, cancellationToken) + .ConfigureAwait(false); + this.UpdateFunctionResultWithOutputs(result); + } + else + { + // loop through steps and execute until completion + while (this.HasNextStep) + { + AddVariablesToContext(this.State, context); + await this.InvokeNextStepAsync(context, cancellationToken).ConfigureAwait(false); + this.UpdateContextWithOutputs(context); + + result = new FunctionResult(this.Name, this.PluginName, context, context.Result); + this.UpdateFunctionResultWithOutputs(result); + } + } + + return result; + } + + /// + public ISKFunction SetAIService(Func serviceFactory) + { + return this.Function is not null ? this.Function.SetAIService(serviceFactory) : this; + } + + /// + public ISKFunction SetAIConfiguration(AIRequestSettings? requestSettings) + { + return this.Function is not null ? this.Function.SetAIConfiguration(requestSettings) : this; + } + + #endregion ISKFunction implementation + + /// + /// Expand variables in the input string. + /// + /// Variables to use for expansion. + /// Input string to expand. + /// Expanded string. + internal string ExpandFromVariables(ContextVariables variables, string input) + { + var result = input; + var matches = s_variablesRegex.Matches(input); + var orderedMatches = matches.Cast().Select(m => m.Groups["var"].Value).Distinct().OrderByDescending(m => m.Length); + + foreach (var varName in orderedMatches) + { + if (variables.TryGetValue(varName, out string? value) || this.State.TryGetValue(varName, out value)) + { + result = result.Replace($"${varName}", value); + } + } + + return result; + } + + /// + /// Set functions for a plan and its steps. + /// + /// Plan to set functions for. + /// The collection of available functions. + /// Whether to throw an exception if a function is not found. + /// The plan with functions set. + private static Plan SetAvailableFunctions(Plan plan, IReadOnlyFunctionCollection functions, bool requireFunctions = true) + { + if (plan.Steps.Count == 0) + { + Verify.NotNull(functions); + + if (functions.TryGetFunction(plan.PluginName, plan.Name, out var planFunction)) + { + plan.SetFunction(planFunction); + } + else if (requireFunctions) + { + throw new SKException($"Function '{plan.PluginName}.{plan.Name}' not found in function collection"); + } + } + else + { + foreach (var step in plan.Steps) + { + SetAvailableFunctions(step, functions, requireFunctions); + } + } + + return plan; + } + + /// + /// Add any missing variables from a plan state variables to the context. + /// + private static void AddVariablesToContext(ContextVariables vars, SKContext context) + { + // Loop through vars and add anything missing to context + foreach (var item in vars) + { + if (!context.Variables.TryGetValue(item.Key, out string? value) || string.IsNullOrEmpty(value)) + { + context.Variables.Set(item.Key, item.Value); + } + } + } + + /// + /// Update the context with the outputs from the current step. + /// + /// The context to update. + /// The updated context. + private SKContext UpdateContextWithOutputs(SKContext context) + { + var resultString = this.State.TryGetValue(DefaultResultKey, out string? result) ? result : this.State.ToString(); + context.Variables.Update(resultString); + + // copy previous step's variables to the next step + foreach (var item in this._steps[this.NextStepIndex - 1].Outputs) + { + if (this.State.TryGetValue(item, out string? val)) + { + context.Variables.Set(item, val); + } + else + { + context.Variables.Set(item, resultString); + } + } + + return context; + } + + /// + /// Update the function result with the outputs from the current state. + /// + /// The function result to update. + /// The updated function result. + private FunctionResult UpdateFunctionResultWithOutputs(FunctionResult functionResult) + { + foreach (var output in this.Outputs) + { + if (this.State.TryGetValue(output, out var value)) + { + functionResult.Metadata[output] = value; + } + else if (functionResult.Context.Variables.TryGetValue(output, out var val)) + { + functionResult.Metadata[output] = val; + } + } + + return functionResult; + } + + /// + /// Get the variables for the next step in the plan. + /// + /// The current context variables. + /// The next step in the plan. + /// The context variables for the next step in the plan. + private ContextVariables GetNextStepVariables(ContextVariables variables, Plan step) + { + // Priority for Input + // - Parameters (expand from variables if needed) + // - SKContext.Variables + // - Plan.State + // - Empty if sending to another plan + // - Plan.Description + + var input = string.Empty; + if (!string.IsNullOrEmpty(step.Parameters.Input)) + { + input = this.ExpandFromVariables(variables, step.Parameters.Input!); + } + else if (!string.IsNullOrEmpty(variables.Input)) + { + input = variables.Input; + } + else if (!string.IsNullOrEmpty(this.State.Input)) + { + input = this.State.Input; + } + else if (step.Steps.Count > 0) + { + input = string.Empty; + } + else if (!string.IsNullOrEmpty(this.Description)) + { + input = this.Description; + } + + var stepVariables = new ContextVariables(input); + + // Priority for remaining stepVariables is: + // - Function Parameters (pull from variables or state by a key value) + // - Step Parameters (pull from variables or state by a key value) + // - All other variables. These are carried over in case the function wants access to the ambient content. + var functionParameters = step.Describe(); + foreach (var param in functionParameters.Parameters) + { + if (param.Name.Equals(ContextVariables.MainKey, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (variables.TryGetValue(param.Name, out string? value)) + { + stepVariables.Set(param.Name, value); + } + else if (this.State.TryGetValue(param.Name, out value) && !string.IsNullOrEmpty(value)) + { + stepVariables.Set(param.Name, value); + } + } + + foreach (var item in step.Parameters) + { + // Don't overwrite variable values that are already set + if (stepVariables.ContainsKey(item.Key)) + { + continue; + } + + var expandedValue = this.ExpandFromVariables(variables, item.Value); + if (!expandedValue.Equals(item.Value, StringComparison.OrdinalIgnoreCase)) + { + stepVariables.Set(item.Key, expandedValue); + } + else if (variables.TryGetValue(item.Key, out string? value)) + { + stepVariables.Set(item.Key, value); + } + else if (this.State.TryGetValue(item.Key, out value)) + { + stepVariables.Set(item.Key, value); + } + else + { + stepVariables.Set(item.Key, expandedValue); + } + } + + foreach (KeyValuePair item in variables) + { + if (!stepVariables.ContainsKey(item.Key)) + { + stepVariables.Set(item.Key, item.Value); + } + } + + return stepVariables; + } + + private void SetFunction(ISKFunction function) + { + this.Function = function; + this.Name = function.Name; + this.PluginName = function.PluginName; + this.Description = function.Description; + this.RequestSettings = function.RequestSettings; + +#pragma warning disable CS0618 // Type or member is obsolete + this.IsSemantic = function.IsSemantic; +#pragma warning restore CS0618 // Type or member is obsolete + } + + private static string GetRandomPlanName() => "plan" + Guid.NewGuid().ToString("N"); + + private ISKFunction? Function { get; set; } + + private readonly List _steps = new(); + + private static readonly Regex s_variablesRegex = new(@"\$(?\w+)"); + + private const string DefaultResultKey = "PLAN.RESULT"; + + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + private string DebuggerDisplay + { + get + { + string display = this.Description; + + if (!string.IsNullOrWhiteSpace(this.Name)) + { + display = $"{this.Name} ({display})"; + } + + if (this._steps.Count > 0) + { + display += $", Steps = {this._steps.Count}, NextStep = {this.NextStepIndex}"; + } + + return display; + } + } + + #region Obsolete + + /// + [JsonIgnore] + [Obsolete("Methods, properties and classes which include Skill in the name have been renamed. Use ISKFunction.PluginName instead. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public string SkillName => this.PluginName; + + /// + [JsonIgnore] + [Obsolete("Kernel no longer differentiates between Semantic and Native functions. This will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public bool IsSemantic { get; private set; } + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultSkillCollection(IReadOnlyFunctionCollection skills) => this; + + /// + [Obsolete("This method is a nop and will be removed in a future release.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public ISKFunction SetDefaultFunctionCollection(IReadOnlyFunctionCollection functions) => this; + + #endregion +} diff --git a/dotnet/src/SemanticKernel/Planning/PlanExtensions.cs b/dotnet/src/SemanticKernel.Core/Planning/PlanExtensions.cs similarity index 81% rename from dotnet/src/SemanticKernel/Planning/PlanExtensions.cs rename to dotnet/src/SemanticKernel.Core/Planning/PlanExtensions.cs index 485a1a22e806..c163c617c48c 100644 --- a/dotnet/src/SemanticKernel/Planning/PlanExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/PlanExtensions.cs @@ -21,10 +21,10 @@ public static string ToSafePlanString(this Plan plan, string indent = " ") { if (step.Steps.Count == 0) { - string skillName = step.SkillName; + string pluginName = step.PluginName; string stepName = step.Name; - return $"{indent}{indent}- {string.Join(".", skillName, stepName)}"; + return $"{indent}{indent}- {string.Join(".", pluginName, stepName)}"; } return step.ToSafePlanString(indent + indent); @@ -44,7 +44,7 @@ public static string ToPlanString(this Plan plan, string indent = " ") { if (step.Steps.Count == 0) { - string skillName = step.SkillName; + string pluginName = step.PluginName; string stepName = step.Name; string parameters = string.Join(" ", step.Parameters.Select(param => $"{param.Key}='{param.Value}'")); @@ -59,7 +59,7 @@ public static string ToPlanString(this Plan plan, string indent = " ") outputs = $" => {outputs}"; } - return $"{indent}{indent}- {string.Join(".", skillName, stepName)}{parameters}{outputs}"; + return $"{indent}{indent}- {string.Join(".", pluginName, stepName)}{parameters}{outputs}"; } return step.ToPlanString(indent + indent); @@ -72,9 +72,9 @@ public static string ToPlanString(this Plan plan, string indent = " ") /// Returns decorated instance of with enabled instrumentation. /// /// Instance of to decorate. - /// Optional logger. - public static IPlan WithInstrumentation(this IPlan plan, ILogger? logger = null) + /// The to use for logging. If null, no logging will be performed. + public static IPlan WithInstrumentation(this IPlan plan, ILoggerFactory? loggerFactory = null) { - return new InstrumentedPlan(plan, logger); + return new InstrumentedPlan(plan, loggerFactory); } } diff --git a/dotnet/src/SemanticKernel.Core/Reliability/NullHttpRetryHandler.cs b/dotnet/src/SemanticKernel.Core/Reliability/NullHttpRetryHandler.cs new file mode 100644 index 000000000000..a12528df0526 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Reliability/NullHttpRetryHandler.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Http; + +namespace Microsoft.SemanticKernel.Reliability; + +/// +/// A factory for creating instances of . +/// +public class NullHttpRetryHandlerFactory : IDelegatingHandlerFactory +{ + /// + /// Creates a new instance of . + /// + /// The to use for logging. If null, no logging will be performed. + /// A new instance of . + public DelegatingHandler Create(ILoggerFactory? loggerFactory) + { + return new NullHttpRetryHandler(); + } +} + +/// +/// A HTTP retry handler that does not retry. +/// +/// +/// This handler is useful when you want to disable retry functionality in your HTTP requests. +/// +public class NullHttpRetryHandler : DelegatingHandler +{ +} diff --git a/dotnet/src/SemanticKernel.Core/SemanticKernel.Core.csproj b/dotnet/src/SemanticKernel.Core/SemanticKernel.Core.csproj new file mode 100644 index 000000000000..116f47301d39 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/SemanticKernel.Core.csproj @@ -0,0 +1,37 @@ + + + + + Microsoft.SemanticKernel.Core + Microsoft.SemanticKernel + netstandard2.0 + true + true + + + + + + + Semantic Kernel Core + + Semantic Kernel core orchestration, runtime and functions. + This package is automatically installed by 'Microsoft.SemanticKernel' package with other useful packages. + Install this package manually only if you are selecting individual Semantic Kernel components. + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/SemanticKernel/Services/AIServiceCollection.cs b/dotnet/src/SemanticKernel.Core/Services/AIServiceCollection.cs similarity index 93% rename from dotnet/src/SemanticKernel/Services/AIServiceCollection.cs rename to dotnet/src/SemanticKernel.Core/Services/AIServiceCollection.cs index dd165d955e26..abb861239cb3 100644 --- a/dotnet/src/SemanticKernel/Services/AIServiceCollection.cs +++ b/dotnet/src/SemanticKernel.Core/Services/AIServiceCollection.cs @@ -6,6 +6,9 @@ namespace Microsoft.SemanticKernel.Services; +/// +/// A collection of AI services that can be registered and built into an . +/// [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix")] public class AIServiceCollection { @@ -13,7 +16,6 @@ public class AIServiceCollection private const string DefaultKey = "__DEFAULT__"; // A dictionary that maps a service type to a nested dictionary of names and service instances or factories - //private readonly Dictionary> _services = new(); private readonly Dictionary>> _services = new(); // A dictionary that maps a service type to the name of the default service @@ -90,9 +92,9 @@ public void SetService(string? name, Func factory, bool setAsDefault = fal } /// - /// Builds an from the registered services and default names. + /// Builds an from the registered services and default names. /// - /// + /// An containing the registered services. public IAIServiceProvider Build() { // Create a clone of the services and defaults Dictionaries to prevent further changes @@ -112,5 +114,5 @@ public IAIServiceProvider Build() private bool HasDefault() where T : IAIService => this._defaultIds.TryGetValue(typeof(T), out var defaultName) - && !string.IsNullOrEmpty(defaultName); + && !string.IsNullOrEmpty(defaultName); } diff --git a/dotnet/src/SemanticKernel.Core/Services/AIServiceProvider.cs b/dotnet/src/SemanticKernel.Core/Services/AIServiceProvider.cs new file mode 100644 index 000000000000..d3cec01a3031 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Services/AIServiceProvider.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; + +namespace Microsoft.SemanticKernel.Services; +/// +/// Provides AI services by managing a collection of named service instances. +/// +public class AIServiceProvider : NamedServiceProvider, IAIServiceProvider +{ + /// + /// Initializes a new instance of the class. + /// + /// A dictionary of service types and their corresponding named instances. + /// A dictionary of service types and their default instance names. + public AIServiceProvider(Dictionary>> services, Dictionary defaultIds) + : base(services, defaultIds) + { + } +} diff --git a/dotnet/src/SemanticKernel/Services/NamedServiceProvider.cs b/dotnet/src/SemanticKernel.Core/Services/NamedServiceProvider.cs similarity index 77% rename from dotnet/src/SemanticKernel/Services/NamedServiceProvider.cs rename to dotnet/src/SemanticKernel.Core/Services/NamedServiceProvider.cs index 24069e092b29..ba5c903e4cd2 100644 --- a/dotnet/src/SemanticKernel/Services/NamedServiceProvider.cs +++ b/dotnet/src/SemanticKernel.Core/Services/NamedServiceProvider.cs @@ -5,15 +5,23 @@ namespace Microsoft.SemanticKernel.Services; +/// +/// Provides named services of type . Allows for the registration and retrieval of services by name. +/// +/// The type of service provided by this provider. public class NamedServiceProvider : INamedServiceProvider { // A dictionary that maps a service type to a nested dictionary of names and service instances or factories - //private readonly Dictionary> _services = new(); private readonly Dictionary>> _services; // A dictionary that maps a service type to the name of the default service private readonly Dictionary _defaultIds; + /// + /// Initializes a new instance of the class. + /// + /// A dictionary that maps a service type to a nested dictionary of names and service instances or factories. + /// A dictionary that maps a service type to the name of the default service. public NamedServiceProvider( Dictionary>> services, Dictionary defaultIds) diff --git a/dotnet/src/SemanticKernel.Core/TemplateEngine/PromptTemplate.cs b/dotnet/src/SemanticKernel.Core/TemplateEngine/PromptTemplate.cs new file mode 100644 index 000000000000..74b9912c9786 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/TemplateEngine/PromptTemplate.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.TemplateEngine; + +/// +/// Prompt template. +/// +public sealed class PromptTemplate : IPromptTemplate +{ + private readonly string _template; + private readonly IPromptTemplateEngine _templateEngine; + + // ReSharper disable once NotAccessedField.Local + private readonly PromptTemplateConfig _promptConfig; + + /// + /// Constructor for PromptTemplate. + /// + /// Template. + /// Prompt template configuration. + /// Kernel in which template is to take effect. + public PromptTemplate(string template, PromptTemplateConfig promptTemplateConfig, IKernel kernel) + : this(template, promptTemplateConfig, kernel.PromptTemplateEngine) + { + } + + /// + /// Constructor for PromptTemplate. + /// + /// Template. + /// Prompt template configuration. + /// Prompt template engine. + public PromptTemplate( + string template, + PromptTemplateConfig promptTemplateConfig, + IPromptTemplateEngine promptTemplateEngine) + { + this._template = template; + this._templateEngine = promptTemplateEngine; + this._promptConfig = promptTemplateConfig; + + this._params = new(() => this.InitParameters()); + } + + /// + /// The list of parameters used by the function, using JSON settings and template variables. + /// + /// List of parameters + public IReadOnlyList Parameters + => this._params.Value; + + /// + /// Render the template using the information in the context + /// + /// Kernel execution context helpers + /// The to monitor for cancellation requests. The default is . + /// Prompt rendered to string + public async Task RenderAsync(SKContext executionContext, CancellationToken cancellationToken) + { + return await this._templateEngine.RenderAsync(this._template, executionContext, cancellationToken).ConfigureAwait(false); + } + + private readonly Lazy> _params; + + private List InitParameters() + { + // Parameters from config.json + Dictionary result = new(this._promptConfig.Input.Parameters.Count, StringComparer.OrdinalIgnoreCase); + foreach (var p in this._promptConfig.Input.Parameters) + { + result[p.Name] = new ParameterView(p.Name, p.Description, p.DefaultValue); + } + + return result.Values.ToList(); + } +} diff --git a/dotnet/src/SemanticKernel.Core/TemplateEngine/PromptTemplateConfig.cs b/dotnet/src/SemanticKernel.Core/TemplateEngine/PromptTemplateConfig.cs new file mode 100644 index 000000000000..9e5817d27c0d --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/TemplateEngine/PromptTemplateConfig.cs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Text; + +namespace Microsoft.SemanticKernel.TemplateEngine; + +/// +/// Prompt template configuration. +/// +public class PromptTemplateConfig +{ + /// + /// Input parameter for semantic functions. + /// + public class InputParameter + { + /// + /// Name of the parameter to pass to the function. + /// e.g. when using "{{$input}}" the name is "input", when using "{{$style}}" the name is "style", etc. + /// + [JsonPropertyName("name")] + [JsonPropertyOrder(1)] + public string Name { get; set; } = string.Empty; + + /// + /// Parameter description for UI apps and planner. Localization is not supported here. + /// + [JsonPropertyName("description")] + [JsonPropertyOrder(2)] + public string Description { get; set; } = string.Empty; + + /// + /// Default value when nothing is provided. + /// + [JsonPropertyName("defaultValue")] + [JsonPropertyOrder(3)] + public string DefaultValue { get; set; } = string.Empty; + } + + /// + /// Input configuration (list of all input parameters for a semantic function). + /// + public class InputConfig + { + /// + /// Gets or sets the list of input parameters. + /// + [JsonPropertyName("parameters")] + [JsonPropertyOrder(1)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List Parameters { get; set; } = new(); + } + + /// + /// Schema - Not currently used. + /// + [JsonPropertyName("schema")] + [JsonPropertyOrder(1)] + public int Schema { get; set; } = 1; + + /// + /// Description + /// + [JsonPropertyName("description")] + [JsonPropertyOrder(2)] + public string Description { get; set; } = string.Empty; + + /// + /// Input configuration (that is, list of all input parameters). + /// + [JsonPropertyName("input")] + [JsonPropertyOrder(3)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public InputConfig Input { get; set; } = new(); + + /// + /// Model request settings. + /// Initially only a single model request settings is supported. + /// + [JsonPropertyName("models")] + [JsonPropertyOrder(4)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List ModelSettings { get; set; } = new(); + + /// + /// Return the default + /// + public AIRequestSettings GetDefaultRequestSettings() + { + return this.ModelSettings.FirstOrDefault(); + } + + #region Obsolete + /// + /// Type, such as "completion", "embeddings", etc. + /// + /// TODO: use enum + [JsonPropertyName("type")] + [JsonPropertyOrder(5)] + [Obsolete("Type property is no longer used. This will be removed in a future release.")] + public string Type { get; set; } = "completion"; + + /// + /// Completion configuration parameters. + /// + [JsonPropertyName("completion")] + [JsonPropertyOrder(6)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Obsolete("Completion is no longer no longer supported. Use PromptTemplateConfig.Models collection instead. This will be removed in a future release.")] + public AIRequestSettings? Completion + { + get { return this.GetDefaultRequestSettings(); } + set + { + if (value is not null) + { + this.ModelSettings.Add(value); + } + } + } + + /// + /// Default AI services to use. + /// + [JsonPropertyName("default_services")] + [JsonPropertyOrder(7)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Obsolete("DefaultServices property is no longer used. This will be removed in a future release.")] + public List DefaultServices { get; set; } = new(); + #endregion + + /// + /// Creates a prompt template configuration from JSON. + /// + /// JSON of the prompt template configuration. + /// Prompt template configuration. + /// Thrown when the deserialization returns null. + public static PromptTemplateConfig FromJson(string json) + { + var result = Json.Deserialize(json); + return result ?? throw new ArgumentException("Unable to deserialize prompt template config from argument. The deserialization returned null.", nameof(json)); + } +} diff --git a/dotnet/src/SemanticKernel.Core/Text/TextChunker.cs b/dotnet/src/SemanticKernel.Core/Text/TextChunker.cs new file mode 100644 index 000000000000..a31a915c9233 --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Text/TextChunker.cs @@ -0,0 +1,317 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; + +namespace Microsoft.SemanticKernel.Text; + +/// +/// Split text in chunks, attempting to leave meaning intact. +/// For plain text, split looking at new lines first, then periods, and so on. +/// For markdown, split looking at punctuation first, and so on. +/// +public static class TextChunker +{ + /// + /// Delegate for counting tokens in a string. + /// + /// The input string to count tokens in. + /// The number of tokens in the input string. + public delegate int TokenCounter(string input); + + private static readonly char[] s_spaceChar = new[] { ' ' }; + private static readonly string?[] s_plaintextSplitOptions = new[] { "\n\r", ".", "?!", ";", ":", ",", ")]}", " ", "-", null }; + private static readonly string?[] s_markdownSplitOptions = new[] { ".", "?!", ";", ":", ",", ")]}", " ", "-", "\n\r", null }; + + /// + /// Split plain text into lines. + /// + /// Text to split + /// Maximum number of tokens per line. + /// Function to count tokens in a string. If not supplied, the default counter will be used. + /// List of lines. + public static List SplitPlainTextLines(string text, int maxTokensPerLine, TokenCounter? tokenCounter = null) + { + tokenCounter ??= DefaultTokenCounter; + + return InternalSplitLines(text, maxTokensPerLine, trim: true, s_plaintextSplitOptions, tokenCounter); + } + + /// + /// Split markdown text into lines. + /// + /// Text to split + /// Maximum number of tokens per line. + /// Function to count tokens in a string. If not supplied, the default counter will be used. + /// List of lines. + public static List SplitMarkDownLines(string text, int maxTokensPerLine, TokenCounter? tokenCounter = null) + { + tokenCounter ??= DefaultTokenCounter; + + return InternalSplitLines(text, maxTokensPerLine, trim: true, s_markdownSplitOptions, tokenCounter); + } + + /// + /// Split plain text into paragraphs. + /// + /// Lines of text. + /// Maximum number of tokens per paragraph. + /// Number of tokens to overlap between paragraphs. + /// Text to be prepended to each individual chunk. + /// Function to count tokens in a string. If not supplied, the default counter will be used. + /// List of paragraphs. + public static List SplitPlainTextParagraphs(List lines, int maxTokensPerParagraph, int overlapTokens = 0, string? chunkHeader = null, TokenCounter? tokenCounter = null) + { + tokenCounter ??= DefaultTokenCounter; + + return InternalSplitTextParagraphs(lines, maxTokensPerParagraph, overlapTokens, chunkHeader, (text, maxTokens) => InternalSplitLines(text, maxTokens, trim: false, s_plaintextSplitOptions, tokenCounter), tokenCounter); + } + + /// + /// Split markdown text into paragraphs. + /// + /// Lines of text. + /// Maximum number of tokens per paragraph. + /// Number of tokens to overlap between paragraphs. + /// Text to be prepended to each individual chunk. + /// Function to count tokens in a string. If not supplied, the default counter will be used. + /// List of paragraphs. + public static List SplitMarkdownParagraphs(List lines, int maxTokensPerParagraph, int overlapTokens = 0, string? chunkHeader = null, TokenCounter? tokenCounter = null) + { + tokenCounter ??= DefaultTokenCounter; + + return InternalSplitTextParagraphs(lines, maxTokensPerParagraph, overlapTokens, chunkHeader, (text, maxTokens) => InternalSplitLines(text, maxTokens, trim: false, s_markdownSplitOptions, tokenCounter), tokenCounter); + } + + private static List InternalSplitTextParagraphs(List lines, int maxTokensPerParagraph, int overlapTokens, string? chunkHeader, Func> longLinesSplitter, TokenCounter tokenCounter) + { + if (maxTokensPerParagraph <= 0) + { + throw new ArgumentException("maxTokensPerParagraph should be a positive number"); + } + + if (maxTokensPerParagraph <= overlapTokens) + { + throw new ArgumentException("overlapTokens cannot be larger than maxTokensPerParagraph"); + } + + if (lines.Count == 0) + { + return new List(); + } + + var chunkHeaderTokens = chunkHeader is { Length: > 0 } ? tokenCounter(chunkHeader) : 0; + var adjustedMaxTokensPerParagraph = maxTokensPerParagraph - overlapTokens - chunkHeaderTokens; + + // Split long lines first + IEnumerable truncatedLines = lines.SelectMany(line => longLinesSplitter(line, adjustedMaxTokensPerParagraph)); + + var paragraphs = BuildParagraph(truncatedLines, adjustedMaxTokensPerParagraph, longLinesSplitter, tokenCounter); + var processedParagraphs = ProcessParagraphs(paragraphs, adjustedMaxTokensPerParagraph, overlapTokens, chunkHeader, longLinesSplitter, tokenCounter); + + return processedParagraphs; + } + + private static List BuildParagraph(IEnumerable truncatedLines, int maxTokensPerParagraph, Func> longLinesSplitter, TokenCounter tokenCounter) + { + StringBuilder paragraphBuilder = new(); + List paragraphs = new(); + + foreach (string line in truncatedLines) + { + if (paragraphBuilder.Length > 0 && tokenCounter(paragraphBuilder.ToString()) + tokenCounter(line) + 1 >= maxTokensPerParagraph) + { + // Complete the paragraph and prepare for the next + paragraphs.Add(paragraphBuilder.ToString().Trim()); + paragraphBuilder.Clear(); + } + + paragraphBuilder.AppendLine(line); + } + + if (paragraphBuilder.Length > 0) + { + // Add the final paragraph if there's anything remaining + paragraphs.Add(paragraphBuilder.ToString().Trim()); + } + + return paragraphs; + } + + private static List ProcessParagraphs(List paragraphs, int adjustedMaxTokensPerParagraph, int overlapTokens, string? chunkHeader, Func> longLinesSplitter, TokenCounter tokenCounter) + { + var processedParagraphs = new List(); + var paragraphStringBuilder = new StringBuilder(); + + // distribute text more evenly in the last paragraphs when the last paragraph is too short. + if (paragraphs.Count > 1) + { + var lastParagraph = paragraphs[paragraphs.Count - 1]; + var secondLastParagraph = paragraphs[paragraphs.Count - 2]; + + if (tokenCounter(lastParagraph) < adjustedMaxTokensPerParagraph / 4) + { + var lastParagraphTokens = lastParagraph.Split(s_spaceChar, StringSplitOptions.RemoveEmptyEntries); + var secondLastParagraphTokens = secondLastParagraph.Split(s_spaceChar, StringSplitOptions.RemoveEmptyEntries); + + var lastParagraphTokensCount = lastParagraphTokens.Length; + var secondLastParagraphTokensCount = secondLastParagraphTokens.Length; + + if (lastParagraphTokensCount + secondLastParagraphTokensCount <= adjustedMaxTokensPerParagraph) + { + var newSecondLastParagraph = string.Join(" ", secondLastParagraphTokens); + var newLastParagraph = string.Join(" ", lastParagraphTokens); + + paragraphs[paragraphs.Count - 2] = $"{newSecondLastParagraph} {newLastParagraph}"; + paragraphs.RemoveAt(paragraphs.Count - 1); + } + } + } + + for (int i = 0; i < paragraphs.Count; i++) + { + paragraphStringBuilder.Clear(); + + if (chunkHeader is not null) + { + paragraphStringBuilder.Append(chunkHeader); + } + + var paragraph = paragraphs[i]; + + if (overlapTokens > 0 && i < paragraphs.Count - 1) + { + var nextParagraph = paragraphs[i + 1]; + var split = longLinesSplitter(nextParagraph, overlapTokens); + + paragraphStringBuilder.Append(paragraph); + + if (split.FirstOrDefault() is string overlap) + { + paragraphStringBuilder.Append(' '); + paragraphStringBuilder.Append(overlap); + } + } + else + { + paragraphStringBuilder.Append(paragraph); + } + + processedParagraphs.Add(paragraphStringBuilder.ToString()); + } + + return processedParagraphs; + } + + private static List InternalSplitLines(string text, int maxTokensPerLine, bool trim, string?[] splitOptions, TokenCounter tokenCounter) + { + var result = new List(); + + text = text.NormalizeLineEndings(); + result.Add(text); + for (int i = 0; i < splitOptions.Length; i++) + { + int count = result.Count; // track where the original input left off + var (splits2, inputWasSplit2) = Split(result, maxTokensPerLine, splitOptions[i].AsSpan(), trim, tokenCounter); + result.AddRange(splits2); + result.RemoveRange(0, count); // remove the original input + if (!inputWasSplit2) + { + break; + } + } + return result; + } + + private static (List, bool) Split(List input, int maxTokens, ReadOnlySpan separators, bool trim, TokenCounter tokenCounter) + { + bool inputWasSplit = false; + List result = new(); + int count = input.Count; + for (int i = 0; i < count; i++) + { + var (splits, split) = Split(input[i].AsSpan(), input[i], maxTokens, separators, trim, tokenCounter); + result.AddRange(splits); + inputWasSplit |= split; + } + return (result, inputWasSplit); + } + + private static (List, bool) Split(ReadOnlySpan input, string? inputString, int maxTokens, ReadOnlySpan separators, bool trim, TokenCounter tokenCounter) + { + Debug.Assert(inputString is null || input.SequenceEqual(inputString.AsSpan())); + List result = new(); + var inputWasSplit = false; + if (tokenCounter(input.ToString()) > maxTokens) + { + inputWasSplit = true; + + int half = input.Length / 2; + int cutPoint = -1; + + if (separators.IsEmpty) + { + cutPoint = half; + } + else if (input.Length > 2) + { + int pos = 0; + while (true) + { + int index = input.Slice(pos, input.Length - 1 - pos).IndexOfAny(separators); + if (index < 0) + { + break; + } + + index += pos; + + if (Math.Abs(half - index) < Math.Abs(half - cutPoint)) + { + cutPoint = index + 1; + } + + pos = index + 1; + } + } + + if (cutPoint > 0) + { + var firstHalf = input.Slice(0, cutPoint); + var secondHalf = input.Slice(cutPoint); + if (trim) + { + firstHalf = firstHalf.Trim(); + secondHalf = secondHalf.Trim(); + } + + // Recursion + var (splits1, split1) = Split(firstHalf, null, maxTokens, separators, trim, tokenCounter); + result.AddRange(splits1); + var (splits2, split2) = Split(secondHalf, null, maxTokens, separators, trim, tokenCounter); + result.AddRange(splits2); + + inputWasSplit = split1 || split2; + return (result, inputWasSplit); + } + } + + result.Add((inputString is not null, trim) switch + { + (true, true) => inputString!.Trim(), + (true, false) => inputString!, + (false, true) => input.Trim().ToString(), + (false, false) => input.ToString(), + }); + + return (result, inputWasSplit); + } + + private static int DefaultTokenCounter(string input) + { + return input.Length / 4; + } +} diff --git a/dotnet/src/SemanticKernel.MetaPackage/SemanticKernel.MetaPackage.csproj b/dotnet/src/SemanticKernel.MetaPackage/SemanticKernel.MetaPackage.csproj index 73e4cb561aac..3bd7bdee934d 100644 --- a/dotnet/src/SemanticKernel.MetaPackage/SemanticKernel.MetaPackage.csproj +++ b/dotnet/src/SemanticKernel.MetaPackage/SemanticKernel.MetaPackage.csproj @@ -1,26 +1,23 @@  - Microsoft.SemanticKernel $(AssemblyName) netstandard2.0 - - Semantic Kernel Semantic Kernel common package collection, including SK Core, OpenAI, Azure OpenAI, DALL-E 2. Empowers app owners to integrate cutting-edge LLM technology quickly and easily into their apps. - - + + + - - - + + + - - + \ No newline at end of file diff --git a/dotnet/src/SemanticKernel.UnitTests/.editorconfig b/dotnet/src/SemanticKernel.UnitTests/.editorconfig index 8f4c52fa9f51..394eef685f21 100644 --- a/dotnet/src/SemanticKernel.UnitTests/.editorconfig +++ b/dotnet/src/SemanticKernel.UnitTests/.editorconfig @@ -2,4 +2,5 @@ [*.cs] dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave - +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/SemanticKernel.UnitTests/AI/AIExceptionTests.cs b/dotnet/src/SemanticKernel.UnitTests/AI/AIExceptionTests.cs deleted file mode 100644 index b7186b044672..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/AI/AIExceptionTests.cs +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.AI; -using Xunit; - -namespace SemanticKernel.UnitTests.AI; - -public class AIExceptionTests -{ - [Fact] - public void ItRoundtripsArgsToErrorCodeCtor() - { - // Arrange - var e = new AIException(AIException.ErrorCodes.InvalidRequest); - - // Assert - Assert.Equal(AIException.ErrorCodes.InvalidRequest, e.ErrorCode); - Assert.Contains("Invalid request", e.Message, StringComparison.Ordinal); - Assert.Null(e.Detail); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageCtor() - { - // Arrange - const string Message = "this is a test"; - var e = new AIException(AIException.ErrorCodes.InvalidRequest, Message); - - // Assert - Assert.Equal(AIException.ErrorCodes.InvalidRequest, e.ErrorCode); - Assert.Contains("Invalid request", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Null(e.Detail); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageExceptionCtor() - { - // Arrange - const string Message = "this is a test"; - var inner = new FormatException(); - var e = new AIException(AIException.ErrorCodes.InvalidRequest, Message, inner); - - // Assert - Assert.Equal(AIException.ErrorCodes.InvalidRequest, e.ErrorCode); - Assert.Contains("Invalid request", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Null(e.Detail); - Assert.Same(inner, e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageDetailCtor() - { - // Arrange - const string Message = "this is a test"; - const string Detail = "so is this"; - var e = new AIException(AIException.ErrorCodes.InvalidRequest, Message, Detail); - - // Assert - Assert.Equal(AIException.ErrorCodes.InvalidRequest, e.ErrorCode); - Assert.Contains("Invalid request", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Equal(Detail, e.Detail); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageDetailExceptionCtor() - { - // Arrange - const string Message = "this is a test"; - const string Detail = "so is this"; - var inner = new FormatException(); - var e = new AIException(AIException.ErrorCodes.InvalidRequest, Message, Detail, inner); - - // Assert - Assert.Equal(AIException.ErrorCodes.InvalidRequest, e.ErrorCode); - Assert.Contains("Invalid request", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Equal(Detail, e.Detail); - Assert.Same(inner, e.InnerException); - } - - [Fact] - public void ItAllowsNullMessageAndInnerExceptionInCtors() - { - // Arrange - var e = new AIException(AIException.ErrorCodes.AccessDenied, null, null, null); - - // Assert - Assert.Equal(AIException.ErrorCodes.AccessDenied, e.ErrorCode); - Assert.Contains("Access denied", e.Message, StringComparison.Ordinal); - Assert.Null(e.Detail); - Assert.Null(e.InnerException); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/AI/Embeddings/EmbeddingTests.cs b/dotnet/src/SemanticKernel.UnitTests/AI/Embeddings/EmbeddingTests.cs deleted file mode 100644 index 060e82b6bb67..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/AI/Embeddings/EmbeddingTests.cs +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Text.Json; -using Microsoft.SemanticKernel.AI.Embeddings; -using Xunit; - -namespace SemanticKernel.UnitTests.AI.Embeddings; - -public class EmbeddingTests -{ - // Vector has length of 3, magnitude of 5 - private readonly float[] _vector = new float[] { 0, 3, -4 }; - private readonly float[] _empty = Array.Empty(); - - [Fact] - public void ItTreatsDefaultEmbeddingAsEmpty() - { - // Arrange - Embedding target = default; - - // Assert - Assert.True(target.IsEmpty); - Assert.Equal(0, target.Count); - Assert.Empty(target.Vector); - Assert.Same(Array.Empty(), target.Vector); - Assert.Same(Array.Empty(), (float[])target); - Assert.True(target.AsReadOnlySpan().IsEmpty); - Assert.True(((ReadOnlySpan)target).IsEmpty); - Assert.True(target.Equals(Embedding.Empty)); - Assert.True(target.Equals(new Embedding())); - Assert.True(target == Embedding.Empty); - Assert.True(target == new Embedding()); - Assert.False(target != Embedding.Empty); - Assert.Equal(0, target.GetHashCode()); - } - - [Fact] - public void ItThrowsFromCtorWithUnsupportedType() - { - // Assert - Assert.Throws(() => new Embedding(new int[] { 1, 2, 3 })); - Assert.Throws(() => new Embedding(Array.Empty())); - } - - [Fact] - public void ItThrowsFromEmptyWithUnsupportedType() - { - // Assert - Assert.Throws(() => Embedding.Empty); - } - - [Fact] - public void ItAllowsUnsupportedTypesOnEachOperation() - { - // Arrange - Embedding target = default; - - // Act - Assert.True(target.IsEmpty); - Assert.Equal(0, target.Count); - } - - [Fact] - public void ItThrowsWithNullVector() - { - // Assert - Assert.Throws("vector", () => new Embedding(null!)); - } - - [Fact] - public void ItCreatesEmptyEmbedding() - { - // Arrange - var target = new Embedding(this._empty); - - // Assert - Assert.Empty(target.Vector); - Assert.Equal(0, target.Count); - Assert.False(Embedding.IsSupported()); - } - - [Fact] - public void ItCreatesExpectedEmbedding() - { - // Arrange - var target = new Embedding(this._vector); - - // Assert - Assert.True(target.Vector.SequenceEqual(this._vector)); - } - - [Fact] - public void ItSerializesEmbedding() - { - // Arrange - var target = new Embedding(this._vector); - - // Act - string json = JsonSerializer.Serialize(target); - var copy = JsonSerializer.Deserialize>(json); - - // Assert - Assert.True(copy.Vector.SequenceEqual(this._vector)); - } - - [Fact] - public void ItDoesntCopyVectorWhenCastingToSpan() - { - // Arrange - var target = new Embedding(this._vector); - - // Act - ReadOnlySpan span1 = target.AsReadOnlySpan(); - ReadOnlySpan span2 = (ReadOnlySpan)target; - - // Assert - Assert.False(Unsafe.AreSame(ref MemoryMarshal.GetReference(span1), ref MemoryMarshal.GetArrayDataReference(this._vector))); - Assert.True(Unsafe.AreSame(ref MemoryMarshal.GetReference(span1), ref MemoryMarshal.GetReference(span2))); - } - - [Fact] - public void ItTransfersOwnershipWhenRequested() - { - // Assert - Assert.False(ReferenceEquals(this._vector, new Embedding(this._vector).Vector)); - Assert.False(ReferenceEquals(this._vector, new Embedding(this._vector, transferOwnership: false).Vector)); - Assert.True(ReferenceEquals(this._vector, new Embedding(this._vector, transferOwnership: true).Vector)); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/FunctionCollectionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/FunctionCollectionTests.cs new file mode 100644 index 000000000000..addbedc144e4 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/FunctionCollectionTests.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; +using Moq; +using Xunit; + +namespace SemanticKernel.UnitTests.Functions; + +public class FunctionCollectionTests +{ + [Fact] + public void ItAllowsToReplaceFunctions() + { + // Arrange + var functionOne = new Mock(); + functionOne.SetupGet(x => x.Name).Returns("fName"); + functionOne.SetupGet(x => x.PluginName).Returns("sName"); + functionOne.SetupGet(x => x.Description).Returns("ONE"); + functionOne.SetupGet(x => x.RequestSettings).Returns(new AIRequestSettings()); + + var functionTwo = new Mock(); + functionTwo.SetupGet(x => x.Name).Returns("fName"); + functionTwo.SetupGet(x => x.PluginName).Returns("sName"); + functionTwo.SetupGet(x => x.Description).Returns("TWO"); + functionTwo.SetupGet(x => x.RequestSettings).Returns(new AIRequestSettings()); + + var target = new FunctionCollection(); + + // Act + target.AddFunction(functionOne.Object); + + // Assert + Assert.True(target.TryGetFunction("sName", "fName", out var func)); + Assert.Equal("ONE", func.Description); + + // Act + target.AddFunction(functionTwo.Object); + + // Assert + Assert.True(target.TryGetFunction("sName", "fName", out func)); + Assert.Equal("TWO", func.Description); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/FunctionViewTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/FunctionViewTests.cs new file mode 100644 index 000000000000..16de94bd4e09 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/FunctionViewTests.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.SemanticKernel; +using Xunit; + +namespace SemanticKernel.UnitTests.Functions; + +public class FunctionViewTests +{ + [Fact] + public void ItReturnsFunctionParams() + { + // Arrange + var paramsA = new List + { + new("p1", "param 1", "default 1"), + new("p2", "param 2", "default 2") + }; + + // Act + var funcViewA = new FunctionView("funcA", "s1", "", paramsA); + + // Assert + Assert.NotNull(funcViewA); + + Assert.Equal("p1", funcViewA.Parameters[0].Name); + Assert.Equal("p2", funcViewA.Parameters[1].Name); + Assert.Equal("param 1", funcViewA.Parameters[0].Description); + Assert.Equal("param 2", funcViewA.Parameters[1].Description); + Assert.Equal("default 1", funcViewA.Parameters[0].DefaultValue); + Assert.Equal("default 2", funcViewA.Parameters[1].DefaultValue); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/ParameterViewTypeTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/ParameterViewTypeTests.cs new file mode 100644 index 000000000000..497cf97b63f2 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/ParameterViewTypeTests.cs @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel; +using Xunit; + +namespace SemanticKernel.UnitTests.Functions; + +public class ParameterViewTypeTests +{ + [Theory] + [InlineData("string")] + [InlineData("number")] + [InlineData("object")] + [InlineData("array")] + [InlineData("boolean")] + public void ItCanConvertParameterDataTypeToString(string name) + { + //Arrange + var sut = new ParameterViewType(name); + + //Act + var result = sut.ToString(); + + //Assert + Assert.Equal(name, result); + } + + [Fact] + public void ItCanCreateStringParameterDataType() + { + //Act + var sut = ParameterViewType.String; + + //Assert + Assert.Equal("string", sut.Name); + } + + [Fact] + public void ItCanCreateNumberParameterDataType() + { + //Act + var sut = ParameterViewType.Number; + + //Assert + Assert.Equal("number", sut.Name); + } + + [Fact] + public void ItCanCreateObjectParameterDataType() + { + //Act + var sut = ParameterViewType.Object; + + //Assert + Assert.Equal("object", sut.Name); + } + + [Fact] + public void ItCanArrayParameterDataType() + { + //Act + var sut = ParameterViewType.Array; + + //Assert + Assert.Equal("array", sut.Name); + } + + [Fact] + public void ItCanCreateBooleanParameterDataType() + { + //Act + var sut = ParameterViewType.Boolean; + + //Assert + Assert.Equal("boolean", sut.Name); + } + + [Fact] + public void ItCanCheckTwoParameterDataTypesAreEqual() + { + //Arrange + var sut1 = new ParameterViewType("array"); + var sut2 = new ParameterViewType("array"); + + //Assert + Assert.True(sut1.Equals(sut2)); + } + + [Fact] + public void ItCanCheckTwoParameterDataTypesAreUnequal() + { + //Arrange + var sut1 = new ParameterViewType("array"); + var sut2 = new ParameterViewType("string"); + + //Assert + Assert.False(sut1.Equals(sut2)); + } + + [Fact] + public void ItCanCheckParameterDataTypeIsEqualToAnotherOneRepresentedByObject() + { + //Arrange + var sut1 = new ParameterViewType("array"); + object sut2 = new ParameterViewType("array"); + + //Assert + Assert.True(sut1.Equals(sut2)); + } + + [Fact] + public void ItCanCheckParameterDataTypeIsUnequalToAnotherOneRepresentedByObject() + { + //Arrange + var sut1 = new ParameterViewType("array"); + object sut2 = new ParameterViewType("string"); + + //Assert + Assert.False(sut1.Equals(sut2)); + } + + [Fact] + public void ItCanCheckParameterDataTypeIsUnequalToAnotherType() + { + //Arrange + var sut1 = new ParameterViewType("array"); + var sut2 = "array"; + + //Assert + Assert.False(sut1.Equals(sut2)); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/SKContextTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/SKContextTests.cs new file mode 100644 index 000000000000..2686395430a1 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/SKContextTests.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Orchestration; +using Moq; +using Xunit; + +namespace SemanticKernel.UnitTests.Functions; + +public class SKContextTests +{ + private readonly Mock _functions = new(); + + [Fact] + public void ItHasHelpersForContextVariables() + { + // Arrange + var variables = new ContextVariables(); + var target = new SKContext(new Mock().Object, variables); + variables.Set("foo1", "bar1"); + + // Act + target.Variables["foo2"] = "bar2"; + target.Variables["INPUT"] = Guid.NewGuid().ToString("N"); + + // Assert + Assert.Equal("bar1", target.Variables["foo1"]); + Assert.Equal("bar1", target.Variables["foo1"]); + Assert.Equal("bar2", target.Variables["foo2"]); + Assert.Equal("bar2", target.Variables["foo2"]); + Assert.Equal(target.Variables["INPUT"], target.Result); + Assert.Equal(target.Variables["INPUT"], target.ToString()); + Assert.Equal(target.Variables["INPUT"], target.Variables.Input); + Assert.Equal(target.Variables["INPUT"], target.Variables.ToString()); + } + + [Fact] + public async Task ItHasHelpersForFunctionCollectionAsync() + { + // Arrange + IDictionary functions = KernelBuilder.Create().ImportFunctions(new Parrot(), "test"); + this._functions.Setup(x => x.GetFunction("func")).Returns(functions["say"]); + var (kernel, functionRunner) = this.SetupKernelMock(this._functions.Object); + var target = new SKContext(functionRunner.Object, new ContextVariables(), this._functions.Object); + Assert.NotNull(target.Functions); + + // Act + var say = target.Functions.GetFunction("func"); + + FunctionResult result = await say.InvokeAsync("ciao", kernel.Object); + + // Assert + Assert.Equal("ciao", result.Context.Result); + Assert.Equal("ciao", result.GetValue()); + } + + private (Mock kernelMock, Mock functionRunnerMock) SetupKernelMock(IReadOnlyFunctionCollection? functions = null) + { + functions ??= new Mock().Object; + + var kernel = new Mock(); + var functionRunner = new Mock(); + + kernel.SetupGet(x => x.Functions).Returns(functions); + kernel.Setup(k => k.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((contextVariables, skills, loggerFactory, culture) => + { + return new SKContext(functionRunner.Object, contextVariables); + }); + + return (kernel, functionRunner); + } + + private sealed class Parrot + { + [SKFunction, Description("say something")] + // ReSharper disable once UnusedMember.Local + public string Say(string input) + { + return input; + } + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/SKFunctionTests2.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/SKFunctionTests2.cs new file mode 100644 index 000000000000..8a128a37a551 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/SKFunctionTests2.cs @@ -0,0 +1,1053 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Orchestration; +using Moq; +using Xunit; + +namespace SemanticKernel.UnitTests.Functions; + +public sealed class SKFunctionTests2 +{ + private readonly Mock _logger; + private readonly Mock _functions; + private readonly Mock _kernel; + + private static string s_expected = string.Empty; + private static string s_actual = string.Empty; + + public SKFunctionTests2() + { + this._logger = new Mock(); + this._functions = new Mock(); + this._kernel = new Mock(); + + s_expected = Guid.NewGuid().ToString("D"); + } + + [Fact] + public async Task ItSupportsStaticVoidVoidAsync() + { + // Arrange + static void Test() + { + s_actual = s_expected; + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + } + + [Fact] + public async Task ItSupportsStaticVoidStringAsync() + { + // Arrange + static string Test() + { + s_actual = s_expected; + return s_expected; + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Result); + Assert.Equal(s_expected, result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticVoidTaskStringAsync() + { + // Arrange + static Task Test() + { + s_actual = s_expected; + return Task.FromResult(s_expected); + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Result); + Assert.Equal(s_expected, result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticVoidValueTaskStringAsync() + { + // Arrange + static async ValueTask Test() + { + s_actual = s_expected; + await Task.Delay(1); + return s_expected; + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Result); + Assert.Equal(s_expected, result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticContextVoidAsync() + { + // Arrange + static void Test(SKContext context) + { + s_actual = s_expected; + context.Variables["canary"] = s_expected; + } + + var context = this.MockContext("xy"); + context.Variables["someVar"] = "qz"; + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Variables["canary"]); + } + + [Fact] + public async Task ItSupportsStaticContextStringAsync() + { + // Arrange + static string Test(SKContext context) + { + s_actual = context.Variables["someVar"]; + return "abc"; + } + + var context = this.MockContext(""); + context.Variables["someVar"] = s_expected; + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal("abc", context.Result); + Assert.Equal("abc", result.GetValue()); + } + + [Fact] + public async Task ItSupportsInstanceContextStringNullableAsync() + { + // Arrange + int invocationCount = 0; + + string? Test(SKContext context) + { + invocationCount++; + s_actual = context.Variables["someVar"]; + return "abc"; + } + + var context = this.MockContext(""); + context.Variables["someVar"] = s_expected; + + // Act + Func method = Test; + var function = SKFunction.FromNativeMethod(Method(method), method.Target, loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Equal(s_expected, s_actual); + Assert.Equal("abc", context.Result); + Assert.Equal("abc", result.GetValue()); + } + + [Fact] + public async Task ItSupportsInstanceContextTaskStringAsync() + { + // Arrange + int invocationCount = 0; + + Task Test(SKContext context) + { + invocationCount++; + s_actual = s_expected; + context.Variables["canary"] = s_expected; + return Task.FromResult(s_expected); + } + + var context = this.MockContext(""); + + // Act + Func> method = Test; + var function = SKFunction.FromNativeMethod(Method(method), method.Target, loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_actual, context.Result); + Assert.Equal(s_actual, result.GetValue()); + Assert.Equal(s_expected, context.Variables["canary"]); + } + + [Fact] + public async Task ItSupportsInstanceContextTaskContextAsync() + { + // Arrange + int invocationCount = 0; + + async Task TestAsync(SKContext context) + { + await Task.Delay(0); + invocationCount++; + s_actual = s_expected; + context.Variables.Update("foo"); + context.Variables["canary"] = s_expected; + return context; + } + + var context = this.MockContext(""); + + // Act + Func> method = TestAsync; + var function = SKFunction.FromNativeMethod(Method(method), method.Target, loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Variables["canary"]); + Assert.Equal("foo", context.Result); + Assert.Equal("foo", result.GetValue()); + } + + [Fact] + public async Task ItSupportsInstanceStringVoidAsync() + { + // Arrange + int invocationCount = 0; + + void Test(string input) + { + invocationCount++; + s_actual = s_expected + input; + } + + var context = this.MockContext(".blah"); + + // Act + Action method = Test; + var function = SKFunction.FromNativeMethod(Method(method), method.Target, loggerFactory: this._logger.Object); + Assert.NotNull(function); + + await function.InvokeAsync(context); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Equal(s_expected + ".blah", s_actual); + } + + [Fact] + public async Task ItSupportsInstanceStringStringAsync() + { + // Arrange + int invocationCount = 0; + + string Test(string input) + { + invocationCount++; + s_actual = s_expected; + return "foo-bar"; + } + + var context = this.MockContext(""); + + // Act + Func method = Test; + var function = SKFunction.FromNativeMethod(Method(method), method.Target, loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Equal(s_expected, s_actual); + Assert.Equal("foo-bar", context.Result); + Assert.Equal("foo-bar", result.GetValue()); + } + + [Fact] + public async Task ItSupportsInstanceStringTaskStringAsync() + { + // Arrange + int invocationCount = 0; + + Task Test(string input) + { + invocationCount++; + s_actual = s_expected; + return Task.FromResult("hello there"); + } + + var context = this.MockContext(""); + + // Act + Func> method = Test; + var function = SKFunction.FromNativeMethod(Method(method), method.Target, loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Equal(s_expected, s_actual); + Assert.Equal("hello there", context.Result); + Assert.Equal("hello there", result.GetValue()); + } + + [Fact] + public async Task ItSupportsInstanceStringContextVoidAsync() + { + // Arrange + int invocationCount = 0; + + void Test(string input, SKContext context) + { + invocationCount++; + s_actual = s_expected; + context.Variables.Update("x y z"); + context.Variables["canary"] = s_expected; + } + + var context = this.MockContext(""); + + // Act + Action method = Test; + var function = SKFunction.FromNativeMethod(Method(method), method.Target, loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Variables["canary"]); + Assert.Equal("x y z", context.Result); + Assert.Null(result.GetValue()); + } + + [Fact] + public async Task ItSupportsInstanceContextStringVoidAsync() + { + // Arrange + int invocationCount = 0; + + void Test(SKContext context, string input) + { + invocationCount++; + s_actual = s_expected; + context.Variables.Update("x y z"); + context.Variables["canary"] = s_expected; + } + + var context = this.MockContext(""); + + // Act + Action method = Test; + var function = SKFunction.FromNativeMethod(Method(method), method.Target, loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Variables["canary"]); + Assert.Equal("x y z", context.Result); + Assert.Null(result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticStringContextStringAsync() + { + // Arrange + static string Test(string input, SKContext context) + { + s_actual = s_expected; + context.Variables["canary"] = s_expected; + context.Variables.Update("x y z"); + // This value should overwrite "x y z" + return "new data"; + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Variables["canary"]); + Assert.Equal("new data", context.Result); + Assert.Equal("new data", result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticStringContextTaskStringAsync() + { + // Arrange + static Task Test(string input, SKContext context) + { + s_actual = s_expected; + context.Variables["canary"] = s_expected; + context.Variables.Update("x y z"); + // This value should overwrite "x y z" + return Task.FromResult("new data"); + } + + var context = this.MockContext(""); + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Variables["canary"]); + Assert.Equal("new data", context.Result); + Assert.Equal("new data", result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticStringContextTaskContextAsync() + { + // Arrange + static Task Test(string input, SKContext context) + { + s_actual = s_expected; + context.Variables["canary"] = s_expected; + context.Variables.Update("x y z"); + + var newContext = context.Clone(); + newContext.Variables.Clear(); + + // This value should overwrite "x y z". Contexts are merged. + newContext.Variables.Update("new data"); + newContext.Variables["canary2"] = "222"; + + return Task.FromResult(newContext); + } + + var oldContext = this.MockContext(""); + oldContext.Variables["legacy"] = "something"; + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(oldContext); + var newContext = result.Context; + + // Assert + Assert.Equal(s_expected, s_actual); + + Assert.True(oldContext.Variables.ContainsKey("canary")); + Assert.False(oldContext.Variables.ContainsKey("canary2")); + + Assert.False(newContext.Variables.ContainsKey("canary")); + Assert.True(newContext.Variables.ContainsKey("canary2")); + + Assert.Equal(s_expected, oldContext.Variables["canary"]); + Assert.Equal("222", newContext.Variables["canary2"]); + + Assert.True(oldContext.Variables.ContainsKey("legacy")); + Assert.False(newContext.Variables.ContainsKey("legacy")); + + Assert.Equal("x y z", oldContext.Result); + Assert.Equal("new data", newContext.Result); + + Assert.Equal("new data", result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticContextValueTaskContextAsync() + { + // Arrange + static ValueTask Test(string input, SKContext context) + { + // This value should overwrite "x y z". Contexts are merged. + var newCx = context.Clone(); + newCx.Variables.Update(input + "abc"); + + return new ValueTask(newCx); + } + + var oldContext = this.MockContext("test"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(oldContext); + + // Assert + Assert.Equal("testabc", result.Context.Variables.Input); + } + + [Fact] + public async Task ItSupportsStaticStringTaskAsync() + { + // Arrange + static Task TestAsync(string input) + { + s_actual = s_expected; + return Task.CompletedTask; + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(TestAsync), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + } + + [Fact] + public async Task ItSupportsStaticStringValueTaskAsync() + { + // Arrange + static ValueTask TestAsync(string input) + { + s_actual = s_expected; + return default; + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(TestAsync), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + } + + [Fact] + public async Task ItSupportsStaticContextTaskAsync() + { + // Arrange + static Task TestAsync(SKContext context) + { + s_actual = s_expected; + context.Variables["canary"] = s_expected; + context.Variables.Update("x y z"); + return Task.CompletedTask; + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(TestAsync), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Variables["canary"]); + Assert.Equal("x y z", context.Result); + Assert.Null(result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticStringContextTaskAsync() + { + // Arrange + static Task TestAsync(string input, SKContext context) + { + s_actual = s_expected; + context.Variables["canary"] = s_expected; + context.Variables.Update(input + "x y z"); + return Task.CompletedTask; + } + + var context = this.MockContext("input:"); + + // Act + var function = SKFunction.FromNativeMethod(Method(TestAsync), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + var result = await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + Assert.Equal(s_expected, context.Variables["canary"]); + Assert.Equal("input:x y z", context.Result); + Assert.Null(result.GetValue()); + } + + [Fact] + public async Task ItSupportsStaticVoidTaskAsync() + { + // Arrange + static Task TestAsync() + { + s_actual = s_expected; + return Task.CompletedTask; + } + + var context = this.MockContext(""); + + // Act + var function = SKFunction.FromNativeMethod(Method(TestAsync), loggerFactory: this._logger.Object); + Assert.NotNull(function); + + await function.InvokeAsync(context); + + // Assert + Assert.Equal(s_expected, s_actual); + } + + [Fact] + public async Task ItSupportsUsingNamedInputValueFromContextAsync() + { + static string Test(string input) => "Result: " + input; + + var context = this.MockContext("input value"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal("Result: input value", result.Context.Variables.Input); + } + + [Fact] + public async Task ItSupportsUsingNonNamedInputValueFromContextAsync() + { + static string Test(string other) => "Result: " + other; + + var context = this.MockContext("input value"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal("Result: input value", result.Context.Variables.Input); + } + + [Fact] + public async Task ItSupportsUsingNonNamedInputValueFromContextEvenWhenThereAreMultipleParametersAsync() + { + static string Test(int something, long orother) => "Result: " + (something + orother); + + var context = this.MockContext("42"); + context.Variables.Set("orother", "8"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal("Result: 50", result.Context.Variables.Input); + } + + [Fact] + public async Task ItSupportsPreferringNamedValueOverInputFromContextAsync() + { + static string Test(string other) => "Result: " + other; + + var context = this.MockContext("input value"); + context.Variables.Set("other", "other value"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal("Result: other value", result.Context.Variables.Input); + } + + [Fact] + public async Task ItSupportsOverridingNameWithAttributeAsync() + { + static string Test([SKName("input"), Description("description")] string other) => "Result: " + other; + + var context = this.MockContext("input value"); + context.Variables.Set("other", "other value"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal("Result: input value", result.Context.Variables.Input); + } + + [Fact] + public async Task ItSupportNullDefaultValuesOverInputAsync() + { + static string Test(string? input = null, string? other = null) => "Result: " + (other is null); + + var context = this.MockContext("input value"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal("Result: True", result.Context.Variables.Input); + } + + [Fact] + public async Task ItSupportsConvertingFromManyTypesAsync() + { + static string Test(int a, long b, decimal c, Guid d, DateTimeOffset e, DayOfWeek? f) => + $"{a} {b} {c} {d} {e:R} {f}"; + + var context = this.MockContext(""); + context.Variables.Set("a", "1"); + context.Variables.Set("b", "-2"); + context.Variables.Set("c", "1234"); + context.Variables.Set("d", "7e08cc00-1d71-4558-81ed-69929499dea1"); + context.Variables.Set("e", "Thu, 25 May 2023 20:17:30 GMT"); + context.Variables.Set("f", "Monday"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal("1 -2 1234 7e08cc00-1d71-4558-81ed-69929499dea1 Thu, 25 May 2023 20:17:30 GMT Monday", result.Context.Variables.Input); + } + + [Fact] + public async Task ItSupportsConvertingFromTypeConverterAttributedTypesAsync() + { + static int Test(MyCustomType mct) => mct.Value * 2; + + var context = this.MockContext(""); + context.Variables.Set("mct", "42"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + FunctionResult result = await function.InvokeAsync(context); + + // Assert + Assert.Equal("84", result.Context.Variables.Input); + } + + [TypeConverter(typeof(MyCustomTypeConverter))] + private sealed class MyCustomType + { + public int Value { get; set; } + } + +#pragma warning disable CA1812 // Instantiated by reflection + private sealed class MyCustomTypeConverter : TypeConverter + { + public override bool CanConvertFrom(ITypeDescriptorContext? context, Type sourceType) => + sourceType == typeof(string); + public override object? ConvertFrom(ITypeDescriptorContext? context, CultureInfo? culture, object value) => + new MyCustomType { Value = int.Parse((string)value, culture) }; + } +#pragma warning restore CA1812 + + [Fact] + public async Task ItSupportsConvertingFromToManyTypesAsync() + { + // Arrange + var context = this.MockContext("1"); + + static async Task AssertResult(Delegate d, SKContext context, string expected) + { + var result = await SKFunction.FromNativeFunction(d, functionName: "Test")!.InvokeAsync(context); + context = result.Context; + + Assert.Equal(expected, context.Variables.Input); + } + + // Act/Assert + await AssertResult((sbyte input) => input * 2, context, "2"); + await AssertResult((byte input) => input * 2, context, "4"); + await AssertResult((short input) => input * 2, context, "8"); + await AssertResult((ushort input) => input * 2, context, "16"); + await AssertResult((int input) => input * 2, context, "32"); + await AssertResult((uint input) => input * 2, context, "64"); + await AssertResult((long input) => input * 2, context, "128"); + await AssertResult((ulong input) => input * 2, context, "256"); + await AssertResult((float input) => input * 2, context, "512"); + await AssertResult((double input) => input * 2, context, "1024"); + await AssertResult((int input) => Task.FromResult(input * 2), context, "2048"); + await AssertResult((long input) => Task.FromResult(input * 2), context, "4096"); + await AssertResult((int input) => ValueTask.FromResult(input * 2), context, "8192"); + await AssertResult((long input) => ValueTask.FromResult(input * 2), context, "16384"); + await AssertResult((long? input) => input!.Value * 2, context, "32768"); + await AssertResult((TimeSpan input) => input * 2, context, "65536.00:00:00"); + await AssertResult((TimeSpan? input) => (int?)null, context, ""); + + context.Variables.Update("http://example.com/semantic"); + await AssertResult((Uri input) => new Uri(input, "kernel"), context, "http://example.com/kernel"); + } + + [Fact] + public async Task ItUsesContextCultureForParsingFormattingAsync() + { + // Arrange + var context = this.MockContext(""); + ISKFunction func = SKFunction.FromNativeFunction((double input) => input * 2, functionName: "Test"); + FunctionResult result; + + // Act/Assert + + context.Culture = new CultureInfo("fr-FR"); + context.Variables.Update("12,34"); // tries first to parse with the specified culture + result = await func.InvokeAsync(context); + Assert.Equal("24,68", result.Context.Variables.Input); + + context.Culture = new CultureInfo("fr-FR"); + context.Variables.Update("12.34"); // falls back to invariant culture + result = await func.InvokeAsync(context); + Assert.Equal("24,68", result.Context.Variables.Input); + + context.Culture = new CultureInfo("en-US"); + context.Variables.Update("12.34"); // works with current culture + result = await func.InvokeAsync(context); + Assert.Equal("24.68", result.Context.Variables.Input); + + context.Culture = new CultureInfo("en-US"); + context.Variables.Update("12,34"); // not parsable with current or invariant culture + await Assert.ThrowsAsync(() => func.InvokeAsync(context)); + } + + [Fact] + public async Task ItThrowsWhenItFailsToConvertAnArgumentAsync() + { + static string Test(Guid g) => g.ToString(); + + var context = this.MockContext(""); + context.Variables.Set("g", "7e08cc00-1d71-4558-81ed-69929499dxyz"); + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + Assert.NotNull(function); + + var ex = await Assert.ThrowsAsync(() => function.InvokeAsync(context)); + + //Assert + AssertExtensions.AssertIsArgumentOutOfRange(ex, "g", context.Variables["g"]); + } + + [Fact] + public void ItExposesMetadataFromDelegate() + { + [Description("Concat information")] + static string Test(Guid id, string name, [SKName("old")] int age) => $"{id} {name} {age}"; + + // Act + var function = SKFunction.FromNativeFunction(Test); + + // Assert + Assert.Contains("Test", function.Name, StringComparison.Ordinal); + Assert.Equal("Concat information", function.Description); + Assert.Equal("id", function.Describe().Parameters[0].Name); + Assert.Equal("name", function.Describe().Parameters[1].Name); + Assert.Equal("old", function.Describe().Parameters[2].Name); + } + + [Fact] + public void ItExposesMetadataFromMethodInfo() + { + [Description("Concat information")] + static string Test(Guid id, string name, [SKName("old")] int age) => $"{id} {name} {age}"; + + // Act + var function = SKFunction.FromNativeMethod(Method(Test)); + + // Assert + Assert.Contains("Test", function.Name, StringComparison.Ordinal); + Assert.Equal("Concat information", function.Description); + Assert.Equal("id", function.Describe().Parameters[0].Name); + Assert.Equal("name", function.Describe().Parameters[1].Name); + Assert.Equal("old", function.Describe().Parameters[2].Name); + } + + [Fact] + public async Task ItCanReturnBasicTypesAsync() + { + // Arrange + static int TestInt(int number) => number; + static double TestDouble(double number) => number; + static string TestString(string str) => str; + static bool TestBool(bool flag) => flag; + + var function1 = SKFunction.FromNativeMethod(Method(TestInt)); + var function2 = SKFunction.FromNativeMethod(Method(TestDouble)); + var function3 = SKFunction.FromNativeMethod(Method(TestString)); + var function4 = SKFunction.FromNativeMethod(Method(TestBool)); + + // Act + FunctionResult result1 = await function1.InvokeAsync(this.MockContext("42")); + FunctionResult result2 = await function2.InvokeAsync(this.MockContext("3.14")); + FunctionResult result3 = await function3.InvokeAsync(this.MockContext("test-string")); + FunctionResult result4 = await function4.InvokeAsync(this.MockContext("true")); + + // Assert + Assert.Equal(42, result1.GetValue()); + Assert.Equal(3.14, result2.GetValue()); + Assert.Equal("test-string", result3.GetValue()); + Assert.True(result4.GetValue()); + } + + [Fact] + public async Task ItCanReturnComplexTypeAsync() + { + // Arrange + static MyCustomType TestCustomType(MyCustomType instance) => instance; + + var context = this.MockContext(""); + context.Variables.Set("instance", "42"); + + var function = SKFunction.FromNativeMethod(Method(TestCustomType)); + + // Act + FunctionResult result = await function.InvokeAsync(context); + + var actualInstance = result.GetValue(); + + // Assert + Assert.NotNull(actualInstance); + Assert.Equal(42, actualInstance.Value); + } + + [Fact] + public async Task ItCanReturnAsyncEnumerableTypeAsync() + { + // Arrange + static async IAsyncEnumerable TestAsyncEnumerableTypeAsync() + { + yield return 1; + + await Task.Delay(50); + + yield return 2; + + await Task.Delay(50); + + yield return 3; + } + + var function = SKFunction.FromNativeMethod(Method(TestAsyncEnumerableTypeAsync)); + + // Act + FunctionResult result = await function.InvokeAsync(this.MockContext(string.Empty)); + + // Assert + Assert.NotNull(result); + + var asyncEnumerableResult = result.GetValue>(); + + Assert.NotNull(asyncEnumerableResult); + + var assertResult = new List(); + + await foreach (var value in asyncEnumerableResult) + { + assertResult.Add(value); + } + + Assert.True(assertResult.SequenceEqual(new List { 1, 2, 3 })); + } + + private static MethodInfo Method(Delegate method) + { + return method.Method; + } + + private SKContext MockContext(string input) + { + var functionRunner = new Mock(); + + return new SKContext( + functionRunner.Object, + new ContextVariables(input) + ); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests3.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/SKFunctionTests3.cs similarity index 88% rename from dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests3.cs rename to dotnet/src/SemanticKernel.UnitTests/Functions/SKFunctionTests3.cs index 0e8acbbb1d35..89c380123989 100644 --- a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests3.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/SKFunctionTests3.cs @@ -7,11 +7,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; using Xunit; -namespace SemanticKernel.UnitTests.SkillDefinition; +namespace SemanticKernel.UnitTests.Functions; public sealed class SKFunctionTests3 { @@ -19,13 +19,13 @@ public sealed class SKFunctionTests3 public void ItDoesntThrowForValidFunctionsViaDelegate() { // Arrange - var skillInstance = new LocalExampleSkill(); - MethodInfo[] methods = skillInstance.GetType() + var pluginInstance = new LocalExamplePlugin(); + MethodInfo[] methods = pluginInstance.GetType() .GetMethods(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.InvokeMethod) .Where(m => m.Name is not "GetType" and not "Equals" and not "GetHashCode" and not "ToString") .ToArray(); - ISKFunction[] functions = (from method in methods select SKFunction.FromNativeMethod(method, skillInstance, "skill")).ToArray(); + ISKFunction[] functions = (from method in methods select SKFunction.FromNativeMethod(method, pluginInstance, "plugin")).ToArray(); // Act Assert.Equal(methods.Length, functions.Length); @@ -33,16 +33,16 @@ public void ItDoesntThrowForValidFunctionsViaDelegate() } [Fact] - public void ItDoesntThrowForValidFunctionsViaSkill() + public void ItDoesNotThrowForValidFunctionsViaPlugin() { // Arrange - var skillInstance = new LocalExampleSkill(); - MethodInfo[] methods = skillInstance.GetType() + var pluginInstance = new LocalExamplePlugin(); + MethodInfo[] methods = pluginInstance.GetType() .GetMethods(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.InvokeMethod) .Where(m => m.Name is not "GetType" and not "Equals" and not "GetHashCode" and not "ToString") .ToArray(); - ISKFunction[] functions = Kernel.Builder.Build().ImportSkill(skillInstance).Select(s => s.Value).ToArray(); + ISKFunction[] functions = Kernel.Builder.Build().ImportFunctions(pluginInstance).Select(s => s.Value).ToArray(); // Act Assert.Equal(methods.Length, functions.Length); @@ -53,7 +53,7 @@ public void ItDoesntThrowForValidFunctionsViaSkill() public void ItThrowsForInvalidFunctions() { // Arrange - var instance = new InvalidSkill(); + var instance = new InvalidPlugin(); MethodInfo[] methods = instance.GetType() .GetMethods(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.InvokeMethod) .Where(m => m.Name is not "GetType" and not "Equals" and not "GetHashCode") @@ -65,9 +65,9 @@ public void ItThrowsForInvalidFunctions() { try { - SKFunction.FromNativeMethod(method, instance, "skill"); + SKFunction.FromNativeMethod(method, instance, "plugin"); } - catch (KernelException e) when (e.ErrorCode is KernelException.ErrorCodes.FunctionTypeNotSupported or KernelException.ErrorCodes.InvalidFunctionDescription) + catch (SKException) { count++; } @@ -99,14 +99,14 @@ async Task ExecuteAsync(SKContext contextIn) nativeFunction: ExecuteAsync, parameters: null, description: "description", - skillName: "skillName", + pluginName: "pluginName", functionName: "functionName"); - SKContext result = await function.InvokeAsync(context); + FunctionResult result = await function.InvokeAsync(context); // Assert Assert.Equal("YES", context.Variables["canary"]); - Assert.Equal("YES", result.Variables["canary"]); + Assert.Equal("YES", result.Context.Variables["canary"]); } [Fact] @@ -134,16 +134,16 @@ async Task ExecuteAsync(SKContext contextIn) ISKFunction function = SKFunction.FromNativeFunction( nativeFunction: ExecuteAsync, description: "description", - skillName: "skillName", + pluginName: "pluginName", functionName: "functionName"); - SKContext result = await function.InvokeAsync(context); + FunctionResult result = await function.InvokeAsync(context); // Assert - Assert.Equal("YES", result.Variables["canary"]); + Assert.Equal("YES", result.Context.Variables["canary"]); } - private sealed class InvalidSkill + private sealed class InvalidPlugin { [SKFunction] public void Invalid1([SKName("input"), Description("The x parameter")] string x, [SKName("input"), Description("The y parameter")] string y) @@ -168,7 +168,7 @@ public void Invalid4(CancellationToken ct1, CancellationToken ct2) public struct CustomUnknownType { } } - private sealed class LocalExampleSkill + private sealed class LocalExamplePlugin { [SKFunction] public void Type01() diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs new file mode 100644 index 000000000000..b80a68a8f2ab --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs @@ -0,0 +1,440 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.TemplateEngine; +using Moq; +using Xunit; + +// ReSharper disable StringLiteralTypo + +namespace SemanticKernel.UnitTests.Functions; + +public class SemanticFunctionTests +{ + [Fact] + public void ItProvidesAccessToFunctionsViaFunctionCollection() + { + // Arrange + var factory = new Mock>(); + var kernel = Kernel.Builder + .WithDefaultAIService(factory.Object) + .Build(); + + kernel.CreateSemanticFunction(promptTemplate: "Tell me a joke", functionName: "joker", pluginName: "jk", description: "Nice fun"); + + // Act & Assert - 3 functions, var name is not case sensitive + Assert.True(kernel.Functions.TryGetFunction("jk", "joker", out _)); + Assert.True(kernel.Functions.TryGetFunction("JK", "JOKER", out _)); + } + + [Theory] + [InlineData(null, "Assistant is a large language model.")] + [InlineData("My Chat Prompt", "My Chat Prompt")] + public async Task ItUsesChatSystemPromptWhenProvidedAsync(string providedSystemChatPrompt, string expectedSystemChatPrompt) + { + // Arrange + var mockTextCompletion = new Mock(); + var mockCompletionResult = new Mock(); + + mockTextCompletion.Setup(c => c.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new[] { mockCompletionResult.Object }); + mockCompletionResult.Setup(cr => cr.GetCompletionAsync(It.IsAny())).ReturnsAsync("llmResult"); + + var kernel = Kernel.Builder + .WithAIService("x", mockTextCompletion.Object) + .Build(); + + var templateConfig = new PromptTemplateConfig(); + templateConfig.ModelSettings.Add(new OpenAIRequestSettings() + { + ChatSystemPrompt = providedSystemChatPrompt + }); + + var func = kernel.CreateSemanticFunction("template", templateConfig, "functionName", "pluginName"); + + // Act + await kernel.RunAsync(func); + + // Assert + mockTextCompletion.Verify(a => a.GetCompletionsAsync("template", It.Is(c => c.ChatSystemPrompt == expectedSystemChatPrompt), It.IsAny()), Times.Once()); + } + + [Fact] + public void ItAllowsToCreateFunctionsInTheGlobalNamespace() + { + // Arrange + var kernel = Kernel.Builder.Build(); + var templateConfig = new PromptTemplateConfig(); + + // Act + var func = kernel.CreateSemanticFunction("template", templateConfig, "functionName"); + + // Assert + Assert.Equal(FunctionCollection.GlobalFunctionsPluginName, func.PluginName); + } + + [Fact] + public async Task ItUsesDefaultServiceWhenSpecifiedAsync() + { + // Arrange + var mockTextCompletion1 = new Mock(); + var mockTextCompletion2 = new Mock(); + var mockCompletionResult = new Mock(); + + mockTextCompletion1.Setup(c => c.GetCompletionsAsync(It.IsAny(), null, It.IsAny())).ReturnsAsync(new[] { mockCompletionResult.Object }); + mockTextCompletion2.Setup(c => c.GetCompletionsAsync(It.IsAny(), null, It.IsAny())).ReturnsAsync(new[] { mockCompletionResult.Object }); + mockCompletionResult.Setup(cr => cr.GetCompletionAsync(It.IsAny())).ReturnsAsync("llmResult"); + + var kernel = Kernel.Builder + .WithAIService("service1", mockTextCompletion1.Object, false) + .WithAIService("service2", mockTextCompletion2.Object, true) + .Build(); + + var templateConfig = new PromptTemplateConfig(); + var func = kernel.CreateSemanticFunction("template", templateConfig, "functionName", "pluginName"); + + // Act + await kernel.RunAsync(func); + + // Assert + mockTextCompletion1.Verify(a => a.GetCompletionsAsync("template", null, It.IsAny()), Times.Never()); + mockTextCompletion2.Verify(a => a.GetCompletionsAsync("template", null, It.IsAny()), Times.Once()); + } + + [Fact] + public async Task ItUsesServiceIdWhenProvidedAsync() + { + // Arrange + var mockTextCompletion1 = new Mock(); + var mockTextCompletion2 = new Mock(); + var mockCompletionResult = new Mock(); + + mockTextCompletion1.Setup(c => c.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new[] { mockCompletionResult.Object }); + mockTextCompletion2.Setup(c => c.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new[] { mockCompletionResult.Object }); + mockCompletionResult.Setup(cr => cr.GetCompletionAsync(It.IsAny())).ReturnsAsync("llmResult"); + + var kernel = Kernel.Builder + .WithAIService("service1", mockTextCompletion1.Object, false) + .WithAIService("service2", mockTextCompletion2.Object, true) + .Build(); + + var templateConfig = new PromptTemplateConfig(); + templateConfig.ModelSettings.Add(new AIRequestSettings() { ServiceId = "service1" }); + var func = kernel.CreateSemanticFunction("template", templateConfig, "functionName", "pluginName"); + + // Act + await kernel.RunAsync(func); + + // Assert + mockTextCompletion1.Verify(a => a.GetCompletionsAsync("template", It.IsAny(), It.IsAny()), Times.Once()); + mockTextCompletion2.Verify(a => a.GetCompletionsAsync("template", It.IsAny(), It.IsAny()), Times.Never()); + } + + [Fact] + public async Task ItFailsIfInvalidServiceIdIsProvidedAsync() + { + // Arrange + var mockTextCompletion1 = new Mock(); + var mockTextCompletion2 = new Mock(); + + var kernel = Kernel.Builder + .WithAIService("service1", mockTextCompletion1.Object, false) + .WithAIService("service2", mockTextCompletion2.Object, true) + .Build(); + + var templateConfig = new PromptTemplateConfig(); + templateConfig.ModelSettings.Add(new AIRequestSettings() { ServiceId = "service3" }); + var func = kernel.CreateSemanticFunction("template", templateConfig, "functionName", "pluginName"); + + // Act + var exception = await Assert.ThrowsAsync(() => kernel.RunAsync(func)); + + // Assert + Assert.Equal("Service of type Microsoft.SemanticKernel.AI.TextCompletion.ITextCompletion and name service3 not registered.", exception.Message); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public async Task RunAsyncHandlesPreInvocationAsync(int pipelineCount) + { + // Arrange + var sut = Kernel.Builder.Build(); + var semanticFunction = sut.CreateSemanticFunction("Write a simple phrase about UnitTests"); + var (mockTextResult, mockTextCompletion) = this.SetupMocks(); + + semanticFunction.SetAIService(() => mockTextCompletion.Object); + var invoked = 0; + sut.FunctionInvoking += (sender, e) => + { + invoked++; + }; + List functions = new(); + for (int i = 0; i < pipelineCount; i++) + { + functions.Add(semanticFunction); + } + + // Act + var result = await sut.RunAsync(functions.ToArray()); + + // Assert + Assert.Equal(pipelineCount, invoked); + mockTextCompletion.Verify(m => m.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(pipelineCount)); + } + + [Fact] + public async Task RunAsyncHandlesPreInvocationWasCancelledAsync() + { + // Arrange + var sut = Kernel.Builder.Build(); + var semanticFunction = sut.CreateSemanticFunction("Write a simple phrase about UnitTests"); + var input = "Test input"; + var invoked = false; + sut.FunctionInvoking += (sender, e) => + { + invoked = true; + e.Cancel(); + }; + + // Act + var result = await sut.RunAsync(input, semanticFunction); + + // Assert + Assert.True(invoked); + Assert.Null(result.GetValue()); + } + + [Fact] + public async Task RunAsyncHandlesPreInvocationCancelationDontRunSubsequentFunctionsInThePipelineAsync() + { + // Arrange + var sut = Kernel.Builder.Build(); + var (mockTextResult, mockTextCompletion) = this.SetupMocks(); + var semanticFunction = sut.CreateSemanticFunction("Write a simple phrase about UnitTests"); + semanticFunction.SetAIService(() => mockTextCompletion.Object); + + var invoked = 0; + sut.FunctionInvoking += (sender, e) => + { + invoked++; + e.Cancel(); + }; + + // Act + var result = await sut.RunAsync(semanticFunction, semanticFunction); + + // Assert + Assert.Equal(1, invoked); + mockTextCompletion.Verify(m => m.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsyncPreInvocationCancelationDontTriggerInvokedHandlerAsync() + { + // Arrange + var sut = Kernel.Builder.Build(); + var semanticFunction = sut.CreateSemanticFunction("Write a simple phrase about UnitTests"); + var invoked = 0; + + sut.FunctionInvoking += (sender, e) => + { + e.Cancel(); + }; + + sut.FunctionInvoked += (sender, e) => + { + invoked++; + }; + + // Act + var result = await sut.RunAsync(semanticFunction); + + // Assert + Assert.Equal(0, invoked); + } + + [Fact] + public async Task RunAsyncPreInvocationSkipDontTriggerInvokedHandlerAsync() + { + // Arrange + var sut = Kernel.Builder.Build(); + var (mockTextResult, mockTextCompletion) = this.SetupMocks(); + var semanticFunction1 = sut.CreateSemanticFunction("Write one phrase about UnitTests", functionName: "SkipMe"); + var semanticFunction2 = sut.CreateSemanticFunction("Write two phrases about UnitTests", functionName: "DontSkipMe"); + semanticFunction2.SetAIService(() => mockTextCompletion.Object); + var invoked = 0; + var invoking = 0; + string invokedFunction = string.Empty; + + sut.FunctionInvoking += (sender, e) => + { + invoking++; + if (e.FunctionView.Name == "SkipMe") + { + e.Skip(); + } + }; + + sut.FunctionInvoked += (sender, e) => + { + invokedFunction = e.FunctionView.Name; + invoked++; + }; + + // Act + var result = await sut.RunAsync( + semanticFunction1, + semanticFunction2); + + // Assert + Assert.Equal(2, invoking); + Assert.Equal(1, invoked); + Assert.Equal("DontSkipMe", invokedFunction); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public async Task RunAsyncHandlesPostInvocationAsync(int pipelineCount) + { + // Arrange + var sut = Kernel.Builder.Build(); + var semanticFunction = sut.CreateSemanticFunction("Write a simple phrase about UnitTests"); + var (mockTextResult, mockTextCompletion) = this.SetupMocks(); + + semanticFunction.SetAIService(() => mockTextCompletion.Object); + var invoked = 0; + + sut.FunctionInvoked += (sender, e) => + { + invoked++; + }; + + List functions = new(); + for (int i = 0; i < pipelineCount; i++) + { + functions.Add(semanticFunction); + } + + // Act + var result = await sut.RunAsync(functions.ToArray()); + + // Assert + Assert.Equal(pipelineCount, invoked); + mockTextCompletion.Verify(m => m.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(pipelineCount)); + } + + [Fact] + public async Task RunAsyncChangeVariableInvokingHandlerAsync() + { + var sut = Kernel.Builder.Build(); + var prompt = "Write a simple phrase about UnitTests {{$input}}"; + var semanticFunction = sut.CreateSemanticFunction(prompt); + var (mockTextResult, mockTextCompletion) = this.SetupMocks(); + semanticFunction.SetAIService(() => mockTextCompletion.Object); + + var originalInput = "Importance"; + var newInput = "Problems"; + + sut.FunctionInvoking += (sender, e) => + { + originalInput = newInput; + }; + + // Act + await sut.RunAsync(originalInput, semanticFunction); + + // Assert + Assert.Equal(newInput, originalInput); + } + + [Fact] + public async Task RunAsyncChangeVariableInvokedHandlerAsync() + { + var sut = Kernel.Builder.Build(); + var prompt = "Write a simple phrase about UnitTests {{$input}}"; + var semanticFunction = sut.CreateSemanticFunction(prompt); + var (mockTextResult, mockTextCompletion) = this.SetupMocks(); + semanticFunction.SetAIService(() => mockTextCompletion.Object); + + var originalInput = "Importance"; + var newInput = "Problems"; + + sut.FunctionInvoked += (sender, e) => + { + originalInput = newInput; + }; + + // Act + await sut.RunAsync(originalInput, semanticFunction); + + // Assert + Assert.Equal(newInput, originalInput); + } + + [Fact] + public async Task ItReturnsFunctionResultsCorrectlyAsync() + { + // Arrange + [SKName("Function1")] + static string Function1() => "Result1"; + + [SKName("Function2")] + static string Function2() => "Result2"; + + const string PluginName = "MyPlugin"; + const string Prompt = "Write a simple phrase about UnitTests"; + + var kernel = Kernel.Builder.Build(); + + var function1 = SKFunction.FromNativeMethod(Method(Function1), pluginName: PluginName); + var function2 = SKFunction.FromNativeMethod(Method(Function2), pluginName: PluginName); + + var function3 = kernel.CreateSemanticFunction(Prompt, functionName: "Function3", pluginName: PluginName); + var (mockTextResult, mockTextCompletion) = this.SetupMocks("Result3"); + + function3.SetAIService(() => mockTextCompletion.Object); + + // Act + var kernelResult = await kernel.RunAsync(function1, function2, function3); + + // Assert + Assert.NotNull(kernelResult); + Assert.Equal("Result3", kernelResult.GetValue()); + + var functionResult1 = kernelResult.FunctionResults.First(l => l.FunctionName == "Function1" && l.PluginName == PluginName); + var functionResult2 = kernelResult.FunctionResults.First(l => l.FunctionName == "Function2" && l.PluginName == PluginName); + var functionResult3 = kernelResult.FunctionResults.First(l => l.FunctionName == "Function3" && l.PluginName == PluginName); + + Assert.Equal("Result1", functionResult1.GetValue()); + Assert.Equal("Result2", functionResult2.GetValue()); + Assert.Equal("Result3", functionResult3.GetValue()); + } + + private (Mock textResultMock, Mock textCompletionMock) SetupMocks(string? completionResult = null) + { + var mockTextResult = new Mock(); + mockTextResult.Setup(m => m.GetCompletionAsync(It.IsAny())).ReturnsAsync(completionResult ?? "LLM Result about UnitTests"); + + var mockTextCompletion = new Mock(); + mockTextCompletion.Setup(m => m.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new List { mockTextResult.Object }); + + return (mockTextResult, mockTextCompletion); + } + + private static MethodInfo Method(Delegate method) + { + return method.Method; + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/HttpMessageHandlerStub.cs b/dotnet/src/SemanticKernel.UnitTests/HttpMessageHandlerStub.cs new file mode 100644 index 000000000000..e42837c6a602 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/HttpMessageHandlerStub.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Mime; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace SemanticKernel.UnitTests; + +internal sealed class HttpMessageHandlerStub : DelegatingHandler +{ + public HttpRequestHeaders? RequestHeaders { get; private set; } + + public HttpContentHeaders? ContentHeaders { get; private set; } + + public byte[]? RequestContent { get; private set; } + + public Uri? RequestUri { get; private set; } + + public HttpMethod? Method { get; private set; } + + public HttpResponseMessage ResponseToReturn { get; set; } + + public HttpMessageHandlerStub() + { + this.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK); + this.ResponseToReturn.Content = new StringContent("{}", Encoding.UTF8, MediaTypeNames.Application.Json); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + this.Method = request.Method; + this.RequestUri = request.RequestUri; + this.RequestHeaders = request.Headers; + this.RequestContent = request.Content == null ? null : await request.Content.ReadAsByteArrayAsync(cancellationToken); + this.ContentHeaders = request.Content?.Headers; + + return await Task.FromResult(this.ResponseToReturn); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/KernelConfigTests.cs b/dotnet/src/SemanticKernel.UnitTests/KernelConfigTests.cs deleted file mode 100644 index 3c310302e372..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/KernelConfigTests.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Reliability; -using Moq; -using Xunit; - -namespace SemanticKernel.UnitTests; - -/// -/// Unit tests of . -/// -public class KernelConfigTests -{ - private readonly Mock _kernel; - - public KernelConfigTests() - { - var kernelConfig = new KernelConfig(); - this._kernel = new Mock(); - this._kernel.SetupGet(x => x.Logger).Returns(NullLogger.Instance); - this._kernel.SetupGet(x => x.Config).Returns(kernelConfig); - } - - [Fact] - public void HttpRetryHandlerFactoryIsSet() - { - // Arrange - var retry = new NullHttpRetryHandlerFactory(); - var config = new KernelConfig(); - - // Act - config.SetHttpRetryHandlerFactory(retry); - - // Assert - Assert.Equal(retry, config.HttpHandlerFactory); - } - - [Fact] - public void HttpRetryHandlerFactoryIsSetWithCustomImplementation() - { - // Arrange - var retry = new Mock(); - var config = new KernelConfig(); - - // Act - config.SetHttpRetryHandlerFactory(retry.Object); - - // Assert - Assert.Equal(retry.Object, config.HttpHandlerFactory); - } - - [Fact] - public void HttpRetryHandlerFactoryIsSetToDefaultHttpRetryHandlerFactoryIfNull() - { - // Arrange - var config = new KernelConfig(); - - // Act - config.SetHttpRetryHandlerFactory(null); - - // Assert - Assert.IsType(config.HttpHandlerFactory); - } - - [Fact] - public void HttpRetryHandlerFactoryIsSetToDefaultHttpRetryHandlerFactoryIfNotSet() - { - // Arrange - var config = new KernelConfig(); - - // Act - // Assert - Assert.IsType(config.HttpHandlerFactory); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/KernelExceptionTests.cs b/dotnet/src/SemanticKernel.UnitTests/KernelExceptionTests.cs deleted file mode 100644 index 3bb684cda1ec..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/KernelExceptionTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel; -using Xunit; - -namespace SemanticKernel.UnitTests; - -public class KernelExceptionTests -{ - [Fact] - public void ItRoundtripsArgsToErrorCodeCtor() - { - // Arrange - var e = new KernelException(KernelException.ErrorCodes.FunctionNotAvailable); - - // Assert - Assert.Equal(KernelException.ErrorCodes.FunctionNotAvailable, e.ErrorCode); - Assert.Contains("Function not available", e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageCtor() - { - // Arrange - const string Message = "this is a test"; - var e = new KernelException(KernelException.ErrorCodes.FunctionNotAvailable, Message); - - // Assert - Assert.Equal(KernelException.ErrorCodes.FunctionNotAvailable, e.ErrorCode); - Assert.Contains("Function not available", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageExceptionCtor() - { - // Arrange - const string Message = "this is a test"; - var inner = new FormatException(); - var e = new KernelException(KernelException.ErrorCodes.FunctionNotAvailable, Message, inner); - - // Assert - Assert.Equal(KernelException.ErrorCodes.FunctionNotAvailable, e.ErrorCode); - Assert.Contains("Function not available", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Same(inner, e.InnerException); - } - - [Fact] - public void ItAllowsNullMessageAndInnerExceptionInCtors() - { - // Arrange - var e = new KernelException(KernelException.ErrorCodes.FunctionNotAvailable, null, null); - - // Assert - Assert.Equal(KernelException.ErrorCodes.FunctionNotAvailable, e.ErrorCode); - Assert.Contains("Function not available", e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs index cf3662aa3290..dd41db6af420 100644 --- a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs @@ -3,13 +3,16 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Events; using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; using Moq; using Xunit; @@ -20,181 +23,640 @@ namespace SemanticKernel.UnitTests; public class KernelTests { [Fact] - public void ItProvidesAccessToFunctionsViaSkillCollection() + public void ItProvidesAccessToFunctionsViaFunctionCollection() { // Arrange - var factory = new Mock>(); + var factory = new Mock>(); var kernel = Kernel.Builder .WithDefaultAIService(factory.Object) .Build(); - var nativeSkill = new MySkill(); - kernel.CreateSemanticFunction(promptTemplate: "Tell me a joke", functionName: "joker", skillName: "jk", description: "Nice fun"); - kernel.ImportSkill(nativeSkill, "mySk"); + var nativePlugin = new MyPlugin(); + kernel.ImportFunctions(nativePlugin, "mySk"); - // Act - FunctionsView data = kernel.Skills.GetFunctionsView(); - - // Assert - 3 functions, var name is not case sensitive - Assert.True(data.IsSemantic("jk", "joker")); - Assert.True(data.IsSemantic("JK", "JOKER")); - Assert.False(data.IsNative("jk", "joker")); - Assert.False(data.IsNative("JK", "JOKER")); - Assert.True(data.IsNative("mySk", "sayhello")); - Assert.True(data.IsNative("MYSK", "SayHello")); - Assert.True(data.IsNative("mySk", "ReadSkillCollectionAsync")); - Assert.True(data.IsNative("MYSK", "readskillcollectionasync")); - Assert.Single(data.SemanticFunctions["Jk"]); - Assert.Equal(3, data.NativeFunctions["mySk"].Count); + // Act & Assert - 3 functions, var name is not case sensitive + Assert.True(kernel.Functions.TryGetFunction("mySk", "sayhello", out _)); + Assert.True(kernel.Functions.TryGetFunction("MYSK", "SayHello", out _)); + Assert.True(kernel.Functions.TryGetFunction("mySk", "ReadFunctionCollectionAsync", out _)); + Assert.True(kernel.Functions.TryGetFunction("MYSK", "ReadFunctionCollectionAsync", out _)); } [Fact] - public async Task ItProvidesAccessToFunctionsViaSKContextAsync() + public async Task RunAsyncDoesNotRunWhenCancelledAsync() { // Arrange - var factory = new Mock>(); - var kernel = Kernel.Builder - .WithAIService("x", factory.Object) - .Build(); + var kernel = Kernel.Builder.Build(); + var nativePlugin = new MyPlugin(); + var functions = kernel.ImportFunctions(nativePlugin, "mySk"); - var nativeSkill = new MySkill(); - kernel.CreateSemanticFunction("Tell me a joke", functionName: "joker", skillName: "jk", description: "Nice fun"); - var skill = kernel.ImportSkill(nativeSkill, "mySk"); + using CancellationTokenSource cts = new(); + cts.Cancel(); // Act - SKContext result = await kernel.RunAsync(skill["ReadSkillCollectionAsync"]); - - // Assert - 3 functions, var name is not case sensitive - Assert.Equal("Nice fun", result.Variables["jk.joker"]); - Assert.Equal("Nice fun", result.Variables["JK.JOKER"]); - Assert.Equal("Just say hello", result.Variables["mySk.sayhello"]); - Assert.Equal("Just say hello", result.Variables["mySk.SayHello"]); - Assert.Equal("Export info.", result.Variables["mySk.ReadSkillCollectionAsync"]); - Assert.Equal("Export info.", result.Variables["mysk.readskillcollectionasync"]); + await Assert.ThrowsAsync(() => kernel.RunAsync(cts.Token, functions["GetAnyValue"])); } [Fact] - public async Task RunAsyncDoesNotRunWhenCancelledAsync() + public async Task RunAsyncRunsWhenNotCancelledAsync() { // Arrange var kernel = Kernel.Builder.Build(); - var nativeSkill = new MySkill(); - var skill = kernel.ImportSkill(nativeSkill, "mySk"); + var nativePlugin = new MyPlugin(); + kernel.ImportFunctions(nativePlugin, "mySk"); using CancellationTokenSource cts = new(); - cts.Cancel(); // Act - SKContext result = await kernel.RunAsync(cts.Token, skill["GetAnyValue"]); + KernelResult result = await kernel.RunAsync(cts.Token, kernel.Functions.GetFunction("mySk", "GetAnyValue")); // Assert - Assert.True(string.IsNullOrEmpty(result.Result)); - Assert.True(result.ErrorOccurred); - Assert.True(result.LastException is OperationCanceledException); + Assert.False(string.IsNullOrEmpty(result.GetValue())); } [Fact] - public async Task RunAsyncRunsWhenNotCancelledAsync() + public void ItImportsPluginsNotCaseSensitive() + { + // Act + IDictionary functions = Kernel.Builder.Build().ImportFunctions(new MyPlugin(), "test"); + + // Assert + Assert.Equal(3, functions.Count); + Assert.True(functions.ContainsKey("GetAnyValue")); + Assert.True(functions.ContainsKey("getanyvalue")); + Assert.True(functions.ContainsKey("GETANYVALUE")); + } + + [Fact] + public void ItAllowsToImportFunctionsInTheGlobalNamespace() { // Arrange var kernel = Kernel.Builder.Build(); - var nativeSkill = new MySkill(); - kernel.ImportSkill(nativeSkill, "mySk"); - using CancellationTokenSource cts = new(); + // Act + IDictionary functions = kernel.ImportFunctions(new MyPlugin()); + + // Assert + Assert.Equal(3, functions.Count); + Assert.True(kernel.Functions.TryGetFunction("GetAnyValue", out ISKFunction? functionInstance)); + Assert.NotNull(functionInstance); + } + + [Fact] + public void ItAllowsToImportTheSamePluginMultipleTimes() + { + // Arrange + var kernel = Kernel.Builder.Build(); + + // Act - Assert no exception occurs + kernel.ImportFunctions(new MyPlugin()); + kernel.ImportFunctions(new MyPlugin()); + kernel.ImportFunctions(new MyPlugin()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public async Task RunAsyncHandlesPreInvocationAsync(int pipelineCount) + { + // Arrange + var sut = Kernel.Builder.Build(); + var myPlugin = new Mock(); + var functions = sut.ImportFunctions(myPlugin.Object, "MyPlugin"); + + var invoked = 0; + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + invoked++; + }; + List pipeline = new(); + for (int i = 0; i < pipelineCount; i++) + { + pipeline.Add(functions["SayHello"]); + } + + // Act + var result = await sut.RunAsync(pipeline.ToArray()); + + // Assert + Assert.Equal(pipelineCount, invoked); + myPlugin.Verify(m => m.SayHello(), Times.Exactly(pipelineCount)); + } + + [Fact] + public async Task RunAsyncHandlesPreInvocationWasCancelledAsync() + { + // Arrange + var sut = Kernel.Builder.Build(); + var functions = sut.ImportFunctions(new MyPlugin(), "MyPlugin"); + + var invoked = false; + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + invoked = true; + e.Cancel(); + }; + + // Act + var result = await sut.RunAsync(functions["GetAnyValue"]); + + // Assert + Assert.True(invoked); + Assert.Null(result.GetValue()); + } + + [Fact] + public async Task RunAsyncHandlesPreInvocationCancelationDontRunSubsequentFunctionsInThePipelineAsync() + { + // Arrange + var sut = Kernel.Builder.Build(); + var (mockTextResult, mockTextCompletion) = this.SetupMocks(); + var myPlugin = new Mock(); + var functions = sut.ImportFunctions(myPlugin.Object, "MyPlugin"); + + var invoked = 0; + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + invoked++; + e.Cancel(); + }; + + // Act + var result = await sut.RunAsync(functions["GetAnyValue"], functions["SayHello"]); + + // Assert + Assert.Equal(1, invoked); + myPlugin.Verify(m => m.GetAnyValue(), Times.Never); + myPlugin.Verify(m => m.SayHello(), Times.Never); + } + + [Fact] + public async Task RunAsyncPreInvocationCancelationDontTriggerInvokedHandlerAsync() + { + // Arrange + var sut = Kernel.Builder.Build(); + var functions = sut.ImportFunctions(new MyPlugin(), "MyPlugin"); + + var invoked = 0; + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + e.Cancel(); + }; + + sut.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + invoked++; + }; + + // Act + var result = await sut.RunAsync(functions["GetAnyValue"]); + + // Assert + Assert.Equal(0, invoked); + } + + [Fact] + public async Task RunAsyncPreInvocationSkipDontTriggerInvokedHandlerAsync() + { + // Arrange + var sut = Kernel.Builder.Build(); + var (mockTextResult, mockTextCompletion) = this.SetupMocks(); + var myPlugin = new Mock(); + var functions = sut.ImportFunctions(myPlugin.Object, "MyPlugin"); + + var invoked = 0; + var invoking = 0; + string invokedFunction = string.Empty; + + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + invoking++; + if (e.FunctionView.Name == "GetAnyValue") + { + e.Skip(); + } + }; + + sut.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + invokedFunction = e.FunctionView.Name; + invoked++; + }; + + // Act + var result = await sut.RunAsync(functions["GetAnyValue"], functions["SayHello"]); + + // Assert + Assert.Equal(2, invoking); + Assert.Equal(1, invoked); + myPlugin.Verify(m => m.GetAnyValue(), Times.Never); + myPlugin.Verify(m => m.SayHello(), Times.Once); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public async Task RunAsyncHandlesPostInvocationAsync(int pipelineCount) + { + // Arrange + var sut = Kernel.Builder.Build(); + var myPlugin = new Mock(); + var functions = sut.ImportFunctions(myPlugin.Object, "MyPlugin"); + + var invoked = 0; + sut.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + invoked++; + }; + + List pipeline = new(); + for (int i = 0; i < pipelineCount; i++) + { + pipeline.Add(functions["GetAnyValue"]); + } // Act - SKContext result = await kernel.RunAsync(cts.Token, kernel.Func("mySk", "GetAnyValue")); + var result = await sut.RunAsync(pipeline.ToArray()); // Assert - Assert.False(string.IsNullOrEmpty(result.Result)); - Assert.False(result.ErrorOccurred); - Assert.False(result.LastException is OperationCanceledException); + Assert.Equal(pipelineCount, invoked); + myPlugin.Verify(m => m.GetAnyValue(), Times.Exactly(pipelineCount)); } [Fact] - public void ItImportsSkillsNotCaseSensitive() + public async Task RunAsyncChangeVariableInvokingHandlerAsync() { + var sut = Kernel.Builder.Build(); + var myPlugin = new Mock(); + var functions = sut.ImportFunctions(myPlugin.Object, "MyPlugin"); + + var originalInput = "Importance"; + var newInput = "Problems"; + + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + originalInput = newInput; + }; + + // Act + await sut.RunAsync(originalInput, functions["GetAnyValue"]); + + // Assert + Assert.Equal(newInput, originalInput); + } + + [Fact] + public async Task RunAsyncChangeVariableInvokedHandlerAsync() + { + var sut = Kernel.Builder.Build(); + var myPlugin = new Mock(); + var functions = sut.ImportFunctions(myPlugin.Object, "MyPlugin"); + + var originalInput = "Importance"; + var newInput = "Problems"; + + sut.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + originalInput = newInput; + }; + // Act - IDictionary skill = Kernel.Builder.Build().ImportSkill(new MySkill(), "test"); + await sut.RunAsync(originalInput, functions["GetAnyValue"]); // Assert - Assert.Equal(3, skill.Count); - Assert.True(skill.ContainsKey("GetAnyValue")); - Assert.True(skill.ContainsKey("getanyvalue")); - Assert.True(skill.ContainsKey("GETANYVALUE")); + Assert.Equal(newInput, originalInput); } [Fact] - public void ItAllowsToImportSkillsInTheGlobalNamespace() + public async Task ItReturnsFunctionResultsCorrectlyAsync() { // Arrange + [SKName("Function1")] + static string Function1() => "Result1"; + + [SKName("Function2")] + static string Function2() => "Result2"; + + const string PluginName = "MyPlugin"; + var kernel = Kernel.Builder.Build(); + var function1 = SKFunction.FromNativeMethod(Method(Function1), pluginName: PluginName); + var function2 = SKFunction.FromNativeMethod(Method(Function2), pluginName: PluginName); + // Act - IDictionary skill = kernel.ImportSkill(new MySkill()); + var kernelResult = await kernel.RunAsync(function1, function2); + var functionResult1 = kernelResult.FunctionResults.First(l => l.FunctionName == "Function1" && l.PluginName == PluginName); + var functionResult2 = kernelResult.FunctionResults.First(l => l.FunctionName == "Function2" && l.PluginName == PluginName); // Assert - Assert.Equal(3, skill.Count); - Assert.True(kernel.Skills.TryGetFunction("GetAnyValue", out ISKFunction? functionInstance)); - Assert.NotNull(functionInstance); + Assert.NotNull(kernelResult); + Assert.Equal("Result2", kernelResult.GetValue()); + Assert.Equal("Result1", functionResult1.GetValue()); + Assert.Equal("Result2", functionResult2.GetValue()); + } + + [Fact] + public async Task ItReturnsChangedResultsFromFunctionInvokedEventsAsync() + { + var kernel = Kernel.Builder.Build(); + + // Arrange + [SKName("Function1")] + static string Function1() => "Result1"; + var function1 = SKFunction.FromNativeMethod(Method(Function1), pluginName: "MyPlugin"); + const string ExpectedValue = "new result"; + + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs args) => + { + args.SKContext.Variables.Update(ExpectedValue); + }; + + // Act + var kernelResult = await kernel.RunAsync(function1); + + // Assert + Assert.NotNull(kernelResult); + Assert.Equal(ExpectedValue, kernelResult.GetValue()); + Assert.Equal(ExpectedValue, kernelResult.FunctionResults.Single().GetValue()); + Assert.Equal(ExpectedValue, kernelResult.FunctionResults.Single().Context.Result); } [Fact] - public void ItAllowsToImportTheSameSkillMultipleTimes() + public async Task ItReturnsChangedResultsFromFunctionInvokingEventsAsync() { // Arrange var kernel = Kernel.Builder.Build(); - // Act - Assert no exception occurs - kernel.ImportSkill(new MySkill()); - kernel.ImportSkill(new MySkill()); - kernel.ImportSkill(new MySkill()); + [SKName("Function1")] + static string Function1(SKContext context) => context.Variables["injected variable"]; + var function1 = SKFunction.FromNativeMethod(Method(Function1), pluginName: "MyPlugin"); + const string ExpectedValue = "injected value"; + + kernel.FunctionInvoking += (object? sender, FunctionInvokingEventArgs args) => + { + args.SKContext.Variables["injected variable"] = ExpectedValue; + }; + + // Act + var kernelResult = await kernel.RunAsync(function1); + + // Assert + Assert.NotNull(kernelResult); + Assert.Equal(ExpectedValue, kernelResult.GetValue()); + Assert.Equal(ExpectedValue, kernelResult.FunctionResults.Single().GetValue()); + Assert.Equal(ExpectedValue, kernelResult.FunctionResults.Single().Context.Result); + } + + [Theory] + [InlineData("Function1", 5)] + [InlineData("Function2", 1)] + public async Task ItRepeatsFunctionInvokedEventsAsync(string retryFunction, int numberOfRepeats) + { + // Arrange + var kernel = Kernel.Builder.Build(); + + [SKName("Function1")] + static string Function1(SKContext context) => "Result1"; + var function1 = SKFunction.FromNativeMethod(Method(Function1), pluginName: "MyPlugin"); + + [SKName("Function2")] + static string Function2(SKContext context) => "Result2"; + var function2 = SKFunction.FromNativeMethod(Method(Function2), pluginName: "MyPlugin"); + + int numberOfInvocations = 0; + int repeatCount = 0; + + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs args) => + { + if (args.FunctionView.Name == retryFunction && repeatCount < numberOfRepeats) + { + args.Repeat(); + repeatCount++; + } + + numberOfInvocations++; + }; + + // Act + var kernelResult = await kernel.RunAsync(function1, function2); + + // Assert + Assert.NotNull(kernelResult); + Assert.Equal(2 + numberOfRepeats, kernelResult.FunctionResults.Count); + Assert.Equal(2 + numberOfRepeats, numberOfInvocations); + } + + [Theory] + [InlineData("Function1", "Result2 Result3")] + [InlineData("Function2", "Result1 Result3")] + [InlineData("Function3", "Result1 Result2")] + public async Task ItSkipsFunctionsFromFunctionInvokingEventsAsync(string skipFunction, string expectedResult) + { + // Arrange + [SKName("Function1")] + static string Function1(string input) => input + " Result1"; + + [SKName("Function2")] + static string Function2(string input) => input + " Result2"; + + [SKName("Function3")] + static string Function3(string input) => input + " Result3"; + + const string PluginName = "MyPlugin"; + + var kernel = Kernel.Builder.Build(); + + var function1 = SKFunction.FromNativeMethod(Method(Function1), pluginName: PluginName); + var function2 = SKFunction.FromNativeMethod(Method(Function2), pluginName: PluginName); + var function3 = SKFunction.FromNativeMethod(Method(Function3), pluginName: PluginName); + + const int ExpectedInvocations = 2; + + int numberOfInvocations = 0; + + kernel.FunctionInvoking += (object? sender, FunctionInvokingEventArgs args) => + { + if (args.FunctionView.Name == skipFunction) + { + args.Skip(); + } + }; + + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs args) => + { + numberOfInvocations++; + }; + + // Act + var kernelResult = await kernel.RunAsync(string.Empty, function1, function2, function3); + + // Assert + Assert.NotNull(kernelResult); + Assert.Equal(expectedResult, kernelResult.GetValue()!.Trim()); + Assert.Equal(ExpectedInvocations, numberOfInvocations); + Assert.Equal(ExpectedInvocations, kernelResult.FunctionResults.Count); + } + + [Theory] + [InlineData(1, 0, 0)] + [InlineData(2, 0, 0)] + [InlineData(2, 1, 1)] + [InlineData(5, 2, 2)] + public async Task ItCancelsPipelineFromFunctionInvokingEventsAsync(int numberOfFunctions, int functionCancelIndex, int expectedInvocations) + { + List functions = new(); + const string PluginName = "MyPlugin"; + + // Arrange + [SKName("Function1")] + static string Function1() => "Result1"; + functions.Add(SKFunction.FromNativeMethod(Method(Function1), pluginName: PluginName)); + + [SKName("Function2")] + static string Function2() => "Result2"; + functions.Add(SKFunction.FromNativeMethod(Method(Function2), pluginName: PluginName)); + + [SKName("Function3")] + static string Function3() => "Result3"; + functions.Add(SKFunction.FromNativeMethod(Method(Function3), pluginName: PluginName)); + + [SKName("Function4")] + static string Function4() => "Result4"; + functions.Add(SKFunction.FromNativeMethod(Method(Function4), pluginName: PluginName)); + + [SKName("Function5")] + static string Function5() => "Result5"; + functions.Add(SKFunction.FromNativeMethod(Method(Function5), pluginName: PluginName)); + + var kernel = Kernel.Builder.Build(); + + int numberOfInvocations = 0; + + kernel.FunctionInvoking += (object? sender, FunctionInvokingEventArgs args) => + { + if (args.FunctionView.Name == functions[functionCancelIndex].Name) + { + args.Cancel(); + } + }; + + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs args) => + { + numberOfInvocations++; + }; + + // Act + var kernelResult = await kernel.RunAsync(functions.Take(numberOfFunctions).ToArray()); + + // Assert + Assert.NotNull(kernelResult); + + // Kernel result is the same as the last invoked function + Assert.Equal(expectedInvocations, numberOfInvocations); + Assert.Equal(expectedInvocations, kernelResult.FunctionResults.Count); + } + + [Theory] + [InlineData(1, 0, 1)] + [InlineData(2, 0, 1)] + [InlineData(2, 1, 2)] + [InlineData(5, 2, 3)] + public async Task ItCancelsPipelineFromFunctionInvokedEventsAsync(int numberOfFunctions, int functionCancelIndex, int expectedInvocations) + { + List functions = new(); + const string PluginName = "MyPlugin"; + + // Arrange + [SKName("Function1")] + static string Function1() => "Result1"; + functions.Add(SKFunction.FromNativeMethod(Method(Function1), pluginName: PluginName)); + + [SKName("Function2")] + static string Function2() => "Result2"; + functions.Add(SKFunction.FromNativeMethod(Method(Function2), pluginName: PluginName)); + + [SKName("Function3")] + static string Function3() => "Result3"; + functions.Add(SKFunction.FromNativeMethod(Method(Function3), pluginName: PluginName)); + + [SKName("Function4")] + static string Function4() => "Result4"; + functions.Add(SKFunction.FromNativeMethod(Method(Function4), pluginName: PluginName)); + + [SKName("Function5")] + static string Function5() => "Result5"; + functions.Add(SKFunction.FromNativeMethod(Method(Function5), pluginName: PluginName)); + + var kernel = Kernel.Builder.Build(); + + int numberOfInvocations = 0; + + kernel.FunctionInvoked += (object? sender, FunctionInvokedEventArgs args) => + { + numberOfInvocations++; + if (args.FunctionView.Name == functions[functionCancelIndex].Name) + { + args.Cancel(); + } + }; + + // Act + var kernelResult = await kernel.RunAsync(functions.Take(numberOfFunctions).ToArray()); + + // Assert + Assert.NotNull(kernelResult); + + // Kernel result is the same as the last invoked function + Assert.Equal($"Result{functionCancelIndex + 1}", kernelResult.GetValue()); + Assert.Equal(expectedInvocations, numberOfInvocations); } - public class MySkill + public class MyPlugin { [SKFunction, Description("Return any value.")] - public string GetAnyValue() + public virtual string GetAnyValue() { return Guid.NewGuid().ToString(); } [SKFunction, Description("Just say hello")] - public void SayHello() + public virtual void SayHello() { Console.WriteLine("Hello folks!"); } - [SKFunction, Description("Export info."), SKName("ReadSkillCollectionAsync")] - public async Task ReadSkillCollectionAsync(SKContext context) + [SKFunction, Description("Export info."), SKName("ReadFunctionCollectionAsync")] + public async Task ReadFunctionCollectionAsync(SKContext context) { await Task.Delay(0); - if (context.Skills == null) - { - Assert.Fail("Skills collection is missing"); - } - - FunctionsView procMem = context.Skills.GetFunctionsView(); - - foreach (KeyValuePair> list in procMem.SemanticFunctions) + if (context.Functions == null) { - foreach (FunctionView f in list.Value) - { - context.Variables[$"{list.Key}.{f.Name}"] = f.Description; - } + Assert.Fail("Functions collection is missing"); } - foreach (KeyValuePair> list in procMem.NativeFunctions) + foreach (var function in context.Functions.GetFunctionViews()) { - foreach (FunctionView f in list.Value) - { - context.Variables[$"{list.Key}.{f.Name}"] = f.Description; - } + context.Variables[$"{function.PluginName}.{function.Name}"] = function.Description; } return context; } } + + private (Mock textResultMock, Mock textCompletionMock) SetupMocks(string? completionResult = null) + { + var mockTextResult = new Mock(); + mockTextResult.Setup(m => m.GetCompletionAsync(It.IsAny())).ReturnsAsync(completionResult ?? "LLM Result about UnitTests"); + + var mockTextCompletion = new Mock(); + mockTextCompletion.Setup(m => m.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new List { mockTextResult.Object }); + + return (mockTextResult, mockTextCompletion); + } + + private static MethodInfo Method(Delegate method) + { + return method.Method; + } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Memory/Collections/MinHeapTests.cs b/dotnet/src/SemanticKernel.UnitTests/Memory/Collections/MinHeapTests.cs deleted file mode 100644 index 50e25fbc3e37..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Memory/Collections/MinHeapTests.cs +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.SemanticKernel.Memory.Collections; -using Xunit; - -namespace SemanticKernel.UnitTests.Memory.Collections; - -/// -/// Contains tests for the class. -/// -public class MinHeapTests -{ - private const int MinValue = 1; - - [Fact] - public void ItThrowsExceptionWhenCapacityIsInvalid() - { - // Arrange - const int InvalidCapacity = -1; - - void Action() - { - var minHeap = new MinHeap(MinValue, InvalidCapacity); - } - - // Act - var exception = Assert.Throws("capacity", Action); - - // Assert - Assert.Equal(-1, exception.ActualValue); - } - - [Fact] - public void ItAddsItemsInCorrectOrder() - { - // Arrange - const int ExpectedTopItem = 1; - const int ExpectedCount = 3; - - // Act - var minHeap = new MinHeap(MinValue) { 3, 1, 2 }; - - // Assert - Assert.Equal(ExpectedTopItem, minHeap.Top); - Assert.Equal(ExpectedCount, minHeap.Count); - } - - [Fact] - public void ItErasesItemsCorrectly() - { - // Arrange - const int ExpectedCount = 0; - var minHeap = new MinHeap(MinValue) { 3, 1, 2 }; - - // Act - minHeap.Erase(); - - // Assert - Assert.Equal(ExpectedCount, minHeap.Count); - } - - [Fact] - public void ItReturnsItemsOnBufferDetaching() - { - // Arrange - const int ExpectedHeapCount = 0; - - var minHeap = new MinHeap(MinValue) { 3, 1, 2 }; - - // Act - var items = minHeap.DetachBuffer(); - - // Assert - Assert.True(items.Length > 0); - Assert.Equal(ExpectedHeapCount, minHeap.Count); - } - - [Fact] - public void ItThrowsExceptionOnAddingItemsAtInvalidIndex() - { - // Arrange - const int StartIndex = 4; - - var items = new List { 3, 1, 2 }; - var minHeap = new MinHeap(MinValue); - - var action = () => { minHeap.Add(items, StartIndex); }; - - // Act - var exception = Assert.Throws("startAt", () => action()); - - // Assert - Assert.Equal(StartIndex, exception.ActualValue); - } - - [Fact] - public void ItRemovesTopItemCorrectly() - { - // Arrange - const int ExpectedTopItem = 2; - const int ExpectedHeapCount = 2; - - var minHeap = new MinHeap(MinValue) { 3, 1, 2 }; - - // Act - minHeap.RemoveTop(); - - // Assert - Assert.Equal(ExpectedTopItem, minHeap.Top); - Assert.Equal(ExpectedHeapCount, minHeap.Count); - } - - [Fact] - public void ItRemovesAllItemsCorrectly() - { - // Arrange - const int ExpectedHeapCount = 0; - - var minHeap = new MinHeap(MinValue) { 3, 1, 2 }; - - // Act - var items = minHeap.RemoveAll().ToList(); - - // Assert - Assert.Equal(1, items[0]); - Assert.Equal(2, items[1]); - Assert.Equal(3, items[2]); - - Assert.Equal(ExpectedHeapCount, minHeap.Count); - } - - [Fact] - public void ItEnsuresCapacityToExpectedValue() - { - // Arrange - const int ExpectedCapacity = 16; - - var minHeap = new MinHeap(MinValue); - - // Act - minHeap.EnsureCapacity(ExpectedCapacity); - - // Assert - Assert.Equal(ExpectedCapacity, minHeap.Capacity); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Memory/Collections/TopNCollectionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Memory/Collections/TopNCollectionTests.cs deleted file mode 100644 index 60f27ac1d288..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Memory/Collections/TopNCollectionTests.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel.Memory.Collections; -using Xunit; - -namespace SemanticKernel.UnitTests.Memory.Collections; - -/// -/// Contains tests for the class. -/// -public class TopNCollectionTests -{ - private const int MaxItemsCount = 5; - - [Fact] - public void ItResetsCollectionCorrectly() - { - // Arrange - const int ExpectedItemsCount = 0; - - var topNCollection = this.GetTestCollection(MaxItemsCount); - - // Act - topNCollection.Reset(); - - // Assert - Assert.Equal(ExpectedItemsCount, topNCollection.Count); - } - - [Fact] - public void ItKeepsMaxItemsCountWhenMoreItemsWereAdded() - { - // Arrange - const int ExpectedCollectionCount = 5; - - // Act - var topNCollection = this.GetTestCollection(ExpectedCollectionCount); - - // Assert - Assert.Equal(ExpectedCollectionCount, topNCollection.Count); - } - - [Fact] - public void ItSortsCollectionByScoreInDescendingOrder() - { - // Arrange - var topNCollection = this.GetTestCollection(MaxItemsCount); - - // Act - topNCollection.SortByScore(); - - // Assert - for (var i = 0; i < topNCollection.Count - 1; i++) - { - Assert.True(topNCollection[i].Score > topNCollection[i + 1].Score); - } - } - - private TopNCollection GetTestCollection(int maxItemsCount) - { - return new TopNCollection(maxItemsCount) - { - new ScoredValue(1, 0.5), - new ScoredValue(2, 0.6), - new ScoredValue(3, 0.4), - new ScoredValue(4, 0.2), - new ScoredValue(5, 0.9), - new ScoredValue(6, 0.1), - }; - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Memory/MemoryExceptionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Memory/MemoryExceptionTests.cs deleted file mode 100644 index f59fffe24f6f..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Memory/MemoryExceptionTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Memory; -using Xunit; - -namespace SemanticKernel.UnitTests.Memory; - -public class MemoryExceptionTests -{ - [Fact] - public void ItRoundtripsArgsToErrorCodeCtor() - { - // Arrange - var e = new MemoryException(MemoryException.ErrorCodes.FailedToCreateCollection); - - // Assert - Assert.Equal(MemoryException.ErrorCodes.FailedToCreateCollection, e.ErrorCode); - Assert.Contains("Failed to create collection", e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageCtor() - { - // Arrange - const string Message = "this is a test"; - var e = new MemoryException(MemoryException.ErrorCodes.FailedToCreateCollection, Message); - - // Assert - Assert.Equal(MemoryException.ErrorCodes.FailedToCreateCollection, e.ErrorCode); - Assert.Contains("Failed to create collection", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageExceptionCtor() - { - // Arrange - const string Message = "this is a test"; - var inner = new FormatException(); - var e = new MemoryException(MemoryException.ErrorCodes.FailedToCreateCollection, Message, inner); - - // Assert - Assert.Equal(MemoryException.ErrorCodes.FailedToCreateCollection, e.ErrorCode); - Assert.Contains("Failed to create collection", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Same(inner, e.InnerException); - } - - [Fact] - public void ItAllowsNullMessageAndInnerExceptionInCtors() - { - // Arrange - var e = new MemoryException(MemoryException.ErrorCodes.FailedToCreateCollection, null, null); - - // Assert - Assert.Equal(MemoryException.ErrorCodes.FailedToCreateCollection, e.ErrorCode); - Assert.Contains("Failed to create collection", e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Memory/MemoryRecordTests.cs b/dotnet/src/SemanticKernel.UnitTests/Memory/MemoryRecordTests.cs index 4c85a9366fdd..d1200af829c6 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Memory/MemoryRecordTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Memory/MemoryRecordTests.cs @@ -2,7 +2,6 @@ using System; using System.Text.Json; -using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.Memory; using Xunit; @@ -16,7 +15,7 @@ public class MemoryRecordTests private readonly string _description = "description"; private readonly string _externalSourceName = "externalSourceName"; private readonly string _additionalMetadata = "value"; - private readonly Embedding _embedding = new(new float[] { 1, 2, 3 }); + private readonly ReadOnlyMemory _embedding = new(new float[] { 1, 2, 3 }); [Fact] public void ItCanBeConstructedFromMetadataAndVector() @@ -39,7 +38,7 @@ public void ItCanBeConstructedFromMetadataAndVector() Assert.Equal(this._text, memoryRecord.Metadata.Text); Assert.Equal(this._description, memoryRecord.Metadata.Description); Assert.Equal(this._externalSourceName, memoryRecord.Metadata.ExternalSourceName); - Assert.Equal(this._embedding.Vector, memoryRecord.Embedding.Vector); + Assert.True(this._embedding.Span.SequenceEqual(memoryRecord.Embedding.Span)); } [Fact] @@ -58,7 +57,7 @@ public void ItCanBeCreatedToRepresentLocalData() Assert.Equal(this._text, memoryRecord.Metadata.Text); Assert.Equal(this._description, memoryRecord.Metadata.Description); Assert.Equal(string.Empty, memoryRecord.Metadata.ExternalSourceName); - Assert.Equal(this._embedding.Vector, memoryRecord.Embedding.Vector); + Assert.True(this._embedding.Span.SequenceEqual(memoryRecord.Embedding.Span)); } [Fact] @@ -77,7 +76,7 @@ public void ItCanBeCreatedToRepresentExternalData() Assert.Equal(string.Empty, memoryRecord.Metadata.Text); Assert.Equal(this._description, memoryRecord.Metadata.Description); Assert.Equal(this._externalSourceName, memoryRecord.Metadata.ExternalSourceName); - Assert.Equal(this._embedding.Vector, memoryRecord.Embedding.Vector); + Assert.True(this._embedding.Span.SequenceEqual(memoryRecord.Embedding.Span)); } [Fact] @@ -103,7 +102,7 @@ public void ItCanBeCreatedFromSerializedMetadata() Assert.Equal(this._description, memoryRecord.Metadata.Description); Assert.Equal(this._externalSourceName, memoryRecord.Metadata.ExternalSourceName); Assert.Equal(this._additionalMetadata, memoryRecord.Metadata.AdditionalMetadata); - Assert.Equal(this._embedding.Vector, memoryRecord.Embedding.Vector); + Assert.True(this._embedding.Span.SequenceEqual(memoryRecord.Embedding.Span)); } [Fact] @@ -119,13 +118,12 @@ public void ItCanBeDeserializedFromJson() ""external_source_name"": ""externalSourceName"", ""additional_metadata"": ""value"" }, - ""embedding"": { - ""vector"": [ - 1, - 2, - 3 - ] - } + ""embedding"": + [ + 1, + 2, + 3 + ] }"; // Act @@ -139,7 +137,7 @@ public void ItCanBeDeserializedFromJson() Assert.Equal(this._description, memoryRecord.Metadata.Description); Assert.Equal(this._externalSourceName, memoryRecord.Metadata.ExternalSourceName); Assert.Equal(this._externalSourceName, memoryRecord.Metadata.ExternalSourceName); - Assert.Equal(this._embedding.Vector, memoryRecord.Embedding.Vector); + Assert.True(this._embedding.Span.SequenceEqual(memoryRecord.Embedding.Span)); } [Fact] @@ -147,13 +145,12 @@ public void ItCanBeSerialized() { // Arrange string jsonString = @"{ - ""embedding"": { - ""vector"": [ - 1, - 2, - 3 - ] - }, + ""embedding"": + [ + 1, + 2, + 3 + ], ""metadata"": { ""is_reference"": false, ""external_source_name"": ""externalSourceName"", diff --git a/dotnet/src/SemanticKernel.UnitTests/Orchestration/ContextVariablesConverterTests.cs b/dotnet/src/SemanticKernel.UnitTests/Orchestration/ContextVariablesConverterTests.cs index b102c26dd000..327669744c1d 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Orchestration/ContextVariablesConverterTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Orchestration/ContextVariablesConverterTests.cs @@ -26,7 +26,8 @@ public void ReadFromJsonSucceeds() // Assert Assert.Equal("b", result!["a"]); - Assert.Equal(string.Empty, result!["INPUT"]); + Assert.Throws(() => result!["INPUT"]); + Assert.Equal(string.Empty, result!.Input); } [Fact] @@ -60,10 +61,6 @@ public void ReadFromJsonSucceedsWithInput() // input value // params key/value [Theory] - [InlineData(null, new[] { "a", "b" }, new[] - { - /*lang=json,strict*/ @"{""Key"":""INPUT"",""Value"":""""}", /*lang=json,strict*/ @"{""Key"":""a"",""Value"":""b""}" - })] [InlineData("", new[] { "a", "b" }, new[] { /*lang=json,strict*/ @"{""Key"":""INPUT"",""Value"":""""}", /*lang=json,strict*/ @"{""Key"":""a"",""Value"":""b""}" @@ -157,7 +154,8 @@ public void ReadFromJsonReturnsDefaultWithEmpty() // Assert Assert.NotNull(result); - Assert.Equal(string.Empty, result!["INPUT"]); + Assert.Throws(() => result!["INPUT"]); + Assert.Equal(string.Empty, result!.Input); } [Fact] diff --git a/dotnet/src/SemanticKernel.UnitTests/Orchestration/ContextVariablesTests.cs b/dotnet/src/SemanticKernel.UnitTests/Orchestration/ContextVariablesTests.cs index e624ef885a94..e8d775888482 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Orchestration/ContextVariablesTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Orchestration/ContextVariablesTests.cs @@ -248,14 +248,13 @@ public void UpdateOriginalDoesNotAffectClonedSucceeds() string anyContent = Guid.NewGuid().ToString(); string someOtherMainContent = Guid.NewGuid().ToString(); string someOtherContent = Guid.NewGuid().ToString(); - ContextVariables target = new(); - ContextVariables original = new(mainContent); + ContextVariables original = new(mainContent); original.Set(anyName, anyContent); // Act // Clone original into target - target.Update(original); + ContextVariables target = original.Clone(); // Update original original.Update(someOtherMainContent); original.Set(anyName, someOtherContent); diff --git a/dotnet/src/SemanticKernel.UnitTests/Orchestration/FunctionResultTests.cs b/dotnet/src/SemanticKernel.UnitTests/Orchestration/FunctionResultTests.cs new file mode 100644 index 000000000000..caf17df58a85 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Orchestration/FunctionResultTests.cs @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.SemanticKernel.Orchestration; +using Moq; +using Xunit; + +namespace SemanticKernel.UnitTests.Orchestration; + +/// +/// Unit tests of . +/// +public class FunctionResultTests +{ + private readonly Mock _functionRunner = new(); + + private SKContext CreateContext() + { + return new SKContext(this._functionRunner.Object); + } + + [Fact] + public void TryGetMetadataValueReturnsTrueWhenKeyExists() + { + // Arrange + string key = Guid.NewGuid().ToString(); + string value = Guid.NewGuid().ToString(); + FunctionResult target = new("functionName", "pluginName", this.CreateContext()); + + // Act + target.Metadata.Add(key, value); + + // Assert + Assert.True(target.TryGetMetadataValue(key, out string result)); + Assert.Equal(value, result); + } + + [Fact] + public void TryGetMetadataValueReturnsFalseWhenKeyDoesNotExist() + { + // Arrange + string key = Guid.NewGuid().ToString(); + FunctionResult target = new("functionName", "pluginName", this.CreateContext()); + + // Act,Assert + Assert.False(target.TryGetMetadataValue(key, out string result)); + Assert.Null(result); + } + + [Fact] + public void TryGetMetadataValueReturnsFalseWhenKeyExistsButTypeDoesNotMatch() + { + // Arrange + string key = Guid.NewGuid().ToString(); + int value = 42; + FunctionResult target = new("functionName", "pluginName", this.CreateContext()); + + // Act + target.Metadata.Add(key, value); + + // Assert + Assert.False(target.TryGetMetadataValue(key, out string result)); + Assert.Null(result); + } + + [Fact] + public void GetValueReturnsValueWhenValueIsNotNull() + { + // Arrange + string value = Guid.NewGuid().ToString(); + FunctionResult target = new("functionName", "pluginName", this.CreateContext(), value); + + // Act,Assert + Assert.Equal(value, target.GetValue()); + } + + [Fact] + public void GetValueReturnsNullWhenValueIsNull() + { + // Arrange + FunctionResult target = new("functionName", "pluginName", this.CreateContext(), null); + + // Act,Assert + Assert.Null(target.GetValue()); + } + + [Fact] + public void GetValueThrowsWhenValueIsNotNullButTypeDoesNotMatch() + { + // Arrange + int value = 42; + FunctionResult target = new("functionName", "pluginName", this.CreateContext(), value); + + // Act,Assert + Assert.Throws(() => target.GetValue()); + } + + [Fact] + public void ConstructorSetsProperties() + { + // Arrange + string functionName = Guid.NewGuid().ToString(); + string pluginName = Guid.NewGuid().ToString(); + SKContext context = this.CreateContext(); + + // Act + FunctionResult target = new(functionName, pluginName, context); + + // Assert + Assert.Equal(functionName, target.FunctionName); + Assert.Equal(pluginName, target.PluginName); + Assert.Equal(context, target.Context); + } + + [Fact] + public void ConstructorSetsPropertiesAndValue() + { + // Arrange + string functionName = Guid.NewGuid().ToString(); + string pluginName = Guid.NewGuid().ToString(); + SKContext context = this.CreateContext(); + string value = Guid.NewGuid().ToString(); + + // Act + FunctionResult target = new(functionName, pluginName, context, value); + + // Assert + Assert.Equal(functionName, target.FunctionName); + Assert.Equal(pluginName, target.PluginName); + Assert.Equal(context, target.Context); + Assert.Equal(value, target.Value); + } + + [Fact] + public void ToStringWorksCorrectly() + { + // Arrange + string value = Guid.NewGuid().ToString(); + FunctionResult target = new("functionName", "pluginName", this.CreateContext(), value); + + // Act and Assert + Assert.Equal(value, target.ToString()); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Orchestration/KernelResultTests.cs b/dotnet/src/SemanticKernel.UnitTests/Orchestration/KernelResultTests.cs new file mode 100644 index 000000000000..f37023ab8f39 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Orchestration/KernelResultTests.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.SemanticKernel.Orchestration; +using Moq; +using Xunit; + +namespace SemanticKernel.UnitTests.Orchestration; + +/// +/// Unit tests for class. +/// +public class KernelResultTests +{ + private readonly Mock _functionRunner = new(); + private readonly SKContext _context; + + public KernelResultTests() + { + this._context = new SKContext(this._functionRunner.Object); + } + + [Fact] + public void ItReturnsCorrectValuesFromFunctionResults() + { + // Arrange + var functionResults = new List + { + new("function1", "plugin1", this._context, "value1"), + new("function2", "plugin2", this._context, "value2"), + }; + + // Act + var kernelResult = KernelResult.FromFunctionResults("value2", functionResults); + var actualFunctionResults = kernelResult.FunctionResults.ToList(); + + // Assert + Assert.Equal("value2", kernelResult.GetValue()); + Assert.Equal(functionResults.Count, actualFunctionResults.Count); + + for (var i = 0; i < functionResults.Count; i++) + { + this.AssertFunctionResult(functionResults[i], actualFunctionResults[i]); + } + } + + [Fact] + public void ToStringWorksCorrectly() + { + // Arrange + var kernelResult = KernelResult.FromFunctionResults("value", Array.Empty()); + + // Act and Assert + Assert.Equal("value", kernelResult.ToString()); + } + + private void AssertFunctionResult(FunctionResult expected, FunctionResult actual) + { + Assert.Equal(expected.FunctionName, actual.FunctionName); + Assert.Equal(expected.PluginName, actual.PluginName); + Assert.Equal(expected.Context, actual.Context); + Assert.Equal(expected.Value, actual.Value); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanSerializationTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanSerializationTests.cs index 09b64562d5cd..bfea57b58058 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanSerializationTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanSerializationTests.cs @@ -1,16 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SkillDefinition; using Moq; using Xunit; @@ -18,6 +17,8 @@ namespace SemanticKernel.UnitTests.Planning; public sealed class PlanSerializationTests { + private readonly Mock _functionRunner = new(); + [Fact] public void CanSerializePlan() { @@ -84,21 +85,15 @@ public void CanSerializePlanWithPlanStep() var plan = new Plan(goal); // Arrange Mocks - var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); - var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object - ); + var returnContext = new SKContext(this._functionRunner.Object, new ContextVariables(stepOutput)); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); plan.AddSteps(new Plan(mockFunction.Object)); @@ -122,21 +117,18 @@ public void CanSerializePlanWithFunctionStep() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); plan.AddSteps(mockFunction.Object); @@ -160,21 +152,18 @@ public void CanSerializePlanWithFunctionSteps() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); plan.AddSteps(mockFunction.Object, mockFunction.Object); @@ -198,21 +187,18 @@ public void CanSerializePlanWithStepsAndFunction() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); plan.AddSteps(new Plan(mockFunction.Object), mockFunction.Object); @@ -235,21 +221,18 @@ public void CanSerializePlanWithSteps() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); plan.AddSteps(new Plan(mockFunction.Object), new Plan(mockFunction.Object)); @@ -272,20 +255,35 @@ public async Task CanStepAndSerializePlanWithStepsAsync() // Arrange var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); + var functionRunner = new Mock(); + kernel.SetupGet(x => x.Functions).Returns(functions.Object); - var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + kernel.Setup(k => k.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((contextVariables, functions, loggerFactory, culture) => + { + return new SKContext(this._functionRunner.Object, contextVariables, functions); + }); + + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); + + this._functionRunner.Setup(k => k.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((function, variables, ct) => + { + var c = new SKContext(new Mock().Object, variables); + returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input); + var functionResult = new FunctionResult(function.Name, function.PluginName, returnContext); + return Task.FromResult(functionResult); + }); + + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(mockFunction.Object, mockFunction.Object); @@ -329,28 +327,49 @@ public async Task CanStepAndSerializePlanWithStepsAndContextAsync() // Arrange var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); + + kernel.SetupGet(x => x.Functions).Returns(functions.Object); + + kernel.Setup(k => k.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((contextVariables, functions, loggerFactory, culture) => + { + return new SKContext(this._functionRunner.Object, contextVariables); + }); var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => { c.Variables.TryGetValue("variables", out string? v); returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input + v); }) - .Returns(() => Task.FromResult(returnContext)); - mockFunction.Setup(x => x.Describe()).Returns(new FunctionView() + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); + + mockFunction.Setup(x => x.Describe()).Returns(new FunctionView("functionName", "pluginName")); + + this._functionRunner.Setup(k => k.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((function, variables, ct) => + { + var c = new SKContext(new Mock().Object, variables); + c.Variables.TryGetValue("variables", out string? v); + + returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input + v); + var functionResult = new FunctionResult(function.Name, function.PluginName, returnContext); + + return Task.FromResult(functionResult); + }); + + mockFunction.Setup(x => x.Describe()).Returns(new FunctionView("functionName", "pluginName") { - Parameters = new List() + Parameters = new ParameterView[] { - new ParameterView() { Name = "variables" } + new("variables") } }); @@ -376,7 +395,7 @@ public async Task CanStepAndSerializePlanWithStepsAndContextAsync() // Assert Assert.NotNull(plan); Assert.Equal($"{stepOutput}{planInput}foo{stepOutput}{planInput}foobar", plan.State.ToString()); - mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Exactly(2)); + this._functionRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); // Act var serializedPlan2 = plan.ToJson(); @@ -399,38 +418,55 @@ public async Task CanStepAndSerializeAndDeserializePlanWithStepsAndContextAsync( // Arrange var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); + + kernel.SetupGet(x => x.Functions).Returns(functions.Object); var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables(stepOutput) ); + kernel.Setup(k => k.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((contextVariables, functions, loggerFactory, culture) => + { + return new SKContext(this._functionRunner.Object, contextVariables); + }); + var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => { c.Variables.TryGetValue("variables", out string? v); returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input + v); }) - .Returns(() => Task.FromResult(returnContext)); - mockFunction.Setup(x => x.Describe()).Returns(new FunctionView() + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); + mockFunction.Setup(x => x.Describe()).Returns(new FunctionView("functionName", "pluginName") { - Parameters = new List + Parameters = new ParameterView[] { - new() { Name = "variables" } + new("variables") } }); ISKFunction? outFunc = mockFunction.Object; - skills.Setup(x => x.TryGetFunction(It.IsAny(), out outFunc)).Returns(true); - skills.Setup(x => x.TryGetFunction(It.IsAny(), It.IsAny(), out outFunc)).Returns(true); - skills.Setup(x => x.GetFunction(It.IsAny(), It.IsAny())).Returns(mockFunction.Object); + functions.Setup(x => x.TryGetFunction(It.IsAny(), out outFunc)).Returns(true); + functions.Setup(x => x.TryGetFunction(It.IsAny(), It.IsAny(), out outFunc)).Returns(true); + functions.Setup(x => x.GetFunction(It.IsAny(), It.IsAny())).Returns(mockFunction.Object); plan.AddSteps(mockFunction.Object, mockFunction.Object); + this._functionRunner.Setup(k => k.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((function, variables, ct) => + { + var c = new SKContext(new Mock().Object, variables); + c.Variables.TryGetValue("variables", out string? v); + + returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input + v); + var functionResult = new FunctionResult(function.Name, function.PluginName, returnContext); + return Task.FromResult(functionResult); + }); + var serializedPlan = plan.ToJson(); var cv = new ContextVariables(planInput); @@ -449,18 +485,18 @@ public async Task CanStepAndSerializeAndDeserializePlanWithStepsAndContextAsync( // Act cv.Set("variables", "bar"); cv.Update(string.Empty); + var nextContext = new SKContext( - new ContextVariables(), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables() ); - plan = Plan.FromJson(serializedPlan1, nextContext); + plan = Plan.FromJson(serializedPlan1, functions.Object); plan = await kernel.Object.StepAsync(cv, plan); // Assert Assert.NotNull(plan); Assert.Equal($"{stepOutput}{planInput}foo{stepOutput}{planInput}foobar", plan.State.ToString()); - mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Exactly(2)); + this._functionRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); // Act var serializedPlan2 = plan.ToJson(); @@ -483,36 +519,33 @@ public void CanDeserializePlan(bool requireFunctions) var plan = new Plan(goal); // Arrange - var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); if (requireFunctions) { mockFunction.Setup(x => x.Name).Returns(string.Empty); ISKFunction? outFunc = mockFunction.Object; - skills.Setup(x => x.TryGetFunction(It.IsAny(), out outFunc)).Returns(true); - skills.Setup(x => x.TryGetFunction(It.IsAny(), It.IsAny(), out outFunc)).Returns(true); - skills.Setup(x => x.GetFunction(It.IsAny(), It.IsAny())).Returns(mockFunction.Object); + functions.Setup(x => x.TryGetFunction(It.IsAny(), out outFunc)).Returns(true); + functions.Setup(x => x.TryGetFunction(It.IsAny(), It.IsAny(), out outFunc)).Returns(true); + functions.Setup(x => x.GetFunction(It.IsAny(), It.IsAny())).Returns(mockFunction.Object); } plan.AddSteps(new Plan("Step1", mockFunction.Object), mockFunction.Object); // Act var serializedPlan = plan.ToJson(); - var deserializedPlan = Plan.FromJson(serializedPlan, returnContext, requireFunctions); + var deserializedPlan = Plan.FromJson(serializedPlan, functions.Object, requireFunctions); // Assert Assert.NotNull(deserializedPlan); @@ -541,20 +574,18 @@ public void DeserializeWithMissingFunctions(bool requireFunctions) // Arrange var kernel = new Mock(); - var logger = new Mock(); - var skills = new Mock(); + var functions = new Mock(); var returnContext = new SKContext( - new ContextVariables(stepOutput), - skills.Object, - logger.Object + this._functionRunner.Object, + new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); plan.AddSteps(new Plan("Step1", mockFunction.Object), mockFunction.Object); @@ -563,12 +594,12 @@ public void DeserializeWithMissingFunctions(bool requireFunctions) if (requireFunctions) { // Act + Assert - Assert.Throws(() => Plan.FromJson(serializedPlan, returnContext)); + Assert.Throws(() => Plan.FromJson(serializedPlan, functions.Object)); } else { // Act - var deserializedPlan = Plan.FromJson(serializedPlan, returnContext, requireFunctions); + var deserializedPlan = Plan.FromJson(serializedPlan, functions.Object, requireFunctions); // Assert Assert.NotNull(deserializedPlan); diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanTests.cs index 12c0c9240900..1b157593b8ba 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanTests.cs @@ -1,15 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; +using System.Globalization; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SkillDefinition; using Moq; using Xunit; @@ -35,15 +34,17 @@ public Task CanCreatePlanAsync() public async Task CanExecutePlanAsync() { // Arrange + var (kernel, functionRunner) = this.SetupKernelMock(); var goal = "Write a poem or joke and send it in an e-mail to Kai."; var plan = new Plan(goal); // Act - var result = await plan.InvokeAsync("Some input"); + var result = await plan.InvokeAsync("Some input", kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal("Some input", result.Result); + Assert.Equal("Some input", result.Context.Result); + Assert.Null(result.GetValue()); } [Fact] @@ -52,18 +53,17 @@ public async Task CanExecutePlanWithContextAsync() // Arrange var goal = "Write a poem or joke and send it in an e-mail to Kai."; var plan = new Plan(goal); - var kernel = new Mock(); + var kernel = new Mock(); - var context = new SKContext( - new ContextVariables("Some input") - ); + var context = new SKContext(kernel.Object, new ContextVariables("Some input")); // Act var result = await plan.InvokeAsync(context); // Assert Assert.NotNull(result); - Assert.Equal("Some input", result.Result); + Assert.Equal("Some input", result.Context.Result); + Assert.Null(result.GetValue()); plan = new Plan(goal); // Act @@ -71,7 +71,8 @@ public async Task CanExecutePlanWithContextAsync() result = await plan.InvokeAsync(context); // Assert Assert.NotNull(result); - Assert.Equal("other input", result.Result); + Assert.Equal("other input", result.Context.Result); + Assert.Null(result.GetValue()); } [Fact] @@ -84,26 +85,26 @@ public async Task CanExecutePlanWithPlanStepAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext( - new ContextVariables(stepOutput) - ); + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput)); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(new Plan(mockFunction.Object)); // Act - var result = await plan.InvokeAsync(planInput); + var result = await plan.InvokeAsync(planInput, kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal($"{stepOutput}{planInput}", result.Result); + Assert.Equal($"{stepOutput}{planInput}", result.Context.Result); + Assert.Equal($"{stepOutput}{planInput}", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Once); } @@ -117,26 +118,26 @@ public async Task CanExecutePlanWithFunctionStepAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext( - new ContextVariables(stepOutput) - ); + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput)); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(mockFunction.Object); // Act - var result = await plan.InvokeAsync(planInput); + var result = await plan.InvokeAsync(planInput, kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal($"{stepOutput}{planInput}", result.Result); + Assert.Equal($"{stepOutput}{planInput}", result.Context.Result); + Assert.Equal($"{stepOutput}{planInput}", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Once); } @@ -150,26 +151,26 @@ public async Task CanExecutePlanWithFunctionStepsAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext( - new ContextVariables(stepOutput) - ); + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput)); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(mockFunction.Object, mockFunction.Object); // Act - var result = await plan.InvokeAsync(planInput); + var result = await plan.InvokeAsync(planInput, kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.Result); + Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.Context.Result); + Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Exactly(2)); } @@ -183,26 +184,26 @@ public async Task CanExecutePlanWithStepsAndFunctionAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext( - new ContextVariables(stepOutput) - ); + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput)); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(new Plan(mockFunction.Object), mockFunction.Object); // Act - var result = await plan.InvokeAsync(planInput); + var result = await plan.InvokeAsync(planInput, kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.Result); + Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.Context.Result); + Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Exactly(2)); } @@ -216,26 +217,26 @@ public async Task CanExecutePlanWithStepsAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext( - new ContextVariables(stepOutput) - ); + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput)); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(new Plan(mockFunction.Object), new Plan(mockFunction.Object)); // Act - var result = await plan.InvokeAsync(planInput); + var result = await plan.InvokeAsync(planInput, kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.Result); + Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.Context.Result); + Assert.Equal($"{stepOutput}{planInput}{stepOutput}{planInput}", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Exactly(2)); } @@ -249,17 +250,17 @@ public async Task CanStepPlanWithStepsAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext( - new ContextVariables(stepOutput) + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput) ); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(mockFunction.Object, mockFunction.Object); @@ -289,26 +290,21 @@ public async Task CanStepPlanWithStepsAndContextAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext( - new ContextVariables(stepOutput) - ); + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput)); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => { c.Variables.TryGetValue("variables", out string? v); returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input + v); }) - .Returns(() => Task.FromResult(returnContext)); - mockFunction.Setup(x => x.Describe()).Returns(new FunctionView + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext))); + mockFunction.Setup(x => x.Describe()).Returns(new FunctionView("functionName", "pluginName", "description") { - Parameters = new List - { - new() { Name = "variables" } - } + Parameters = new ParameterView[] { new("variables") } }); plan.AddSteps(mockFunction.Object, mockFunction.Object); @@ -343,23 +339,20 @@ public async Task StepExceptionIsThrownAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); - - var returnContext = new SKContext( - new ContextVariables(stepOutput) - ); + var (kernel, functionRunner) = this.SetupKernelMock(); - returnContext.Fail("Error description", new ArgumentException("Error message")); + var returnContext = new SKContext(functionRunner.Object, new ContextVariables(stepOutput)); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Returns(() => Task.FromResult(returnContext)); + .Throws(new ArgumentException("Error message")); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(mockFunction.Object, mockFunction.Object); // Act var cv = new ContextVariables(planInput); - await Assert.ThrowsAsync(async () => await kernel.Object.StepAsync(cv, plan)); + await Assert.ThrowsAsync(async () => await kernel.Object.StepAsync(cv, plan)); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Once); } @@ -372,28 +365,27 @@ public async Task PlanStepExceptionIsThrownAsync() var plan = new Plan(goal); // Arrange - var kernel = new Mock(); var logger = new Mock(); - var skills = new Mock(); - - var returnContext = new SKContext(); + var functions = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - returnContext.Fail("Error description", new ArgumentException("Error message")); + var returnContext = new SKContext(functionRunner.Object); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Returns(() => Task.FromResult(returnContext)); + .Throws(new ArgumentException("Error message")); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); plan.AddSteps(new Plan(mockFunction.Object), new Plan(mockFunction.Object)); // Act var cv = new ContextVariables(planInput); - await Assert.ThrowsAsync(async () => await kernel.Object.StepAsync(cv, plan)); + await Assert.ThrowsAsync(async () => await kernel.Object.StepAsync(cv, plan)); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Once); } [Fact] - public async Task CanExecutePanWithTreeStepsAsync() + public async Task CanExecutePlanWithTreeStepsAsync() { // Arrange var goal = "Write a poem or joke and send it in an e-mail to Kai."; @@ -401,31 +393,37 @@ public async Task CanExecutePanWithTreeStepsAsync() var subPlan = new Plan("Write a poem or joke"); // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext(); + var returnContext = new SKContext(functionRunner.Object); var childFunction1 = new Mock(); childFunction1.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update("Child 1 output!" + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("child1", "pluginName", returnContext, returnContext.Result))); + childFunction1.Setup(x => x.Describe()).Returns(() => new FunctionView("child1", "pluginName")); + var childFunction2 = new Mock(); childFunction2.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update("Child 2 is happy about " + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("child2", "pluginName", returnContext, returnContext.Result))); + childFunction2.Setup(x => x.Describe()).Returns(() => new FunctionView("child2", "pluginName")); + var childFunction3 = new Mock(); childFunction3.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update("Child 3 heard " + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("child3", "pluginName", returnContext, returnContext.Result))); + childFunction3.Setup(x => x.Describe()).Returns(() => new FunctionView("child3", "pluginName")); var nodeFunction1 = new Mock(); nodeFunction1.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update(c.Variables.Input + " - this just happened.")) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("node1", "pluginName", returnContext, returnContext.Result))); + nodeFunction1.Setup(x => x.Describe()).Returns(() => new FunctionView("node1", "pluginName")); subPlan.AddSteps(childFunction1.Object, childFunction2.Object, childFunction3.Object); plan.AddSteps(subPlan); @@ -476,25 +474,27 @@ public void CanCreatePlanWithGoalAndSubPlans() public async Task CanExecutePlanWithOneStepAndStateAsync() { // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext(); + var returnContext = new SKContext(functionRunner.Object); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update("Here is a poem about " + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); var plan = new Plan(mockFunction.Object); plan.State.Set("input", "Cleopatra"); // Act - var result = await plan.InvokeAsync(); + var result = await plan.InvokeAsync(kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal("Here is a poem about Cleopatra", result.Result); + Assert.Equal("Here is a poem about Cleopatra", result.Context.Result); + Assert.Equal("Here is a poem about Cleopatra", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Once); } @@ -502,18 +502,27 @@ public async Task CanExecutePlanWithOneStepAndStateAsync() public async Task CanExecutePlanWithStateAsync() { // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); + + functionRunner.Setup(k => k.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (function, variables, ct) => + { + var c = new SKContext(functionRunner.Object, variables); + var functionResult = await function.InvokeAsync(c, cancellationToken: ct); + return functionResult; + }); - var returnContext = new SKContext(); + var returnContext = new SKContext(functionRunner.Object); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => { c.Variables.TryGetValue("type", out string? t); returnContext.Variables.Update($"Here is a {t} about " + c.Variables.Input); }) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); var planStep = new Plan(mockFunction.Object); planStep.Parameters.Set("type", string.Empty); @@ -523,11 +532,12 @@ public async Task CanExecutePlanWithStateAsync() plan.State.Set("type", "poem"); // Act - var result = await plan.InvokeAsync(); + var result = await plan.InvokeAsync(kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal("Here is a poem about Cleopatra", result.Result); + Assert.Equal("Here is a poem about Cleopatra", result.Context.Result); + Assert.Equal("Here is a poem about Cleopatra", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Once); } @@ -535,36 +545,38 @@ public async Task CanExecutePlanWithStateAsync() public async Task CanExecutePlanWithCustomContextAsync() { // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext(); + var returnContext = new SKContext(functionRunner.Object); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => { c.Variables.TryGetValue("type", out string? t); returnContext.Variables.Update($"Here is a {t} about " + c.Variables.Input); }) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); var plan = new Plan(mockFunction.Object); plan.State.Set("input", "Cleopatra"); plan.State.Set("type", "poem"); // Act - var result = await plan.InvokeAsync(); + var result = await plan.InvokeAsync(kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal("Here is a poem about Cleopatra", result.Result); + Assert.Equal("Here is a poem about Cleopatra", result.Context.Result); + Assert.Equal("Here is a poem about Cleopatra", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Once); plan = new Plan(mockFunction.Object); plan.State.Set("input", "Cleopatra"); plan.State.Set("type", "poem"); - var contextOverride = new SKContext(); + var contextOverride = new SKContext(functionRunner.Object); contextOverride.Variables.Set("type", "joke"); contextOverride.Variables.Update("Medusa"); @@ -573,7 +585,8 @@ public async Task CanExecutePlanWithCustomContextAsync() // Assert Assert.NotNull(result); - Assert.Equal("Here is a joke about Medusa", result.Result); + Assert.Equal("Here is a joke about Medusa", result.Context.Result); + Assert.Equal("Here is a joke about Medusa", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Exactly(2)); } @@ -581,18 +594,19 @@ public async Task CanExecutePlanWithCustomContextAsync() public async Task CanExecutePlanWithCustomStateAsync() { // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext(); + var returnContext = new SKContext(functionRunner.Object); var mockFunction = new Mock(); mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => { c.Variables.TryGetValue("type", out string? t); returnContext.Variables.Update($"Here is a {t} about " + c.Variables.Input); }) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + mockFunction.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); var planStep = new Plan(mockFunction.Object); planStep.Parameters.Set("type", string.Empty); @@ -602,11 +616,12 @@ public async Task CanExecutePlanWithCustomStateAsync() plan.AddSteps(planStep); // Act - var result = await plan.InvokeAsync(); + var result = await plan.InvokeAsync(kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal("Here is a joke about Medusa", result.Result); + Assert.Equal("Here is a joke about Medusa", result.Context.Result); + Assert.Equal("Here is a joke about Medusa", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Once); planStep = new Plan(mockFunction.Object); @@ -618,11 +633,12 @@ public async Task CanExecutePlanWithCustomStateAsync() plan.AddSteps(planStep); // Act - result = await plan.InvokeAsync(); + result = await plan.InvokeAsync(kernel.Object); // Assert Assert.NotNull(result); - Assert.Equal("Here is a poem about Medusa", result.Result); + Assert.Equal("Here is a poem about Medusa", result.Context.Result); + Assert.Equal("Here is a poem about Medusa", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Exactly(2)); planStep = new Plan(mockFunction.Object); @@ -630,7 +646,7 @@ public async Task CanExecutePlanWithCustomStateAsync() planStep.Parameters.Set("input", "Cleopatra"); planStep.Parameters.Set("type", "poem"); plan.AddSteps(planStep); - var contextOverride = new SKContext(); + var contextOverride = new SKContext(functionRunner.Object); contextOverride.Variables.Set("type", "joke"); contextOverride.Variables.Update("Medusa"); // context input will not override parameters @@ -639,7 +655,8 @@ public async Task CanExecutePlanWithCustomStateAsync() // Assert Assert.NotNull(result); - Assert.Equal("Here is a joke about Cleopatra", result.Result); + Assert.Equal("Here is a joke about Cleopatra", result.Context.Result); + Assert.Equal("Here is a joke about Cleopatra", result.GetValue()); mockFunction.Verify(x => x.InvokeAsync(It.IsAny(), null, It.IsAny()), Times.Exactly(3)); } @@ -647,43 +664,54 @@ public async Task CanExecutePlanWithCustomStateAsync() public async Task CanExecutePlanWithJoinedResultAsync() { // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); + + functionRunner.Setup(k => k.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (function, variables, ct) => + { + var c = new SKContext(functionRunner.Object, variables); + var functionResult = await function.InvokeAsync(c, cancellationToken: ct); + return functionResult; + }); - var returnContext = new SKContext(); + var returnContext = new SKContext(functionRunner.Object); var outlineMock = new Mock(); outlineMock.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update($"Here is a {c.Variables["chapterCount"]} chapter outline about " + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("outline", "pluginName", returnContext, returnContext.Result))); + outlineMock.Setup(x => x.Describe()).Returns(() => new FunctionView("outline", "pluginName")); var elementAtIndexMock = new Mock(); elementAtIndexMock.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => { returnContext.Variables.Update($"Outline section #{c.Variables["index"]} of {c.Variables["count"]}: " + c.Variables.Input); }) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("elementAt", "pluginName", returnContext, returnContext.Result))); + elementAtIndexMock.Setup(x => x.Describe()).Returns(() => new FunctionView("elementAt", "pluginName")); var novelChapterMock = new Mock(); novelChapterMock.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => { returnContext.Variables.Update( $"Chapter #{c.Variables["chapterIndex"]}: {c.Variables.Input}\nTheme:{c.Variables["theme"]}\nPreviously:{c.Variables["previousChapter"]}"); }) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("novelChapter", "pluginName", returnContext, returnContext.Result))); + novelChapterMock.Setup(x => x.Describe()).Returns(() => new FunctionView("novelChapter", "pluginName")); var plan = new Plan("A plan with steps that alternate appending to the plan result."); // Steps: - // - WriterSkill.NovelOutline chapterCount='3' INPUT='A group of kids in a club called 'The Thinking Caps' that solve mysteries and puzzles using their creativity and logic.' endMarker='' => OUTLINE - // - MiscSkill.ElementAtIndex count='3' INPUT='$OUTLINE' index='0' => CHAPTER_1_SYNOPSIS - // - WriterSkill.NovelChapter chapterIndex='1' previousChapter='' INPUT='$CHAPTER_1_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_1 - // - MiscSkill.ElementAtIndex count='3' INPUT='$OUTLINE' index='1' => CHAPTER_2_SYNOPSIS - // - WriterSkill.NovelChapter chapterIndex='2' previousChapter='$CHAPTER_1_SYNOPSIS' INPUT='$CHAPTER_2_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_2 - // - MiscSkill.ElementAtIndex count='3' INPUT='$OUTLINE' index='2' => CHAPTER_3_SYNOPSIS - // - WriterSkill.NovelChapter chapterIndex='3' previousChapter='$CHAPTER_2_SYNOPSIS' INPUT='$CHAPTER_3_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_3 + // - WriterPlugin.NovelOutline chapterCount='3' INPUT='A group of kids in a club called 'The Thinking Caps' that solve mysteries and puzzles using their creativity and logic.' endMarker='' => OUTLINE + // - MiscPlugin.ElementAtIndex count='3' INPUT='$OUTLINE' index='0' => CHAPTER_1_SYNOPSIS + // - WriterPlugin.NovelChapter chapterIndex='1' previousChapter='' INPUT='$CHAPTER_1_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_1 + // - MiscPlugin.ElementAtIndex count='3' INPUT='$OUTLINE' index='1' => CHAPTER_2_SYNOPSIS + // - WriterPlugin.NovelChapter chapterIndex='2' previousChapter='$CHAPTER_1_SYNOPSIS' INPUT='$CHAPTER_2_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_2 + // - MiscPlugin.ElementAtIndex count='3' INPUT='$OUTLINE' index='2' => CHAPTER_3_SYNOPSIS + // - WriterPlugin.NovelChapter chapterIndex='3' previousChapter='$CHAPTER_2_SYNOPSIS' INPUT='$CHAPTER_3_SYNOPSIS' theme='Children's mystery' => RESULT__CHAPTER_3 var planStep = new Plan(outlineMock.Object); planStep.Parameters.Set("input", "NovelOutline function input."); @@ -740,7 +768,7 @@ public async Task CanExecutePlanWithJoinedResultAsync() plan.AddSteps(planStep); // Act - var result = await plan.InvokeAsync(); + var result = await plan.InvokeAsync(kernel.Object); var expected = @"Chapter #1: Outline section #0 of 3: Here is a 3 chapter outline about NovelOutline function input. @@ -754,22 +782,28 @@ public async Task CanExecutePlanWithJoinedResultAsync() Previously:Outline section #1 of 3: Here is a 3 chapter outline about NovelOutline function input."; // Assert - Assert.Equal(expected, result.Result); + Assert.Equal(expected, result.GetValue()); + Assert.Equal(expected, result.Context.Result); + Assert.True(result.TryGetMetadataValue("RESULT__CHAPTER_1", out var chapter1)); + Assert.True(result.TryGetMetadataValue("RESULT__CHAPTER_2", out var chapter2)); + Assert.True(result.TryGetMetadataValue("CHAPTER_3", out var chapter3)); + Assert.False(result.TryGetMetadataValue("CHAPTER_3_SYNOPSIS", out var chapter3Synopsis)); } [Fact] public async Task CanExecutePlanWithExpandedAsync() { // Arrange - var kernel = new Mock(); + var (kernel, functionRunner) = this.SetupKernelMock(); - var returnContext = new SKContext(); + var returnContext = new SKContext(functionRunner.Object); var functionMock = new Mock(); functionMock.Setup(x => x.InvokeAsync(It.IsAny(), null, It.IsAny())) - .Callback((c, s, ct) => + .Callback((c, s, ct) => returnContext.Variables.Update($"Here is a payload '{c.Variables["payload"]}' for " + c.Variables.Input)) - .Returns(() => Task.FromResult(returnContext)); + .Returns(() => Task.FromResult(new FunctionResult("functionName", "pluginName", returnContext, returnContext.Result))); + functionMock.Setup(x => x.Describe()).Returns(() => new FunctionView("functionName", "pluginName")); var plan = new Plan("A plan with steps that have variables with a $ in them but not associated with an output"); @@ -781,12 +815,38 @@ public async Task CanExecutePlanWithExpandedAsync() plan.State.Set("var", "foobar"); // Act - var result = await plan.InvokeAsync(); + var result = await plan.InvokeAsync(kernel.Object); var expected = @"Here is a payload '{""prop"":""value"", ""$prop"": 3, ""prop2"": ""my name is $pop and foobar""}' for Function input."; // Assert - Assert.Equal(expected, result.Result); + Assert.Equal(expected, result.Context.Result); + Assert.Equal(expected, result.GetValue()); + } + + private (Mock kernelMock, Mock functionRunnerMock) SetupKernelMock(IFunctionCollection? functions = null) + { + functions ??= new Mock().Object; + + var kernel = new Mock(); + var functionRunner = new Mock(); + + kernel.SetupGet(x => x.Functions).Returns(functions); + kernel.Setup(k => k.CreateNewContext(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((contextVariables, skills, loggerFactory, culture) => + { + return new SKContext(functionRunner.Object, contextVariables, functions); + }); + + functionRunner.Setup(k => k.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (function, variables, ct) => + { + var c = new SKContext(functionRunner.Object, variables); + var functionResult = await function.InvokeAsync(c, cancellationToken: ct); + return functionResult; + }); + + return (kernel, functionRunner); } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningExceptionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningExceptionTests.cs deleted file mode 100644 index 5cae12c12cec..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningExceptionTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Planning; -using Xunit; - -namespace SemanticKernel.UnitTests.Planning; - -public class PlanningExceptionTests -{ - [Fact] - public void ItRoundtripsArgsToErrorCodeCtor() - { - // Arrange - var e = new PlanningException(PlanningException.ErrorCodes.InvalidGoal); - - // Assert - Assert.Equal(PlanningException.ErrorCodes.InvalidGoal, e.ErrorCode); - Assert.Contains("Invalid goal", e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageCtor() - { - // Arrange - const string Message = "this is a test"; - var e = new PlanningException(PlanningException.ErrorCodes.InvalidGoal, Message); - - // Assert - Assert.Equal(PlanningException.ErrorCodes.InvalidGoal, e.ErrorCode); - Assert.Contains("Invalid goal", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageExceptionCtor() - { - // Arrange - const string Message = "this is a test"; - var inner = new FormatException(); - var e = new PlanningException(PlanningException.ErrorCodes.InvalidGoal, Message, inner); - - // Assert - Assert.Equal(PlanningException.ErrorCodes.InvalidGoal, e.ErrorCode); - Assert.Contains("Invalid goal", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Same(inner, e.InnerException); - } - - [Fact] - public void ItAllowsNullMessageAndInnerExceptionInCtors() - { - // Arrange - var e = new PlanningException(PlanningException.ErrorCodes.InvalidGoal, null, null); - - // Assert - Assert.Equal(PlanningException.ErrorCodes.InvalidGoal, e.ErrorCode); - Assert.Contains("Invalid goal", e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Reliability/DefaultHttpRetryHandlerTests.cs b/dotnet/src/SemanticKernel.UnitTests/Reliability/DefaultHttpRetryHandlerTests.cs deleted file mode 100644 index d89088cf5136..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Reliability/DefaultHttpRetryHandlerTests.cs +++ /dev/null @@ -1,677 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Reliability; -using Moq; -using Moq.Protected; -using Xunit; - -namespace SemanticKernel.UnitTests.Reliability; - -public class DefaultHttpRetryHandlerTests -{ - [Theory] - [InlineData(HttpStatusCode.RequestTimeout)] - [InlineData(HttpStatusCode.ServiceUnavailable)] - [InlineData(HttpStatusCode.GatewayTimeout)] - [InlineData(HttpStatusCode.TooManyRequests)] - public async Task NoMaxRetryCountCallsOnceForStatusAsync(HttpStatusCode statusCode) - { - // Arrange - using var retry = new DefaultHttpRetryHandler(new HttpRetryConfig() { MaxRetryCount = 0 }, Mock.Of()); - using var mockResponse = new HttpResponseMessage(statusCode); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(statusCode, response.StatusCode); - } - - [Theory] - [InlineData(HttpStatusCode.RequestTimeout)] - [InlineData(HttpStatusCode.ServiceUnavailable)] - [InlineData(HttpStatusCode.GatewayTimeout)] - [InlineData(HttpStatusCode.TooManyRequests)] - public async Task ItRetriesOnceOnRetryableStatusAsync(HttpStatusCode statusCode) - { - // Arrange - using var retry = ConfigureRetryHandler(); - using var mockResponse = new HttpResponseMessage(statusCode); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(statusCode, response.StatusCode); - } - - [Theory] - [InlineData(typeof(HttpRequestException))] - public async Task ItRetriesOnceOnRetryableExceptionAsync(Type exceptionType) - { - // Arrange - using var retry = ConfigureRetryHandler(); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(exceptionType); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await Assert.ThrowsAsync(exceptionType, - async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - } - - [Theory] - [InlineData(typeof(HttpRequestException))] - public async Task NoMaxRetryCountCallsOnceForExceptionAsync(Type exceptionType) - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 0 }); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(exceptionType); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await Assert.ThrowsAsync(exceptionType, - async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); - } - - [Theory] - [InlineData(HttpStatusCode.RequestTimeout)] - [InlineData(HttpStatusCode.ServiceUnavailable)] - [InlineData(HttpStatusCode.GatewayTimeout)] - [InlineData(HttpStatusCode.TooManyRequests)] - public async Task ItRetriesOnceOnTransientStatusWithExponentialBackoffAsync(HttpStatusCode statusCode) - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig() { UseExponentialBackoff = true }); - using var mockResponse = new HttpResponseMessage(statusCode); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(statusCode, response.StatusCode); - } - - [Theory] - [InlineData(typeof(HttpRequestException))] - public async Task ItRetriesOnceOnRetryableExceptionWithExponentialBackoffAsync(Type exceptionType) - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig() { UseExponentialBackoff = true }); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(exceptionType); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await Assert.ThrowsAsync(exceptionType, - async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - } - - [Theory] - [InlineData(HttpStatusCode.RequestTimeout)] - [InlineData(HttpStatusCode.ServiceUnavailable)] - [InlineData(HttpStatusCode.GatewayTimeout)] - public async Task ItRetriesExponentiallyWithExponentialBackoffAsync(HttpStatusCode statusCode) - { - // Arrange - var currentTime = DateTimeOffset.UtcNow; - var mockTimeProvider = new Mock(); - var mockDelayProvider = new Mock(); - mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) - .Returns(() => currentTime) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(510)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(1015)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(1520)); - using var retry = ConfigureRetryHandler(new HttpRetryConfig() - { - UseExponentialBackoff = true, MaxRetryCount = 3, - MinRetryDelay = TimeSpan.FromMilliseconds(500) - }, mockTimeProvider, mockDelayProvider); - using var mockResponse = new HttpResponseMessage(statusCode); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(statusCode, response.StatusCode); - mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); - mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); - mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(1000), It.IsAny()), Times.Once); - mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(2000), It.IsAny()), Times.Once); - } - - [Theory] - [InlineData(HttpStatusCode.RequestTimeout)] - [InlineData(HttpStatusCode.ServiceUnavailable)] - [InlineData(HttpStatusCode.GatewayTimeout)] - public async Task ItRetriesOnceOnTransientStatusCodeWithRetryValueAsync(HttpStatusCode statusCode) - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig(), null); - using var mockResponse = new HttpResponseMessage() - { - StatusCode = statusCode, - Headers = { RetryAfter = new RetryConditionHeaderValue(new TimeSpan(0, 0, 0, 1)) }, - }; - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - using var testContent = new StringContent("test"); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(statusCode, response.StatusCode); - Assert.Equal(new TimeSpan(0, 0, 0, 1), response.Headers.RetryAfter?.Delta); - } - - [Theory] - [InlineData(HttpStatusCode.RequestTimeout)] - [InlineData(HttpStatusCode.ServiceUnavailable)] - [InlineData(HttpStatusCode.GatewayTimeout)] - public async Task ItRetriesStatusCustomCountAsync(HttpStatusCode expectedStatus) - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); - using var mockResponse = new HttpResponseMessage(expectedStatus); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(expectedStatus, response.StatusCode); - } - - [Theory] - [InlineData(typeof(HttpRequestException))] - public async Task ItRetriesExceptionsCustomCountAsync(Type expectedException) - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(expectedException); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await Assert.ThrowsAsync(expectedException, - async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); - } - - [Fact] - public async Task NoExceptionNoRetryAsync() - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); - using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - } - - [Fact] - public async Task ItDoesNotExecuteOnCancellationTokenAsync() - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); - using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - var cancellationToken = new CancellationToken(true); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await Assert.ThrowsAsync(async () => - await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken)); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Never(), ItExpr.IsAny(), ItExpr.IsAny()); - } - - [Fact] - public async Task ItDoestExecuteOnFalseCancellationTokenAsync() - { - // Arrange - using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); - using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - var cancellationToken = new CancellationToken(false); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - } - - [Fact] - public async Task ItRetriesWithMinRetryDelayAsync() - { - var httpRetryConfig = new HttpRetryConfig - { - MinRetryDelay = TimeSpan.FromMilliseconds(500) - }; - - var mockDelayProvider = new Mock(); - var mockTimeProvider = new Mock(); - - var currentTime = DateTimeOffset.UtcNow; - - mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) - .Returns(() => currentTime) - .Returns(() => currentTime.AddMilliseconds(5)) - .Returns(() => currentTime.AddMilliseconds(510)); - - mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) - .Returns(() => Task.CompletedTask); - - using var retry = ConfigureRetryHandler(httpRetryConfig, mockTimeProvider, mockDelayProvider); - using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); - mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); - } - - [Fact] - public async Task ItRetriesWithMaxRetryDelayAsync() - { - var httpRetryConfig = new HttpRetryConfig - { - MinRetryDelay = TimeSpan.FromMilliseconds(1), - MaxRetryDelay = TimeSpan.FromMilliseconds(500) - }; - - var mockDelayProvider = new Mock(); - var mockTimeProvider = new Mock(); - - var currentTime = DateTimeOffset.UtcNow; - - mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) - .Returns(() => currentTime) - .Returns(() => currentTime.AddMilliseconds(5)) - .Returns(() => currentTime.AddMilliseconds(505)); - - mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) - .Returns(() => Task.CompletedTask); - - using var retry = ConfigureRetryHandler(httpRetryConfig, mockTimeProvider, mockDelayProvider); - using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests) - { - Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(2000)) } - }; - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); - mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); - Assert.Equal(TimeSpan.FromMilliseconds(2000), response.Headers.RetryAfter?.Delta); - } - - [Theory] - [InlineData(HttpStatusCode.TooManyRequests)] - [InlineData(HttpStatusCode.ServiceUnavailable)] - [InlineData(HttpStatusCode.GatewayTimeout)] - [InlineData(HttpStatusCode.RequestTimeout)] - public async Task ItRetriesWithMaxTotalDelayAsync(HttpStatusCode statusCode) - { - // Arrange - var httpRetryConfig = new HttpRetryConfig - { - MaxRetryCount = 5, - MinRetryDelay = TimeSpan.FromMilliseconds(50), - MaxRetryDelay = TimeSpan.FromMilliseconds(50), - MaxTotalRetryTime = TimeSpan.FromMilliseconds(350) - }; - - var mockDelayProvider = new Mock(); - var mockTimeProvider = new Mock(); - - var currentTime = DateTimeOffset.UtcNow; - mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) - .Returns(() => currentTime) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); - - using var retry = ConfigureRetryHandler(httpRetryConfig, mockTimeProvider, mockDelayProvider); - - using var mockResponse = new HttpResponseMessage(statusCode); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(6)); - mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(5)); - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(6), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(statusCode, response.StatusCode); - } - - [Fact] - public async Task ItRetriesFewerWithMaxTotalDelayAsync() - { - // Arrange - var httpRetryConfig = new HttpRetryConfig - { - MaxRetryCount = 5, - MinRetryDelay = TimeSpan.FromMilliseconds(50), - MaxRetryDelay = TimeSpan.FromMilliseconds(50), - MaxTotalRetryTime = TimeSpan.FromMilliseconds(100) - }; - - var mockDelayProvider = new Mock(); - var mockTimeProvider = new Mock(); - - var currentTime = DateTimeOffset.UtcNow; - mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) - .Returns(() => currentTime) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); - - using var retry = ConfigureRetryHandler(httpRetryConfig, mockTimeProvider, mockDelayProvider); - - using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); // 1 initial, 2 retries, 1 for logging time taken. - mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(1)); - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); - } - - [Fact] - public async Task ItRetriesFewerWithMaxTotalDelayOnExceptionAsync() - { - // Arrange - var httpRetryConfig = new HttpRetryConfig - { - MaxRetryCount = 5, - MinRetryDelay = TimeSpan.FromMilliseconds(50), - MaxRetryDelay = TimeSpan.FromMilliseconds(50), - MaxTotalRetryTime = TimeSpan.FromMilliseconds(100) - }; - - var mockDelayProvider = new Mock(); - var mockTimeProvider = new Mock(); - - var currentTime = DateTimeOffset.UtcNow; - mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) - .Returns(() => currentTime) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) - .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); - - using var retry = ConfigureRetryHandler(httpRetryConfig, mockTimeProvider, mockDelayProvider); - var mockHandler = GetHttpMessageHandlerMock(typeof(HttpRequestException)); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - await Assert.ThrowsAsync(() => httpClient.GetAsync(new Uri("https://www.microsoft.com"), CancellationToken.None)); - - // Assert - mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); // 1 initial, 2 retries, 1 for logging time taken. - mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(1)); - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - } - - [Fact] - public async Task ItRetriesOnRetryableStatusCodesAsync() - { - // Arrange - var config = new HttpRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; - using var retry = ConfigureRetryHandler(config); - using var mockResponse = new HttpResponseMessage(HttpStatusCode.Unauthorized); - - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); - } - - [Fact] - public async Task ItDoesNotRetryOnNonRetryableStatusCodesAsync() - { - // Arrange - var config = new HttpRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; - using var retry = ConfigureRetryHandler(config); - using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); - - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(mockResponse); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); - Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); - } - - [Fact] - public async Task ItRetriesOnRetryableExceptionsAsync() - { - // Arrange - var config = new HttpRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; - using var retry = ConfigureRetryHandler(config); - - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(typeof(InvalidOperationException)); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - await Assert.ThrowsAsync(async () => - await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); - } - - [Fact] - public async Task ItDoesNotRetryOnNonRetryableExceptionsAsync() - { - // Arrange - var config = new HttpRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; - using var retry = ConfigureRetryHandler(config); - - using var testContent = new StringContent("test"); - var mockHandler = GetHttpMessageHandlerMock(typeof(ArgumentException)); - - retry.InnerHandler = mockHandler.Object; - using var httpClient = new HttpClient(retry); - - // Act - await Assert.ThrowsAsync(async () => - await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); - - // Assert - mockHandler.Protected() - .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); - } - - private static DefaultHttpRetryHandler ConfigureRetryHandler(HttpRetryConfig? config = null, - Mock? timeProvider = null, Mock? delayProvider = null) - { - delayProvider ??= new Mock(); - timeProvider ??= new Mock(); - var retry = new DefaultHttpRetryHandler(config ?? new HttpRetryConfig(), Mock.Of(), delayProvider.Object, timeProvider.Object); - return retry; - } - - private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) - { - var mockHandler = new Mock(); - mockHandler.Protected() - .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) - .ReturnsAsync(mockResponse); - return mockHandler; - } - - private static Mock GetHttpMessageHandlerMock(Type exceptionType) - { - var mockHandler = new Mock(); - mockHandler.Protected() - .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) - .ThrowsAsync(Activator.CreateInstance(exceptionType) as Exception); - return mockHandler; - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Reliability/HttpRetryConfigTests.cs b/dotnet/src/SemanticKernel.UnitTests/Reliability/HttpRetryConfigTests.cs deleted file mode 100644 index 2501217a46a4..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Reliability/HttpRetryConfigTests.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Reliability; -using Xunit; - -namespace SemanticKernel.UnitTests.Reliability; - -/// -/// Unit tests of . -/// -public class HttpRetryConfigTests -{ - [Fact] - public async Task NegativeMaxRetryCountThrowsAsync() - { - // Act - await Assert.ThrowsAsync(() => - { - var httpRetryConfig = new HttpRetryConfig() { MaxRetryCount = -1 }; - return Task.CompletedTask; - }); - } - - [Fact] - public void SetDefaultHttpRetryConfig() - { - // Arrange - var config = new KernelConfig(); - var httpRetryConfig = new HttpRetryConfig() { MaxRetryCount = 1 }; - - // Act - config.SetDefaultHttpRetryConfig(httpRetryConfig); - - // Assert - Assert.Equal(httpRetryConfig, config.DefaultHttpRetryConfig); - } - - [Fact] - public void SetDefaultHttpRetryConfigToDefaultIfNotSet() - { - // Arrange - var config = new KernelConfig(); - - // Act - // Assert - var defaultConfig = new HttpRetryConfig(); - Assert.Equal(defaultConfig.MaxRetryCount, config.DefaultHttpRetryConfig.MaxRetryCount); - Assert.Equal(defaultConfig.MaxRetryDelay, config.DefaultHttpRetryConfig.MaxRetryDelay); - Assert.Equal(defaultConfig.MinRetryDelay, config.DefaultHttpRetryConfig.MinRetryDelay); - Assert.Equal(defaultConfig.MaxTotalRetryTime, config.DefaultHttpRetryConfig.MaxTotalRetryTime); - Assert.Equal(defaultConfig.UseExponentialBackoff, config.DefaultHttpRetryConfig.UseExponentialBackoff); - Assert.Equal(defaultConfig.RetryableStatusCodes, config.DefaultHttpRetryConfig.RetryableStatusCodes); - Assert.Equal(defaultConfig.RetryableExceptionTypes, config.DefaultHttpRetryConfig.RetryableExceptionTypes); - } - - [Fact] - public void SetDefaultHttpRetryConfigToDefaultIfNull() - { - // Arrange - var config = new KernelConfig(); - - // Act - config.SetDefaultHttpRetryConfig(null); - - // Assert - var defaultConfig = new HttpRetryConfig(); - Assert.Equal(defaultConfig.MaxRetryCount, config.DefaultHttpRetryConfig.MaxRetryCount); - Assert.Equal(defaultConfig.MaxRetryDelay, config.DefaultHttpRetryConfig.MaxRetryDelay); - Assert.Equal(defaultConfig.MinRetryDelay, config.DefaultHttpRetryConfig.MinRetryDelay); - Assert.Equal(defaultConfig.MaxTotalRetryTime, config.DefaultHttpRetryConfig.MaxTotalRetryTime); - Assert.Equal(defaultConfig.UseExponentialBackoff, config.DefaultHttpRetryConfig.UseExponentialBackoff); - Assert.Equal(defaultConfig.RetryableStatusCodes, config.DefaultHttpRetryConfig.RetryableStatusCodes); - Assert.Equal(defaultConfig.RetryableExceptionTypes, config.DefaultHttpRetryConfig.RetryableExceptionTypes); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj b/dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj index 8b351704c6d8..7bb88e3fa440 100644 --- a/dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj +++ b/dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj @@ -30,7 +30,7 @@ - + diff --git a/dotnet/src/SemanticKernel.UnitTests/Services/ServiceConfigTests.cs b/dotnet/src/SemanticKernel.UnitTests/Services/ServiceConfigTests.cs deleted file mode 100644 index 9285ed34a3d8..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Services/ServiceConfigTests.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Services; -using Xunit; - -namespace SemanticKernel.UnitTests.Services; - -public class ServiceConfigTests -{ - [Fact] - public void ConstructorWithValidParametersSetsProperties() - { - // Arrange - string serviceId = "testId"; - - // Act - var config = new FakeAIServiceConfig(serviceId); - - // Assert - Assert.Equal(serviceId, config.ServiceId); - } - - [Fact] - public void ConstructorWithEmptyRequiredParameterThrowsArgumentException() - { - // Act + Assert - Assert.Throws(() => new FakeAIServiceConfig(string.Empty)); - } - - private sealed class FakeAIServiceConfig : ServiceConfig - { - public FakeAIServiceConfig(string serviceId) : base(serviceId) - { - } - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/FunctionsViewTests.cs b/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/FunctionsViewTests.cs deleted file mode 100644 index f6f9b63003fd..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/FunctionsViewTests.cs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using Microsoft.SemanticKernel.SkillDefinition; -using Xunit; - -namespace SemanticKernel.UnitTests.SkillDefinition; - -public class FunctionsViewTests -{ - [Fact] - public void ItIsEmptyByDefault() - { - // Act - var target = new FunctionsView(); - - // Assert - Assert.Empty(target.SemanticFunctions); - Assert.Empty(target.NativeFunctions); - } - - [Fact] - public void ItProvidesCorrectNativeFunctionInfo() - { - // Arrange - var target = new FunctionsView() - .AddFunction(new FunctionView("f1", "s1", "", new List(), false)) - .AddFunction(new FunctionView("f2", "s1", "", new List(), false)) - .AddFunction(new FunctionView("f1", "s2", "", new List(), false)); - - // Assert - Assert.Equal(2, target.NativeFunctions.Count); - Assert.Equal(2, target.NativeFunctions["s1"].Count); - Assert.Single(target.NativeFunctions["s2"]); - Assert.True(target.IsNative("s1", "f1")); - Assert.True(target.IsNative("s1", "f2")); - Assert.True(target.IsNative("s2", "f1")); - Assert.False(target.IsNative("s5", "f5")); - Assert.False(target.IsSemantic("s1", "f1")); - } - - [Fact] - public void ItProvidesCorrectSemanticFunctionInfo() - { - // Arrange - var target = new FunctionsView() - .AddFunction(new FunctionView("f1", "s1", "", new List(), true)) - .AddFunction(new FunctionView("f2", "s1", "", new List(), true)) - .AddFunction(new FunctionView("f1", "s2", "", new List(), true)); - - // Assert - Assert.Equal(2, target.SemanticFunctions.Count); - Assert.Equal(2, target.SemanticFunctions["s1"].Count); - Assert.Single(target.SemanticFunctions["s2"]); - Assert.True(target.IsSemantic("s1", "f1")); - Assert.True(target.IsSemantic("s1", "f2")); - Assert.True(target.IsSemantic("s2", "f1")); - Assert.False(target.IsSemantic("s5", "f5")); - Assert.False(target.IsNative("s1", "f1")); - } - - [Fact] - public void ItThrowsOnConflict() - { - // Arrange - var target = new FunctionsView() - .AddFunction(new FunctionView("f1", "s1", "", new List(), true)) - .AddFunction(new FunctionView("f1", "s1", "", new List(), false)); - - // Assert - Assert.Throws(() => target.IsSemantic("s1", "f1")); - Assert.Throws(() => target.IsNative("s1", "f1")); - } - - [Fact] - public void ItReturnsFunctionParams() - { - // Arrange - var params1 = new List - { - new("p1", "param 1", "default 1"), - new("p2", "param 2", "default 2") - }; - var params2 = new List - { - new("p3", "param 3", "default 3"), - new("p4", "param 4", "default 4") - }; - var target = new FunctionsView() - .AddFunction(new FunctionView("semFun", "s1", "", params1, true)) - .AddFunction(new FunctionView("natFun", "s1", "", params2, false)); - - // Act - List semFun = target.SemanticFunctions["s1"]; - List natFun = target.NativeFunctions["s1"]; - - // Assert - Assert.Single(semFun); - Assert.Single(natFun); - Assert.Equal("p1", semFun.First().Parameters[0].Name); - Assert.Equal("p2", semFun.First().Parameters[1].Name); - Assert.Equal("p3", natFun.First().Parameters[0].Name); - Assert.Equal("p4", natFun.First().Parameters[1].Name); - Assert.Equal("param 1", semFun.First().Parameters[0].Description); - Assert.Equal("param 2", semFun.First().Parameters[1].Description); - Assert.Equal("param 3", natFun.First().Parameters[0].Description); - Assert.Equal("param 4", natFun.First().Parameters[1].Description); - Assert.Equal("default 1", semFun.First().Parameters[0].DefaultValue); - Assert.Equal("default 2", semFun.First().Parameters[1].DefaultValue); - Assert.Equal("default 3", natFun.First().Parameters[0].DefaultValue); - Assert.Equal("default 4", natFun.First().Parameters[1].DefaultValue); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKContextTests.cs b/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKContextTests.cs deleted file mode 100644 index f313277b28ad..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKContextTests.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; - -namespace SemanticKernel.UnitTests.SkillDefinition; - -public class SKContextTests -{ - private readonly Mock _skills; - private readonly Mock _logger; - - public SKContextTests() - { - this._skills = new Mock(); - this._logger = new Mock(); - } - - [Fact] - public void ItHasHelpersForContextVariables() - { - // Arrange - var variables = new ContextVariables(); - var target = new SKContext(variables, skills: this._skills.Object, logger: this._logger.Object); - variables.Set("foo1", "bar1"); - - // Act - target.Variables["foo2"] = "bar2"; - target.Variables["INPUT"] = Guid.NewGuid().ToString("N"); - - // Assert - Assert.Equal("bar1", target.Variables["foo1"]); - Assert.Equal("bar1", target.Variables["foo1"]); - Assert.Equal("bar2", target.Variables["foo2"]); - Assert.Equal("bar2", target.Variables["foo2"]); - Assert.Equal(target.Variables["INPUT"], target.Result); - Assert.Equal(target.Variables["INPUT"], target.ToString()); - Assert.Equal(target.Variables["INPUT"], target.Variables.Input); - Assert.Equal(target.Variables["INPUT"], target.Variables.ToString()); - } - - [Fact] - public async Task ItHasHelpersForSkillCollectionAsync() - { - // Arrange - IDictionary skill = KernelBuilder.Create().ImportSkill(new Parrot(), "test"); - this._skills.Setup(x => x.GetFunction("func")).Returns(skill["say"]); - var target = new SKContext(new ContextVariables(), this._skills.Object, this._logger.Object); - Assert.NotNull(target.Skills); - - // Act - var say = target.Skills.GetFunction("func"); - SKContext result = await say.InvokeAsync("ciao"); - - // Assert - Assert.Equal("ciao", result.Result); - } - - private sealed class Parrot - { - [SKFunction, Description("say something")] - // ReSharper disable once UnusedMember.Local - public string Say(string input) - { - return input; - } - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests1.cs b/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests1.cs deleted file mode 100644 index 24a330a74379..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests1.cs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; - -namespace SemanticKernel.UnitTests.SkillDefinition; - -public sealed class SKFunctionTests1 -{ - private readonly Mock _promptTemplate; - - public SKFunctionTests1() - { - this._promptTemplate = new Mock(); - this._promptTemplate.Setup(x => x.RenderAsync(It.IsAny(), It.IsAny())).ReturnsAsync("foo"); - this._promptTemplate.Setup(x => x.GetParameters()).Returns(new List()); - } - - [Fact] - public void ItHasDefaultRequestSettings() - { - // Arrange - var templateConfig = new PromptTemplateConfig(); - var functionConfig = new SemanticFunctionConfig(templateConfig, this._promptTemplate.Object); - - // Act - var skFunction = SKFunction.FromSemanticConfig("sk", "name", functionConfig); - - // Assert - Assert.Equal(0, skFunction.RequestSettings.Temperature); - Assert.Equal(null, skFunction.RequestSettings.MaxTokens); - } - - [Fact] - public void ItAllowsToUpdateRequestSettings() - { - // Arrange - var templateConfig = new PromptTemplateConfig(); - var functionConfig = new SemanticFunctionConfig(templateConfig, this._promptTemplate.Object); - var skFunction = SKFunction.FromSemanticConfig("sk", "name", functionConfig); - var settings = new CompleteRequestSettings - { - Temperature = 0.9, - MaxTokens = 2001, - }; - - // Act - skFunction.RequestSettings.Temperature = 1.3; - skFunction.RequestSettings.MaxTokens = 130; - - // Assert - Assert.Equal(1.3, skFunction.RequestSettings.Temperature); - Assert.Equal(130, skFunction.RequestSettings.MaxTokens); - - // Act - skFunction.RequestSettings.Temperature = 0.7; - - // Assert - Assert.Equal(0.7, skFunction.RequestSettings.Temperature); - - // Act - skFunction.SetAIConfiguration(settings); - - // Assert - Assert.Equal(settings.Temperature, skFunction.RequestSettings.Temperature); - Assert.Equal(settings.MaxTokens, skFunction.RequestSettings.MaxTokens); - } - - private static Mock MockPromptTemplate() - { - var promptTemplate = new Mock(); - - promptTemplate.Setup(x => x.RenderAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync("some prompt"); - - promptTemplate - .Setup(x => x.GetParameters()) - .Returns(new List()); - - return promptTemplate; - } - - private static Mock MockAIService(string result) - { - var aiService = new Mock(); - var textCompletionResult = new Mock(); - - textCompletionResult - .Setup(x => x.GetCompletionAsync(It.IsAny())) - .ReturnsAsync(result); - - aiService - .Setup(x => x.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(new List { textCompletionResult.Object }); - - return aiService; - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests2.cs b/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests2.cs deleted file mode 100644 index d1b6f88b9c21..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SKFunctionTests2.cs +++ /dev/null @@ -1,914 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ComponentModel; -using System.Globalization; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; - -namespace SemanticKernel.UnitTests.SkillDefinition; - -public sealed class SKFunctionTests2 -{ - private readonly Mock _logger; - private readonly Mock _skills; - - private static string s_expected = string.Empty; - private static string s_actual = string.Empty; - - public SKFunctionTests2() - { - this._logger = new Mock(); - this._skills = new Mock(); - - s_expected = Guid.NewGuid().ToString("D"); - } - - [Fact] - public async Task ItSupportsStaticVoidVoidAsync() - { - // Arrange - static void Test() - { - s_actual = s_expected; - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - } - - [Fact] - public async Task ItSupportsStaticVoidStringAsync() - { - // Arrange - static string Test() - { - s_actual = s_expected; - return s_expected; - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, result.Result); - Assert.Equal(s_expected, context.Result); - } - - [Fact] - public async Task ItSupportsStaticVoidTaskStringAsync() - { - // Arrange - static Task Test() - { - s_actual = s_expected; - return Task.FromResult(s_expected); - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Result); - Assert.Equal(s_expected, result.Result); - } - - [Fact] - public async Task ItSupportsStaticVoidValueTaskStringAsync() - { - // Arrange - static async ValueTask Test() - { - s_actual = s_expected; - await Task.Delay(1); - return s_expected; - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Result); - Assert.Equal(s_expected, result.Result); - } - - [Fact] - public async Task ItSupportsStaticContextVoidAsync() - { - // Arrange - static void Test(SKContext context) - { - s_actual = s_expected; - context.Variables["canary"] = s_expected; - } - - var context = this.MockContext("xy"); - context.Variables["someVar"] = "qz"; - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Variables["canary"]); - } - - [Fact] - public async Task ItSupportsStaticContextStringAsync() - { - // Arrange - static string Test(SKContext context) - { - s_actual = context.Variables["someVar"]; - return "abc"; - } - - var context = this.MockContext(""); - context.Variables["someVar"] = s_expected; - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal("abc", context.Result); - } - - [Fact] - public async Task ItSupportsInstanceContextStringNullableAsync() - { - // Arrange - int invocationCount = 0; - - string? Test(SKContext context) - { - invocationCount++; - s_actual = context.Variables["someVar"]; - return "abc"; - } - - var context = this.MockContext(""); - context.Variables["someVar"] = s_expected; - - // Act - Func method = Test; - var function = SKFunction.FromNativeMethod(Method(method), method.Target, logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(1, invocationCount); - Assert.Equal(s_expected, s_actual); - Assert.Equal("abc", context.Result); - } - - [Fact] - public async Task ItSupportsInstanceContextTaskStringAsync() - { - // Arrange - int invocationCount = 0; - - Task Test(SKContext context) - { - invocationCount++; - s_actual = s_expected; - context.Variables["canary"] = s_expected; - return Task.FromResult(s_expected); - } - - var context = this.MockContext(""); - - // Act - Func> method = Test; - var function = SKFunction.FromNativeMethod(Method(method), method.Target, logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(1, invocationCount); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_actual, context.Result); - Assert.Equal(s_expected, context.Variables["canary"]); - } - - [Fact] - public async Task ItSupportsInstanceContextTaskContextAsync() - { - // Arrange - int invocationCount = 0; - - async Task TestAsync(SKContext context) - { - await Task.Delay(0); - invocationCount++; - s_actual = s_expected; - context.Variables.Update("foo"); - context.Variables["canary"] = s_expected; - return context; - } - - var context = this.MockContext(""); - - // Act - Func> method = TestAsync; - var function = SKFunction.FromNativeMethod(Method(method), method.Target, logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(1, invocationCount); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Variables["canary"]); - Assert.Equal("foo", context.Result); - } - - [Fact] - public async Task ItSupportsInstanceStringVoidAsync() - { - // Arrange - int invocationCount = 0; - - void Test(string input) - { - invocationCount++; - s_actual = s_expected + input; - } - - var context = this.MockContext(".blah"); - - // Act - Action method = Test; - var function = SKFunction.FromNativeMethod(Method(method), method.Target, logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(1, invocationCount); - Assert.Equal(s_expected + ".blah", s_actual); - } - - [Fact] - public async Task ItSupportsInstanceStringStringAsync() - { - // Arrange - int invocationCount = 0; - - string Test(string input) - { - invocationCount++; - s_actual = s_expected; - return "foo-bar"; - } - - var context = this.MockContext(""); - - // Act - Func method = Test; - var function = SKFunction.FromNativeMethod(Method(method), method.Target, logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(1, invocationCount); - Assert.Equal(s_expected, s_actual); - Assert.Equal("foo-bar", context.Result); - } - - [Fact] - public async Task ItSupportsInstanceStringTaskStringAsync() - { - // Arrange - int invocationCount = 0; - - Task Test(string input) - { - invocationCount++; - s_actual = s_expected; - return Task.FromResult("hello there"); - } - - var context = this.MockContext(""); - - // Act - Func> method = Test; - var function = SKFunction.FromNativeMethod(Method(method), method.Target, logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(1, invocationCount); - Assert.Equal(s_expected, s_actual); - Assert.Equal("hello there", context.Result); - } - - [Fact] - public async Task ItSupportsInstanceStringContextVoidAsync() - { - // Arrange - int invocationCount = 0; - - void Test(string input, SKContext context) - { - invocationCount++; - s_actual = s_expected; - context.Variables.Update("x y z"); - context.Variables["canary"] = s_expected; - } - - var context = this.MockContext(""); - - // Act - Action method = Test; - var function = SKFunction.FromNativeMethod(Method(method), method.Target, logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(1, invocationCount); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Variables["canary"]); - Assert.Equal("x y z", context.Result); - } - - [Fact] - public async Task ItSupportsInstanceContextStringVoidAsync() - { - // Arrange - int invocationCount = 0; - - void Test(SKContext context, string input) - { - invocationCount++; - s_actual = s_expected; - context.Variables.Update("x y z"); - context.Variables["canary"] = s_expected; - } - - var context = this.MockContext(""); - - // Act - Action method = Test; - var function = SKFunction.FromNativeMethod(Method(method), method.Target, logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(1, invocationCount); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Variables["canary"]); - Assert.Equal("x y z", context.Result); - } - - [Fact] - public async Task ItSupportsStaticStringContextStringAsync() - { - // Arrange - static string Test(string input, SKContext context) - { - s_actual = s_expected; - context.Variables["canary"] = s_expected; - context.Variables.Update("x y z"); - // This value should overwrite "x y z" - return "new data"; - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Variables["canary"]); - Assert.Equal("new data", context.Result); - } - - [Fact] - public async Task ItSupportsStaticStringContextTaskStringAsync() - { - // Arrange - static Task Test(string input, SKContext context) - { - s_actual = s_expected; - context.Variables["canary"] = s_expected; - context.Variables.Update("x y z"); - // This value should overwrite "x y z" - return Task.FromResult("new data"); - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Variables["canary"]); - Assert.Equal("new data", context.Result); - } - - [Fact] - public async Task ItSupportsStaticStringContextTaskContextAsync() - { - // Arrange - static Task Test(string input, SKContext context) - { - s_actual = s_expected; - context.Variables["canary"] = s_expected; - context.Variables.Update("x y z"); - - // This value should overwrite "x y z". Contexts are merged. - var newContext = new SKContext( - new ContextVariables(input), - skills: new Mock().Object); - - newContext.Variables.Update("new data"); - newContext.Variables["canary2"] = "222"; - - return Task.FromResult(newContext); - } - - var oldContext = this.MockContext(""); - oldContext.Variables["legacy"] = "something"; - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext newContext = await function.InvokeAsync(oldContext); - - // Assert - Assert.False(oldContext.ErrorOccurred); - Assert.False(newContext.ErrorOccurred); - - Assert.Equal(s_expected, s_actual); - - Assert.True(oldContext.Variables.ContainsKey("canary")); - Assert.False(oldContext.Variables.ContainsKey("canary2")); - - Assert.False(newContext.Variables.ContainsKey("canary")); - Assert.True(newContext.Variables.ContainsKey("canary2")); - - Assert.Equal(s_expected, oldContext.Variables["canary"]); - Assert.Equal("222", newContext.Variables["canary2"]); - - Assert.True(oldContext.Variables.ContainsKey("legacy")); - Assert.False(newContext.Variables.ContainsKey("legacy")); - - Assert.Equal("x y z", oldContext.Result); - Assert.Equal("new data", newContext.Result); - } - - [Fact] - public async Task ItSupportsStaticContextValueTaskContextAsync() - { - // Arrange - static ValueTask Test(string input, SKContext context) - { - // This value should overwrite "x y z". Contexts are merged. - var newCx = new SKContext( - new ContextVariables(input + "abc"), - skills: new Mock().Object); - - return new ValueTask(newCx); - } - - var oldContext = this.MockContext("test"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test), logger: this._logger.Object); - Assert.NotNull(function); - SKContext newContext = await function.InvokeAsync(oldContext); - - // Assert - Assert.Equal("testabc", newContext.Variables.Input); - } - - [Fact] - public async Task ItSupportsStaticStringTaskAsync() - { - // Arrange - static Task TestAsync(string input) - { - s_actual = s_expected; - return Task.CompletedTask; - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(TestAsync), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - } - - [Fact] - public async Task ItSupportsStaticStringValueTaskAsync() - { - // Arrange - static ValueTask TestAsync(string input) - { - s_actual = s_expected; - return default; - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(TestAsync), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - } - - [Fact] - public async Task ItSupportsStaticContextTaskAsync() - { - // Arrange - static Task TestAsync(SKContext context) - { - s_actual = s_expected; - context.Variables["canary"] = s_expected; - context.Variables.Update("x y z"); - return Task.CompletedTask; - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(TestAsync), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Variables["canary"]); - Assert.Equal("x y z", context.Result); - } - - [Fact] - public async Task ItSupportsStaticStringContextTaskAsync() - { - // Arrange - static Task TestAsync(string input, SKContext context) - { - s_actual = s_expected; - context.Variables["canary"] = s_expected; - context.Variables.Update(input + "x y z"); - return Task.CompletedTask; - } - - var context = this.MockContext("input:"); - - // Act - var function = SKFunction.FromNativeMethod(Method(TestAsync), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - Assert.Equal(s_expected, context.Variables["canary"]); - Assert.Equal("input:x y z", context.Result); - } - - [Fact] - public async Task ItSupportsStaticVoidTaskAsync() - { - // Arrange - static Task TestAsync() - { - s_actual = s_expected; - return Task.CompletedTask; - } - - var context = this.MockContext(""); - - // Act - var function = SKFunction.FromNativeMethod(Method(TestAsync), logger: this._logger.Object); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal(s_expected, s_actual); - } - - [Fact] - public async Task ItSupportsUsingNamedInputValueFromContext() - { - static string Test(string input) => "Result: " + input; - - var context = this.MockContext("input value"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal("Result: input value", result.Variables.Input); - } - - [Fact] - public async Task ItSupportsUsingNonNamedInputValueFromContext() - { - static string Test(string other) => "Result: " + other; - - var context = this.MockContext("input value"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal("Result: input value", result.Variables.Input); - } - - [Fact] - public async Task ItSupportsUsingNonNamedInputValueFromContextEvenWhenThereAreMultipleParameters() - { - static string Test(int something, long orother) => "Result: " + (something + orother); - - var context = this.MockContext("42"); - context.Variables.Set("orother", "8"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal("Result: 50", result.Variables.Input); - } - - [Fact] - public async Task ItSupportsPreferringNamedValueOverInputFromContext() - { - static string Test(string other) => "Result: " + other; - - var context = this.MockContext("input value"); - context.Variables.Set("other", "other value"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal("Result: other value", result.Variables.Input); - } - - [Fact] - public async Task ItSupportsOverridingNameWithAttribute() - { - static string Test([SKName("input"), Description("description")] string other) => "Result: " + other; - - var context = this.MockContext("input value"); - context.Variables.Set("other", "other value"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal("Result: input value", result.Variables.Input); - } - - [Fact] - public async Task ItSupportNullDefaultValuesOverInput() - { - static string Test(string? input = null, string? other = null) => "Result: " + (other is null); - - var context = this.MockContext("input value"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal("Result: True", result.Variables.Input); - } - - [Fact] - public async Task ItSupportsConvertingFromManyTypes() - { - static string Test(int a, long b, decimal c, Guid d, DateTimeOffset e, DayOfWeek? f) => - $"{a} {b} {c} {d} {e:R} {f}"; - - var context = this.MockContext(""); - context.Variables.Set("a", "1"); - context.Variables.Set("b", "-2"); - context.Variables.Set("c", "1234"); - context.Variables.Set("d", "7e08cc00-1d71-4558-81ed-69929499dea1"); - context.Variables.Set("e", "Thu, 25 May 2023 20:17:30 GMT"); - context.Variables.Set("f", "Monday"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal("1 -2 1234 7e08cc00-1d71-4558-81ed-69929499dea1 Thu, 25 May 2023 20:17:30 GMT Monday", result.Variables.Input); - } - - [Fact] - public async Task ItSupportsConvertingFromTypeConverterAttributedTypes() - { - static int Test(MyCustomType mct) => mct.Value * 2; - - var context = this.MockContext(""); - context.Variables.Set("mct", "42"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - Assert.False(result.ErrorOccurred); - Assert.Equal("84", result.Variables.Input); - } - - [TypeConverter(typeof(MyCustomTypeConverter))] - private sealed class MyCustomType - { - public int Value { get; set; } - } - -#pragma warning disable CA1812 // Instantiated by reflection - private sealed class MyCustomTypeConverter : TypeConverter - { - public override bool CanConvertFrom(ITypeDescriptorContext? context, Type sourceType) => - sourceType == typeof(string); - public override object? ConvertFrom(ITypeDescriptorContext? context, CultureInfo? culture, object value) => - new MyCustomType { Value = int.Parse((string)value, culture) }; - } -#pragma warning restore CA1812 - - [Fact] - public async Task ItSupportsConvertingFromToManyTypes() - { - // Arrange - var context = this.MockContext("1"); - - static async Task AssertResult(Delegate d, SKContext context, string expected) - { - context = await SKFunction.FromNativeFunction(d, functionName: "Test")!.InvokeAsync(context); - Assert.False(context.ErrorOccurred, context.LastErrorDescription); - Assert.Equal(expected, context.Variables.Input); - } - - // Act/Assert - await AssertResult((sbyte input) => input * 2, context, "2"); - await AssertResult((byte input) => input * 2, context, "4"); - await AssertResult((short input) => input * 2, context, "8"); - await AssertResult((ushort input) => input * 2, context, "16"); - await AssertResult((int input) => input * 2, context, "32"); - await AssertResult((uint input) => input * 2, context, "64"); - await AssertResult((long input) => input * 2, context, "128"); - await AssertResult((ulong input) => input * 2, context, "256"); - await AssertResult((float input) => input * 2, context, "512"); - await AssertResult((double input) => input * 2, context, "1024"); - await AssertResult((int input) => Task.FromResult(input * 2), context, "2048"); - await AssertResult((long input) => Task.FromResult(input * 2), context, "4096"); - await AssertResult((int input) => ValueTask.FromResult(input * 2), context, "8192"); - await AssertResult((long input) => ValueTask.FromResult(input * 2), context, "16384"); - await AssertResult((long? input) => input!.Value * 2, context, "32768"); - await AssertResult((TimeSpan input) => input * 2, context, "65536.00:00:00"); - await AssertResult((TimeSpan? input) => (int?)null, context, ""); - - context.Variables.Update("http://example.com/semantic"); - await AssertResult((Uri input) => new Uri(input, "kernel"), context, "http://example.com/kernel"); - } - - [Fact] - public async Task ItUsesContextCultureForParsingFormatting() - { - // Arrange - var context = this.MockContext(""); - ISKFunction func = SKFunction.FromNativeFunction((double input) => input * 2, functionName: "Test"); - - // Act/Assert - - context.Culture = new CultureInfo("fr-FR"); - context.Variables.Update("12,34"); // tries first to parse with the specified culture - context = await func.InvokeAsync(context); - Assert.Equal("24,68", context.Variables.Input); - - context.Culture = new CultureInfo("fr-FR"); - context.Variables.Update("12.34"); // falls back to invariant culture - context = await func.InvokeAsync(context); - Assert.Equal("24,68", context.Variables.Input); - - context.Culture = new CultureInfo("en-US"); - context.Variables.Update("12.34"); // works with current culture - context = await func.InvokeAsync(context); - Assert.Equal("24.68", context.Variables.Input); - - context.Culture = new CultureInfo("en-US"); - context.Variables.Update("12,34"); // not parsable with current or invariant culture - context = await func.InvokeAsync(context); - Assert.True(context.ErrorOccurred); - Assert.IsType(context.LastException); - } - - [Fact] - public async Task ItThrowsWhenItFailsToConvertAnArgument() - { - static string Test(Guid g) => g.ToString(); - - var context = this.MockContext(""); - context.Variables.Set("g", "7e08cc00-1d71-4558-81ed-69929499dxyz"); - - // Act - var function = SKFunction.FromNativeMethod(Method(Test)); - Assert.NotNull(function); - SKContext result = await function.InvokeAsync(context); - - // Assert - AssertExtensions.AssertIsArgumentOutOfRange(result.LastException, "g", context.Variables["g"]); - } - - private static MethodInfo Method(Delegate method) - { - return method.Method; - } - - private SKContext MockContext(string input) - { - return new SKContext( - new ContextVariables(input), - skills: this._skills.Object, - logger: this._logger.Object); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SkillCollectionTests.cs b/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SkillCollectionTests.cs deleted file mode 100644 index 730fa4118b7c..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/SkillDefinition/SkillCollectionTests.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; - -namespace SemanticKernel.UnitTests.SkillDefinition; - -public class SkillCollectionTests -{ - [Fact] - public void ItAllowsToReplaceFunctions() - { - // Arrange - var functionOne = new Mock(); - functionOne.SetupGet(x => x.Name).Returns("fName"); - functionOne.SetupGet(x => x.SkillName).Returns("sName"); - functionOne.SetupGet(x => x.Description).Returns("ONE"); - functionOne.SetupGet(x => x.IsSemantic).Returns(false); - functionOne.SetupGet(x => x.RequestSettings).Returns(new CompleteRequestSettings()); - - var functionTwo = new Mock(); - functionTwo.SetupGet(x => x.Name).Returns("fName"); - functionTwo.SetupGet(x => x.SkillName).Returns("sName"); - functionTwo.SetupGet(x => x.Description).Returns("TWO"); - functionTwo.SetupGet(x => x.IsSemantic).Returns(false); - functionTwo.SetupGet(x => x.RequestSettings).Returns(new CompleteRequestSettings()); - - var target = new SkillCollection(); - - // Act - target.AddFunction(functionOne.Object); - - // Assert - Assert.True(target.TryGetFunction("sName", "fName", out var func)); - Assert.Equal("ONE", func.Description); - - // Act - target.AddFunction(functionTwo.Object); - - // Assert - Assert.True(target.TryGetFunction("sName", "fName", out func)); - Assert.Equal("TWO", func.Description); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/CodeBlockTests.cs b/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/CodeBlockTests.cs deleted file mode 100644 index 1d2fda34c8ac..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/Blocks/CodeBlockTests.cs +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; -using Moq; -using Xunit; - -namespace SemanticKernel.UnitTests.TemplateEngine.Blocks; - -public class CodeBlockTests -{ - private readonly Mock _skills; - private readonly Mock _logger; - - public CodeBlockTests() - { - this._skills = new Mock(); - this._logger = new Mock(); - } - - [Fact] - public async Task ItThrowsIfAFunctionDoesntExistAsync() - { - // Arrange - var context = new SKContext(skills: this._skills.Object, logger: this._logger.Object); - this._skills.Setup(x => x.TryGetFunction("functionName", out It.Ref.IsAny)).Returns(false); - var target = new CodeBlock("functionName", this._logger.Object); - - // Act - var exception = await Assert.ThrowsAsync(async () => await target.RenderCodeAsync(context)); - - // Assert - Assert.Equal(TemplateException.ErrorCodes.FunctionNotFound, exception.ErrorCode); - } - - [Fact] - public async Task ItThrowsIfAFunctionCallThrowsAsync() - { - // Arrange - var context = new SKContext(skills: this._skills.Object, logger: this._logger.Object); - var function = new Mock(); - function - .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Throws(new RuntimeWrappedException("error")); - ISKFunction? outFunc = function.Object; - this._skills.Setup(x => x.TryGetFunction("functionName", out outFunc)).Returns(true); - this._skills.Setup(x => x.GetFunction("functionName")).Returns(function.Object); - var target = new CodeBlock("functionName", this._logger.Object); - - // Act - var exception = await Assert.ThrowsAsync(async () => await target.RenderCodeAsync(context)); - - // Assert - Assert.Equal(TemplateException.ErrorCodes.RuntimeError, exception.ErrorCode); - } - - [Fact] - public void ItHasTheCorrectType() - { - // Act - var target = new CodeBlock("", NullLogger.Instance); - - // Assert - Assert.Equal(BlockTypes.Code, target.Type); - } - - [Fact] - public void ItTrimsSpaces() - { - // Act + Assert - Assert.Equal("aa", new CodeBlock(" aa ", NullLogger.Instance).Content); - } - - [Fact] - public void ItChecksValidityOfInternalBlocks() - { - // Arrange - var validBlock1 = new FunctionIdBlock("x"); - var validBlock2 = new ValBlock("''"); - var invalidBlock = new VarBlock(""); - - // Act - var codeBlock1 = new CodeBlock(new List { validBlock1, validBlock2 }, "", NullLogger.Instance); - var codeBlock2 = new CodeBlock(new List { validBlock1, invalidBlock }, "", NullLogger.Instance); - - // Assert - Assert.True(codeBlock1.IsValid(out _)); - Assert.False(codeBlock2.IsValid(out _)); - } - - [Fact] - public void ItRequiresAValidFunctionCall() - { - // Arrange - var funcId = new FunctionIdBlock("funcName"); - var valBlock = new ValBlock("'value'"); - var varBlock = new VarBlock("$var"); - - // Act - var codeBlock1 = new CodeBlock(new List { funcId, valBlock }, "", NullLogger.Instance); - var codeBlock2 = new CodeBlock(new List { funcId, varBlock }, "", NullLogger.Instance); - var codeBlock3 = new CodeBlock(new List { funcId, funcId }, "", NullLogger.Instance); - var codeBlock4 = new CodeBlock(new List { funcId, varBlock, varBlock }, "", NullLogger.Instance); - - // Assert - Assert.True(codeBlock1.IsValid(out _)); - Assert.True(codeBlock2.IsValid(out _)); - - // Assert - Can't pass a function to a function - Assert.False(codeBlock3.IsValid(out _)); - - // Assert - Can't pass more than one param - Assert.False(codeBlock4.IsValid(out _)); - } - - [Fact] - public async Task ItRendersCodeBlockConsistingOfJustAVarBlock1Async() - { - // Arrange - var variables = new ContextVariables { ["varName"] = "foo" }; - var context = new SKContext(variables); - - // Act - var codeBlock = new CodeBlock("$varName", NullLogger.Instance); - var result = await codeBlock.RenderCodeAsync(context); - - // Assert - Assert.Equal("foo", result); - } - - [Fact] - public async Task ItRendersCodeBlockConsistingOfJustAVarBlock2Async() - { - // Arrange - var variables = new ContextVariables { ["varName"] = "bar" }; - var context = new SKContext(variables); - var varBlock = new VarBlock("$varName"); - - // Act - var codeBlock = new CodeBlock(new List { varBlock }, "", NullLogger.Instance); - var result = await codeBlock.RenderCodeAsync(context); - - // Assert - Assert.Equal("bar", result); - } - - [Fact] - public async Task ItRendersCodeBlockConsistingOfJustAValBlock1Async() - { - // Arrange - var context = new SKContext(); - - // Act - var codeBlock = new CodeBlock("'ciao'", NullLogger.Instance); - var result = await codeBlock.RenderCodeAsync(context); - - // Assert - Assert.Equal("ciao", result); - } - - [Fact] - public async Task ItRendersCodeBlockConsistingOfJustAValBlock2Async() - { - // Arrange - var context = new SKContext(); - var valBlock = new ValBlock("'arrivederci'"); - - // Act - var codeBlock = new CodeBlock(new List { valBlock }, "", NullLogger.Instance); - var result = await codeBlock.RenderCodeAsync(context); - - // Assert - Assert.Equal("arrivederci", result); - } - - [Fact] - public async Task ItInvokesFunctionCloningAllVariablesAsync() - { - // Arrange - const string Func = "funcName"; - - var variables = new ContextVariables { ["input"] = "zero", ["var1"] = "uno", ["var2"] = "due" }; - var context = new SKContext(variables, skills: this._skills.Object); - var funcId = new FunctionIdBlock(Func); - - var canary0 = string.Empty; - var canary1 = string.Empty; - var canary2 = string.Empty; - var function = new Mock(); - function - .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback((context, _, _) => - { - canary0 = context!.Variables["input"]; - canary1 = context.Variables["var1"]; - canary2 = context.Variables["var2"]; - - context.Variables["input"] = "overridden"; - context.Variables["var1"] = "overridden"; - context.Variables["var2"] = "overridden"; - }) - .ReturnsAsync((SKContext inputcontext, CompleteRequestSettings _, CancellationToken _) => inputcontext); - - ISKFunction? outFunc = function.Object; - this._skills.Setup(x => x.TryGetFunction(Func, out outFunc)).Returns(true); - this._skills.Setup(x => x.GetFunction(Func)).Returns(function.Object); - - // Act - var codeBlock = new CodeBlock(new List { funcId }, "", NullLogger.Instance); - string result = await codeBlock.RenderCodeAsync(context); - - // Assert - Values are received - Assert.Equal("zero", canary0); - Assert.Equal("uno", canary1); - Assert.Equal("due", canary2); - - // Assert - Original context is intact - Assert.Equal("zero", variables["input"]); - Assert.Equal("uno", variables["var1"]); - Assert.Equal("due", variables["var2"]); - } - - [Fact] - public async Task ItInvokesFunctionWithCustomVariableAsync() - { - // Arrange - const string Func = "funcName"; - const string Var = "varName"; - const string VarValue = "varValue"; - - var variables = new ContextVariables { [Var] = VarValue }; - var context = new SKContext(variables, skills: this._skills.Object); - var funcId = new FunctionIdBlock(Func); - var varBlock = new VarBlock($"${Var}"); - - var canary = string.Empty; - var function = new Mock(); - function - .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback((context, _, _) => - { - canary = context!.Variables["input"]; - }) - .ReturnsAsync((SKContext inputcontext, CompleteRequestSettings _, CancellationToken _) => inputcontext); - - ISKFunction? outFunc = function.Object; - this._skills.Setup(x => x.TryGetFunction(Func, out outFunc)).Returns(true); - this._skills.Setup(x => x.GetFunction(Func)).Returns(function.Object); - - // Act - var codeBlock = new CodeBlock(new List { funcId, varBlock }, "", NullLogger.Instance); - string result = await codeBlock.RenderCodeAsync(context); - - // Assert - Assert.Equal(VarValue, result); - Assert.Equal(VarValue, canary); - } - - [Fact] - public async Task ItInvokesFunctionWithCustomValueAsync() - { - // Arrange - const string Func = "funcName"; - const string Value = "value"; - - var context = new SKContext(skills: this._skills.Object); - var funcId = new FunctionIdBlock(Func); - var valBlock = new ValBlock($"'{Value}'"); - - var canary = string.Empty; - var function = new Mock(); - function - .Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback((context, _, _) => - { - canary = context!.Variables["input"]; - }) - .ReturnsAsync((SKContext inputcontext, CompleteRequestSettings _, CancellationToken _) => inputcontext); - - ISKFunction? outFunc = function.Object; - this._skills.Setup(x => x.TryGetFunction(Func, out outFunc)).Returns(true); - this._skills.Setup(x => x.GetFunction(Func)).Returns(function.Object); - - // Act - var codeBlock = new CodeBlock(new List { funcId, valBlock }, "", NullLogger.Instance); - string result = await codeBlock.RenderCodeAsync(context); - - // Assert - Assert.Equal(Value, result); - Assert.Equal(Value, canary); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/CodeTokenizerTests.cs b/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/CodeTokenizerTests.cs deleted file mode 100644 index 5490f09440b9..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/CodeTokenizerTests.cs +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; -using Xunit; - -namespace SemanticKernel.UnitTests.TemplateEngine; - -public class CodeTokenizerTests -{ - private readonly CodeTokenizer _target; - - public CodeTokenizerTests() - { - this._target = new CodeTokenizer(); - } - - [Fact] - public void ItParsesEmptyText() - { - // Act + Assert - Assert.Empty(this._target.Tokenize(null)); - Assert.Empty(this._target.Tokenize("")); - Assert.Empty(this._target.Tokenize(" ")); - Assert.Empty(this._target.Tokenize(" \n ")); - } - - [Theory] - [InlineData("$", "$")] - [InlineData(" $ ", "$")] - [InlineData("$foo", "$foo")] - [InlineData("$foo ", "$foo")] - [InlineData(" $foo", "$foo")] - [InlineData(" $bar ", "$bar")] - public void ItParsesVarBlocks(string template, string content) - { - // Act - var blocks = this._target.Tokenize(template); - - // Assert - Assert.Single(blocks); - Assert.Equal(content, blocks[0].Content); - Assert.Equal(BlockTypes.Variable, blocks[0].Type); - } - - [Theory] - [InlineData("'", "'")] - [InlineData(" \" ", "\"")] - [InlineData("'foo'", "'foo'")] - [InlineData("'foo' ", "'foo'")] - [InlineData(" 'foo'", "'foo'")] - [InlineData(" \"bar\" ", "\"bar\"")] - public void ItParsesValBlocks(string template, string content) - { - // Act - var blocks = this._target.Tokenize(template); - - // Assert - Assert.Single(blocks); - Assert.Equal(content, blocks[0].Content); - Assert.Equal(BlockTypes.Value, blocks[0].Type); - } - - [Theory] - [InlineData("f", "f")] - [InlineData(" x ", "x")] - [InlineData("foo", "foo")] - [InlineData("fo.o ", "fo.o")] - [InlineData(" f.oo", "f.oo")] - [InlineData(" bar ", "bar")] - public void ItParsesFunctionIdBlocks(string template, string content) - { - // Act - var blocks = this._target.Tokenize(template); - - // Assert - Assert.Single(blocks); - Assert.Equal(content, blocks[0].Content); - Assert.Equal(BlockTypes.FunctionId, blocks[0].Type); - } - - [Fact] - public void ItParsesFunctionCalls() - { - // Arrange - var template1 = "x.y $foo"; - var template2 = "xy $foo"; - var template3 = "xy '$value'"; - - // Act - var blocks1 = this._target.Tokenize(template1); - var blocks2 = this._target.Tokenize(template2); - var blocks3 = this._target.Tokenize(template3); - - // Assert - Assert.Equal(2, blocks1.Count); - Assert.Equal(2, blocks2.Count); - Assert.Equal(2, blocks3.Count); - - Assert.Equal("x.y", blocks1[0].Content); - Assert.Equal("xy", blocks2[0].Content); - Assert.Equal("xy", blocks3[0].Content); - - Assert.Equal(BlockTypes.FunctionId, blocks1[0].Type); - Assert.Equal(BlockTypes.FunctionId, blocks2[0].Type); - Assert.Equal(BlockTypes.FunctionId, blocks3[0].Type); - - Assert.Equal("$foo", blocks1[1].Content); - Assert.Equal("$foo", blocks2[1].Content); - Assert.Equal("'$value'", blocks3[1].Content); - - Assert.Equal(BlockTypes.Variable, blocks1[1].Type); - Assert.Equal(BlockTypes.Variable, blocks2[1].Type); - Assert.Equal(BlockTypes.Value, blocks3[1].Type); - } - - [Fact] - public void ItSupportsEscaping() - { - // Arrange - var template = "func 'f\\'oo'"; - - // Act - var blocks = this._target.Tokenize(template); - - // Assert - Assert.Equal(2, blocks.Count); - Assert.Equal("func", blocks[0].Content); - Assert.Equal("'f\'oo'", blocks[1].Content); - } - - [Fact] - public void ItThrowsWhenSeparatorsAreMissing() - { - // Arrange - var template1 = @"call 'f\\'xy'"; - var template2 = @"call 'f\\'x"; - - // Act - Assert.Throws(() => this._target.Tokenize(template1)); - Assert.Throws(() => this._target.Tokenize(template2)); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/PromptTemplateConfigTests.cs b/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/PromptTemplateConfigTests.cs new file mode 100644 index 000000000000..30461bebf4df --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/PromptTemplateConfigTests.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.TemplateEngine; +using Xunit; + +namespace SemanticKernel.UnitTests.TemplateEngine; + +public class PromptTemplateConfigTests +{ + [Fact] + public void DeserializingDoNotExpectChatSystemPromptToExist() + { + // Arrange + string configPayload = @"{ + ""max_tokens"": 60, + ""temperature"": 0.5, + ""top_p"": 0.0, + ""presence_penalty"": 0.0, + ""frequency_penalty"": 0.0 + }"; + + // Act + var requestSettings = JsonSerializer.Deserialize(configPayload); + + // Assert + Assert.NotNull(requestSettings); + Assert.NotNull(requestSettings.ChatSystemPrompt); + Assert.Equal("Assistant is a large language model.", requestSettings.ChatSystemPrompt); + } + + [Fact] + public void DeserializingExpectChatSystemPromptToExists() + { + // Arrange + string configPayload = @"{ + ""max_tokens"": 60, + ""temperature"": 0.5, + ""top_p"": 0.0, + ""presence_penalty"": 0.0, + ""frequency_penalty"": 0.0, + ""chat_system_prompt"": ""I am a prompt"" + }"; + + // Act + var requestSettings = JsonSerializer.Deserialize(configPayload); + + // Assert + Assert.NotNull(requestSettings); + Assert.NotNull(requestSettings.ChatSystemPrompt); + Assert.Equal("I am a prompt", requestSettings.ChatSystemPrompt); + } + + [Fact] + public void DeserializingExpectMultipleModels() + { + // Arrange + string configPayload = @" +{ + ""schema"": 1, + ""description"": """", + ""models"": + [ + { + ""model_id"": ""gpt-4"", + ""max_tokens"": 200, + ""temperature"": 0.2, + ""top_p"": 0.0, + ""presence_penalty"": 0.0, + ""frequency_penalty"": 0.0, + ""stop_sequences"": + [ + ""Human"", + ""AI"" + ] + }, + { + ""model_id"": ""gpt-3.5_turbo"", + ""max_tokens"": 256, + ""temperature"": 0.3, + ""top_p"": 0.0, + ""presence_penalty"": 0.0, + ""frequency_penalty"": 0.0, + ""stop_sequences"": + [ + ""Human"", + ""AI"" + ] + } + ] +} + "; + + // Act + var promptTemplateConfig = JsonSerializer.Deserialize(configPayload); + + // Assert + Assert.NotNull(promptTemplateConfig); + Assert.NotNull(promptTemplateConfig.ModelSettings); + Assert.Equal(2, promptTemplateConfig.ModelSettings.Count); + } + + [Fact] + public void DeserializingExpectCompletion() + { + // Arrange + string configPayload = @" +{ + ""schema"": 1, + ""description"": """", + ""models"": + [ + { + ""model_id"": ""gpt-4"", + ""max_tokens"": 200, + ""temperature"": 0.2, + ""top_p"": 0.0, + ""presence_penalty"": 0.0, + ""frequency_penalty"": 0.0, + ""stop_sequences"": + [ + ""Human"", + ""AI"" + ] + } + ] +} + "; + + // Act + var promptTemplateConfig = JsonSerializer.Deserialize(configPayload); + + // Assert + Assert.NotNull(promptTemplateConfig); +#pragma warning disable CS0618 // Ensure backward compatibility + Assert.NotNull(promptTemplateConfig.Completion); + Assert.Equal("gpt-4", promptTemplateConfig.Completion.ModelId); +#pragma warning restore CS0618 // Ensure backward compatibility + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/PromptTemplateEngineTests.cs b/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/PromptTemplateEngineTests.cs deleted file mode 100644 index 255d8a570c20..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/PromptTemplateEngineTests.cs +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; -using Moq; -using SemanticKernel.UnitTests.XunitHelpers; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.UnitTests.TemplateEngine; - -public sealed class PromptTemplateEngineTests -{ - private readonly PromptTemplateEngine _target; - private readonly ContextVariables _variables; - private readonly Mock _skills; - private readonly ITestOutputHelper _logger; - - public PromptTemplateEngineTests(ITestOutputHelper testOutputHelper) - { - this._logger = testOutputHelper; - this._target = new PromptTemplateEngine(TestConsoleLogger.Logger); - this._variables = new ContextVariables(Guid.NewGuid().ToString("X")); - this._skills = new Mock(); - } - - [Fact] - public void ItRendersVariables() - { - // Arrange - var template = "{$x11} This {$a} is {$_a} a {{$x11}} test {{$x11}} " + - "template {{foo}}{{bar $a}}{{baz $_a}}{{yay $x11}}"; - - // Act - var blocks = this._target.ExtractBlocks(template); - var updatedBlocks = this._target.RenderVariables(blocks, this._variables); - - // Assert - Assert.Equal(9, blocks.Count); - Assert.Equal(9, updatedBlocks.Count); - - Assert.Equal("$x11", blocks[1].Content); - Assert.Equal("", updatedBlocks[1].Content); - Assert.Equal(BlockTypes.Variable, blocks[1].Type); - Assert.Equal(BlockTypes.Text, updatedBlocks[1].Type); - - Assert.Equal("$x11", blocks[3].Content); - Assert.Equal("", updatedBlocks[3].Content); - Assert.Equal(BlockTypes.Variable, blocks[3].Type); - Assert.Equal(BlockTypes.Text, updatedBlocks[3].Type); - - Assert.Equal("foo", blocks[5].Content); - Assert.Equal("foo", updatedBlocks[5].Content); - Assert.Equal(BlockTypes.Code, blocks[5].Type); - Assert.Equal(BlockTypes.Code, updatedBlocks[5].Type); - - Assert.Equal("bar $a", blocks[6].Content); - Assert.Equal("bar $a", updatedBlocks[6].Content); - Assert.Equal(BlockTypes.Code, blocks[6].Type); - Assert.Equal(BlockTypes.Code, updatedBlocks[6].Type); - - Assert.Equal("baz $_a", blocks[7].Content); - Assert.Equal("baz $_a", updatedBlocks[7].Content); - Assert.Equal(BlockTypes.Code, blocks[7].Type); - Assert.Equal(BlockTypes.Code, updatedBlocks[7].Type); - - Assert.Equal("yay $x11", blocks[8].Content); - Assert.Equal("yay $x11", updatedBlocks[8].Content); - Assert.Equal(BlockTypes.Code, blocks[8].Type); - Assert.Equal(BlockTypes.Code, updatedBlocks[8].Type); - - // Arrange - this._variables.Set("x11", "x11 value"); - this._variables.Set("a", "a value"); - this._variables.Set("_a", "_a value"); - - // Act - blocks = this._target.ExtractBlocks(template); - updatedBlocks = this._target.RenderVariables(blocks, this._variables); - - // Assert - Assert.Equal(9, blocks.Count); - Assert.Equal(9, updatedBlocks.Count); - - Assert.Equal("$x11", blocks[1].Content); - Assert.Equal("x11 value", updatedBlocks[1].Content); - Assert.Equal(BlockTypes.Variable, blocks[1].Type); - Assert.Equal(BlockTypes.Text, updatedBlocks[1].Type); - - Assert.Equal("$x11", blocks[3].Content); - Assert.Equal("x11 value", updatedBlocks[3].Content); - Assert.Equal(BlockTypes.Variable, blocks[3].Type); - Assert.Equal(BlockTypes.Text, updatedBlocks[3].Type); - - Assert.Equal("foo", blocks[5].Content); - Assert.Equal("foo", updatedBlocks[5].Content); - Assert.Equal(BlockTypes.Code, blocks[5].Type); - Assert.Equal(BlockTypes.Code, updatedBlocks[5].Type); - - Assert.Equal("bar $a", blocks[6].Content); - Assert.Equal("bar $a", updatedBlocks[6].Content); - Assert.Equal(BlockTypes.Code, blocks[6].Type); - Assert.Equal(BlockTypes.Code, updatedBlocks[6].Type); - - Assert.Equal("baz $_a", blocks[7].Content); - Assert.Equal("baz $_a", updatedBlocks[7].Content); - Assert.Equal(BlockTypes.Code, blocks[7].Type); - Assert.Equal(BlockTypes.Code, updatedBlocks[7].Type); - - Assert.Equal("yay $x11", blocks[8].Content); - Assert.Equal("yay $x11", updatedBlocks[8].Content); - Assert.Equal(BlockTypes.Code, blocks[8].Type); - Assert.Equal(BlockTypes.Code, updatedBlocks[8].Type); - } - - [Fact] - public async Task ItRendersCodeUsingInputAsync() - { - // Arrange - string MyFunctionAsync(SKContext context) - { - this._logger.WriteLine("MyFunction call received, input: {0}", context.Variables.Input); - return $"F({context.Variables.Input})"; - } - - ISKFunction func = SKFunction.FromNativeMethod(Method(MyFunctionAsync), this); - Assert.NotNull(func); - - this._variables.Update("INPUT-BAR"); - var template = "foo-{{function}}-baz"; - { - ISKFunction? outFunc = func; - this._skills.Setup(x => x.TryGetFunction("function", out outFunc)).Returns(true); - } - this._skills.Setup(x => x.GetFunction("function")).Returns(func); - var context = this.MockContext(); - - // Act - var result = await this._target.RenderAsync(template, context); - - // Assert - Assert.Equal("foo-F(INPUT-BAR)-baz", result); - } - - [Fact] - public async Task ItRendersCodeUsingVariablesAsync() - { - // Arrange - string MyFunctionAsync(SKContext context) - { - this._logger.WriteLine("MyFunction call received, input: {0}", context.Variables.Input); - return $"F({context.Variables.Input})"; - } - - ISKFunction func = SKFunction.FromNativeMethod(Method(MyFunctionAsync), this); - Assert.NotNull(func); - - this._variables.Set("myVar", "BAR"); - var template = "foo-{{function $myVar}}-baz"; - { - ISKFunction? outFunc = func; - this._skills.Setup(x => x.TryGetFunction("function", out outFunc)).Returns(true); - } - this._skills.Setup(x => x.GetFunction("function")).Returns(func); - var context = this.MockContext(); - - // Act - var result = await this._target.RenderAsync(template, context); - - // Assert - Assert.Equal("foo-F(BAR)-baz", result); - } - - [Fact] - public async Task ItRendersAsyncCodeUsingVariablesAsync() - { - // Arrange - Task MyFunctionAsync(SKContext context) - { - // Input value should be "BAR" because the variable $myVar is passed in - this._logger.WriteLine("MyFunction call received, input: {0}", context.Variables.Input); - return Task.FromResult(context.Variables.Input); - } - - ISKFunction func = SKFunction.FromNativeMethod(Method(MyFunctionAsync), this); - Assert.NotNull(func); - - this._variables.Set("myVar", "BAR"); - - var template = "foo-{{function $myVar}}-baz"; - { - ISKFunction? outFunc = func; - this._skills.Setup(x => x.TryGetFunction("function", out outFunc)).Returns(true); - } - this._skills.Setup(x => x.GetFunction("function")).Returns(func); - var context = this.MockContext(); - - // Act - var result = await this._target.RenderAsync(template, context); - - // Assert - Assert.Equal("foo-BAR-baz", result); - } - - [Fact] - public async Task ItRendersAsyncCodeUsingImmutableVariablesAsync() - { - // Arrange - var template = "{{func1}} {{func2}} {{func3 $myVar}}"; - this._variables.Update("BAR"); - this._variables.Set("myVar", "BAZ"); - - string MyFunction1Async(SKContext context) - { - this._logger.WriteLine("MyFunction1 call received, input: {0}", context.Variables.Input); - context.Variables.Update("foo"); - return "F(OUTPUT-FOO)"; - } - string MyFunction2Async(SKContext context) - { - // Input value should be "BAR" because the variable $input is immutable in MyFunction1 - this._logger.WriteLine("MyFunction2 call received, input: {0}", context.Variables.Input); - context.Variables.Set("myVar", "bar"); - return context.Variables.Input; - } - string MyFunction3Async(SKContext context) - { - // Input value should be "BAZ" because the variable $myVar is immutable in MyFunction2 - this._logger.WriteLine("MyFunction3 call received, input: {0}", context.Variables.Input); - return context.Variables.TryGetValue("myVar", out string? value) ? value : ""; - } - - var functions = new List() - { - SKFunction.FromNativeMethod(Method(MyFunction1Async), this, "func1"), - SKFunction.FromNativeMethod(Method(MyFunction2Async), this, "func2"), - SKFunction.FromNativeMethod(Method(MyFunction3Async), this, "func3") - }; - - foreach (var func in functions) - { - Assert.NotNull(func); - ISKFunction? outFunc = func; - this._skills.Setup(x => x.GetFunction(It.Is(s => s == func.SkillName))).Returns(func); - this._skills.Setup(x => x.TryGetFunction(It.Is(s => s == func.SkillName), out outFunc)).Returns(true); - } - - // Act - var result = await this._target.RenderAsync(template, this.MockContext()); - - // Assert - Assert.Equal("F(OUTPUT-FOO) BAR BAZ", result); - } - - private static MethodInfo Method(Delegate method) - { - return method.Method; - } - - private SKContext MockContext() - { - return new SKContext( - this._variables, - skills: this._skills.Object, - logger: TestConsoleLogger.Logger); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/TemplateExceptionTests.cs b/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/TemplateExceptionTests.cs deleted file mode 100644 index 48c1fc1fc292..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/TemplateEngine/TemplateExceptionTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.TemplateEngine; -using Xunit; - -namespace SemanticKernel.UnitTests.TemplateEngine; - -public class TemplateExceptionTests -{ - [Fact] - public void ItRoundtripsArgsToErrorCodeCtor() - { - // Arrange - var e = new TemplateException(TemplateException.ErrorCodes.RuntimeError); - - // Assert - Assert.Equal(TemplateException.ErrorCodes.RuntimeError, e.ErrorCode); - Assert.Contains("Runtime error", e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageCtor() - { - // Arrange - const string Message = "this is a test"; - var e = new TemplateException(TemplateException.ErrorCodes.RuntimeError, Message); - - // Assert - Assert.Equal(TemplateException.ErrorCodes.RuntimeError, e.ErrorCode); - Assert.Contains("Runtime error", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } - - [Fact] - public void ItRoundtripsArgsToErrorCodeMessageExceptionCtor() - { - // Arrange - const string Message = "this is a test"; - var inner = new FormatException(); - var e = new TemplateException(TemplateException.ErrorCodes.RuntimeError, Message, inner); - - // Assert - Assert.Equal(TemplateException.ErrorCodes.RuntimeError, e.ErrorCode); - Assert.Contains("Runtime error", e.Message, StringComparison.Ordinal); - Assert.Contains(Message, e.Message, StringComparison.Ordinal); - Assert.Same(inner, e.InnerException); - } - - [Fact] - public void ItAllowsNullMessageAndInnerExceptionInCtors() - { - // Arrange - var e = new TemplateException(TemplateException.ErrorCodes.RuntimeError, null, null); - - // Assert - Assert.Equal(TemplateException.ErrorCodes.RuntimeError, e.ErrorCode); - Assert.Contains("Runtime error", e.Message, StringComparison.Ordinal); - Assert.Null(e.InnerException); - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerTests.cs b/dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerTests.cs index ecac27fa740c..e14b69de1aa3 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerTests.cs @@ -478,4 +478,303 @@ public void CanSplitVeryLargeDocumentsWithoutStackOverflowing() Assert.NotEmpty(paragraphs); #pragma warning restore CA5394 } + + [Fact] + public void CanSplitPlainTextLinesWithCustomTokenCounter() + { + const string Input = "This is a test of the emergency broadcast system. This is only a test."; + var expected = new[] + { + "This is a test of the emergency broadcast system.", + "This is only a test." + }; + + var result = TextChunker.SplitPlainTextLines(Input, 60, (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitMarkdownParagraphsWithCustomTokenCounter() + { + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + var expected = new[] + { + "This is a test of the emergency broadcast system.", + "This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var result = TextChunker.SplitMarkdownParagraphs(input, 52, tokenCounter: (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitMarkdownParagraphsWithOverlapAndCustomTokenCounter() + { + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + "This is a test of the emergency broadcast system.", + "emergency broadcast system. This is only a test.", + "This is only a test. We repeat, this is only a test.", + "We repeat, this is only a test. A unit test.", + "A unit test." + }; + + var result = TextChunker.SplitMarkdownParagraphs(input, 75, 40, tokenCounter: (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitTextParagraphsWithCustomTokenCounter() + { + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + "This is a test of the emergency broadcast system.", + "This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var result = TextChunker.SplitPlainTextParagraphs(input, 52, tokenCounter: (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitTextParagraphsWithOverlapAndCustomTokenCounter() + { + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + "This is a test of the emergency broadcast system.", + "emergency broadcast system. This is only a test.", + "This is only a test. We repeat, this is only a test.", + "We repeat, this is only a test. A unit test.", + "A unit test." + }; + + var result = TextChunker.SplitPlainTextParagraphs(input, 75, 40, tokenCounter: (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitMarkDownLinesWithCustomTokenCounter() + { + const string Input = "This is a test of the emergency broadcast system. This is only a test."; + var expected = new[] + { + "This is a test of the emergency broadcast system.", + "This is only a test." + }; + + var result = TextChunker.SplitMarkDownLines(Input, 60, (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitMarkdownParagraphsWithHeader() + { + const string ChunkHeader = "DOCUMENT NAME: test.txt\n\n"; + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + var expected = new[] + { + $"{ChunkHeader}This is a test of the emergency broadcast system.", + $"{ChunkHeader}This is only a test.", + $"{ChunkHeader}We repeat, this is only a test. A unit test." + }; + + var result = TextChunker.SplitMarkdownParagraphs(input, 20, chunkHeader: ChunkHeader); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitMarkdownParagraphsWithOverlapAndHeader() + { + const string ChunkHeader = "DOCUMENT NAME: test.txt\n\n"; + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + $"{ChunkHeader}This is a test of the emergency broadcast system.", + $"{ChunkHeader}emergency broadcast system. This is only a test.", + $"{ChunkHeader}This is only a test. We repeat, this is only a test.", + $"{ChunkHeader}We repeat, this is only a test. A unit test.", + $"{ChunkHeader}A unit test." + }; + + var result = TextChunker.SplitMarkdownParagraphs(input, 22, 8, chunkHeader: ChunkHeader); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitTextParagraphsWithHeader() + { + const string ChunkHeader = "DOCUMENT NAME: test.txt\n\n"; + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + $"{ChunkHeader}This is a test of the emergency broadcast system.", + $"{ChunkHeader}This is only a test.", + $"{ChunkHeader}We repeat, this is only a test. A unit test." + }; + + var result = TextChunker.SplitPlainTextParagraphs(input, 20, chunkHeader: ChunkHeader); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitTextParagraphsWithOverlapAndHeader() + { + const string ChunkHeader = "DOCUMENT NAME: test.txt\n\n"; + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + $"{ChunkHeader}This is a test of the emergency broadcast system.", + $"{ChunkHeader}emergency broadcast system. This is only a test.", + $"{ChunkHeader}This is only a test. We repeat, this is only a test.", + $"{ChunkHeader}We repeat, this is only a test. A unit test.", + $"{ChunkHeader}A unit test." + }; + + var result = TextChunker.SplitPlainTextParagraphs(input, 22, 8, chunkHeader: ChunkHeader); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitMarkdownParagraphsWithHeaderAndCustomTokenCounter() + { + const string ChunkHeader = "DOCUMENT NAME: test.txt\n\n"; + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + var expected = new[] + { + $"{ChunkHeader}This is a test of the emergency broadcast system.", + $"{ChunkHeader}This is only a test.", + $"{ChunkHeader}We repeat, this is only a test. A unit test." + }; + + var result = TextChunker.SplitMarkdownParagraphs(input, 77, chunkHeader: ChunkHeader, tokenCounter: (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitMarkdownParagraphsWithOverlapAndHeaderAndCustomTokenCounter() + { + const string ChunkHeader = "DOCUMENT NAME: test.txt\n\n"; + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + $"{ChunkHeader}This is a test of the emergency broadcast system.", + $"{ChunkHeader}emergency broadcast system. This is only a test.", + $"{ChunkHeader}This is only a test. We repeat, this is only a test.", + $"{ChunkHeader}We repeat, this is only a test. A unit test.", + $"{ChunkHeader}A unit test." + }; + + var result = TextChunker.SplitMarkdownParagraphs(input, 100, 40, chunkHeader: ChunkHeader, tokenCounter: (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitTextParagraphsWithHeaderAndCustomTokenCounter() + { + const string ChunkHeader = "DOCUMENT NAME: test.txt\n\n"; + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + $"{ChunkHeader}This is a test of the emergency broadcast system.", + $"{ChunkHeader}This is only a test.", + $"{ChunkHeader}We repeat, this is only a test. A unit test." + }; + + var result = TextChunker.SplitPlainTextParagraphs(input, 77, chunkHeader: ChunkHeader, tokenCounter: (input) => input.Length); + + Assert.Equal(expected, result); + } + + [Fact] + public void CanSplitTextParagraphsWithOverlapAndHeaderAndCustomTokenCounter() + { + const string ChunkHeader = "DOCUMENT NAME: test.txt\n\n"; + List input = new() + { + "This is a test of the emergency broadcast system. This is only a test.", + "We repeat, this is only a test. A unit test." + }; + + var expected = new[] + { + $"{ChunkHeader}This is a test of the emergency broadcast system.", + $"{ChunkHeader}emergency broadcast system. This is only a test.", + $"{ChunkHeader}This is only a test. We repeat, this is only a test.", + $"{ChunkHeader}We repeat, this is only a test. A unit test.", + $"{ChunkHeader}A unit test." + }; + + var result = TextChunker.SplitPlainTextParagraphs(input, 100, 40, chunkHeader: ChunkHeader, tokenCounter: (input) => input.Length); + + Assert.Equal(expected, result); + } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Utilities/HttpClientExtensionsTests.cs b/dotnet/src/SemanticKernel.UnitTests/Utilities/HttpClientExtensionsTests.cs new file mode 100644 index 000000000000..250ef3e19c5c --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Utilities/HttpClientExtensionsTests.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; +using System.Net.Http; +using System.Net.Mime; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Xunit; + +namespace SemanticKernel.UnitTests.Utilities; + +public sealed class HttpClientExtensionsTests : IDisposable +{ + /// + /// An instance of HttpMessageHandlerStub class used to get access to various properties of HttpRequestMessage sent by HTTP client. + /// + private readonly HttpMessageHandlerStub _httpMessageHandlerStub; + + /// + /// An instance of HttpClient class used by the tests. + /// + private readonly HttpClient _httpClient; + + /// + /// Creates an instance of a class. + /// + public HttpClientExtensionsTests() + { + this._httpMessageHandlerStub = new HttpMessageHandlerStub(); + + this._httpClient = new HttpClient(this._httpMessageHandlerStub); + } + + [Fact] + public async Task ShouldReturnHttpResponseForSuccessfulRequestAsync() + { + //Arrange + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, "https://fake-random-test-host"); + + //Act + using var responseMessage = await this._httpClient.SendWithSuccessCheckAsync(requestMessage, CancellationToken.None); + + //Assert + Assert.NotNull(responseMessage); + + Assert.Equal(HttpMethod.Get, this._httpMessageHandlerStub.Method); + + Assert.NotNull(this._httpMessageHandlerStub.ResponseToReturn); + Assert.Equal(System.Net.HttpStatusCode.OK, this._httpMessageHandlerStub.ResponseToReturn.StatusCode); + } + + [Fact] + public async Task ShouldThrowHttpOperationExceptionForFailedRequestAsync() + { + //Arrange + this._httpMessageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError); + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("{\"details\": \"fake-response-content\"}", Encoding.UTF8, MediaTypeNames.Application.Json); + + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, "https://fake-random-test-host"); + + //Act + var exception = await Assert.ThrowsAsync(() => this._httpClient.SendWithSuccessCheckAsync(requestMessage, CancellationToken.None)); + + //Assert + Assert.NotNull(exception); + + Assert.Equal(HttpStatusCode.InternalServerError, exception.StatusCode); + + Assert.Equal("Response status code does not indicate success: 500 (Internal Server Error).", exception.Message); + + Assert.Equal("{\"details\": \"fake-response-content\"}", exception.ResponseContent); + + Assert.True(exception.InnerException is HttpRequestException); + } + + /// + /// Disposes resources used by this class. + /// + public void Dispose() + { + this._httpMessageHandlerStub.Dispose(); + + this._httpClient.Dispose(); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Utilities/HttpContentExtensionsTests.cs b/dotnet/src/SemanticKernel.UnitTests/Utilities/HttpContentExtensionsTests.cs new file mode 100644 index 000000000000..433643a34c39 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Utilities/HttpContentExtensionsTests.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Net.Http; +using System.Net.Mime; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace SemanticKernel.UnitTests.Utilities; + +public sealed class HttpContentExtensionsTests : IDisposable +{ + /// + /// An instance of HttpMessageHandlerStub class used to get access to various properties of HttpRequestMessage sent by HTTP client. + /// + private readonly HttpMessageHandlerStub _httpMessageHandlerStub; + + /// + /// An instance of HttpClient class used by the tests. + /// + private readonly HttpClient _httpClient; + + /// + /// Creates an instance of a class. + /// + public HttpContentExtensionsTests() + { + this._httpMessageHandlerStub = new HttpMessageHandlerStub(); + + this._httpClient = new HttpClient(this._httpMessageHandlerStub); + } + + [Fact] + public async Task ShouldReturnHttpContentAsStringAsync() + { + //Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("{\"details\": \"fake-response-content\"}", Encoding.UTF8, MediaTypeNames.Application.Json); + + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, "https://fake-random-test-host"); + + using var responseMessage = await this._httpClient.SendAsync(requestMessage, CancellationToken.None); + + //Act + var result = await responseMessage.Content.ReadAsStringWithExceptionMappingAsync(); + + //Assert + Assert.False(string.IsNullOrEmpty(result)); + + Assert.Equal("{\"details\": \"fake-response-content\"}", result); + } + + [Fact] + public async Task ShouldReturnHttpContentAsStreamAsync() + { + //Arrange + using var expectedStream = new MemoryStream(Encoding.Default.GetBytes("{\"details\": \"fake-response-content\"}")); + + this._httpMessageHandlerStub.ResponseToReturn.Content = new StreamContent(expectedStream); + + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, "https://fake-random-test-host"); + + using var responseMessage = await this._httpClient.SendAsync(requestMessage, CancellationToken.None); + + //Act + var actualStream = await responseMessage.Content.ReadAsStreamAndTranslateExceptionAsync(); + + //Assert + Assert.NotNull(actualStream); + + using var streamReader = new StreamReader(actualStream); + var content = await streamReader.ReadToEndAsync(); + Assert.Equal("{\"details\": \"fake-response-content\"}", content); + } + + [Fact] + public async Task ShouldReturnHttpContentAsByteArrayAsync() + { + //Arrange + this._httpMessageHandlerStub.ResponseToReturn.Content = new ByteArrayContent(new byte[] { 1, 2, 3 }); + + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, "https://fake-random-test-host"); + + using var responseMessage = await this._httpClient.SendAsync(requestMessage, CancellationToken.None); + + //Act + var bytes = await responseMessage.Content.ReadAsByteArrayAsync(); + + //Assert + Assert.NotNull(bytes); + + Assert.Equal(new byte[] { 1, 2, 3 }, bytes); + } + + /// + /// Disposes resources used by this class. + /// + public void Dispose() + { + this._httpMessageHandlerStub.Dispose(); + + this._httpClient.Dispose(); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/VectorOperations/VectorOperationTests.cs b/dotnet/src/SemanticKernel.UnitTests/VectorOperations/VectorOperationTests.cs deleted file mode 100644 index 30e970f11d24..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/VectorOperations/VectorOperationTests.cs +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; -using Xunit; - -namespace SemanticKernel.UnitTests.VectorOperations; - -public class VectorOperationTests -{ - private readonly float[] _floatV1 = new float[] { 1.0F, 2.0F, -4.0F, 10.0F }; - private readonly float[] _floatV2 = new float[] { 3.0F, -7.0F, 1.0F, 6.0F }; - - private readonly double[] _doubleV1 = new double[] { 1.0, 2.0, -4.0, 10.0 }; - private readonly double[] _doubleV2 = new double[] { 3.0, -7.0, 1.0, 6.0 }; - - [Fact] - public void ItOnlySupportsFPDataTypes() - { - // Arrange - var target = Embedding.SupportedTypes; - - // Assert - Assert.Equal(2, target.Count()); - Assert.Contains(typeof(float), target); - Assert.Contains(typeof(double), target); - } - - [Fact] - public void ItComputesCosineSimilarityFloat() - { - // Arrange - var target = this._floatV1.CosineSimilarity(this._floatV2); - - // Assert - Assert.Equal(0.41971841676, target, 5); - } - - [Fact] - public void ItComputesCosineSimilarityDouble() - { - // Arrange - var target = this._doubleV1.CosineSimilarity(this._doubleV2); - - // Assert - Assert.Equal(0.41971841676, target, 5); - } - - [Fact] - public void ItThrowsOnCosineSimilarityWithDifferentLengthVectorsFP() - { - // Arrange - var shortVector = new float[] { -1.0F, 4.0F }; - - // Assert - Assert.Throws(() => shortVector.CosineSimilarity(this._floatV2)); - } - - [Fact] - public void ItThrowsOnCosineSimilarityWithDifferentLengthVectorsDouble() - { - // Arrange - var shortVector = new double[] { -1.0, 4.0 }; - - // Assert - Assert.Throws(() => shortVector.CosineSimilarity(this._doubleV2)); - } - - [Fact] - public void ItComputesEuclideanLengthFloat() - { - // Arrange - var target = this._floatV1.EuclideanLength(); - - // Assert - Assert.Equal(11.0, target, 5); - } - - [Fact] - public void ItComputesEuclideanLengthDouble() - { - // Arrange - var target = this._doubleV1.EuclideanLength(); - - // Assert - Assert.Equal(11.0, target, 5); - } - - [Theory] - [InlineData(0, 0)] - [InlineData(1, 3.0)] - [InlineData(2, -11.0)] - [InlineData(3, -15.0)] - [InlineData(4, 45.0)] - public void ItComputesDotProductFloat(int length, double expectedResult) - { - // Arrange - var target = this._floatV1.AsSpan(0, length).DotProduct(this._floatV2.AsSpan(0, length)); - - // Assert - Assert.Equal(expectedResult, target, 5); - } - - [Theory] - [InlineData(0, 0)] - [InlineData(1, 3.0)] - [InlineData(2, -11.0)] - [InlineData(3, -15.0)] - [InlineData(4, 45.0)] - public void ItComputesDotProductDouble(int length, double expectedResult) - { - // Arrange - var target = this._doubleV1.AsSpan(0, length).DotProduct(this._doubleV2.AsSpan(0, length)); - - // Assert - Assert.Equal(expectedResult, target, 5); - } - - [Fact] - public void ItNormalizesInPlaceFloat() - { - // Arrange - var target = this._floatV1; - target.NormalizeInPlace(); - var expected = new float[] { 0.09090909F, 0.18181819F, -0.3636364F, 0.90909094F }; - - // Assert - Assert.Equal(expected.Length, target.Length); - for (int i = 0; i < expected.Length; i++) - { - Assert.Equal(expected[i], target[i], .00001F); - } - } - - [Fact] - public void ItNormalizesInPlaceDouble() - { - // Arrange - var target = this._doubleV1; - target.NormalizeInPlace(); - var expected = new double[] { 0.09090909, 0.18181819, -0.3636364, 0.90909094 }; - - // Assert - Assert.Equal(expected.Length, target.Length); - for (int i = 0; i < expected.Length; i++) - { - Assert.Equal(expected[i], target[i], .00001); - } - } - - [Fact] - public void ItMultipliesInPlaceFloat() - { - // Arrange - var target = this._floatV1; - target.MultiplyByInPlace(2); - var expected = new float[] { 2.0F, 4.0F, -8.0F, 20.0F }; - - // Assert - Assert.Equal(expected.Length, target.Length); - for (int i = 0; i < expected.Length; i++) - { - Assert.Equal(expected[i], target[i], .00001F); - } - } - - [Fact] - public void ItMultipliesInPlaceDouble() - { - // Arrange - var target = this._doubleV1; - target.MultiplyByInPlace(2); - var expected = new double[] { 2.0, 4.0, -8.0, 20.0 }; - - // Assert - Assert.Equal(expected.Length, target.Length); - for (int i = 0; i < expected.Length; i++) - { - Assert.Equal(expected[i], target[i], .00001); - } - } - - [Fact] - public void ItDividesInPlaceFloat() - { - // Arrange - var target = this._floatV1; - target.DivideByInPlace(2); - var expected = new float[] { 0.5F, 1.0F, -2.0F, 5.0F }; - - // Assert - Assert.Equal(expected.Length, target.Length); - for (int i = 0; i < expected.Length; i++) - { - Assert.Equal(expected[i], target[i], .00001F); - } - } - - [Fact] - public void ItDividesInPlaceDouble() - { - // Arrange - var target = this._doubleV1; - target.DivideByInPlace(2); - var expected = new double[] { 0.5, 1.0, -2.0, 5.0 }; - - // Assert - Assert.Equal(expected.Length, target.Length); - for (int i = 0; i < expected.Length; i++) - { - Assert.Equal(expected[i], target[i], .00001); - } - } - - [Fact] - public void ItProducesExpectedCosineSimilarityResultsFloat() - { - // Arrange - var vectorList = new List(); - var comparisonVector = new float[] { 1.0F, 1.0F, 1.0F, 1.0F }; - vectorList.Add(new float[] { 1.0F, 1.0F, 1.0F, 1.0F }); // identical - vectorList.Add(new float[] { 1.0F, 1.0F, 1.0F, 2.0F }); - vectorList.Add(new float[] { 1.0F, 1.0F, -1.0F, -1.0F }); - vectorList.Add(new float[] { -1.0F, -1.0F, -1.0F, -1.0F }); // least similar - - // Act - var target = vectorList.Select(x => x.CosineSimilarity(comparisonVector)).ToArray(); - - // Assert - Assert.Equal(1.0, target[0]); // identical vectors results in similarity of 1 - Assert.True(target[0] > target[1]); - Assert.True(target[1] > target[2]); - Assert.True(target[2] > target[3]); - Assert.Equal(-1.0, target[3]); // opposing vectors results in similarity of -1 - } - - [Fact] - public void ItProducesExpectedCosineSimilarityResultsDouble() - { - // Arrange - var vectorList = new List(); - var comparisonVector = new double[] { 1.0, 1.0, 1.0, 1.0 }; - vectorList.Add(new double[] { 1.0, 1.0, 1.0, 1.0 }); // identical - vectorList.Add(new double[] { 1.0, 1.0, 1.0, 2.0 }); - vectorList.Add(new double[] { 1.0, 1.0, -1.0, -1.0 }); - vectorList.Add(new double[] { -1.0, -1.0, -1.0, -1.0 }); // least similar - - // Act - var target = vectorList.Select(x => x.CosineSimilarity(comparisonVector)).ToArray(); - - // Assert - Assert.Equal(1.0, target[0]); // identical vectors results in similarity of 1 - Assert.True(target[0] > target[1]); - Assert.True(target[1] > target[2]); - Assert.True(target[2] > target[3]); - Assert.Equal(-1.0, target[3]); // opposing vectors results in similarity of -1 - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/VectorOperations/VectorSpanTests.cs b/dotnet/src/SemanticKernel.UnitTests/VectorOperations/VectorSpanTests.cs deleted file mode 100644 index db829cb6cb98..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/VectorOperations/VectorSpanTests.cs +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.AI.Embeddings; -using Xunit; - -namespace SemanticKernel.UnitTests.VectorOperations; - -public class VectorSpanTests -{ - private readonly float[] _floatV1 = new float[] { 1.0F, 2.0F, -4.0F, 10.0F }; - private readonly float[] _floatV2 = new float[] { 3.0F, -7.0F, 1.0F, 6.0F }; - - private readonly double[] _doubleV1 = new double[] { 1.0, 2.0, -4.0, 10.0 }; - private readonly double[] _doubleV2 = new double[] { 3.0, -7.0, 1.0, 6.0 }; - - [Fact] - public void ItOnlySupportsFPDataTypes() - { - // Assert - TestTypeUnmanaged(true); - TestTypeUnmanaged(true); - - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestTypeUnmanaged(false); - TestType(false); - TestType(false); - - static void TestType(bool expected) - { - if (expected) - { - Assert.True(Embedding.IsSupported()); - Assert.True(Embedding.IsSupported(typeof(T))); - Assert.Contains(typeof(T), Embedding.SupportedTypes); - } - else - { - Assert.False(Embedding.IsSupported()); - Assert.False(Embedding.IsSupported(typeof(T))); - Assert.DoesNotContain(typeof(T), Embedding.SupportedTypes); - } - } - - static void TestTypeUnmanaged(bool expected) where T : unmanaged - { - TestType(expected); - if (expected) - { - _ = new Embedding(Array.Empty()); - _ = new Embedding(Array.Empty(), transferOwnership: true); - _ = new EmbeddingSpan(Array.Empty()); - _ = new EmbeddingReadOnlySpan(Array.Empty()); - } - else - { - Assert.False(Embedding.IsSupported()); - Assert.False(Embedding.IsSupported(typeof(T))); - Assert.DoesNotContain(typeof(T), Embedding.SupportedTypes); - Assert.Throws(() => new Embedding(Array.Empty())); - Assert.Throws(() => new Embedding(Array.Empty(), transferOwnership: true)); - Assert.Throws(() => new EmbeddingSpan(Array.Empty())); - Assert.Throws(() => new EmbeddingReadOnlySpan(Array.Empty())); - } - } - } - - [Fact] - public void ItCanComputeCosineSimilarityFloats() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._floatV1); - var vSpan2 = new EmbeddingSpan(this._floatV2); - var target = vSpan1.CosineSimilarity(vSpan2); - - // Assert - Assert.Equal(0.41971841676, target, 5); - } - - [Fact] - public void ItCanComputeCosineSimilarityDouble() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._doubleV1); - var vSpan2 = new EmbeddingSpan(this._doubleV2); - var target = vSpan1.CosineSimilarity(vSpan2); - - // Assert - Assert.Equal(0.41971841676, target, 5); - } - - [Fact] - public void ItThrowsOnCosineSimilarityWithDifferentLengthVectorsFloat() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._floatV1); - var vSpan2 = new EmbeddingSpan(new float[] { -1.0F, 4.0F }); - - // Assert - try - { - vSpan1.CosineSimilarity(vSpan2); - Assert.True(false, "No exception thrown"); - } - catch (ArgumentException target) - { - Assert.IsType(target); - } - } - - [Fact] - public void ItThrowsOnCosineSimilarityWithDifferentLengthVectorsDouble() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._doubleV1); - var vSpan2 = new EmbeddingSpan(new double[] { -1.0, 4.0 }); - - // Assert - try - { - vSpan1.CosineSimilarity(vSpan2); - Assert.True(false, "No exception thrown"); - } - catch (ArgumentException target) - { - Assert.IsType(target); - } - } - - [Fact] - public void ItCanComputeEuclideanLengthFloat() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._floatV1); - - // Act - var target = vSpan1.EuclideanLength(); - - // Assert - Assert.Equal(11.0, target, 5); - } - - [Fact] - public void ItCanComputeEuclideanLengthDouble() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._doubleV1); - - // Act - var target = vSpan1.EuclideanLength(); - - // Assert - Assert.Equal(11.0, target, 5); - } - - [Fact] - public void ItCanComputeDotProductFloat() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._floatV1); - var vSpan2 = new EmbeddingSpan(this._floatV2); - - // Act - var target = vSpan1.Dot(vSpan2); - - // Assert - Assert.Equal(45.0, target, 5); - } - - [Fact] - public void ItCanComputeDotProductDouble() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._doubleV1); - var vSpan2 = new EmbeddingSpan(this._doubleV2); - - // Act - var target = vSpan1.Dot(vSpan2); - - // Assert - Assert.Equal(45.0, target, 5); - } - - [Fact] - public void ItThrowsOnDotProductWithDifferentLengthVectorsFP() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._floatV1); - var vSpan2 = new EmbeddingSpan(new float[] { -1.0F, 4.0F }); - - // Assert - try - { - vSpan1.Dot(vSpan2); - Assert.True(false, "No exception thrown"); - } - catch (ArgumentException target) - { - Assert.IsType(target); - } - } - - [Fact] - public void ItThrowsOnDotProductWithDifferentLengthVectorsDouble() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._doubleV1); - var vSpan2 = new EmbeddingSpan(new double[] { -1.0, 4.0 }); - - // Assert - try - { - vSpan1.Dot(vSpan2); - Assert.True(false, "No exception thrown"); - } - catch (ArgumentException target) - { - Assert.IsType(target); - } - } - - [Fact] - public void ItCanBeNormalizedFloat() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._floatV1); - - // Act - var target = vSpan1.Normalize(); - var expected = new EmbeddingSpan(new float[] { 0.09090909F, 0.18181819F, -0.3636364F, 0.90909094F }); - - // Assert - Assert.True(target.IsNormalized); - Assert.Equal(vSpan1.Span.Length, target.ReadOnlySpan.Length); - for (int i = 0; i < vSpan1.Span.Length; i++) - { - Assert.Equal(expected.Span[i], target.ReadOnlySpan[i], .00001F); - } - } - - [Fact] - public void ItCanBeNormalizedDouble() - { - // Arrange - var vSpan1 = new EmbeddingSpan(this._doubleV1); - - // Act - var target = vSpan1.Normalize(); - var expected = new EmbeddingSpan(new double[] { 0.09090909, 0.18181819, -0.3636364, 0.90909094 }); - - // Assert - Assert.True(target.IsNormalized); - Assert.Equal(vSpan1.Span.Length, target.ReadOnlySpan.Length); - for (int i = 0; i < vSpan1.Span.Length; i++) - { - Assert.Equal(expected.Span[i], target.ReadOnlySpan[i], .00001); - } - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/XunitHelpers/TestConsoleLogger.cs b/dotnet/src/SemanticKernel.UnitTests/XunitHelpers/TestConsoleLogger.cs index 6efbc88bd646..9990760dc746 100644 --- a/dotnet/src/SemanticKernel.UnitTests/XunitHelpers/TestConsoleLogger.cs +++ b/dotnet/src/SemanticKernel.UnitTests/XunitHelpers/TestConsoleLogger.cs @@ -10,14 +10,14 @@ namespace SemanticKernel.UnitTests.XunitHelpers; /// internal static class TestConsoleLogger { - internal static ILogger Logger => LogFactory.CreateLogger(); + internal static ILogger Logger => LoggerFactory.CreateLogger(); - private static ILoggerFactory LogFactory => s_loggerFactory.Value; + internal static ILoggerFactory LoggerFactory => s_loggerFactory.Value; private static readonly Lazy s_loggerFactory = new(LogBuilder); private static ILoggerFactory LogBuilder() { - return LoggerFactory.Create(builder => + return Microsoft.Extensions.Logging.LoggerFactory.Create(builder => { builder.SetMinimumLevel(LogLevel.Trace); // builder.AddFilter("Microsoft", LogLevel.Trace); diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/EmbeddingReadOnlySpan.cs b/dotnet/src/SemanticKernel/AI/Embeddings/EmbeddingReadOnlySpan.cs deleted file mode 100644 index d85ad1e88a97..000000000000 --- a/dotnet/src/SemanticKernel/AI/Embeddings/EmbeddingReadOnlySpan.cs +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; - -namespace Microsoft.SemanticKernel.AI.Embeddings; - -/// -/// A view of a vector that allows for low-level, optimized, read-only mathematical operations. -/// -/// The unmanaged data type (, currently supported). -public readonly ref struct EmbeddingReadOnlySpan - where TEmbedding : unmanaged -{ - /// - /// Constructor - /// - /// A a vector of contiguous, unmanaged data. - /// Indicates whether the data was pre-normalized. - /// - /// This does not verify that the data is normalized, nor make any guarantees that it remains so, - /// as the data can be modified at its source. The parameter simply - /// directs these operations to perform faster if the data is known to be normalized. - /// - public EmbeddingReadOnlySpan(ReadOnlySpan vector, bool isNormalized = false) - { - if (!Embedding.IsSupported()) - { - EmbeddingSpan.ThrowTEmbeddingNotSupported(); - } - - this.ReadOnlySpan = vector; - this.IsNormalized = isNormalized; - } - - /// - /// Constructor - /// - /// A vector of contiguous, unmanaged data. - /// Indicates whether the data was pre-normalized. - /// - /// This does not verify that the data is normalized, nor make any guarantees that it remains so, - /// as the data can be modified at its source. The parameter simply - /// directs these operations to perform faster if the data is known to be normalized. - /// - public EmbeddingReadOnlySpan(TEmbedding[] vector, bool isNormalized = false) - : this(vector.AsReadOnlySpan(), isNormalized) - { - } - - /// - /// Constructor - /// - /// A vector of contiguous, unmanaged data. - /// Indicates whether the data was pre-normalized. - /// - /// This does not verify that the data is normalized, nor make any guarantees that it remains so, - /// as the data can be modified at its source. The parameter simply - /// directs these operations to perform faster if the data is known to be normalized. - /// - public EmbeddingReadOnlySpan(EmbeddingSpan span, bool isNormalized = false) - : this(span.Span.AsReadOnlySpan(), isNormalized) - { - } - - /// - /// Gets the underlying of unmanaged data. - /// - public ReadOnlySpan ReadOnlySpan { get; } - - /// - /// True if the data was specified to be normalized at construction. - /// - public bool IsNormalized { get; } - - /// - /// Calculates the dot product of this vector with another. - /// - /// The second vector. - /// The dot product as a - public double Dot(EmbeddingReadOnlySpan other) - { - return this.ReadOnlySpan.DotProduct(other.ReadOnlySpan); - } - - /// - /// Calculates the Euclidean length of this vector. - /// - /// The Euclidean length as a - public double EuclideanLength() - { - return this.ReadOnlySpan.EuclideanLength(); - } - - /// - /// Calculates the cosine similarity of this vector with another. - /// - /// The second vector. - /// The cosine similarity as a . - public double CosineSimilarity(EmbeddingReadOnlySpan other) - { - if (this.IsNormalized && other.IsNormalized) - { - // Because Normalized embeddings already have normalized lengths, cosine similarity is much - // faster - just a dot product. Don't have to compute lengths, square roots, etc. - return this.Dot(other); - } - - return this.ReadOnlySpan.CosineSimilarity(other.ReadOnlySpan); - } -} diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/EmbeddingSpan.cs b/dotnet/src/SemanticKernel/AI/Embeddings/EmbeddingSpan.cs deleted file mode 100644 index 6c49551c5f5c..000000000000 --- a/dotnet/src/SemanticKernel/AI/Embeddings/EmbeddingSpan.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; - -namespace Microsoft.SemanticKernel.AI.Embeddings; - -/// -/// A view of a vector that allows for low-level, optimized, read-write mathematical operations. -/// -/// The unmanaged data type (, currently supported). -public readonly ref struct EmbeddingSpan - where TEmbedding : unmanaged -{ - /// - /// Constructor - /// - /// A a vector of contiguous, unmanaged data. - public EmbeddingSpan(Span vector) - { - if (!Embedding.IsSupported()) - { - ThrowTEmbeddingNotSupported(); - } - - this.Span = vector; - } - - internal static void ThrowTEmbeddingNotSupported() => - throw new NotSupportedException($"Embeddings do not support type '{typeof(TEmbedding).Name}'. Supported types include: [ Single, Double ]"); - - /// - /// Constructor - /// - /// A vector of contiguous, unmanaged data. - public EmbeddingSpan(TEmbedding[] vector) - : this(vector.AsSpan()) - { - } - - /// - /// Gets the underlying of unmanaged data. - /// - public Span Span { get; } - - /// - /// Normalizes the underlying vector in-place, such that the Euclidean length is 1. - /// - /// A with 'IsNormalized' set to true. - public EmbeddingReadOnlySpan Normalize() - { - this.Span.NormalizeInPlace(); - return new EmbeddingReadOnlySpan(this.Span, true); - } - - /// - /// Calculates the dot product of this vector with another. - /// - /// The second vector. - /// The dot product as a - public double Dot(EmbeddingSpan other) - { - return this.Span.DotProduct(other.Span); - } - - /// - /// Calculates the Euclidean length of this vector. - /// - /// The Euclidean length as a - public double EuclideanLength() - { - return this.Span.EuclideanLength(); - } - - /// - /// Calculates the cosine similarity of this vector with another. - /// - /// The second vector. - /// The cosine similarity as a . - /// This operation can be performed much faster if the vectors are known to be normalized, by - /// converting to a with constructor parameter 'isNormalized' true. - public double CosineSimilarity(EmbeddingSpan other) - { - return this.Span.CosineSimilarity(other.Span); - } -} diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/IEmbeddingIndex.cs b/dotnet/src/SemanticKernel/AI/Embeddings/IEmbeddingIndex.cs deleted file mode 100644 index 86de0b5ff697..000000000000 --- a/dotnet/src/SemanticKernel/AI/Embeddings/IEmbeddingIndex.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.AI.Embeddings; - -/// -/// Represents an searchable index of structs. -/// -/// The data type of the embedding. -public interface IEmbeddingIndex - where TEmbedding : unmanaged -{ - /// - /// Gets the nearest matches to the . - /// - /// The storage collection to search. - /// The input to use as the search. - /// The max number of results to return. - /// The minimum score to consider in the distance calculation. - /// A tuple consisting of the and the similarity score as a . - IAsyncEnumerable<(IEmbeddingWithMetadata, double)> GetNearestMatchesAsync( - string collection, - Embedding embedding, - int limit = 1, - double minRelevanceScore = 0.0); -} - -/// -/// Common extension methods for objects. -/// -public static class EmbeddingIndexExtensions -{ - /// - /// Searches the index for the nearest match to the . - /// - public static async Task<(IEmbeddingWithMetadata, double)> GetNearestMatchAsync(this IEmbeddingIndex index, - string collection, - Embedding embedding, - double minScore = 0.0) - where TEmbedding : unmanaged - { - Verify.NotNull(index); - await foreach (var match in index.GetNearestMatchesAsync(collection, embedding, 1, minScore)) - { - return match; - } - - return default; - } -} diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/IEmbeddingWithMetadata.cs b/dotnet/src/SemanticKernel/AI/Embeddings/IEmbeddingWithMetadata.cs deleted file mode 100644 index 2d0b6e704263..000000000000 --- a/dotnet/src/SemanticKernel/AI/Embeddings/IEmbeddingWithMetadata.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.SemanticKernel.AI.Embeddings; - -/// -/// Represents an object that has an . -/// -/// The embedding data type. -public interface IEmbeddingWithMetadata - where TEmbedding : unmanaged -{ - /// - /// Gets the . - /// - Embedding Embedding { get; } - - /// - /// Returns a string representing the metadata. - /// - string GetSerializedMetadata(); -} diff --git a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/SpanExtensions.cs b/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/SpanExtensions.cs deleted file mode 100644 index d45504ef0de7..000000000000 --- a/dotnet/src/SemanticKernel/AI/Embeddings/VectorOperations/SpanExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; - -/// -/// Extension methods to convert from array and to . -/// -internal static class SpanExtensions -{ - internal static ReadOnlySpan AsReadOnlySpan(this TNumber[] vector) - { - return new ReadOnlySpan(vector); - } - - internal static ReadOnlySpan AsReadOnlySpan(this Span span) - { - return span; - } -} diff --git a/dotnet/src/SemanticKernel/Kernel.cs b/dotnet/src/SemanticKernel/Kernel.cs deleted file mode 100644 index 079bd503cac0..000000000000 --- a/dotnet/src/SemanticKernel/Kernel.cs +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.Services; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.TemplateEngine; - -namespace Microsoft.SemanticKernel; - -/// -/// Semantic kernel class. -/// The kernel provides a skill collection to define native and semantic functions, an orchestrator to execute a list of functions. -/// Semantic functions are automatically rendered and executed using an internal prompt template rendering engine. -/// Future versions will allow to: -/// * customize the rendering engine -/// * include branching logic in the functions pipeline -/// * persist execution state for long running pipelines -/// * distribute pipelines over a network -/// * RPC functions and secure environments, e.g. sandboxing and credentials management -/// * auto-generate pipelines given a higher level goal -/// -public sealed class Kernel : IKernel, IDisposable -{ - /// - public KernelConfig Config { get; } - - /// - [Obsolete("Use Logger instead. This will be removed in a future release.")] - public ILogger Log => this.Logger; - - /// - public ILogger Logger { get; } - - /// - public ISemanticTextMemory Memory => this._memory; - - /// - public IReadOnlySkillCollection Skills => this._skillCollection; - - /// - public IPromptTemplateEngine PromptTemplateEngine { get; } - - /// - /// Return a new instance of the kernel builder, used to build and configure kernel instances. - /// - public static KernelBuilder Builder => new(); - - /// - /// Kernel constructor. See KernelBuilder for an easier and less error prone approach to create kernel instances. - /// - /// - /// - /// - /// - /// - /// - public Kernel( - ISkillCollection skillCollection, - IAIServiceProvider aiServiceProvider, - IPromptTemplateEngine promptTemplateEngine, - ISemanticTextMemory memory, - KernelConfig config, - ILogger logger) - { - this.Logger = logger; - this.Config = config; - this.PromptTemplateEngine = promptTemplateEngine; - this._memory = memory; - this._aiServiceProvider = aiServiceProvider; - this._promptTemplateEngine = promptTemplateEngine; - this._skillCollection = skillCollection; - } - - /// - public ISKFunction RegisterSemanticFunction(string functionName, SemanticFunctionConfig functionConfig) - { - return this.RegisterSemanticFunction(SkillCollection.GlobalSkill, functionName, functionConfig); - } - - /// - public ISKFunction RegisterSemanticFunction(string skillName, string functionName, SemanticFunctionConfig functionConfig) - { - // Future-proofing the name not to contain special chars - Verify.ValidSkillName(skillName); - Verify.ValidFunctionName(functionName); - - ISKFunction function = this.CreateSemanticFunction(skillName, functionName, functionConfig); - this._skillCollection.AddFunction(function); - - return function; - } - - /// - public IDictionary ImportSkill(object skillInstance, string? skillName = null) - { - Verify.NotNull(skillInstance); - - if (string.IsNullOrWhiteSpace(skillName)) - { - skillName = SkillCollection.GlobalSkill; - this.Logger.LogTrace("Importing skill {0} in the global namespace", skillInstance.GetType().FullName); - } - else - { - this.Logger.LogTrace("Importing skill {0}", skillName); - } - - Dictionary skill = ImportSkill( - skillInstance, - skillName!, - this.Logger - ); - foreach (KeyValuePair f in skill) - { - f.Value.SetDefaultSkillCollection(this.Skills); - this._skillCollection.AddFunction(f.Value); - } - - return skill; - } - - /// - public ISKFunction RegisterCustomFunction(ISKFunction customFunction) - { - Verify.NotNull(customFunction); - - customFunction.SetDefaultSkillCollection(this.Skills); - this._skillCollection.AddFunction(customFunction); - - return customFunction; - } - - /// - public void RegisterMemory(ISemanticTextMemory memory) - { - this._memory = memory; - } - - /// - public Task RunAsync(ISKFunction skFunction, CancellationToken cancellationToken = default) - => this.RunAsync(new ContextVariables(), cancellationToken, skFunction); - - /// - public Task RunAsync(params ISKFunction[] pipeline) - => this.RunAsync(new ContextVariables(), pipeline); - - /// - public Task RunAsync(string input, params ISKFunction[] pipeline) - => this.RunAsync(new ContextVariables(input), pipeline); - - /// - public Task RunAsync(ContextVariables variables, params ISKFunction[] pipeline) - => this.RunAsync(variables, CancellationToken.None, pipeline); - - /// - public Task RunAsync(CancellationToken cancellationToken, params ISKFunction[] pipeline) - => this.RunAsync(new ContextVariables(), cancellationToken, pipeline); - - /// - public Task RunAsync(string input, CancellationToken cancellationToken, params ISKFunction[] pipeline) - => this.RunAsync(new ContextVariables(input), cancellationToken, pipeline); - - /// - public async Task RunAsync(ContextVariables variables, CancellationToken cancellationToken, params ISKFunction[] pipeline) - { - var context = new SKContext( - variables, - this._skillCollection, - this.Logger); - - int pipelineStepCount = -1; - foreach (ISKFunction f in pipeline) - { - if (context.ErrorOccurred) - { - this.Logger.LogError( - context.LastException, - "Something went wrong in pipeline step {0}:'{1}'", pipelineStepCount, context.LastErrorDescription); - return context; - } - - pipelineStepCount++; - - try - { - cancellationToken.ThrowIfCancellationRequested(); - context = await f.InvokeAsync(context, cancellationToken: cancellationToken).ConfigureAwait(false); - - if (context.ErrorOccurred) - { - this.Logger.LogError("Function call fail during pipeline step {0}: {1}.{2}. Error: {3}", - pipelineStepCount, f.SkillName, f.Name, context.LastErrorDescription); - return context; - } - } - catch (Exception e) when (!e.IsCriticalException()) - { - this.Logger.LogError(e, "Something went wrong in pipeline step {0}: {1}.{2}. Error: {3}", - pipelineStepCount, f.SkillName, f.Name, e.Message); - context.Fail(e.Message, e); - return context; - } - } - - return context; - } - - /// - public ISKFunction Func(string skillName, string functionName) - { - return this.Skills.GetFunction(skillName, functionName); - } - - /// - public SKContext CreateNewContext() - { - return new SKContext( - skills: this._skillCollection, - logger: this.Logger); - } - - /// - /// Create a new instance of a context, linked to the kernel internal state. - /// - /// Cancellation token for operations in context. - /// SK context - [Obsolete("SKContext no longer contains the CancellationToken. Use CreateNewContext().")] - public SKContext CreateNewContext(CancellationToken cancellationToken) - { - return this.CreateNewContext(); - } - - /// - public T GetService(string? name = null) where T : IAIService - { - var service = this._aiServiceProvider.GetService(name); - if (service != null) - { - return service; - } - - throw new KernelException(KernelException.ErrorCodes.ServiceNotFound, $"Service of type {typeof(T)} and name {name ?? ""} not registered."); - } - - /// - /// Dispose of resources. - /// - public void Dispose() - { - // ReSharper disable once SuspiciousTypeConversion.Global - if (this._memory is IDisposable mem) { mem.Dispose(); } - - // ReSharper disable once SuspiciousTypeConversion.Global - if (this._skillCollection is IDisposable reg) { reg.Dispose(); } - } - - #region private ================================================================================ - - private readonly ISkillCollection _skillCollection; - private ISemanticTextMemory _memory; - private readonly IPromptTemplateEngine _promptTemplateEngine; - private readonly IAIServiceProvider _aiServiceProvider; - - private ISKFunction CreateSemanticFunction( - string skillName, - string functionName, - SemanticFunctionConfig functionConfig) - { - if (!functionConfig.PromptTemplateConfig.Type.Equals("completion", StringComparison.OrdinalIgnoreCase)) - { - throw new AIException( - AIException.ErrorCodes.FunctionTypeNotSupported, - $"Function type not supported: {functionConfig.PromptTemplateConfig}"); - } - - ISKFunction func = SKFunction.FromSemanticConfig( - skillName, - functionName, - functionConfig, - this.Logger - ); - - // Connect the function to the current kernel skill collection, in case the function - // is invoked manually without a context and without a way to find other functions. - func.SetDefaultSkillCollection(this.Skills); - - func.SetAIConfiguration(CompleteRequestSettings.FromCompletionConfig(functionConfig.PromptTemplateConfig.Completion)); - - // Note: the service is instantiated using the kernel configuration state when the function is invoked - func.SetAIService(() => this.GetService()); - - return func; - } - - /// - /// Import a skill into the kernel skill collection, so that semantic functions and pipelines can consume its functions. - /// - /// Skill class instance - /// Skill name, used to group functions under a shared namespace - /// Application logger - /// Dictionary of functions imported from the given class instance, case-insensitively indexed by name. - private static Dictionary ImportSkill(object skillInstance, string skillName, ILogger logger) - { - MethodInfo[] methods = skillInstance.GetType().GetMethods(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public); - logger.LogTrace("Importing skill name: {0}. Potential methods found: {1}", skillName, methods.Length); - - // Filter out non-SKFunctions and fail if two functions have the same name - Dictionary result = new(StringComparer.OrdinalIgnoreCase); - foreach (MethodInfo method in methods) - { - if (method.GetCustomAttribute() is not null) - { - ISKFunction function = SKFunction.FromNativeMethod(method, skillInstance, skillName, logger); - if (result.ContainsKey(function.Name)) - { - throw new KernelException( - KernelException.ErrorCodes.FunctionOverloadNotSupported, - "Function overloads are not supported, please differentiate function names"); - } - - result.Add(function.Name, function); - } - } - - logger.LogTrace("Methods imported {0}", result.Count); - - return result; - } - - #endregion -} diff --git a/dotnet/src/SemanticKernel/KernelBuilder.cs b/dotnet/src/SemanticKernel/KernelBuilder.cs deleted file mode 100644 index 223ff8e5c92a..000000000000 --- a/dotnet/src/SemanticKernel/KernelBuilder.cs +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Reliability; -using Microsoft.SemanticKernel.Services; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.TemplateEngine; - -namespace Microsoft.SemanticKernel; - -/// -/// A builder for Semantic Kernel. -/// -public sealed class KernelBuilder -{ - private KernelConfig _config = new(); - private Func _memoryFactory = () => NullMemory.Instance; - private ILogger _logger = NullLogger.Instance; - private Func? _memoryStorageFactory = null; - private IDelegatingHandlerFactory? _httpHandlerFactory = null; - private IPromptTemplateEngine? _promptTemplateEngine; - private readonly AIServiceCollection _aiServices = new(); - - /// - /// Create a new kernel instance - /// - /// New kernel instance - public static IKernel Create() - { - var builder = new KernelBuilder(); - return builder.Build(); - } - - /// - /// Build a new kernel instance using the settings passed so far. - /// - /// Kernel instance - public IKernel Build() - { - if (this._httpHandlerFactory != null) - { - this._config.SetHttpRetryHandlerFactory(this._httpHandlerFactory); - } - - var instance = new Kernel( - new SkillCollection(this._logger), - this._aiServices.Build(), - this._promptTemplateEngine ?? new PromptTemplateEngine(this._logger), - this._memoryFactory.Invoke(), - this._config, - this._logger - ); - - // TODO: decouple this from 'UseMemory' kernel extension - if (this._memoryStorageFactory != null) - { - instance.UseMemory(this._memoryStorageFactory.Invoke()); - } - - return instance; - } - - /// - /// Add a logger to the kernel to be built. - /// - /// Logger to add. - /// Updated kernel builder including the logger. - public KernelBuilder WithLogger(ILogger logger) - { - Verify.NotNull(logger); - this._logger = logger; - return this; - } - - /// - /// Add a semantic text memory entity to the kernel to be built. - /// - /// Semantic text memory entity to add. - /// Updated kernel builder including the semantic text memory entity. - public KernelBuilder WithMemory(ISemanticTextMemory memory) - { - Verify.NotNull(memory); - this._memoryFactory = () => memory; - return this; - } - - /// - /// Add a semantic text memory store factory. - /// - /// The store factory. - /// Updated kernel builder including the semantic text memory entity. - public KernelBuilder WithMemory(Func<(ILogger Logger, KernelConfig Config), TStore> factory) where TStore : ISemanticTextMemory - { - Verify.NotNull(factory); - this._memoryFactory = () => factory((this._logger, this._config)); - return this; - } - - /// - /// Add memory storage to the kernel to be built. - /// - /// Storage to add. - /// Updated kernel builder including the memory storage. - public KernelBuilder WithMemoryStorage(IMemoryStore storage) - { - Verify.NotNull(storage); - this._memoryStorageFactory = () => storage; - return this; - } - - /// - /// Add memory storage factory to the kernel. - /// - /// The storage factory. - /// Updated kernel builder including the memory storage. - public KernelBuilder WithMemoryStorage(Func<(ILogger Logger, KernelConfig Config), TStore> factory) where TStore : IMemoryStore - { - Verify.NotNull(factory); - this._memoryStorageFactory = () => factory((this._logger, this._config)); - return this; - } - - /// - /// Add prompt template engine to the kernel to be built. - /// - /// Prompt template engine to add. - /// Updated kernel builder including the prompt template engine. - public KernelBuilder WithPromptTemplateEngine(IPromptTemplateEngine promptTemplateEngine) - { - Verify.NotNull(promptTemplateEngine); - this._promptTemplateEngine = promptTemplateEngine; - return this; - } - - /// - /// Add a retry handler factory to the kernel to be built. - /// - /// Retry handler factory to add. - /// Updated kernel builder including the retry handler factory. - public KernelBuilder WithRetryHandlerFactory(IDelegatingHandlerFactory httpHandlerFactory) - { - Verify.NotNull(httpHandlerFactory); - this._httpHandlerFactory = httpHandlerFactory; - return this; - } - - /// - /// Use the given configuration with the kernel to be built. - /// - /// Configuration to use. - /// Updated kernel builder including the given configuration. - public KernelBuilder WithConfiguration(KernelConfig config) - { - Verify.NotNull(config); - this._config = config; - return this; - } - - /// - /// Update the configuration using the instructions provided. - /// - /// Action that updates the current configuration. - /// Updated kernel builder including the updated configuration. - public KernelBuilder Configure(Action configure) - { - Verify.NotNull(configure); - configure.Invoke(this._config); - return this; - } - - /// - /// Adds a instance to the services collection - /// - /// The instance. - public KernelBuilder WithDefaultAIService(TService instance) where TService : IAIService - { - this._aiServices.SetService(instance); - return this; - } - - /// - /// Adds a instance to the services collection - /// - /// The service ID - /// The instance. - /// Optional: set as the default AI service for type - public KernelBuilder WithAIService( - string? serviceId, - TService instance, - bool setAsDefault = false) where TService : IAIService - { - this._aiServices.SetService(serviceId, instance, setAsDefault); - return this; - } - - /// - /// Adds a factory method to the services collection - /// - /// The factory method that creates the AI service instances of type . - public KernelBuilder WithDefaultAIService(Func factory) where TService : IAIService - { - this._aiServices.SetService(() => factory(this._logger)); - return this; - } - - /// - /// Adds a factory method to the services collection - /// - /// The service ID - /// The factory method that creates the AI service instances of type . - /// Optional: set as the default AI service for type - public KernelBuilder WithAIService( - string? serviceId, - Func<(ILogger Logger, KernelConfig Config), TService> factory, - bool setAsDefault = false) where TService : IAIService - { - this._aiServices.SetService(serviceId, () => factory((this._logger, this._config)), setAsDefault); - return this; - } -} diff --git a/dotnet/src/SemanticKernel/Memory/Collections/MinHeap.cs b/dotnet/src/SemanticKernel/Memory/Collections/MinHeap.cs deleted file mode 100644 index c8964f88ae97..000000000000 --- a/dotnet/src/SemanticKernel/Memory/Collections/MinHeap.cs +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Memory.Collections; - -/// -/// Implements the classic 'heap' data structure. By default, the item with the lowest value is at the top of the heap. -/// -/// Data type. -internal sealed class MinHeap : IEnumerable where T : IComparable -{ - private const int DefaultCapacity = 7; - private const int MinCapacity = 0; - - private static readonly T[] s_emptyBuffer = Array.Empty(); - - private T[] _items; - private int _count; - - public MinHeap(T minValue, int capacity = DefaultCapacity) - { - if (capacity < MinCapacity) - { - Verify.ThrowArgumentOutOfRangeException(nameof(capacity), capacity, $"MinHeap capacity must be greater than {MinCapacity}."); - } - - this._items = new T[capacity + 1]; - // - // The 0'th item is a sentinel entry that simplifies the code - // - this._items[0] = minValue; - } - - public MinHeap(T minValue, IList items) - : this(minValue, items.Count) - { - this.Add(items); - } - - public int Count - { - get => this._count; - internal set - { - Debug.Assert(value <= this.Capacity); - this._count = value; - } - } - - public int Capacity => this._items.Length - 1; // 0'th item is always a sentinel to simplify code - - public T this[int index] - { - get => this._items[index + 1]; - internal set { this._items[index + 1] = value; } - } - - public T Top => this._items[1]; - - public bool IsEmpty => (this._count == 0); - - public void Clear() - { - this._count = 0; - } - - public void Erase() - { - Array.Clear(this._items, 1, this._count); - this._count = 0; - } - - public T[] DetachBuffer() - { - T[] buf = this._items; - this._items = s_emptyBuffer; - this._count = 0; - return buf; - } - - public void Add(T item) - { - // - // the 0'th item is always a sentinel and not included in this._count. - // The length of the buffer is always this._count + 1 - // - this._count++; - this.EnsureCapacity(); - this._items[this._count] = item; - this.UpHeap(this._count); - } - - public void Add(IEnumerable items) - { - foreach (T item in items) - { - this.Add(item); - } - } - - public void Add(IList items, int startAt = 0) - { - Verify.NotNull(items); - - int newItemCount = items.Count; - if (startAt >= newItemCount) - { - Verify.ThrowArgumentOutOfRangeException(nameof(startAt), startAt, $"{nameof(startAt)} value must be less than {nameof(items)}.{nameof(items.Count)}."); - } - - this.EnsureCapacity(this._count + (newItemCount - startAt)); - for (int i = startAt; i < newItemCount; ++i) - { - // - // the 0'th item is always a sentinel and not included in this._count. - // The length of the buffer is always this._count + 1 - // - this._count++; - this._items[this._count] = items[i]; - this.UpHeap(this._count); - } - } - - public T RemoveTop() - { - if (this._count == 0) - { - throw new InvalidOperationException("MinHeap is empty."); - } - - T item = this._items[1]; - this._items[1] = this._items[this._count--]; - this.DownHeap(1); - return item; - } - - public IEnumerable RemoveAll() - { - while (this._count > 0) - { - yield return this.RemoveTop(); - } - } - - public void EnsureCapacity(int capacity) - { - if (capacity < MinCapacity) - { - Verify.ThrowArgumentOutOfRangeException(nameof(capacity), capacity, $"MinHeap capacity must be greater than {MinCapacity}."); - } - - // 0th item is always a sentinel - capacity++; - if (capacity > this._items.Length) - { - Array.Resize(ref this._items, capacity); - } - } - - public void EnsureCapacity() - { - if (this._count == this._items.Length) - { - Array.Resize(ref this._items, (this._count * 2) + 1); - } - } - - private void UpHeap(int startAt) - { - int i = startAt; - T[] items = this._items; - T item = items[i]; - int parent = i >> 1; //i / 2; - - while (parent > 0 && items[parent].CompareTo(item) > 0) - { - // Child > parent. Exchange with parent, thus moving the child up the queue - items[i] = items[parent]; - i = parent; - parent = i >> 1; //i / 2; - } - - items[i] = item; - } - - private void DownHeap(int startAt) - { - int i = startAt; - int count = this._count; - int maxParent = count >> 1; - T[] items = this._items; - T item = items[i]; - - while (i <= maxParent) - { - int child = i + i; - // - // Exchange the item with the smaller of its two children - if one is smaller, i.e. - // - // First, find the smaller child - // - if (child < count && items[child].CompareTo(items[child + 1]) > 0) - { - child++; - } - - if (item.CompareTo(items[child]) <= 0) - { - // Heap condition is satisfied. Parent <= both its children - break; - } - - // Else, swap parent with the smallest child - items[i] = items[child]; - i = child; - } - - items[i] = item; - } - - public IEnumerator GetEnumerator() - { - // The 0'th item in the queue is a sentinel. i is 1 based. - for (int i = 1; i <= this._count; ++i) - { - yield return this._items[i]; - } - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - - /// - /// Heap Sort in-place. - /// This is destructive. Once you do this, the heap order is lost. - /// The advantage on in-place is that we don't need to do another allocation - /// - public void SortDescending() - { - int count = this._count; - int i = count; // remember that the 0'th item in the queue is always a sentinel. So i is 1 based - - while (this._count > 0) - { - // - // this dequeues the item with the current LOWEST relevancy - // We take that and place it at the 'back' of the array - thus inverting it - // - T item = this.RemoveTop(); - this._items[i--] = item; - } - - this._count = count; - } - - /// - /// Restores heap order - /// - internal void Restore() - { - this.Clear(); - this.Add(this._items, 1); - } - - internal void Sort(IComparer comparer) - { - Array.Sort(this._items, 1, this._count, comparer); - } -} diff --git a/dotnet/src/SemanticKernel/Memory/Collections/Score.cs b/dotnet/src/SemanticKernel/Memory/Collections/Score.cs deleted file mode 100644 index b03d36075307..000000000000 --- a/dotnet/src/SemanticKernel/Memory/Collections/Score.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Globalization; - -namespace Microsoft.SemanticKernel.Memory.Collections; - -/// -/// Structure for storing score value. -/// -public readonly struct Score : IComparable, IEquatable -{ - public double Value { get; } - - public Score(double value) - { - this.Value = value; - } - - internal static Score Min => double.MinValue; - - public static implicit operator Score(double score) - { - return new Score(score); - } - - public static implicit operator double(Score src) - { - return src.Value; - } - - public int CompareTo(Score other) - { - return this.Value.CompareTo(other.Value); - } - - public override string ToString() - { - return this.Value.ToString(CultureInfo.InvariantCulture.NumberFormat); - } - - public override bool Equals(object obj) - { - return (obj is Score other) && this.Equals(other); - } - - public bool Equals(Score other) - { - return this.Value == other.Value; - } - - public override int GetHashCode() - { - return HashCode.Combine(this.Value); - } - - public static bool operator ==(Score left, Score right) - { - return left.Equals(right); - } - - public static bool operator !=(Score left, Score right) - { - return !(left == right); - } - - public static bool operator <(Score left, Score right) - { - return left.CompareTo(right) < 0; - } - - public static bool operator <=(Score left, Score right) - { - return left.CompareTo(right) <= 0; - } - - public static bool operator >(Score left, Score right) - { - return left.CompareTo(right) > 0; - } - - public static bool operator >=(Score left, Score right) - { - return left.CompareTo(right) >= 0; - } -} diff --git a/dotnet/src/SemanticKernel/Memory/Collections/ScoredValue.cs b/dotnet/src/SemanticKernel/Memory/Collections/ScoredValue.cs deleted file mode 100644 index c6b73f3fe75a..000000000000 --- a/dotnet/src/SemanticKernel/Memory/Collections/ScoredValue.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; - -namespace Microsoft.SemanticKernel.Memory.Collections; - -/// -/// Structure for storing data which can be scored. -/// -/// Data type. -public readonly struct ScoredValue : IComparable>, IEquatable> -{ - public ScoredValue(T item, double score) - { - this.Value = item; - this.Score = score; - } - - public T Value { get; } - public Score Score { get; } - - public int CompareTo(ScoredValue other) - { - return this.Score.CompareTo(other.Score); - } - - public override string ToString() - { - return $"{this.Score}, {this.Value}"; - } - - public static explicit operator double(ScoredValue src) - { - return src.Score; - } - - public static explicit operator T(ScoredValue src) - { - return src.Value; - } - - public static implicit operator ScoredValue(KeyValuePair src) - { - return new ScoredValue(src.Key, src.Value); - } - - public override bool Equals(object obj) - { - return (obj is ScoredValue other) && this.Equals(other); - } - - public bool Equals(ScoredValue other) - { - return EqualityComparer.Default.Equals(other.Value) && - this.Score.Equals(other.Score); - } - - public override int GetHashCode() - { - return HashCode.Combine(this.Value, this.Score); - } - - public static bool operator ==(ScoredValue left, ScoredValue right) - { - return left.Equals(right); - } - - public static bool operator !=(ScoredValue left, ScoredValue right) - { - return !(left == right); - } - - public static bool operator <(ScoredValue left, ScoredValue right) - { - return left.CompareTo(right) < 0; - } - - public static bool operator <=(ScoredValue left, ScoredValue right) - { - return left.CompareTo(right) <= 0; - } - - public static bool operator >(ScoredValue left, ScoredValue right) - { - return left.CompareTo(right) > 0; - } - - public static bool operator >=(ScoredValue left, ScoredValue right) - { - return left.CompareTo(right) >= 0; - } - - internal static ScoredValue Min() - { - return new ScoredValue(default!, Score.Min); - } -} diff --git a/dotnet/src/SemanticKernel/Memory/Collections/TopNCollection.cs b/dotnet/src/SemanticKernel/Memory/Collections/TopNCollection.cs deleted file mode 100644 index d84daf1b174c..000000000000 --- a/dotnet/src/SemanticKernel/Memory/Collections/TopNCollection.cs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections; -using System.Collections.Generic; - -namespace Microsoft.SemanticKernel.Memory.Collections; - -/// -/// A collector for Top N matches. Keeps only the best N matches by Score. -/// Automatically flushes out any not in the top N. -/// By default, items are not sorted by score until you call . -/// -public class TopNCollection : IEnumerable> -{ - private readonly MinHeap> _heap; - private bool _sorted = false; - - public TopNCollection(int maxItems) - { - this.MaxItems = maxItems; - this._heap = new MinHeap>(ScoredValue.Min(), maxItems); - } - - public int MaxItems { get; } - - public int Count => this._heap.Count; - - internal ScoredValue this[int i] => this._heap[i]; - internal ScoredValue Top => this._heap.Top; - - /// - /// Call this to reuse the buffer - /// - public void Reset() - { - this._heap.Clear(); - } - - /// - /// Adds a single scored value - /// - public void Add(ScoredValue value) - { - if (this._sorted) - { - this._heap.Restore(); - this._sorted = false; - } - - if (this._heap.Count == this.MaxItems) - { - // Queue is full. We will need to dequeue the item with lowest weight - if (value.Score <= this.Top.Score) - { - // This score is lower than the lowest score on the queue right now. Ignore it - return; - } - - this._heap.RemoveTop(); - } - - this._heap.Add(value); - } - - public void Add(T value, Score score) - { - this.Add(new ScoredValue(value, score)); - } - - /// - /// Sort in relevancy order. - /// - public void SortByScore() - { - if (!this._sorted && this._heap.Count > 0) - { - this._heap.SortDescending(); - this._sorted = true; - } - } - - public IList> ToList() - { - var list = new List>(this.Count); - for (int i = 0, count = this.Count; i < count; ++i) - { - list.Add(this[i]); - } - - return list; - } - - public IEnumerator> GetEnumerator() - { - return this._heap.GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return this._heap.GetEnumerator(); - } -} diff --git a/dotnet/src/SemanticKernel/Memory/MemoryConfiguration.cs b/dotnet/src/SemanticKernel/Memory/MemoryConfiguration.cs deleted file mode 100644 index caa177670924..000000000000 --- a/dotnet/src/SemanticKernel/Memory/MemoryConfiguration.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; -using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Memory; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using NS of IKernel -namespace Microsoft.SemanticKernel; -#pragma warning restore IDE0130 - -/// -/// Kernel extension to configure the semantic memory with custom settings -/// -public static class MemoryConfiguration -{ - /// - /// Set the semantic memory to use the given memory storage and embeddings service. - /// - /// Kernel instance - /// Memory storage - /// Kernel service id for embedding generation - public static void UseMemory(this IKernel kernel, IMemoryStore storage, string? embeddingsServiceId = null) - { - var embeddingGenerator = kernel.GetService(embeddingsServiceId); - - UseMemory(kernel, embeddingGenerator, storage); - } - - /// - /// Set the semantic memory to use the given memory storage and embedding generator. - /// - /// Kernel instance - /// Embedding generator - /// Memory storage - [SuppressMessage("Reliability", "CA2000:Dispose objects before losing scope", Justification = "The embeddingGenerator object is disposed by the kernel")] - public static void UseMemory(this IKernel kernel, ITextEmbeddingGeneration embeddingGenerator, IMemoryStore storage) - { - Verify.NotNull(storage); - Verify.NotNull(embeddingGenerator); - - kernel.RegisterMemory(new SemanticTextMemory(storage, embeddingGenerator)); - } -} diff --git a/dotnet/src/SemanticKernel/Orchestration/ContextVariablesExtensions.cs b/dotnet/src/SemanticKernel/Orchestration/ContextVariablesExtensions.cs deleted file mode 100644 index 4f2891c7f10c..000000000000 --- a/dotnet/src/SemanticKernel/Orchestration/ContextVariablesExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// ReSharper disable once CheckNamespace - Using NS of SKContext - -namespace Microsoft.SemanticKernel.Orchestration; - -/// -/// Class that holds extension methods for ContextVariables. -/// -public static class ContextVariablesExtensions -{ - /// - /// Simple extension method to turn a string into a instance. - /// - /// The text to transform - /// An instance of - public static ContextVariables ToContextVariables(this string text) - { - return new ContextVariables(text); - } -} diff --git a/dotnet/src/SemanticKernel/Planning/InstrumentedPlan.cs b/dotnet/src/SemanticKernel/Planning/InstrumentedPlan.cs deleted file mode 100644 index 8eeeb54a24ad..000000000000 --- a/dotnet/src/SemanticKernel/Planning/InstrumentedPlan.cs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics; -using System.Diagnostics.Metrics; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Planning; - -/// -/// Standard Semantic Kernel callable plan with instrumentation. -/// -public sealed class InstrumentedPlan : IPlan -{ - /// - public string Name => this._plan.Name; - - /// - public string SkillName => this._plan.SkillName; - - /// - public string Description => this._plan.Description; - - /// - public bool IsSemantic => this._plan.IsSemantic; - - /// - public CompleteRequestSettings RequestSettings => this._plan.RequestSettings; - - /// - /// Initialize a new instance of the class. - /// - /// Instance of to decorate. - /// Optional logger. - public InstrumentedPlan( - IPlan plan, - ILogger? logger = null) - { - this._plan = plan; - this._logger = logger ?? NullLogger.Instance; - } - - /// - public FunctionView Describe() - { - return this._plan.Describe(); - } - - /// - public async Task InvokeAsync( - SKContext context, - CompleteRequestSettings? settings = null, - CancellationToken cancellationToken = default) - { - return await this.InvokeWithInstrumentationAsync(() => - this._plan.InvokeAsync(context, settings, cancellationToken)).ConfigureAwait(false); - } - - /// - public async Task InvokeAsync( - string? input = null, - CompleteRequestSettings? settings = null, - ILogger? logger = null, - CancellationToken cancellationToken = default) - { - return await this.InvokeWithInstrumentationAsync(() => - this._plan.InvokeAsync(input, settings, logger ?? this._logger, cancellationToken)).ConfigureAwait(false); - } - - /// - public ISKFunction SetAIConfiguration(CompleteRequestSettings settings) => - this._plan.SetAIConfiguration(settings); - - /// - public ISKFunction SetAIService(Func serviceFactory) => - this._plan.SetAIService(serviceFactory); - - /// - public ISKFunction SetDefaultSkillCollection(IReadOnlySkillCollection skills) => - this._plan.SetDefaultSkillCollection(skills); - - #region private ================================================================================ - - private readonly IPlan _plan; - private readonly ILogger _logger; - - /// - /// Instance of for plan-related metrics. - /// - private static Meter s_meter = new(typeof(Plan).FullName); - - /// - /// Instance of to measure and track the time of plan execution. - /// - private static Histogram s_executionTimeHistogram = - s_meter.CreateHistogram( - name: "SK.Plan.Execution.ExecutionTime", - unit: "ms", - description: "Duration of plan execution"); - - /// - /// Instance of to keep track of the total number of plan executions. - /// - private static Counter s_executionTotalCounter = - s_meter.CreateCounter( - name: "SK.Plan.Execution.ExecutionTotal", - description: "Total number of plan executions"); - - /// - /// Instance of to keep track of the number of successful plan executions. - /// - private static Counter s_executionSuccessCounter = - s_meter.CreateCounter( - name: "SK.Plan.Execution.ExecutionSuccess", - description: "Number of successful plan executions"); - - /// - /// Instance of to keep track of the number of failed plan executions. - /// - private static Counter s_executionFailureCounter = - s_meter.CreateCounter( - name: "SK.Plan.Execution.ExecutionFailure", - description: "Number of failed plan executions"); - - /// - /// Wrapper for instrumentation to be used in multiple invocation places. - /// - /// Delegate to instrument. - private async Task InvokeWithInstrumentationAsync(Func> func) - { - this._logger.LogInformation("Plan execution started."); - - var stopwatch = new Stopwatch(); - - stopwatch.Start(); - - var result = await func().ConfigureAwait(false); - - stopwatch.Stop(); - - if (result.ErrorOccurred) - { - this._logger.LogWarning("Plan execution status: {Status}", "Failed"); - this._logger.LogError(result.LastException, "Plan execution exception details: {Message}", result.LastErrorDescription); - - s_executionFailureCounter.Add(1); - } - else - { - this._logger.LogInformation("Plan execution status: {Status}", "Success"); - this._logger.LogInformation("Plan execution finished in {ExecutionTime}ms", stopwatch.ElapsedMilliseconds); - - s_executionSuccessCounter.Add(1); - } - - s_executionTotalCounter.Add(1); - s_executionTimeHistogram.Record(stopwatch.ElapsedMilliseconds); - - return result; - } - - #endregion -} diff --git a/dotnet/src/SemanticKernel/Planning/Plan.cs b/dotnet/src/SemanticKernel/Planning/Plan.cs deleted file mode 100644 index 32a7fea6b0c3..000000000000 --- a/dotnet/src/SemanticKernel/Planning/Plan.cs +++ /dev/null @@ -1,626 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Planning; - -/// -/// Standard Semantic Kernel callable plan. -/// Plan is used to create trees of s. -/// -[DebuggerDisplay("{DebuggerDisplay,nq}")] -public sealed class Plan : IPlan -{ - /// - /// State of the plan - /// - [JsonPropertyName("state")] - [JsonConverter(typeof(ContextVariablesConverter))] - public ContextVariables State { get; } = new(); - - /// - /// Steps of the plan - /// - [JsonPropertyName("steps")] - public IReadOnlyList Steps => this._steps.AsReadOnly(); - - /// - /// Parameters for the plan, used to pass information to the next step - /// - [JsonPropertyName("parameters")] - [JsonConverter(typeof(ContextVariablesConverter))] - public ContextVariables Parameters { get; set; } = new(); - - /// - /// Outputs for the plan, used to pass information to the caller - /// - [JsonPropertyName("outputs")] - public IList Outputs { get; set; } = new List(); - - /// - /// Gets whether the plan has a next step. - /// - [JsonIgnore] - public bool HasNextStep => this.NextStepIndex < this.Steps.Count; - - /// - /// Gets the next step index. - /// - [JsonPropertyName("next_step_index")] - public int NextStepIndex { get; private set; } - - #region ISKFunction implementation - - /// - [JsonPropertyName("name")] - public string Name { get; set; } = string.Empty; - - /// - [JsonPropertyName("skill_name")] - public string SkillName { get; set; } = string.Empty; - - /// - [JsonPropertyName("description")] - public string Description { get; set; } = string.Empty; - - /// - [JsonIgnore] - public bool IsSemantic { get; private set; } - - /// - [JsonIgnore] - public CompleteRequestSettings RequestSettings { get; private set; } = new(); - - #endregion ISKFunction implementation - - /// - /// Initializes a new instance of the class with a goal description. - /// - /// The goal of the plan used as description. - public Plan(string goal) - { - this.Description = goal; - this.SkillName = this.GetType().FullName; - } - - /// - /// Initializes a new instance of the class with a goal description and steps. - /// - /// The goal of the plan used as description. - /// The steps to add. - public Plan(string goal, params ISKFunction[] steps) : this(goal) - { - this.AddSteps(steps); - } - - /// - /// Initializes a new instance of the class with a goal description and steps. - /// - /// The goal of the plan used as description. - /// The steps to add. - public Plan(string goal, params Plan[] steps) : this(goal) - { - this.AddSteps(steps); - } - - /// - /// Initializes a new instance of the class with a function. - /// - /// The function to execute. - public Plan(ISKFunction function) - { - this.SetFunction(function); - } - - /// - /// Initializes a new instance of the class with a function and steps. - /// - /// The name of the plan. - /// The name of the skill. - /// The description of the plan. - /// The index of the next step. - /// The state of the plan. - /// The parameters of the plan. - /// The outputs of the plan. - /// The steps of the plan. - [JsonConstructor] - public Plan( - string name, - string skillName, - string description, - int nextStepIndex, - ContextVariables state, - ContextVariables parameters, - IList outputs, - IReadOnlyList steps) - { - this.Name = name; - this.SkillName = skillName; - this.Description = description; - this.NextStepIndex = nextStepIndex; - this.State = state; - this.Parameters = parameters; - this.Outputs = outputs; - this._steps.Clear(); - this.AddSteps(steps.ToArray()); - } - - /// - /// Deserialize a JSON string into a Plan object. - /// TODO: the context should never be null, it's required internally - /// - /// JSON string representation of a Plan - /// The context to use for function registrations. - /// Whether to require functions to be registered. Only used when context is not null. - /// An instance of a Plan object. - /// If Context is not supplied, plan will not be able to execute. - public static Plan FromJson(string json, SKContext? context = null, bool requireFunctions = true) - { - var plan = JsonSerializer.Deserialize(json, new JsonSerializerOptions { IncludeFields = true }) ?? new Plan(string.Empty); - - if (context != null) - { - plan = SetAvailableFunctions(plan, context, requireFunctions); - } - - return plan; - } - - /// - /// Get JSON representation of the plan. - /// - /// Whether to emit indented JSON - /// Plan serialized using JSON format - public string ToJson(bool indented = false) - { - return JsonSerializer.Serialize(this, new JsonSerializerOptions { WriteIndented = indented }); - } - - /// - /// Adds one or more existing plans to the end of the current plan as steps. - /// - /// The plans to add as steps to the current plan. - /// - /// When you add a plan as a step to the current plan, the steps of the added plan are executed after the steps of the current plan have completed. - /// - public void AddSteps(params Plan[] steps) - { - this._steps.AddRange(steps); - } - - /// - /// Adds one or more new steps to the end of the current plan. - /// - /// The steps to add to the current plan. - /// - /// When you add a new step to the current plan, it is executed after the previous step in the plan has completed. Each step can be a function call or another plan. - /// - public void AddSteps(params ISKFunction[] steps) - { - this._steps.AddRange(steps.Select(step => new Plan(step))); - } - - /// - /// Runs the next step in the plan using the provided kernel instance and variables. - /// - /// The kernel instance to use for executing the plan. - /// The variables to use for the execution of the plan. - /// The to monitor for cancellation requests. The default is . - /// A task representing the asynchronous execution of the plan's next step. - /// - /// This method executes the next step in the plan using the specified kernel instance and context variables. - /// The context variables contain the necessary information for executing the plan, such as the skills, and logger. - /// The method returns a task representing the asynchronous execution of the plan's next step. - /// - public Task RunNextStepAsync(IKernel kernel, ContextVariables variables, CancellationToken cancellationToken = default) - { - var context = new SKContext( - variables, - kernel.Skills, - kernel.Logger); - - return this.InvokeNextStepAsync(context, cancellationToken); - } - - /// - /// Invoke the next step of the plan - /// - /// Context to use - /// The to monitor for cancellation requests. The default is . - /// The updated plan - /// If an error occurs while running the plan - public async Task InvokeNextStepAsync(SKContext context, CancellationToken cancellationToken = default) - { - if (this.HasNextStep) - { - var step = this.Steps[this.NextStepIndex]; - - // Merge the state with the current context variables for step execution - var functionVariables = this.GetNextStepVariables(context.Variables, step); - - // Execute the step - var functionContext = new SKContext(functionVariables, context.Skills, context.Logger); - var result = await step.InvokeAsync(functionContext, cancellationToken: cancellationToken).ConfigureAwait(false); - var resultValue = result.Result.Trim(); - - if (result.ErrorOccurred) - { - throw new KernelException(KernelException.ErrorCodes.FunctionInvokeError, - $"Error occurred while running plan step: {result.LastErrorDescription}", result.LastException); - } - - #region Update State - - // Update state with result - this.State.Update(resultValue); - - // Update Plan Result in State with matching outputs (if any) - if (this.Outputs.Intersect(step.Outputs).Any()) - { - this.State.TryGetValue(DefaultResultKey, out string? currentPlanResult); - this.State.Set(DefaultResultKey, string.Join("\n", currentPlanResult?.Trim(), resultValue)); - } - - // Update state with outputs (if any) - foreach (var item in step.Outputs) - { - if (result.Variables.TryGetValue(item, out string? val)) - { - this.State.Set(item, val); - } - else - { - this.State.Set(item, resultValue); - } - } - - #endregion Update State - - this.NextStepIndex++; - } - - return this; - } - - #region ISKFunction implementation - - /// - public FunctionView Describe() - { - return this.Function?.Describe() ?? new(); - } - - /// - public Task InvokeAsync( - string? input = null, - CompleteRequestSettings? settings = null, - ILogger? logger = null, - CancellationToken cancellationToken = default) - { - if (input != null) { this.State.Update(input); } - - SKContext context = new( - this.State, - logger: logger); - - return this.InvokeAsync(context, settings, cancellationToken); - } - - /// - public async Task InvokeAsync( - SKContext context, - CompleteRequestSettings? settings = null, - CancellationToken cancellationToken = default) - { - if (this.Function is not null) - { - var result = await this.Function - .WithInstrumentation(context.Logger) - .InvokeAsync(context, settings, cancellationToken) - .ConfigureAwait(false); - - if (result.ErrorOccurred) - { - return result; - } - - context.Variables.Update(result.Result); - } - else - { - // loop through steps and execute until completion - while (this.HasNextStep) - { - var functionContext = context; - - AddVariablesToContext(this.State, functionContext); - - await this.InvokeNextStepAsync(functionContext, cancellationToken).ConfigureAwait(false); - - this.UpdateContextWithOutputs(context); - } - } - - return context; - } - - /// - public ISKFunction SetDefaultSkillCollection(IReadOnlySkillCollection skills) - { - return this.Function is null - ? throw new NotImplementedException() - : this.Function.SetDefaultSkillCollection(skills); - } - - /// - public ISKFunction SetAIService(Func serviceFactory) - { - return this.Function is null - ? throw new NotImplementedException() - : this.Function.SetAIService(serviceFactory); - } - - /// - public ISKFunction SetAIConfiguration(CompleteRequestSettings settings) - { - return this.Function is null - ? throw new NotImplementedException() - : this.Function.SetAIConfiguration(settings); - } - - #endregion ISKFunction implementation - - /// - /// Expand variables in the input string. - /// - /// Variables to use for expansion. - /// Input string to expand. - /// Expanded string. - internal string ExpandFromVariables(ContextVariables variables, string input) - { - var result = input; - var matches = s_variablesRegex.Matches(input); - var orderedMatches = matches.Cast().Select(m => m.Groups["var"].Value).Distinct().OrderByDescending(m => m.Length); - - foreach (var varName in orderedMatches) - { - if (variables.TryGetValue(varName, out string? value) || this.State.TryGetValue(varName, out value)) - { - result = result.Replace($"${varName}", value); - } - } - - return result; - } - - /// - /// Set functions for a plan and its steps. - /// - /// Plan to set functions for. - /// Context to use. - /// Whether to throw an exception if a function is not found. - /// The plan with functions set. - private static Plan SetAvailableFunctions(Plan plan, SKContext context, bool requireFunctions = true) - { - if (plan.Steps.Count == 0) - { - if (context.Skills == null) - { - throw new KernelException( - KernelException.ErrorCodes.SkillCollectionNotSet, - "Skill collection not found in the context"); - } - - if (context.Skills.TryGetFunction(plan.SkillName, plan.Name, out var skillFunction)) - { - plan.SetFunction(skillFunction); - } - else if (requireFunctions) - { - throw new KernelException( - KernelException.ErrorCodes.FunctionNotAvailable, - $"Function '{plan.SkillName}.{plan.Name}' not found in skill collection"); - } - } - else - { - foreach (var step in plan.Steps) - { - SetAvailableFunctions(step, context, requireFunctions); - } - } - - return plan; - } - - /// - /// Add any missing variables from a plan state variables to the context. - /// - private static void AddVariablesToContext(ContextVariables vars, SKContext context) - { - // Loop through vars and add anything missing to context - foreach (var item in vars) - { - if (!context.Variables.ContainsKey(item.Key)) - { - context.Variables.Set(item.Key, item.Value); - } - } - } - - /// - /// Update the context with the outputs from the current step. - /// - /// The context to update. - /// The updated context. - private SKContext UpdateContextWithOutputs(SKContext context) - { - var resultString = this.State.TryGetValue(DefaultResultKey, out string? result) ? result : this.State.ToString(); - context.Variables.Update(resultString); - - // copy previous step's variables to the next step - foreach (var item in this._steps[this.NextStepIndex - 1].Outputs) - { - if (this.State.TryGetValue(item, out string? val)) - { - context.Variables.Set(item, val); - } - else - { - context.Variables.Set(item, resultString); - } - } - - return context; - } - - /// - /// Get the variables for the next step in the plan. - /// - /// The current context variables. - /// The next step in the plan. - /// The context variables for the next step in the plan. - private ContextVariables GetNextStepVariables(ContextVariables variables, Plan step) - { - // Priority for Input - // - Parameters (expand from variables if needed) - // - SKContext.Variables - // - Plan.State - // - Empty if sending to another plan - // - Plan.Description - - var input = string.Empty; - if (!string.IsNullOrEmpty(step.Parameters.Input)) - { - input = this.ExpandFromVariables(variables, step.Parameters.Input); - } - else if (!string.IsNullOrEmpty(variables.Input)) - { - input = variables.Input; - } - else if (!string.IsNullOrEmpty(this.State.Input)) - { - input = this.State.Input; - } - else if (step.Steps.Count > 0) - { - input = string.Empty; - } - else if (!string.IsNullOrEmpty(this.Description)) - { - input = this.Description; - } - - var stepVariables = new ContextVariables(input); - - // Priority for remaining stepVariables is: - // - Function Parameters (pull from variables or state by a key value) - // - Step Parameters (pull from variables or state by a key value) - // - All other variables. These are carried over in case the function wants access to the ambient content. - var functionParameters = step.Describe(); - foreach (var param in functionParameters.Parameters) - { - if (param.Name.Equals(ContextVariables.MainKey, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (variables.TryGetValue(param.Name, out string? value)) - { - stepVariables.Set(param.Name, value); - } - else if (this.State.TryGetValue(param.Name, out value) && !string.IsNullOrEmpty(value)) - { - stepVariables.Set(param.Name, value); - } - } - - foreach (var item in step.Parameters) - { - // Don't overwrite variable values that are already set - if (stepVariables.ContainsKey(item.Key)) - { - continue; - } - - var expandedValue = this.ExpandFromVariables(variables, item.Value); - if (!expandedValue.Equals(item.Value, StringComparison.OrdinalIgnoreCase)) - { - stepVariables.Set(item.Key, expandedValue); - } - else if (variables.TryGetValue(item.Key, out string? value)) - { - stepVariables.Set(item.Key, value); - } - else if (this.State.TryGetValue(item.Key, out value)) - { - stepVariables.Set(item.Key, value); - } - else - { - stepVariables.Set(item.Key, expandedValue); - } - } - - foreach (KeyValuePair item in variables) - { - if (!stepVariables.ContainsKey(item.Key)) - { - stepVariables.Set(item.Key, item.Value); - } - } - - return stepVariables; - } - - private void SetFunction(ISKFunction function) - { - this.Function = function; - this.Name = function.Name; - this.SkillName = function.SkillName; - this.Description = function.Description; - this.IsSemantic = function.IsSemantic; - this.RequestSettings = function.RequestSettings; - } - - private ISKFunction? Function { get; set; } = null; - - private readonly List _steps = new(); - - private static readonly Regex s_variablesRegex = new(@"\$(?\w+)"); - - private const string DefaultResultKey = "PLAN.RESULT"; - - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - private string DebuggerDisplay - { - get - { - string display = this.Description; - - if (!string.IsNullOrWhiteSpace(this.Name)) - { - display = $"{this.Name} ({display})"; - } - - if (this._steps.Count > 0) - { - display += $", Steps = {this._steps.Count}, NextStep = {this.NextStepIndex}"; - } - - return display; - } - } -} diff --git a/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs deleted file mode 100644 index c227ca74abd3..000000000000 --- a/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Net.Http; -using Microsoft.Extensions.Logging; - -namespace Microsoft.SemanticKernel.Reliability; - -public class NullHttpRetryHandlerFactory : IDelegatingHandlerFactory -{ - public DelegatingHandler Create(ILogger? logger) - { - return new NullHttpRetryHandler(); - } -} - -/// -/// A http retry handler that does not retry. -/// -public class NullHttpRetryHandler : DelegatingHandler -{ -} diff --git a/dotnet/src/SemanticKernel/SemanticFunctions/PromptTemplate.cs b/dotnet/src/SemanticKernel/SemanticFunctions/PromptTemplate.cs deleted file mode 100644 index 023b7a2f9b63..000000000000 --- a/dotnet/src/SemanticKernel/SemanticFunctions/PromptTemplate.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.TemplateEngine; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; - -namespace Microsoft.SemanticKernel.SemanticFunctions; - -/// -/// Prompt template. -/// -public sealed class PromptTemplate : IPromptTemplate -{ - private readonly string _template; - private readonly IPromptTemplateEngine _templateEngine; - - // ReSharper disable once NotAccessedField.Local - private readonly PromptTemplateConfig _promptConfig; - - /// - /// Constructor for PromptTemplate. - /// - /// Template. - /// Prompt template configuration. - /// Kernel in which template is to take effect. - public PromptTemplate(string template, PromptTemplateConfig promptTemplateConfig, IKernel kernel) - : this(template, promptTemplateConfig, kernel.PromptTemplateEngine) - { - } - - /// - /// Constructor for PromptTemplate. - /// - /// Template. - /// Prompt template configuration. - /// Prompt template engine. - public PromptTemplate( - string template, - PromptTemplateConfig promptTemplateConfig, - IPromptTemplateEngine promptTemplateEngine) - { - this._template = template; - this._templateEngine = promptTemplateEngine; - this._promptConfig = promptTemplateConfig; - } - - /// - /// Get the list of parameters used by the function, using JSON settings and template variables. - /// TODO: consider caching results - though cache invalidation will add extra complexity - /// - /// List of parameters - public IList GetParameters() - { - // Parameters from config.json - Dictionary result = new(StringComparer.OrdinalIgnoreCase); - foreach (var p in this._promptConfig.Input.Parameters) - { - result[p.Name] = new ParameterView(p.Name, p.Description, p.DefaultValue); - } - - // Parameters from the template - foreach (var block in this._templateEngine.ExtractBlocks(this._template)) - { - string? blockName = (block as VarBlock)?.Name; - if (!string.IsNullOrEmpty(blockName) && !result.ContainsKey(blockName!)) - { - result.Add(blockName!, new ParameterView(blockName!)); - } - } - - return result.Values.ToList(); - } - - /// - /// Render the template using the information in the context - /// - /// Kernel execution context helpers - /// The to monitor for cancellation requests. The default is . - /// Prompt rendered to string - public async Task RenderAsync(SKContext executionContext, CancellationToken cancellationToken) - { - return await this._templateEngine.RenderAsync(this._template, executionContext, cancellationToken).ConfigureAwait(false); - } -} diff --git a/dotnet/src/SemanticKernel/SemanticKernel.csproj b/dotnet/src/SemanticKernel/SemanticKernel.csproj deleted file mode 100644 index 32764f5e4829..000000000000 --- a/dotnet/src/SemanticKernel/SemanticKernel.csproj +++ /dev/null @@ -1,36 +0,0 @@ - - - - - Microsoft.SemanticKernel.Core - Microsoft.SemanticKernel - netstandard2.0 - true - true - - - - - - - Semantic Kernel Core - - Semantic Kernel core orchestration, runtime and skills. - This package is automatically installed by 'Microsoft.SemanticKernel' package with other useful packages. - Install this package manually only if you are selecting individual Semantic Kernel components. - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/dotnet/src/SemanticKernel/Services/AIServiceProvider.cs b/dotnet/src/SemanticKernel/Services/AIServiceProvider.cs deleted file mode 100644 index cd04485d3f7e..000000000000 --- a/dotnet/src/SemanticKernel/Services/AIServiceProvider.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; - -namespace Microsoft.SemanticKernel.Services; - -public class AIServiceProvider : NamedServiceProvider, IAIServiceProvider -{ - public AIServiceProvider(Dictionary>> services, Dictionary defaultIds) - : base(services, defaultIds) - { - } -} diff --git a/dotnet/src/SemanticKernel/Services/ServiceConfig.cs b/dotnet/src/SemanticKernel/Services/ServiceConfig.cs deleted file mode 100644 index e4a821f3565d..000000000000 --- a/dotnet/src/SemanticKernel/Services/ServiceConfig.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Services; - -public abstract class ServiceConfig : IServiceConfig -{ - /// - /// An identifier used to map semantic functions to AI services, - /// decoupling prompts configurations from the actual provider and model used. - /// - public string ServiceId { get; } - - /// - /// Creates a new with supplied values. - /// - /// An identifier used to map semantic functions to AI services and models. - protected ServiceConfig(string serviceId) - { - Verify.NotNullOrWhiteSpace(serviceId); - this.ServiceId = serviceId; - } -} diff --git a/dotnet/src/SemanticKernel/SkillDefinition/IReadOnlySkillCollectionTypeProxy.cs b/dotnet/src/SemanticKernel/SkillDefinition/IReadOnlySkillCollectionTypeProxy.cs deleted file mode 100644 index 37d0decb6f4a..000000000000 --- a/dotnet/src/SemanticKernel/SkillDefinition/IReadOnlySkillCollectionTypeProxy.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Debugger type proxy for . -/// -// ReSharper disable once InconsistentNaming -internal sealed class IReadOnlySkillCollectionTypeProxy -{ - private readonly IReadOnlySkillCollection _collection; - - public IReadOnlySkillCollectionTypeProxy(IReadOnlySkillCollection collection) => this._collection = collection; - - [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)] - public SkillProxy[] Items - { - get - { - var view = this._collection.GetFunctionsView(); - return view.NativeFunctions - .Concat(view.SemanticFunctions) - .GroupBy(f => f.Key) - .Select(g => new SkillProxy(g.SelectMany(f => f.Value)) { Name = g.Key }) - .ToArray(); - } - } - - [DebuggerDisplay("{Name}")] - public sealed class SkillProxy : List - { - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - public string? Name; - - public SkillProxy(IEnumerable functions) : base(functions) { } - } -} diff --git a/dotnet/src/SemanticKernel/SkillDefinition/ImportSemanticSkillFromDirectory.cs b/dotnet/src/SemanticKernel/SkillDefinition/ImportSemanticSkillFromDirectory.cs deleted file mode 100644 index b38044246852..000000000000 --- a/dotnet/src/SemanticKernel/SkillDefinition/ImportSemanticSkillFromDirectory.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.IO; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Text; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using the namespace of IKernel -namespace Microsoft.SemanticKernel; -#pragma warning restore IDE0130 - -/// -/// Class for extensions methods for importing semantic functions from a directory. -/// -public static class ImportSemanticSkillFromDirectoryExtension -{ - /// - /// Loads semantic functions, defined by prompt templates stored in the filesystem. - /// - /// - /// - /// A skill directory contains a set of subdirectories, one for each semantic function. - /// - /// - /// This method accepts the path of the parent directory (e.g. "d:\skills") and the name of the skill directory - /// (e.g. "OfficeSkill"), which is used also as the "skill name" in the internal skill collection (note that - /// skill and function names can contain only alphanumeric chars and underscore). - /// - /// - /// Example: - /// D:\skills\ # parentDirectory = "D:\skills" - /// - /// |__ OfficeSkill\ # skillDirectoryName = "SummarizeEmailThread" - /// - /// |__ ScheduleMeeting # semantic function - /// |__ skprompt.txt # prompt template - /// |__ config.json # settings (optional file) - /// - /// |__ SummarizeEmailThread # semantic function - /// |__ skprompt.txt # prompt template - /// |__ config.json # settings (optional file) - /// - /// |__ MergeWordAndExcelDocs # semantic function - /// |__ skprompt.txt # prompt template - /// |__ config.json # settings (optional file) - /// - /// |__ XboxSkill\ # another skill, etc. - /// - /// |__ MessageFriend - /// |__ skprompt.txt - /// |__ config.json - /// |__ LaunchGame - /// |__ skprompt.txt - /// |__ config.json - /// - /// - /// See https://github.com/microsoft/semantic-kernel/tree/main/samples/skills for examples in the Semantic Kernel repository. - /// - /// - /// Semantic Kernel instance - /// Directory containing the skill directory, e.g. "d:\myAppSkills" - /// Name of the directories containing the selected skills, e.g. "StrategySkill" - /// A list of all the semantic functions found in the directory, indexed by function name. - public static IDictionary ImportSemanticSkillFromDirectory( - this IKernel kernel, string parentDirectory, params string[] skillDirectoryNames) - { - const string ConfigFile = "config.json"; - const string PromptFile = "skprompt.txt"; - - var skill = new Dictionary(); - - foreach (string skillDirectoryName in skillDirectoryNames) - { - Verify.ValidSkillName(skillDirectoryName); - var skillDir = Path.Combine(parentDirectory, skillDirectoryName); - Verify.DirectoryExists(skillDir); - - string[] directories = Directory.GetDirectories(skillDir); - foreach (string dir in directories) - { - var functionName = Path.GetFileName(dir); - - // Continue only if prompt template exists - var promptPath = Path.Combine(dir, PromptFile); - if (!File.Exists(promptPath)) { continue; } - - // Load prompt configuration. Note: the configuration is optional. - var config = new PromptTemplateConfig(); - var configPath = Path.Combine(dir, ConfigFile); - if (File.Exists(configPath)) - { - config = PromptTemplateConfig.FromJson(File.ReadAllText(configPath)); - } - - kernel.Logger.LogTrace("Config {0}: {1}", functionName, config.ToJson()); - - // Load prompt template - var template = new PromptTemplate(File.ReadAllText(promptPath), config, kernel.PromptTemplateEngine); - - var functionConfig = new SemanticFunctionConfig(config, template); - - kernel.Logger.LogTrace("Registering function {0}.{1} loaded from {2}", skillDirectoryName, functionName, dir); - skill[functionName] = kernel.RegisterSemanticFunction(skillDirectoryName, functionName, functionConfig); - } - } - - return skill; - } -} diff --git a/dotnet/src/SemanticKernel/SkillDefinition/InlineFunctionsDefinitionExtension.cs b/dotnet/src/SemanticKernel/SkillDefinition/InlineFunctionsDefinitionExtension.cs deleted file mode 100644 index c34883cf49c5..000000000000 --- a/dotnet/src/SemanticKernel/SkillDefinition/InlineFunctionsDefinitionExtension.cs +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.SkillDefinition; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using the namespace of IKernel -namespace Microsoft.SemanticKernel; -#pragma warning restore IDE0130 - -/// -/// Class for extensions methods to define semantic functions. -/// -public static class InlineFunctionsDefinitionExtension -{ - /// - /// Define a string-to-string semantic function, with no direct support for input context. - /// The function can be referenced in templates and will receive the context, but when invoked programmatically you - /// can only pass in a string in input and receive a string in output. - /// - /// Semantic Kernel instance - /// Plain language definition of the semantic function, using SK template language - /// A name for the given function. The name can be referenced in templates and used by the pipeline planner. - /// Optional skill name, for namespacing and avoid collisions - /// Optional description, useful for the planner - /// Max number of tokens to generate - /// Temperature parameter passed to LLM - /// Top P parameter passed to LLM - /// Presence Penalty parameter passed to LLM - /// Frequency Penalty parameter passed to LLM - /// Strings the LLM will detect to stop generating (before reaching max tokens) - /// A function ready to use - public static ISKFunction CreateSemanticFunction( - this IKernel kernel, - string promptTemplate, - string? functionName = null, - string? skillName = null, - string? description = null, - int? maxTokens = null, - double temperature = 0, - double topP = 0, - double presencePenalty = 0, - double frequencyPenalty = 0, - IEnumerable? stopSequences = null) - { - functionName ??= RandomFunctionName(); - - var config = new PromptTemplateConfig - { - Description = description ?? "Generic function, unknown purpose", - Type = "completion", - Completion = new PromptTemplateConfig.CompletionConfig - { - Temperature = temperature, - TopP = topP, - PresencePenalty = presencePenalty, - FrequencyPenalty = frequencyPenalty, - MaxTokens = maxTokens, - StopSequences = stopSequences?.ToList() ?? new List() - } - }; - - return kernel.CreateSemanticFunction( - promptTemplate: promptTemplate, - config: config, - functionName: functionName, - skillName: skillName); - } - - /// - /// Allow to define a semantic function passing in the definition in natural language, i.e. the prompt template. - /// - /// Semantic Kernel instance - /// Plain language definition of the semantic function, using SK template language - /// Optional function settings - /// A name for the given function. The name can be referenced in templates and used by the pipeline planner. - /// An optional skill name, e.g. to namespace functions with the same name. When empty, - /// the function is added to the global namespace, overwriting functions with the same name - /// A function ready to use - public static ISKFunction CreateSemanticFunction( - this IKernel kernel, - string promptTemplate, - PromptTemplateConfig config, - string? functionName = null, - string? skillName = null) - { - functionName ??= RandomFunctionName(); - Verify.ValidFunctionName(functionName); - if (!string.IsNullOrEmpty(skillName)) { Verify.ValidSkillName(skillName); } - - var template = new PromptTemplate(promptTemplate, config, kernel.PromptTemplateEngine); - - // Prepare lambda wrapping AI logic - var functionConfig = new SemanticFunctionConfig(config, template); - - // TODO: manage overwrites, potentially error out - return string.IsNullOrEmpty(skillName) - ? kernel.RegisterSemanticFunction(functionName, functionConfig) - : kernel.RegisterSemanticFunction(skillName!, functionName, functionConfig); - } - - /// - /// Invoke a semantic function using the provided prompt template. - /// - /// Semantic Kernel instance - /// Plain language definition of the semantic function, using SK template language - /// A name for the given function. The name can be referenced in templates and used by the pipeline planner. - /// Optional skill name, for namespacing and avoid collisions - /// Optional description, useful for the planner - /// Max number of tokens to generate - /// Temperature parameter passed to LLM - /// Top P parameter passed to LLM - /// Presence Penalty parameter passed to LLM - /// Frequency Penalty parameter passed to LLM - /// Strings the LLM will detect to stop generating (before reaching max tokens) - /// A function ready to use - public static Task InvokeSemanticFunctionAsync( - this IKernel kernel, - string promptTemplate, - string? functionName = null, - string? skillName = null, - string? description = null, - int maxTokens = 256, - double temperature = 0, - double topP = 0, - double presencePenalty = 0, - double frequencyPenalty = 0, - IEnumerable? stopSequences = null) - { - var skfunction = kernel.CreateSemanticFunction( - promptTemplate, - functionName, - skillName, - description, - maxTokens, - temperature, - topP, - presencePenalty, - frequencyPenalty, - stopSequences); - - return skfunction.InvokeAsync(); - } - - private static string RandomFunctionName() => "func" + Guid.NewGuid().ToString("N"); -} diff --git a/dotnet/src/SemanticKernel/SkillDefinition/InstrumentedSKFunction.cs b/dotnet/src/SemanticKernel/SkillDefinition/InstrumentedSKFunction.cs deleted file mode 100644 index 003e7a9e8614..000000000000 --- a/dotnet/src/SemanticKernel/SkillDefinition/InstrumentedSKFunction.cs +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics; -using System.Diagnostics.Metrics; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Standard Semantic Kernel callable function with instrumentation. -/// -public sealed class InstrumentedSKFunction : ISKFunction -{ - /// - public string Name => this._function.Name; - - /// - public string SkillName => this._function.SkillName; - - /// - public string Description => this._function.Description; - - /// - public bool IsSemantic => this._function.IsSemantic; - - /// - public CompleteRequestSettings RequestSettings => this._function.RequestSettings; - - /// - /// Initialize a new instance of the class. - /// - /// Instance of to decorate. - /// Optional logger. - public InstrumentedSKFunction( - ISKFunction function, - ILogger? logger = null) - { - this._function = function; - this._logger = logger ?? NullLogger.Instance; - - this._executionTimeHistogram = s_meter.CreateHistogram( - name: $"SK.{this.SkillName}.{this.Name}.ExecutionTime", - unit: "ms", - description: "Duration of function execution"); - - this._executionTotalCounter = s_meter.CreateCounter( - name: $"SK.{this.SkillName}.{this.Name}.ExecutionTotal", - description: "Total number of function executions"); - - this._executionSuccessCounter = s_meter.CreateCounter( - name: $"SK.{this.SkillName}.{this.Name}.ExecutionSuccess", - description: "Number of successful function executions"); - - this._executionFailureCounter = s_meter.CreateCounter( - name: $"SK.{this.SkillName}.{this.Name}.ExecutionFailure", - description: "Number of failed function executions"); - } - - /// - public FunctionView Describe() => - this._function.Describe(); - - /// - public async Task InvokeAsync( - SKContext context, - CompleteRequestSettings? settings = null, - CancellationToken cancellationToken = default) - { - return await this.InvokeWithInstrumentationAsync(() => - this._function.InvokeAsync(context, settings, cancellationToken)).ConfigureAwait(false); - } - - /// - public async Task InvokeAsync( - string? input = null, - CompleteRequestSettings? settings = null, - ILogger? logger = null, - CancellationToken cancellationToken = default) - { - return await this.InvokeWithInstrumentationAsync(() => - this._function.InvokeAsync(input, settings, logger ?? this._logger, cancellationToken)).ConfigureAwait(false); - } - - /// - public ISKFunction SetAIConfiguration(CompleteRequestSettings settings) => - this._function.SetAIConfiguration(settings); - - /// - public ISKFunction SetAIService(Func serviceFactory) => - this._function.SetAIService(serviceFactory); - - /// - public ISKFunction SetDefaultSkillCollection(IReadOnlySkillCollection skills) => - this._function.SetDefaultSkillCollection(skills); - - #region private ================================================================================ - - private readonly ISKFunction _function; - private readonly ILogger _logger; - - /// - /// Instance of for function-related activities. - /// - private static ActivitySource s_activitySource = new(typeof(SKFunction).FullName); - - /// - /// Instance of for function-related metrics. - /// - private static Meter s_meter = new(typeof(SKFunction).FullName); - - /// - /// Instance of to measure and track the time of function execution. - /// - private Histogram _executionTimeHistogram; - - /// - /// Instance of to keep track of the total number of function executions. - /// - private Counter _executionTotalCounter; - - /// - /// Instance of to keep track of the number of successful function executions. - /// - private Counter _executionSuccessCounter; - - /// - /// Instance of to keep track of the number of failed function executions. - /// - private Counter _executionFailureCounter; - - /// - /// Wrapper for instrumentation to be used in multiple invocation places. - /// - /// Delegate to instrument. - private async Task InvokeWithInstrumentationAsync(Func> func) - { - using var activity = s_activitySource.StartActivity($"{this.SkillName}.{this.Name}"); - - this._logger.LogInformation("{SkillName}.{FunctionName}: Function execution started.", this.SkillName, this.Name); - - var stopwatch = new Stopwatch(); - - stopwatch.Start(); - - var result = await func().ConfigureAwait(false); - - stopwatch.Stop(); - - if (result.ErrorOccurred) - { - this._logger.LogWarning("{SkillName}.{FunctionName}: Function execution status: {Status}", - this.SkillName, this.Name, "Failed"); - - this._logger.LogError(result.LastException, "{SkillName}.{FunctionName}: Function execution exception details: {Message}", - this.SkillName, this.Name, result.LastErrorDescription); - - this._executionFailureCounter.Add(1); - } - else - { - this._logger.LogInformation("{SkillName}.{FunctionName}: Function execution status: {Status}", - this.SkillName, this.Name, "Success"); - - this._logger.LogInformation("{SkillName}.{FunctionName}: Function execution finished in {ExecutionTime}ms", - this.SkillName, this.Name, stopwatch.ElapsedMilliseconds); - - this._executionSuccessCounter.Add(1); - } - - this._executionTotalCounter.Add(1); - this._executionTimeHistogram.Record(stopwatch.ElapsedMilliseconds); - - return result; - } - - #endregion -} diff --git a/dotnet/src/SemanticKernel/SkillDefinition/SKFunction.cs b/dotnet/src/SemanticKernel/SkillDefinition/SKFunction.cs deleted file mode 100644 index 3871e46d226e..000000000000 --- a/dotnet/src/SemanticKernel/SkillDefinition/SKFunction.cs +++ /dev/null @@ -1,979 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.ComponentModel; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Linq; -using System.Reflection; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SemanticFunctions; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -#pragma warning disable format - -/// -/// Standard Semantic Kernel callable function. -/// SKFunction is used to extend one C# , , , -/// with additional methods required by the kernel. -/// -[DebuggerDisplay("{DebuggerDisplay,nq}")] -public sealed class SKFunction : ISKFunction, IDisposable -{ - /// - public string Name { get; } - - /// - public string SkillName { get; } - - /// - public string Description { get; } - - /// - public bool IsSemantic { get; } - - /// - public CompleteRequestSettings RequestSettings => this._aiRequestSettings; - - /// - /// List of function parameters - /// - public IList Parameters { get; } - - /// - /// Create a native function instance, wrapping a native object method - /// - /// Signature of the method to invoke - /// Object containing the method to invoke - /// SK skill name - /// Application logger - /// SK function instance - public static ISKFunction FromNativeMethod( - MethodInfo method, - object? target = null, - string? skillName = null, - ILogger? logger = null) - { - if (!method.IsStatic && target is null) - { - throw new ArgumentNullException(nameof(target), "Argument cannot be null for non-static methods"); - } - - if (string.IsNullOrWhiteSpace(skillName)) - { - skillName = SkillCollection.GlobalSkill; - } - - MethodDetails methodDetails = GetMethodDetails(method, target, logger); - - return new SKFunction( - delegateFunction: methodDetails.Function, - parameters: methodDetails.Parameters, - skillName: skillName!, - functionName: methodDetails.Name, - isSemantic: false, - description: methodDetails.Description, - logger: logger); - } - - /// - /// Create a native function instance, wrapping a delegate function - /// - /// Function to invoke - /// SK skill name - /// SK function name - /// SK function description - /// SK function parameters - /// Application logger - /// SK function instance - public static ISKFunction FromNativeFunction( - Delegate nativeFunction, - string? skillName = null, - string? functionName = null, - string? description = null, - IEnumerable? parameters = null, - ILogger? logger = null) - { - MethodDetails methodDetails = GetMethodDetails(nativeFunction.Method, nativeFunction.Target, logger); - - functionName ??= nativeFunction.Method.Name; - description ??= string.Empty; - - if (string.IsNullOrWhiteSpace(skillName)) - { - skillName = SkillCollection.GlobalSkill; - } - - return new SKFunction( - delegateFunction: methodDetails.Function, - parameters: parameters is not null ? parameters.ToList() : (IList)Array.Empty(), - description: description, - skillName: skillName!, - functionName: functionName, - isSemantic: false, - logger: logger); - } - - /// - /// Create a native function instance, given a semantic function configuration. - /// - /// Name of the skill to which the function to create belongs. - /// Name of the function to create. - /// Semantic function configuration. - /// Optional logger for the function. - /// The to monitor for cancellation requests. The default is . - /// SK function instance. - public static ISKFunction FromSemanticConfig( - string skillName, - string functionName, - SemanticFunctionConfig functionConfig, - ILogger? logger = null, - CancellationToken cancellationToken = default) - { - Verify.NotNull(functionConfig); - - Task LocalFuncTmp( - ITextCompletion? client, - CompleteRequestSettings? requestSettings, - SKContext context, - CancellationToken cancellationToken) - { - return Task.FromResult(context); - } - - var func = new SKFunction( - // Start with an empty delegate, so we can have a reference to func - // to be used in the LocalFunc below - // Before returning the delegateFunction will be updated to be LocalFunc - delegateFunction: LocalFuncTmp, - parameters: functionConfig.PromptTemplate.GetParameters(), - description: functionConfig.PromptTemplateConfig.Description, - skillName: skillName, - functionName: functionName, - isSemantic: true, - logger: logger - ); - - async Task LocalFunc( - ITextCompletion? client, - CompleteRequestSettings? requestSettings, - SKContext context, - CancellationToken cancellationToken) - { - Verify.NotNull(client); - Verify.NotNull(requestSettings); - - try - { - string renderedPrompt = await functionConfig.PromptTemplate.RenderAsync(context, cancellationToken).ConfigureAwait(false); - var completionResults = await client.GetCompletionsAsync(renderedPrompt, requestSettings, cancellationToken).ConfigureAwait(false); - string completion = await GetCompletionsResultContentAsync(completionResults, cancellationToken).ConfigureAwait(false); - - // Update the result with the completion - context.Variables.Update(completion); - - context.ModelResults = completionResults.Select(c => c.ModelResult).ToArray(); - } - catch (AIException ex) - { - const string Message = "Something went wrong while rendering the semantic function" + - " or while executing the text completion. Function: {0}.{1}. Error: {2}. Details: {3}"; - logger?.LogError(ex, Message, skillName, functionName, ex.Message, ex.Detail); - context.Fail(ex.Message, ex); - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - const string Message = "Something went wrong while rendering the semantic function" + - " or while executing the text completion. Function: {0}.{1}. Error: {2}"; - logger?.LogError(ex, Message, skillName, functionName, ex.Message); - context.Fail(ex.Message, ex); - } - - return context; - } - - // Update delegate function with a reference to the LocalFunc created - func._function = LocalFunc; - - return func; - } - - /// - public FunctionView Describe() - { - return new FunctionView - { - IsSemantic = this.IsSemantic, - Name = this.Name, - SkillName = this.SkillName, - Description = this.Description, - Parameters = this.Parameters, - }; - } - - /// - public async Task InvokeAsync(SKContext context, CompleteRequestSettings? settings = null, CancellationToken cancellationToken = default) - { - if (this.IsSemantic) - { - this.AddDefaultValues(context.Variables); - - var resultContext = await this._function(this._aiService?.Value, settings ?? this._aiRequestSettings, context, cancellationToken).ConfigureAwait(false); - context.Variables.Update(resultContext.Variables); - } - else - { - try - { - context = await this._function(null, settings, context, cancellationToken).ConfigureAwait(false); - } - catch (Exception e) when (!e.IsCriticalException()) - { - const string Message = "Something went wrong while executing the native function. Function: {0}. Error: {1}"; - this._logger.LogError(e, Message, this._function.Method.Name, e.Message); - context.Fail(e.Message, e); - } - } - - return context; - } - - /// - public Task InvokeAsync( - string? input = null, - CompleteRequestSettings? settings = null, - ILogger? logger = null, - CancellationToken cancellationToken = default) - { - SKContext context = new( - new ContextVariables(input), - skills: this._skillCollection, - logger: logger); - - return this.InvokeAsync(context, settings, cancellationToken: cancellationToken); - } - - /// - public ISKFunction SetDefaultSkillCollection(IReadOnlySkillCollection skills) - { - this._skillCollection = skills; - return this; - } - - /// - public ISKFunction SetAIService(Func serviceFactory) - { - Verify.NotNull(serviceFactory); - this.VerifyIsSemantic(); - this._aiService = new Lazy(serviceFactory); - return this; - } - - /// - public ISKFunction SetAIConfiguration(CompleteRequestSettings settings) - { - Verify.NotNull(settings); - this.VerifyIsSemantic(); - this._aiRequestSettings = settings; - return this; - } - - /// - /// Dispose of resources. - /// - public void Dispose() - { - if (this._aiService is { IsValueCreated: true } aiService) - { - (aiService.Value as IDisposable)?.Dispose(); - } - } - - /// - /// JSON serialized string representation of the function. - /// - public override string ToString() - => this.ToString(false); - - /// - /// JSON serialized string representation of the function. - /// - public string ToString(bool writeIndented) - => JsonSerializer.Serialize(this, options: writeIndented ? s_toStringIndentedSerialization : s_toStringStandardSerialization); - - #region private - - private static readonly JsonSerializerOptions s_toStringStandardSerialization = new(); - private static readonly JsonSerializerOptions s_toStringIndentedSerialization = new() { WriteIndented = true }; - private Func> _function; - private readonly ILogger _logger; - private IReadOnlySkillCollection? _skillCollection; - private Lazy? _aiService = null; - private CompleteRequestSettings _aiRequestSettings = new(); - - private struct MethodDetails - { - public Func> Function { get; set; } - public List Parameters { get; set; } - public string Name { get; set; } - public string Description { get; set; } - } - - private static async Task GetCompletionsResultContentAsync(IReadOnlyList completions, CancellationToken cancellationToken = default) - { - // To avoid any unexpected behavior we only take the first completion result (when running from the Kernel) - return await completions[0].GetCompletionAsync(cancellationToken).ConfigureAwait(false); - } - - internal SKFunction( - Func> delegateFunction, - IList parameters, - string skillName, - string functionName, - string description, - bool isSemantic = false, - ILogger? logger = null) - { - Verify.NotNull(delegateFunction); - Verify.ValidSkillName(skillName); - Verify.ValidFunctionName(functionName); - Verify.ParametersUniqueness(parameters); - - this._logger = logger ?? NullLogger.Instance; - - this._function = delegateFunction; - this.Parameters = parameters; - - this.IsSemantic = isSemantic; - this.Name = functionName; - this.SkillName = skillName; - this.Description = description; - } - - /// - /// Throw an exception if the function is not semantic, use this method when some logic makes sense only for semantic functions. - /// - /// - private void VerifyIsSemantic() - { - if (this.IsSemantic) { return; } - - this._logger.LogError("The function is not semantic"); - throw new KernelException( - KernelException.ErrorCodes.InvalidFunctionType, - "Invalid operation, the method requires a semantic function"); - } - - private static MethodDetails GetMethodDetails( - MethodInfo method, - object? target, - ILogger? logger = null) - { - Verify.NotNull(method); - - // Get the name to use for the function. If the function has an SKName attribute, we use that. - // Otherwise, we use the name of the method, but strip off any "Async" suffix if it's {Value}Task-returning. - // We don't apply any heuristics to the value supplied by SKName so that it can always be used - // as a definitive override. - string? functionName = method.GetCustomAttribute(inherit: true)?.Name?.Trim(); - if (string.IsNullOrEmpty(functionName)) - { - functionName = SanitizeMetadataName(method.Name!); - Verify.ValidFunctionName(functionName); - - if (IsAsyncMethod(method) && - functionName.EndsWith("Async", StringComparison.Ordinal) && - functionName.Length > "Async".Length) - { - functionName = functionName.Substring(0, functionName.Length - "Async".Length); - } - } - - SKFunctionAttribute? functionAttribute = method.GetCustomAttribute(inherit: true); - - string? description = method.GetCustomAttribute(inherit: true)?.Description; - - var result = new MethodDetails - { - Name = functionName!, - Description = description ?? string.Empty, - }; - - (result.Function, result.Parameters) = GetDelegateInfo(target, method); - - logger?.LogTrace("Method '{0}' found", result.Name); - - return result; - } - - /// Gets whether a method has a known async return type. - private static bool IsAsyncMethod(MethodInfo method) - { - Type t = method.ReturnType; - - if (t == typeof(Task) || t == typeof(ValueTask)) - { - return true; - } - - if (t.IsGenericType) - { - t = t.GetGenericTypeDefinition(); - if (t == typeof(Task<>) || t == typeof(ValueTask<>)) - { - return true; - } - } - - return false; - } - - // Inspect a method and returns the corresponding delegate and related info - private static (Func> function, List) GetDelegateInfo(object? instance, MethodInfo method) - { - ThrowForInvalidSignatureIf(method.IsGenericMethodDefinition, method, "Generic methods are not supported"); - - var stringParameterViews = new List(); - var parameters = method.GetParameters(); - - // Get marshaling funcs for parameters and build up the parameter views. - var parameterFuncs = new Func[parameters.Length]; - bool sawFirstParameter = false, hasSKContextParam = false, hasCancellationTokenParam = false, hasLoggerParam = false, hasMemoryParam = false, hasCultureParam = false; - for (int i = 0; i < parameters.Length; i++) - { - (parameterFuncs[i], ParameterView? parameterView) = GetParameterMarshalerDelegate( - method, parameters[i], - ref sawFirstParameter, ref hasSKContextParam, ref hasCancellationTokenParam, ref hasLoggerParam, ref hasMemoryParam, ref hasCultureParam); - if (parameterView is not null) - { - stringParameterViews.Add(parameterView); - } - } - - // Get marshaling func for the return value. - Func> returnFunc = GetReturnValueMarshalerDelegate(method); - - // Create the func - Func> function = (_, _, context, cancellationToken) => - { - // Create the arguments. - object?[] args = parameterFuncs.Length != 0 ? new object?[parameterFuncs.Length] : Array.Empty(); - for (int i = 0; i < args.Length; i++) - { - args[i] = parameterFuncs[i](context, cancellationToken); - } - - // Invoke the method. - object? result = method.Invoke(instance, args); - - // Extract and return the result. - return returnFunc(result, context); - }; - - // Add parameters applied to the method that aren't part of the signature. - stringParameterViews.AddRange(method - .GetCustomAttributes(inherit: true) - .Select(x => new ParameterView(x.Name ?? string.Empty, x.Description ?? string.Empty, x.DefaultValue ?? string.Empty))); - - // Check for param names conflict - Verify.ParametersUniqueness(stringParameterViews); - - // Return the function and its parameter views. - return (function, stringParameterViews); - } - - /// - /// Gets a delegate for handling the marshaling of a parameter. - /// - private static (Func, ParameterView?) GetParameterMarshalerDelegate( - MethodInfo method, ParameterInfo parameter, - ref bool sawFirstParameter, ref bool hasSKContextParam, ref bool hasCancellationTokenParam, ref bool hasLoggerParam, ref bool hasMemoryParam, ref bool hasCultureParam) - { - Type type = parameter.ParameterType; - - // Handle special types based on SKContext data. These can each show up at most once in the method signature, - // with the SKContext itself or the primary data from it mapped directly into the method's parameter. - // They do not get parameter views as they're not supplied from context variables. - - if (type == typeof(SKContext)) - { - TrackUniqueParameterType(ref hasSKContextParam, method, $"At most one {nameof(SKContext)} parameter is permitted."); - return (static (SKContext context, CancellationToken _) => context, null); - } - - if (type == typeof(ILogger)) - { - TrackUniqueParameterType(ref hasLoggerParam, method, $"At most one {nameof(ILogger)} parameter is permitted."); - return (static (SKContext context, CancellationToken _) => context.Logger, null); - } - - if (type == typeof(CultureInfo) || type == typeof(IFormatProvider)) - { - TrackUniqueParameterType(ref hasCultureParam, method, $"At most one {nameof(CultureInfo)}/{nameof(IFormatProvider)} parameter is permitted."); - return (static (SKContext context, CancellationToken _) => context.Culture, null); - } - - if (type == typeof(CancellationToken)) - { - TrackUniqueParameterType(ref hasCancellationTokenParam, method, $"At most one {nameof(CancellationToken)} parameter is permitted."); - return (static (SKContext _, CancellationToken cancellationToken) => cancellationToken, null); - } - - // Handle context variables. These are supplied from the SKContext's Variables dictionary. - - if (!type.IsByRef && GetParser(type) is Func parser) - { - // Use either the parameter's name or an override from an applied SKName attribute. - SKNameAttribute? nameAttr = parameter.GetCustomAttribute(inherit: true); - string name = nameAttr?.Name?.Trim() ?? SanitizeMetadataName(parameter.Name); - bool nameIsInput = name.Equals("input", StringComparison.OrdinalIgnoreCase); - ThrowForInvalidSignatureIf(name.Length == 0, method, $"Parameter {parameter.Name}'s context attribute defines an invalid name."); - ThrowForInvalidSignatureIf(sawFirstParameter && nameIsInput, method, "Only the first parameter may be named 'input'"); - - // Use either the parameter's optional default value as contained in parameter metadata (e.g. `string s = "hello"`) - // or an override from an applied SKParameter attribute. Note that a default value may be null. - DefaultValueAttribute defaultValueAttribute = parameter.GetCustomAttribute(inherit: true); - bool hasDefaultValue = defaultValueAttribute is not null; - object? defaultValue = defaultValueAttribute?.Value; - if (!hasDefaultValue && parameter.HasDefaultValue) - { - hasDefaultValue = true; - defaultValue = parameter.DefaultValue; - } - - if (hasDefaultValue) - { - // If we got a default value, make sure it's of the right type. This currently supports - // null values if the target type is a reference type or a Nullable, strings, - // anything that can be parsed from a string via a registered TypeConverter, - // and a value that's already the same type as the parameter. - if (defaultValue is string defaultStringValue && defaultValue.GetType() != typeof(string)) - { - // Invariant culture is used here as this value comes from the C# source - // and it should be deterministic across cultures. - defaultValue = parser(defaultStringValue, CultureInfo.InvariantCulture); - } - else - { - ThrowForInvalidSignatureIf( - defaultValue is null && type.IsValueType && Nullable.GetUnderlyingType(type) is null, - method, - $"Type {type} is a non-nullable value type but a null default value was specified."); - ThrowForInvalidSignatureIf( - defaultValue is not null && !type.IsAssignableFrom(defaultValue.GetType()), - method, - $"Default value {defaultValue} for parameter {name} is not assignable to type {type}."); - } - } - - bool fallBackToInput = !sawFirstParameter && !nameIsInput; - Func parameterFunc = (SKContext context, CancellationToken _) => - { - // 1. Use the value of the variable if it exists. - if (context.Variables.TryGetValue(name, out string? value)) - { - return Process(value); - } - - // 2. Otherwise, use the default value if there is one, sourced either from an attribute or the parameter's default. - if (hasDefaultValue) - { - return defaultValue; - } - - // 3. Otherwise, use "input" if this is the first (or only) parameter. - if (fallBackToInput) - { - return Process(context.Variables.Input); - } - - // 4. Otherwise, fail. - throw new KernelException(KernelException.ErrorCodes.FunctionInvokeError, $"Missing value for parameter '{name}'"); - - object? Process(string value) - { - if (type == typeof(string)) - { - return value; - } - - try - { - return parser(value, context.Culture); - } - catch (Exception e) when (!e.IsCriticalException()) - { - throw new ArgumentOutOfRangeException(name, value, e.Message); - } - } - }; - - sawFirstParameter = true; - - var parameterView = new ParameterView( - name, - parameter.GetCustomAttribute(inherit: true)?.Description ?? string.Empty, - defaultValue?.ToString() ?? string.Empty); - - return (parameterFunc, parameterView); - } - - // Fail for unknown parameter types. - throw GetExceptionForInvalidSignature(method, $"Unknown parameter type {parameter.ParameterType}"); - } - - /// - /// Gets a delegate for handling the result value of a method, converting it into the to return from the invocation. - /// - private static Func> GetReturnValueMarshalerDelegate(MethodInfo method) - { - // Handle each known return type for the method - Type returnType = method.ReturnType; - - // No return value, either synchronous (void) or asynchronous (Task / ValueTask). - - if (returnType == typeof(void)) - { - return static (result, context) => Task.FromResult(context); - } - - if (returnType == typeof(Task)) - { - return async static (result, context) => - { - await ((Task)ThrowIfNullResult(result)).ConfigureAwait(false); - return context; - }; - } - - if (returnType == typeof(ValueTask)) - { - return async static (result, context) => - { - await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(false); - return context; - }; - } - - // SKContext, either synchronous (SKContext) or asynchronous (Task / ValueTask). - - if (returnType == typeof(SKContext)) - { - return static (result, _) => Task.FromResult((SKContext)ThrowIfNullResult(result)); - } - - if (returnType == typeof(Task)) - { - return static (result, _) => (Task)ThrowIfNullResult(result); - } - - if (returnType == typeof(ValueTask)) - { - return static (result, context) => ((ValueTask)ThrowIfNullResult(result)).AsTask(); - } - - // string (which is special as no marshaling is required), either synchronous (string) or asynchronous (Task / ValueTask) - - if (returnType == typeof(string)) - { - return static (result, context) => - { - context.Variables.Update((string?)result); - return Task.FromResult(context); - }; - } - - if (returnType == typeof(Task)) - { - return async static (result, context) => - { - context.Variables.Update(await ((Task)ThrowIfNullResult(result)).ConfigureAwait(false)); - return context; - }; - } - - if (returnType == typeof(ValueTask)) - { - return async static (result, context) => - { - context.Variables.Update(await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(false)); - return context; - }; - } - - // All other synchronous return types T. - - if (!returnType.IsGenericType || returnType.GetGenericTypeDefinition() == typeof(Nullable<>)) - { - if (GetFormatter(returnType) is not Func formatter) - { - throw GetExceptionForInvalidSignature(method, $"Unknown return type {returnType}"); - } - - return (result, context) => - { - context.Variables.Update(formatter(result, context.Culture)); - return Task.FromResult(context); - }; - } - - // All other asynchronous return types - - // Task - if (returnType.GetGenericTypeDefinition() is Type genericTask && - genericTask == typeof(Task<>) && - returnType.GetProperty("Result", BindingFlags.Public | BindingFlags.Instance)?.GetGetMethod() is MethodInfo taskResultGetter && - GetFormatter(taskResultGetter.ReturnType) is Func taskResultFormatter) - { - return async (result, context) => - { - await ((Task)ThrowIfNullResult(result)).ConfigureAwait(false); - context.Variables.Update(taskResultFormatter(taskResultGetter.Invoke(result!, Array.Empty()), context.Culture)); - return context; - }; - } - - // ValueTask - if (returnType.GetGenericTypeDefinition() is Type genericValueTask && - genericValueTask == typeof(ValueTask<>) && - returnType.GetMethod("AsTask", BindingFlags.Public | BindingFlags.Instance) is MethodInfo valueTaskAsTask && - valueTaskAsTask.ReturnType.GetProperty("Result", BindingFlags.Public | BindingFlags.Instance)?.GetGetMethod() is MethodInfo asTaskResultGetter && - GetFormatter(asTaskResultGetter.ReturnType) is Func asTaskResultFormatter) - { - return async (result, context) => - { - Task task = (Task)valueTaskAsTask.Invoke(ThrowIfNullResult(result), Array.Empty()); - await task.ConfigureAwait(false); - context.Variables.Update(asTaskResultFormatter(asTaskResultGetter.Invoke(task!, Array.Empty()), context.Culture)); - return context; - }; - } - - // Unrecognized return type. - throw GetExceptionForInvalidSignature(method, $"Unknown return type {returnType}"); - - // Throws an exception if a result is found to be null unexpectedly - static object ThrowIfNullResult(object? result) => - result ?? - throw new KernelException(KernelException.ErrorCodes.FunctionInvokeError, "Function returned null unexpectedly."); - } - - /// Gets an exception that can be thrown indicating an invalid signature. - [DoesNotReturn] - private static Exception GetExceptionForInvalidSignature(MethodInfo method, string reason) => - throw new KernelException( - KernelException.ErrorCodes.FunctionTypeNotSupported, - $"Function '{method.Name}' is not supported by the kernel. {reason}"); - - /// Throws an exception indicating an invalid SKFunction signature if the specified condition is not met. - private static void ThrowForInvalidSignatureIf([DoesNotReturnIf(true)] bool condition, MethodInfo method, string reason) - { - if (condition) - { - throw GetExceptionForInvalidSignature(method, reason); - } - } - - /// Tracks whether a particular kind of parameter has been seen, throwing an exception if it has, and marking it as seen if it hasn't - private static void TrackUniqueParameterType(ref bool hasParameterType, MethodInfo method, string failureMessage) - { - ThrowForInvalidSignatureIf(hasParameterType, method, failureMessage); - hasParameterType = true; - } - - /// - /// Gets a TypeConverter-based parser for parsing a string as the target type. - /// - /// Specifies the target type into which a string should be parsed. - /// The parsing function if the target type is supported; otherwise, null. - /// - /// The parsing function uses whatever TypeConverter is registered for the target type. - /// Parsing is first attempted using the current culture, and if that fails, it tries again - /// with the invariant culture. If both fail, an exception is thrown. - /// - private static Func? GetParser(Type targetType) => - s_parsers.GetOrAdd(targetType, static targetType => - { - // Strings just parse to themselves. - if (targetType == typeof(string)) - { - return (input, cultureInfo) => input; - } - - // For nullables, parse as the inner type. We then just need to be careful to treat null as null, - // as the underlying parser might not be expecting null. - bool wasNullable = false; - if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) - { - wasNullable = true; - targetType = Nullable.GetUnderlyingType(targetType); - } - - // For enums, delegate to Enum.Parse, special-casing null if it was actually Nullable. - if (targetType.IsEnum) - { - return (input, cultureInfo) => - { - if (wasNullable && input is null) - { - return null!; - } - - return Enum.Parse(targetType, input, ignoreCase: true); - }; - } - - // Finally, look up and use a type converter. Again, special-case null if it was actually Nullable. - if (GetTypeConverter(targetType) is TypeConverter converter && converter.CanConvertFrom(typeof(string))) - { - return (input, cultureInfo) => - { - if (wasNullable && input is null) - { - return null!; - } - - // First try to parse using the supplied culture (or current if none was supplied). - // If that fails, try with the invariant culture and allow any exception to propagate. - try - { - return converter.ConvertFromString(context: null, cultureInfo, input); - } - catch (Exception e) when (!e.IsCriticalException() && cultureInfo != CultureInfo.InvariantCulture) - { - return converter.ConvertFromInvariantString(input); - } - }; - } - - // Unsupported type. - return null; - }); - - /// - /// Gets a TypeConverter-based formatter for formatting an object as a string. - /// - /// - /// Formatting is performed in the invariant culture whenever possible. - /// - private static Func? GetFormatter(Type targetType) => - s_formatters.GetOrAdd(targetType, static targetType => - { - // For nullables, render as the underlying type. - bool wasNullable = false; - if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) - { - wasNullable = true; - targetType = Nullable.GetUnderlyingType(targetType); - } - - // For enums, just ToString() and allow the object override to do the right thing. - if (targetType.IsEnum) - { - return (input, cultureInfo) => input?.ToString()!; - } - - // Strings just render as themselves. - if (targetType == typeof(string)) - { - return (input, cultureInfo) => (string)input!; - } - - // Finally, look up and use a type converter. - if (GetTypeConverter(targetType) is TypeConverter converter && converter.CanConvertTo(typeof(string))) - { - return (input, cultureInfo) => - { - if (wasNullable && input is null) - { - return null!; - } - - return converter.ConvertToString(context: null, cultureInfo, input); - }; - } - - return null; - }); - - private static TypeConverter? GetTypeConverter(Type targetType) - { - // In an ideal world, this would use TypeDescriptor.GetConverter. However, that is not friendly to - // any form of ahead-of-time compilation, as it could end up requiring functionality that was trimmed. - // Instead, we just use a hard-coded set of converters for the types we know about and then also support - // types that are explicitly attributed with TypeConverterAttribute. - - if (targetType == typeof(byte)) { return new ByteConverter(); } - if (targetType == typeof(sbyte)) { return new SByteConverter(); } - if (targetType == typeof(bool)) { return new BooleanConverter(); } - if (targetType == typeof(ushort)) { return new UInt16Converter(); } - if (targetType == typeof(short)) { return new Int16Converter(); } - if (targetType == typeof(char)) { return new CharConverter(); } - if (targetType == typeof(uint)) { return new UInt32Converter(); } - if (targetType == typeof(int)) { return new Int32Converter(); } - if (targetType == typeof(ulong)) { return new UInt64Converter(); } - if (targetType == typeof(long)) { return new Int64Converter(); } - if (targetType == typeof(float)) { return new SingleConverter(); } - if (targetType == typeof(double)) { return new DoubleConverter(); } - if (targetType == typeof(decimal)) { return new DecimalConverter(); } - if (targetType == typeof(TimeSpan)) { return new TimeSpanConverter(); } - if (targetType == typeof(DateTime)) { return new DateTimeConverter(); } - if (targetType == typeof(DateTimeOffset)) { return new DateTimeOffsetConverter(); } - if (targetType == typeof(Uri)) { return new UriTypeConverter(); } - if (targetType == typeof(Guid)) { return new GuidConverter(); } - - if (targetType.GetCustomAttribute() is TypeConverterAttribute tca && - Type.GetType(tca.ConverterTypeName, throwOnError: false) is Type converterType && - Activator.CreateInstance(converterType) is TypeConverter converter) - { - return converter; - } - - return null; - } - - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - private string DebuggerDisplay => $"{this.Name} ({this.Description})"; - - /// - /// Remove characters from method name that are valid in metadata but invalid for SK. - /// - private static string SanitizeMetadataName(string methodName) => - s_invalidNameCharsRegex.Replace(methodName, "_"); - - /// Regex that flags any character other than ASCII digits or letters or the underscore. - private static readonly Regex s_invalidNameCharsRegex = new("[^0-9A-Za-z_]"); - - /// Parser functions for converting strings to parameter types. - private static readonly ConcurrentDictionary?> s_parsers = new(); - - /// Formatter functions for converting parameter types to strings. - private static readonly ConcurrentDictionary?> s_formatters = new(); - - /// Add default values to the context variables if the variable is not defined - private void AddDefaultValues(ContextVariables variables) - { - foreach (var parameter in this.Parameters) - { - if (!variables.ContainsKey(parameter.Name) && parameter.DefaultValue != null) - { - variables[parameter.Name] = parameter.DefaultValue; - } - } - } - - #endregion -} diff --git a/dotnet/src/SemanticKernel/SkillDefinition/SKFunctionExtensions.cs b/dotnet/src/SemanticKernel/SkillDefinition/SKFunctionExtensions.cs deleted file mode 100644 index 2b3577d1a1a5..000000000000 --- a/dotnet/src/SemanticKernel/SkillDefinition/SKFunctionExtensions.cs +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Class that holds extension methods for objects implementing ISKFunction. -/// -public static class SKFunctionExtensions -{ - /// - /// Configure the LLM settings used by semantic function. - /// - /// Semantic function - /// Completion settings - /// Self instance - public static ISKFunction UseCompletionSettings(this ISKFunction skFunction, CompleteRequestSettings settings) - { - return skFunction.SetAIConfiguration(settings); - } - - /// - /// Change the LLM Max Tokens configuration - /// - /// Semantic function - /// Tokens count - /// Self instance - public static ISKFunction UseMaxTokens(this ISKFunction skFunction, int? maxTokens) - { - skFunction.RequestSettings.MaxTokens = maxTokens; - return skFunction; - } - - /// - /// Change the LLM Temperature configuration - /// - /// Semantic function - /// Temperature value - /// Self instance - public static ISKFunction UseTemperature(this ISKFunction skFunction, double temperature) - { - skFunction.RequestSettings.Temperature = temperature; - return skFunction; - } - - /// - /// Change the Max Tokens configuration - /// - /// Semantic function - /// TopP value - /// Self instance - public static ISKFunction UseTopP(this ISKFunction skFunction, double topP) - { - skFunction.RequestSettings.TopP = topP; - return skFunction; - } - - /// - /// Change the Max Tokens configuration - /// - /// Semantic function - /// Presence penalty value - /// Self instance - public static ISKFunction UsePresencePenalty(this ISKFunction skFunction, double presencePenalty) - { - skFunction.RequestSettings.PresencePenalty = presencePenalty; - return skFunction; - } - - /// - /// Change the Max Tokens configuration - /// - /// Semantic function - /// Frequency penalty value - /// Self instance - public static ISKFunction UseFrequencyPenalty(this ISKFunction skFunction, double frequencyPenalty) - { - skFunction.RequestSettings.FrequencyPenalty = frequencyPenalty; - return skFunction; - } - - /// - /// Execute a function allowing to pass the main input separately from the rest of the context - /// and the cancellation token without the need to name the parameter, to have a shorter more readable syntax. - /// Note: if the context contains an INPUT key/value, that value is ignored, logging a warning. - /// - /// Function to execute - /// Main input string - /// /// The to monitor for cancellation requests. The default is . - /// The result of the function execution - public static Task InvokeAsync(this ISKFunction function, - string? input = null, CancellationToken cancellationToken = default) - { - return function.InvokeAsync(input, settings: null, logger: null, cancellationToken: cancellationToken); - } - - /// - /// Execute a function allowing to pass the main input separately from the rest of the context. - /// Note: if the context contains an INPUT key/value, that value is ignored, logging a warning. - /// - /// Function to execute - /// Main input string - /// Execution context, including variables other than input - /// Whether the function can modify the context variables, True by default - /// LLM completion settings (for semantic functions only) - /// The result of the function execution - public static Task InvokeAsync(this ISKFunction function, - string input, - SKContext context, - bool mutableContext = true, - CompleteRequestSettings? settings = null) - { - // Log a warning if the given input is overriding a different input in the context - var inputInContext = context.Variables.Input; - if (!string.IsNullOrEmpty(inputInContext) && !string.Equals(input, inputInContext, StringComparison.Ordinal)) - { - context.Logger.LogWarning( - "Function {0}.{1} has been invoked with an explicit input text that is different and overrides the input text defined in the context", - function.SkillName, function.Name); - } - - if (!mutableContext) - { - // Create a copy of the context, to avoid editing the original set of variables - context = context.Clone(); - } - - // Store the input in the context - context.Variables.Update(input); - return function.InvokeAsync(context, settings); - } - - /// - /// Returns decorated instance of with enabled instrumentation. - /// - /// Instance of to decorate. - /// Optional logger. - public static ISKFunction WithInstrumentation(this ISKFunction function, ILogger? logger = null) - { - return new InstrumentedSKFunction(function, logger); - } -} diff --git a/dotnet/src/SemanticKernel/SkillDefinition/SkillCollection.cs b/dotnet/src/SemanticKernel/SkillDefinition/SkillCollection.cs deleted file mode 100644 index a0d0ca5833ba..000000000000 --- a/dotnet/src/SemanticKernel/SkillDefinition/SkillCollection.cs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.SkillDefinition; - -/// -/// Semantic Kernel default skill collection class. -/// The class holds a list of all the functions, native and semantic, known to the kernel instance. -/// The list is used by the planner and when executing pipelines of function compositions. -/// -[SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix")] -[DebuggerTypeProxy(typeof(IReadOnlySkillCollectionTypeProxy))] -[DebuggerDisplay("{DebuggerDisplay,nq}")] -public class SkillCollection : ISkillCollection -{ - internal const string GlobalSkill = "_GLOBAL_FUNCTIONS_"; - - public SkillCollection(ILogger? logger = null) - { - this._logger = logger ?? NullLogger.Instance; - - // Important: names are case insensitive - this._skillCollection = new(StringComparer.OrdinalIgnoreCase); - } - - public ISkillCollection AddFunction(ISKFunction functionInstance) - { - Verify.NotNull(functionInstance); - - ConcurrentDictionary skill = this._skillCollection.GetOrAdd(functionInstance.SkillName, static _ => new(StringComparer.OrdinalIgnoreCase)); - skill[functionInstance.Name] = functionInstance; - - return this; - } - - /// - public ISKFunction GetFunction(string functionName) => - this.GetFunction(GlobalSkill, functionName); - - /// - public ISKFunction GetFunction(string skillName, string functionName) - { - if (!this.TryGetFunction(skillName, functionName, out ISKFunction? functionInstance)) - { - this.ThrowFunctionNotAvailable(skillName, functionName); - } - - return functionInstance; - } - - /// - public bool TryGetFunction(string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction) => - this.TryGetFunction(GlobalSkill, functionName, out availableFunction); - - /// - public bool TryGetFunction(string skillName, string functionName, [NotNullWhen(true)] out ISKFunction? availableFunction) - { - Verify.NotNull(skillName); - Verify.NotNull(functionName); - - if (this._skillCollection.TryGetValue(skillName, out ConcurrentDictionary? skill)) - { - return skill.TryGetValue(functionName, out availableFunction); - } - - availableFunction = null; - return false; - } - - /// - public FunctionsView GetFunctionsView(bool includeSemantic = true, bool includeNative = true) - { - var result = new FunctionsView(); - - if (includeSemantic || includeNative) - { - foreach (var skill in this._skillCollection) - { - foreach (KeyValuePair f in skill.Value) - { - if (f.Value.IsSemantic ? includeSemantic : includeNative) - { - result.AddFunction(f.Value.Describe()); - } - } - } - } - - return result; - } - - [DebuggerBrowsable(DebuggerBrowsableState.Never)] - internal string DebuggerDisplay => $"Count = {this._skillCollection.Count}"; - - #region private ================================================================================ - - [DoesNotReturn] - private void ThrowFunctionNotAvailable(string skillName, string functionName) - { - this._logger.LogError("Function not available: skill:{0} function:{1}", skillName, functionName); - throw new KernelException( - KernelException.ErrorCodes.FunctionNotAvailable, - $"Function not available {skillName}.{functionName}"); - } - - private readonly ILogger _logger; - - private readonly ConcurrentDictionary> _skillCollection; - - #endregion -} diff --git a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/CodeBlock.cs b/dotnet/src/SemanticKernel/TemplateEngine/Blocks/CodeBlock.cs deleted file mode 100644 index 82e7c4481afc..000000000000 --- a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/CodeBlock.cs +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; - -#pragma warning disable CA2254 // error strings are used also internally, not just for logging -// ReSharper disable TemplateIsNotCompileTimeConstantProblem -internal sealed class CodeBlock : Block, ICodeRendering -{ - internal override BlockTypes Type => BlockTypes.Code; - - /// - /// Initializes a new instance of the class. - /// - /// Block content - /// App logger - public CodeBlock(string? content, ILogger logger) - : this(new CodeTokenizer(logger).Tokenize(content), content?.Trim(), logger) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// A list of blocks - /// Block content - /// App logger - public CodeBlock(List tokens, string? content, ILogger logger) - : base(content?.Trim(), logger) - { - this._tokens = tokens; - } - - /// - public override bool IsValid(out string errorMsg) - { - errorMsg = ""; - - foreach (Block token in this._tokens) - { - if (!token.IsValid(out errorMsg)) - { - this.Logger.LogError(errorMsg); - return false; - } - } - - if (this._tokens.Count > 1) - { - if (this._tokens[0].Type != BlockTypes.FunctionId) - { - errorMsg = $"Unexpected second token found: {this._tokens[1].Content}"; - this.Logger.LogError(errorMsg); - return false; - } - - if (this._tokens[1].Type is not BlockTypes.Value and not BlockTypes.Variable) - { - errorMsg = "Functions support only one parameter"; - this.Logger.LogError(errorMsg); - return false; - } - } - - if (this._tokens.Count > 2) - { - errorMsg = $"Unexpected second token found: {this._tokens[1].Content}"; - this.Logger.LogError(errorMsg); - return false; - } - - this._validated = true; - - return true; - } - - /// - public async Task RenderCodeAsync(SKContext context, CancellationToken cancellationToken = default) - { - if (!this._validated && !this.IsValid(out var error)) - { - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, error); - } - - this.Logger.LogTrace("Rendering code: `{0}`", this.Content); - - switch (this._tokens[0].Type) - { - case BlockTypes.Value: - case BlockTypes.Variable: - return ((ITextRendering)this._tokens[0]).Render(context.Variables); - - case BlockTypes.FunctionId: - return await this.RenderFunctionCallAsync((FunctionIdBlock)this._tokens[0], context).ConfigureAwait(false); - } - - throw new TemplateException(TemplateException.ErrorCodes.UnexpectedBlockType, - $"Unexpected first token type: {this._tokens[0].Type:G}"); - } - - #region private ================================================================================ - - private bool _validated; - private readonly List _tokens; - - private async Task RenderFunctionCallAsync(FunctionIdBlock fBlock, SKContext context) - { - if (context.Skills == null) - { - throw new KernelException( - KernelException.ErrorCodes.SkillCollectionNotSet, - "Skill collection not found in the context"); - } - - if (!this.GetFunctionFromSkillCollection(context.Skills!, fBlock, out ISKFunction? function)) - { - var errorMsg = $"Function `{fBlock.Content}` not found"; - this.Logger.LogError(errorMsg); - throw new TemplateException(TemplateException.ErrorCodes.FunctionNotFound, errorMsg); - } - - SKContext contextClone = context.Clone(); - - // If the code syntax is {{functionName $varName}} use $varName instead of $input - // If the code syntax is {{functionName 'value'}} use "value" instead of $input - if (this._tokens.Count > 1) - { - // Sensitive data, logging as trace, disabled by default - this.Logger.LogTrace("Passing variable/value: `{0}`", this._tokens[1].Content); - - string input = ((ITextRendering)this._tokens[1]).Render(contextClone.Variables); - // Keep previous trust information when updating the input - contextClone.Variables.Update(input); - } - - try - { - contextClone = await function.InvokeAsync(contextClone).ConfigureAwait(false); - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - this.Logger.LogError(ex, "Something went wrong when invoking function with custom input: {0}.{1}. Error: {2}", - function.SkillName, function.Name, ex.Message); - contextClone.Fail(ex.Message, ex); - } - - if (contextClone.ErrorOccurred) - { - var errorMsg = $"Function `{fBlock.Content}` execution failed. {contextClone.LastException?.GetType().FullName}: {contextClone.LastErrorDescription}"; - this.Logger.LogError(errorMsg); - throw new TemplateException(TemplateException.ErrorCodes.RuntimeError, errorMsg, contextClone.LastException); - } - - return contextClone.Result; - } - - private bool GetFunctionFromSkillCollection( - IReadOnlySkillCollection skills, - FunctionIdBlock fBlock, - [NotNullWhen(true)] out ISKFunction? function) - { - if (string.IsNullOrEmpty(fBlock.SkillName)) - { - // Function in the global skill - return skills.TryGetFunction(fBlock.FunctionName, out function); - } - - // Function within a specific skill - return skills.TryGetFunction(fBlock.SkillName, fBlock.FunctionName, out function); - } - - #endregion -} -// ReSharper restore TemplateIsNotCompileTimeConstantProblem -#pragma warning restore CA2254 diff --git a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/FunctionIdBlock.cs b/dotnet/src/SemanticKernel/TemplateEngine/Blocks/FunctionIdBlock.cs deleted file mode 100644 index e72f5f82e01f..000000000000 --- a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/FunctionIdBlock.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Linq; -using System.Text.RegularExpressions; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Orchestration; - -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; - -internal sealed class FunctionIdBlock : Block, ITextRendering -{ - internal override BlockTypes Type => BlockTypes.FunctionId; - - internal string SkillName { get; } = string.Empty; - - internal string FunctionName { get; } = string.Empty; - - public FunctionIdBlock(string? text, ILogger? logger = null) - : base(text?.Trim(), logger) - { - var functionNameParts = this.Content.Split('.'); - if (functionNameParts.Length > 2) - { - this.Logger.LogError("Invalid function name `{0}`", this.Content); - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, - "A function name can contain at most one dot separating the skill name from the function name"); - } - - if (functionNameParts.Length == 2) - { - this.SkillName = functionNameParts[0]; - this.FunctionName = functionNameParts[1]; - return; - } - - this.FunctionName = this.Content; - } - - public override bool IsValid(out string errorMsg) - { - if (!s_validContentRegex.IsMatch(this.Content)) - { - errorMsg = "The function identifier is empty"; - return false; - } - - if (HasMoreThanOneDot(this.Content)) - { - errorMsg = "The function identifier can contain max one '.' char separating skill name from function name"; - return false; - } - - errorMsg = ""; - return true; - } - - public string Render(ContextVariables? variables) - { - return this.Content; - } - - private static bool HasMoreThanOneDot(string? value) - { - if (value == null || value.Length < 2) { return false; } - - int count = 0; - return value.Any(t => t == '.' && ++count > 1); - } - - private static readonly Regex s_validContentRegex = new("^[a-zA-Z0-9_.]*$"); -} diff --git a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/TextBlock.cs b/dotnet/src/SemanticKernel/TemplateEngine/Blocks/TextBlock.cs deleted file mode 100644 index c4aac3edcd2e..000000000000 --- a/dotnet/src/SemanticKernel/TemplateEngine/Blocks/TextBlock.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Orchestration; - -namespace Microsoft.SemanticKernel.TemplateEngine.Blocks; - -internal sealed class TextBlock : Block, ITextRendering -{ - internal override BlockTypes Type => BlockTypes.Text; - - public TextBlock(string? text, ILogger? logger = null) - : base(text, logger) - { - } - - public TextBlock(string text, int startIndex, int stopIndex, ILogger logger) - : base(text.Substring(startIndex, stopIndex - startIndex), logger) - { - } - - public override bool IsValid(out string errorMsg) - { - errorMsg = ""; - return true; - } - - public string Render(ContextVariables? variables) - { - return this.Content; - } -} diff --git a/dotnet/src/SemanticKernel/TemplateEngine/CodeTokenizer.cs b/dotnet/src/SemanticKernel/TemplateEngine/CodeTokenizer.cs deleted file mode 100644 index 84ace98c53bc..000000000000 --- a/dotnet/src/SemanticKernel/TemplateEngine/CodeTokenizer.cs +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Text; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; -using Microsoft.SemanticKernel.Text; - -namespace Microsoft.SemanticKernel.TemplateEngine; - -/// -/// Simple tokenizer used for default SK template code language. -/// -/// BNF parsed by TemplateTokenizer: -/// [template] ::= "" | [block] | [block] [template] -/// [block] ::= [sk-block] | [text-block] -/// [sk-block] ::= "{{" [variable] "}}" | "{{" [value] "}}" | "{{" [function-call] "}}" -/// [text-block] ::= [any-char] | [any-char] [text-block] -/// [any-char] ::= any char -/// -/// BNF parsed by CodeTokenizer: -/// [template] ::= "" | [variable] " " [template] | [value] " " [template] | [function-call] " " [template] -/// [variable] ::= "$" [valid-name] -/// [value] ::= "'" [text] "'" | '"' [text] '"' -/// [function-call] ::= [function-id] | [function-id] [parameter] -/// [parameter] ::= [variable] | [value] -/// -/// BNF parsed by dedicated blocks -/// [function-id] ::= [valid-name] | [valid-name] "." [valid-name] -/// [valid-name] ::= [valid-symbol] | [valid-symbol] [valid-name] -/// [valid-symbol] ::= [letter] | [digit] | "_" -/// [letter] ::= "a" | "b" ... | "z" | "A" | "B" ... | "Z" -/// [digit] ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" -/// -internal sealed class CodeTokenizer -{ - private enum TokenTypes - { - None = 0, - Value = 1, - Variable = 2, - FunctionId = 3, - } - - private readonly ILogger _logger; - - public CodeTokenizer(ILogger? logger = null) - { - this._logger = logger ?? NullLogger.Instance; - } - - /// - /// Tokenize a code block, without checking for syntax errors - /// - /// Text to parse - /// A list of blocks - public List Tokenize(string? text) - { - // Remove spaces, which are ignored anyway - text = text?.Trim(); - - // Render NULL to "" - if (text.IsNullOrEmpty()) { return new List(); } - - // Track what type of token we're reading - TokenTypes currentTokenType = TokenTypes.None; - - // Track the content of the current token - var currentTokenContent = new StringBuilder(); - - char textValueDelimiter = '\0'; - - var blocks = new List(); - char nextChar = text[0]; - - // Tokens must be separated by spaces, track their presence - bool spaceSeparatorFound = false; - - // 1 char only edge case - if (text.Length == 1) - { - switch (nextChar) - { - case Symbols.VarPrefix: - blocks.Add(new VarBlock(text, this._logger)); - break; - - case Symbols.DblQuote: - case Symbols.SglQuote: - blocks.Add(new ValBlock(text, this._logger)); - break; - - default: - blocks.Add(new FunctionIdBlock(text, this._logger)); - break; - } - - return blocks; - } - - bool skipNextChar = false; - for (int nextCharCursor = 1; nextCharCursor < text.Length; nextCharCursor++) - { - char currentChar = nextChar; - nextChar = text[nextCharCursor]; - - if (skipNextChar) - { - skipNextChar = false; - continue; - } - - // First char is easy - if (nextCharCursor == 1) - { - if (IsVarPrefix(currentChar)) - { - currentTokenType = TokenTypes.Variable; - } - else if (IsQuote(currentChar)) - { - currentTokenType = TokenTypes.Value; - textValueDelimiter = currentChar; - } - else - { - currentTokenType = TokenTypes.FunctionId; - } - - currentTokenContent.Append(currentChar); - continue; - } - - // While reading a values between quotes - if (currentTokenType == TokenTypes.Value) - { - // If the current char is escaping the next special char: - // - skip the current char (escape char) - // - add the next (special char) - // - jump to the one after (to handle "\\" properly) - if (currentChar == Symbols.EscapeChar && CanBeEscaped(nextChar)) - { - currentTokenContent.Append(nextChar); - skipNextChar = true; - continue; - } - - currentTokenContent.Append(currentChar); - - // When we reach the end of the value - if (currentChar == textValueDelimiter) - { - blocks.Add(new ValBlock(currentTokenContent.ToString(), this._logger)); - currentTokenContent.Clear(); - currentTokenType = TokenTypes.None; - spaceSeparatorFound = false; - } - - continue; - } - - // If we're not between quotes, a space signals the end of the current token - // Note: there might be multiple consecutive spaces - if (IsBlankSpace(currentChar)) - { - if (currentTokenType == TokenTypes.Variable) - { - blocks.Add(new VarBlock(currentTokenContent.ToString(), this._logger)); - currentTokenContent.Clear(); - } - else if (currentTokenType == TokenTypes.FunctionId) - { - blocks.Add(new FunctionIdBlock(currentTokenContent.ToString(), this._logger)); - currentTokenContent.Clear(); - } - - spaceSeparatorFound = true; - currentTokenType = TokenTypes.None; - - continue; - } - - // If we're not inside a quoted value and we're not processing a space - currentTokenContent.Append(currentChar); - - if (currentTokenType == TokenTypes.None) - { - if (!spaceSeparatorFound) - { - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, - "Tokens must be separated by one space least"); - } - - if (IsQuote(currentChar)) - { - // A quoted value starts here - currentTokenType = TokenTypes.Value; - textValueDelimiter = currentChar; - } - else if (IsVarPrefix(currentChar)) - { - // A variable starts here - currentTokenType = TokenTypes.Variable; - } - else - { - // A function Id starts here - currentTokenType = TokenTypes.FunctionId; - } - } - } - - // Capture last token - currentTokenContent.Append(nextChar); - switch (currentTokenType) - { - case TokenTypes.Value: - blocks.Add(new ValBlock(currentTokenContent.ToString(), this._logger)); - break; - - case TokenTypes.Variable: - blocks.Add(new VarBlock(currentTokenContent.ToString(), this._logger)); - break; - - case TokenTypes.FunctionId: - blocks.Add(new FunctionIdBlock(currentTokenContent.ToString(), this._logger)); - break; - - case TokenTypes.None: - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, - "Tokens must be separated by one space least"); - } - - return blocks; - } - - private static bool IsVarPrefix(char c) - { - return (c == Symbols.VarPrefix); - } - - private static bool IsBlankSpace(char c) - { - return c is Symbols.Space or Symbols.NewLine or Symbols.CarriageReturn or Symbols.Tab; - } - - private static bool IsQuote(char c) - { - return c is Symbols.DblQuote or Symbols.SglQuote; - } - - private static bool CanBeEscaped(char c) - { - return c is Symbols.DblQuote or Symbols.SglQuote or Symbols.EscapeChar; - } -} diff --git a/dotnet/src/SemanticKernel/TemplateEngine/PromptTemplateEngine.cs b/dotnet/src/SemanticKernel/TemplateEngine/PromptTemplateEngine.cs deleted file mode 100644 index 5b063d84ef7e..000000000000 --- a/dotnet/src/SemanticKernel/TemplateEngine/PromptTemplateEngine.cs +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.TemplateEngine.Blocks; - -namespace Microsoft.SemanticKernel.TemplateEngine; - -/// -/// Given a prompt, that might contain references to variables and functions: -/// - Get the list of references -/// - Resolve each reference -/// - Variable references are resolved using the context variables -/// - Function references are resolved invoking those functions -/// - Functions can be invoked passing in variables -/// - Functions do not receive the context variables, unless specified using a special variable -/// - Functions can be invoked in order and in parallel so the context variables must be immutable when invoked within the template -/// -public class PromptTemplateEngine : IPromptTemplateEngine -{ - private readonly ILogger _logger; - - private readonly TemplateTokenizer _tokenizer; - - public PromptTemplateEngine(ILogger? logger = null) - { - this._logger = logger ?? NullLogger.Instance; - this._tokenizer = new TemplateTokenizer(this._logger); - } - - /// - public IList ExtractBlocks(string? templateText, bool validate = true) - { - this._logger.LogTrace("Extracting blocks from template: {0}", templateText); - var blocks = this._tokenizer.Tokenize(templateText); - - if (validate) - { - foreach (var block in blocks) - { - if (!block.IsValid(out var error)) - { - throw new TemplateException(TemplateException.ErrorCodes.SyntaxError, error); - } - } - } - - return blocks; - } - - /// - public async Task RenderAsync(string templateText, SKContext context, CancellationToken cancellationToken = default) - { - this._logger.LogTrace("Rendering string template: {0}", templateText); - var blocks = this.ExtractBlocks(templateText); - return await this.RenderAsync(blocks, context, cancellationToken).ConfigureAwait(false); - } - - /// - /// Given a list of blocks render each block and compose the final result - /// - /// Template blocks generated by ExtractBlocks - /// Access into the current kernel execution context - /// The to monitor for cancellation requests. The default is . - /// The prompt template ready to be used for an AI request - internal async Task RenderAsync(IList blocks, SKContext context, CancellationToken cancellationToken = default) - { - this._logger.LogTrace("Rendering list of {0} blocks", blocks.Count); - var tasks = new List>(blocks.Count); - foreach (var block in blocks) - { - switch (block) - { - case ITextRendering staticBlock: - tasks.Add(Task.FromResult(staticBlock.Render(context.Variables))); - break; - - case ICodeRendering dynamicBlock: - tasks.Add(dynamicBlock.RenderCodeAsync(context, cancellationToken)); - break; - - default: - const string Error = "Unexpected block type, the block doesn't have a rendering method"; - this._logger.LogError(Error); - throw new TemplateException(TemplateException.ErrorCodes.UnexpectedBlockType, Error); - } - } - - var result = new StringBuilder(); - foreach (Task t in tasks) - { - result.Append(await t.ConfigureAwait(false)); - } - - // Sensitive data, logging as trace, disabled by default - this._logger.LogTrace("Rendered prompt: {0}", result); - - return result.ToString(); - } - - /// - /// Given a list of blocks, render the Variable Blocks, replacing placeholders with the actual value in memory - /// - /// List of blocks, typically all the blocks found in a template - /// Container of all the temporary variables known to the kernel - /// An updated list of blocks where Variable Blocks have rendered to Text Blocks - internal IList RenderVariables(IList blocks, ContextVariables? variables) - { - this._logger.LogTrace("Rendering variables"); - return blocks.Select(block => block.Type != BlockTypes.Variable - ? block - : new TextBlock(((ITextRendering)block).Render(variables), this._logger)).ToList(); - } -} diff --git a/dotnet/src/SemanticKernel/TemplateEngine/TemplateException.cs b/dotnet/src/SemanticKernel/TemplateEngine/TemplateException.cs deleted file mode 100644 index 6eafbe90d9df..000000000000 --- a/dotnet/src/SemanticKernel/TemplateEngine/TemplateException.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.TemplateEngine; - -#pragma warning disable RCS1194 // Implement exception constructors - -/// -/// Exception thrown for errors related to templating. -/// -public class TemplateException : SKException -{ - /// - /// Initializes a new instance of the class with a provided error code and message. - /// - /// The error code. - /// The exception message. - public TemplateException(ErrorCodes errorCode, string? message) - : this(errorCode, message, innerException: null) - { - } - - /// - /// Initializes a new instance of the class with a provided error code, message, and inner exception. - /// - /// The error code. - /// A string that describes the error. - /// The exception that is the cause of the current exception. - public TemplateException(ErrorCodes errorCode, string? message = null, Exception? innerException = null) - : base(GetDefaultMessage(errorCode, message), innerException) - { - this.ErrorCode = errorCode; - } - - /// - /// Gets the error code for this exception. - /// - public ErrorCodes ErrorCode { get; } - - /// Translate the error code into a default message. - private static string GetDefaultMessage(ErrorCodes errorCode, string? message) - { - string description = errorCode switch - { - ErrorCodes.SyntaxError => "Syntax error", - ErrorCodes.UnexpectedBlockType => "Unexpected block type", - ErrorCodes.FunctionNotFound => "Function not found", - ErrorCodes.RuntimeError => "Runtime error", - _ => $"Unknown error ({errorCode:G})", - }; - - return message is not null ? $"{description}: {message}" : description; - } - - /// - /// Error codes for . - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError = -1, - - /// - /// Syntax error, the template syntax used is not valid. - /// - SyntaxError = 0, - - /// - /// The block type produced be the tokenizer was not expected - /// - UnexpectedBlockType = 1, - - /// - /// The template requires an unknown function. - /// - FunctionNotFound = 2, - - /// - /// The template execution failed, e.g. a function call threw an exception. - /// - RuntimeError = 3, - } -} diff --git a/dotnet/src/SemanticKernel/Text/SKFunctionTextExtensions.cs b/dotnet/src/SemanticKernel/Text/SKFunctionTextExtensions.cs deleted file mode 100644 index 7036ac9e46cc..000000000000 --- a/dotnet/src/SemanticKernel/Text/SKFunctionTextExtensions.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Orchestration; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using NS of ISKFunction -namespace Microsoft.SemanticKernel.SkillDefinition; -#pragma warning restore IDE0130 - -/// -/// Class with extension methods for semantic functions. -/// -public static class SKFunctionTextExtensions -{ - /// - /// Extension method to aggregate partitioned results of a semantic function. - /// - /// Semantic Kernel function - /// Input to aggregate. - /// Semantic Kernel context. - /// Aggregated results. - public static async Task AggregatePartitionedResultsAsync( - this ISKFunction func, - List partitionedInput, - SKContext context) - { - var results = new List(); - foreach (var partition in partitionedInput) - { - context.Variables.Update(partition); - context = await func.InvokeAsync(context).ConfigureAwait(false); - - results.Add(context.Variables.ToString()); - } - - context.Variables.Update(string.Join("\n", results)); - return context; - } -} diff --git a/dotnet/src/SemanticKernel/Text/TextChunker.cs b/dotnet/src/SemanticKernel/Text/TextChunker.cs deleted file mode 100644 index 3bbe91ac0740..000000000000 --- a/dotnet/src/SemanticKernel/Text/TextChunker.cs +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; - -namespace Microsoft.SemanticKernel.Text; - -/// -/// Split text in chunks, attempting to leave meaning intact. -/// For plain text, split looking at new lines first, then periods, and so on. -/// For markdown, split looking at punctuation first, and so on. -/// -public static class TextChunker -{ - private static readonly char[] s_spaceChar = new[] { ' ' }; - private static readonly string?[] s_plaintextSplitOptions = new[] { "\n\r", ".", "?!", ";", ":", ",", ")]}", " ", "-", null }; - private static readonly string?[] s_markdownSplitOptions = new[] { ".", "?!", ";", ":", ",", ")]}", " ", "-", "\n\r", null }; - - /// - /// Split plain text into lines. - /// - /// Text to split - /// Maximum number of tokens per line. - /// List of lines. - public static List SplitPlainTextLines(string text, int maxTokensPerLine) - { - return InternalSplitLines(text, maxTokensPerLine, trim: true, s_plaintextSplitOptions); - } - - /// - /// Split markdown text into lines. - /// - /// Text to split - /// Maximum number of tokens per line. - /// List of lines. - public static List SplitMarkDownLines(string text, int maxTokensPerLine) - { - return InternalSplitLines(text, maxTokensPerLine, trim: true, s_markdownSplitOptions); - } - - /// - /// Split plain text into paragraphs. - /// - /// Lines of text. - /// Maximum number of tokens per paragraph. - /// Number of tokens to overlap between paragraphs. - /// List of paragraphs. - public static List SplitPlainTextParagraphs(List lines, int maxTokensPerParagraph, int overlapTokens = 0) - { - return InternalSplitTextParagraphs(lines, maxTokensPerParagraph, overlapTokens, (text, maxTokens) => InternalSplitLines(text, maxTokens, trim: false, s_plaintextSplitOptions)); - } - - /// - /// Split markdown text into paragraphs. - /// - /// Lines of text. - /// Maximum number of tokens per paragraph. - /// Number of tokens to overlap between paragraphs. - /// List of paragraphs. - public static List SplitMarkdownParagraphs(List lines, int maxTokensPerParagraph, int overlapTokens = 0) - { - return InternalSplitTextParagraphs(lines, maxTokensPerParagraph, overlapTokens, (text, maxTokens) => InternalSplitLines(text, maxTokens, trim: false, s_markdownSplitOptions)); - } - - private static List InternalSplitTextParagraphs(List lines, int maxTokensPerParagraph, int overlapTokens, Func> longLinesSplitter) - { - if (maxTokensPerParagraph <= 0) - { - throw new ArgumentException("maxTokensPerParagraph should be a positive number"); - } - - if (maxTokensPerParagraph <= overlapTokens) - { - throw new ArgumentException("overlapTokens cannot be larger than maxTokensPerParagraph"); - } - - if (lines.Count == 0) - { - return new List(); - } - - var adjustedMaxTokensPerParagraph = maxTokensPerParagraph - overlapTokens; - - // Split long lines first - IEnumerable truncatedLines = lines.SelectMany(line => longLinesSplitter(line, adjustedMaxTokensPerParagraph)); - - var paragraphs = BuildParagraph(truncatedLines, adjustedMaxTokensPerParagraph, longLinesSplitter); - // distribute text more evenly in the last paragraphs when the last paragraph is too short. - if (paragraphs.Count > 1) - { - var lastParagraph = paragraphs[paragraphs.Count - 1]; - var secondLastParagraph = paragraphs[paragraphs.Count - 2]; - - if (TokenCount(lastParagraph.Length) < adjustedMaxTokensPerParagraph / 4) - { - var lastParagraphTokens = lastParagraph.Split(s_spaceChar, StringSplitOptions.RemoveEmptyEntries); - var secondLastParagraphTokens = secondLastParagraph.Split(s_spaceChar, StringSplitOptions.RemoveEmptyEntries); - - var lastParagraphTokensCount = lastParagraphTokens.Length; - var secondLastParagraphTokensCount = secondLastParagraphTokens.Length; - - if (lastParagraphTokensCount + secondLastParagraphTokensCount <= adjustedMaxTokensPerParagraph) - { - var newSecondLastParagraph = string.Join(" ", secondLastParagraphTokens); - var newLastParagraph = string.Join(" ", lastParagraphTokens); - - paragraphs[paragraphs.Count - 2] = $"{newSecondLastParagraph} {newLastParagraph}"; - paragraphs.RemoveAt(paragraphs.Count - 1); - } - } - } - - if (overlapTokens > 0 && paragraphs.Count > 1) - { - var lastParagraph = paragraphs.Last(); - - paragraphs = paragraphs.Zip(paragraphs.Skip(1), (currentParagraph, nextParagraph) => - { - var split = longLinesSplitter(nextParagraph, overlapTokens); - return $"{currentParagraph} {split.FirstOrDefault()}"; - }).ToList(); - - paragraphs.Add(lastParagraph); - } - - return paragraphs; - } - - private static List BuildParagraph(IEnumerable truncatedLines, int maxTokensPerParagraph, Func> longLinesSplitter) - { - StringBuilder paragraphBuilder = new(); - List paragraphs = new(); - - foreach (string line in truncatedLines) - { - if (paragraphBuilder.Length > 0 && TokenCount(paragraphBuilder.Length) + TokenCount(line.Length) + 1 >= maxTokensPerParagraph) - { - // Complete the paragraph and prepare for the next - paragraphs.Add(paragraphBuilder.ToString().Trim()); - paragraphBuilder.Clear(); - } - - paragraphBuilder.AppendLine(line); - } - - if (paragraphBuilder.Length > 0) - { - // Add the final paragraph if there's anything remaining - paragraphs.Add(paragraphBuilder.ToString().Trim()); - } - - return paragraphs; - } - - private static List InternalSplitLines(string text, int maxTokensPerLine, bool trim, string?[] splitOptions) - { - var result = new List(); - - text = text.NormalizeLineEndings(); - result.Add(text); - for (int i = 0; i < splitOptions.Length; i++) - { - int count = result.Count; // track where the original input left off - var (splits2, inputWasSplit2) = Split(result, maxTokensPerLine, splitOptions[i].AsSpan(), trim); - result.AddRange(splits2); - result.RemoveRange(0, count); // remove the original input - if (!inputWasSplit2) - { - break; - } - } - return result; - } - - private static (List, bool) Split(List input, int maxTokens, ReadOnlySpan separators, bool trim) - { - bool inputWasSplit = false; - List result = new(); - int count = input.Count; - for (int i = 0; i < count; i++) - { - var (splits, split) = Split(input[i].AsSpan(), input[i], maxTokens, separators, trim); - result.AddRange(splits); - inputWasSplit |= split; - } - return (result, inputWasSplit); - } - - private static (List, bool) Split(ReadOnlySpan input, string? inputString, int maxTokens, ReadOnlySpan separators, bool trim) - { - Debug.Assert(inputString is null || input.SequenceEqual(inputString.AsSpan())); - List result = new(); - var inputWasSplit = false; - if (TokenCount(input.Length) > maxTokens) - { - inputWasSplit = true; - - int half = input.Length / 2; - int cutPoint = -1; - - if (separators.IsEmpty) - { - cutPoint = half; - } - else if (input.Length > 2) - { - int pos = 0; - while (true) - { - int index = input.Slice(pos, input.Length - 1 - pos).IndexOfAny(separators); - if (index < 0) - { - break; - } - - index += pos; - - if (Math.Abs(half - index) < Math.Abs(half - cutPoint)) - { - cutPoint = index + 1; - } - - pos = index + 1; - } - } - - if (cutPoint > 0) - { - var firstHalf = input.Slice(0, cutPoint); - var secondHalf = input.Slice(cutPoint); - if (trim) - { - firstHalf = firstHalf.Trim(); - secondHalf = secondHalf.Trim(); - } - - // Recursion - var (splits1, split1) = Split(firstHalf, null, maxTokens, separators, trim); - result.AddRange(splits1); - var (splits2, split2) = Split(secondHalf, null, maxTokens, separators, trim); - result.AddRange(splits2); - - inputWasSplit = split1 || split2; - return (result, inputWasSplit); - } - } - - result.Add((inputString is not null, trim) switch - { - (true, true) => inputString!.Trim(), - (true, false) => inputString!, - (false, true) => input.Trim().ToString(), - (false, false) => input.ToString(), - }); - - return (result, inputWasSplit); - } - - private static int TokenCount(int inputLength) - { - // TODO: partitioning methods should be configurable to allow for different tokenization strategies - // depending on the model to be called. For now, we use an extremely rough estimate. - return inputLength / 4; - } -} diff --git a/dotnet/src/Skills/Skills.Core/ConversationSummarySkill.cs b/dotnet/src/Skills/Skills.Core/ConversationSummarySkill.cs deleted file mode 100644 index 30fb4689a3e0..000000000000 --- a/dotnet/src/Skills/Skills.Core/ConversationSummarySkill.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Text; - -namespace Microsoft.SemanticKernel.Skills.Core; - -/// -/// Semantic skill that enables conversations summarization. -/// -/// -/// -/// var kernel Kernel.Builder.Build(); -/// kernel.ImportSkill(new ConversationSummarySkill(kernel)); -/// -/// -public class ConversationSummarySkill -{ - /// - /// The max tokens to process in a single semantic function call. - /// - private const int MaxTokens = 1024; - - private readonly ISKFunction _summarizeConversationFunction; - private readonly ISKFunction _conversationActionItemsFunction; - private readonly ISKFunction _conversationTopicsFunction; - - /// - /// Initializes a new instance of the class. - /// - /// Kernel instance - public ConversationSummarySkill(IKernel kernel) - { - this._summarizeConversationFunction = kernel.CreateSemanticFunction( - SemanticFunctionConstants.SummarizeConversationDefinition, - skillName: nameof(ConversationSummarySkill), - description: "Given a section of a conversation transcript, summarize the part of the conversation.", - maxTokens: MaxTokens, - temperature: 0.1, - topP: 0.5); - - this._conversationActionItemsFunction = kernel.CreateSemanticFunction( - SemanticFunctionConstants.GetConversationActionItemsDefinition, - skillName: nameof(ConversationSummarySkill), - description: "Given a section of a conversation transcript, identify action items.", - maxTokens: MaxTokens, - temperature: 0.1, - topP: 0.5); - - this._conversationTopicsFunction = kernel.CreateSemanticFunction( - SemanticFunctionConstants.GetConversationTopicsDefinition, - skillName: nameof(ConversationSummarySkill), - description: "Analyze a conversation transcript and extract key topics worth remembering.", - maxTokens: MaxTokens, - temperature: 0.1, - topP: 0.5); - } - - /// - /// Given a long conversation transcript, summarize the conversation. - /// - /// A long conversation transcript. - /// The SKContext for function execution. - [SKFunction, Description("Given a long conversation transcript, summarize the conversation.")] - public Task SummarizeConversationAsync( - [Description("A long conversation transcript.")] string input, - SKContext context) - { - List lines = TextChunker.SplitPlainTextLines(input, MaxTokens); - List paragraphs = TextChunker.SplitPlainTextParagraphs(lines, MaxTokens); - - return this._summarizeConversationFunction - .AggregatePartitionedResultsAsync(paragraphs, context); - } - - /// - /// Given a long conversation transcript, identify action items. - /// - /// A long conversation transcript. - /// The SKContext for function execution. - [SKFunction, Description("Given a long conversation transcript, identify action items.")] - public Task GetConversationActionItemsAsync( - [Description("A long conversation transcript.")] string input, - SKContext context) - { - List lines = TextChunker.SplitPlainTextLines(input, MaxTokens); - List paragraphs = TextChunker.SplitPlainTextParagraphs(lines, MaxTokens); - - return this._conversationActionItemsFunction - .AggregatePartitionedResultsAsync(paragraphs, context); - } - - /// - /// Given a long conversation transcript, identify topics. - /// - /// A long conversation transcript. - /// The SKContext for function execution. - [SKFunction, Description("Given a long conversation transcript, identify topics worth remembering.")] - public Task GetConversationTopicsAsync( - [Description("A long conversation transcript.")] string input, - SKContext context) - { - List lines = TextChunker.SplitPlainTextLines(input, MaxTokens); - List paragraphs = TextChunker.SplitPlainTextParagraphs(lines, MaxTokens); - - return this._conversationTopicsFunction - .AggregatePartitionedResultsAsync(paragraphs, context); - } -} diff --git a/dotnet/src/Skills/Skills.Core/FileIOSkill.cs b/dotnet/src/Skills/Skills.Core/FileIOSkill.cs deleted file mode 100644 index 7835516b95e7..000000000000 --- a/dotnet/src/Skills/Skills.Core/FileIOSkill.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ComponentModel; -using System.IO; -using System.Text; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Core; - -/// -/// Read and write from a file. -/// -/// -/// Usage: kernel.ImportSkill("file", new FileIOSkill()); -/// Examples: -/// {{file.readAsync $path }} => "hello world" -/// {{file.writeAsync}} -/// -public sealed class FileIOSkill -{ - /// - /// Read a file - /// - /// - /// {{file.readAsync $path }} => "hello world" - /// - /// Source file - /// File content - [SKFunction, Description("Read a file")] - public async Task ReadAsync([Description("Source file")] string path) - { - using var reader = File.OpenText(path); - return await reader.ReadToEndAsync().ConfigureAwait(false); - } - - /// - /// Write a file - /// - /// - /// {{file.writeAsync}} - /// - /// The destination file path - /// The file content to write - /// An awaitable task - [SKFunction, Description("Write a file")] - public async Task WriteAsync( - [Description("Destination file")] string path, - [Description("File content")] string content) - { - byte[] text = Encoding.UTF8.GetBytes(content); - if (File.Exists(path) && File.GetAttributes(path).HasFlag(FileAttributes.ReadOnly)) - { - // Most environments will throw this with OpenWrite, but running inside docker on Linux will not. - throw new UnauthorizedAccessException($"File is read-only: {path}"); - } - - using var writer = File.OpenWrite(path); - await writer.WriteAsync(text, 0, text.Length).ConfigureAwait(false); - } -} diff --git a/dotnet/src/Skills/Skills.Core/HttpSkill.cs b/dotnet/src/Skills/Skills.Core/HttpSkill.cs deleted file mode 100644 index 20a05f3f4e01..000000000000 --- a/dotnet/src/Skills/Skills.Core/HttpSkill.cs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Core; - -/// -/// A skill that provides HTTP functionality. -/// -/// -/// Usage: kernel.ImportSkill("http", new HttpSkill()); -/// Examples: -/// SKContext.Variables["url"] = "https://www.bing.com" -/// {{http.getAsync $url}} -/// {{http.postAsync $url}} -/// {{http.putAsync $url}} -/// {{http.deleteAsync $url}} -/// -[System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1054:URI-like parameters should not be strings", - Justification = "Semantic Kernel operates on strings")] -public sealed class HttpSkill -{ - private readonly HttpClient _client; - - /// - /// Initializes a new instance of the class. - /// - public HttpSkill() : this(new HttpClient(NonDisposableHttpClientHandler.Instance, disposeHandler: false)) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The HTTP client to use. - /// - /// assumes ownership of the instance and will dispose it when the skill is disposed. - /// - public HttpSkill(HttpClient client) => - this._client = client; - - /// - /// Sends an HTTP GET request to the specified URI and returns the response body as a string. - /// - /// URI of the request - /// The token to use to request cancellation. - /// The response body as a string. - [SKFunction, Description("Makes a GET request to a uri")] - public Task GetAsync( - [Description("The URI of the request")] string uri, - CancellationToken cancellationToken = default) => - this.SendRequestAsync(uri, HttpMethod.Get, requestContent: null, cancellationToken); - - /// - /// Sends an HTTP POST request to the specified URI and returns the response body as a string. - /// - /// URI of the request - /// The body of the request - /// The token to use to request cancellation. - /// The response body as a string. - [SKFunction, Description("Makes a POST request to a uri")] - public Task PostAsync( - [Description("The URI of the request")] string uri, - [Description("The body of the request")] string body, - CancellationToken cancellationToken = default) => - this.SendRequestAsync(uri, HttpMethod.Post, new StringContent(body), cancellationToken); - - /// - /// Sends an HTTP PUT request to the specified URI and returns the response body as a string. - /// - /// URI of the request - /// The body of the request - /// The token to use to request cancellation. - /// The response body as a string. - [SKFunction, Description("Makes a PUT request to a uri")] - public Task PutAsync( - [Description("The URI of the request")] string uri, - [Description("The body of the request")] string body, - CancellationToken cancellationToken = default) => - this.SendRequestAsync(uri, HttpMethod.Put, new StringContent(body), cancellationToken); - - /// - /// Sends an HTTP DELETE request to the specified URI and returns the response body as a string. - /// - /// URI of the request - /// The token to use to request cancellation. - /// The response body as a string. - [SKFunction, Description("Makes a DELETE request to a uri")] - public Task DeleteAsync( - [Description("The URI of the request")] string uri, - CancellationToken cancellationToken = default) => - this.SendRequestAsync(uri, HttpMethod.Delete, requestContent: null, cancellationToken); - - /// Sends an HTTP request and returns the response content as a string. - /// The URI of the request. - /// The HTTP method for the request. - /// Optional request content. - /// The token to use to request cancellation. - private async Task SendRequestAsync(string uri, HttpMethod method, HttpContent? requestContent, CancellationToken cancellationToken) - { - using var request = new HttpRequestMessage(method, uri) { Content = requestContent }; - using var response = await this._client.SendAsync(request, cancellationToken).ConfigureAwait(false); - return await response.Content.ReadAsStringAsync().ConfigureAwait(false); - } -} diff --git a/dotnet/src/Skills/Skills.Core/MathSkill.cs b/dotnet/src/Skills/Skills.Core/MathSkill.cs deleted file mode 100644 index 7878f3d7726d..000000000000 --- a/dotnet/src/Skills/Skills.Core/MathSkill.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Core; - -/// -/// MathSkill provides a set of functions to make Math calculations. -/// -/// -/// Usage: kernel.ImportSkill("math", new MathSkill()); -/// Examples: -/// {{math.Add}} => Returns the sum of FirstNumber and SecondNumber (provided in the SKContext) -/// -public sealed class MathSkill -{ - /// - /// Returns the Addition result of initial and amount values provided. - /// - /// Initial value to which to add the specified amount - /// The amount to add as a string. - /// The resulting sum as a string. - [SKFunction, Description("Adds an amount to a value")] - public int Add( - [Description("The value to add")] int value, - [Description("Amount to add")] int amount) => - value + amount; - - /// - /// Returns the Sum of two SKContext numbers provided. - /// - /// Initial value from which to subtract the specified amount - /// The amount to subtract as a string. - /// The resulting subtraction as a string. - [SKFunction, Description("Subtracts an amount from a value")] - public int Subtract( - [Description("The value to subtract")] int value, - [Description("Amount to subtract")] int amount) => - value - amount; -} diff --git a/dotnet/src/Skills/Skills.Core/Skills.Core.csproj b/dotnet/src/Skills/Skills.Core/Skills.Core.csproj deleted file mode 100644 index 3d9e6e4b1a89..000000000000 --- a/dotnet/src/Skills/Skills.Core/Skills.Core.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - - Microsoft.SemanticKernel.Skills.Core - $(AssemblyName) - netstandard2.0 - - - - - - - - Semantic Kernel - Core Skills - Semantic Kernel core skills. - - - - - - - diff --git a/dotnet/src/Skills/Skills.Core/TextMemorySkill.cs b/dotnet/src/Skills/Skills.Core/TextMemorySkill.cs deleted file mode 100644 index 693e48bac379..000000000000 --- a/dotnet/src/Skills/Skills.Core/TextMemorySkill.cs +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Core; - -/// -/// TextMemorySkill provides a skill to save or recall information from the long or short term memory. -/// -/// -/// Usage: kernel.ImportSkill("memory", new TextMemorySkill()); -/// Examples: -/// SKContext.Variables["input"] = "what is the capital of France?" -/// {{memory.recall $input }} => "Paris" -/// -public sealed class TextMemorySkill -{ - /// - /// Name of the context variable used to specify which memory collection to use. - /// - public const string CollectionParam = "collection"; - - /// - /// Name of the context variable used to specify memory search relevance score. - /// - public const string RelevanceParam = "relevance"; - - /// - /// Name of the context variable used to specify a unique key associated with stored information. - /// - public const string KeyParam = "key"; - - /// - /// Name of the context variable used to specify the number of memories to recall - /// - public const string LimitParam = "limit"; - - private const string DefaultCollection = "generic"; - private const double DefaultRelevance = 0.0; - private const int DefaultLimit = 1; - - private ISemanticTextMemory _memory; - - /// - /// Creates a new instance of the TextMemorySkill - /// - public TextMemorySkill(ISemanticTextMemory memory) - { - this._memory = memory; - } - - /// - /// Key-based lookup for a specific memory - /// - /// Memories collection associated with the memory to retrieve - /// The key associated with the memory to retrieve. - /// Application logger - /// The to monitor for cancellation requests. The default is . - /// - /// SKContext.Variables[TextMemorySkill.KeyParam] = "countryInfo1" - /// {{memory.retrieve }} - /// - [SKFunction, Description("Key-based lookup for a specific memory")] - public async Task RetrieveAsync( - [SKName(CollectionParam), Description("Memories collection associated with the memory to retrieve"), DefaultValue(DefaultCollection)] string? collection, - [SKName(KeyParam), Description("The key associated with the memory to retrieve")] string key, - ILogger? logger, - CancellationToken cancellationToken = default) - { - Verify.NotNullOrWhiteSpace(collection); - Verify.NotNullOrWhiteSpace(key); - logger ??= NullLogger.Instance; - - logger.LogDebug("Recalling memory with key '{0}' from collection '{1}'", key, collection); - - var memory = await this._memory.GetAsync(collection, key, cancellationToken: cancellationToken).ConfigureAwait(false); - - return memory?.Metadata.Text ?? string.Empty; - } - - /// - /// Semantic search and return up to N memories related to the input text - /// - /// - /// SKContext.Variables["input"] = "what is the capital of France?" - /// {{memory.recall $input }} => "Paris" - /// - /// The input text to find related memories for. - /// Memories collection to search. - /// The relevance score, from 0.0 to 1.0, where 1.0 means perfect match. - /// The maximum number of relevant memories to recall. - /// Application logger - /// The to monitor for cancellation requests. The default is . - [SKFunction, Description("Semantic search and return up to N memories related to the input text")] - public async Task RecallAsync( - [Description("The input text to find related memories for")] string input, - [SKName(CollectionParam), Description("Memories collection to search"), DefaultValue(DefaultCollection)] string collection, - [SKName(RelevanceParam), Description("The relevance score, from 0.0 to 1.0, where 1.0 means perfect match"), DefaultValue(DefaultRelevance)] double? relevance, - [SKName(LimitParam), Description("The maximum number of relevant memories to recall"), DefaultValue(DefaultLimit)] int? limit, - ILogger? logger, - CancellationToken cancellationToken = default) - { - Verify.NotNullOrWhiteSpace(collection); - logger ??= NullLogger.Instance; - relevance ??= DefaultRelevance; - limit ??= DefaultLimit; - - logger.LogDebug("Searching memories in collection '{0}', relevance '{1}'", collection, relevance); - - // Search memory - List memories = await this._memory - .SearchAsync(collection, input, limit.Value, relevance.Value, cancellationToken: cancellationToken) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - if (memories.Count == 0) - { - logger.LogWarning("Memories not found in collection: {0}", collection); - return string.Empty; - } - - logger.LogTrace("Done looking for memories in collection '{0}')", collection); - return limit == 1 ? memories[0].Metadata.Text : JsonSerializer.Serialize(memories.Select(x => x.Metadata.Text)); - } - - /// - /// Save information to semantic memory - /// - /// - /// SKContext.Variables["input"] = "the capital of France is Paris" - /// SKContext.Variables[TextMemorySkill.KeyParam] = "countryInfo1" - /// {{memory.save $input }} - /// - /// The information to save - /// Memories collection associated with the information to save - /// The key associated with the information to save - /// Application logger - /// The to monitor for cancellation requests. The default is . - [SKFunction, Description("Save information to semantic memory")] - public async Task SaveAsync( - [Description("The information to save")] string input, - [SKName(CollectionParam), Description("Memories collection associated with the information to save"), DefaultValue(DefaultCollection)] string collection, - [SKName(KeyParam), Description("The key associated with the information to save")] string key, - ILogger? logger, - CancellationToken cancellationToken = default) - { - Verify.NotNullOrWhiteSpace(collection); - Verify.NotNullOrWhiteSpace(key); - logger ??= NullLogger.Instance; - - logger.LogDebug("Saving memory to collection '{0}'", collection); - - await this._memory.SaveInformationAsync(collection, text: input, id: key, cancellationToken: cancellationToken).ConfigureAwait(false); - } - - /// - /// Remove specific memory - /// - /// - /// SKContext.Variables[TextMemorySkill.KeyParam] = "countryInfo1" - /// {{memory.remove }} - /// - /// Memories collection associated with the information to save - /// The key associated with the information to save - /// Application logger - /// The to monitor for cancellation requests. The default is . - [SKFunction, Description("Remove specific memory")] - public async Task RemoveAsync( - [SKName(CollectionParam), Description("Memories collection associated with the information to save"), DefaultValue(DefaultCollection)] string collection, - [SKName(KeyParam), Description("The key associated with the information to save")] string key, - ILogger? logger, - CancellationToken cancellationToken = default) - { - Verify.NotNullOrWhiteSpace(collection); - Verify.NotNullOrWhiteSpace(key); - logger ??= NullLogger.Instance; - - logger.LogDebug("Removing memory from collection '{0}'", collection); - - await this._memory.RemoveAsync(collection, key, cancellationToken: cancellationToken).ConfigureAwait(false); - } -} diff --git a/dotnet/src/Skills/Skills.Core/TextSkill.cs b/dotnet/src/Skills/Skills.Core/TextSkill.cs deleted file mode 100644 index 5103f8caf5de..000000000000 --- a/dotnet/src/Skills/Skills.Core/TextSkill.cs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using System.Globalization; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Core; - -/// -/// TextSkill provides a set of functions to manipulate strings. -/// -/// -/// Usage: kernel.ImportSkill("text", new TextSkill()); -/// -/// Examples: -/// SKContext.Variables["input"] = " hello world " -/// {{text.trim $input}} => "hello world" -/// {{text.trimStart $input} => "hello world " -/// {{text.trimEnd $input} => " hello world" -/// SKContext.Variables["input"] = "hello world" -/// {{text.uppercase $input}} => "HELLO WORLD" -/// SKContext.Variables["input"] = "HELLO WORLD" -/// {{text.lowercase $input}} => "hello world" -/// -public sealed class TextSkill -{ - /// - /// Trim whitespace from the start and end of a string. - /// - /// - /// SKContext.Variables["input"] = " hello world " - /// {{text.trim $input}} => "hello world" - /// - /// The string to trim. - /// The trimmed string. - [SKFunction, Description("Trim whitespace from the start and end of a string.")] - public string Trim(string input) => input.Trim(); - - /// - /// Trim whitespace from the start of a string. - /// - /// - /// SKContext.Variables["input"] = " hello world " - /// {{text.trimStart $input} => "hello world " - /// - /// The string to trim. - /// The trimmed string. - [SKFunction, Description("Trim whitespace from the start of a string.")] - public string TrimStart(string input) => input.TrimStart(); - - /// - /// Trim whitespace from the end of a string. - /// - /// - /// SKContext.Variables["input"] = " hello world " - /// {{text.trimEnd $input} => " hello world" - /// - /// The string to trim. - /// The trimmed string. - [SKFunction, Description("Trim whitespace from the end of a string.")] - public string TrimEnd(string input) => input.TrimEnd(); - - /// - /// Convert a string to uppercase. - /// - /// - /// SKContext.Variables["input"] = "hello world" - /// {{text.uppercase $input}} => "HELLO WORLD" - /// - /// The string to convert. - /// An object that supplies culture-specific casing rules. - /// The converted string. - [SKFunction, Description("Convert a string to uppercase.")] - public string Uppercase(string input, CultureInfo? cultureInfo = null) => input.ToUpper(cultureInfo); - - /// - /// Convert a string to lowercase. - /// - /// - /// SKContext.Variables["input"] = "HELLO WORLD" - /// {{text.lowercase $input}} => "hello world" - /// - /// The string to convert. - /// An object that supplies culture-specific casing rules. - /// The converted string. - [SKFunction, Description("Convert a string to lowercase.")] - public string Lowercase(string input, CultureInfo? cultureInfo = null) => input.ToLower(cultureInfo); - - /// - /// Get the length of a string. Returns 0 if null or empty - /// - /// - /// SKContext.Variables["input"] = "HELLO WORLD" - /// {{text.length $input}} => "11" - /// - /// The string to get length. - /// The length size of string (0) if null or empty. - [SKFunction, Description("Get the length of a string.")] - public int Length(string input) => input?.Length ?? 0; - - /// - /// Concatenate two strings into one - /// - /// - /// text = "HELLO " - /// SKContext.Variables["input2"] = "WORLD" - /// Result: "HELLO WORLD" - /// - /// First input to concatenate with - /// Second input to concatenate with - /// Concatenation result from both inputs. - [SKFunction, Description("Concat two strings into one.")] - public string Concat( - [Description("First input to concatenate with")] string input, - [Description("Second input to concatenate with")] string input2) => - string.Concat(input, input2); - - [SKFunction, Description("Echo the input string. Useful for capturing plan input for use in multiple functions.")] - public string Echo( - [Description("Input string to echo.")] string text) - { - return text; - } -} diff --git a/dotnet/src/Skills/Skills.Core/TimeSkill.cs b/dotnet/src/Skills/Skills.Core/TimeSkill.cs deleted file mode 100644 index a3ca9782d9bf..000000000000 --- a/dotnet/src/Skills/Skills.Core/TimeSkill.cs +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ComponentModel; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Core; - -/// -/// TimeSkill provides a set of functions to get the current time and date. -/// -/// -/// Usage: kernel.ImportSkill("time", new TimeSkill()); -/// Examples: -/// {{time.date}} => Sunday, 12 January, 2031 -/// {{time.today}} => Sunday, 12 January, 2031 -/// {{time.now}} => Sunday, January 12, 2031 9:15 PM -/// {{time.utcNow}} => Sunday, January 13, 2031 5:15 AM -/// {{time.time}} => 09:15:07 PM -/// {{time.year}} => 2031 -/// {{time.month}} => January -/// {{time.monthNumber}} => 01 -/// {{time.day}} => 12 -/// {{time.dayOfMonth}} => 12 -/// {{time.dayOfWeek}} => Sunday -/// {{time.hour}} => 9 PM -/// {{time.hourNumber}} => 21 -/// {{time.daysAgo $days}} => Sunday, January 12, 2025 9:15 PM -/// {{time.lastMatchingDay $dayName}} => Sunday, 7 May, 2023 -/// {{time.minute}} => 15 -/// {{time.minutes}} => 15 -/// {{time.second}} => 7 -/// {{time.seconds}} => 7 -/// {{time.timeZoneOffset}} => -08:00 -/// {{time.timeZoneName}} => PST -/// -/// -/// Note: the time represents the time on the hw/vm/machine where the kernel is running. -/// TODO: import and use user's timezone -/// -public sealed class TimeSkill -{ - /// - /// Get the current date - /// - /// - /// {{time.date}} => Sunday, 12 January, 2031 - /// - /// The current date - [SKFunction, Description("Get the current date")] - public string Date(IFormatProvider? formatProvider = null) => - // Example: Sunday, 12 January, 2025 - DateTimeOffset.Now.ToString("D", formatProvider); - - /// - /// Get the current date - /// - /// - /// {{time.today}} => Sunday, 12 January, 2031 - /// - /// The current date - [SKFunction, Description("Get the current date")] - public string Today(IFormatProvider? formatProvider = null) => - // Example: Sunday, 12 January, 2025 - this.Date(formatProvider); - - /// - /// Get the current date and time in the local time zone" - /// - /// - /// {{time.now}} => Sunday, January 12, 2025 9:15 PM - /// - /// The current date and time in the local time zone - [SKFunction, Description("Get the current date and time in the local time zone")] - public string Now(IFormatProvider? formatProvider = null) => - // Sunday, January 12, 2025 9:15 PM - DateTimeOffset.Now.ToString("f", formatProvider); - - /// - /// Get the current UTC date and time - /// - /// - /// {{time.utcNow}} => Sunday, January 13, 2025 5:15 AM - /// - /// The current UTC date and time - [SKFunction, Description("Get the current UTC date and time")] - public string UtcNow(IFormatProvider? formatProvider = null) => - // Sunday, January 13, 2025 5:15 AM - DateTimeOffset.UtcNow.ToString("f", formatProvider); - - /// - /// Get the current time - /// - /// - /// {{time.time}} => 09:15:07 PM - /// - /// The current time - [SKFunction, Description("Get the current time")] - public string Time(IFormatProvider? formatProvider = null) => - // Example: 09:15:07 PM - DateTimeOffset.Now.ToString("hh:mm:ss tt", formatProvider); - - /// - /// Get the current year - /// - /// - /// {{time.year}} => 2025 - /// - /// The current year - [SKFunction, Description("Get the current year")] - public string Year(IFormatProvider? formatProvider = null) => - // Example: 2025 - DateTimeOffset.Now.ToString("yyyy", formatProvider); - - /// - /// Get the current month name - /// - /// - /// {time.month}} => January - /// - /// The current month name - [SKFunction, Description("Get the current month name")] - public string Month(IFormatProvider? formatProvider = null) => - // Example: January - DateTimeOffset.Now.ToString("MMMM", formatProvider); - - /// - /// Get the current month number - /// - /// - /// {{time.monthNumber}} => 01 - /// - /// The current month number - [SKFunction, Description("Get the current month number")] - public string MonthNumber(IFormatProvider? formatProvider = null) => - // Example: 01 - DateTimeOffset.Now.ToString("MM", formatProvider); - - /// - /// Get the current day of the month - /// - /// - /// {{time.day}} => 12 - /// - /// The current day of the month - [SKFunction, Description("Get the current day of the month")] - public string Day(IFormatProvider? formatProvider = null) => - // Example: 12 - DateTimeOffset.Now.ToString("dd", formatProvider); - - /// - /// Get the date a provided number of days in the past - /// - /// - /// SKContext.Variables["input"] = "3" - /// {{time.daysAgo}} => Sunday, January 12, 2025 9:15 PM - /// - /// The date the provided number of days before today - [SKFunction] - [Description("Get the date offset by a provided number of days from today")] - public string DaysAgo([Description("The number of days to offset from today"), SKName("input")] double daysOffset, IFormatProvider? formatProvider = null) => - DateTimeOffset.Now.AddDays(-daysOffset).ToString("D", formatProvider); - - /// - /// Get the current day of the week - /// - /// - /// {{time.dayOfWeek}} => Sunday - /// - /// The current day of the week - [SKFunction, Description("Get the current day of the week")] - public string DayOfWeek(IFormatProvider? formatProvider = null) => - // Example: Sunday - DateTimeOffset.Now.ToString("dddd", formatProvider); - - /// - /// Get the current clock hour - /// - /// - /// {{time.hour}} => 9 PM - /// - /// The current clock hour - [SKFunction, Description("Get the current clock hour")] - public string Hour(IFormatProvider? formatProvider = null) => - // Example: 9 PM - DateTimeOffset.Now.ToString("h tt", formatProvider); - - /// - /// Get the current clock 24-hour number - /// - /// - /// {{time.hourNumber}} => 21 - /// - /// The current clock 24-hour number - [SKFunction, Description("Get the current clock 24-hour number")] - public string HourNumber(IFormatProvider? formatProvider = null) => - // Example: 21 - DateTimeOffset.Now.ToString("HH", formatProvider); - - /// - /// Get the date of the previous day matching the supplied day name - /// - /// - /// {{time.lastMatchingDay $dayName}} => Sunday, 7 May, 2023 - /// - /// The date of the last instance of this day name - /// dayName is not a recognized name of a day of the week - [SKFunction] - [Description("Get the date of the last day matching the supplied week day name in English. Example: Che giorno era 'Martedi' scorso -> dateMatchingLastDayName 'Tuesday' => Tuesday, 16 May, 2023")] - public string DateMatchingLastDayName( - [Description("The day name to match"), SKName("input")] DayOfWeek dayName, - IFormatProvider? formatProvider = null) - { - DateTimeOffset dateTime = DateTimeOffset.Now; - - // Walk backwards from the previous day for up to a week to find the matching day - for (int i = 1; i <= 7; ++i) - { - dateTime = dateTime.AddDays(-1); - if (dateTime.DayOfWeek == dayName) - { - break; - } - } - - return dateTime.ToString("D", formatProvider); - } - - /// - /// Get the minutes on the current hour - /// - /// - /// {{time.minute}} => 15 - /// - /// The minutes on the current hour - [SKFunction, Description("Get the minutes on the current hour")] - public string Minute(IFormatProvider? formatProvider = null) => - // Example: 15 - DateTimeOffset.Now.ToString("mm", formatProvider); - - /// - /// Get the seconds on the current minute - /// - /// - /// {{time.second}} => 7 - /// - /// The seconds on the current minute - [SKFunction, Description("Get the seconds on the current minute")] - public string Second(IFormatProvider? formatProvider = null) => - // Example: 07 - DateTimeOffset.Now.ToString("ss", formatProvider); - - /// - /// Get the local time zone offset from UTC - /// - /// - /// {{time.timeZoneOffset}} => -08:00 - /// - /// The local time zone offset from UTC - [SKFunction, Description("Get the local time zone offset from UTC")] - public string TimeZoneOffset(IFormatProvider? formatProvider = null) => - // Example: -08:00 - DateTimeOffset.Now.ToString("%K", formatProvider); - - /// - /// Get the local time zone name - /// - /// - /// {{time.timeZoneName}} => PST - /// - /// - /// Note: this is the "current" timezone and it can change over the year, e.g. from PST to PDT - /// - /// The local time zone name - [SKFunction, Description("Get the local time zone name")] - public string TimeZoneName() => - // Example: PST - // Note: this is the "current" timezone and it can change over the year, e.g. from PST to PDT - TimeZoneInfo.Local.DisplayName; -} diff --git a/dotnet/src/Skills/Skills.Core/WaitSkill.cs b/dotnet/src/Skills/Skills.Core/WaitSkill.cs deleted file mode 100644 index c3930123aee6..000000000000 --- a/dotnet/src/Skills/Skills.Core/WaitSkill.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Core; - -/// -/// WaitSkill provides a set of functions to wait before making the rest of operations. -/// -/// -/// Usage: kernel.ImportSkill("wait", new WaitSkill()); -/// Examples: -/// {{wait.seconds 10}} => Wait 10 seconds -/// -public sealed class WaitSkill -{ - private readonly IWaitProvider _waitProvider; - - public interface IWaitProvider - { - Task DelayAsync(int milliSeconds); - } - - private sealed class WaitProvider : IWaitProvider - { - public Task DelayAsync(int milliSeconds) - { - return Task.Delay(milliSeconds); - } - } - - public WaitSkill(IWaitProvider? waitProvider = null) - { - this._waitProvider = waitProvider ?? new WaitProvider(); - } - - /// - /// Wait a given amount of seconds - /// - /// - /// {{wait.seconds 10}} (Wait 10 seconds) - /// - [SKFunction, Description("Wait a given amount of seconds")] - public async Task SecondsAsync([Description("The number of seconds to wait")] decimal seconds) - { - var milliseconds = seconds * 1000; - milliseconds = milliseconds > 0 ? milliseconds : 0; - - await this._waitProvider.DelayAsync((int)milliseconds).ConfigureAwait(false); - } -} diff --git a/dotnet/src/Skills/Skills.Document/DocumentSkill.cs b/dotnet/src/Skills/Skills.Document/DocumentSkill.cs deleted file mode 100644 index b88e68cac813..000000000000 --- a/dotnet/src/Skills/Skills.Document/DocumentSkill.cs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ComponentModel; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.Document.FileSystem; - -namespace Microsoft.SemanticKernel.Skills.Document; - -//********************************************************************************************************************** -// EXAMPLE USAGE -// Option #1: as a standalone C# function -// -// DocumentSkill documentSkill = new(new WordDocumentConnector(), new LocalDriveConnector()); -// string filePath = "PATH_TO_DOCX_FILE.docx"; -// string text = await documentSkill.ReadTextAsync(filePath); -// Console.WriteLine(text); -// -// -// Option #2: with the Semantic Kernel -// -// DocumentSkill documentSkill = new(new WordDocumentConnector(), new LocalDriveConnector()); -// string filePath = "PATH_TO_DOCX_FILE.docx"; -// ISemanticKernel kernel = SemanticKernel.Build(); -// var result = await kernel.RunAsync( -// filePath, -// documentSkill.ReadTextAsync); -// Console.WriteLine(result); -//********************************************************************************************************************** - -/// -/// Skill for interacting with documents (e.g. Microsoft Word) -/// -public sealed class DocumentSkill -{ - /// - /// parameter names. - /// - public static class Parameters - { - /// - /// Document file path. - /// - public const string FilePath = "filePath"; - } - - private readonly IDocumentConnector _documentConnector; - private readonly IFileSystemConnector _fileSystemConnector; - private readonly ILogger _logger; - - /// - /// Initializes a new instance of the class. - /// - /// Document connector - /// File system connector - /// Optional logger - public DocumentSkill(IDocumentConnector documentConnector, IFileSystemConnector fileSystemConnector, ILogger? logger = null) - { - this._documentConnector = documentConnector ?? throw new ArgumentNullException(nameof(documentConnector)); - this._fileSystemConnector = fileSystemConnector ?? throw new ArgumentNullException(nameof(fileSystemConnector)); - this._logger = logger ?? new NullLogger(); - } - - /// - /// Read all text from a document, using as the file path. - /// - [SKFunction, Description("Read all text from a document")] - public async Task ReadTextAsync( - [Description("Path to the file to read")] string filePath, - CancellationToken cancellationToken = default) - { - this._logger.LogDebug("Reading text from {0}", filePath); - using var stream = await this._fileSystemConnector.GetFileContentStreamAsync(filePath, cancellationToken).ConfigureAwait(false); - return this._documentConnector.ReadText(stream); - } - - /// - /// Append the text in to a document. If the document doesn't exist, it will be created. - /// - [SKFunction, Description("Append text to a document. If the document doesn't exist, it will be created.")] - public async Task AppendTextAsync( - [Description("Text to append")] string text, - [Description("Destination file path")] string filePath, - CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(filePath)) - { - throw new ArgumentException("Variable was null or whitespace", nameof(filePath)); - } - - // If the document already exists, open it. If not, create it. - if (await this._fileSystemConnector.FileExistsAsync(filePath, cancellationToken).ConfigureAwait(false)) - { - this._logger.LogDebug("Writing text to file {0}", filePath); - using Stream stream = await this._fileSystemConnector.GetWriteableFileStreamAsync(filePath, cancellationToken).ConfigureAwait(false); - this._documentConnector.AppendText(stream, text); - } - else - { - this._logger.LogDebug("File does not exist. Creating file at {0}", filePath); - using Stream stream = await this._fileSystemConnector.CreateFileAsync(filePath, cancellationToken).ConfigureAwait(false); - this._documentConnector.Initialize(stream); - - this._logger.LogDebug("Writing text to {0}", filePath); - this._documentConnector.AppendText(stream, text); - } - } -} diff --git a/dotnet/src/Skills/Skills.Document/Skills.Document.csproj b/dotnet/src/Skills/Skills.Document/Skills.Document.csproj deleted file mode 100644 index 80f6be777539..000000000000 --- a/dotnet/src/Skills/Skills.Document/Skills.Document.csproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - - Microsoft.SemanticKernel.Skills.Document - $(AssemblyName) - netstandard2.0 - - - - - - - Semantic Kernel - Document Skill - Semantic Kernel Document Skill: Word processing, OpenXML, etc. - - - - - - - - - - - diff --git a/dotnet/src/Skills/Skills.Grpc/Extensions/KernelGrpcExtensions.cs b/dotnet/src/Skills/Skills.Grpc/Extensions/KernelGrpcExtensions.cs deleted file mode 100644 index 5dc1086e8f7b..000000000000 --- a/dotnet/src/Skills/Skills.Grpc/Extensions/KernelGrpcExtensions.cs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.Grpc.Model; -using Microsoft.SemanticKernel.Skills.Grpc.Protobuf; - -namespace Microsoft.SemanticKernel.Skills.Grpc.Extensions; - -/// -/// extensions methods for gRPC functionality. -/// -public static class KernelGrpcExtensions -{ - /// - /// Imports gRPC document from a directory. - /// - /// Semantic Kernel instance. - /// Directory containing the skill directory. - /// Name of the directory containing the selected skill. - /// HttpClient to use for sending requests. - /// A list of all the semantic functions representing the skill. - public static IDictionary ImportGrpcSkillFromDirectory( - this IKernel kernel, - string parentDirectory, - string skillDirectoryName, - HttpClient? httpClient = null) - { - const string ProtoFile = "grpc.proto"; - - Verify.ValidSkillName(skillDirectoryName); - - var skillDir = Path.Combine(parentDirectory, skillDirectoryName); - Verify.DirectoryExists(skillDir); - - var filePath = Path.Combine(skillDir, ProtoFile); - if (!File.Exists(filePath)) - { - throw new FileNotFoundException($"No .proto document for the specified path - {filePath} is found."); - } - - kernel.Logger.LogTrace("Registering gRPC functions from {0} .proto document", filePath); - - using var stream = File.OpenRead(filePath); - - return kernel.RegisterGrpcSkill(stream, skillDirectoryName, httpClient); - } - - /// - /// Imports gRPC document from a file. - /// - /// Semantic Kernel instance. - /// Name of the skill to register. - /// File path to .proto document. - /// HttpClient to use for sending requests. - /// A list of all the semantic functions representing the skill. - public static IDictionary ImportGrpcSkillFromFile( - this IKernel kernel, - string skillName, - string filePath, - HttpClient? httpClient = null) - { - if (!File.Exists(filePath)) - { - throw new FileNotFoundException($"No .proto document for the specified path - {filePath} is found."); - } - - kernel.Logger.LogTrace("Registering gRPC functions from {0} .proto document", filePath); - - using var stream = File.OpenRead(filePath); - - return kernel.RegisterGrpcSkill(stream, skillName, httpClient); - } - - /// - /// Registers an gRPC skill. - /// - /// Semantic Kernel instance. - /// .proto document stream. - /// Skill name. - /// HttpClient to use for sending requests. - /// A list of all the semantic functions representing the skill. - public static IDictionary RegisterGrpcSkill( - this IKernel kernel, - Stream documentStream, - string skillName, - HttpClient? httpClient = null) - { - Verify.NotNull(kernel); - Verify.ValidSkillName(skillName); - - // Parse - var parser = new ProtoDocumentParser(); - - var operations = parser.Parse(documentStream, skillName); - - var skill = new Dictionary(); - - var client = HttpClientProvider.GetHttpClient(kernel.Config, httpClient, kernel.Logger); - - var runner = new GrpcOperationRunner(client); - - foreach (var operation in operations) - { - try - { - kernel.Logger.LogTrace("Registering gRPC function {0}.{1}", skillName, operation.Name); - var function = kernel.RegisterGrpcFunction(runner, skillName, operation); - skill[function.Name] = function; - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - //Logging the exception and keep registering other gRPC functions - kernel.Logger.LogWarning(ex, "Something went wrong while rendering the gRPC function. Function: {0}.{1}. Error: {2}", - skillName, operation.Name, ex.Message); - } - } - - return skill; - } - - #region private - - /// - /// Registers SKFunction for a gRPC operation. - /// - /// Semantic Kernel instance. - /// gRPC operation runner. - /// Skill name. - /// The gRPC operation. - /// An instance of class. - private static ISKFunction RegisterGrpcFunction( - this IKernel kernel, - GrpcOperationRunner runner, - string skillName, - GrpcOperation operation) - { - var operationParameters = operation.GetParameters(); - - async Task ExecuteAsync(SKContext context) - { - try - { - var arguments = new Dictionary(); - - //Extract function arguments from context - foreach (var parameter in operationParameters) - { - //A try to resolve argument parameter name. - if (context.Variables.TryGetValue(parameter.Name, out string? value)) - { - arguments.Add(parameter.Name, value); - continue; - } - - throw new KeyNotFoundException($"No variable found in context to use as an argument for the '{parameter.Name}' parameter of the '{skillName}.{operation.Name}' gRPC function."); - } - - //SKFunction should be extended to pass cancellation token for delegateFunction calls. - var result = await runner.RunAsync(operation, arguments, CancellationToken.None).ConfigureAwait(false); - - if (result != null) - { - context.Variables.Update(result.ToString()); - } - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - kernel.Logger.LogWarning(ex, "Something went wrong while rendering the gRPC function. Function: {0}.{1}. Error: {2}", skillName, operation.Name, - ex.Message); - context.Fail(ex.Message, ex); - } - - return context; - } - - var function = SKFunction.FromNativeFunction( - nativeFunction: ExecuteAsync, - parameters: operationParameters.ToList(), - description: operation.Name, - skillName: skillName, - functionName: operation.Name, - logger: kernel.Logger); - - return kernel.RegisterCustomFunction(function); - } - - #endregion -} diff --git a/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationException.cs b/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationException.cs deleted file mode 100644 index 2d7e43b419c2..000000000000 --- a/dotnet/src/Skills/Skills.Grpc/Model/GrpcOperationException.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.Skills.Grpc.Model; - -/// -/// Exception to be throw if a gRPC operation has failed. -/// -public class GrpcOperationException : Exception -{ - /// - /// Creates an instance of a class. - /// - /// The exception message. - internal GrpcOperationException(string message) : base(message) - { - } - - /// - /// Creates an instance of a class. - /// - /// The exception message. - /// The inner exception. - internal GrpcOperationException(string message, Exception innerException) : base(message, innerException) - { - } - - /// - /// Creates an instance of a class. - /// - internal GrpcOperationException() - { - } -} diff --git a/dotnet/src/Skills/Skills.Grpc/Protobuf/ProtobufParsingException.cs b/dotnet/src/Skills/Skills.Grpc/Protobuf/ProtobufParsingException.cs deleted file mode 100644 index 0950a98a8ff6..000000000000 --- a/dotnet/src/Skills/Skills.Grpc/Protobuf/ProtobufParsingException.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.Skills.Grpc.Protobuf; - -/// -/// Exception to be throw if a .proto file parsing has failed. -/// -public class ProtobufParsingException : Exception -{ - /// - /// Creates an instance of a class. - /// - /// The exception message. - internal ProtobufParsingException(string message) : base(message) - { - } - - /// - /// Creates an instance of a class. - /// - /// The exception message. - /// The inner exception. - internal ProtobufParsingException(string message, Exception innerException) : base(message, innerException) - { - } - - /// - /// Creates an instance of a class. - /// - internal ProtobufParsingException() - { - } -} diff --git a/dotnet/src/Skills/Skills.Grpc/Skills.Grpc.csproj b/dotnet/src/Skills/Skills.Grpc/Skills.Grpc.csproj deleted file mode 100644 index 43a13a2d3a9b..000000000000 --- a/dotnet/src/Skills/Skills.Grpc/Skills.Grpc.csproj +++ /dev/null @@ -1,39 +0,0 @@ - - - - - Microsoft.SemanticKernel.Skills.Grpc - $(AssemblyName) - netstandard2.0 - - - - - - - - Semantic Kernel - gRPC Skills - Semantic Kernel gRPC Skill - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dotnet/src/Skills/Skills.MsGraph/CalendarSkill.cs b/dotnet/src/Skills/Skills.MsGraph/CalendarSkill.cs deleted file mode 100644 index 63f34d2510ac..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/CalendarSkill.cs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.MsGraph.Diagnostics; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; - -namespace Microsoft.SemanticKernel.Skills.MsGraph; - -/// -/// Skill for calendar operations. -/// -public sealed class CalendarSkill -{ - /// - /// parameter names. - /// - public static class Parameters - { - /// - /// Event start as DateTimeOffset. - /// - public const string Start = "start"; - - /// - /// Event end as DateTimeOffset. - /// - public const string End = "end"; - - /// - /// Event's location. - /// - public const string Location = "location"; - - /// - /// Event's content. - /// - public const string Content = "content"; - - /// - /// Event's attendees, separated by ',' or ';'. - /// - public const string Attendees = "attendees"; - - /// - /// The name of the top parameter used to limit the number of results returned in the response. - /// - public const string MaxResults = "maxResults"; - - /// - /// The name of the skip parameter used to skip a certain number of results in the response. - /// - public const string Skip = "skip"; - } - - private readonly ICalendarConnector _connector; - private readonly ILogger _logger; - private static readonly JsonSerializerOptions s_options = new() - { - WriteIndented = false, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - /// - /// Initializes a new instance of the class. - /// - /// Calendar connector. - /// Logger. - public CalendarSkill(ICalendarConnector connector, ILogger? logger = null) - { - Ensure.NotNull(connector, nameof(connector)); - - this._connector = connector; - this._logger = logger ?? new NullLogger(); - } - - /// - /// Add an event to my calendar using as the subject. - /// - [SKFunction, Description("Add an event to my calendar.")] - public async Task AddEventAsync( - [Description("Event subject"), SKName("input")] string subject, - [Description("Event start date/time as DateTimeOffset")] DateTimeOffset start, - [Description("Event end date/time as DateTimeOffset")] DateTimeOffset end, - [Description("Event location (optional)")] string? location = null, - [Description("Event content/body (optional)")] string? content = null, - [Description("Event attendees, separated by ',' or ';'.")] string? attendees = null) - { - if (string.IsNullOrWhiteSpace(subject)) - { - throw new ArgumentException($"{nameof(subject)} variable was null or whitespace", nameof(subject)); - } - - CalendarEvent calendarEvent = new() - { - Subject = subject, - Start = start, - End = end, - Location = location, - Content = content, - Attendees = attendees is not null ? attendees.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries) : Enumerable.Empty(), - }; - - // Sensitive data, logging as trace, disabled by default - this._logger.LogTrace("Adding calendar event '{0}'", calendarEvent.Subject); - await this._connector.AddEventAsync(calendarEvent).ConfigureAwait(false); - } - - /// - /// Get calendar events with specified optional clauses used to query for messages. - /// - [SKFunction, Description("Get calendar events.")] - public async Task GetCalendarEventsAsync( - [Description("Optional limit of the number of events to retrieve.")] int? maxResults = 10, - [Description("Optional number of events to skip before retrieving results.")] int? skip = 0, - CancellationToken cancellationToken = default) - { - this._logger.LogDebug("Getting calendar events with query options top: '{0}', skip:'{1}'.", maxResults, skip); - - const string SelectString = "start,subject,organizer,location"; - - IEnumerable events = await this._connector.GetEventsAsync( - top: maxResults, - skip: skip, - select: SelectString, - cancellationToken - ).ConfigureAwait(false); - - return JsonSerializer.Serialize(value: events, options: s_options); - } -} diff --git a/dotnet/src/Skills/Skills.MsGraph/CloudDriveSkill.cs b/dotnet/src/Skills/Skills.MsGraph/CloudDriveSkill.cs deleted file mode 100644 index 572d8317f2b1..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/CloudDriveSkill.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ComponentModel; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.MsGraph.Diagnostics; - -namespace Microsoft.SemanticKernel.Skills.MsGraph; - -/// -/// Cloud drive skill (e.g. OneDrive). -/// -public sealed class CloudDriveSkill -{ - /// - /// parameter names. - /// - public static class Parameters - { - /// - /// Document file path. - /// - public const string DestinationPath = "destinationPath"; - } - - private readonly ICloudDriveConnector _connector; - private readonly ILogger _logger; - - public CloudDriveSkill(ICloudDriveConnector connector, ILogger? logger = null) - { - Ensure.NotNull(connector, nameof(connector)); - - this._connector = connector; - this._logger = logger ?? new NullLogger(); - } - - /// - /// Get the contents of a file stored in a cloud drive. - /// - [SKFunction, Description("Get the contents of a file in a cloud drive.")] - public async Task GetFileContentAsync( - [Description("Path to file")] string filePath, - CancellationToken cancellationToken = default) - { - this._logger.LogDebug("Getting file content for '{0}'", filePath); - Stream fileContentStream = await this._connector.GetFileContentStreamAsync(filePath, cancellationToken).ConfigureAwait(false); - - using StreamReader sr = new(fileContentStream); - string content = await sr.ReadToEndAsync().ConfigureAwait(false); - - return content; - } - - /// - /// Upload a small file to OneDrive (less than 4MB). - /// - [SKFunction, Description("Upload a small file to OneDrive (less than 4MB).")] - public async Task UploadFileAsync( - [Description("Path to file")] string filePath, - [Description("Remote path to store the file")] string destinationPath, - CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(destinationPath)) - { - throw new ArgumentException("Variable was null or whitespace", nameof(destinationPath)); - } - - this._logger.LogDebug("Uploading file '{0}'", filePath); - - // TODO Add support for large file uploads (i.e. upload sessions) - await this._connector.UploadSmallFileAsync(filePath, destinationPath, cancellationToken).ConfigureAwait(false); - } - - /// - /// Create a sharable link to a file stored in a cloud drive. - /// - [SKFunction, Description("Create a sharable link to a file stored in a cloud drive.")] - public async Task CreateLinkAsync( - [Description("Path to file")] string filePath, - CancellationToken cancellationToken = default) - { - this._logger.LogDebug("Creating link for '{0}'", filePath); - const string Type = "view"; // TODO expose this as an SK variable - const string Scope = "anonymous"; // TODO expose this as an SK variable - - return await this._connector.CreateShareLinkAsync(filePath, Type, Scope, cancellationToken).ConfigureAwait(false); - } -} diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/Diagnostics/Ensure.cs b/dotnet/src/Skills/Skills.MsGraph/Connectors/Diagnostics/Ensure.cs deleted file mode 100644 index 8b3e3d3282cd..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/Diagnostics/Ensure.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; - -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Diagnostics; - -/// -/// Internal data validation class. -/// -internal static class Ensure -{ - /// - /// Ensures the given parameter is not null or does not contain only white-space characters. - /// Throws an if the parameter is invalid. - /// - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void NotNullOrWhitespace([NotNull] string parameter, [NotNull] string parameterName) - { - if (string.IsNullOrWhiteSpace(parameter)) - { - throw new ArgumentException($"Parameter '{parameterName}' cannot be null or whitespace.", parameterName); - } - } - - /// - /// Ensures the given parameter is not null. - /// Throws an if the parameter is invalid. - /// - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void NotNull([NotNull] object parameter, [NotNull] string parameterName) - { - if (parameter == null) - { - throw new ArgumentNullException($"Parameter '{parameterName}' cannot be null.", parameterName); - } - } -} diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/Exceptions/MsGraphConnectorException.cs b/dotnet/src/Skills/Skills.MsGraph/Connectors/Exceptions/MsGraphConnectorException.cs deleted file mode 100644 index 0eb430b38926..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/Exceptions/MsGraphConnectorException.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Exceptions; - -/// -/// Exception thrown by the MsGraph connectors -/// -public class MsGraphConnectorException : Exception -{ - /// - /// Initializes a new instance of the class. - /// - /// Exception message. - public MsGraphConnectorException(string message) : base(message) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// Exception message. - /// Inner exception. - public MsGraphConnectorException(string message, Exception innerException) : base(message, innerException) - { - } - - private MsGraphConnectorException() - { - // Do not use, error message is required - } -} diff --git a/dotnet/src/Skills/Skills.MsGraph/Connectors/OneDriveConnector.cs b/dotnet/src/Skills/Skills.MsGraph/Connectors/OneDriveConnector.cs deleted file mode 100644 index 7fe80f22ad3c..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/Connectors/OneDriveConnector.cs +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Graph; -using Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Diagnostics; -using Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Exceptions; - -namespace Microsoft.SemanticKernel.Skills.MsGraph.Connectors; - -/// -/// Connector for OneDrive API -/// -public class OneDriveConnector : ICloudDriveConnector -{ - private readonly GraphServiceClient _graphServiceClient; - - /// - /// Initializes a new instance of the class. - /// - /// A graph service client. - public OneDriveConnector(GraphServiceClient graphServiceClient) - { - this._graphServiceClient = graphServiceClient; - } - - /// - public async Task GetFileContentStreamAsync(string filePath, CancellationToken cancellationToken = default) - { - Ensure.NotNullOrWhitespace(filePath, nameof(filePath)); - - return await this._graphServiceClient.Me - .Drive.Root - .ItemWithPath(filePath).Content - .Request().GetAsync(cancellationToken).ConfigureAwait(false); - } - - public async Task FileExistsAsync(string filePath, CancellationToken cancellationToken = default) - { - Ensure.NotNullOrWhitespace(filePath, nameof(filePath)); - - try - { - await this._graphServiceClient.Me - .Drive.Root - .ItemWithPath(filePath).Request().GetAsync(cancellationToken).ConfigureAwait(false); - - // If no exception is thrown, the file exists. - return true; - } - catch (ServiceException ex) - { - // If the exception is a 404 Not Found, the file does not exist. - if (ex.StatusCode == HttpStatusCode.NotFound) - { - return false; - } - - // Otherwise, rethrow the exception. - throw; - } - } - - /// - public async Task UploadSmallFileAsync(string filePath, string destinationPath, CancellationToken cancellationToken = default) - { - Ensure.NotNullOrWhitespace(filePath, nameof(filePath)); - Ensure.NotNullOrWhitespace(destinationPath, nameof(destinationPath)); - - filePath = Environment.ExpandEnvironmentVariables(filePath); - - long fileSize = new FileInfo(filePath).Length; - if (fileSize > 4 * 1024 * 1024) - { - throw new IOException("File is too large to upload - function currently only supports files up to 4MB."); - } - - using FileStream fileContentStream = new(filePath, FileMode.Open, FileAccess.Read); - - GraphResponse response = await this._graphServiceClient.Me - .Drive.Root - .ItemWithPath(destinationPath).Content - .Request().PutResponseAsync(fileContentStream, cancellationToken, HttpCompletionOption.ResponseContentRead).ConfigureAwait(false); - - response.ToHttpResponseMessage().EnsureSuccessStatusCode(); - } - - /// - public async Task CreateShareLinkAsync(string filePath, string type = "view", string scope = "anonymous", - CancellationToken cancellationToken = default) - { - Ensure.NotNullOrWhitespace(filePath, nameof(filePath)); - Ensure.NotNullOrWhitespace(type, nameof(type)); - Ensure.NotNullOrWhitespace(scope, nameof(scope)); - - GraphResponse response = await this._graphServiceClient.Me - .Drive.Root - .ItemWithPath(filePath) - .CreateLink(type, scope) - .Request().PostResponseAsync(cancellationToken).ConfigureAwait(false); - - response.ToHttpResponseMessage().EnsureSuccessStatusCode(); - - string? result = (await response.GetResponseObjectAsync().ConfigureAwait(false)).Link?.WebUrl; - if (string.IsNullOrWhiteSpace(result)) - { - throw new MsGraphConnectorException("Shareable file link was null or whitespace."); - } - - return result!; - } -} diff --git a/dotnet/src/Skills/Skills.MsGraph/Diagnostics/Ensure.cs b/dotnet/src/Skills/Skills.MsGraph/Diagnostics/Ensure.cs deleted file mode 100644 index 69256dbfe2e8..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/Diagnostics/Ensure.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; - -namespace Microsoft.SemanticKernel.Skills.MsGraph.Diagnostics; - -internal static class Ensure -{ - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void NotNullOrWhitespace([NotNull] string parameter, [NotNull] string parameterName) - { - if (string.IsNullOrWhiteSpace(parameter)) - { - throw new ArgumentException($"Parameter '{parameterName}' cannot be null or whitespace.", parameterName); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void NotNull([NotNull] object parameter, [NotNull] string parameterName) - { - if (parameter == null) - { - throw new ArgumentNullException($"Parameter '{parameterName}' cannot be null.", parameterName); - } - } -} diff --git a/dotnet/src/Skills/Skills.MsGraph/EmailSkill.cs b/dotnet/src/Skills/Skills.MsGraph/EmailSkill.cs deleted file mode 100644 index f6f7222d686b..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/EmailSkill.cs +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.MsGraph.Diagnostics; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; - -namespace Microsoft.SemanticKernel.Skills.MsGraph; - -/// -/// Email skill (e.g. Outlook). -/// -public sealed class EmailSkill -{ - /// - /// parameter names. - /// - public static class Parameters - { - /// - /// Email recipients, separated by ',' or ';'. - /// - public const string Recipients = "recipients"; - - /// - /// Email subject. - /// - public const string Subject = "subject"; - - /// - /// The name of the top parameter used to limit the number of results returned in the response. - /// - public const string MaxResults = "maxResults"; - - /// - /// The name of the skip parameter used to skip a certain number of results in the response. - /// - public const string Skip = "skip"; - } - - private readonly IEmailConnector _connector; - private readonly ILogger _logger; - private static readonly JsonSerializerOptions s_options = new() - { - WriteIndented = false, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - /// - /// Initializes a new instance of the class. - /// - /// Email connector. - /// Logger. - public EmailSkill(IEmailConnector connector, ILogger? logger = null) - { - Ensure.NotNull(connector, nameof(connector)); - - this._connector = connector; - this._logger = logger ?? new NullLogger(); - } - - /// - /// Get my email address. - /// - [SKFunction, Description("Gets the email address for me.")] - public async Task GetMyEmailAddressAsync() - => await this._connector.GetMyEmailAddressAsync().ConfigureAwait(false); - - /// - /// Send an email using as the body. - /// - [SKFunction, Description("Send an email to one or more recipients.")] - public async Task SendEmailAsync( - [Description("Email content/body")] string content, - [Description("Recipients of the email, separated by ',' or ';'.")] string recipients, - [Description("Subject of the email")] string subject, - CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(recipients)) - { - throw new ArgumentException("Variable was null or whitespace", nameof(recipients)); - } - - if (string.IsNullOrWhiteSpace(subject)) - { - throw new ArgumentException("Variable was null or whitespace", nameof(subject)); - } - - // Sensitive data, logging as trace, disabled by default - this._logger.LogTrace("Sending email to '{0}' with subject '{1}'", recipients, subject); - string[] recipientList = recipients.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries); - await this._connector.SendEmailAsync(subject, content, recipientList, cancellationToken).ConfigureAwait(false); - } - - /// - /// Get email messages with specified optional clauses used to query for messages. - /// - [SKFunction, Description("Get email messages.")] - public async Task GetEmailMessagesAsync( - [Description("Optional limit of the number of message to retrieve.")] int? maxResults = 10, - [Description("Optional number of message to skip before retrieving results.")] int? skip = 0, - CancellationToken cancellationToken = default) - { - this._logger.LogDebug("Getting email messages with query options top: '{0}', skip:'{1}'.", maxResults, skip); - - const string SelectString = "subject,receivedDateTime,bodyPreview"; - - IEnumerable messages = await this._connector.GetMessagesAsync( - top: maxResults, - skip: skip, - select: SelectString, - cancellationToken) - .ConfigureAwait(false); - - return JsonSerializer.Serialize(value: messages, options: s_options); - } -} diff --git a/dotnet/src/Skills/Skills.MsGraph/OrganizationHierarchySkill.cs b/dotnet/src/Skills/Skills.MsGraph/OrganizationHierarchySkill.cs deleted file mode 100644 index 4ca33d5a21fc..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/OrganizationHierarchySkill.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.MsGraph.Diagnostics; - -namespace Microsoft.SemanticKernel.Skills.MsGraph; - -/// -/// Organizational Hierarchy skill. -/// -public sealed class OrganizationHierarchySkill -{ - private readonly IOrganizationHierarchyConnector _connector; - - public OrganizationHierarchySkill(IOrganizationHierarchyConnector connector) - { - Ensure.NotNull(connector, nameof(connector)); - - this._connector = connector; - } - - /// - /// Get the emails of the direct reports of the current user. - /// - [SKFunction, Description("Get my direct report's email addresses.")] - public async Task GetMyDirectReportsEmailAsync(CancellationToken cancellationToken = default) - => JsonSerializer.Serialize(await this._connector.GetDirectReportsEmailAsync(cancellationToken).ConfigureAwait(false)); - - /// - /// Get the email of the manager of the current user. - /// - [SKFunction, Description("Get my manager's email address.")] - public async Task GetMyManagerEmailAsync(CancellationToken cancellationToken = default) - => await this._connector.GetManagerEmailAsync(cancellationToken).ConfigureAwait(false); - - /// - /// Get the name of the manager of the current user. - /// - [SKFunction, Description("Get my manager's name.")] - public async Task GetMyManagerNameAsync(CancellationToken cancellationToken = default) - => await this._connector.GetManagerNameAsync(cancellationToken).ConfigureAwait(false); -} diff --git a/dotnet/src/Skills/Skills.MsGraph/Skills.MsGraph.csproj b/dotnet/src/Skills/Skills.MsGraph/Skills.MsGraph.csproj deleted file mode 100644 index fb6116695520..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/Skills.MsGraph.csproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - - Microsoft.SemanticKernel.Skills.MsGraph - $(AssemblyName) - netstandard2.0 - - - - - - - Semantic Kernel - Microsoft Graph Connector - Semantic Kernel Microsoft Graph Skill: access your tenant data, schedule meetings, send emails, etc. - - - - - - - - - - - - - diff --git a/dotnet/src/Skills/Skills.MsGraph/TaskListSkill.cs b/dotnet/src/Skills/Skills.MsGraph/TaskListSkill.cs deleted file mode 100644 index 550f1d6dadd7..000000000000 --- a/dotnet/src/Skills/Skills.MsGraph/TaskListSkill.cs +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.MsGraph.Diagnostics; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; - -namespace Microsoft.SemanticKernel.Skills.MsGraph; - -/// -/// Task list skill (e.g. Microsoft To-Do) -/// -public sealed class TaskListSkill -{ - /// - /// parameter names. - /// - public static class Parameters - { - /// - /// Task reminder as DateTimeOffset. - /// - public const string Reminder = "reminder"; - - /// - /// Whether to include completed tasks. - /// - public const string IncludeCompleted = "includeCompleted"; - } - - private readonly ITaskManagementConnector _connector; - private readonly ILogger _logger; - - /// - /// Initializes a new instance of the class. - /// - /// Task list connector. - /// Logger. - public TaskListSkill(ITaskManagementConnector connector, ILogger? logger = null) - { - Ensure.NotNull(connector, nameof(connector)); - - this._connector = connector; - this._logger = logger ?? new NullLogger(); - } - - /// - /// Calculates an upcoming day of week (e.g. 'next Monday'). - /// - public static DateTimeOffset GetNextDayOfWeek(DayOfWeek dayOfWeek, TimeSpan timeOfDay) - { - DateTimeOffset today = new(DateTime.Today); - int nextDayOfWeekOffset = dayOfWeek - today.DayOfWeek; - if (nextDayOfWeekOffset <= 0) - { - nextDayOfWeekOffset += 7; - } - - DateTimeOffset nextDayOfWeek = today.AddDays(nextDayOfWeekOffset); - DateTimeOffset nextDayOfWeekAtTimeOfDay = nextDayOfWeek.Add(timeOfDay); - - return nextDayOfWeekAtTimeOfDay; - } - - /// - /// Add a task to a To-Do list with an optional reminder. - /// - [SKFunction, Description("Add a task to a task list with an optional reminder.")] - public async Task AddTaskAsync( - [Description("Title of the task.")] string title, - [Description("Reminder for the task in DateTimeOffset (optional)")] string? reminder = null, - CancellationToken cancellationToken = default) - { - TaskManagementTaskList? defaultTaskList = await this._connector.GetDefaultTaskListAsync(cancellationToken).ConfigureAwait(false); - if (defaultTaskList == null) - { - throw new InvalidOperationException("No default task list found."); - } - - TaskManagementTask task = new( - id: Guid.NewGuid().ToString(), - title: title, - reminder: reminder); - - // Sensitive data, logging as trace, disabled by default - this._logger.LogTrace("Adding task '{0}' to task list '{1}'", task.Title, defaultTaskList.Name); - - await this._connector.AddTaskAsync(defaultTaskList.Id, task, cancellationToken).ConfigureAwait(false); - } - - /// - /// Get tasks from the default task list. - /// - [SKFunction, Description("Get tasks from the default task list.")] - public async Task GetDefaultTasksAsync( - [Description("Whether to include completed tasks (optional)")] string includeCompleted = "false", - CancellationToken cancellationToken = default) - { - TaskManagementTaskList? defaultTaskList = await this._connector.GetDefaultTaskListAsync(cancellationToken).ConfigureAwait(false); - if (defaultTaskList == null) - { - throw new InvalidOperationException("No default task list found."); - } - - if (!bool.TryParse(includeCompleted, out bool includeCompletedValue)) - { - this._logger.LogWarning("Invalid value for '{0}' variable: '{1}'", nameof(includeCompleted), includeCompleted); - } - - IEnumerable tasks = await this._connector.GetTasksAsync(defaultTaskList.Id, includeCompletedValue, cancellationToken).ConfigureAwait(false); - return JsonSerializer.Serialize(tasks); - } -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Authentication/AuthenticateRequestAsyncCallback.cs b/dotnet/src/Skills/Skills.OpenAPI/Authentication/AuthenticateRequestAsyncCallback.cs deleted file mode 100644 index 60a9d91c46f8..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Authentication/AuthenticateRequestAsyncCallback.cs +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Net.Http; -using System.Threading.Tasks; - -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; - -public delegate Task AuthenticateRequestAsyncCallback(HttpRequestMessage request); diff --git a/dotnet/src/Skills/Skills.OpenAPI/Authentication/README.md b/dotnet/src/Skills/Skills.OpenAPI/Authentication/README.md deleted file mode 100644 index 495d7cbe7123..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Authentication/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Authentication for the OpenAPI Skill - -The Semantic Kernel OpenAPI Skill enables developers to take any REST API that follows the OpenAPI specification and import it as a skill to the Semantic Kernel. However, the Kernel needs to be able to authenticate outgoing requests per the requirements of the target API. This document outlines the authentication model for the OpenAPI skill as well as the reference implementations provided by the Semantic Kernel. - -## The `AuthenticateRequestAsyncCallback` delegate - -[`AuthenticateRequestAsyncCallback`](AuthenticateRequestAsyncCallback.cs) is a delegate type that serves as a callback function for adding authentication information to HTTP requests sent by the OpenAPI skill. - -```csharp -public delegate Task AuthenticateRequestAsyncCallback(HttpRequestMessage request); -``` - -Developers may optionally provide an implementation of this delegate when importing an OpenAPI skill to the Kernel. The delegate is then passed through to the `RestApiOperationRunner`, which is responsible for building the HTTP payload and sending the request for each REST API operation. Before the API request is sent, the delegate is executed with the HTTP request message as the parameter, allowing the request message to be updated with any necessary authentication information. - -This pattern was designed to be flexible enough to support a wide variety of authentication frameworks. Developers can provide the delegate function directly or define a class or interface that exposes one or more implementations. They have the option of writing their own custom implementation or using one of the Semantic Kernel's reference authentication providers as a starting point. - -## Reference Authentication Providers - -### [`BasicAuthenticationProvider`](./BasicAuthenticationProvider.cs) -This class implements the HTTP "basic" authentication scheme. The constructor accepts a `Func` which defines how to retrieve the user's credentials. When the `AuthenticateRequestAsync` method is called, it retrieves the credentials, encodes them as a UTF-8 encoded Base64 string, and adds them to the `HttpRequestMessage`'s authorization header. - -The following code demonstrates how to use this provider: -```csharp -var basicAuthProvider = new BasicAuthenticationProvider(() => -{ - // JIRA API expects credentials in the format "email:apikey" - return Task.FromResult( - Env.Var("MY_EMAIL_ADDRESS") + ":" + Env.Var("JIRA_API_KEY") - ); -}); -var skill = kernel.ImportOpenApiSkillFromResource(SkillResourceNames.Jira, new OpenApiSkillExecutionParameters { AuthCallback = basicAuthProvider.AuthenticateRequestAsync } ); -``` - -### [`BearerAuthenticationProvider`](./BearerAuthenticationProvider.cs) -This class implements the HTTP "bearer" authentication scheme. The constructor accepts a `Func` which defines how to retrieve the bearer token. When the `AuthenticateRequestAsync` method is called, it retrieves the token and adds it to the `HttpRequestMessage`'s authorization header. - -The following code demonstrates how to use this provider: -```csharp -var bearerAuthProvider = new BearerAuthenticationProvider(() => -{ - return Task.FromResult(Env.Var("AZURE_KEYVAULT_TOKEN")); -}); -var skill = kernel.ImportOpenApiSkillFromResource(SkillResourceNames.AzureKeyVault, new OpenApiSkillExecutionParameters { AuthCallback = bearerAuthProvider.AuthenticateRequestAsync } ) -``` - -### [`InteractiveMsalAuthenticationProvider`](./InteractiveMsalAuthenticationProvider.cs) - -This class uses the [Microsoft Authentication Library (MSAL)](https://learn.microsoft.com/en-us/azure/active-directory/develop/msal-overview)'s .NET library to authenticate the user and acquire an OAuth token. It follows the interactive [authorization code flow](https://learn.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-auth-code-flow), requiring the user to sign in with a Microsoft or Azure identity. This is particularly useful for authenticating requests to the Microsoft Graph or Azure APIs. - -Once the token is acquired, it is added to the HTTP authentication header via the `AuthenticateRequestAsync` method, which is inherited from `BearerAuthenticationProvider`. - -To construct this provider, the caller must specify: -- *Client ID* – identifier of the calling application. This is acquired by [registering your application with the Microsoft Identity platform](https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app). -- *Tenant ID* – identifier of the target service tenant, or “common” -- *Scopes* – permissions being requested -- *Redirect URI* – for redirecting the user back to the application. (When running locally, this is typically http://localhost.) - -```csharp -var msalAuthProvider = new InteractiveMsalAuthenticationProvider( - Env.Var("AZURE_KEYVAULT_CLIENTID"), // clientId - Env.Var("AZURE_KEYVAULT_TENANTID"), // tenantId - new string[] { ".default" }, // scopes - new Uri("http://localhost") // redirectUri -); -var skill = kernel.ImportOpenApiSkillFromResource(SkillResourceNames.AzureKeyVault, new OpenApiSkillExecutionParameters { AuthCallback = msalAuthProvider.AuthenticateRequestAsync } ) -``` \ No newline at end of file diff --git a/dotnet/src/Skills/Skills.OpenAPI/Extensions/KernelChatGptPluginExtensions.cs b/dotnet/src/Skills/Skills.OpenAPI/Extensions/KernelChatGptPluginExtensions.cs deleted file mode 100644 index 227c0e9981aa..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Extensions/KernelChatGptPluginExtensions.cs +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Resources; -using System.Text.Json.Nodes; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.OpenAPI.Extensions; -using Microsoft.SemanticKernel.Skills.OpenAPI.Skills; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace -namespace Microsoft.SemanticKernel; -#pragma warning restore IDE0130 - -/// -/// Class for extensions methods for IKernel interface. -/// -public static class KernelChatGptPluginExtensions -{ - /// - /// Imports ChatGPT plugin document from a URL. - /// - /// Semantic Kernel instance. - /// Skill name. - /// Url to in which to retrieve the ChatGPT plugin. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static async Task> ImportChatGptPluginSkillFromUrlAsync( - this IKernel kernel, - string skillName, - Uri url, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - Verify.ValidSkillName(skillName); - -#pragma warning disable CA2000 // Dispose objects before losing scope. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. - var internalHttpClient = HttpClientProvider.GetHttpClient(kernel.Config, executionParameters?.HttpClient, kernel.Logger); -#pragma warning restore CA2000 // Dispose objects before losing scope. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. - - using var requestMessage = new HttpRequestMessage(HttpMethod.Get, url); - - if (!string.IsNullOrEmpty(executionParameters?.UserAgent)) - { - requestMessage.Headers.UserAgent.Add(ProductInfoHeaderValue.Parse(executionParameters!.UserAgent)); - } - - using var response = await internalHttpClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - string gptPluginJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - string? openApiUrl = ParseOpenApiUrl(gptPluginJson); - - return await kernel - .ImportOpenApiSkillFromUrlAsync(skillName, new Uri(openApiUrl), executionParameters, cancellationToken: cancellationToken) - .ConfigureAwait(false); - } - - /// - /// Imports ChatGPT plugin from assembly resource. - /// - /// Semantic Kernel instance. - /// Skill name. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static async Task> ImportChatGptPluginSkillFromResourceAsync( - this IKernel kernel, - string skillName, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - Verify.ValidSkillName(skillName); - - var type = typeof(SkillResourceNames); - - var resourceName = $"{skillName}.ai-plugin.json"; - - var stream = type.Assembly.GetManifestResourceStream(type, resourceName) - ?? throw new MissingManifestResourceException($"Unable to load OpenApi skill from assembly resource '{resourceName}'"); - - using StreamReader reader = new(stream); - string gptPluginJson = await reader.ReadToEndAsync().ConfigureAwait(false); - - string? openApiUrl = ParseOpenApiUrl(gptPluginJson); - - return await kernel - .ImportOpenApiSkillFromUrlAsync(skillName, new Uri(openApiUrl), executionParameters, cancellationToken: cancellationToken) - .ConfigureAwait(false); - } - - /// - /// Imports ChatGPT plugin from a directory. - /// - /// Semantic Kernel instance. - /// Directory containing the skill directory. - /// Name of the directory containing the selected skill. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static async Task> ImportChatGptPluginSkillSkillFromDirectoryAsync( - this IKernel kernel, - string parentDirectory, - string skillDirectoryName, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - const string ChatGptPluginFile = "ai-plugin.json"; - - Verify.ValidSkillName(skillDirectoryName); - - var skillDir = Path.Combine(parentDirectory, skillDirectoryName); - Verify.DirectoryExists(skillDir); - - var chatGptPluginPath = Path.Combine(skillDir, ChatGptPluginFile); - if (!File.Exists(chatGptPluginPath)) - { - throw new FileNotFoundException($"No ChatGPT plugin for the specified path - {chatGptPluginPath} is found"); - } - - kernel.Logger.LogTrace("Registering Rest functions from {0} ChatGPT Plugin", chatGptPluginPath); - - using var stream = File.OpenRead(chatGptPluginPath); - - return await kernel - .RegisterOpenApiSkillAsync(stream, skillDirectoryName, executionParameters, cancellationToken: cancellationToken) - .ConfigureAwait(false); - } - - /// - /// Imports ChatGPT plugin from a file. - /// - /// Semantic Kernel instance. - /// Name of the skill to register. - /// File path to the ChatGPT plugin definition. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static async Task> ImportChatGptPluginSkillSkillFromFileAsync( - this IKernel kernel, - string skillName, - string filePath, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - if (!File.Exists(filePath)) - { - throw new FileNotFoundException($"No ChatGPT plugin for the specified path - {filePath} is found"); - } - - kernel.Logger.LogTrace("Registering Rest functions from {0} ChatGPT Plugin", filePath); - - using var stream = File.OpenRead(filePath); - - return await kernel - .RegisterOpenApiSkillAsync(stream, skillName, executionParameters, cancellationToken: cancellationToken) - .ConfigureAwait(false); - } - - private static string ParseOpenApiUrl(string gptPluginJson) - { - JsonNode? gptPlugin = JsonNode.Parse(gptPluginJson); - - string? apiType = gptPlugin?["api"]?["type"]?.ToString(); - if (string.IsNullOrWhiteSpace(apiType) || apiType != "openapi") - { - throw new InvalidOperationException("Invalid ChatGPT plugin document. Supported api types are: openapi"); - } - - string? openApiUrl = gptPlugin?["api"]?["url"]?.ToString(); - if (string.IsNullOrWhiteSpace(openApiUrl)) - { - throw new InvalidOperationException("Invalid ChatGPT plugin document, OpenAPI URL is missing"); - } - - return openApiUrl!; - } -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Extensions/KernelOpenApiExtensions.cs b/dotnet/src/Skills/Skills.OpenAPI/Extensions/KernelOpenApiExtensions.cs deleted file mode 100644 index bf99544911d4..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Extensions/KernelOpenApiExtensions.cs +++ /dev/null @@ -1,349 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Resources; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.OpenAPI; -using Microsoft.SemanticKernel.Skills.OpenAPI.Extensions; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; -using Microsoft.SemanticKernel.Skills.OpenAPI.OpenApi; -using Microsoft.SemanticKernel.Skills.OpenAPI.Skills; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace -namespace Microsoft.SemanticKernel; -#pragma warning restore IDE0130 - -/// -/// Class for extensions methods for interface. -/// -public static class KernelOpenApiExtensions -{ - /// - /// Imports OpenAPI document from a URL. - /// - /// Semantic Kernel instance. - /// Skill name. - /// Url to in which to retrieve the OpenAPI definition. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static async Task> ImportOpenApiSkillFromUrlAsync( - this IKernel kernel, - string skillName, - Uri url, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - Verify.ValidSkillName(skillName); - -#pragma warning disable CA2000 // Dispose objects before losing scope. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. - var internalHttpClient = HttpClientProvider.GetHttpClient(kernel.Config, executionParameters?.HttpClient, kernel.Logger); -#pragma warning restore CA2000 // Dispose objects before losing scope. No need to dispose the Http client here. It can either be an internal client using NonDisposableHttpClientHandler or an external client managed by the calling code, which should handle its disposal. - - using var requestMessage = new HttpRequestMessage(HttpMethod.Get, url); - - if (!string.IsNullOrEmpty(executionParameters?.UserAgent)) - { - requestMessage.Headers.UserAgent.Add(ProductInfoHeaderValue.Parse(executionParameters!.UserAgent)); - } - - using var response = await internalHttpClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); - if (stream == null) - { - throw new MissingManifestResourceException($"Unable to load OpenApi skill from url '{url}'."); - } - - return await kernel.RegisterOpenApiSkillAsync(stream, skillName, executionParameters, cancellationToken).ConfigureAwait(false); - } - - /// - /// Imports OpenApi document from assembly resource. - /// - /// Semantic Kernel instance. - /// Skill name. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static Task> ImportOpenApiSkillFromResourceAsync( - this IKernel kernel, - string skillName, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - Verify.ValidSkillName(skillName); - - var type = typeof(SkillResourceNames); - - var resourceName = $"{skillName}.openapi.json"; - - var stream = type.Assembly.GetManifestResourceStream(type, resourceName); //TODO: support yaml resources - if (stream == null) - { - throw new MissingManifestResourceException($"Unable to load OpenApi skill from assembly resource '{resourceName}'."); - } - - return kernel.RegisterOpenApiSkillAsync(stream, skillName, executionParameters, cancellationToken); - } - - /// - /// Imports OpenApi document from a directory. - /// - /// Semantic Kernel instance. - /// Directory containing the skill directory. - /// Name of the directory containing the selected skill. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static async Task> ImportOpenApiSkillFromDirectoryAsync( - this IKernel kernel, - string parentDirectory, - string skillDirectoryName, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - const string OpenApiFile = "openapi.json"; - - Verify.ValidSkillName(skillDirectoryName); - - var skillDir = Path.Combine(parentDirectory, skillDirectoryName); - Verify.DirectoryExists(skillDir); - - var openApiDocumentPath = Path.Combine(skillDir, OpenApiFile); - if (!File.Exists(openApiDocumentPath)) - { - throw new FileNotFoundException($"No OpenApi document for the specified path - {openApiDocumentPath} is found."); - } - - kernel.Logger.LogTrace("Registering Rest functions from {0} OpenApi document", openApiDocumentPath); - - var skill = new Dictionary(); - - using var stream = File.OpenRead(openApiDocumentPath); - - return await kernel.RegisterOpenApiSkillAsync(stream, skillDirectoryName, executionParameters, cancellationToken).ConfigureAwait(false); - } - - /// - /// Imports OpenApi document from a file. - /// - /// Semantic Kernel instance. - /// Name of the skill to register. - /// File path to the OpenAPI document. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static async Task> ImportOpenApiSkillFromFileAsync( - this IKernel kernel, - string skillName, - string filePath, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - if (!File.Exists(filePath)) - { - throw new FileNotFoundException($"No OpenApi document for the specified path - {filePath} is found."); - } - - kernel.Logger.LogTrace("Registering Rest functions from {0} OpenApi document", filePath); - - using var stream = File.OpenRead(filePath); - - return await kernel.RegisterOpenApiSkillAsync(stream, skillName, executionParameters, cancellationToken).ConfigureAwait(false); - } - - /// - /// Registers an OpenApi skill. - /// - /// Semantic Kernel instance. - /// OpenApi document stream. - /// Skill name. - /// Skill execution parameters. - /// The cancellation token. - /// A list of all the semantic functions representing the skill. - public static async Task> RegisterOpenApiSkillAsync( - this IKernel kernel, - Stream documentStream, - string skillName, - OpenApiSkillExecutionParameters? executionParameters = null, - CancellationToken cancellationToken = default) - { - Verify.NotNull(kernel); - Verify.ValidSkillName(skillName); - - // Parse - var parser = new OpenApiDocumentParser(kernel.Logger); - - var operations = await parser.ParseAsync(documentStream, executionParameters?.IgnoreNonCompliantErrors ?? false, cancellationToken).ConfigureAwait(false); - - var internalHttpClient = HttpClientProvider.GetHttpClient(kernel.Config, executionParameters?.HttpClient, kernel.Logger); - - var runner = new RestApiOperationRunner(internalHttpClient, executionParameters?.AuthCallback, executionParameters?.UserAgent); - - var skill = new Dictionary(); - - foreach (var operation in operations) - { - try - { - kernel.Logger.LogTrace("Registering Rest function {0}.{1}", skillName, operation.Id); - var function = kernel.RegisterRestApiFunction(skillName, runner, operation, executionParameters?.ServerUrlOverride, cancellationToken); - skill[function.Name] = function; - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - //Logging the exception and keep registering other Rest functions - kernel.Logger.LogWarning(ex, "Something went wrong while rendering the Rest function. Function: {0}.{1}. Error: {2}", - skillName, operation.Id, ex.Message); - } - } - - return skill; - } - - #region private - - /// - /// Registers SKFunction for a REST API operation. - /// - /// Semantic Kernel instance. - /// Skill name. - /// The REST API operation runner. - /// The REST API operation. - /// Optional override for REST API server URL if user input required - /// The cancellation token. - /// An instance of class. - private static ISKFunction RegisterRestApiFunction( - this IKernel kernel, - string skillName, - RestApiOperationRunner runner, - RestApiOperation operation, - Uri? serverUrlOverride = null, - CancellationToken cancellationToken = default) - { - var restOperationParameters = operation.GetParameters(serverUrlOverride); - - var logger = kernel.Logger ?? NullLogger.Instance; - - async Task ExecuteAsync(SKContext context) - { - try - { - // Extract function arguments from context - var arguments = new Dictionary(); - foreach (var parameter in restOperationParameters) - { - // A try to resolve argument by alternative parameter name - if (!string.IsNullOrEmpty(parameter.AlternativeName) && context.Variables.TryGetValue(parameter.AlternativeName!, out string? value)) - { - arguments.Add(parameter.Name, value); - continue; - } - - // A try to resolve argument by original parameter name - if (context.Variables.TryGetValue(parameter.Name, out value)) - { - arguments.Add(parameter.Name, value); - continue; - } - - if (parameter.IsRequired) - { - throw new KeyNotFoundException( - $"No variable found in context to use as an argument for the '{parameter.Name}' parameter of the '{skillName}.{operation.Id}' Rest function."); - } - } - - var result = await runner.RunAsync(operation, arguments, cancellationToken).ConfigureAwait(false); - if (result != null) - { - context.Variables.Update(result.ToString()); - } - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - logger.LogWarning(ex, "Something went wrong while rendering the Rest function. Function: {0}.{1}. Error: {2}", skillName, operation.Id, - ex.Message); - context.Fail(ex.Message, ex); - } - - return context; - } - - var parameters = restOperationParameters - .Select(p => new ParameterView - { - Name = p.AlternativeName ?? p.Name, - Description = $"{p.Description ?? p.Name}{(p.IsRequired ? " (required)" : string.Empty)}", - DefaultValue = p.DefaultValue - }) - .ToList(); - - var function = SKFunction.FromNativeFunction( - nativeFunction: ExecuteAsync, - parameters: parameters, - description: operation.Description, - skillName: skillName, - functionName: ConvertOperationIdToValidFunctionName(operation.Id, logger), - logger: logger); - - return kernel.RegisterCustomFunction(function); - } - - /// - /// Converts operation id to valid SK Function name. - /// A function name can contain only ASCII letters, digits, and underscores. - /// - /// The operation id. - /// The logger. - /// Valid SK Function name. - private static string ConvertOperationIdToValidFunctionName(string operationId, ILogger logger) - { - try - { - Verify.ValidFunctionName(operationId); - return operationId; - } - catch (KernelException) - { - } - - // Tokenize operation id on forward and back slashes - string[] tokens = operationId.Split('/', '\\'); - string result = string.Empty; - - foreach (string token in tokens) - { - // Removes all characters that are not ASCII letters, digits, and underscores. - string formattedToken = s_removeInvalidCharsRegex.Replace(token, ""); - result += CultureInfo.CurrentCulture.TextInfo.ToTitleCase(formattedToken.ToLower(CultureInfo.CurrentCulture)); - } - - logger.LogDebug("Operation name \"{0}\" converted to \"{1}\" to comply with SK Function name requirements. Use \"{2}\" when invoking function.", operationId, result, result); - - return result; - } - - /// - /// Used to convert operationId to SK function names. - /// - private static readonly Regex s_removeInvalidCharsRegex = new("[^0-9A-Za-z_]"); - - #endregion -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Extensions/OpenApiSkillExecutionParameters.cs b/dotnet/src/Skills/Skills.OpenAPI/Extensions/OpenApiSkillExecutionParameters.cs deleted file mode 100644 index c992fdd0e656..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Extensions/OpenApiSkillExecutionParameters.cs +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Net.Http; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; - -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Extensions; - -/// -/// OpenAPI skill execution parameters. -/// -public class OpenApiSkillExecutionParameters -{ - private const string HttpUserAgent = "Microsoft-Semantic-Kernel"; - - /// - /// HttpClient to use for sending HTTP requests. - /// - public HttpClient? HttpClient { get; set; } - - /// - /// Callback for adding authentication data to HTTP requests. - /// - public AuthenticateRequestAsyncCallback? AuthCallback { get; set; } - - /// - /// Override for RESP API operation server url. - /// - public Uri? ServerUrlOverride { get; set; } - - /// - /// Flag indicating whether to ignore non-compliant errors or not. - /// If set to true, the operation execution will not throw exceptions for non-compliant documents. - /// Please note that enabling this option may result in incomplete or inaccurate execution results. - /// - public bool IgnoreNonCompliantErrors { get; set; } - - /// - /// Optional user agent header value. - /// - public string? UserAgent { get; set; } - - /// - /// Initializes a new instance of the class. - /// - /// The HttpClient to use for sending HTTP requests. - /// The callback for adding authentication data to HTTP requests. - /// The override for the RESP API operation server URL. - /// Optional user agent header value. - /// A flag indicating whether to ignore non-compliant errors or not - /// If set to true, the operation execution will not throw exceptions for non-compliant documents. - /// Please note that enabling this option may result in incomplete or inaccurate execution results. - public OpenApiSkillExecutionParameters( - HttpClient? httpClient = null, - AuthenticateRequestAsyncCallback? authCallback = null, - Uri? serverUrlOverride = null, - string? userAgent = HttpUserAgent, - bool ignoreNonCompliantErrors = false) - { - this.HttpClient = httpClient; - this.AuthCallback = authCallback; - this.ServerUrlOverride = serverUrlOverride; - this.UserAgent = userAgent; - this.IgnoreNonCompliantErrors = ignoreNonCompliantErrors; - } -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Extensions/RestApiOperationExtensions.cs b/dotnet/src/Skills/Skills.OpenAPI/Extensions/RestApiOperationExtensions.cs deleted file mode 100644 index cd69782e5e43..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Extensions/RestApiOperationExtensions.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Text.RegularExpressions; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; -#pragma warning restore IDE0130 - -/// -/// Class for extensions methods for the class. -/// -internal static class RestApiOperationExtensions -{ - /// - /// Returns list of REST API operation parameters. - /// - /// The REST API operation. - /// The server URL override. - /// The list of parameters. - public static IReadOnlyList GetParameters(this RestApiOperation operation, Uri? serverUrlOverride = null) - { - var parameters = new List(operation.Parameters) - { - // Register the "server-url" parameter if override is provided - new RestApiOperationParameter( - RestApiOperation.ServerUrlArgumentName, - "string", - false, - RestApiOperationParameterLocation.Path, - RestApiOperationParameterStyle.Simple, - defaultValue: serverUrlOverride?.AbsoluteUri ?? operation.ServerUrl?.AbsoluteUri) - }; - - // Register the "payload" parameter to be advertised for Put and Post operations. - if (operation.Method == HttpMethod.Put || operation.Method == HttpMethod.Post) - { - var type = operation.Payload?.MediaType == MediaTypeTextPlain ? "string" : "object"; - - parameters.Add(new RestApiOperationParameter( - RestApiOperation.PayloadArgumentName, - type, - true, - RestApiOperationParameterLocation.Body, - RestApiOperationParameterStyle.Simple, - description: operation.Payload?.Description ?? "REST API request body.")); - - parameters.Add(new RestApiOperationParameter( - RestApiOperation.ContentTypeArgumentName, - "string", - false, - RestApiOperationParameterLocation.Body, - RestApiOperationParameterStyle.Simple, - description: "Content type of REST API request body.")); - } - - // Create a property alternative name without special symbols that are not supported by SK template language. - foreach (var parameter in parameters) - { - parameter.AlternativeName = s_invalidSymbolsRegex.Replace(parameter.Name, "_"); - } - - return parameters; - } - - private const string MediaTypeTextPlain = "text/plain"; - private static readonly Regex s_invalidSymbolsRegex = new("[^0-9A-Za-z_]+"); -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/JsonPathSkill.cs b/dotnet/src/Skills/Skills.OpenAPI/JsonPathSkill.cs deleted file mode 100644 index 43a1dea42162..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/JsonPathSkill.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ComponentModel; -using System.Linq; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Microsoft.SemanticKernel.Skills.OpenAPI; - -public sealed class JsonPathSkill -{ - /// - /// parameter names. - /// - public static class Parameters - { - /// - /// JSON path. - /// - public const string JsonPath = "jsonpath"; - } - - /// - /// Retrieve the value of a JSON element from a JSON string using a JsonPath query. - /// - [SKFunction, Description("Retrieve the value of a JSON element from a JSON string using a JsonPath query.")] - public string GetJsonElementValue( - [Description("JSON string")] string json, - [Description("JSON path query.")] string jsonPath) - { - if (string.IsNullOrWhiteSpace(json)) - { - throw new ArgumentException("Variable was null or whitespace", nameof(json)); - } - - JObject jsonObject = JObject.Parse(json); - - JToken? token = jsonObject.SelectToken(jsonPath); - - return token?.Value() ?? string.Empty; - } - - /// - /// Retrieve a collection of JSON elements from a JSON string using a JsonPath query. - /// - [SKFunction, Description("Retrieve a collection of JSON elements from a JSON string using a JsonPath query.")] - public string GetJsonElements( - [Description("JSON string")] string json, - [Description("JSON path query.")] string jsonPath) - { - if (string.IsNullOrWhiteSpace(json)) - { - throw new ArgumentException("Variable was null or whitespace", nameof(json)); - } - - JObject jsonObject = JObject.Parse(json); - - JToken[] tokens = jsonObject.SelectTokens(jsonPath).ToArray(); - - return JsonConvert.SerializeObject(tokens, Formatting.None); - } -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperation.cs b/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperation.cs deleted file mode 100644 index 63f40b13e466..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperation.cs +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text.RegularExpressions; -using System.Web; - -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; - -/// -/// The REST API operation. -/// -public sealed class RestApiOperation -{ - /// - /// An artificial parameter that is added to be able to override RESP API operation server url. - /// - public const string ServerUrlArgumentName = "server-url"; - - /// - /// An artificial parameter to be used for operation having "text/plain" payload media type. - /// - public const string PayloadArgumentName = "payload"; - - /// - /// An artificial parameter to be used for indicate payload media-type if it's missing in payload metadata. - /// - public const string ContentTypeArgumentName = "content-type"; - - /// - /// The operation identifier. - /// - public string Id { get; } - - /// - /// The operation description. - /// - public string Description { get; } - - /// - /// The operation path. - /// - public string Path { get; } - - /// - /// The operation method - GET, POST, PUT, DELETE. - /// - public HttpMethod Method { get; } - - /// - /// The server URL. - /// - public Uri? ServerUrl { get; } - - /// - /// The operation headers. - /// - public IDictionary Headers { get; } - - /// - /// The operation parameters. - /// - public IList Parameters { get; } - - /// - /// The operation payload. - /// - public RestApiOperationPayload? Payload { get; } - - /// - /// Creates an instance of a class. - /// - /// The operation identifier. - /// The server URL. - /// The operation path. - /// The operation method. - /// The operation description. - /// The operation parameters. - /// The operation headers. - /// The operation payload. - public RestApiOperation( - string id, - Uri? serverUrl, - string path, - HttpMethod method, - string description, - IList parameters, - IDictionary headers, - RestApiOperationPayload? payload = null) - { - this.Id = id; - this.ServerUrl = serverUrl; - this.Path = path; - this.Method = method; - this.Description = description; - this.Parameters = parameters; - this.Headers = headers; - this.Payload = payload; - } - - /// - /// Builds operation Url. - /// - /// The operation arguments. - /// The operation Url. - public Uri BuildOperationUrl(IDictionary arguments) - { - var path = this.ReplacePathParameters(this.Path, arguments); - - path = this.AddQueryString(path, arguments); - - Uri serverUrl; - - //Override defined server url - https://api.example.com/v1 by the one from arguments. - if (arguments.TryGetValue(ServerUrlArgumentName, out string serverUrlString)) - { - serverUrl = new Uri(serverUrlString); - } - else - { - serverUrl = this.ServerUrl ?? throw new InvalidOperationException($"Server url is not defined for operation {this.Id}"); - } - - // make sure base url ends with trailing slash - if (!serverUrl.AbsoluteUri.EndsWith("/", StringComparison.OrdinalIgnoreCase)) - { - serverUrl = new Uri(serverUrl.AbsoluteUri + "/"); - } - - return new Uri(serverUrl, $"{path.TrimStart('/')}"); - } - - /// - /// Renders operation request headers. - /// - /// The operation arguments. - /// The rendered request headers. - public IDictionary RenderHeaders(IDictionary arguments) - { - var headers = new Dictionary(); - - foreach (var header in this.Headers) - { - var headerName = header.Key; - var headerValue = header.Value; - - //A try to resolve header value in arguments. - if (arguments.TryGetValue(headerName, out var value)) - { - headers.Add(headerName, value); - continue; - } - - //Header value is already supplied. - if (!string.IsNullOrEmpty(headerValue)) - { - headers.Add(headerName, headerValue); - continue; - } - - //Getting metadata for the header - var headerMetadata = this.Parameters.FirstOrDefault(p => p.Location == RestApiOperationParameterLocation.Header && p.Name == headerName) - ?? throw new RestApiOperationException($"No value for the '{headerName} header is found.'"); - - //If parameter is required it's value should always be provided. - if (headerMetadata.IsRequired) - { - throw new RestApiOperationException($"No value for the '{headerName} header is found.'"); - } - - //Parameter is not required and no default value provided. - if (string.IsNullOrEmpty(headerMetadata.DefaultValue)) - { - continue; - } - - //Using default value. - headers.Add(headerName, headerMetadata.DefaultValue!); - } - - return headers; - } - - #region private - - /// - /// Replaces path parameters by corresponding arguments. - /// - /// Operation path to replace parameters in. - /// Arguments to replace parameters by. - /// Path with replaced parameters - private string ReplacePathParameters(string path, IDictionary arguments) - { - string ReplaceParameter(Match match) - { - var parameterName = match.Groups[1].Value; - - //A try to find parameter value in arguments - if (arguments.TryGetValue(parameterName, out var value)) - { - return value; - } - - //A try to find default value for the parameter - var parameterMetadata = this.Parameters.First(p => p.Location == RestApiOperationParameterLocation.Path && p.Name == parameterName); - if (parameterMetadata?.DefaultValue == null) - { - throw new RestApiOperationException($"No argument found for parameter - '{parameterName}' for operation - '{this.Id}'"); - } - - return parameterMetadata.DefaultValue; - } - - return s_urlParameterMatch.Replace(path, ReplaceParameter); - } - - /// - /// Adds query string to the operation path. - /// - /// The operation path. - /// The operation arguments. - /// Path with query string. - private string AddQueryString(string path, IDictionary arguments) - { - var queryStringSegments = new List(); - - var queryStringParameters = this.Parameters.Where(p => p.Location == RestApiOperationParameterLocation.Query); - - foreach (var parameter in queryStringParameters) - { - //Resolve argument for the parameter. - if (!arguments.TryGetValue(parameter.Name, out var argument)) - { - argument = parameter.DefaultValue; - } - - //Add the parameter to the query string if there's an argument for it. - if (!string.IsNullOrEmpty(argument)) - { - queryStringSegments.Add($"{parameter.Name}={HttpUtility.UrlEncode(argument)}"); - continue; - } - - //Throw an exception if the parameter is a required one but no value is provided. - if (parameter.IsRequired) - { - throw new RestApiOperationException($"No argument found for required query string parameter - '{parameter.Name}' for operation - '{this.Id}'"); - } - } - - var queryString = string.Join("&", queryStringSegments); - - return string.IsNullOrEmpty(queryString) ? path : $"{path}?{queryString}"; - } - - private static readonly Regex s_urlParameterMatch = new(@"\{([\w-]+)\}"); - - # endregion -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationException.cs b/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationException.cs deleted file mode 100644 index c95b09a40a00..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationException.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; - -/// -/// Exception to be throw if a REST API operation has failed. E.g. mandatory property is missing or empty, value is out of range -/// -public class RestApiOperationException : Exception -{ - /// - /// Creates an instance of a class. - /// - /// The exception message. - internal RestApiOperationException(string message) : base(message) - { - } - - /// - /// Creates an instance of a class. - /// - /// The exception message. - /// The inner exception. - internal RestApiOperationException(string message, Exception innerException) : base(message, innerException) - { - } - - /// - /// Creates an instance of a class. - /// - internal RestApiOperationException() - { - } -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationResponse.cs b/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationResponse.cs deleted file mode 100644 index 58986b1e0e90..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Model/RestApiOperationResponse.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; - -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Model; - -/// -/// The REST API operation response. -/// -public sealed class RestApiOperationResponse -{ - /// - /// Gets the content of the response. - /// - [JsonPropertyName("content")] - public string Content { get; } - - /// - /// Gets the content type of the response. - /// - [JsonPropertyName("contentType")] - public string ContentType { get; } - - /// - /// Initializes a new instance of the class. - /// - /// The content of the response. - /// The content type of the response. - public RestApiOperationResponse(string content, string contentType) - { - this.Content = content; - this.ContentType = contentType; - } -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/OpenApi/OpenApiDocumentParsingException.cs b/dotnet/src/Skills/Skills.OpenAPI/OpenApi/OpenApiDocumentParsingException.cs deleted file mode 100644 index 18e9ad8d4e0c..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/OpenApi/OpenApiDocumentParsingException.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.SemanticKernel.Skills.OpenAPI.OpenApi; - -/// -/// Exception to be throw in case parsing of OpenApi document failed. E.g. mandatory property is missing or empty, value is out of range -/// -public class OpenApiDocumentParsingException : Exception -{ - /// - /// Creates an instance of a class. - /// - /// The exception message. - public OpenApiDocumentParsingException(string message) : base(message) - { - } - - /// - /// Creates an instance of a class. - /// - /// The exception message. - /// The inner exception. - public OpenApiDocumentParsingException(string message, Exception innerException) : base(message, innerException) - { - } - - /// - /// Creates an instance of a class. - /// - public OpenApiDocumentParsingException() - { - } -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/RestApiOperationRunner.cs b/dotnet/src/Skills/Skills.OpenAPI/RestApiOperationRunner.cs deleted file mode 100644 index 98e7ce9ffac3..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/RestApiOperationRunner.cs +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; - -namespace Microsoft.SemanticKernel.Skills.OpenAPI; - -/// -/// Runs REST API operation represented by RestApiOperation model class. -/// -internal sealed class RestApiOperationRunner -{ - private const string MediaTypeApplicationJson = "application/json"; - private const string MediaTypeTextPlain = "text/plain"; - - /// - /// An instance of the HttpClient class. - /// - private readonly HttpClient _httpClient; - - /// - /// Delegate for authorizing the HTTP request. - /// - private readonly AuthenticateRequestAsyncCallback _authCallback; - - /// - /// Request-header field containing information about the user agent originating the request - /// - private readonly string? _userAgent; - - /// - /// Creates an instance of a class. - /// - /// An instance of the HttpClient class. - /// Optional callback for adding auth data to the API requests. - /// Optional request-header field containing information about the user agent originating the request - public RestApiOperationRunner(HttpClient httpClient, AuthenticateRequestAsyncCallback? authCallback = null, string? userAgent = null) - { - this._httpClient = httpClient; - this._userAgent = userAgent; - - // If no auth callback provided, use empty function - if (authCallback == null) - { - this._authCallback = _ => Task.CompletedTask; - } - else - { - this._authCallback = authCallback; - } - } - - public Task RunAsync(RestApiOperation operation, IDictionary arguments, CancellationToken cancellationToken = default) - { - var url = operation.BuildOperationUrl(arguments); - - var headers = operation.RenderHeaders(arguments); - - var payload = BuildOperationPayload(operation, arguments); - - return this.SendAsync(url, operation.Method, headers, payload, cancellationToken); - } - - #region private - - /// - /// Sends an HTTP request. - /// - /// The url to send request to. - /// The HTTP request method. - /// Headers to include into the HTTP request. - /// HTTP request payload. - /// The cancellation token. - /// Response content and content type - private async Task SendAsync( - Uri url, - HttpMethod method, - IDictionary? headers = null, - HttpContent? payload = null, - CancellationToken cancellationToken = default) - { - using var requestMessage = new HttpRequestMessage(method, url); - - await this._authCallback(requestMessage).ConfigureAwait(false); - - if (payload != null) - { - requestMessage.Content = payload; - } - - if (!string.IsNullOrWhiteSpace(this._userAgent)) - { - requestMessage.Headers.Add("User-Agent", this._userAgent); - } - - if (headers != null) - { - foreach (var header in headers) - { - requestMessage.Headers.Add(header.Key, header.Value); - } - } - - using var responseMessage = await this._httpClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false); - - responseMessage.EnsureSuccessStatusCode(); - - var content = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false); - - // First iteration allowing to associate additional metadata with the returned content. - var result = new RestApiOperationResponse( - content, - responseMessage.Content.Headers.ContentType.ToString()); - - return JsonSerializer.SerializeToNode(result); - } - - /// - /// Builds operation payload. - /// - /// The operation. - /// The payload arguments. - /// The HttpContent representing the payload. - private static HttpContent? BuildOperationPayload(RestApiOperation operation, IDictionary arguments) - { - if (operation?.Method != HttpMethod.Put && operation?.Method != HttpMethod.Post) - { - return null; - } - - var mediaType = operation.Payload?.MediaType; - - // A try to resolve payload content type from the operation arguments if it's missing in the payload metadata. - if (string.IsNullOrEmpty(mediaType)) - { - if (!arguments.TryGetValue(RestApiOperation.ContentTypeArgumentName, out mediaType)) - { - throw new RestApiOperationException($"No content type is provided for the {operation.Id} operation."); - } - } - - if (!s_payloadFactoryByMediaType.TryGetValue(mediaType!, out var payloadFactory)) - { - throw new RestApiOperationException($"The media type {mediaType} of the {operation.Id} operation is not supported by {nameof(RestApiOperationRunner)}."); - } - - return payloadFactory.Invoke(arguments); - } - - /// - /// Builds "application/json" payload. - /// - /// The payload arguments. - /// The HttpContent representing the payload. - private static HttpContent BuildAppJsonPayload(IDictionary arguments) - { - if (!arguments.TryGetValue(RestApiOperation.PayloadArgumentName, out var content)) - { - throw new RestApiOperationException($"No argument is found for the '{RestApiOperation.PayloadArgumentName}' payload content."); - } - - return new StringContent(content, Encoding.UTF8, MediaTypeApplicationJson); - } - - /// - /// Builds "text/plain" payload. - /// - /// The payload arguments. - /// The HttpContent representing the payload. - private static HttpContent BuildPlainTextPayload(IDictionary arguments) - { - if (!arguments.TryGetValue(RestApiOperation.PayloadArgumentName, out var propertyValue)) - { - throw new RestApiOperationException($"No argument is found for the '{RestApiOperation.PayloadArgumentName}' payload content."); - } - - return new StringContent(propertyValue, Encoding.UTF8, MediaTypeTextPlain); - } - - /// - /// List of payload builders/factories. - /// - private static readonly Dictionary, HttpContent>> s_payloadFactoryByMediaType = - new() - { - { MediaTypeApplicationJson, BuildAppJsonPayload }, - { MediaTypeTextPlain, BuildPlainTextPayload } - }; - - #endregion -} diff --git a/dotnet/src/Skills/Skills.OpenAPI/Skills.OpenAPI.csproj b/dotnet/src/Skills/Skills.OpenAPI/Skills.OpenAPI.csproj deleted file mode 100644 index 28d54517963b..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Skills.OpenAPI.csproj +++ /dev/null @@ -1,40 +0,0 @@ - - - - - Microsoft.SemanticKernel.Skills.OpenAPI - $(AssemblyName) - netstandard2.0 - - - - - - - - Semantic Kernel - OpenAPI Skills - Semantic Kernel OpenAPI Skill - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dotnet/src/Skills/Skills.OpenAPI/Skills/SkillResourceNames.cs b/dotnet/src/Skills/Skills.OpenAPI/Skills/SkillResourceNames.cs deleted file mode 100644 index 3bfb22d145fb..000000000000 --- a/dotnet/src/Skills/Skills.OpenAPI/Skills/SkillResourceNames.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.SemanticKernel.Skills.OpenAPI.Skills; - -/// -/// Skill resource names. -/// -public static class SkillResourceNames -{ - /// - /// Azure KeyVault skill name. - /// - public const string AzureKeyVault = "AzureKeyVaultSkill"; -} diff --git a/dotnet/src/Skills/Skills.UnitTests/.editorconfig b/dotnet/src/Skills/Skills.UnitTests/.editorconfig deleted file mode 100644 index 8f4c52fa9f51..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/.editorconfig +++ /dev/null @@ -1,5 +0,0 @@ -# Suppressing errors for Test projects under dotnet folder -[*.cs] -dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task -dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave - diff --git a/dotnet/src/Skills/Skills.UnitTests/Connectors/WebApi/Rest/RestApiOperationRunnerTests.cs b/dotnet/src/Skills/Skills.UnitTests/Connectors/WebApi/Rest/RestApiOperationRunnerTests.cs deleted file mode 100644 index ea7053489d16..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Connectors/WebApi/Rest/RestApiOperationRunnerTests.cs +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Net.Mime; -using System.Text; -using System.Text.Json.Nodes; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.OpenAPI; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; -using Moq; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Connectors.WebApi.Rest; - -public sealed class RestApiOperationRunnerTests : IDisposable -{ - /// - /// A mock instance of the authentication callback. - /// - private readonly Mock _authenticationHandlerMock; - - /// - /// An instance of HttpMessageHandlerStub class used to get access to various properties of HttpRequestMessage sent by HTTP client. - /// - private readonly HttpMessageHandlerStub _httpMessageHandlerStub; - - /// - /// An instance of HttpClient class used by the tests. - /// - private readonly HttpClient _httpClient; - - /// - /// Creates an instance of a class. - /// - public RestApiOperationRunnerTests() - { - this._authenticationHandlerMock = new Mock(); - - this._httpMessageHandlerStub = new HttpMessageHandlerStub(); - - this._httpClient = new HttpClient(this._httpMessageHandlerStub); - } - - [Fact] - public async Task ItCanRunCreateAndUpdateOperationsWithJsonPayloadSuccessfullyAsync() - { - // Arrange - this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); - - var payloadMetadata = new RestApiOperationPayload(MediaTypeNames.Application.Json, new List()); - - var operation = new RestApiOperation( - "fake-id", - new Uri("https://fake-random-test-host"), - "fake-path", - HttpMethod.Post, - "fake-description", - new List(), - new Dictionary(), - payloadMetadata - ); - - var payload = new - { - value = "fake-value", - attributes = new - { - enabled = true - } - }; - - var arguments = new Dictionary - { - { "payload", System.Text.Json.JsonSerializer.Serialize(payload) } - }; - - var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); - - // Act - var result = await sut.RunAsync(operation, arguments); - - // Assert - Assert.NotNull(this._httpMessageHandlerStub.RequestUri); - Assert.Equal("https://fake-random-test-host/fake-path", this._httpMessageHandlerStub.RequestUri.AbsoluteUri); - - Assert.Equal(HttpMethod.Post, this._httpMessageHandlerStub.Method); - - Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); - Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("application/json; charset=utf-8")); - - var messageContent = this._httpMessageHandlerStub.RequestContent; - Assert.NotNull(messageContent); - Assert.True(messageContent.Length != 0); - - var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent)); - Assert.NotNull(deserializedPayload); - - var valueProperty = deserializedPayload["value"]?.ToString(); - Assert.Equal("fake-value", valueProperty); - - var attributesProperty = deserializedPayload["attributes"]; - Assert.NotNull(attributesProperty); - - var enabledProperty = attributesProperty["enabled"]?.AsValue(); - Assert.NotNull(enabledProperty); - Assert.Equal("true", enabledProperty.ToString()); - - Assert.NotNull(result); - - var contentProperty = result["content"]?.ToString(); - Assert.Equal("fake-content", contentProperty); - - var contentTypeProperty = result["contentType"]?.ToString(); - Assert.Equal("application/json; charset=utf-8", contentTypeProperty); - - this._authenticationHandlerMock.Verify(x => x(It.IsAny()), Times.Once); - } - - [Fact] - public async Task ItCanRunCreateAndUpdateOperationsWithPlainTextPayloadSuccessfullyAsync() - { - // Arrange - this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Text.Plain); - - var payload = new RestApiOperationPayload(MediaTypeNames.Text.Plain, new List(), "fake-description"); - - var operation = new RestApiOperation( - "fake-id", - new Uri("https://fake-random-test-host"), - "fake-path", - HttpMethod.Post, - "fake-description", - new List(), - new Dictionary(), - payload - ); - - var arguments = new Dictionary - { - { "payload", "fake-input-value" } - }; - - var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); - - // Act - var result = await sut.RunAsync(operation, arguments); - - // Assert - Assert.NotNull(this._httpMessageHandlerStub.RequestUri); - Assert.Equal("https://fake-random-test-host/fake-path", this._httpMessageHandlerStub.RequestUri.AbsoluteUri); - - Assert.Equal(HttpMethod.Post, this._httpMessageHandlerStub.Method); - - Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); - Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("text/plain; charset=utf-8")); - - var messageContent = this._httpMessageHandlerStub.RequestContent; - Assert.NotNull(messageContent); - Assert.True(messageContent.Length != 0); - - var payloadText = System.Text.Encoding.UTF8.GetString(messageContent, 0, messageContent.Length); - Assert.Equal("fake-input-value", payloadText); - - Assert.NotNull(result); - - var contentProperty = result["content"]?.ToString(); - Assert.Equal("fake-content", contentProperty); - - var contentTypeProperty = result["contentType"]?.ToString(); - Assert.Equal("text/plain; charset=utf-8", contentTypeProperty); - - this._authenticationHandlerMock.Verify(x => x(It.IsAny()), Times.Once); - } - - [Fact] - public async Task ItShouldAddHeadersToHttpRequestAsync() - { - // Arrange - var headers = new Dictionary - { - { "fake-header", string.Empty } - }; - - var operation = new RestApiOperation( - "fake-id", - new Uri("https://fake-random-test-host"), - "fake-path", - HttpMethod.Get, - "fake-description", - new List(), - headers - ); - - var arguments = new Dictionary - { - { "fake-header", "fake-header-value" } - }; - - var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); - - // Act - await sut.RunAsync(operation, arguments); - - // Assert - Assert.NotNull(this._httpMessageHandlerStub.RequestHeaders); - Assert.Single(this._httpMessageHandlerStub.RequestHeaders); - - Assert.Contains(this._httpMessageHandlerStub.RequestHeaders, h => h.Key == "fake-header" && h.Value.Contains("fake-header-value")); - } - - [Fact] - public async Task ItShouldAddUserAgentHeaderToHttpRequestIfConfiguredAsync() - { - // Arrange - var headers = new Dictionary - { - { "fake-header", string.Empty } - }; - - var operation = new RestApiOperation( - "fake-id", - new Uri("https://fake-random-test-host"), - "fake-path", - HttpMethod.Get, - "fake-description", - new List(), - headers - ); - - var arguments = new Dictionary - { - { "fake-header", "fake-header-value" } - }; - - var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object, "fake-user-agent"); - - // Act - await sut.RunAsync(operation, arguments); - - // Assert - Assert.NotNull(this._httpMessageHandlerStub.RequestHeaders); - Assert.Equal(2, this._httpMessageHandlerStub.RequestHeaders.Count()); - - Assert.Contains(this._httpMessageHandlerStub.RequestHeaders, h => h.Key == "fake-header" && h.Value.Contains("fake-header-value")); - Assert.Contains(this._httpMessageHandlerStub.RequestHeaders, h => h.Key == "User-Agent" && h.Value.Contains("fake-user-agent")); - } - - [Fact] - public async Task ItShouldUsePayloadAndContentTypeArgumentsIfPayloadMetadataIsMissingAsync() - { - // Arrange - this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json); - - var operation = new RestApiOperation( - "fake-id", - new Uri("https://fake-random-test-host"), - "fake-path", - HttpMethod.Post, - "fake-description", - new List(), - new Dictionary() - ); - - var payload = new - { - value = "fake-value", - attributes = new - { - enabled = true - } - }; - - var arguments = new Dictionary - { - { "payload", System.Text.Json.JsonSerializer.Serialize(payload) }, - { "content-type", "application/json" } - }; - - var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object); - - // Act - var result = await sut.RunAsync(operation, arguments); - - // Assert - Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders); - Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("application/json; charset=utf-8")); - - var messageContent = this._httpMessageHandlerStub.RequestContent; - Assert.NotNull(messageContent); - Assert.True(messageContent.Length != 0); - - var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent)); - Assert.NotNull(deserializedPayload); - - var valueProperty = deserializedPayload["value"]?.ToString(); - Assert.Equal("fake-value", valueProperty); - - var attributesProperty = deserializedPayload["attributes"]; - Assert.NotNull(attributesProperty); - - var enabledProperty = attributesProperty["enabled"]?.AsValue(); - Assert.NotNull(enabledProperty); - Assert.Equal("true", enabledProperty.ToString()); - } - - /// - /// Disposes resources used by this class. - /// - public void Dispose() - { - this._httpMessageHandlerStub.Dispose(); - - this._httpClient.Dispose(); - } - - private sealed class HttpMessageHandlerStub : DelegatingHandler - { - public HttpRequestHeaders? RequestHeaders { get; private set; } - - public HttpContentHeaders? ContentHeaders { get; private set; } - - public byte[]? RequestContent { get; private set; } - - public Uri? RequestUri { get; private set; } - - public HttpMethod? Method { get; private set; } - - public HttpResponseMessage ResponseToReturn { get; set; } - - public HttpMessageHandlerStub() - { - this.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent("{}", Encoding.UTF8, MediaTypeNames.Application.Json) - }; - } - - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - this.Method = request.Method; - this.RequestUri = request.RequestUri; - this.RequestHeaders = request.Headers; - this.RequestContent = request.Content == null ? null : await request.Content.ReadAsByteArrayAsync(cancellationToken); - this.ContentHeaders = request.Content?.Headers; - - return await Task.FromResult(this.ResponseToReturn); - } - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Connectors/WebApi/Rest/RestApiOperationTests.cs b/dotnet/src/Skills/Skills.UnitTests/Connectors/WebApi/Rest/RestApiOperationTests.cs deleted file mode 100644 index f043f20f15ae..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Connectors/WebApi/Rest/RestApiOperationTests.cs +++ /dev/null @@ -1,530 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Web; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Connectors.WebApi.Rest; - -public class RestApiOperationTests -{ - [Fact] - public void ShouldUseHostUrlIfNoOverrideProvided() - { - // Arrange - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "/", - HttpMethod.Get, - "fake_description", - new List(), - new Dictionary() - ); - - var arguments = new Dictionary(); - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.Equal("https://fake-random-test-host/", url.OriginalString); - } - - [Fact] - public void ShouldUseHostUrlOverrideIfProvided() - { - // Arrange - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "/", - HttpMethod.Get, - "fake_description", - new List(), - new Dictionary() - ); - - var arguments = new Dictionary - { - { "server-url", "https://fake-random-test-host-override" } - }; - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.Equal("https://fake-random-test-host-override/", url.OriginalString); - } - - [Fact] - public void ShouldReplacePathParametersByValuesFromArguments() - { - // Arrange - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "/{fake-path-parameter}/other_fake_path_section", - HttpMethod.Get, - "fake_description", - new List(), - new Dictionary() - ); - - var arguments = new Dictionary - { - { "fake-path-parameter", "fake-path-value" } - }; - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.Equal("https://fake-random-test-host/fake-path-value/other_fake_path_section", url.OriginalString); - } - - [Fact] - public void ShouldReplacePathParametersByDefaultValues() - { - // Arrange - var parameterMetadata = new RestApiOperationParameter( - "fake-path-parameter", - "fake_type", - true, - RestApiOperationParameterLocation.Path, - defaultValue: "fake-default-path"); - - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "/{fake-path-parameter}/other_fake_path_section", - HttpMethod.Get, - "fake_description", - new List { parameterMetadata }, - new Dictionary()); - - var arguments = new Dictionary(); - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.Equal("https://fake-random-test-host/fake-default-path/other_fake_path_section", url.OriginalString); - } - - [Fact] - public void ShouldAddQueryStringParametersAndUseValuesFromArguments() - { - // Arrange - var firstParameterMetadata = new RestApiOperationParameter( - "p1", - "fake_type", - true, - RestApiOperationParameterLocation.Query); - - var secondParameterMetadata = new RestApiOperationParameter( - "p2", - "fake_type", - true, - RestApiOperationParameterLocation.Query); - - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "/", - HttpMethod.Get, - "fake_description", - new List { firstParameterMetadata, secondParameterMetadata }, - new Dictionary()); - - var arguments = new Dictionary - { - { "p1", "v1" }, - { "p2", "v2" } - }; - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.Equal("https://fake-random-test-host/?p1=v1&p2=v2", url.OriginalString); - } - - [Fact] - public void ShouldAddQueryStringParametersAndUseTheirDefaultValues() - { - // Arrange - var firstParameterMetadata = new RestApiOperationParameter( - "p1", - "fake_type", - true, - RestApiOperationParameterLocation.Query, - defaultValue: "dv1"); - - var secondParameterMetadata = new RestApiOperationParameter( - "p2", - "fake_type", - true, - RestApiOperationParameterLocation.Query, - defaultValue: "dv2"); - - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "/", - HttpMethod.Get, - "fake_description", - new List { firstParameterMetadata, secondParameterMetadata }, - new Dictionary()); - - var arguments = new Dictionary(); - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.Equal("https://fake-random-test-host/?p1=dv1&p2=dv2", url.OriginalString); - } - - [Fact] - public void ShouldSkipNotRequiredQueryStringParametersIfTheirValuesMissing() - { - // Arrange - var firstParameterMetadata = new RestApiOperationParameter( - "p1", - "fake_type", - false, - RestApiOperationParameterLocation.Query); - - var secondParameterMetadata = new RestApiOperationParameter( - "p2", - "fake_type", - false, - RestApiOperationParameterLocation.Query); - - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "/", - HttpMethod.Get, - "fake_description", - new List { firstParameterMetadata, secondParameterMetadata }, - new Dictionary()); - - var arguments = new Dictionary - { - { "p2", "v2" } - }; - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.Equal("https://fake-random-test-host/?p2=v2", url.OriginalString); - } - - [Fact] - public void ShouldThrowExceptionIfNoValueIsProvideForRequiredQueryStringParameter() - { - // Arrange - var firstParameterMetadata = new RestApiOperationParameter( - "p1", - "fake_type", - true, - RestApiOperationParameterLocation.Query); - - var secondParameterMetadata = new RestApiOperationParameter( - "p2", - "fake_type", - false, - RestApiOperationParameterLocation.Query); - - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "/", - HttpMethod.Get, - "fake_description", - new List { firstParameterMetadata, secondParameterMetadata }, - new Dictionary()); - - var arguments = new Dictionary - { - { "p2", "v2" } - }; - - //Act and assert - Assert.Throws(() => sut.BuildOperationUrl(arguments)); - } - - [Theory] - [InlineData(":", "%3a")] - [InlineData("/", "%2f")] - [InlineData("?", "%3f")] - [InlineData("#", "%23")] - public void ItShouldEncodeSpecialSymbolsInQueryStringValues(string specialSymbol, string encodedEquivalent) - { - // Arrange - var metadata = new List - { - new RestApiOperationParameter("fake_query_param", "string", false, RestApiOperationParameterLocation.Query, RestApiOperationParameterStyle.Simple) - }; - - var arguments = new Dictionary - { - { "fake_query_param", $"fake_query_param_value{specialSymbol}" } - }; - - var sut = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.NotNull(url); - - Assert.EndsWith(encodedEquivalent, url.OriginalString, StringComparison.Ordinal); - } - - [Fact] - public void ShouldBuildResourceUrlThatIncludesAllUrlComponents() - { - // Arrange - var firstParameterMetadata = new RestApiOperationParameter( - "p1", - "fake_type", - false, - RestApiOperationParameterLocation.Query, - defaultValue: "dv1"); - - var secondParameterMetadata = new RestApiOperationParameter( - "p2", - "fake_type", - false, - RestApiOperationParameterLocation.Query); - - var sut = new RestApiOperation( - "fake_id", - new Uri("https://fake-random-test-host"), - "{fake-path}/", - HttpMethod.Get, - "fake_description", - new List { firstParameterMetadata, secondParameterMetadata }, - new Dictionary()); - - var arguments = new Dictionary - { - { "server-url", "https://fake-random-test-host-override" }, - { "fake-path", "fake-path-value" }, - { "p2", "v2" } - }; - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.Equal("https://fake-random-test-host-override/fake-path-value/?p1=dv1&p2=v2", url.OriginalString); - } - - [Fact] - public void ItShouldRenderHeaderValuesFromArguments() - { - // Arrange - var rawHeaders = new Dictionary - { - { "fake_header_one", string.Empty }, - { "fake_header_two", string.Empty } - }; - - var arguments = new Dictionary - { - { "fake_header_one", "fake_header_one_value" }, - { "fake_header_two", "fake_header_two_value" } - }; - - var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", new List(), rawHeaders); - - // Act - var headers = sut.RenderHeaders(arguments); - - // Assert - Assert.Equal(2, headers.Count); - - var headerOne = headers["fake_header_one"]; - Assert.Equal("fake_header_one_value", headerOne); - - var headerTwo = headers["fake_header_two"]; - Assert.Equal("fake_header_two_value", headerTwo); - } - - [Fact] - public void ItShouldUseHeaderValuesIfTheyAreAlreadyProvided() - { - // Arrange - var rawHeaders = new Dictionary - { - { "fake_header_one", "fake_header_one_value" }, - { "fake_header_two", "fake_header_two_value" } - }; - - var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", new List(), - rawHeaders); - - // Act - var headers = sut.RenderHeaders(new Dictionary()); - - // Assert - Assert.Equal(2, headers.Count); - - var headerOne = headers["fake_header_one"]; - Assert.Equal("fake_header_one_value", headerOne); - - var headerTwo = headers["fake_header_two"]; - Assert.Equal("fake_header_two_value", headerTwo); - } - - [Fact] - public void ItShouldThrowExceptionIfHeadersHaveNoValuesAndHeadersMetadataNotSupplied() - { - // Arrange - var rawHeaders = new Dictionary - { - { "fake_header_one", string.Empty }, - { "fake_header_two", string.Empty } - }; - - var metadata = new List(); - - var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", metadata, rawHeaders); - - // Act - void Act() => sut.RenderHeaders(new Dictionary()); - - // Assert - Assert.Throws(Act); - } - - [Fact] - public void ShouldThrowExceptionIfNoValueProvidedForRequiredHeader() - { - // Arrange - var rawHeaders = new Dictionary - { - { "fake_header_one", string.Empty }, - { "fake_header_two", string.Empty } - }; - - var metadata = new List - { - new RestApiOperationParameter("fake_header_one", "string", true, RestApiOperationParameterLocation.Header, RestApiOperationParameterStyle.Simple), - new RestApiOperationParameter("fake_header_two", "string", false, RestApiOperationParameterLocation.Header, RestApiOperationParameterStyle.Simple) - }; - - var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", metadata, rawHeaders); - - // Act - void Act() => sut.RenderHeaders(new Dictionary()); - - // Assert - Assert.Throws(Act); - } - - [Fact] - public void ItShouldSkipOptionalHeaderHavingNeitherValueNorDefaultValue() - { - // Arrange - var rawHeaders = new Dictionary - { - { "fake_header_one", string.Empty }, - { "fake_header_two", string.Empty } - }; - - var metadata = new List - { - new RestApiOperationParameter("fake_header_one", "string", true, RestApiOperationParameterLocation.Header, RestApiOperationParameterStyle.Simple), - new RestApiOperationParameter("fake_header_two", "string", false, RestApiOperationParameterLocation.Header, RestApiOperationParameterStyle.Simple) - }; - - var arguments = new Dictionary - { - { "fake_header_one", "fake_header_one_value" } - }; - - var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", metadata, rawHeaders); - - // Act - var headers = sut.RenderHeaders(arguments); - - // Assert - Assert.Equal(1, headers.Count); - - var headerOne = headers["fake_header_one"]; - Assert.Equal("fake_header_one_value", headerOne); - } - - [Fact] - public void ShouldUseDefaultValueForOptionalHeaderIfNoValueProvided() - { - // Arrange - var rawHeaders = new Dictionary - { - { "fake_header_one", string.Empty }, - { "fake_header_two", string.Empty } - }; - - var metadata = new List - { - new RestApiOperationParameter("fake_header_one", "string", true, RestApiOperationParameterLocation.Header, RestApiOperationParameterStyle.Simple), - new RestApiOperationParameter("fake_header_two", "string", false, RestApiOperationParameterLocation.Header, RestApiOperationParameterStyle.Simple, defaultValue: "fake_header_two_default_value") - }; - - var arguments = new Dictionary - { - { "fake_header_one", "fake_header_one_value" } //Argument is only provided for the first parameter and not for the second one - }; - - var sut = new RestApiOperation("fake_id", new Uri("http://fake_url"), "fake_path", HttpMethod.Get, "fake_description", metadata, rawHeaders); - - // Act - var headers = sut.RenderHeaders(arguments); - - // Assert - Assert.Equal(2, headers.Count); - - var headerOne = headers["fake_header_one"]; - Assert.Equal("fake_header_one_value", headerOne); - - var headerTwo = headers["fake_header_two"]; - Assert.Equal("fake_header_two_default_value", headerTwo); - } - - [Fact] - public void ItShouldNotWrapQueryStringValuesOfStringTypeIntoSingleQuotes() - { - // Arrange - var metadata = new List - { - new RestApiOperationParameter("fake_query_string_param", "string", false, RestApiOperationParameterLocation.Query, RestApiOperationParameterStyle.Simple) - }; - - var arguments = new Dictionary - { - { "fake_query_string_param", "fake_query_string_param_value" } - }; - - var sut = new RestApiOperation("fake_id", new Uri("https://fake-random-test-host"), "fake_path", HttpMethod.Get, "fake_description", metadata, new Dictionary()); - - // Act - var url = sut.BuildOperationUrl(arguments); - - // Assert - Assert.NotNull(url); - - var parameterValue = HttpUtility.ParseQueryString(url.Query)["fake_query_string_param"]; - Assert.NotNull(parameterValue); - Assert.Equal("fake_query_string_param_value", parameterValue); // Making sure that query string value of string type is not wrapped with quotes. - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Core/FileIOSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/Core/FileIOSkillTests.cs deleted file mode 100644 index b66c28c0a42d..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Core/FileIOSkillTests.cs +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.IO; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Core; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Core; - -public class FileIOSkillTests -{ - [Fact] - public void ItCanBeInstantiated() - { - // Act - Assert no exception occurs - _ = new FileIOSkill(); - } - - [Fact] - public void ItCanBeImported() - { - // Arrange - var kernel = Kernel.Builder.Build(); - - // Act - Assert no exception occurs e.g. due to reflection - _ = kernel.ImportSkill(new FileIOSkill(), "fileIO"); - } - - [Fact] - public async Task ItCanReadAsync() - { - // Arrange - var skill = new FileIOSkill(); - var path = Path.GetTempFileName(); - File.WriteAllText(path, "hello world"); - - // Act - var result = await skill.ReadAsync(path); - - // Assert - Assert.Equal("hello world", result); - } - - [Fact] - public async Task ItCannotReadAsync() - { - // Arrange - var skill = new FileIOSkill(); - var path = Path.GetTempFileName(); - File.Delete(path); - - // Act - Task Fn() - { - return skill.ReadAsync(path); - } - - // Assert - _ = await Assert.ThrowsAsync(Fn); - } - - [Fact] - public async Task ItCanWriteAsync() - { - // Arrange - var skill = new FileIOSkill(); - var path = Path.GetTempFileName(); - - // Act - await skill.WriteAsync(path, "hello world"); - - // Assert - Assert.Equal("hello world", await File.ReadAllTextAsync(path)); - } - - [Fact] - public async Task ItCannotWriteAsync() - { - // Arrange - var skill = new FileIOSkill(); - var path = Path.GetTempFileName(); - File.SetAttributes(path, FileAttributes.ReadOnly); - - // Act - Task Fn() - { - return skill.WriteAsync(path, "hello world"); - } - - // Assert - _ = await Assert.ThrowsAsync(Fn); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Core/HttpSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/Core/HttpSkillTests.cs deleted file mode 100644 index fb99493fcd73..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Core/HttpSkillTests.cs +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Core; -using Moq; -using Moq.Protected; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Core; - -public class HttpSkillTests : IDisposable -{ - private readonly string _content = "hello world"; - private readonly string _uriString = "http://www.example.com"; - - private readonly HttpResponseMessage _response = new() - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent("hello world"), - }; - - [Fact] - public void ItCanBeInstantiated() - { - // Act - Assert no exception occurs - var skill = new HttpSkill(); - } - - [Fact] - public void ItCanBeImported() - { - // Arrange - var kernel = KernelBuilder.Create(); - var skill = new HttpSkill(); - - // Act - Assert no exception occurs e.g. due to reflection - kernel.ImportSkill(skill, "http"); - } - - [Fact] - public async Task ItCanGetAsync() - { - // Arrange - var mockHandler = this.CreateMock(); - using var client = new HttpClient(mockHandler.Object); - var skill = new HttpSkill(client); - - // Act - var result = await skill.GetAsync(this._uriString); - - // Assert - Assert.Equal(this._content, result); - this.VerifyMock(mockHandler, HttpMethod.Get); - } - - [Fact] - public async Task ItCanPostAsync() - { - // Arrange - var mockHandler = this.CreateMock(); - using var client = new HttpClient(mockHandler.Object); - var skill = new HttpSkill(client); - - // Act - var result = await skill.PostAsync(this._uriString, this._content); - - // Assert - Assert.Equal(this._content, result); - this.VerifyMock(mockHandler, HttpMethod.Post); - } - - [Fact] - public async Task ItCanPutAsync() - { - // Arrange - var mockHandler = this.CreateMock(); - using var client = new HttpClient(mockHandler.Object); - var skill = new HttpSkill(client); - - // Act - var result = await skill.PutAsync(this._uriString, this._content); - - // Assert - Assert.Equal(this._content, result); - this.VerifyMock(mockHandler, HttpMethod.Put); - } - - [Fact] - public async Task ItCanDeleteAsync() - { - // Arrange - var mockHandler = this.CreateMock(); - using var client = new HttpClient(mockHandler.Object); - var skill = new HttpSkill(client); - - // Act - var result = await skill.DeleteAsync(this._uriString); - - // Assert - Assert.Equal(this._content, result); - this.VerifyMock(mockHandler, HttpMethod.Delete); - } - - private Mock CreateMock() - { - var mockHandler = new Mock(); - mockHandler.Protected() - .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) - .ReturnsAsync(this._response); - return mockHandler; - } - - private void VerifyMock(Mock mockHandler, HttpMethod method) - { - mockHandler.Protected().Verify( - "SendAsync", - Times.Exactly(1), // we expected a single external request - ItExpr.Is(req => - req.Method == method // we expected a POST request - && req.RequestUri == new Uri(this._uriString) // to this uri - ), - ItExpr.IsAny() - ); - } - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool disposing) - { - if (disposing) - { - this._response.Dispose(); - } - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Core/MathSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/Core/MathSkillTests.cs deleted file mode 100644 index 422874a6479e..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Core/MathSkillTests.cs +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Core; -using SemanticKernel.UnitTests; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Core; - -public class MathSkillTests -{ - [Fact] - public void ItCanBeInstantiated() - { - // Act - Assert no exception occurs - var _ = new MathSkill(); - } - - [Fact] - public void ItCanBeImported() - { - // Arrange - var kernel = Kernel.Builder.Build(); - - // Act - Assert no exception occurs e.g. due to reflection - kernel.ImportSkill(new MathSkill(), "math"); - } - - [Theory] - [InlineData("10", "10", "20")] - [InlineData("0", "10", "10")] - [InlineData("0", "-10", "-10")] - [InlineData("10", "0", "10")] - [InlineData("-1", "10", "9")] - [InlineData("-10", "10", "0")] - [InlineData("-192", "13", "-179")] - [InlineData("-192", "-13", "-205")] - public async Task AddWhenValidParametersShouldSucceedAsync(string initialValue, string amount, string expectedResult) - { - // Arrange - var target = new MathSkill(); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "Add", ("input", initialValue), ("amount", amount)); - - // Assert - Assert.Equal(expectedResult, context.Variables.Input); - } - - [Theory] - [InlineData("10", "10", "0")] - [InlineData("0", "10", "-10")] - [InlineData("10", "0", "10")] - [InlineData("100", "-10", "110")] - [InlineData("100", "102", "-2")] - [InlineData("-1", "10", "-11")] - [InlineData("-10", "10", "-20")] - [InlineData("-192", "13", "-205")] - public async Task SubtractWhenValidParametersShouldSucceedAsync(string initialValue, string amount, string expectedResult) - { - // Arrange - var target = new MathSkill(); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "Subtract", ("input", initialValue), ("amount", amount)); // Assert - - // Assert - Assert.Equal(expectedResult, context.Variables.Input); - } - - [Theory] - [InlineData("$0")] - [InlineData("one hundred")] - [InlineData("20..,,2,1")] - [InlineData(".2,2.1")] - [InlineData("0.1.0")] - [InlineData("00-099")] - [InlineData("¹²¹")] - [InlineData("2²")] - [InlineData("zero")] - [InlineData("-100 units")] - [InlineData("1 banana")] - public async Task AddWhenInvalidInitialValueShouldThrowAsync(string initialValue) - { - // Arrange - var target = new MathSkill(); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "Add", ("input", initialValue), ("amount", "1")); - - // Assert - AssertExtensions.AssertIsArgumentOutOfRange(context.LastException, "value", initialValue); - } - - [Theory] - [InlineData("$0")] - [InlineData("one hundred")] - [InlineData("20..,,2,1")] - [InlineData(".2,2.1")] - [InlineData("0.1.0")] - [InlineData("00-099")] - [InlineData("¹²¹")] - [InlineData("2²")] - [InlineData("zero")] - [InlineData("-100 units")] - [InlineData("1 banana")] - public async Task AddWhenInvalidAmountShouldThrowAsync(string amount) - { - // Arrange - var target = new MathSkill(); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "Add", ("input", "1"), ("amount", amount)); - - // Assert - AssertExtensions.AssertIsArgumentOutOfRange(context.LastException, "amount", amount); - } - - [Theory] - [InlineData("$0")] - [InlineData("one hundred")] - [InlineData("20..,,2,1")] - [InlineData(".2,2.1")] - [InlineData("0.1.0")] - [InlineData("00-099")] - [InlineData("¹²¹")] - [InlineData("2²")] - [InlineData("zero")] - [InlineData("-100 units")] - [InlineData("1 banana")] - public async Task SubtractWhenInvalidInitialValueShouldThrowAsync(string initialValue) - { - // Arrange - var target = new MathSkill(); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "Subtract", ("input", initialValue), ("amount", "1")); - - // Assert - AssertExtensions.AssertIsArgumentOutOfRange(context.LastException, "value", initialValue); - } - - [Theory] - [InlineData("$0")] - [InlineData("one hundred")] - [InlineData("20..,,2,1")] - [InlineData(".2,2.1")] - [InlineData("0.1.0")] - [InlineData("00-099")] - [InlineData("¹²¹")] - [InlineData("2²")] - [InlineData("zero")] - [InlineData("-100 units")] - [InlineData("1 banana")] - public async Task SubtractAsyncWhenInvalidAmountShouldThrowAsync(string amount) - { - // Arrange - var target = new MathSkill(); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "Subtract", ("input", "1"), ("amount", amount)); - - // Assert - AssertExtensions.AssertIsArgumentOutOfRange(context.LastException, "amount", amount); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Core/TextSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/Core/TextSkillTests.cs deleted file mode 100644 index 8a0f6f5af3f8..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Core/TextSkillTests.cs +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Core; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Core; - -public class TextSkillTests -{ - [Fact] - public void ItCanBeInstantiated() - { - // Act - Assert no exception occurs - var _ = new TextSkill(); - } - - [Fact] - public void ItCanBeImported() - { - // Arrange - var kernel = Kernel.Builder.Build(); - - // Act - Assert no exception occurs e.g. due to reflection - kernel.ImportSkill(new TextSkill(), "text"); - } - - [Fact] - public void ItCanTrim() - { - // Arrange - var skill = new TextSkill(); - - // Act - var result = skill.Trim(" hello world "); - - // Assert - Assert.Equal("hello world", result); - } - - [Fact] - public void ItCanTrimStart() - { - // Arrange - var skill = new TextSkill(); - - // Act - var result = skill.TrimStart(" hello world "); - - // Assert - Assert.Equal("hello world ", result); - } - - [Fact] - public void ItCanTrimEnd() - { - // Arrange - var skill = new TextSkill(); - - // Act - var result = skill.TrimEnd(" hello world "); - - // Assert - Assert.Equal(" hello world", result); - } - - [Fact] - public void ItCanUppercase() - { - // Arrange - var skill = new TextSkill(); - - // Act - var result = skill.Uppercase("hello world"); - - // Assert - Assert.Equal("HELLO WORLD", result); - } - - [Fact] - public void ItCanLowercase() - { - // Arrange - var skill = new TextSkill(); - - // Act - var result = skill.Lowercase("HELLO WORLD"); - - // Assert - Assert.Equal("hello world", result); - } - - [Theory] - [InlineData("hello world ", 12)] - [InlineData("hello World", 11)] - [InlineData("HELLO", 5)] - [InlineData("World", 5)] - [InlineData("", 0)] - [InlineData(" ", 1)] - [InlineData(null, 0)] - public void ItCanLength(string textToLength, int expectedLength) - { - // Arrange - var target = new TextSkill(); - - // Act - var result = target.Length(textToLength); - - // Assert - Assert.Equal(expectedLength, result); - } - - [Theory] - [InlineData("hello world", "hello world")] - [InlineData("hello World", "hello World")] - [InlineData("HELLO", "HELLO")] - [InlineData("World", "World")] - [InlineData("", "")] - [InlineData(" ", " ")] - [InlineData(null, "")] - public void ItCanConcat(string textToConcat, string text2ToConcat) - { - // Arrange - var target = new TextSkill(); - var expected = string.Concat(textToConcat, text2ToConcat); - - // Act - string result = target.Concat(textToConcat, text2ToConcat); - - // Assert - Assert.Equal(expected, result); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Core/TimeSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/Core/TimeSkillTests.cs deleted file mode 100644 index 13982a0fe5bb..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Core/TimeSkillTests.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Core; -using SemanticKernel.UnitTests; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Core; - -// TODO: allow clock injection and test all functions -public class TimeSkillTests -{ - [Fact] - public void ItCanBeInstantiated() - { - // Act - Assert no exception occurs - var _ = new TimeSkill(); - } - - [Fact] - public void ItCanBeImported() - { - // Arrange - var kernel = Kernel.Builder.Build(); - - // Act - Assert no exception occurs e.g. due to reflection - kernel.ImportSkill(new TimeSkill(), "time"); - } - - [Fact] - public void DaysAgo() - { - double interval = 2; - DateTime expected = DateTime.Now.AddDays(-interval); - var skill = new TimeSkill(); - string result = skill.DaysAgo(interval, CultureInfo.CurrentCulture); - DateTime returned = DateTime.Parse(result, CultureInfo.CurrentCulture); - Assert.Equal(expected.Day, returned.Day); - Assert.Equal(expected.Month, returned.Month); - Assert.Equal(expected.Year, returned.Year); - } - - [Fact] - public void Day() - { - string expected = DateTime.Now.ToString("dd", CultureInfo.CurrentCulture); - var skill = new TimeSkill(); - string result = skill.Day(CultureInfo.CurrentCulture); - Assert.Equal(expected, result); - Assert.True(int.TryParse(result, out _)); - } - - [Fact] - public async Task LastMatchingDayBadInput() - { - var skill = new TimeSkill(); - var context = await FunctionHelpers.CallViaKernel(skill, "DateMatchingLastDayName", ("input", "not a day name")); - AssertExtensions.AssertIsArgumentOutOfRange(context.LastException, "input", "not a day name"); - } - - [Theory] - [MemberData(nameof(DayOfWeekEnumerator))] - public void LastMatchingDay(DayOfWeek dayName) - { - int steps = 0; - DateTime date = DateTime.Now.Date.AddDays(-1); - while (date.DayOfWeek != dayName && steps <= 7) - { - date = date.AddDays(-1); - steps++; - } - bool found = date.DayOfWeek == dayName; - Assert.True(found); - - var skill = new TimeSkill(); - string result = skill.DateMatchingLastDayName(dayName, CultureInfo.CurrentCulture); - DateTime returned = DateTime.Parse(result, CultureInfo.CurrentCulture); - Assert.Equal(date.Day, returned.Day); - Assert.Equal(date.Month, returned.Month); - Assert.Equal(date.Year, returned.Year); - } - - public static IEnumerable DayOfWeekEnumerator() - { - foreach (var day in Enum.GetValues()) - { - yield return new object[] { day }; - } - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Core/WaitSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/Core/WaitSkillTests.cs deleted file mode 100644 index 3352015981bc..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Core/WaitSkillTests.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Core; -using Moq; -using SemanticKernel.UnitTests; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Core; - -// TODO: allow clock injection and test all functions -public class WaitSkillTests -{ - [Fact] - public void ItCanBeInstantiated() - { - // Act - Assert no exception occurs - var _ = new WaitSkill(); - } - - [Fact] - public void ItCanBeImported() - { - // Arrange - var kernel = Kernel.Builder.Build(); - - // Act - Assert no exception occurs e.g. due to reflection - kernel.ImportSkill(new WaitSkill(), "wait"); - } - - [Theory] - [InlineData("0", 0)] - [InlineData("100", 100000)] - [InlineData("20.1", 20100)] - [InlineData("0.1", 100)] - [InlineData("0.01", 10)] - [InlineData("0.001", 1)] - [InlineData("0.0001", 0)] - [InlineData("-0.0001", 0)] - [InlineData("-10000", 0)] - public async Task ItWaitSecondsWhenValidParametersSucceedAsync(string textSeconds, int expectedMilliseconds) - { - // Arrange - var waitProviderMock = new Mock(); - var target = new WaitSkill(waitProviderMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "Seconds", ("input", textSeconds)); - - // Assert - waitProviderMock.Verify(w => w.DelayAsync(It.IsIn(expectedMilliseconds)), Times.Once); - } - - [Theory] - [InlineData("$0")] - [InlineData("one hundred")] - [InlineData("20..,,2,1")] - [InlineData(".2,2.1")] - [InlineData("0.1.0")] - [InlineData("00-099")] - [InlineData("¹²¹")] - [InlineData("2²")] - [InlineData("zero")] - [InlineData("-100 seconds")] - [InlineData("1 second")] - public async Task ItWaitSecondsWhenInvalidParametersFailsAsync(string textSeconds) - { - // Arrange - var waitProviderMock = new Mock(); - var target = new WaitSkill(waitProviderMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "Seconds", ("input", textSeconds)); - - // Assert - AssertExtensions.AssertIsArgumentOutOfRange(context.LastException, "seconds", textSeconds); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Document/DocumentSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/Document/DocumentSkillTests.cs deleted file mode 100644 index cd6cb8c11698..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Document/DocumentSkillTests.cs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.Document; -using Microsoft.SemanticKernel.Skills.Document.FileSystem; -using Moq; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Document; - -public class DocumentSkillTests -{ - [Fact] - public async Task ReadTextAsyncSucceedsAsync() - { - // Arrange - var expectedText = Guid.NewGuid().ToString(); - var anyFilePath = Guid.NewGuid().ToString(); - - var fileSystemConnectorMock = new Mock(); - fileSystemConnectorMock - .Setup(mock => mock.GetFileContentStreamAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), - It.IsAny())) - .ReturnsAsync(Stream.Null); - - var documentConnectorMock = new Mock(); - documentConnectorMock - .Setup(mock => mock.ReadText(It.IsAny())) - .Returns(expectedText); - - var target = new DocumentSkill(documentConnectorMock.Object, fileSystemConnectorMock.Object); - - // Act - string actual = await target.ReadTextAsync(anyFilePath); - - // Assert - Assert.Equal(expectedText, actual); - fileSystemConnectorMock.VerifyAll(); - documentConnectorMock.VerifyAll(); - } - - [Fact] - public async Task AppendTextAsyncFileExistsSucceedsAsync() - { - // Arrange - var anyText = Guid.NewGuid().ToString(); - var anyFilePath = Guid.NewGuid().ToString(); - - var fileSystemConnectorMock = new Mock(); - fileSystemConnectorMock - .Setup(mock => mock.FileExistsAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), - It.IsAny())) - .ReturnsAsync(true); - fileSystemConnectorMock - .Setup(mock => mock.GetWriteableFileStreamAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), - It.IsAny())) - .ReturnsAsync(Stream.Null); - - var documentConnectorMock = new Mock(); - documentConnectorMock - .Setup(mock => mock.AppendText(It.IsAny(), It.Is(text => text.Equals(anyText, StringComparison.Ordinal)))); - - var target = new DocumentSkill(documentConnectorMock.Object, fileSystemConnectorMock.Object); - - // Act - await target.AppendTextAsync(anyText, anyFilePath); - - // Assert - fileSystemConnectorMock.VerifyAll(); - documentConnectorMock.VerifyAll(); - } - - [Fact] - public async Task AppendTextAsyncFileDoesNotExistSucceedsAsync() - { - // Arrange - var anyText = Guid.NewGuid().ToString(); - var anyFilePath = Guid.NewGuid().ToString(); - - var fileSystemConnectorMock = new Mock(); - fileSystemConnectorMock - .Setup(mock => mock.FileExistsAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), - It.IsAny())) - .ReturnsAsync(false); - fileSystemConnectorMock - .Setup(mock => mock.CreateFileAsync(It.Is(filePath => filePath.Equals(anyFilePath, StringComparison.Ordinal)), - It.IsAny())) - .ReturnsAsync(Stream.Null); - - var documentConnectorMock = new Mock(); - documentConnectorMock - .Setup(mock => mock.Initialize(It.IsAny())); - documentConnectorMock - .Setup(mock => mock.AppendText(It.IsAny(), It.Is(text => text.Equals(anyText, StringComparison.Ordinal)))); - - var target = new DocumentSkill(documentConnectorMock.Object, fileSystemConnectorMock.Object); - - // Act - await target.AppendTextAsync(anyText, anyFilePath); - - // Assert - fileSystemConnectorMock.VerifyAll(); - documentConnectorMock.VerifyAll(); - } - - [Fact] - public async Task AppendTextAsyncNoFilePathFailsAsync() - { - // Arrange - var anyText = Guid.NewGuid().ToString(); - - var fileSystemConnectorMock = new Mock(); - var documentConnectorMock = new Mock(); - - var target = new DocumentSkill(documentConnectorMock.Object, fileSystemConnectorMock.Object); - - // Act/Assert - await Assert.ThrowsAnyAsync(() => - target.AppendTextAsync(anyText, null!)); - - // Assert - fileSystemConnectorMock.Verify(mock => mock.GetWriteableFileStreamAsync(It.IsAny(), It.IsAny()), Times.Never()); - documentConnectorMock.Verify(mock => mock.AppendText(It.IsAny(), It.IsAny()), Times.Never()); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Grpc/Protobuf/TestSkills/ResourceSkillsProvider.cs b/dotnet/src/Skills/Skills.UnitTests/Grpc/Protobuf/TestSkills/ResourceSkillsProvider.cs deleted file mode 100644 index 5d2b9c9cffd4..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Grpc/Protobuf/TestSkills/ResourceSkillsProvider.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.IO; -using System.Resources; - -namespace SemanticKernel.Skills.UnitTests.Grpc.Protobuf.TestSkills; - -internal static class ResourceSkillsProvider -{ - /// - /// Loads .proto file from assembly resource. - /// - /// The resource name. - /// The OpenApi document resource stream. - public static Stream LoadFromResource(string resourceName) - { - var type = typeof(ResourceSkillsProvider); - - var stream = type.Assembly.GetManifestResourceStream(type, resourceName); - if (stream == null) - { - throw new MissingManifestResourceException($"Unable to load gRPC skill from assembly resource '{resourceName}'."); - } - - return stream; - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/MsGraph/CalendarSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/MsGraph/CalendarSkillTests.cs deleted file mode 100644 index f9447c499243..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/MsGraph/CalendarSkillTests.cs +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Globalization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.MsGraph; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; -using Moq; -using SemanticKernel.UnitTests; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.MsGraph; - -public class CalendarSkillTests -{ - [Fact] - public async Task AddEventAsyncSucceedsAsync() - { - // Arrange - string anyContent = Guid.NewGuid().ToString(); - string anySubject = Guid.NewGuid().ToString(); - string anyLocation = Guid.NewGuid().ToString(); - DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); - DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); - string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; - - CalendarEvent expected = new() - { - Subject = anySubject, - Location = anyLocation, - Attendees = anyAttendees - }; - - Mock connectorMock = new(); - connectorMock.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(expected); - - CalendarSkill target = new(connectorMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "AddEvent", - ("input", anySubject), - ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), - ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), - ("location", anyLocation), - ("content", anyContent), - ("attendees", string.Join(";", anyAttendees))); - - // Assert - Assert.False(context.ErrorOccurred); - connectorMock.VerifyAll(); - } - - [Fact] - public async Task AddEventAsyncWithoutLocationSucceedsAsync() - { - // Arrange - string anyContent = Guid.NewGuid().ToString(); - string anySubject = Guid.NewGuid().ToString(); - DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); - DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); - string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; - - CalendarEvent expected = new() - { - Content = anyContent, - Subject = anySubject, - Attendees = anyAttendees, - Start = anyStartTime, - End = anyEndTime - }; - - Mock connectorMock = new(); - connectorMock.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(expected); - - CalendarSkill target = new(connectorMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "AddEvent", - ("input", anySubject), - ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), - ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), - ("content", anyContent), - ("attendees", string.Join(";", anyAttendees))); - - // Assert - Assert.False(context.ErrorOccurred); - connectorMock.VerifyAll(); - } - - [Fact] - public async Task AddEventAsyncWithoutContentSucceedsAsync() - { - // Arrange - string anySubject = Guid.NewGuid().ToString(); - string anyLocation = Guid.NewGuid().ToString(); - DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); - DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); - string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; - - CalendarEvent expected = new() - { - Subject = anySubject, - Start = anyStartTime, - End = anyEndTime, - Location = anyLocation, - Attendees = anyAttendees - }; - - Mock connectorMock = new(); - connectorMock.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(expected); - - CalendarSkill target = new(connectorMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "AddEvent", - ("input", anySubject), - ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), - ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), - ("location", anyLocation), - ("attendees", string.Join(";", anyAttendees))); - - // Assert - Assert.False(context.ErrorOccurred); - connectorMock.VerifyAll(); - } - - [Fact] - public async Task AddEventAsyncWithoutAttendeesSucceedsAsync() - { - // Arrange - string anyContent = Guid.NewGuid().ToString(); - string anySubject = Guid.NewGuid().ToString(); - string anyLocation = Guid.NewGuid().ToString(); - DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); - DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); - - CalendarEvent expected = new() - { - Subject = anySubject, - Start = anyStartTime, - End = anyEndTime, - Content = anyContent, - Location = anyLocation - }; - - Mock connectorMock = new(); - connectorMock.Setup(c => c.AddEventAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(expected); - - CalendarSkill target = new(connectorMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "AddEvent", - ("input", anySubject), - ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), - ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), - ("location", anyLocation), - ("content", anyContent)); - - // Assert - Assert.False(context.ErrorOccurred); - connectorMock.VerifyAll(); - } - - [Fact] - public async Task AddEventAsyncWithoutStartFailsAsync() - { - // Arrange - string anyContent = Guid.NewGuid().ToString(); - string anySubject = Guid.NewGuid().ToString(); - string anyLocation = Guid.NewGuid().ToString(); - DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); - string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; - - Mock connectorMock = new(); - - CalendarSkill target = new(connectorMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "AddEvent", - ("input", anySubject), - ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), - ("location", anyLocation), - ("content", anyContent), - ("attendees", string.Join(";", anyAttendees))); - - // Assert - Assert.True(context.ErrorOccurred); - KernelException e = Assert.IsType(context.LastException); - Assert.Equal(KernelException.ErrorCodes.FunctionInvokeError, e.ErrorCode); - } - - [Fact] - public async Task AddEventAsyncWithoutEndFailsAsync() - { - // Arrange - string anyContent = Guid.NewGuid().ToString(); - string anySubject = Guid.NewGuid().ToString(); - string anyLocation = Guid.NewGuid().ToString(); - DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); - string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; - - Mock connectorMock = new(); - - CalendarSkill target = new(connectorMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "AddEvent", - ("input", anySubject), - ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), - ("location", anyLocation), - ("content", anyContent), - ("attendees", string.Join(";", anyAttendees))); - - // Assert - Assert.True(context.ErrorOccurred); - KernelException e = Assert.IsType(context.LastException); - Assert.Equal(KernelException.ErrorCodes.FunctionInvokeError, e.ErrorCode); - } - - [Fact] - public async Task AddEventAsyncWithoutSubjectFailsAsync() - { - // Arrange - string anyContent = Guid.NewGuid().ToString(); - string anyLocation = Guid.NewGuid().ToString(); - DateTimeOffset anyStartTime = DateTimeOffset.Now + TimeSpan.FromDays(1); - DateTimeOffset anyEndTime = DateTimeOffset.Now + TimeSpan.FromDays(1.1); - string[] anyAttendees = new[] { Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; - - Mock connectorMock = new(); - - CalendarSkill target = new(connectorMock.Object); - - // Act - var context = await FunctionHelpers.CallViaKernel(target, "AddEvent", - ("start", anyStartTime.ToString(CultureInfo.InvariantCulture)), - ("end", anyEndTime.ToString(CultureInfo.InvariantCulture)), - ("location", anyLocation), - ("content", anyContent), - ("attendees", string.Join(";", anyAttendees))); - - // Assert - Assert.True(context.ErrorOccurred); - ArgumentException e = Assert.IsType(context.LastException); - Assert.Equal("subject", e.ParamName); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/MsGraph/CloudDriveSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/MsGraph/CloudDriveSkillTests.cs deleted file mode 100644 index 811266e49183..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/MsGraph/CloudDriveSkillTests.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.IO; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.MsGraph; -using Moq; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.MsGraph; - -public class CloudDriveSkillTests -{ - [Fact] - public async Task UploadSmallFileAsyncSucceedsAsync() - { - // Arrange - string anyFilePath = Guid.NewGuid().ToString(); - - Mock connectorMock = new(); - connectorMock.Setup(c => c.UploadSmallFileAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(Task.CompletedTask); - - CloudDriveSkill target = new(connectorMock.Object); - - // Act - await target.UploadFileAsync(anyFilePath, Guid.NewGuid().ToString()); - - // Assert - connectorMock.VerifyAll(); - } - - [Fact] - public async Task CreateLinkAsyncSucceedsAsync() - { - // Arrange - string anyFilePath = Guid.NewGuid().ToString(); - string anyLink = Guid.NewGuid().ToString(); - - Mock connectorMock = new(); - connectorMock.Setup(c => c.CreateShareLinkAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(anyLink); - - CloudDriveSkill target = new(connectorMock.Object); - - // Act - string actual = await target.CreateLinkAsync(anyFilePath); - - // Assert - Assert.Equal(anyLink, actual); - connectorMock.VerifyAll(); - } - - [Fact] - public async Task GetFileContentAsyncSucceedsAsync() - { - string anyFilePath = Guid.NewGuid().ToString(); - string expectedContent = Guid.NewGuid().ToString(); - using MemoryStream expectedStream = new(Encoding.UTF8.GetBytes(expectedContent)); - - // Arrange - Mock connectorMock = new(); - connectorMock.Setup(c => c.GetFileContentStreamAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(expectedStream); - - CloudDriveSkill target = new(connectorMock.Object); - - // Act - string actual = await target.GetFileContentAsync(anyFilePath); - - // Assert - Assert.Equal(expectedContent, actual); - connectorMock.VerifyAll(); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/MsGraph/EmailSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/MsGraph/EmailSkillTests.cs deleted file mode 100644 index 6b0c064d93d3..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/MsGraph/EmailSkillTests.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.MsGraph; -using Moq; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.MsGraph; - -public class EmailSkillTests -{ - [Fact] - public async Task SendEmailAsyncSucceedsAsync() - { - // Arrange - Mock connectorMock = new(); - connectorMock.Setup(c => c.SendEmailAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(Task.CompletedTask); - - EmailSkill target = new(connectorMock.Object); - - string anyContent = Guid.NewGuid().ToString(); - string anySubject = Guid.NewGuid().ToString(); - string anyRecipient = Guid.NewGuid().ToString(); - - // Act - await target.SendEmailAsync(anyContent, anyRecipient, anySubject); - - // Assert - connectorMock.VerifyAll(); - } - - [Fact] - public async Task SendEmailAsyncNoRecipientFailsAsync() - { - // Arrange - Mock connectorMock = new(); - EmailSkill target = new(connectorMock.Object); - - string anyContent = Guid.NewGuid().ToString(); - string anySubject = Guid.NewGuid().ToString(); - - // Act/Assert - await Assert.ThrowsAnyAsync(() => - target.SendEmailAsync(anyContent, null!, anySubject)); - - // Assert - connectorMock.VerifyAll(); - } - - [Fact] - public async Task SendEmailAsyncNoSubjectFailsAsync() - { - // Arrange - Mock connectorMock = new(); - EmailSkill target = new(connectorMock.Object); - - string anyContent = Guid.NewGuid().ToString(); - string anyRecipient = Guid.NewGuid().ToString(); - - // Act/Assert - await Assert.ThrowsAnyAsync(() => - target.SendEmailAsync(anyContent, anyRecipient, null!)); - - // Assert - connectorMock.VerifyAll(); - } - - [Fact] - public async Task GetMyEmailAddressAsyncSucceedsAsync() - { - // Arrange - string anyEmailAddress = Guid.NewGuid().ToString(); - Mock connectorMock = new(); - connectorMock.Setup(c => c.GetMyEmailAddressAsync(It.IsAny())) - .ReturnsAsync(anyEmailAddress); - - EmailSkill target = new(connectorMock.Object); - - // Act - string actual = await target.GetMyEmailAddressAsync(); - - // Assert - Assert.Equal(anyEmailAddress, actual); - connectorMock.VerifyAll(); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/MsGraph/OrganizationHierarchySkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/MsGraph/OrganizationHierarchySkillTests.cs deleted file mode 100644 index 74ed40c4f1ea..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/MsGraph/OrganizationHierarchySkillTests.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.MsGraph; -using Moq; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.MsGraph; - -public class OrganizationHierarchySkillTests -{ - [Fact] - public async Task GetMyDirectReportsEmailAsyncSucceedsAsync() - { - // Arrange - string[] anyDirectReportsEmail = { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }; - Mock connectorMock = new(); - connectorMock.Setup(c => c.GetDirectReportsEmailAsync(It.IsAny())).ReturnsAsync(anyDirectReportsEmail); - OrganizationHierarchySkill target = new(connectorMock.Object); - - // Act - string actual = await target.GetMyDirectReportsEmailAsync(); - - // Assert - var emails = JsonSerializer.Deserialize>(actual); - Assert.NotNull(emails); - foreach (string directReportEmail in anyDirectReportsEmail) - { - Assert.Contains(directReportEmail, emails); - } - - connectorMock.VerifyAll(); - } - - [Fact] - public async Task GetMyManagerEmailAsyncSucceedsAsync() - { - // Arrange - string anyManagerEmail = Guid.NewGuid().ToString(); - Mock connectorMock = new(); - connectorMock.Setup(c => c.GetManagerEmailAsync(It.IsAny())).ReturnsAsync(anyManagerEmail); - OrganizationHierarchySkill target = new(connectorMock.Object); - - // Act - string actual = await target.GetMyManagerEmailAsync(); - - // Assert - Assert.Equal(anyManagerEmail, actual); - connectorMock.VerifyAll(); - } - - [Fact] - public async Task GetMyManagerNameAsyncSucceedsAsync() - { - // Arrange - string anyManagerName = Guid.NewGuid().ToString(); - Mock connectorMock = new(); - connectorMock.Setup(c => c.GetManagerNameAsync(It.IsAny())).ReturnsAsync(anyManagerName); - OrganizationHierarchySkill target = new(connectorMock.Object); - - // Act - string actual = await target.GetMyManagerNameAsync(); - - // Assert - Assert.Equal(anyManagerName, actual); - connectorMock.VerifyAll(); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/MsGraph/TaskListSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/MsGraph/TaskListSkillTests.cs deleted file mode 100644 index 6988f053f83b..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/MsGraph/TaskListSkillTests.cs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.MsGraph; -using Microsoft.SemanticKernel.Skills.MsGraph.Models; -using Moq; -using Xunit; -using static Microsoft.SemanticKernel.Skills.MsGraph.TaskListSkill; - -namespace SemanticKernel.Skills.UnitTests.MsGraph; - -public class TaskListSkillTests -{ - private readonly TaskManagementTaskList _anyTaskList = new( - id: Guid.NewGuid().ToString(), - name: Guid.NewGuid().ToString()); - - private readonly TaskManagementTask _anyTask = new( - id: Guid.NewGuid().ToString(), - title: Guid.NewGuid().ToString(), - reminder: (DateTimeOffset.Now + TimeSpan.FromDays(1)).ToString("o"), - due: DateTimeOffset.Now.ToString("o"), - isCompleted: false); - - [Fact] - public async Task AddTaskAsyncNoReminderSucceedsAsync() - { - // Arrange - string anyTitle = Guid.NewGuid().ToString(); - - Mock connectorMock = new(); - connectorMock.Setup(c => c.GetDefaultTaskListAsync(It.IsAny())) - .ReturnsAsync(this._anyTaskList); - - connectorMock.Setup(c => c.AddTaskAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(this._anyTask); - - TaskListSkill target = new(connectorMock.Object); - - // Act - await target.AddTaskAsync(anyTitle); - - // Assert - connectorMock.VerifyAll(); - } - - [Fact] - public async Task AddTaskAsyncWithReminderSucceedsAsync() - { - // Arrange - string anyTitle = Guid.NewGuid().ToString(); - - Mock connectorMock = new(); - connectorMock.Setup(c => c.GetDefaultTaskListAsync(It.IsAny())) - .ReturnsAsync(this._anyTaskList); - - connectorMock.Setup(c => c.AddTaskAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(this._anyTask); - - string anyReminder = (DateTimeOffset.Now + TimeSpan.FromHours(1)).ToString("o"); - - TaskListSkill target = new(connectorMock.Object); - - // Act - await target.AddTaskAsync(anyTitle, anyReminder); - - // Assert - connectorMock.VerifyAll(); - } - - [Fact] - public async Task AddTaskAsyncNoDefaultTaskListFailsAsync() - { - // Arrange - string anyTitle = Guid.NewGuid().ToString(); - - Mock connectorMock = new(); -#pragma warning disable CS8600 // Converting null literal or possible null value to non-nullable type. - connectorMock.Setup(c => c.GetDefaultTaskListAsync(It.IsAny())) - .ReturnsAsync((TaskManagementTaskList)null); -#pragma warning restore CS8600 // Converting null literal or possible null value to non-nullable type. - - string anyReminder = (DateTimeOffset.Now + TimeSpan.FromHours(1)).ToString("o"); - - TaskListSkill target = new(connectorMock.Object); - - // Act/Assert - await Assert.ThrowsAnyAsync(() => - target.AddTaskAsync(anyTitle, anyReminder)); - - // Assert - connectorMock.VerifyAll(); - } - - [Theory] - [InlineData(DayOfWeek.Sunday)] - [InlineData(DayOfWeek.Monday)] - [InlineData(DayOfWeek.Tuesday)] - [InlineData(DayOfWeek.Wednesday)] - [InlineData(DayOfWeek.Thursday)] - [InlineData(DayOfWeek.Friday)] - [InlineData(DayOfWeek.Saturday)] - public void GetNextDayOfWeekIsCorrect(DayOfWeek dayOfWeek) - { - // Arrange - DateTimeOffset today = new(DateTime.Today); - TimeSpan timeOfDay = TimeSpan.FromHours(13); - - // Act - DateTimeOffset actual = GetNextDayOfWeek(dayOfWeek, timeOfDay); - - // Assert - Assert.Equal(dayOfWeek, actual.DayOfWeek); - Assert.True(today.ToUnixTimeSeconds() < actual.ToUnixTimeSeconds()); - Assert.Equal(timeOfDay.Hours, actual.Hour); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/JsonPathSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/OpenAPI/JsonPathSkillTests.cs deleted file mode 100644 index 9e0ab3a9d416..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/JsonPathSkillTests.cs +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Skills.OpenAPI; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.OpenAPI; - -public class JsonPathSkillTests -{ - private const string Json = @"{ - 'Stores': [ - 'Lambton Quay', - 'Willis Street' - ], - 'Manufacturers': [ - { - 'Name': 'Acme Co', - 'Products': [ - { - 'Name': 'Anvil', - 'Price': 50 - } - ] - }, - { - 'Name': 'Contoso', - 'Products': [ - { - 'Name': 'Elbow Grease', - 'Price': 99.95 - }, - { - 'Name': 'Headlight Fluid', - 'Price': 4 - } - ] - } - ] -}"; - - [Theory] - [InlineData("$.Manufacturers[0].Products[0].Name", "Anvil")] // single value - [InlineData("$.Manufacturers[0].Products[0].Foo", "")] // no value - public void GetJsonElementValueSucceeds(string jsonPath, string expected) - { - var target = new JsonPathSkill(); - - string actual = target.GetJsonElementValue(Json, jsonPath); - - Assert.Equal(expected, actual, StringComparer.OrdinalIgnoreCase); - } - - [Theory] - [InlineData("$..Products[?(@.Price >= 50)].Name", "[\"Anvil\",\"Elbow Grease\"]")] // multiple values - [InlineData("$.Manufacturers", - "[[{\"Name\":\"Acme Co\",\"Products\":[{\"Name\":\"Anvil\",\"Price\":50}]},{\"Name\":\"Contoso\",\"Products\":[{\"Name\":\"Elbow Grease\",\"Price\":99.95},{\"Name\":\"Headlight Fluid\",\"Price\":4}]}]]")] // complex value - public void GetJsonPropertyValueSucceeds(string jsonPath, string expected) - { - var target = new JsonPathSkill(); - - string actual = target.GetJsonElements(Json, jsonPath); - - Assert.Equal(expected, actual, StringComparer.OrdinalIgnoreCase); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV30Tests.cs b/dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV30Tests.cs deleted file mode 100644 index 5a42ae8eba25..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV30Tests.cs +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; -using Microsoft.SemanticKernel.Skills.OpenAPI.OpenApi; -using SemanticKernel.Skills.UnitTests.OpenAPI.TestSkills; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.OpenAPI; - -public sealed class OpenApiDocumentParserV30Tests : IDisposable -{ - /// - /// System under test - an instance of OpenApiDocumentParser class. - /// - private readonly OpenApiDocumentParser _sut; - - /// - /// OpenAPI document stream. - /// - private readonly Stream _openApiDocument; - - /// - /// Creates an instance of a class. - /// - public OpenApiDocumentParserV30Tests() - { - this._openApiDocument = ResourceSkillsProvider.LoadFromResource("documentV3_0.json"); - - this._sut = new OpenApiDocumentParser(); - } - - [Fact] - public async Task ItCanParsePutOperationBodySuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.NotNull(operations); - Assert.True(operations.Any()); - - var putOperation = operations.Single(o => o.Id == "SetSecret"); - Assert.NotNull(putOperation); - - var payload = putOperation.Payload; - Assert.NotNull(payload); - Assert.Equal("application/json", payload.MediaType); - - var properties = payload.Properties; - Assert.NotNull(properties); - Assert.Equal(2, properties.Count); - - var valueProperty = properties.FirstOrDefault(p => p.Name == "value"); - Assert.NotNull(valueProperty); - Assert.True(valueProperty.IsRequired); - Assert.Equal("The value of the secret.", valueProperty.Description); - Assert.NotNull(valueProperty.Properties); - Assert.False(valueProperty.Properties.Any()); - - var attributesProperty = properties.FirstOrDefault(p => p.Name == "attributes"); - Assert.NotNull(attributesProperty); - Assert.False(attributesProperty.IsRequired); - Assert.Equal("attributes", attributesProperty.Description); - Assert.NotNull(attributesProperty.Properties); - Assert.True(attributesProperty.Properties.Any()); - - var enabledProperty = attributesProperty.Properties.FirstOrDefault(p => p.Name == "enabled"); - Assert.NotNull(enabledProperty); - Assert.False(enabledProperty.IsRequired); - Assert.Equal("Determines whether the object is enabled.", enabledProperty.Description); - Assert.NotNull(enabledProperty.Properties); - Assert.False(enabledProperty.Properties.Any()); - } - - [Fact] - public async Task ItCanParsePutOperationMetadataSuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.NotNull(operations); - Assert.True(operations.Any()); - - var putOperation = operations.Single(o => o.Id == "SetSecret"); - Assert.NotNull(putOperation); - Assert.Equal("Sets a secret in a specified key vault.", putOperation.Description); - Assert.Equal("https://my-key-vault.vault.azure.net/", putOperation.ServerUrl?.AbsoluteUri); - Assert.Equal(HttpMethod.Put, putOperation.Method); - Assert.Equal("/secrets/{secret-name}", putOperation.Path); - - var parameters = putOperation.GetParameters(); - Assert.NotNull(parameters); - Assert.True(parameters.Count >= 5); - - var pathParameter = parameters.Single(p => p.Name == "secret-name"); //'secret-name' path parameter. - Assert.True(pathParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Path, pathParameter.Location); - Assert.Null(pathParameter.DefaultValue); - - var apiVersionParameter = parameters.Single(p => p.Name == "api-version"); //'api-version' query string parameter. - Assert.True(apiVersionParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Query, apiVersionParameter.Location); - Assert.Equal("7.0", apiVersionParameter.DefaultValue); - - var serverUrlParameter = parameters.Single(p => p.Name == "server-url"); //'server-url' artificial parameter. - Assert.False(serverUrlParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Path, serverUrlParameter.Location); - Assert.Equal("https://my-key-vault.vault.azure.net/", serverUrlParameter.DefaultValue); - - var payloadParameter = parameters.Single(p => p.Name == "payload"); //'payload' artificial parameter. - Assert.True(payloadParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Body, payloadParameter.Location); - Assert.Null(payloadParameter.DefaultValue); - Assert.Equal("REST API request body.", payloadParameter.Description); - - var contentTypeParameter = parameters.Single(p => p.Name == "content-type"); //'content-type' artificial parameter. - Assert.False(contentTypeParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Body, contentTypeParameter.Location); - Assert.Null(contentTypeParameter.DefaultValue); - Assert.Equal("Content type of REST API request body.", contentTypeParameter.Description); - } - - [Fact] - public async Task ItCanExtractSimpleTypeHeaderParameterMetadataSuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - //Assert string header parameter metadata - var accept = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "Accept"); - - Assert.Equal("string", accept.Type); - Assert.Equal("application/json", accept.DefaultValue); - Assert.Equal("Indicates which content types, expressed as MIME types, the client is able to understand.", accept.Description); - Assert.False(accept.IsRequired); - - //Assert integer header parameter metadata - var apiVersion = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "X-API-Version"); - - Assert.Equal("integer", apiVersion.Type); - Assert.Equal("10", apiVersion.DefaultValue); - Assert.Equal("Requested API version.", apiVersion.Description); - Assert.True(apiVersion.IsRequired); - } - - [Fact] - public async Task ItCanExtractCsvStyleHeaderParameterMetadataSuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - //Assert header parameters metadata - var acceptParameter = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "X-Operation-Csv-Ids"); - - Assert.Null(acceptParameter.DefaultValue); - Assert.False(acceptParameter.IsRequired); - Assert.Equal("array", acceptParameter.Type); - Assert.Equal(RestApiOperationParameterStyle.Simple, acceptParameter.Style); - Assert.Equal("The comma separated list of operation ids.", acceptParameter.Description); - Assert.Equal("string", acceptParameter.ArrayItemType); - } - - [Fact] - public async Task ItCanExtractHeadersSuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.True(operations.Any()); - - var operation = operations.Single(o => o.Id == "SetSecret"); - Assert.NotNull(operation.Headers); - Assert.Equal(3, operation.Headers.Count); - - Assert.True(operation.Headers.ContainsKey("Accept")); - Assert.True(operation.Headers.ContainsKey("X-API-Version")); - Assert.True(operation.Headers.ContainsKey("X-Operation-Csv-Ids")); - } - - [Fact] - public async Task ItCanExtractAllPathsAsOperationsAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.Equal(3, operations.Count); - } - - [Fact] - public async Task ItCanParseOperationHavingTextPlainBodySuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.NotNull(operations); - Assert.True(operations.Any()); - - var operation = operations.Single(o => o.Id == "Excuses"); - Assert.NotNull(operation); - - var payload = operation.Payload; - Assert.NotNull(payload); - Assert.Equal("text/plain", payload.MediaType); - Assert.Equal("excuse event", payload.Description); - - var properties = payload.Properties; - Assert.NotNull(properties); - Assert.Equal(0, properties.Count); - } - - [Fact] - public async Task ItShouldThrowExceptionForNonCompliantDocumentAsync() - { - // Arrange - var nonComplaintOpenApiDocument = ResourceSkillsProvider.LoadFromResource("nonCompliant_documentV3_0.json"); - - // Act and Assert - await Assert.ThrowsAsync(async () => await this._sut.ParseAsync(nonComplaintOpenApiDocument)); - } - - [Fact] - public async Task ItShouldWorkWithNonCompliantDocumentIfAllowedAsync() - { - // Arrange - var nonComplaintOpenApiDocument = ResourceSkillsProvider.LoadFromResource("nonCompliant_documentV3_0.json"); - - // Act - await this._sut.ParseAsync(nonComplaintOpenApiDocument, ignoreNonCompliantErrors: true); - - // Assert - // The absence of any thrown exceptions serves as evidence of the functionality's success. - } - - private static RestApiOperationParameter GetParameterMetadata(IList operations, string operationId, - RestApiOperationParameterLocation location, string name) - { - Assert.True(operations.Any()); - - var operation = operations.Single(o => o.Id == operationId); - Assert.NotNull(operation.Parameters); - Assert.True(operation.Parameters.Any()); - - var parameters = operation.Parameters.Where(p => p.Location == location); - - var parameter = parameters.Single(p => p.Name == name); - Assert.NotNull(parameter); - - return parameter; - } - - public void Dispose() - { - this._openApiDocument.Dispose(); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV31Tests.cs b/dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV31Tests.cs deleted file mode 100644 index d2be70e9841a..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/OpenApiDocumentParserV31Tests.cs +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Skills.OpenAPI.Model; -using Microsoft.SemanticKernel.Skills.OpenAPI.OpenApi; -using SemanticKernel.Skills.UnitTests.OpenAPI.TestSkills; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.OpenAPI; - -public sealed class OpenApiDocumentParserV31Tests : IDisposable -{ - /// - /// System under test - an instance of OpenApiDocumentParser class. - /// - private readonly OpenApiDocumentParser _sut; - - /// - /// OpenAPI document stream. - /// - private readonly Stream _openApiDocument; - - /// - /// Creates an instance of a class. - /// - public OpenApiDocumentParserV31Tests() - { - this._openApiDocument = ResourceSkillsProvider.LoadFromResource("documentV3_1.yaml"); - - this._sut = new OpenApiDocumentParser(); - } - - [Fact] - public async Task ItCanParsePutOperationBodySuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.NotNull(operations); - Assert.True(operations.Any()); - - var putOperation = operations.Single(o => o.Id == "SetSecret"); - Assert.NotNull(putOperation); - - var payload = putOperation.Payload; - Assert.NotNull(payload); - Assert.Equal("application/json", payload.MediaType); - - var properties = payload.Properties; - Assert.NotNull(properties); - Assert.Equal(2, properties.Count); - - var valueProperty = properties.FirstOrDefault(p => p.Name == "value"); - Assert.NotNull(valueProperty); - Assert.True(valueProperty.IsRequired); - Assert.Equal("The value of the secret.", valueProperty.Description); - Assert.NotNull(valueProperty.Properties); - Assert.False(valueProperty.Properties.Any()); - - var attributesProperty = properties.FirstOrDefault(p => p.Name == "attributes"); - Assert.NotNull(attributesProperty); - Assert.False(attributesProperty.IsRequired); - Assert.Equal("attributes", attributesProperty.Description); - Assert.NotNull(attributesProperty.Properties); - Assert.True(attributesProperty.Properties.Any()); - - var enabledProperty = attributesProperty.Properties.FirstOrDefault(p => p.Name == "enabled"); - Assert.NotNull(enabledProperty); - Assert.False(enabledProperty.IsRequired); - Assert.Equal("Determines whether the object is enabled.", enabledProperty.Description); - Assert.NotNull(enabledProperty.Properties); - Assert.False(enabledProperty.Properties.Any()); - } - - [Fact] - public async Task ItCanParsePutOperationMetadataSuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.NotNull(operations); - Assert.True(operations.Any()); - - var putOperation = operations.Single(o => o.Id == "SetSecret"); - Assert.NotNull(putOperation); - Assert.Equal("Sets a secret in a specified key vault.", putOperation.Description); - Assert.Equal("https://my-key-vault.vault.azure.net/", putOperation.ServerUrl?.AbsoluteUri); - Assert.Equal(HttpMethod.Put, putOperation.Method); - Assert.Equal("/secrets/{secret-name}", putOperation.Path); - - var parameters = putOperation.GetParameters(); - Assert.NotNull(parameters); - Assert.True(parameters.Count >= 5); - - var pathParameter = parameters.Single(p => p.Name == "secret-name"); //'secret-name' path parameter. - Assert.True(pathParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Path, pathParameter.Location); - Assert.Null(pathParameter.DefaultValue); - - var apiVersionParameter = parameters.Single(p => p.Name == "api-version"); //'api-version' query string parameter. - Assert.True(apiVersionParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Query, apiVersionParameter.Location); - Assert.Equal("7.0", apiVersionParameter.DefaultValue); - - var serverUrlParameter = parameters.Single(p => p.Name == "server-url"); //'server-url' artificial parameter. - Assert.False(serverUrlParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Path, serverUrlParameter.Location); - Assert.Equal("https://my-key-vault.vault.azure.net/", serverUrlParameter.DefaultValue); - - var payloadParameter = parameters.Single(p => p.Name == "payload"); //'payload' artificial parameter. - Assert.True(payloadParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Body, payloadParameter.Location); - Assert.Null(payloadParameter.DefaultValue); - Assert.Equal("REST API request body.", payloadParameter.Description); - - var contentTypeParameter = parameters.Single(p => p.Name == "content-type"); //'content-type' artificial parameter. - Assert.False(contentTypeParameter.IsRequired); - Assert.Equal(RestApiOperationParameterLocation.Body, contentTypeParameter.Location); - Assert.Null(contentTypeParameter.DefaultValue); - Assert.Equal("Content type of REST API request body.", contentTypeParameter.Description); - } - - [Fact] - public async Task ItCanExtractSimpleTypeHeaderParameterMetadataSuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - //Assert string header parameter metadata - var accept = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "Accept"); - - Assert.Equal("string", accept.Type); - Assert.Equal("application/json", accept.DefaultValue); - Assert.Equal("Indicates which content types, expressed as MIME types, the client is able to understand.", accept.Description); - Assert.False(accept.IsRequired); - - //Assert integer header parameter metadata - var apiVersion = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "X-API-Version"); - - Assert.Equal("integer", apiVersion.Type); - Assert.Equal("10", apiVersion.DefaultValue); - Assert.Equal("Requested API version.", apiVersion.Description); - Assert.True(apiVersion.IsRequired); - } - - [Fact] - public async Task ItCanExtractCsvStyleHeaderParameterMetadataSuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - //Assert header parameters metadata - var acceptParameter = GetParameterMetadata(operations, "SetSecret", RestApiOperationParameterLocation.Header, "X-Operation-Csv-Ids"); - - Assert.Null(acceptParameter.DefaultValue); - Assert.False(acceptParameter.IsRequired); - Assert.Equal("array", acceptParameter.Type); - Assert.Equal(RestApiOperationParameterStyle.Simple, acceptParameter.Style); - Assert.Equal("The comma separated list of operation ids.", acceptParameter.Description); - Assert.Equal("string", acceptParameter.ArrayItemType); - } - - [Fact] - public async Task ItCanExtractHeadersSuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.True(operations.Any()); - - var operation = operations.Single(o => o.Id == "SetSecret"); - Assert.NotNull(operation.Headers); - Assert.Equal(3, operation.Headers.Count); - - Assert.True(operation.Headers.ContainsKey("Accept")); - Assert.True(operation.Headers.ContainsKey("X-API-Version")); - Assert.True(operation.Headers.ContainsKey("X-Operation-Csv-Ids")); - } - - [Fact] - public async Task ItCanExtractAllPathsAsOperationsAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.Equal(3, operations.Count); - } - - [Fact] - public async Task ItCanParseOperationHavingTextPlainBodySuccessfullyAsync() - { - // Act - var operations = await this._sut.ParseAsync(this._openApiDocument); - - // Assert - Assert.NotNull(operations); - Assert.True(operations.Any()); - - var operation = operations.Single(o => o.Id == "Excuses"); - Assert.NotNull(operation); - - var payload = operation.Payload; - Assert.NotNull(payload); - Assert.Equal("text/plain", payload.MediaType); - Assert.Equal("excuse event", payload.Description); - - var properties = payload.Properties; - Assert.NotNull(properties); - Assert.Equal(0, properties.Count); - } - - private static RestApiOperationParameter GetParameterMetadata(IList operations, string operationId, RestApiOperationParameterLocation location, string name) - { - Assert.True(operations.Any()); - - var operation = operations.Single(o => o.Id == operationId); - Assert.NotNull(operation.Parameters); - Assert.True(operation.Parameters.Any()); - - var parameters = operation.Parameters.Where(p => p.Location == location); - - var parameter = parameters.Single(p => p.Name == name); - Assert.NotNull(parameter); - - return parameter; - } - - public void Dispose() - { - this._openApiDocument.Dispose(); - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/ResourceSkillsProvider.cs b/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/ResourceSkillsProvider.cs deleted file mode 100644 index 0346aa0d9cac..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/OpenAPI/TestSkills/ResourceSkillsProvider.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.IO; -using System.Resources; - -namespace SemanticKernel.Skills.UnitTests.OpenAPI.TestSkills; - -internal static class ResourceSkillsProvider -{ - /// - /// Loads OpenApi document from assembly resource. - /// - /// The resource name. - /// The OpenApi document resource stream. - public static Stream LoadFromResource(string resourceName) - { - var type = typeof(ResourceSkillsProvider); - - var stream = type.Assembly.GetManifestResourceStream(type, resourceName); - if (stream == null) - { - throw new MissingManifestResourceException($"Unable to load OpenApi skill from assembly resource '{resourceName}'."); - } - - return stream; - } -} diff --git a/dotnet/src/Skills/Skills.UnitTests/Skills.UnitTests.csproj b/dotnet/src/Skills/Skills.UnitTests/Skills.UnitTests.csproj deleted file mode 100644 index c449a27e0126..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Skills.UnitTests.csproj +++ /dev/null @@ -1,53 +0,0 @@ - - - - SemanticKernel.Skills.UnitTests - SemanticKernel.Skills.UnitTests - net6.0 - LatestMajor - true - enable - disable - false - CA2007,VSTHRD111 - - - - - - - - - - - - - - - - - - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - - - - - - - - diff --git a/dotnet/src/Skills/Skills.UnitTests/Web/SearchUrlSkillTests.cs b/dotnet/src/Skills/Skills.UnitTests/Web/SearchUrlSkillTests.cs deleted file mode 100644 index 0536c2ca67a5..000000000000 --- a/dotnet/src/Skills/Skills.UnitTests/Web/SearchUrlSkillTests.cs +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Encodings.Web; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Skills.Web; -using Xunit; - -namespace SemanticKernel.Skills.UnitTests.Web; - -public class SearchUrlSkillTests -{ - private const string AnyInput = ""; - private readonly string _encodedInput = UrlEncoder.Default.Encode(AnyInput); - - [Fact] - public void ItCanBeInstantiated() - { - // Act - Assert no exception occurs - var _ = new SearchUrlSkill(); - } - - [Fact] - public void ItCanBeImported() - { - // Arrange - IKernel kernel = Kernel.Builder.Build(); - - // Act - Assert no exception occurs e.g. due to reflection - kernel.ImportSkill(new SearchUrlSkill(), "search"); - } - - [Fact] - public void AmazonSearchUrlSucceeds() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.AmazonSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.amazon.com/s?k={this._encodedInput}", actual); - } - - [Fact] - public void BingSearchUrlSucceeds() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.BingSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.bing.com/search?q={this._encodedInput}", actual); - } - - [Fact] - public void BingImagesSearchUrlSucceeds() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.BingImagesSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.bing.com/images/search?q={this._encodedInput}", actual); - } - - [Fact] - public void BingMapsSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.BingMapsSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.bing.com/maps?q={this._encodedInput}", actual); - } - - [Fact] - public void BingShoppingSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.BingShoppingSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.bing.com/shop?q={this._encodedInput}", actual); - } - - [Fact] - public void BingNewsSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.BingNewsSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.bing.com/news/search?q={this._encodedInput}", actual); - } - - [Fact] - public void BingTravelSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.BingTravelSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.bing.com/travel/search?q={this._encodedInput}", actual); - } - - [Fact] - public void FacebookSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.FacebookSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.facebook.com/search/top/?q={this._encodedInput}", actual); - } - - [Fact] - public void GitHubSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.GitHubSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://github.com/search?q={this._encodedInput}", actual); - } - - [Fact] - public void LinkedInSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.LinkedInSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://www.linkedin.com/search/results/index/?keywords={this._encodedInput}", actual); - } - - [Fact] - public void TwitterSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.TwitterSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://twitter.com/search?q={this._encodedInput}", actual); - } - - [Fact] - public void WikipediaSearchUrl() - { - // Arrange - var skill = new SearchUrlSkill(); - - // Act - string actual = skill.WikipediaSearchUrl(AnyInput); - - // Assert - Assert.Equal($"https://wikipedia.org/w/index.php?search={this._encodedInput}", actual); - } -} diff --git a/dotnet/src/Skills/Skills.Web/Bing/BingConnector.cs b/dotnet/src/Skills/Skills.Web/Bing/BingConnector.cs deleted file mode 100644 index a9e52a55e5d9..000000000000 --- a/dotnet/src/Skills/Skills.Web/Bing/BingConnector.cs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Net.Http; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Skills.Web.Bing; - -/// -/// Bing API connector. -/// -public sealed class BingConnector : IWebSearchEngineConnector -{ - private readonly ILogger _logger; - private readonly HttpClient _httpClient; - private readonly string? _apiKey; - - /// - /// Initializes a new instance of the class. - /// - /// The API key to authenticate the connector. - /// An optional logger to log connector-related information. - public BingConnector(string apiKey, ILogger? logger = null) : - this(apiKey, new HttpClient(NonDisposableHttpClientHandler.Instance, false), logger) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The API key to authenticate the connector. - /// The HTTP client to use for making requests. - /// An optional logger to log connector-related information. - public BingConnector(string apiKey, HttpClient httpClient, ILogger? logger = null) - { - Verify.NotNull(httpClient); - - this._apiKey = apiKey; - this._logger = logger ?? NullLogger.Instance; - this._httpClient = httpClient; - } - - /// - public async Task> SearchAsync(string query, int count = 1, int offset = 0, CancellationToken cancellationToken = default) - { - if (count <= 0) { throw new ArgumentOutOfRangeException(nameof(count)); } - - if (count >= 50) { throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} value must be less than 50."); } - - if (offset < 0) { throw new ArgumentOutOfRangeException(nameof(offset)); } - - Uri uri = new($"https://api.bing.microsoft.com/v7.0/search?q={Uri.EscapeDataString(query)}&count={count}&offset={offset}"); - - this._logger.LogDebug("Sending request: {0}", uri); - - using HttpResponseMessage response = await this.SendGetRequest(uri, cancellationToken).ConfigureAwait(false); - - response.EnsureSuccessStatusCode(); - - this._logger.LogDebug("Response received: {0}", response.StatusCode); - - string json = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - - // Sensitive data, logging as trace, disabled by default - this._logger.LogTrace("Response content received: {0}", json); - - BingSearchResponse? data = JsonSerializer.Deserialize(json); - - WebPage[]? results = data?.WebPages?.Value; - - return results == null ? Enumerable.Empty() : results.Select(x => x.Snippet); - } - - /// - /// Sends a GET request to the specified URI. - /// - /// The URI to send the request to. - /// A cancellation token to cancel the request. - /// A representing the response from the request. - private async Task SendGetRequest(Uri uri, CancellationToken cancellationToken = default) - { - using var httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, uri); - - if (!string.IsNullOrEmpty(this._apiKey)) - { - httpRequestMessage.Headers.Add("Ocp-Apim-Subscription-Key", this._apiKey); - } - - return await this._httpClient.SendAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); - } - - [SuppressMessage("Performance", "CA1812:Internal class that is apparently never instantiated", - Justification = "Class is instantiated through deserialization.")] - private sealed class BingSearchResponse - { - [JsonPropertyName("webPages")] - public WebPages? WebPages { get; set; } - } - - [SuppressMessage("Performance", "CA1812:Internal class that is apparently never instantiated", - Justification = "Class is instantiated through deserialization.")] - private sealed class WebPages - { - [JsonPropertyName("value")] - public WebPage[]? Value { get; set; } - } - - [SuppressMessage("Performance", "CA1812:Internal class that is apparently never instantiated", - Justification = "Class is instantiated through deserialization.")] - private sealed class WebPage - { - [JsonPropertyName("name")] - public string Name { get; set; } = string.Empty; - - [JsonPropertyName("url")] - public string Url { get; set; } = string.Empty; - - [JsonPropertyName("snippet")] - public string Snippet { get; set; } = string.Empty; - } -} diff --git a/dotnet/src/Skills/Skills.Web/Google/GoogleConnector.cs b/dotnet/src/Skills/Skills.Web/Google/GoogleConnector.cs deleted file mode 100644 index 4a2c8871bc3f..000000000000 --- a/dotnet/src/Skills/Skills.Web/Google/GoogleConnector.cs +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Google.Apis.CustomSearchAPI.v1; -using Google.Apis.Services; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Skills.Web.Google; - -/// -/// Google search connector. -/// -public sealed class GoogleConnector : IWebSearchEngineConnector, IDisposable -{ - private readonly ILogger _logger; - private readonly CustomSearchAPIService _search; - private readonly string? _searchEngineId; - - /// - /// Google search connector. - /// - /// Google Custom Search API (looks like "ABcdEfG1...") - /// Google Search Engine ID (looks like "a12b345...") - /// Optional logger - public GoogleConnector( - string apiKey, - string searchEngineId, - ILogger? logger = null) : this(new BaseClientService.Initializer { ApiKey = apiKey }, searchEngineId, logger) - { - Verify.NotNullOrWhiteSpace(apiKey); - } - - /// - /// Google search connector. - /// - /// The connector initializer - /// Google Search Engine ID (looks like "a12b345...") - /// Optional logger - public GoogleConnector( - BaseClientService.Initializer initializer, - string searchEngineId, - ILogger? logger = null) - { - Verify.NotNull(initializer); - Verify.NotNullOrWhiteSpace(searchEngineId); - - this._search = new CustomSearchAPIService(initializer); - this._searchEngineId = searchEngineId; - this._logger = logger ?? NullLogger.Instance; - } - - /// - public async Task> SearchAsync( - string query, - int count, - int offset, - CancellationToken cancellationToken) - { - if (count <= 0) { throw new ArgumentOutOfRangeException(nameof(count)); } - - if (count > 10) { throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} value must be between 0 and 10, inclusive."); } - - if (offset < 0) { throw new ArgumentOutOfRangeException(nameof(offset)); } - - var search = this._search.Cse.List(); - search.Cx = this._searchEngineId; - search.Q = query; - search.Num = count; - search.Start = offset; - - var results = await search.ExecuteAsync(cancellationToken).ConfigureAwait(false); - - return results.Items.Select(item => item.Snippet); - } - - private void Dispose(bool disposing) - { - if (disposing) - { - this._search.Dispose(); - } - } - - public void Dispose() - { - this.Dispose(disposing: true); - GC.SuppressFinalize(this); - } -} diff --git a/dotnet/src/Skills/Skills.Web/SearchUrlSkill.cs b/dotnet/src/Skills/Skills.Web/SearchUrlSkill.cs deleted file mode 100644 index ccfbd704217f..000000000000 --- a/dotnet/src/Skills/Skills.Web/SearchUrlSkill.cs +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; -using System.Text.Encodings.Web; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Web; - -/// -/// Get search URLs for various websites -/// -[SuppressMessage("Design", "CA1055:URI return values should not be strings", Justification = "Semantic Kernel operates on strings")] -public sealed class SearchUrlSkill -{ - /** - * Amazon Search URLs - */ - /// - /// Get search URL for Amazon - /// - [SKFunction, Description("Return URL for Amazon search query")] - public string AmazonSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.amazon.com/s?k={encoded}"; - } - - /** - * Bing Search URLs - */ - /// - /// Get search URL for Bing - /// - [SKFunction, Description("Return URL for Bing search query.")] - public string BingSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.bing.com/search?q={encoded}"; - } - - /// - /// Get search URL for Bing Images - /// - [SKFunction, Description("Return URL for Bing Images search query.")] - public string BingImagesSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.bing.com/images/search?q={encoded}"; - } - - /// - /// Get search URL for Bing Maps - /// - [SKFunction, Description("Return URL for Bing Maps search query.")] - public string BingMapsSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.bing.com/maps?q={encoded}"; - } - - /// - /// Get search URL for Bing Shopping - /// - [SKFunction, Description("Return URL for Bing Shopping search query.")] - public string BingShoppingSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.bing.com/shop?q={encoded}"; - } - - /// - /// Get search URL for Bing News - /// - [SKFunction, Description("Return URL for Bing News search query.")] - public string BingNewsSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.bing.com/news/search?q={encoded}"; - } - - /// - /// Get search URL for Bing Travel - /// - [SKFunction, Description("Return URL for Bing Travel search query.")] - public string BingTravelSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.bing.com/travel/search?q={encoded}"; - } - - /** - * Facebook Search URLs - */ - /// - /// Get search URL for Facebook - /// - [SKFunction, Description("Return URL for Facebook search query.")] - public string FacebookSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.facebook.com/search/top/?q={encoded}"; - } - - /** - * GitHub Search URLs - */ - /// - /// Get search URL for GitHub - /// - [SKFunction, Description("Return URL for GitHub search query.")] - public string GitHubSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://github.com/search?q={encoded}"; - } - - /** - * LinkedIn Search URLs - */ - /// - /// Get search URL for LinkedIn - /// - [SKFunction, Description("Return URL for LinkedIn search query.")] - public string LinkedInSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://www.linkedin.com/search/results/index/?keywords={encoded}"; - } - - /** - * Twitter Search URLs - */ - /// - /// Get search URL for Twitter - /// - [SKFunction, Description("Return URL for Twitter search query.")] - public string TwitterSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://twitter.com/search?q={encoded}"; - } - - /** - * Wikipedia Search URLs - */ - /// - /// Get search URL for Wikipedia - /// - [SKFunction, Description("Return URL for Wikipedia search query.")] - public string WikipediaSearchUrl([Description("Text to search for")] string query) - { - string encoded = UrlEncoder.Default.Encode(query); - return $"https://wikipedia.org/w/index.php?search={encoded}"; - } -} diff --git a/dotnet/src/Skills/Skills.Web/Skills.Web.csproj b/dotnet/src/Skills/Skills.Web/Skills.Web.csproj deleted file mode 100644 index f0a7c1073361..000000000000 --- a/dotnet/src/Skills/Skills.Web/Skills.Web.csproj +++ /dev/null @@ -1,29 +0,0 @@ - - - - - Microsoft.SemanticKernel.Skills.Web - $(AssemblyName) - netstandard2.0 - - - - - - - - Semantic Kernel - Microsoft Bing Connector - Semantic Kernel Web Skill: search the web, download files, etc. - - - - - - - - - - - - - \ No newline at end of file diff --git a/dotnet/src/Skills/Skills.Web/WebFileDownloadSkill.cs b/dotnet/src/Skills/Skills.Web/WebFileDownloadSkill.cs deleted file mode 100644 index 9b8a7c5bd60e..000000000000 --- a/dotnet/src/Skills/Skills.Web/WebFileDownloadSkill.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Web; - -/// -/// Skill to download web files. -/// -public sealed class WebFileDownloadSkill -{ - /// - /// Skill parameter: where to save file. - /// - public const string FilePathParamName = "filePath"; - - private readonly ILogger _logger; - private readonly HttpClient _httpClient; - - /// - /// Initializes a new instance of the class. - /// - /// An optional logger to log skill-related information. - public WebFileDownloadSkill(ILogger? logger = null) : - this(new HttpClient(NonDisposableHttpClientHandler.Instance, false), logger) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The HTTP client to use for making requests. - /// An optional logger to log skill-related information. - public WebFileDownloadSkill(HttpClient httpClient, ILogger? logger = null) - { - this._httpClient = httpClient; - this._logger = logger ?? NullLogger.Instance; - } - - /// - /// Downloads a file to a local file path. - /// - /// URI of file to download - /// Path where to save file locally - /// The token to use to request cancellation. - /// Task. - /// Thrown when the location where to download the file is not provided - [SKFunction, Description("Downloads a file to local storage")] - public async Task DownloadToFileAsync( - [Description("URL of file to download")] Uri url, - [Description("Path where to save file locally")] string filePath, - CancellationToken cancellationToken = default) - { - this._logger.LogDebug($"{nameof(this.DownloadToFileAsync)} got called"); - - this._logger.LogDebug("Sending GET request for {0}", url); - using HttpResponseMessage response = await this._httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - this._logger.LogDebug("Response received: {0}", response.StatusCode); - - using Stream webStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); - using FileStream outputFileStream = new(Environment.ExpandEnvironmentVariables(filePath), FileMode.Create); - - await webStream.CopyToAsync(outputFileStream, 81920 /*same value used by default*/, cancellationToken).ConfigureAwait(false); - } -} diff --git a/dotnet/src/Skills/Skills.Web/WebSearchEngineSkill.cs b/dotnet/src/Skills/Skills.Web/WebSearchEngineSkill.cs deleted file mode 100644 index ce55bb5356f3..000000000000 --- a/dotnet/src/Skills/Skills.Web/WebSearchEngineSkill.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ComponentModel; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.SkillDefinition; - -namespace Microsoft.SemanticKernel.Skills.Web; - -/// -/// Web search engine skill (e.g. Bing) -/// -public sealed class WebSearchEngineSkill -{ - public const string CountParam = "count"; - public const string OffsetParam = "offset"; - - private readonly IWebSearchEngineConnector _connector; - - public WebSearchEngineSkill(IWebSearchEngineConnector connector) - { - this._connector = connector; - } - - [SKFunction, Description("Perform a web search.")] - public async Task SearchAsync( - [Description("Text to search for")] string query, - [Description("Number of results")] int count = 1, - [Description("Number of results to skip")] int offset = 0, - CancellationToken cancellationToken = default) - { - var results = await this._connector.SearchAsync(query, count, offset, cancellationToken).ConfigureAwait(false); - if (!results.Any()) - { - throw new InvalidOperationException("Failed to get a response from the web search engine."); - } - - return count == 1 - ? results.FirstOrDefault() ?? string.Empty - : JsonSerializer.Serialize(results); - } -} diff --git a/python/.env.example b/python/.env.example index a1989cae275f..2e7a633e31ba 100644 --- a/python/.env.example +++ b/python/.env.example @@ -5,6 +5,12 @@ AZURE_OPENAI_ENDPOINT="" AZURE_OPENAI_API_KEY="" AZURE_COGNITIVE_SEARCH_ENDPOINT="" AZURE_COGNITIVE_SEARCH_ADMIN_KEY="" +MONGODB_ATLAS_CONNECTION_STRING="" PINECONE_API_KEY="" PINECONE_ENVIRONMENT="" POSTGRES_CONNECTION_STRING="" +WEAVIATE_URL="" +WEAVIATE_API_KEY="" +GOOGLE_PALM_API_KEY="" +GOOGLE_SEARCH_ENGINE_ID="" +REDIS_CONNECTION_STRING="" diff --git a/python/.vscode/settings.json b/python/.vscode/settings.json index 17d438492690..4e6c0ad8e15e 100644 --- a/python/.vscode/settings.json +++ b/python/.vscode/settings.json @@ -7,7 +7,7 @@ "editor.formatOnType": true, "editor.formatOnSave": true, "editor.formatOnPaste": true, - "python.formatting.provider": "autopep8", + "python.formatting.provider": "black", "python.formatting.autopep8Args": [ "--max-line-length=160" ], @@ -18,4 +18,9 @@ "OPENAI", "skfunction" ], -} + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, +} \ No newline at end of file diff --git a/python/DEV_SETUP.md b/python/DEV_SETUP.md index e7122b663023..b851b3b2b8dc 100644 --- a/python/DEV_SETUP.md +++ b/python/DEV_SETUP.md @@ -1,16 +1,18 @@ +# Dev Setup + This document describes how to setup your environment with Python and Poetry, if you're working on new features or a bug fix for Semantic Kernel, or simply want to run the tests included. -# LLM setup +## LLM setup Make sure you have an -[Open AI API Key](https://openai.com/api/) or -[Azure Open AI service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=rest-api) +[OpenAI API Key](https://openai.com/product/) or +[Azure OpenAI service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=rest-api) Copy those keys into a `.env` file (see the `.env.example` file): -``` +```bash OPENAI_API_KEY="" OPENAI_ORG_ID="" AZURE_OPENAI_DEPLOYMENT_NAME="" @@ -21,15 +23,17 @@ AZURE_OPENAI_API_KEY="" We suggest adding a copy of the `.env` file under these folders: - [python/tests](tests) -- [samples/notebooks/python](../samples/notebooks/python). +- [./notebooks](./notebooks). -# System setup +## System setup To get started, you'll need VSCode and a local installation of Python 3.8+. You can run: +```python python3 --version ; pip3 --version ; code -v +``` to verify that you have the required dependencies. @@ -53,7 +57,7 @@ you may need to run `~/.local/bin/poetry install` and `~/.local/bin/poetry shell instead. You can fix this by adding `export PATH="$HOME/.local/bin:$PATH"` to your `~/.bashrc` and closing/re-opening the terminal.\_ -# Using Poetry +## Using Poetry Poetry allows to use SK from the local files, without worrying about paths, as if you had SK pip package installed. @@ -79,7 +83,7 @@ poetry install poetry shell ``` -# VSCode Setup +## VSCode Setup Open any of the `.py` files in the project and run the `Python: Select Interpreter` command from the command palette. Make sure the virtual env (venv) created by @@ -89,35 +93,188 @@ The python you're looking for should be under `~/.cache/pypoetry/virtualenvs/sem If prompted, install `black` and `flake8` (if VSCode doesn't find those packages, it will prompt you to install them). -# Tests +## Tests You can run the unit tests under the [tests/unit](tests/unit/) folder. +```bash cd python poetry install poetry run pytest tests/unit +``` You can run the integration tests under the [tests/integration](tests/integration/) folder. +```bash cd python poetry install poetry run pytest tests/integration +``` You can also run all the tests together under the [tests](tests/) folder. +```bash cd python poetry install poetry run pytest tests +``` + +## Tools and scripts + +## Pydantic and Serialization + +[Pydantic Documentation](https://docs.pydantic.dev/1.10/) + +### Overview + +This section describes how one can enable serialization for their class using Pydantic. + +IMPORTANT: This document (and SemanticKernel) currently use Pydantic 1.x. When SK is upgraded +to use Pydantic 2.x, this document will be upgraded accordingly. + +### Terminology + +There are 3 types of classes you need to be aware of when enabling serialization with Pydantic: + +1. Classes which contain no data - examples are Protocols, ABC subclasses and any other classes + that don't contain any data that needs to be serialized. +2. Classes which contain data that need to be serialized, but don't contain any generic classes. +3. Classes which contain data that need to be serialized, AND contain generic classes. + +### Upgrading existing classes to use Pydantic + +#### Classes without any data + +Let's take the following classes as examples - 1 ABC, 1 Protocol, and 1 class that only contains +data that doesn't need to be serialized. + +```python +class A(Protocol): + def some_method(self, *args, **kwargs): ... + +class B(ABC): + def some_method(self, *args, **kwargs): ... + +class C: + def __init__(self): + # IMPORTANT: These variables are NOT being passed into the initializer + # so they don't need to be serialized. If the are though, you'll have + # to treat this as a class that contains data that needs to be serialized + self._a = ... +``` + +For `Protocol` subclasses, nothing needs to be done, and they can be left as is. + +For the remaining types, SemanticKernel provides a class named `PydanticField`. Subclassing +from this field is sufficient to have these types of classes as valid Pydantic fields, and allows +any class using them as attributes to be serialized. + +```python +from semantic_kernel.sk_pydantic import PydanticField + +class B(PydanticField): ... # correct, B is still an ABC because PydanticField subclasses ABC +class B(PydanticField, ABC): ... # Also correct +class B(ABC, PydanticField): ... # ERROR: Python cannot find a valid super class ordering. + +class C(PydanticField): ... # No other changes needed +``` -# Tools and scripts +The classes B and C can now be used as valid Pydantic Field annotations. + +```python +from pydantic import BaseModel + +class MyModel(BaseModel): + b: B + c: C +``` + +Class A can only be used as a Pydantic Field annotation for a Pydantic BaseModel subclass +which is configured to allow arbitrary field types like so: + +```python +from pydantic import BaseModel +class IncorrectModel(BaseModel): + a: A # Pydantic error + +class CorrectModel(BaseModel): + a: A # Okay + class Config: # Configuration that tells Pydantic to allow field types that it can't serialize + arbitrary_types_allowed = True +``` + +#### Classes with data, but no Generic types that need to be serialized + +If your class has any data that needs to be serialized, but the field annotation for that data type +in your class is not a Generic type, this section applies to you. + +Let's take the following example: + +```python +class A: + def __init__(self, a: int, b: float, c: List[float], d: dict[str, tuple[float, str]] = {}): + # Since a, b, c and d are needed to initialize this class, they need to be serialized + # if can be serialized. + # Although a, b, c and d are builtin python types, any valid pydantic field can be used + # here. This includes the classes defined in the previous category. + self.a = a + self.b = b + self.c = c + self.d = d +``` + +You would convert this to a Pydantic class by subclassing from the `SKBaseModel` class. + +```python +from pydantic import Field +from semantic_kernel.sk_pydantic import SKBaseModel + +class A(SKBaseModel): + # The notation for the fields is similar to dataclasses. + a: int + b: float + c: List[float] + # Only, instead of using dataclasses.field, you would use pydantic.Field + d: dict[str, tuple[flost, str]] = Field(default_factory=dict) +``` + +#### Classes with data that need to be serialized, and some of them are Generic types + +Let's take the following example: + +```python +from typing import TypeVar + +T1 = TypeVar("T1") +T2 = TypeVar("T2", bound=) + +class A: + def __init__(a: int, b: T1, c: T2): + self.a = a + self.b = b + self.c = c +``` + +You can uses the `SKGenericModel` to convert these to pydantic serializable classes. + +```python +from typing import Generic + +from semantic_kernel.sk_pydantic import SKGenericModel + +class A(SKGenericModel, Generic[T1, T2]): + # T1 and T2 must be specified in the Generic argument otherwise, pydantic will + # NOT be able to serialize this class + a: int + b: T1 + c: T2 +``` ## Pipeline checks To run the same checks that run during the GitHub Action build, you can use this command, from the [python](../python) folder: +```bash poetry run pre-commit run -c .conf/.pre-commit-config.yaml -a - -## Running ruff - - poetry run ruff check . +``` diff --git a/python/README.md b/python/README.md index da5be102bb73..381a63047592 100644 --- a/python/README.md +++ b/python/README.md @@ -4,14 +4,13 @@ Install the latest package: python -m pip install --upgrade semantic-kernel - # AI Services ## OpenAI / Azure OpenAI API keys Make sure you have an -[Open AI API Key](https://openai.com/api/) or -[Azure Open AI service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=rest-api) +[OpenAI API Key](https://openai.com/product/) or +[Azure OpenAI service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=rest-api) Copy those keys into a `.env` file (see the `.env.example` file): @@ -27,17 +26,17 @@ AZURE_OPENAI_API_KEY="" ```python import semantic_kernel as sk -from semantic_kernel.connectors.ai.open_ai import OpenAITextCompletion, AzureTextCompletion +from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion, AzureChatCompletion kernel = sk.Kernel() # Prepare OpenAI service using credentials stored in the `.env` file api_key, org_id = sk.openai_settings_from_dot_env() -kernel.add_text_completion_service("dv", OpenAITextCompletion("text-davinci-003", api_key, org_id)) +kernel.add_chat_service("chat-gpt", OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id)) # Alternative using Azure: # deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env() -# kernel.add_text_completion_service("dv", AzureTextCompletion(deployment, endpoint, api_key)) +# kernel.add_chat_service("dv", AzureChatCompletion(deployment, endpoint, api_key)) # Wrap your prompt in a function prompt = kernel.create_semantic_function(""" @@ -92,18 +91,18 @@ get started with the Semantic Kernel. Python notebooks: -* [Getting started with Semantic Kernel](../samples/notebooks/python/00-getting-started.ipynb) -* [Loading and configuring Semantic Kernel](../samples/notebooks/python/01-basic-loading-the-kernel.ipynb) -* [Running AI prompts from file](../samples/notebooks/python/02-running-prompts-from-file.ipynb) -* [Creating Semantic Functions at runtime (i.e. inline functions)](../samples/notebooks/python/03-semantic-function-inline.ipynb) -* [Using Context Variables to Build a Chat Experience](../samples/notebooks/python/04-context-variables-chat.ipynb) -* [Introduction to planners](../samples/notebooks/python/05-using-the-planner.ipynb) -* [Building Memory with Embeddings](../samples/notebooks/python/06-memory-and-embeddings.ipynb) -* [Using Hugging Face for Skills](../samples/notebooks/python/07-hugging-face-for-skills.ipynb) -* [Combining native functions and semantic functions](../samples/notebooks/python/08-native-function-inline.ipynb) -* [Groundedness Checking with Semantic Kernel](../samples/notebooks/python/09-groundedness-checking.ipynb) -* [Returning multiple results per prompt](../samples/notebooks/python/10-multiple-results-per-prompt.ipynb) -* [Streaming completions with Semantic Kernel](../samples/notebooks/python/11-streaming-completions.ipynb) +- [Getting started with Semantic Kernel](./notebooks/00-getting-started.ipynb) +- [Loading and configuring Semantic Kernel](./notebooks/01-basic-loading-the-kernel.ipynb) +- [Running AI prompts from file](./notebooks/02-running-prompts-from-file.ipynb) +- [Creating Semantic Functions at runtime (i.e. inline functions)](./notebooks/03-semantic-function-inline.ipynb) +- [Using Context Variables to Build a Chat Experience](./notebooks/04-context-variables-chat.ipynb) +- [Introduction to planners](./notebooks/05-using-the-planner.ipynb) +- [Building Memory with Embeddings](./notebooks/06-memory-and-embeddings.ipynb) +- [Using Hugging Face for Skills](./notebooks/07-hugging-face-for-skills.ipynb) +- [Combining native functions and semantic functions](./notebooks/08-native-function-inline.ipynb) +- [Groundedness Checking with Semantic Kernel](./notebooks/09-groundedness-checking.ipynb) +- [Returning multiple results per prompt](./notebooks/10-multiple-results-per-prompt.ipynb) +- [Streaming completions with Semantic Kernel](./notebooks/11-streaming-completions.ipynb) # SK Frequently Asked Questions diff --git a/samples/notebooks/python/.env.example b/python/notebooks/.env.example similarity index 100% rename from samples/notebooks/python/.env.example rename to python/notebooks/.env.example diff --git a/python/notebooks/00-getting-started.ipynb b/python/notebooks/00-getting-started.ipynb new file mode 100644 index 000000000000..480f511b25e4 --- /dev/null +++ b/python/notebooks/00-getting-started.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup\n", + "\n", + "**Step 1**: Import Semantic Kernel SDK from pypi.org" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!python -m pip install semantic-kernel==0.3.10.dev0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import semantic_kernel as sk\n", + "\n", + "kernel = sk.Kernel()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Option 1: using OpenAI\n", + "\n", + "**Step 2**: Add your [OpenAI Key](https://openai.com/product/) key to a `.env` file in the same folder (org Id only if you have multiple orgs):\n", + "\n", + "```\n", + "OPENAI_API_KEY=\"sk-...\"\n", + "OPENAI_ORG_ID=\"\"\n", + "```\n", + "\n", + "and add OpenAI Chat Completion to the kernel:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion\n", + "\n", + "api_key, org_id = sk.openai_settings_from_dot_env()\n", + "\n", + "kernel.add_chat_service(\"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Option 2: using Azure OpenAI\n", + "\n", + "**Step 2**: Add your [Azure Open AI Service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=programming-language-studio) settings to a `.env` file in the same folder:\n", + "\n", + "```\n", + "AZURE_OPENAI_API_KEY=\"...\"\n", + "AZURE_OPENAI_ENDPOINT=\"https://...\"\n", + "AZURE_OPENAI_DEPLOYMENT_NAME=\"...\"\n", + "```\n", + "\n", + "and add Azure OpenAI Chat Completion to the kernel:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion\n", + "\n", + "deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", + "\n", + "kernel.add_chat_service(\"chat_completion\", AzureChatCompletion(deployment, endpoint, api_key))\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run a Semantic Function\n", + "\n", + "**Step 3**: Load a Skill and run a semantic function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "skill = kernel.import_semantic_skill_from_directory(\"../../samples/skills\", \"FunSkill\")\n", + "joke_function = skill[\"Joke\"]\n", + "\n", + "print(joke_function(\"time travel to dinosaur age\"))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/notebooks/01-basic-loading-the-kernel.ipynb b/python/notebooks/01-basic-loading-the-kernel.ipynb new file mode 100644 index 000000000000..8de0a12ca9fd --- /dev/null +++ b/python/notebooks/01-basic-loading-the-kernel.ipynb @@ -0,0 +1,174 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic Loading of the Kernel" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To run the notebooks we recommend using Poetry and starting a shell with a virtual environment\n", + "prepared to use SK. \n", + "\n", + "See [DEV_SETUP.md](../../python/DEV_SETUP.md) for more information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!python -m pip install semantic-kernel==0.3.10.dev0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import semantic_kernel as sk\n", + "from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can instantiate the kernel in a few ways, depending on your use case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Simple instance\n", + "kernel_1 = sk.Kernel()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Instance with a custom logger\n", + "my_logger = sk.NullLogger()\n", + "kernel_2 = sk.Kernel(log=my_logger)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When using the kernel for AI requests, the kernel needs some settings like URL and credentials to the AI models.\n", + "\n", + "The SDK currently supports OpenAI and Azure OpenAI, other services will be added over time.\n", + "\n", + "If you need an Azure OpenAI key, go [here](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/quickstart?pivots=rest-api)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kernel = sk.Kernel()\n", + "\n", + "kernel.add_chat_service( # We are adding a text service\n", + " \"Azure_curie\", # The alias we can use in prompt templates' config.json\n", + " AzureChatCompletion(\n", + " \"my-finetuned-Curie\", # Azure OpenAI *Deployment name*\n", + " \"https://contoso.openai.azure.com/\", # Azure OpenAI *Endpoint*\n", + " \"...your Azure OpenAI Key...\" # Azure OpenAI *Key*\n", + " )\n", + ")\n", + "\n", + "kernel.add_chat_service( # We are adding a text service\n", + " \"OpenAI_chat_gpt\", # The alias we can use in prompt templates' config.json\n", + " OpenAIChatCompletion(\n", + " \"gpt-3.5-turbo\", # OpenAI Model Name\n", + " \"...your OpenAI API Key...\", # OpenAI API key\n", + " \"...your OpenAI Org ID...\" # *optional* OpenAI Organization ID\n", + " )\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When working with multiple services and multiple models, the **first service** defined\n", + "is also the \"**default**\" used in these scenarios:\n", + "\n", + "* a prompt configuration doesn't specify which AI service to use\n", + "* a prompt configuration requires a service unknown to the kernel\n", + "\n", + "The default can be set and changed programmatically:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kernel.set_default_text_completion_service(\"Azure_curie\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great, now that you're familiar with setting up the Semantic Kernel, let's see [how we can use it to run prompts](02-running-prompts-from-file.ipynb)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + }, + "polyglot_notebook": { + "kernelInfo": { + "items": [ + { + "aliases": [ + "frontend" + ], + "name": "vscode" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/notebooks/02-running-prompts-from-file.ipynb b/python/notebooks/02-running-prompts-from-file.ipynb new file mode 100644 index 000000000000..c41dc0294caf --- /dev/null +++ b/python/notebooks/02-running-prompts-from-file.ipynb @@ -0,0 +1,198 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "692e361b", + "metadata": {}, + "source": [ + "# How to run a semantic skills from file\n", + "Now that you're familiar with Kernel basics, let's see how the kernel allows you to run Semantic Skills and Semantic Functions stored on disk. \n", + "\n", + "A Semantic Skill is a collection of Semantic Functions, where each function is defined with natural language that can be provided with a text file. \n", + "\n", + "Refer to our [glossary](https://github.com/microsoft/semantic-kernel/blob/main/docs/GLOSSARY.md) for an in-depth guide to the terms.\n", + "\n", + "The repository includes some examples under the [samples](https://github.com/microsoft/semantic-kernel/tree/main/samples) folder.\n", + "\n", + "For instance, [this](../../skills/FunSkill/Joke/skprompt.txt) is the **Joke function** part of the **FunSkill skill**:" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "f3ce1efe", + "metadata": {}, + "source": [ + "```\n", + "WRITE EXACTLY ONE JOKE or HUMOROUS STORY ABOUT THE TOPIC BELOW.\n", + "JOKE MUST BE:\n", + "- G RATED\n", + "- WORKPLACE/FAMILY SAFE\n", + "NO SEXISM, RACISM OR OTHER BIAS/BIGOTRY.\n", + "BE CREATIVE AND FUNNY. I WANT TO LAUGH.\n", + "+++++\n", + "{{$input}}\n", + "+++++\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "afdb96d6", + "metadata": {}, + "source": [ + "Note the special **`{{$input}}`** token, which is a variable that is automatically passed when invoking the function, commonly referred to as a \"function parameter\". \n", + "\n", + "We'll explore later how functions can accept multiple variables, as well as invoke other functions." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c3bd5134", + "metadata": {}, + "source": [ + "\n", + "In the same folder you'll notice a second [config.json](../../skills/FunSkill/Joke/config.json) file. The file is optional, and is used to set some parameters for large language models like Temperature, TopP, Stop Sequences, etc.\n", + "\n", + "```\n", + "{\n", + " \"schema\": 1,\n", + " \"type\": \"completion\",\n", + " \"description\": \"Generate a funny joke\",\n", + " \"completion\": {\n", + " \"max_tokens\": 500,\n", + " \"temperature\": 0.5,\n", + " \"top_p\": 0.5\n", + " }\n", + "}\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "384ff07f", + "metadata": {}, + "source": [ + "Given a semantic function defined by these files, this is how to load and use a file based semantic function.\n", + "\n", + "Load and configure the kernel, as usual, loading also the AI service settings defined in the [Setup notebook](00-getting-started.ipynb):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "365cfc01", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m pip install semantic-kernel==0.3.10.dev0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0062a24", + "metadata": {}, + "outputs": [], + "source": [ + "import semantic_kernel as sk\n", + "from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion\n", + "\n", + "kernel = sk.Kernel()\n", + "\n", + "useAzureOpenAI = False\n", + "\n", + "# Configure AI service used by the kernel\n", + "if useAzureOpenAI:\n", + " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\"chat_completion\", AzureChatCompletion(deployment, endpoint, api_key))\n", + "else:\n", + " api_key, org_id = sk.openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "fd5ff1f4", + "metadata": {}, + "source": [ + "Import the skill and all its functions:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56ee184d", + "metadata": {}, + "outputs": [], + "source": [ + "# note: using skills from the samples folder\n", + "skills_directory = \"../../samples/skills\"\n", + "\n", + "funFunctions = kernel.import_semantic_skill_from_directory(skills_directory, \"FunSkill\")\n", + "\n", + "jokeFunction = funFunctions[\"Joke\"]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "edd99fa0", + "metadata": {}, + "source": [ + "How to use the skill functions, e.g. generate a joke about \"*time travel to dinosaur age*\":" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6effe63b", + "metadata": {}, + "outputs": [], + "source": [ + "result = jokeFunction(\"time travel to dinosaur age\")\n", + "\n", + "print(result)\n", + "\n", + "# You can also invoke functions asynchronously\n", + "# result = await jokeFunction.invoke_async(\"time travel to dinosaur age\")\n", + "# print(result)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "2281a1fc", + "metadata": {}, + "source": [ + "Great, now that you know how to load a skill from disk, let's show how you can [create and run a semantic function inline.](./03-semantic-function-inline.ipynb)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python/notebooks/03-semantic-function-inline.ipynb b/python/notebooks/03-semantic-function-inline.ipynb new file mode 100644 index 000000000000..860de47a5aaa --- /dev/null +++ b/python/notebooks/03-semantic-function-inline.ipynb @@ -0,0 +1,270 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "3c93ac5b", + "metadata": {}, + "source": [ + "# Running Semantic Functions Inline" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "ebcabb91", + "metadata": {}, + "source": [] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "40201641", + "metadata": {}, + "source": [ + "The [previous notebook](./02-running-prompts-from-file.ipynb)\n", + "showed how to define a semantic function using a prompt template stored on a file.\n", + "\n", + "In this notebook, we'll show how to use the Semantic Kernel to define functions inline with your python code. This can be useful in a few scenarios:\n", + "\n", + "* Dynamically generating the prompt using complex rules at runtime\n", + "* Writing prompts by editing Python code instead of TXT files.\n", + "* Easily creating demos, like this document\n", + "\n", + "Prompt templates are defined using the SK template language, which allows to reference variables and functions. Read [this doc](https://aka.ms/sk/howto/configurefunction) to learn more about the design decisions for prompt templating. \n", + "\n", + "For now we'll use only the `{{$input}}` variable, and see more complex templates later.\n", + "\n", + "Almost all semantic function prompts have a reference to `{{$input}}`, which is the default way\n", + "a user can import content from the context variables." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "d90b0c13", + "metadata": {}, + "source": [ + "Prepare a semantic kernel instance first, loading also the AI service settings defined in the [Setup notebook](00-getting-started.ipynb):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1da651d4", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m pip install semantic-kernel==0.3.10.dev0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3712b7c3", + "metadata": {}, + "outputs": [], + "source": [ + "import semantic_kernel as sk\n", + "from semantic_kernel.connectors.ai.open_ai import AzureTextCompletion, OpenAITextCompletion\n", + "\n", + "kernel = sk.Kernel()\n", + "\n", + "useAzureOpenAI = False\n", + "\n", + "# Configure AI service used by the kernel\n", + "if useAzureOpenAI:\n", + " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", + " kernel.add_text_completion_service(\"dv\", AzureTextCompletion(deployment, endpoint, api_key))\n", + "else:\n", + " api_key, org_id = sk.openai_settings_from_dot_env()\n", + " kernel.add_text_completion_service(\"dv\", OpenAITextCompletion(\"text-davinci-003\", api_key, org_id))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "589733c5", + "metadata": {}, + "source": [ + "Let's use a prompt to create a semantic function used to summarize content, allowing for some creativity and a sufficient number of tokens.\n", + "\n", + "The function will take in input the text to summarize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae29c207", + "metadata": {}, + "outputs": [], + "source": [ + "prompt = \"\"\"{{$input}}\n", + "Summarize the content above.\n", + "\"\"\"\n", + "\n", + "summarize = kernel.create_semantic_function(prompt, max_tokens=2000, temperature=0.2, top_p=0.5)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "f26b90c4", + "metadata": {}, + "source": [ + "Set up some content to summarize, here's an extract about Demo, an ancient Greek poet, taken from Wikipedia (https://en.wikipedia.org/wiki/Demo_(ancient_Greek_poet)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "314557fb", + "metadata": {}, + "outputs": [], + "source": [ + "input_text = \"\"\"\n", + "Demo (ancient Greek poet)\n", + "From Wikipedia, the free encyclopedia\n", + "Demo or Damo (Greek: Δεμώ, Δαμώ; fl. c. AD 200) was a Greek woman of the Roman period, known for a single epigram, engraved upon the Colossus of Memnon, which bears her name. She speaks of herself therein as a lyric poetess dedicated to the Muses, but nothing is known of her life.[1]\n", + "Identity\n", + "Demo was evidently Greek, as her name, a traditional epithet of Demeter, signifies. The name was relatively common in the Hellenistic world, in Egypt and elsewhere, and she cannot be further identified. The date of her visit to the Colossus of Memnon cannot be established with certainty, but internal evidence on the left leg suggests her poem was inscribed there at some point in or after AD 196.[2]\n", + "Epigram\n", + "There are a number of graffiti inscriptions on the Colossus of Memnon. Following three epigrams by Julia Balbilla, a fourth epigram, in elegiac couplets, entitled and presumably authored by \"Demo\" or \"Damo\" (the Greek inscription is difficult to read), is a dedication to the Muses.[2] The poem is traditionally published with the works of Balbilla, though the internal evidence suggests a different author.[1]\n", + "In the poem, Demo explains that Memnon has shown her special respect. In return, Demo offers the gift for poetry, as a gift to the hero. At the end of this epigram, she addresses Memnon, highlighting his divine status by recalling his strength and holiness.[2]\n", + "Demo, like Julia Balbilla, writes in the artificial and poetic Aeolic dialect. The language indicates she was knowledgeable in Homeric poetry—'bearing a pleasant gift', for example, alludes to the use of that phrase throughout the Iliad and Odyssey.[a][2] \n", + "\"\"\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "bf0f2330", + "metadata": {}, + "source": [ + "...and run the summary function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b0e3b0c", + "metadata": {}, + "outputs": [], + "source": [ + "# If needed, async is available too: summary = await summarize.invoke_async(input_text)\n", + "summary = summarize(input_text)\n", + "\n", + "print(summary)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "1c2c1262", + "metadata": {}, + "source": [ + "# Using ChatCompletion for Semantic Skills" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "29b59b28", + "metadata": {}, + "source": [ + "You can also use chat completion models (like `gpt-35-turbo` and `gpt4`) for creating skills. Normally you would have to tweak the API to accommodate for a system and user role, but SK abstracts that away for you by using `kernel.add_chat_service` and `AzureChatCompletion` or `OpenAIChatCompletion`" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "4777f447", + "metadata": {}, + "source": [ + "Here's one more example of how to write an inline Semantic Function that gives a TLDR for a piece of text using a ChatCompletion model\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5886aeb", + "metadata": {}, + "outputs": [], + "source": [ + "import semantic_kernel as sk\n", + "from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion\n", + "\n", + "kernel = sk.Kernel()\n", + "\n", + "useAzureOpenAI = False\n", + "\n", + "# Configure AI service used by the kernel\n", + "if useAzureOpenAI:\n", + " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\n", + " \"chat_completion\",\n", + " AzureChatCompletion(deployment, endpoint, api_key),\n", + " )\n", + "else:\n", + " api_key, org_id = sk.openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\n", + " \"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id)\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea8128c8", + "metadata": {}, + "outputs": [], + "source": [ + "sk_prompt = \"\"\"\n", + "{{$input}}\n", + "\n", + "Give me the TLDR in 5 words or less.\n", + "\"\"\"\n", + "\n", + "text = \"\"\"\n", + " 1) A robot may not injure a human being or, through inaction,\n", + " allow a human being to come to harm.\n", + "\n", + " 2) A robot must obey orders given it by human beings except where\n", + " such orders would conflict with the First Law.\n", + "\n", + " 3) A robot must protect its own existence as long as such protection\n", + " does not conflict with the First or Second Law.\n", + "\"\"\"\n", + "\n", + "tldr_function = kernel.create_semantic_function(sk_prompt, max_tokens=200, temperature=0, top_p=0.5)\n", + "\n", + "summary = tldr_function(text)\n", + "\n", + "print(f\"Output: {summary}\") # Output: Robots must not harm humans." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python/notebooks/04-context-variables-chat.ipynb b/python/notebooks/04-context-variables-chat.ipynb new file mode 100644 index 000000000000..e8a5303a8506 --- /dev/null +++ b/python/notebooks/04-context-variables-chat.ipynb @@ -0,0 +1,272 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "fde98ddf", + "metadata": {}, + "source": [ + "# Creating a basic chat experience with context variables\n", + "\n", + "In this example, we show how you can build a simple chat bot by sending and updating context with your requests. \n", + "\n", + "We introduce the Context Variables object which in this demo functions similarly as a key-value store that you can use when running the kernel.\n", + "\n", + "The context is local (i.e. in your computer's RAM) and not persisted anywhere beyond the life of this Jupyter session.\n", + "\n", + "In future examples, we will show how to persist the context on disk so that you can bring it into your applications. \n", + "\n", + "In this chat scenario, as the user talks back and forth with the bot, the context gets populated with the history of the conversation. During each new run of the kernel, the context can provide the AI with its variables' content. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92f69b34", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m pip install semantic-kernel==0.3.10.dev0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "68301108", + "metadata": {}, + "outputs": [], + "source": [ + "import semantic_kernel as sk\n", + "from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion\n", + "\n", + "kernel = sk.Kernel()\n", + "\n", + "useAzureOpenAI = False\n", + "\n", + "# Configure AI service used by the kernel\n", + "if useAzureOpenAI:\n", + " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\"chat_completion\", AzureChatCompletion(deployment, endpoint, api_key))\n", + "else:\n", + " api_key, org_id = sk.openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id))\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "7971783d", + "metadata": {}, + "source": [ + "Let's define a prompt outlining a dialogue chat bot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e84a05fc", + "metadata": {}, + "outputs": [], + "source": [ + "sk_prompt = \"\"\"\n", + "ChatBot can have a conversation with you about any topic.\n", + "It can give explicit instructions or say 'I don't know' if it does not have an answer.\n", + "\n", + "{{$history}}\n", + "User: {{$user_input}}\n", + "ChatBot: \"\"\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "61716b16", + "metadata": {}, + "source": [ + "Register your semantic function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3e4b160", + "metadata": {}, + "outputs": [], + "source": [ + "chat_function = kernel.create_semantic_function(sk_prompt, \"ChatBot\", max_tokens=2000, temperature=0.7, top_p=0.5)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "6e8a676f", + "metadata": {}, + "source": [ + "Initialize your context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4be7394", + "metadata": {}, + "outputs": [], + "source": [ + "context = kernel.create_new_context()\n", + "context[\"history\"] = \"\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "4ce7c497", + "metadata": {}, + "source": [ + "Chat with the Bot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ec41eb8", + "metadata": {}, + "outputs": [], + "source": [ + "context[\"user_input\"] = \"Hi, I'm looking for book suggestions\"\n", + "bot_answer = await chat_function.invoke_async(context=context)\n", + "print(bot_answer)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "a5b03748", + "metadata": {}, + "source": [ + "Update the history with the output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f50f517d", + "metadata": {}, + "outputs": [], + "source": [ + "context[\"history\"] += f\"\\nUser: {context['user_input']}\\nChatBot: {bot_answer}\\n\"\n", + "print(context[\"history\"])" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "23a2eb02", + "metadata": {}, + "source": [ + "Keep Chatting!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c59efe45", + "metadata": {}, + "outputs": [], + "source": [ + "async def chat(input_text: str) -> None:\n", + " # Save new message in the context variables\n", + " print(f\"User: {input_text}\")\n", + " context[\"user_input\"] = input_text\n", + "\n", + " # Process the user message and get an answer\n", + " answer = await chat_function.invoke_async(context=context)\n", + "\n", + " # Show the response\n", + " print(f\"ChatBot: {answer}\")\n", + "\n", + " # Append the new interaction to the chat history\n", + " context[\"history\"] += f\"\\nUser: {input_text}\\nChatBot: {answer}\\n\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06ee244e", + "metadata": {}, + "outputs": [], + "source": [ + "await chat(\"I love history and philosophy, I'd like to learn something new about Greece, any suggestion?\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82be4e7e", + "metadata": {}, + "outputs": [], + "source": [ + "await chat(\"that sounds interesting, what is it about?\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82fe0139", + "metadata": {}, + "outputs": [], + "source": [ + "await chat(\"if I read that book, what exactly will I learn about Greek history?\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55b3a9f2", + "metadata": {}, + "outputs": [], + "source": [ + "await chat(\"could you list some more books I could read about this topic?\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c30bac97", + "metadata": {}, + "source": [ + "After chatting for a while, we have built a growing history, which we are attaching to each prompt and which contains the full conversation. Let's take a look!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e34ae55", + "metadata": {}, + "outputs": [], + "source": [ + "print(context[\"history\"])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python/notebooks/05-using-the-planner.ipynb b/python/notebooks/05-using-the-planner.ipynb new file mode 100644 index 000000000000..44f6e44ec70c --- /dev/null +++ b/python/notebooks/05-using-the-planner.ipynb @@ -0,0 +1,662 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "99a80181", + "metadata": {}, + "source": [ + "# Introduction to the Planner\n", + "\n", + "The Planner is one of the fundamental concepts of the Semantic Kernel.\n", + "\n", + "It makes use of the collection of native and semantic functions that have been registered to the kernel and using AI, will formulate a plan to execute the given ask.\n", + "\n", + "From our own testing, planner works best with more powerful models like `gpt4` but sometimes you might get working plans with cheaper models like `gpt-35-turbo`. We encourage you to implement your own versions of the planner and use different models that fit your user needs. \n", + "\n", + "Read more about planner [here](https://aka.ms/sk/concepts/planner)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07eb35d2", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m pip install semantic-kernel==0.3.12.dev0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11e59885", + "metadata": {}, + "outputs": [], + "source": [ + "import semantic_kernel as sk\n", + "from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion, AzureChatCompletion\n", + "\n", + "kernel = sk.Kernel()\n", + "\n", + "useAzureOpenAI = True\n", + "\n", + "# Configure AI backend used by the kernel\n", + "if useAzureOpenAI:\n", + " \n", + " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\"gpt-3.5\", AzureChatCompletion(deployment, endpoint, api_key))\n", + "else:\n", + " api_key, org_id = sk.openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\"gpt-3.5\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id))" + ] + }, + { + "cell_type": "markdown", + "id": "4ff28070", + "metadata": {}, + "source": [ + "## It all begins with an ask" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93bc6103", + "metadata": {}, + "outputs": [], + "source": [ + "ask = \"\"\"\n", + "Tomorrow is Valentine's day. I need to come up with a few date ideas. She speaks French so write it in French.\n", + "Convert the text to uppercase\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "a5d86739", + "metadata": {}, + "source": [ + "### Providing skills to the planner\n", + "The planner needs to know what skills are available to it. Here we'll give it access to the `SummarizeSkill` and `WriterSkill` we have defined on disk. This will include many semantic functions, of which the planner will intelligently choose a subset. \n", + "\n", + "You can also include native functions as well. Here we'll add the TextSkill." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca0e7604", + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.core_skills.text_skill import TextSkill\n", + "\n", + "skills_directory = \"../../samples/skills/\"\n", + "summarize_skill = kernel.import_semantic_skill_from_directory(skills_directory, \"SummarizeSkill\")\n", + "writer_skill = kernel.import_semantic_skill_from_directory(skills_directory, \"WriterSkill\")\n", + "text_skill = kernel.import_skill(TextSkill(), \"TextSkill\")" + ] + }, + { + "cell_type": "markdown", + "id": "deff5675", + "metadata": {}, + "source": [ + "Define your ASK. What do you want the Kernel to do?" + ] + }, + { + "cell_type": "markdown", + "id": "eee6fe7b", + "metadata": {}, + "source": [ + "# Basic Planner" + ] + }, + { + "cell_type": "markdown", + "id": "590a22f2", + "metadata": {}, + "source": [ + " Let's start by taking a look at a basic planner. The `BasicPlanner` produces a JSON-based plan that aims to solve the provided ask sequentially and evaluated in order." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20d35ed0", + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.planning.basic_planner import BasicPlanner\n", + "planner = BasicPlanner()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5697c09", + "metadata": {}, + "outputs": [], + "source": [ + "basic_plan = await planner.create_plan_async(ask, kernel)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b425ba1e", + "metadata": {}, + "outputs": [], + "source": [ + "print(basic_plan.generated_plan)" + ] + }, + { + "cell_type": "markdown", + "id": "0f3a48f8", + "metadata": {}, + "source": [ + "You can see that the Planner took my ask and converted it into an JSON-based plan detailing how the AI would go about solving this task, making use of the skills that the Kernel has available to it.\n", + "\n", + "As you can see in the above plan, the AI has determined which functions to call in order to fulfill the user ask. The output of each step of the plan becomes the input to the next function." + ] + }, + { + "cell_type": "markdown", + "id": "cd4df0c2", + "metadata": {}, + "source": [ + "Let's also define an inline skill and have it be available to the Planner. Be sure to give it a function name and skill name." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54422ba6", + "metadata": {}, + "outputs": [], + "source": [ + "sk_prompt = \"\"\"\n", + "{{$input}}\n", + "\n", + "Rewrite the above in the style of Shakespeare.\n", + "\"\"\"\n", + "shakespeareFunction = kernel.create_semantic_function(sk_prompt, \"shakespeare\", \"ShakespeareSkill\",\n", + " max_tokens=2000, temperature=0.8)" + ] + }, + { + "cell_type": "markdown", + "id": "5057cf9b", + "metadata": {}, + "source": [ + "Let's update our ask using this new skill" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3161dcf", + "metadata": {}, + "outputs": [], + "source": [ + "ask = \"\"\"\n", + "Tomorrow is Valentine's day. I need to come up with a few date ideas.\n", + "She likes Shakespeare so write using his style. She speaks French so write it in French.\n", + "Convert the text to uppercase.\"\"\"\n", + "\n", + "new_plan = await planner.create_plan_async(ask, kernel)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "997462e8", + "metadata": {}, + "outputs": [], + "source": [ + "print(new_plan.generated_plan)" + ] + }, + { + "cell_type": "markdown", + "id": "b67a052e", + "metadata": {}, + "source": [ + "### Executing the plan" + ] + }, + { + "cell_type": "markdown", + "id": "3b839c90", + "metadata": {}, + "source": [ + "Now that we have a plan, let's try to execute it! The Planner has a function called `execute_plan`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9384831a", + "metadata": {}, + "outputs": [], + "source": [ + "results = await planner.execute_plan_async(new_plan, kernel)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9192b186", + "metadata": {}, + "outputs": [], + "source": [ + "print(results)" + ] + }, + { + "cell_type": "markdown", + "id": "e8a9b6b7", + "metadata": {}, + "source": [ + "# The Plan Object Model" + ] + }, + { + "cell_type": "markdown", + "id": "e50f8859", + "metadata": {}, + "source": [ + "To build more advanced planners, we need to introduce a proper Plan object that can contain all the necessary state and information needed for high quality plans.\n", + "\n", + "To see what that object model is, look at (https://github.com/microsoft/semantic-kernel/blob/main/python/semantic_kernel/planning/plan.py)" + ] + }, + { + "cell_type": "markdown", + "id": "0a0cb2a2", + "metadata": {}, + "source": [ + "# Sequential Planner" + ] + }, + { + "cell_type": "markdown", + "id": "a1c66d83", + "metadata": {}, + "source": [ + "The sequential planner is an XML-based step-by-step planner. You can see the prompt used for it here (https://github.com/microsoft/semantic-kernel/blob/main/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/skprompt.txt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2e90624", + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.planning import SequentialPlanner\n", + "planner = SequentialPlanner(kernel)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d537981", + "metadata": {}, + "outputs": [], + "source": [ + "sequential_plan = await planner.create_plan_async(goal=ask)" + ] + }, + { + "cell_type": "markdown", + "id": "ee2f462b", + "metadata": {}, + "source": [ + "To see the steps that the Sequential Planner will take, we can iterate over them and print their descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7007418", + "metadata": {}, + "outputs": [], + "source": [ + "for step in sequential_plan._steps:\n", + " print(step.description, \":\", step._state.__dict__)" + ] + }, + { + "cell_type": "markdown", + "id": "4db5f844", + "metadata": {}, + "source": [ + "Let's ask the sequential planner to execute the plan." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88411884", + "metadata": {}, + "outputs": [], + "source": [ + "result = await sequential_plan.invoke_async()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36d27aa0", + "metadata": {}, + "outputs": [], + "source": [ + "print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "d6487c75", + "metadata": {}, + "source": [ + "# Action Planner" + ] + }, + { + "cell_type": "markdown", + "id": "b045e26b", + "metadata": {}, + "source": [ + "The action planner takes in a list of functions and the goal, and outputs a **single** function to use that is appropriate to meet that goal." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5bfc0b9f", + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.planning import ActionPlanner\n", + "planner = ActionPlanner(kernel)" + ] + }, + { + "cell_type": "markdown", + "id": "53b1f296", + "metadata": {}, + "source": [ + "Let's add more skills to the kernel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc12642a", + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill\n", + "kernel.import_skill(MathSkill(), \"math\")\n", + "kernel.import_skill(FileIOSkill(), \"fileIO\")\n", + "kernel.import_skill(TimeSkill(), \"time\")\n", + "kernel.import_skill(TextSkill(), \"text\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b938dc0e", + "metadata": {}, + "outputs": [], + "source": [ + "ask = \"What is the sum of 110 and 990?\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3aafd268", + "metadata": {}, + "outputs": [], + "source": [ + "plan = await planner.create_plan_async(goal=ask)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42589835", + "metadata": {}, + "outputs": [], + "source": [ + "result = await plan.invoke_async()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc75e7a9", + "metadata": {}, + "outputs": [], + "source": [ + "print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "789b651a", + "metadata": {}, + "source": [ + "# Stepwise Planner" + ] + }, + { + "cell_type": "markdown", + "id": "8a4bbcc3", + "metadata": {}, + "source": [ + "Stepwise Planner is based off the paper from MRKL (Modular Reasoning, Knowledge and Language) and is similar to other papers like ReACT (Reasoning and Acting in Language Models). At the core, the stepwise planner allows for the AI to form \"thoughts\" and \"observations\" and execute actions based off those to achieve a user's goal. This continues until all required functions are complete and a final output is generated.\n", + "\n", + "See a video walkthrough of Stepwise Planner [here.](https://youtu.be/DG_Ge1v0c4Q?si=T1CHaAm1vV0mWRHu)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32839327", + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.planning import StepwisePlanner\n", + "from semantic_kernel.planning.stepwise_planner.stepwise_planner_config import (\n", + " StepwisePlannerConfig,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e0a00bde", + "metadata": {}, + "source": [ + "Let's create a Bing Search native skill that we can pass in to the Kernel.\n", + "\n", + "Make sure you have a Bing Search API key in your `.env` file\n", + "\n", + "(https://www.microsoft.com/en-us/bing/apis/bing-web-search-api)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92fc5de8", + "metadata": {}, + "outputs": [], + "source": [ + "class WebSearchEngineSkill:\n", + " \"\"\"\n", + " A search engine skill.\n", + " \"\"\"\n", + " from semantic_kernel.orchestration.sk_context import SKContext\n", + " from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter\n", + "\n", + " def __init__(self, connector) -> None:\n", + " self._connector = connector\n", + "\n", + " @sk_function(\n", + " description=\"Performs a web search for a given query\", name=\"searchAsync\"\n", + " )\n", + " @sk_function_context_parameter(\n", + " name=\"query\",\n", + " description=\"The search query\",\n", + " )\n", + " async def search_async(self, query: str, context: SKContext) -> str:\n", + " query = query or context.variables.get(\"query\")[1]\n", + " result = await self._connector.search_async(query, num_results=5, offset=0)\n", + " return str(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "415f7876", + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.connectors.search_engine import BingConnector\n", + "\n", + "BING_API_KEY = sk.bing_search_settings_from_dot_env()\n", + "connector = BingConnector(BING_API_KEY)\n", + "kernel.import_skill(WebSearchEngineSkill(connector), skill_name=\"WebSearch\")" + ] + }, + { + "cell_type": "markdown", + "id": "effdf3ab", + "metadata": {}, + "source": [ + "Let's also add a couple more skills" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "abe150e0", + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.core_skills.math_skill import MathSkill\n", + "from semantic_kernel.core_skills.time_skill import TimeSkill\n", + "\n", + "kernel.import_skill(TimeSkill(), \"time\")\n", + "kernel.import_skill(MathSkill(), \"math\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06d08549", + "metadata": {}, + "outputs": [], + "source": [ + "planner = StepwisePlanner(\n", + " kernel, StepwisePlannerConfig(max_iterations=10, min_iteration_time_ms=1000)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "50699ec3", + "metadata": {}, + "source": [ + "Now let's do a more complicated ask that will require planner to make a call to Bing to get the latest information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "596ade21", + "metadata": {}, + "outputs": [], + "source": [ + "ask = \"\"\"How many total championships combined do the top 5 teams in the NBA have?\"\"\"\n", + "\n", + "plan = planner.create_plan(goal=ask)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "176988ac", + "metadata": {}, + "outputs": [], + "source": [ + "result = await plan.invoke_async()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d00c6f71", + "metadata": {}, + "outputs": [], + "source": [ + "print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "cb40370d", + "metadata": {}, + "source": [ + "Let's see the steps that the AI took to get to the answer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7159ca1b", + "metadata": {}, + "outputs": [], + "source": [ + "for index, step in enumerate(plan._steps):\n", + " print(\"Step:\", index)\n", + " print(\"Description:\",step.description)\n", + " print(\"Function:\", step.skill_name + \".\" + step._function.name)\n", + " if len(step._outputs) > 0:\n", + " print( \" Output:\\n\", str.replace(result[step._outputs[0]],\"\\n\", \"\\n \"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4652ac81", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python/notebooks/06-memory-and-embeddings.ipynb b/python/notebooks/06-memory-and-embeddings.ipynb new file mode 100644 index 000000000000..4e43e925be17 --- /dev/null +++ b/python/notebooks/06-memory-and-embeddings.ipynb @@ -0,0 +1,424 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "68e1c158", + "metadata": {}, + "source": [ + "# Building Semantic Memory with Embeddings\n", + "\n", + "So far, we've mostly been treating the kernel as a stateless orchestration engine.\n", + "We send text into a model API and receive text out. \n", + "\n", + "In a [previous notebook](04-context-variables-chat.ipynb), we used `context variables` to pass in additional\n", + "text into prompts to enrich them with more context. This allowed us to create a basic chat experience. \n", + "\n", + "However, if you solely relied on context variables, you would quickly realize that eventually your prompt\n", + "would grow so large that you would run into a the model's token limit. What we need is a way to persist state\n", + "and build both short-term and long-term memory to empower even more intelligent applications. \n", + "\n", + "To do this, we dive into the key concept of `Semantic Memory` in the Semantic Kernel. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a77bdf89", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m pip install semantic-kernel==0.3.10.dev0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "508ad44f", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Tuple\n", + "\n", + "import semantic_kernel as sk\n", + "from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion, OpenAITextEmbedding, AzureChatCompletion, AzureTextEmbedding" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "d8ddffc1", + "metadata": {}, + "source": [ + "In order to use memory, we need to instantiate the Kernel with a Memory Storage\n", + "and an Embedding service. In this example, we make use of the `VolatileMemoryStore` \"which can be thought of as a temporary in-memory storage (not to be confused with Semantic Memory). This memory is not written to disk and is only available during the app session.\n", + "\n", + "When developing your app you will have the option to plug in persistent storage like Azure Cosmos Db, PostgreSQL, SQLite, etc. Semantic Memory allows also to index external data sources, without duplicating all the information, more on that later." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f8dcbc6", + "metadata": {}, + "outputs": [], + "source": [ + "kernel = sk.Kernel()\n", + "\n", + "useAzureOpenAI = False\n", + "\n", + "# Configure AI service used by the kernel\n", + "if useAzureOpenAI:\n", + " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\"chat_completion\", AzureChatCompletion(deployment, endpoint, api_key))\n", + " # next line assumes embeddings deployment name is \"text-embedding-ada-002\", adjust this if appropriate \n", + " kernel.add_text_embedding_generation_service(\"ada\", AzureTextEmbedding(\"text-embedding-ada-002\", endpoint, api_key))\n", + "else:\n", + " api_key, org_id = sk.openai_settings_from_dot_env()\n", + " kernel.add_chat_service(\"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id))\n", + " kernel.add_text_embedding_generation_service(\"ada\", OpenAITextEmbedding(\"text-embedding-ada-002\", api_key, org_id))\n", + "\n", + "kernel.register_memory_store(memory_store=sk.memory.VolatileMemoryStore())\n", + "kernel.import_skill(sk.core_skills.TextMemorySkill())" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "e7fefb6a", + "metadata": {}, + "source": [ + "At its core, Semantic Memory is a set of data structures that allow you to store the meaning of text that come from different data sources, and optionally to store the source text too. These texts can be from the web, e-mail providers, chats, a database, or from your local directory, and are hooked up to the Semantic Kernel through data source connectors.\n", + "\n", + "The texts are embedded or compressed into a vector of floats representing mathematically the texts' contents and meaning. You can read more about embeddings [here](https://aka.ms/sk/embeddings)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "2a7e7ca4", + "metadata": {}, + "source": [ + "### Manually adding memories\n", + "Let's create some initial memories \"About Me\". We can add memories to our `VolatileMemoryStore` by using `SaveInformationAsync`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d096504c", + "metadata": {}, + "outputs": [], + "source": [ + "async def populate_memory(kernel: sk.Kernel) -> None:\n", + " # Add some documents to the semantic memory\n", + " await kernel.memory.save_information_async(\n", + " \"aboutMe\", id=\"info1\", text=\"My name is Andrea\"\n", + " )\n", + " await kernel.memory.save_information_async(\n", + " \"aboutMe\", id=\"info2\", text=\"I currently work as a tour guide\"\n", + " )\n", + " await kernel.memory.save_information_async(\n", + " \"aboutMe\", id=\"info3\", text=\"I've been living in Seattle since 2005\"\n", + " )\n", + " await kernel.memory.save_information_async(\n", + " \"aboutMe\", id=\"info4\", text=\"I visited France and Italy five times since 2015\"\n", + " )\n", + " await kernel.memory.save_information_async(\n", + " \"aboutMe\", id=\"info5\", text=\"My family is from New York\"\n", + " )" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "2calf857", + "metadata": {}, + "source": [ + "Let's try searching the memory:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "628c843e", + "metadata": {}, + "outputs": [], + "source": [ + "async def search_memory_examples(kernel: sk.Kernel) -> None:\n", + " questions = [\n", + " \"what's my name\",\n", + " \"where do I live?\",\n", + " \"where's my family from?\",\n", + " \"where have I traveled?\",\n", + " \"what do I do for work\",\n", + " ]\n", + "\n", + " for question in questions:\n", + " print(f\"Question: {question}\")\n", + " result = await kernel.memory.search_async(\"aboutMe\", question)\n", + " print(f\"Answer: {result[0].text}\\n\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "e70c2b22", + "metadata": {}, + "source": [ + "Let's now revisit the our chat sample from the [previous notebook](04-context-variables-chat.ipynb).\n", + "If you remember, we used context variables to fill the prompt with a `history` that continuously got populated as we chatted with the bot. Let's add also memory to it!" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "1ed54a32", + "metadata": {}, + "source": [ + "This is done by using the `TextMemorySkill` which exposes the `recall` native function.\n", + "\n", + "`recall` takes an input ask and performs a similarity search on the contents that have\n", + "been embedded in the Memory Store and returns the most relevant memory. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb8549b2", + "metadata": {}, + "outputs": [], + "source": [ + "async def setup_chat_with_memory(\n", + " kernel: sk.Kernel,\n", + ") -> Tuple[sk.SKFunctionBase, sk.SKContext]:\n", + " sk_prompt = \"\"\"\n", + " ChatBot can have a conversation with you about any topic.\n", + " It can give explicit instructions or say 'I don't know' if\n", + " it does not have an answer.\n", + "\n", + " Information about me, from previous conversations:\n", + " - {{$fact1}} {{recall $fact1}}\n", + " - {{$fact2}} {{recall $fact2}}\n", + " - {{$fact3}} {{recall $fact3}}\n", + " - {{$fact4}} {{recall $fact4}}\n", + " - {{$fact5}} {{recall $fact5}}\n", + "\n", + " Chat:\n", + " {{$chat_history}}\n", + " User: {{$user_input}}\n", + " ChatBot: \"\"\".strip()\n", + "\n", + " chat_func = kernel.create_semantic_function(sk_prompt, max_tokens=200, temperature=0.8)\n", + "\n", + " context = kernel.create_new_context()\n", + " context[\"fact1\"] = \"what is my name?\"\n", + " context[\"fact2\"] = \"where do I live?\"\n", + " context[\"fact3\"] = \"where's my family from?\"\n", + " context[\"fact4\"] = \"where have I traveled?\"\n", + " context[\"fact5\"] = \"what do I do for work?\"\n", + "\n", + " context[sk.core_skills.TextMemorySkill.COLLECTION_PARAM] = \"aboutMe\"\n", + " context[sk.core_skills.TextMemorySkill.RELEVANCE_PARAM] = 0.8\n", + "\n", + " context[\"chat_history\"] = \"\"\n", + "\n", + " return chat_func, context" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "1ac62457", + "metadata": {}, + "source": [ + "The `RelevanceParam` is used in memory search and is a measure of the relevance score from 0.0 to 1.0, where 1.0 means a perfect match. We encourage users to experiment with different values." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "645b55a1", + "metadata": {}, + "source": [ + "Now that we've included our memories, let's chat!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75267a2f", + "metadata": {}, + "outputs": [], + "source": [ + "async def chat(\n", + " kernel: sk.Kernel, chat_func: sk.SKFunctionBase, context: sk.SKContext\n", + ") -> bool:\n", + " try:\n", + " user_input = input(\"User:> \")\n", + " context[\"user_input\"] = user_input\n", + " print(f\"User:> {user_input}\")\n", + " except KeyboardInterrupt:\n", + " print(\"\\n\\nExiting chat...\")\n", + " return False\n", + " except EOFError:\n", + " print(\"\\n\\nExiting chat...\")\n", + " return False\n", + "\n", + " if user_input == \"exit\":\n", + " print(\"\\n\\nExiting chat...\")\n", + " return False\n", + "\n", + " answer = await kernel.run_async(chat_func, input_vars=context.variables)\n", + " context[\"chat_history\"] += f\"\\nUser:> {user_input}\\nChatBot:> {answer}\\n\"\n", + "\n", + " print(f\"ChatBot:> {answer}\")\n", + " return True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3875a34", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Populating memory...\")\n", + "await populate_memory(kernel)\n", + "\n", + "print(\"Asking questions... (manually)\")\n", + "await search_memory_examples(kernel)\n", + "\n", + "print(\"Setting up a chat (with memory!)\")\n", + "chat_func, context = await setup_chat_with_memory(kernel)\n", + "\n", + "print(\"Begin chatting (type 'exit' to exit):\\n\")\n", + "chatting = True\n", + "while chatting:\n", + " chatting = await chat(kernel, chat_func, context)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "0a51542b", + "metadata": {}, + "source": [ + "### Adding documents to your memory\n", + "\n", + "Many times in your applications you'll want to bring in external documents into your memory. Let's see how we can do this using our VolatileMemoryStore.\n", + "\n", + "Let's first get some data using some of the links in the Semantic Kernel repo." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d5a1b9", + "metadata": {}, + "outputs": [], + "source": [ + "github_files ={}\n", + "github_files[\"https://github.com/microsoft/semantic-kernel/blob/main/README.md\"] = \\\n", + " \"README: Installation, getting started, and how to contribute\"\n", + "github_files[\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/02-running-prompts-from-file.ipynb\"] = \\\n", + " \"Jupyter notebook describing how to pass prompts from a file to a semantic skill or function\"\n", + "github_files[\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/00-getting-started.ipynb\"] = \\\n", + " \"Jupyter notebook describing how to get started with the Semantic Kernel\"\n", + "github_files[\"https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT\"] = \\\n", + " \"Sample demonstrating how to create a chat skill interfacing with ChatGPT\"\n", + "github_files[\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/Volatile/VolatileMemoryStore.cs\"] = \\\n", + " \"C# class that defines a volatile embedding store\"\n", + "github_files[\"https://github.com/microsoft/semantic-kernel/tree/main/samples/dotnet/KernelHttpServer/README.md\"] = \\\n", + " \"README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4\"\n", + "github_files[\"https://github.com/microsoft/semantic-kernel/tree/main/samples/apps/chat-summary-webapp-react/README.md\"] = \\\n", + " \"README: README associated with a sample starter react-based chat summary webapp\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "75f3ea5e", + "metadata": {}, + "source": [ + "Now let's add these files to our VolatileMemoryStore using `SaveReferenceAsync`. We'll separate these memories from the chat memories by putting them in a different collection." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "170e7142", + "metadata": {}, + "outputs": [], + "source": [ + "memory_collection_name = \"SKGitHub\"\n", + "print(\"Adding some GitHub file URLs and their descriptions to a volatile Semantic Memory.\");\n", + "i = 0\n", + "for entry, value in github_files.items():\n", + " await kernel.memory.save_reference_async(\n", + " collection=memory_collection_name,\n", + " description=value,\n", + " text=value,\n", + " external_id=entry,\n", + " external_source_name=\"GitHub\"\n", + " )\n", + " i += 1\n", + " print(\" URL {} saved\".format(i))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "143911c3", + "metadata": {}, + "outputs": [], + "source": [ + "ask = \"I love Jupyter notebooks, how should I get started?\"\n", + "print(\"===========================\\n\" + \"Query: \" + ask + \"\\n\")\n", + "\n", + "memories = await kernel.memory.search_async(memory_collection_name, ask, limit=5, min_relevance_score=0.77)\n", + "\n", + "i = 0\n", + "for memory in memories:\n", + " i += 1\n", + " print(f\"Result {i}:\")\n", + " print(\" URL: : \" + memory.id)\n", + " print(\" Title : \" + memory.description)\n", + " print(\" Relevance: \" + str(memory.relevance))\n", + " print()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "59294dac", + "metadata": {}, + "source": [ + "Now you might be wondering what happens if you have so much data that it doesn't fit into your RAM? That's where you want to make use of an external Vector Database made specifically for storing and retrieving embeddings.\n", + "\n", + "Stay tuned for that!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/samples/notebooks/python/07-hugging-face-for-skills.ipynb b/python/notebooks/07-hugging-face-for-skills.ipynb similarity index 99% rename from samples/notebooks/python/07-hugging-face-for-skills.ipynb rename to python/notebooks/07-hugging-face-for-skills.ipynb index 0856d57bfb25..057d44216944 100644 --- a/samples/notebooks/python/07-hugging-face-for-skills.ipynb +++ b/python/notebooks/07-hugging-face-for-skills.ipynb @@ -20,7 +20,7 @@ "metadata": {}, "outputs": [], "source": [ - "!python -m pip install semantic-kernel==0.3.1.dev0\n", + "!python -m pip install semantic-kernel==0.3.10.dev0\n", "\n", "# Note that additional dependencies are required for the Hugging Face connectors:\n", "!python -m pip install torch==2.0.0\n", diff --git a/samples/notebooks/python/08-native-function-inline.ipynb b/python/notebooks/08-native-function-inline.ipynb similarity index 96% rename from samples/notebooks/python/08-native-function-inline.ipynb rename to python/notebooks/08-native-function-inline.ipynb index 3162d78e566d..484f43a9a27a 100644 --- a/samples/notebooks/python/08-native-function-inline.ipynb +++ b/python/notebooks/08-native-function-inline.ipynb @@ -46,7 +46,7 @@ "metadata": {}, "outputs": [], "source": [ - "!python -m pip install semantic-kernel==0.3.1.dev0" + "!python -m pip install semantic-kernel==0.3.10.dev0" ] }, { @@ -59,7 +59,7 @@ "import os\n", "import sys\n", "import semantic_kernel as sk\n", - "from semantic_kernel.connectors.ai.open_ai import AzureTextCompletion, OpenAITextCompletion\n", + "from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion\n", "\n", "kernel = sk.Kernel()\n", "\n", @@ -69,10 +69,10 @@ "# Configure AI service used by the kernel\n", "if useAzureOpenAI:\n", " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", - " kernel.add_text_completion_service(\"dv\", AzureTextCompletion(deployment, endpoint, api_key))\n", + " kernel.add_chat_service(\"chat_completion\", AzureChatCompletion(deployment, endpoint, api_key))\n", "else:\n", " api_key, org_id = sk.openai_settings_from_dot_env()\n", - " kernel.add_text_completion_service(\"dv\", OpenAITextCompletion(\"text-davinci-003\", api_key, org_id))" + " kernel.add_chat_service(\"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id))" ] }, { @@ -225,7 +225,7 @@ "import os\n", "import sys\n", "import semantic_kernel as sk\n", - "from semantic_kernel.connectors.ai.open_ai import AzureTextCompletion, OpenAITextCompletion\n", + "from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion\n", "\n", "kernel = sk.Kernel()\n", "\n", @@ -235,10 +235,10 @@ "# Configure AI service used by the kernel\n", "if useAzureOpenAI:\n", " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", - " kernel.add_text_completion_service(\"dv\", AzureTextCompletion(deployment, endpoint, api_key))\n", + " kernel.add_chat_service(\"chat_completion\", AzureChatCompletion(deployment, endpoint, api_key))\n", "else:\n", " api_key, org_id = sk.openai_settings_from_dot_env()\n", - " kernel.add_text_completion_service(\"dv\", OpenAITextCompletion(\"text-davinci-003\", api_key, org_id))" + " kernel.add_chat_service(\"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id))\n" ] }, { diff --git a/samples/notebooks/python/09-groundedness-checking.ipynb b/python/notebooks/09-groundedness-checking.ipynb similarity index 97% rename from samples/notebooks/python/09-groundedness-checking.ipynb rename to python/notebooks/09-groundedness-checking.ipynb index 03f0d60ebc93..8dd15ca7e453 100644 --- a/samples/notebooks/python/09-groundedness-checking.ipynb +++ b/python/notebooks/09-groundedness-checking.ipynb @@ -83,7 +83,7 @@ "outputs": [], "source": [ "import semantic_kernel as sk\n", - "from semantic_kernel.connectors.ai.open_ai import AzureTextCompletion, OpenAITextCompletion\n", + "from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion\n", "\n", "kernel = sk.Kernel()\n", "\n", @@ -92,10 +92,10 @@ "# Configure AI service used by the kernel\n", "if useAzureOpenAI:\n", " deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", - " kernel.add_text_completion_service(\"dv\", AzureTextCompletion(deployment, endpoint, api_key))\n", + " kernel.add_chat_service(\"chat_completion\", AzureChatCompletion(deployment, endpoint, api_key))\n", "else:\n", " api_key, org_id = sk.openai_settings_from_dot_env()\n", - " kernel.add_text_completion_service(\"dv\", OpenAITextCompletion(\"text-davinci-003\", api_key, org_id))" + " kernel.add_chat_service(\"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id))" ] }, { @@ -118,7 +118,7 @@ "from semantic_kernel.core_skills.text_skill import TextSkill\n", "\n", "# note: using skills from the samples folder\n", - "skills_directory = \"../../skills\"\n", + "skills_directory = \"../../samples/skills\"\n", "\n", "groundingSemanticFunctions = kernel.import_semantic_skill_from_directory(skills_directory, \"GroundingSkill\")" ] diff --git a/samples/notebooks/python/10-multiple-results-per-prompt.ipynb b/python/notebooks/10-multiple-results-per-prompt.ipynb similarity index 97% rename from samples/notebooks/python/10-multiple-results-per-prompt.ipynb rename to python/notebooks/10-multiple-results-per-prompt.ipynb index 60b0032ed326..9795fdb8c667 100644 --- a/samples/notebooks/python/10-multiple-results-per-prompt.ipynb +++ b/python/notebooks/10-multiple-results-per-prompt.ipynb @@ -6,7 +6,7 @@ "id": "68e1c158", "metadata": {}, "source": [ - "# Streaming Results" + "# Multiple Results" ] }, { @@ -25,7 +25,7 @@ "metadata": {}, "outputs": [], "source": [ - "!python -m pip install semantic-kernel==0.3.1.dev0" + "!python -m pip install semantic-kernel==0.3.10.dev0" ] }, { @@ -61,8 +61,8 @@ "\n", "# Configure Azure LLM service\n", "deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", - "azure_text_service = AzureTextCompletion(\"text-davinci-003\", endpoint, api_key)\n", - "azure_chat_service = AzureChatCompletion(\"gpt-35-turbo\", endpoint, api_key)\n", + "azure_text_service = AzureTextCompletion(deployment, endpoint, api_key)\n", + "azure_chat_service = AzureChatCompletion(deployment, endpoint, api_key)\n", "\n", "# Configure OpenAI service\n", "api_key, org_id = sk.openai_settings_from_dot_env()\n", diff --git a/samples/notebooks/python/11-streaming-completions.ipynb b/python/notebooks/11-streaming-completions.ipynb similarity index 96% rename from samples/notebooks/python/11-streaming-completions.ipynb rename to python/notebooks/11-streaming-completions.ipynb index e3f4647724f2..8179e69e7e36 100644 --- a/samples/notebooks/python/11-streaming-completions.ipynb +++ b/python/notebooks/11-streaming-completions.ipynb @@ -16,7 +16,7 @@ "metadata": {}, "outputs": [], "source": [ - "!python -m pip install semantic-kernel==0.3.1.dev0" + "!python -m pip install semantic-kernel==0.3.10.dev0" ] }, { @@ -52,8 +52,8 @@ "\n", "# Configure Azure LLM service\n", "deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n", - "azure_text_service = AzureTextCompletion(\"text-davinci-003\", endpoint, api_key)\n", - "azure_chat_service = AzureChatCompletion(\"gpt-35-turbo\", endpoint, api_key)\n", + "azure_text_service = AzureTextCompletion(deployment, endpoint, api_key)\n", + "azure_chat_service = AzureChatCompletion(deployment, endpoint, api_key)\n", "\n", "# Configure OpenAI service\n", "api_key, org_id = sk.openai_settings_from_dot_env()\n", diff --git a/samples/notebooks/python/third_party/.env.example b/python/notebooks/third_party/.env.example similarity index 100% rename from samples/notebooks/python/third_party/.env.example rename to python/notebooks/third_party/.env.example diff --git a/python/notebooks/third_party/weaviate-persistent-memory.ipynb b/python/notebooks/third_party/weaviate-persistent-memory.ipynb new file mode 100644 index 000000000000..58a3fcfd1f4a --- /dev/null +++ b/python/notebooks/third_party/weaviate-persistent-memory.ipynb @@ -0,0 +1,536 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook shows how to replace the `VolatileMemoryStore` memory storage used in a [previous notebook](./06-memory-and-embeddings.ipynb) with a `WeaviateMemoryStore`.\n", + "\n", + "`WeaviateMemoryStore` is an example of a persistent (i.e. long-term) memory store backed by the Weaviate vector database." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# About Weaviate" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[Weaviate](https://weaviate.io/) is an open-source vector database designed to scale seamlessly into billions of data objects. This implementation supports hybrid search out-of-the-box (meaning it will perform better for keyword searches)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can run Weaviate in 5 ways:\n", + "\n", + "- **SaaS** – with [Weaviate Cloud Services (WCS)](https://weaviate.io/pricing).\n", + "\n", + " WCS is a fully managed service that takes care of hosting, scaling, and updating your Weaviate instance. You can try it out for free with a sandbox that lasts for 14 days.\n", + "\n", + " To set up a SaaS Weaviate instance with WCS:\n", + "\n", + " 1. Navigate to [Weaviate Cloud Console](https://console.weaviate.cloud/).\n", + " 2. Register or sign in to your WCS account.\n", + " 3. Create a new cluster with the following settings:\n", + " - `Subscription Tier` – Free sandbox for a free trial, or contact [hello@weaviate.io](mailto:hello@weaviate.io) for other options.\n", + " - `Cluster name` – a unique name for your cluster. The name will become part of the URL used to access this instance.\n", + " - `Enable Authentication?` – Enabled by default. This will generate a static API key that you can use to authenticate.\n", + " 4. Wait for a few minutes until your cluster is ready. You will see a green tick ✔️ when it's done. Copy your cluster URL.\n", + "\n", + "- **Hybrid SaaS**\n", + "\n", + " > If you need to keep your data on-premise for security or compliance reasons, Weaviate also offers a Hybrid SaaS option: Weaviate runs within your cloud instances, but the cluster is managed remotely by Weaviate. This gives you the benefits of a managed service without sending data to an external party.\n", + "\n", + " The Weaviate Hybrid SaaS is a custom solution. If you are interested in this option, please reach out to [hello@weaviate.io](mailto:hello@weaviate.io).\n", + "\n", + "- **Self-hosted** – with a Docker container\n", + "\n", + " To set up a Weaviate instance with Docker:\n", + "\n", + " 1. [Install Docker](https://docs.docker.com/engine/install/) on your local machine if it is not already installed.\n", + " 2. [Install the Docker Compose Plugin](https://docs.docker.com/compose/install/)\n", + " 3. Download a `docker-compose.yml` file with this `curl` command:\n", + "\n", + " ```\n", + " curl -o docker-compose.yml \"https://configuration.weaviate.io/v2/docker-compose/docker-compose.yml?modules=standalone&runtime=docker-compose&weaviate_version=v1.19.6\"\n", + " ```\n", + "\n", + " Alternatively, you can use Weaviate's docker compose [configuration tool](https://weaviate.io/developers/weaviate/installation/docker-compose) to generate your own `docker-compose.yml` file.\n", + "\n", + " 4. Run `docker compose up -d` to spin up a Weaviate instance.\n", + "\n", + " > To shut it down, run `docker compose down`.\n", + "\n", + "- **Self-hosted** – with a Kubernetes cluster\n", + "\n", + " To configure a self-hosted instance with Kubernetes, follow Weaviate's [documentation](https://weaviate.io/developers/weaviate/installation/kubernetes).|\n", + "\n", + "- **Embedded** - start a weaviate instance right from your application code using the client library\n", + " \n", + " This code snippet shows how to instantiate an embedded weaviate instance and upload a document:\n", + "\n", + " ```python\n", + " import weaviate\n", + " from weaviate.embedded import EmbeddedOptions\n", + "\n", + " client = weaviate.Client(\n", + " embedded_options=EmbeddedOptions()\n", + " )\n", + "\n", + " data_obj = {\n", + " \"name\": \"Chardonnay\",\n", + " \"description\": \"Goes with fish\"\n", + " }\n", + "\n", + " client.data_object.create(data_obj, \"Wine\")\n", + " ```\n", + " \n", + " Refer to the [documentation](https://weaviate.io/developers/weaviate/installation/embedded) for more details about this deployment method." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantic-kernel==0.3.8.dev0\n", + "!pip install weaviate-client\n", + "!pip install python-dotenv" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OS-specific notes:\n", + "* if you run into SSL errors when connecting to OpenAI on macOS, see this issue for a [potential solution](https://github.com/microsoft/semantic-kernel/issues/627#issuecomment-1580912248)\n", + "* on Windows, you may need to run Docker Desktop as administrator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Tuple\n", + "\n", + "import semantic_kernel as sk\n", + "from semantic_kernel.connectors.ai.open_ai import (\n", + " OpenAIChatCompletion,\n", + " OpenAITextEmbedding,\n", + ")\n", + "\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we instantiate the Weaviate memory store. Uncomment ONE of the options below, depending on how you want to use Weaviate:\n", + "* from a Docker instance\n", + "* from WCS\n", + "* directly from the client (embedded Weaviate), which works on Linux only at the moment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantic_kernel.connectors.memory.weaviate import weaviate_memory_store\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv(override=True)\n", + "\n", + "# Using Docker\n", + "config = weaviate_memory_store.WeaviateConfig(url=\"http://localhost:8080\")\n", + "\n", + "# Using WCS. Make sure the environment variables `WEAVIATE_URL` and `WEAVIATE_API_KEY`\n", + "# were set in the `.env` file.\n", + "#\n", + "#weaviate_api, weaviate_url = sk.weaviate_settings_from_dot_env()\n", + "#\n", + "#config = weaviate_memory_store.WeaviateConfig(\n", + "# url=weaviate_url,\n", + "# api_key=weaviate_api\n", + "#)\n", + "\n", + "# Using Embedded Weaviate\n", + "#config = weaviate_memory_store.WeaviateConfig(use_embed=True)\n", + "\n", + "store = weaviate_memory_store.WeaviateMemoryStore(config=config)\n", + "store.client.schema.delete_all()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we register the memory store to the kernel:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kernel = sk.Kernel()\n", + "\n", + "api_key, org_id = sk.openai_settings_from_dot_env()\n", + "kernel.add_chat_service(\n", + " \"chat-gpt\", OpenAIChatCompletion(\"gpt-3.5-turbo\", api_key, org_id)\n", + ")\n", + "kernel.add_text_embedding_generation_service(\n", + " \"ada\", OpenAITextEmbedding(\"text-embedding-ada-002\", api_key, org_id)\n", + ")\n", + "\n", + "kernel.register_memory_store(memory_store=store)\n", + "kernel.import_skill(sk.core_skills.TextMemorySkill())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Manually adding memories\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's create some initial memories \"About Me\". We can add memories to our weaviate memory store by using `save_information_async`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "COLLECTION = \"AboutMe\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "async def populate_memory(kernel: sk.Kernel) -> None:\n", + " # Add some documents to the semantic memory\n", + " await kernel.memory.save_information_async(\n", + " COLLECTION, id=\"info1\", text=\"My name is Andrea\"\n", + " )\n", + " await kernel.memory.save_information_async(\n", + " COLLECTION, id=\"info2\", text=\"I currently work as a tour guide\"\n", + " )\n", + " await kernel.memory.save_information_async(\n", + " COLLECTION, id=\"info3\", text=\"I've been living in Seattle since 2005\"\n", + " )\n", + " await kernel.memory.save_information_async(\n", + " COLLECTION, id=\"info4\", text=\"I visited France and Italy five times since 2015\"\n", + " )\n", + " await kernel.memory.save_information_async(\n", + " COLLECTION, id=\"info5\", text=\"My family is from New York\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Searching is done through `search_async`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "async def search_memory_examples(kernel: sk.Kernel) -> None:\n", + " questions = [\n", + " \"what's my name\",\n", + " \"where do I live?\",\n", + " \"where's my family from?\",\n", + " \"where have I traveled?\",\n", + " \"what do I do for work\",\n", + " ]\n", + "\n", + " for question in questions:\n", + " print(f\"Question: {question}\")\n", + " result = await kernel.memory.search_async(COLLECTION, question)\n", + " print(f\"Answer: {result[0].text}\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's see the results of the functions:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Populating memory...\")\n", + "await populate_memory(kernel)\n", + "\n", + "print(\"Asking questions... (manually)\")\n", + "await search_memory_examples(kernel)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's how to use the weaviate memory store in a chat application:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "async def setup_chat_with_memory(\n", + " kernel: sk.Kernel,\n", + ") -> Tuple[sk.SKFunctionBase, sk.SKContext]:\n", + " sk_prompt = \"\"\"\n", + " ChatBot can have a conversation with you about any topic.\n", + " It can give explicit instructions or say 'I don't know' if\n", + " it does not have an answer.\n", + "\n", + " Information about me, from previous conversations:\n", + " - {{$fact1}} {{recall $fact1}}\n", + " - {{$fact2}} {{recall $fact2}}\n", + " - {{$fact3}} {{recall $fact3}}\n", + " - {{$fact4}} {{recall $fact4}}\n", + " - {{$fact5}} {{recall $fact5}}\n", + "\n", + " Chat:\n", + " {{$chat_history}}\n", + " User: {{$user_input}}\n", + " ChatBot: \"\"\".strip()\n", + "\n", + " chat_func = kernel.create_semantic_function(\n", + " sk_prompt, max_tokens=200, temperature=0.8\n", + " )\n", + "\n", + " context = kernel.create_new_context()\n", + " context[\"fact1\"] = \"what is my name?\"\n", + " context[\"fact2\"] = \"where do I live?\"\n", + " context[\"fact3\"] = \"where's my family from?\"\n", + " context[\"fact4\"] = \"where have I traveled?\"\n", + " context[\"fact5\"] = \"what do I do for work?\"\n", + "\n", + " context[sk.core_skills.TextMemorySkill.COLLECTION_PARAM] = COLLECTION\n", + " context[sk.core_skills.TextMemorySkill.RELEVANCE_PARAM] = 0.8\n", + "\n", + " context[\"chat_history\"] = \"\"\n", + "\n", + " return chat_func, context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "async def chat(\n", + " kernel: sk.Kernel, chat_func: sk.SKFunctionBase, context: sk.SKContext\n", + ") -> bool:\n", + " try:\n", + " user_input = input(\"User:> \")\n", + " context[\"user_input\"] = user_input\n", + " except KeyboardInterrupt:\n", + " print(\"\\n\\nExiting chat...\")\n", + " return False\n", + " except EOFError:\n", + " print(\"\\n\\nExiting chat...\")\n", + " return False\n", + "\n", + " if user_input == \"exit\":\n", + " print(\"\\n\\nExiting chat...\")\n", + " return False\n", + "\n", + " answer = await kernel.run_async(chat_func, input_vars=context.variables)\n", + " context[\"chat_history\"] += f\"\\nUser:> {user_input}\\nChatBot:> {answer}\\n\"\n", + "\n", + " print(f\"ChatBot:> {answer}\")\n", + " return True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Setting up a chat (with memory!)\")\n", + "chat_func, context = await setup_chat_with_memory(kernel)\n", + "\n", + "print(\"Begin chatting (type 'exit' to exit):\\n\")\n", + "chatting = True\n", + "while chatting:\n", + " chatting = await chat(kernel, chat_func, context)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Adding documents to your memory" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a dictionary to hold some files. The key is the hyperlink to the file and the value is the file's content:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "github_files = {}\n", + "github_files[\n", + " \"https://github.com/microsoft/semantic-kernel/blob/main/README.md\"\n", + "] = \"README: Installation, getting started, and how to contribute\"\n", + "github_files[\n", + " \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/02-running-prompts-from-file.ipynb\"\n", + "] = \"Jupyter notebook describing how to pass prompts from a file to a semantic skill or function\"\n", + "github_files[\n", + " \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/00-getting-started.ipynb\"\n", + "] = \"Jupyter notebook describing how to get started with the Semantic Kernel\"\n", + "github_files[\n", + " \"https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT\"\n", + "] = \"Sample demonstrating how to create a chat skill interfacing with ChatGPT\"\n", + "github_files[\n", + " \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/Volatile/VolatileMemoryStore.cs\"\n", + "] = \"C# class that defines a volatile embedding store\"\n", + "github_files[\n", + " \"https://github.com/microsoft/semantic-kernel/tree/main/samples/dotnet/KernelHttpServer/README.md\"\n", + "] = \"README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4\"\n", + "github_files[\n", + " \"https://github.com/microsoft/semantic-kernel/tree/main/samples/apps/chat-summary-webapp-react/README.md\"\n", + "] = \"README: README associated with a sample starter react-based chat summary webapp\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use `save_reference_async` to save the file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "COLLECTION = \"SKGitHub\"\n", + "\n", + "print(\n", + " \"Adding some GitHub file URLs and their descriptions to a volatile Semantic Memory.\"\n", + ")\n", + "i = 0\n", + "for entry, value in github_files.items():\n", + " await kernel.memory.save_reference_async(\n", + " collection=COLLECTION,\n", + " description=value,\n", + " text=value,\n", + " external_id=entry,\n", + " external_source_name=\"GitHub\",\n", + " )\n", + " i += 1\n", + " print(\" URL {} saved\".format(i))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use `search_async` to ask a question:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ask = \"I love Jupyter notebooks, how should I get started?\"\n", + "print(\"===========================\\n\" + \"Query: \" + ask + \"\\n\")\n", + "\n", + "memories = await kernel.memory.search_async(\n", + " COLLECTION, ask, limit=5, min_relevance_score=0.77\n", + ")\n", + "\n", + "i = 0\n", + "for memory in memories:\n", + " i += 1\n", + " print(f\"Result {i}:\")\n", + " print(\" URL: : \" + memory.id)\n", + " print(\" Title : \" + memory.description)\n", + " print(\" Relevance: \" + str(memory.relevance))\n", + " print()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/poetry.lock b/python/poetry.lock index 2cc385deed4d..96989acee04d 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -1,22 +1,20 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "aiofiles" -version = "23.1.0" +version = "23.2.1" description = "File support for asyncio." -category = "main" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.7" files = [ - {file = "aiofiles-23.1.0-py3-none-any.whl", hash = "sha256:9312414ae06472eb6f1d163f555e466a23aed1c8f60c30cccf7121dba2e53eb2"}, - {file = "aiofiles-23.1.0.tar.gz", hash = "sha256:edd247df9a19e0db16534d4baaf536d6609a43e1de5401d7a4c1c148753a1635"}, + {file = "aiofiles-23.2.1-py3-none-any.whl", hash = "sha256:19297512c647d4b27a2cf7c34caa7e405c0d60b5560618a29a9fe027b18b0107"}, + {file = "aiofiles-23.2.1.tar.gz", hash = "sha256:84ec2218d8419404abcb9f0c02df3f34c6e0a68ed41072acfb1cef5cbc29051a"}, ] [[package]] name = "aiohttp" version = "3.8.5" description = "Async http client/server framework (asyncio)" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -125,7 +123,6 @@ speedups = ["Brotli", "aiodns", "cchardet"] name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -140,7 +137,6 @@ frozenlist = ">=1.1.0" name = "anyio" version = "3.7.1" description = "High level compatibility layer for multiple asynchronous event loop implementations" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -162,7 +158,6 @@ trio = ["trio (<0.22)"] name = "appnope" version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" -category = "dev" optional = false python-versions = "*" files = [ @@ -170,11 +165,27 @@ files = [ {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, ] +[[package]] +name = "asgiref" +version = "3.7.2" +description = "ASGI specs, helper code, and adapters" +optional = false +python-versions = ">=3.7" +files = [ + {file = "asgiref-3.7.2-py3-none-any.whl", hash = "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e"}, + {file = "asgiref-3.7.2.tar.gz", hash = "sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4", markers = "python_version < \"3.11\""} + +[package.extras] +tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] + [[package]] name = "asttokens" version = "2.2.1" description = "Annotate AST trees with source code positions" -category = "dev" optional = false python-versions = "*" files = [ @@ -190,21 +201,19 @@ test = ["astroid", "pytest"] [[package]] name = "async-timeout" -version = "4.0.2" +version = "4.0.3" description = "Timeout context manager for asyncio programs" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, ] [[package]] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -223,7 +232,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "authlib" version = "1.2.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." -category = "dev" optional = false python-versions = "*" files = [ @@ -238,7 +246,6 @@ cryptography = ">=3.2" name = "azure-common" version = "1.1.28" description = "Microsoft Azure Client Library for Python (Common)" -category = "dev" optional = false python-versions = "*" files = [ @@ -248,34 +255,32 @@ files = [ [[package]] name = "azure-core" -version = "1.28.0" +version = "1.29.4" description = "Microsoft Azure Core Library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "azure-core-1.28.0.zip", hash = "sha256:e9eefc66fc1fde56dab6f04d4e5d12c60754d5a9fa49bdcfd8534fc96ed936bd"}, - {file = "azure_core-1.28.0-py3-none-any.whl", hash = "sha256:dec36dfc8eb0b052a853f30c07437effec2f9e3e1fc8f703d9bdaa5cfc0043d9"}, + {file = "azure-core-1.29.4.tar.gz", hash = "sha256:500b3aa9bf2e90c5ccc88bb105d056114ca0ce7d0ce73afb8bc4d714b2fc7568"}, + {file = "azure_core-1.29.4-py3-none-any.whl", hash = "sha256:b03261bcba22c0b9290faf9999cedd23e849ed2577feee90515694cea6bc74bf"}, ] [package.dependencies] requests = ">=2.18.4" six = ">=1.11.0" -typing-extensions = ">=4.3.0" +typing-extensions = ">=4.6.0" [package.extras] aio = ["aiohttp (>=3.0)"] [[package]] name = "azure-identity" -version = "1.13.0" +version = "1.14.0" description = "Microsoft Azure Identity Library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "azure-identity-1.13.0.zip", hash = "sha256:c931c27301ffa86b07b4dcf574e29da73e3deba9ab5d1fe4f445bb6a3117e260"}, - {file = "azure_identity-1.13.0-py3-none-any.whl", hash = "sha256:bd700cebb80cd9862098587c29d8677e819beca33c62568ced6d5a8e5e332b82"}, + {file = "azure-identity-1.14.0.zip", hash = "sha256:72441799f8c5c89bfe21026965e266672a7c5d050c2c65119ef899dd5362e2b1"}, + {file = "azure_identity-1.14.0-py3-none-any.whl", hash = "sha256:edabf0e010eb85760e1dd19424d5e8f97ba2c9caff73a16e7b30ccbdbcce369b"}, ] [package.dependencies] @@ -283,18 +288,16 @@ azure-core = ">=1.11.0,<2.0.0" cryptography = ">=2.5" msal = ">=1.20.0,<2.0.0" msal-extensions = ">=0.3.0,<2.0.0" -six = ">=1.12.0" [[package]] name = "azure-search-documents" -version = "11.4.0b6" +version = "11.4.0b9" description = "Microsoft Azure Cognitive Search Client Library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "azure-search-documents-11.4.0b6.zip", hash = "sha256:c9ebd7d99d3c7b879f48acad66141e1f50eae4468cfb8389a4b25d4c620e8df1"}, - {file = "azure_search_documents-11.4.0b6-py3-none-any.whl", hash = "sha256:24ff85bf2680c36b38d8092bcbbe2d90699aac7c4a228b0839c0ce595a41628c"}, + {file = "azure-search-documents-11.4.0b9.tar.gz", hash = "sha256:c1b65c0385b52428bee9f3b2bdd5ffe61b46efbacdd2d1f39f825b416cf24e95"}, + {file = "azure_search_documents-11.4.0b9-py3-none-any.whl", hash = "sha256:37e85587c138557aef7a748bb71338daec7681b8732936658cbb67d900f090c4"}, ] [package.dependencies] @@ -306,7 +309,6 @@ isodate = ">=0.6.0" name = "backcall" version = "0.2.0" description = "Specifications for callback functions passed in to an API" -category = "dev" optional = false python-versions = "*" files = [ @@ -318,7 +320,6 @@ files = [ name = "backoff" version = "2.2.1" description = "Function decoration for backoff and retry" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -330,7 +331,6 @@ files = [ name = "backports-zoneinfo" version = "0.2.1" description = "Backport of the standard library zoneinfo module" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -355,11 +355,44 @@ files = [ [package.extras] tzdata = ["tzdata"] +[[package]] +name = "bcrypt" +version = "4.0.1" +description = "Modern password hashing for your software and your servers" +optional = false +python-versions = ">=3.6" +files = [ + {file = "bcrypt-4.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b1023030aec778185a6c16cf70f359cbb6e0c289fd564a7cfa29e727a1c38f8f"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:08d2947c490093a11416df18043c27abe3921558d2c03e2076ccb28a116cb6d0"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0eaa47d4661c326bfc9d08d16debbc4edf78778e6aaba29c1bc7ce67214d4410"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae88eca3024bb34bb3430f964beab71226e761f51b912de5133470b649d82344"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:a522427293d77e1c29e303fc282e2d71864579527a04ddcfda6d4f8396c6c36a"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fbdaec13c5105f0c4e5c52614d04f0bca5f5af007910daa8b6b12095edaa67b3"}, + {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ca3204d00d3cb2dfed07f2d74a25f12fc12f73e606fcaa6975d1f7ae69cacbb2"}, + {file = "bcrypt-4.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:089098effa1bc35dc055366740a067a2fc76987e8ec75349eb9484061c54f535"}, + {file = "bcrypt-4.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:e9a51bbfe7e9802b5f3508687758b564069ba937748ad7b9e890086290d2f79e"}, + {file = "bcrypt-4.0.1-cp36-abi3-win32.whl", hash = "sha256:2caffdae059e06ac23fce178d31b4a702f2a3264c20bfb5ff541b338194d8fab"}, + {file = "bcrypt-4.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:8a68f4341daf7522fe8d73874de8906f3a339048ba406be6ddc1b3ccb16fc0d9"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf4fa8b2ca74381bb5442c089350f09a3f17797829d958fad058d6e44d9eb83c"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:67a97e1c405b24f19d08890e7ae0c4f7ce1e56a712a016746c8b2d7732d65d4b"}, + {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b3b85202d95dd568efcb35b53936c5e3b3600c7cdcc6115ba461df3a8e89f38d"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbb03eec97496166b704ed663a53680ab57c5084b2fc98ef23291987b525cb7d"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:5ad4d32a28b80c5fa6671ccfb43676e8c1cc232887759d1cd7b6f56ea4355215"}, + {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b57adba8a1444faf784394de3436233728a1ecaeb6e07e8c22c8848f179b893c"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:705b2cea8a9ed3d55b4491887ceadb0106acf7c6387699fca771af56b1cdeeda"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:2b3ac11cf45161628f1f3733263e63194f22664bf4d0c0f3ab34099c02134665"}, + {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3100851841186c25f127731b9fa11909ab7b1df6fc4b9f8353f4f1fd952fbf71"}, + {file = "bcrypt-4.0.1.tar.gz", hash = "sha256:27d375903ac8261cfe4047f6709d16f7d18d39b1ec92aaf72af989552a650ebd"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + [[package]] name = "black" version = "23.7.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -402,11 +435,21 @@ d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "cachetools" +version = "5.3.1" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, +] + [[package]] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -418,7 +461,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "dev" optional = false python-versions = "*" files = [ @@ -493,21 +535,30 @@ pycparser = "*" [[package]] name = "cfgv" -version = "3.3.1" +version = "3.4.0" description = "Validate configuration and produce human readable error messages." -category = "dev" optional = false -python-versions = ">=3.6.1" +python-versions = ">=3.8" +files = [ + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, +] + +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" files = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, ] [[package]] name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -588,16 +639,52 @@ files = [ {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, ] +[[package]] +name = "cheap-repr" +version = "0.5.1" +description = "Better version of repr/reprlib for short, cheap string representations." +optional = false +python-versions = "*" +files = [ + {file = "cheap_repr-0.5.1-py2.py3-none-any.whl", hash = "sha256:30096998aeb49367a4a153988d7a99dce9dc59bbdd4b19740da6b4f3f97cf2ff"}, + {file = "cheap_repr-0.5.1.tar.gz", hash = "sha256:31ec63b9d8394aa23d746c8376c8307f75f9fca0b983566b8bcf13cc661fe6dd"}, +] + +[package.extras] +tests = ["Django", "Django (<2)", "Django (<3)", "chainmap", "numpy (>=1.16.3)", "numpy (>=1.16.3,<1.17)", "numpy (>=1.16.3,<1.19)", "pandas (>=0.24.2)", "pandas (>=0.24.2,<0.25)", "pandas (>=0.24.2,<0.26)", "pytest"] + [[package]] name = "chroma-hnswlib" -version = "0.7.1" +version = "0.7.3" description = "Chromas fork of hnswlib" -category = "dev" optional = false python-versions = "*" files = [ - {file = "chroma-hnswlib-0.7.1.tar.gz", hash = "sha256:f72592dc7d0522c25cc1f8864db7a3781f179ba989f209cc3ea01694c0d76493"}, - {file = "chroma_hnswlib-0.7.1-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:38f51585d81a5072db70b17207afd1f57670c209836d0fbbf2a1aa7e8bece6b7"}, + {file = "chroma-hnswlib-0.7.3.tar.gz", hash = "sha256:b6137bedde49fffda6af93b0297fe00429fc61e5a072b1ed9377f909ed95a932"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59d6a7c6f863c67aeb23e79a64001d537060b6995c3eca9a06e349ff7b0998ca"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d71a3f4f232f537b6152947006bd32bc1629a8686df22fd97777b70f416c127a"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c92dc1ebe062188e53970ba13f6b07e0ae32e64c9770eb7f7ffa83f149d4210"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49da700a6656fed8753f68d44b8cc8ae46efc99fc8a22a6d970dc1697f49b403"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-win_amd64.whl", hash = "sha256:108bc4c293d819b56476d8f7865803cb03afd6ca128a2a04d678fffc139af029"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:11e7ca93fb8192214ac2b9c0943641ac0daf8f9d4591bb7b73be808a83835667"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6f552e4d23edc06cdeb553cdc757d2fe190cdeb10d43093d6a3319f8d4bf1c6b"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f96f4d5699e486eb1fb95849fe35ab79ab0901265805be7e60f4eaa83ce263ec"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:368e57fe9ebae05ee5844840fa588028a023d1182b0cfdb1d13f607c9ea05756"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-win_amd64.whl", hash = "sha256:b7dca27b8896b494456db0fd705b689ac6b73af78e186eb6a42fea2de4f71c6f"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:70f897dc6218afa1d99f43a9ad5eb82f392df31f57ff514ccf4eeadecd62f544"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aef10b4952708f5a1381c124a29aead0c356f8d7d6e0b520b778aaa62a356f4"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee2d8d1529fca3898d512079144ec3e28a81d9c17e15e0ea4665697a7923253"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-win_amd64.whl", hash = "sha256:a4021a70e898783cd6f26e00008b494c6249a7babe8774e90ce4766dd288c8ba"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a8f61fa1d417fda848e3ba06c07671f14806a2585272b175ba47501b066fe6b1"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7563be58bc98e8f0866907368e22ae218d6060601b79c42f59af4eccbbd2e0a"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51b8d411486ee70d7b66ec08cc8b9b6620116b650df9c19076d2d8b6ce2ae914"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d706782b628e4f43f1b8a81e9120ac486837fbd9bcb8ced70fe0d9b95c72d77"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-win_amd64.whl", hash = "sha256:54f053dedc0e3ba657f05fec6e73dd541bc5db5b09aa8bc146466ffb734bdc86"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e607c5a71c610a73167a517062d302c0827ccdd6e259af6e4869a5c1306ffb5d"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2358a795870156af6761890f9eb5ca8cade57eb10c5f046fe94dae1faa04b9e"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cea425df2e6b8a5e201fff0d922a1cc1d165b3cfe762b1408075723c8892218"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:454df3dd3e97aa784fba7cf888ad191e0087eef0fd8c70daf28b753b3b591170"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:df587d15007ca701c6de0ee7d5585dd5e976b7edd2b30ac72bc376b3c3f85882"}, ] [package.dependencies] @@ -605,25 +692,24 @@ numpy = "*" [[package]] name = "chromadb" -version = "0.4.3" +version = "0.4.10" description = "Chroma." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "chromadb-0.4.3-py3-none-any.whl", hash = "sha256:01f146338e9696ccb7fb8ad2b4ef1b9e4e44900c40a78dda67430a0c041de9f3"}, - {file = "chromadb-0.4.3.tar.gz", hash = "sha256:d86c61005b138d1676ca7963b3bd46cbbf2f4d86b7c06f0288048e8d668ade92"}, + {file = "chromadb-0.4.10-py3-none-any.whl", hash = "sha256:69e8c627cebb093cb211cd2e33959ec6edf66c9cdfcddf9f30902bd3c9bd23ac"}, + {file = "chromadb-0.4.10.tar.gz", hash = "sha256:1bbb72f5f69b7a0fa9c7f1d74c6ca6197d2991a4333598aa97fd90d89a8bd112"}, ] [package.dependencies] -chroma-hnswlib = "0.7.1" +bcrypt = ">=4.0.1" +chroma-hnswlib = "0.7.3" fastapi = ">=0.95.2,<0.100.0" graphlib-backport = {version = ">=1.0.3", markers = "python_version < \"3.9\""} importlib-resources = "*" -numpy = ">=1.21.6" +numpy = {version = ">=1.22.5", markers = "python_version >= \"3.8\""} onnxruntime = ">=1.14.1" overrides = ">=7.3.1" -pandas = ">=1.3" posthog = ">=2.4.0" pulsar-client = ">=3.1.0" pydantic = ">=1.9,<2.0" @@ -636,14 +722,13 @@ uvicorn = {version = ">=0.18.3", extras = ["standard"]} [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] @@ -651,29 +736,28 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cmake" -version = "3.27.0" +version = "3.27.2" description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" -category = "dev" optional = false python-versions = "*" files = [ - {file = "cmake-3.27.0-py2.py3-none-macosx_10_10_universal2.macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:9ccab4cd93578d3c2df32e66b44b313b75a7484032645040431dc06a583ca4aa"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2010_i686.manylinux_2_12_i686.whl", hash = "sha256:199bfaefb752e82d8067aeee5d6a6e0414fe0d60e9a3fd08e95d537a97e0db16"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:8745eff805f36762d3e8e904698b853cb4a9da8b4b07d1c12bcd1e1a6c4a1709"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58a3f39d3d1bc897f05e531bfa676246a2b25d424c6a47e4b6bbc193fb560db7"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b470ccd3f86cf19a63f6b221c9cceebcc58e32d3787d0d5f9f43d1d91a095090"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:35a8d397ce883e93b5e6561e2803ce9470df52283862264093c1078530f98189"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1f38d87b2c65763a0113f4a6c652e6f4b5adf90b384c1e1d69e4f8a3104a57d6"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b9d5811954dcedcaa6c915c4a9bb6d64b55ac189e9cbc74be726307d9d084804"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:073e4f196d0888216e6794c08cd984ddabc108c0e4e66f48fbd7610d1e6d726d"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:e58e48643903e6fad76274337f9a4d3c575b8e21cd05c6214780b2c98bb0c706"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:9740ed9f61a3bd8708a41cadd5c057c04f38e5b89bd773e369df2e210a1c55a3"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:1b3189171665f5c8d748ae7fe10a29fff1ebeedeaef57b16f1ea54b1ec0fe514"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:c4c968c188e7518deb463a14e64f3a19f242c9dcf7f24e1dbcc1419690cd54e0"}, - {file = "cmake-3.27.0-py2.py3-none-win32.whl", hash = "sha256:5561aca62b65aac844f3931e74cfeb696e4534de145e3307bf942e735736541e"}, - {file = "cmake-3.27.0-py2.py3-none-win_amd64.whl", hash = "sha256:48be3afe62c9513a49be007896a4058fafec512cb1f269a50126da30aacad97f"}, - {file = "cmake-3.27.0-py2.py3-none-win_arm64.whl", hash = "sha256:6f46a170b0c9c552d52da4346534570f818195dfc4f1d0c03264e24cc348fc60"}, - {file = "cmake-3.27.0.tar.gz", hash = "sha256:d03f0a76a2b96805044ad1178b92aeeb5f695caa6776a32522bb5c430a55b4e8"}, + {file = "cmake-3.27.2-py2.py3-none-macosx_10_10_universal2.macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:96ac856c4d6b2104408848f0005a8ab2229d4135b171ea9a03e8c33039ede420"}, + {file = "cmake-3.27.2-py2.py3-none-manylinux2010_i686.manylinux_2_12_i686.whl", hash = "sha256:11fe6129d07982721c5965fd804a4056b8c6e9c4f482ac9e0fe41bb3abc1ab5f"}, + {file = "cmake-3.27.2-py2.py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:f0c64e89e2ea59592980c4fe3821d712fee0e74cf87c2aaec5b3ab9aa809a57c"}, + {file = "cmake-3.27.2-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ca7650477dff2a1138776b28b79c0e99127be733d3978922e8f87b56a433eed6"}, + {file = "cmake-3.27.2-py2.py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:ab2e40fe09e76a7ef67da2bbbf7a4cd1f52db4f1c7b6ccdda2539f918830343a"}, + {file = "cmake-3.27.2-py2.py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:980ee19f12c808cb8ddb56fdcee832501a9f9631799d8b4fc625c0a0b5fb4c55"}, + {file = "cmake-3.27.2-py2.py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:115d30ca0760e3861d9ad6b3288cd11ee72a785b81227da0c1765d3b84e2c009"}, + {file = "cmake-3.27.2-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efc338c939d6d435890a52458a260bf0942bd8392b648d7532a72c1ec0764e18"}, + {file = "cmake-3.27.2-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:7f7438c60ccc01765b67abfb1797787c3b9459d500a804ed70a4cc181bc02204"}, + {file = "cmake-3.27.2-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:294f008734267e0eee1574ad1b911bed137bc907ab19d60a618dab4615aa1fca"}, + {file = "cmake-3.27.2-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:197a34dc62ee149ced343545fac67e5a30b93fda65250b065726f86ce92bdada"}, + {file = "cmake-3.27.2-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:afb46ad883b174fb64347802ba5878423551dbd5847bb64669c39a5957c06eb7"}, + {file = "cmake-3.27.2-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:83611ffd155e270a6b13bbf0cfd4e8688ebda634f448aa2e3734006c745bf33f"}, + {file = "cmake-3.27.2-py2.py3-none-win32.whl", hash = "sha256:53e12deb893da935e236f93accd47dbe2806620cd7654986234dc4487cc49652"}, + {file = "cmake-3.27.2-py2.py3-none-win_amd64.whl", hash = "sha256:611f9722c68c40352d38a6c01960ab038c3d0419e7aee3bf18f95b23031e0dfe"}, + {file = "cmake-3.27.2-py2.py3-none-win_arm64.whl", hash = "sha256:30620326b51ac2ce0d8f476747af6367a7ea21075c4d065fad9443904b07476a"}, + {file = "cmake-3.27.2.tar.gz", hash = "sha256:7cd6e2d7d5a1125f8c26c4f65214f8c942e3f276f98c16cb62ae382c35609f25"}, ] [package.extras] @@ -683,7 +767,6 @@ test = ["coverage (>=4.2)", "flake8 (>=3.0.4)", "path.py (>=11.5.0)", "pytest (> name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -695,7 +778,6 @@ files = [ name = "coloredlogs" version = "15.0.1" description = "Colored terminal output for Python's logging module" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -711,18 +793,17 @@ cron = ["capturer (>=2.4)"] [[package]] name = "comm" -version = "0.1.3" +version = "0.1.4" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." -category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "comm-0.1.3-py3-none-any.whl", hash = "sha256:16613c6211e20223f215fc6d3b266a247b6e2641bf4e0a3ad34cb1aff2aa3f37"}, - {file = "comm-0.1.3.tar.gz", hash = "sha256:a61efa9daffcfbe66fd643ba966f846a624e4e6d6767eda9cf6e993aadaab93e"}, + {file = "comm-0.1.4-py3-none-any.whl", hash = "sha256:6d52794cba11b36ed9860999cd10fd02d6b2eac177068fdd585e1e2f8a96e67a"}, + {file = "comm-0.1.4.tar.gz", hash = "sha256:354e40a59c9dd6db50c5cc6b4acc887d82e9603787f83b68c01a80a923984d15"}, ] [package.dependencies] -traitlets = ">=5.3" +traitlets = ">=4" [package.extras] lint = ["black (>=22.6.0)", "mdformat (>0.7)", "mdformat-gfm (>=0.3.5)", "ruff (>=0.0.156)"] @@ -731,35 +812,34 @@ typing = ["mypy (>=0.990)"] [[package]] name = "cryptography" -version = "41.0.2" +version = "41.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711"}, - {file = "cryptography-41.0.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182"}, - {file = "cryptography-41.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83"}, - {file = "cryptography-41.0.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5"}, - {file = "cryptography-41.0.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58"}, - {file = "cryptography-41.0.2-cp37-abi3-win32.whl", hash = "sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76"}, - {file = "cryptography-41.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766"}, - {file = "cryptography-41.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa"}, - {file = "cryptography-41.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f"}, - {file = "cryptography-41.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0"}, - {file = "cryptography-41.0.2.tar.gz", hash = "sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"}, + {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"}, + {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"}, + {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"}, ] [package.dependencies] @@ -777,37 +857,35 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "debugpy" -version = "1.6.7" +version = "1.6.7.post1" description = "An implementation of the Debug Adapter Protocol for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "debugpy-1.6.7-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b3e7ac809b991006ad7f857f016fa92014445085711ef111fdc3f74f66144096"}, - {file = "debugpy-1.6.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3876611d114a18aafef6383695dfc3f1217c98a9168c1aaf1a02b01ec7d8d1e"}, - {file = "debugpy-1.6.7-cp310-cp310-win32.whl", hash = "sha256:33edb4afa85c098c24cc361d72ba7c21bb92f501104514d4ffec1fb36e09c01a"}, - {file = "debugpy-1.6.7-cp310-cp310-win_amd64.whl", hash = "sha256:ed6d5413474e209ba50b1a75b2d9eecf64d41e6e4501977991cdc755dc83ab0f"}, - {file = "debugpy-1.6.7-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:38ed626353e7c63f4b11efad659be04c23de2b0d15efff77b60e4740ea685d07"}, - {file = "debugpy-1.6.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:279d64c408c60431c8ee832dfd9ace7c396984fd7341fa3116aee414e7dcd88d"}, - {file = "debugpy-1.6.7-cp37-cp37m-win32.whl", hash = "sha256:dbe04e7568aa69361a5b4c47b4493d5680bfa3a911d1e105fbea1b1f23f3eb45"}, - {file = "debugpy-1.6.7-cp37-cp37m-win_amd64.whl", hash = "sha256:f90a2d4ad9a035cee7331c06a4cf2245e38bd7c89554fe3b616d90ab8aab89cc"}, - {file = "debugpy-1.6.7-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:5224eabbbeddcf1943d4e2821876f3e5d7d383f27390b82da5d9558fd4eb30a9"}, - {file = "debugpy-1.6.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae1123dff5bfe548ba1683eb972329ba6d646c3a80e6b4c06cd1b1dd0205e9b"}, - {file = "debugpy-1.6.7-cp38-cp38-win32.whl", hash = "sha256:9cd10cf338e0907fdcf9eac9087faa30f150ef5445af5a545d307055141dd7a4"}, - {file = "debugpy-1.6.7-cp38-cp38-win_amd64.whl", hash = "sha256:aaf6da50377ff4056c8ed470da24632b42e4087bc826845daad7af211e00faad"}, - {file = "debugpy-1.6.7-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:0679b7e1e3523bd7d7869447ec67b59728675aadfc038550a63a362b63029d2c"}, - {file = "debugpy-1.6.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de86029696e1b3b4d0d49076b9eba606c226e33ae312a57a46dca14ff370894d"}, - {file = "debugpy-1.6.7-cp39-cp39-win32.whl", hash = "sha256:d71b31117779d9a90b745720c0eab54ae1da76d5b38c8026c654f4a066b0130a"}, - {file = "debugpy-1.6.7-cp39-cp39-win_amd64.whl", hash = "sha256:c0ff93ae90a03b06d85b2c529eca51ab15457868a377c4cc40a23ab0e4e552a3"}, - {file = "debugpy-1.6.7-py2.py3-none-any.whl", hash = "sha256:53f7a456bc50706a0eaabecf2d3ce44c4d5010e46dfc65b6b81a518b42866267"}, - {file = "debugpy-1.6.7.zip", hash = "sha256:c4c2f0810fa25323abfdfa36cbbbb24e5c3b1a42cb762782de64439c575d67f2"}, + {file = "debugpy-1.6.7.post1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:903bd61d5eb433b6c25b48eae5e23821d4c1a19e25c9610205f5aeaccae64e32"}, + {file = "debugpy-1.6.7.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d16882030860081e7dd5aa619f30dec3c2f9a421e69861125f83cc372c94e57d"}, + {file = "debugpy-1.6.7.post1-cp310-cp310-win32.whl", hash = "sha256:eea8d8cfb9965ac41b99a61f8e755a8f50e9a20330938ad8271530210f54e09c"}, + {file = "debugpy-1.6.7.post1-cp310-cp310-win_amd64.whl", hash = "sha256:85969d864c45f70c3996067cfa76a319bae749b04171f2cdeceebe4add316155"}, + {file = "debugpy-1.6.7.post1-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:890f7ab9a683886a0f185786ffbda3b46495c4b929dab083b8c79d6825832a52"}, + {file = "debugpy-1.6.7.post1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4ac7a4dba28801d184b7fc0e024da2635ca87d8b0a825c6087bb5168e3c0d28"}, + {file = "debugpy-1.6.7.post1-cp37-cp37m-win32.whl", hash = "sha256:3370ef1b9951d15799ef7af41f8174194f3482ee689988379763ef61a5456426"}, + {file = "debugpy-1.6.7.post1-cp37-cp37m-win_amd64.whl", hash = "sha256:65b28435a17cba4c09e739621173ff90c515f7b9e8ea469b92e3c28ef8e5cdfb"}, + {file = "debugpy-1.6.7.post1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:92b6dae8bfbd497c90596bbb69089acf7954164aea3228a99d7e43e5267f5b36"}, + {file = "debugpy-1.6.7.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72f5d2ecead8125cf669e62784ef1e6300f4067b0f14d9f95ee00ae06fc7c4f7"}, + {file = "debugpy-1.6.7.post1-cp38-cp38-win32.whl", hash = "sha256:f0851403030f3975d6e2eaa4abf73232ab90b98f041e3c09ba33be2beda43fcf"}, + {file = "debugpy-1.6.7.post1-cp38-cp38-win_amd64.whl", hash = "sha256:3de5d0f97c425dc49bce4293df6a04494309eedadd2b52c22e58d95107e178d9"}, + {file = "debugpy-1.6.7.post1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:38651c3639a4e8bbf0ca7e52d799f6abd07d622a193c406be375da4d510d968d"}, + {file = "debugpy-1.6.7.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:038c51268367c9c935905a90b1c2d2dbfe304037c27ba9d19fe7409f8cdc710c"}, + {file = "debugpy-1.6.7.post1-cp39-cp39-win32.whl", hash = "sha256:4b9eba71c290852f959d2cf8a03af28afd3ca639ad374d393d53d367f7f685b2"}, + {file = "debugpy-1.6.7.post1-cp39-cp39-win_amd64.whl", hash = "sha256:973a97ed3b434eab0f792719a484566c35328196540676685c975651266fccf9"}, + {file = "debugpy-1.6.7.post1-py2.py3-none-any.whl", hash = "sha256:1093a5c541af079c13ac8c70ab8b24d1d35c8cacb676306cf11e57f699c02926"}, + {file = "debugpy-1.6.7.post1.zip", hash = "sha256:fe87ec0182ef624855d05e6ed7e0b7cb1359d2ffa2a925f8ec2d22e98b75d0ca"}, ] [[package]] name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -819,7 +897,6 @@ files = [ name = "distlib" version = "0.3.7" description = "Distribution utilities" -category = "dev" optional = false python-versions = "*" files = [ @@ -829,23 +906,18 @@ files = [ [[package]] name = "dnspython" -version = "2.4.0" +version = "2.4.2" description = "DNS toolkit" -category = "dev" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "dnspython-2.4.0-py3-none-any.whl", hash = "sha256:46b4052a55b56beea3a3bdd7b30295c292bd6827dd442348bc116f2d35b17f0a"}, - {file = "dnspython-2.4.0.tar.gz", hash = "sha256:758e691dbb454d5ccf4e1b154a19e52847f79e21a42fef17b969144af29a4e6c"}, + {file = "dnspython-2.4.2-py3-none-any.whl", hash = "sha256:57c6fbaaeaaf39c891292012060beb141791735dbb4004798328fc2c467402d8"}, + {file = "dnspython-2.4.2.tar.gz", hash = "sha256:8dcfae8c7460a2f84b4072e26f1c9f4101ca20c071649cb7c34e8b6a93d58984"}, ] -[package.dependencies] -httpcore = {version = ">=0.17.3", markers = "python_version >= \"3.8\""} -sniffio = ">=1.1,<2.0" - [package.extras] dnssec = ["cryptography (>=2.6,<42.0)"] -doh = ["h2 (>=4.1.0)", "httpx (>=0.24.1)"] +doh = ["h2 (>=4.1.0)", "httpcore (>=0.17.3)", "httpx (>=0.24.1)"] doq = ["aioquic (>=0.9.20)"] idna = ["idna (>=2.1,<4.0)"] trio = ["trio (>=0.14,<0.23)"] @@ -855,7 +927,6 @@ wmi = ["wmi (>=1.5.1,<2.0.0)"] name = "environs" version = "9.5.0" description = "simplified environment variable parsing" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -875,14 +946,13 @@ tests = ["dj-database-url", "dj-email-url", "django-cache-url", "pytest"] [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.1.3" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, ] [package.extras] @@ -892,7 +962,6 @@ test = ["pytest (>=6)"] name = "executing" version = "1.2.0" description = "Get the currently executing AST node of a frame, and other information" -category = "dev" optional = false python-versions = "*" files = [ @@ -907,7 +976,6 @@ tests = ["asttokens", "littleutils", "pytest", "rich"] name = "fastapi" version = "0.99.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -927,7 +995,6 @@ all = ["email-validator (>=1.1.1)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)" name = "filelock" version = "3.12.2" description = "A platform independent file lock." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -943,7 +1010,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "flatbuffers" version = "23.5.26" description = "The FlatBuffers serialization format for Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -955,7 +1021,6 @@ files = [ name = "frozenlist" version = "1.4.0" description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1026,7 +1091,6 @@ files = [ name = "fsspec" version = "2023.6.0" description = "File-system specification" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1058,11 +1122,121 @@ smb = ["smbprotocol"] ssh = ["paramiko"] tqdm = ["tqdm"] +[[package]] +name = "google-ai-generativelanguage" +version = "0.3.3" +description = "Google Ai Generativelanguage API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-ai-generativelanguage-0.3.3.tar.gz", hash = "sha256:4b59993e0fd63593171cbb089e7f76f71a4333a62741d3929159aeb2e3532a83"}, + {file = "google_ai_generativelanguage-0.3.3-py3-none-any.whl", hash = "sha256:2696fe952ceea233e1a95b89a428b7dd587eac6687bd20cc62edd5c8abc32b98"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +proto-plus = [ + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, + {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[[package]] +name = "google-api-core" +version = "2.11.1" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, + {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +grpcio = [ + {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, +] +grpcio-status = [ + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.22.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce"}, + {file = "google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" +six = ">=1.9.0" +urllib3 = "<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-generativeai" +version = "0.2.1" +description = "Google Generative AI High level API client library and tools." +optional = false +python-versions = ">=3.9" +files = [ + {file = "google_generativeai-0.2.1-py3-none-any.whl", hash = "sha256:892c80f33fda68f531e97de67c7796f9c10f68708599506bba2388c53d1d332e"}, +] + +[package.dependencies] +google-ai-generativelanguage = "0.3.3" +google-api-core = "*" +google-auth = "*" +protobuf = "*" +tqdm = "*" + +[package.extras] +dev = ["absl-py", "black", "nose2", "pandas", "pytype", "pyyaml"] + +[[package]] +name = "googleapis-common-protos" +version = "1.60.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, + {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + [[package]] name = "graphlib-backport" version = "1.0.3" description = "Backport of the Python 3.9 graphlib module for Python 3.6+" -category = "dev" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -1074,7 +1248,6 @@ files = [ name = "grpcio" version = "1.56.0" description = "HTTP/2-based RPC framework" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1128,11 +1301,26 @@ files = [ [package.extras] protobuf = ["grpcio-tools (>=1.56.0)"] +[[package]] +name = "grpcio-status" +version = "1.56.0" +description = "Status proto mapping for gRPC" +optional = false +python-versions = ">=3.6" +files = [ + {file = "grpcio-status-1.56.0.tar.gz", hash = "sha256:9eca0b2dcda0782d3702df225918efd6d820f75f93cd5c51c7fb6a4ffbfea12c"}, + {file = "grpcio_status-1.56.0-py3-none-any.whl", hash = "sha256:e5f101c96686e9d4e94a114567960fdb00052aa3c818b029745e3db37dc9c613"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.56.0" +protobuf = ">=4.21.6" + [[package]] name = "grpcio-tools" version = "1.56.0" description = "Protobuf code generator for gRPC" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1192,7 +1380,6 @@ setuptools = "*" name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1204,7 +1391,6 @@ files = [ name = "h2" version = "4.1.0" description = "HTTP/2 State-Machine based protocol implementation" -category = "dev" optional = false python-versions = ">=3.6.1" files = [ @@ -1220,7 +1406,6 @@ hyperframe = ">=6.0,<7" name = "hpack" version = "4.0.0" description = "Pure-Python HPACK header compression" -category = "dev" optional = false python-versions = ">=3.6.1" files = [ @@ -1232,7 +1417,6 @@ files = [ name = "httpcore" version = "0.17.3" description = "A minimal low-level HTTP client." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1244,17 +1428,16 @@ files = [ anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = ">=1.0.0,<2.0.0" +sniffio = "==1.*" [package.extras] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] [[package]] name = "httptools" version = "0.6.0" description = "A collection of framework independent HTTP protocol utils." -category = "dev" optional = false python-versions = ">=3.5.0" files = [ @@ -1302,7 +1485,6 @@ test = ["Cython (>=0.29.24,<0.30.0)"] name = "httpx" version = "0.24.1" description = "The next generation HTTP client." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1319,15 +1501,14 @@ sniffio = "*" [package.extras] brotli = ["brotli", "brotlicffi"] -cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] [[package]] name = "huggingface-hub" version = "0.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -1360,7 +1541,6 @@ typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "typ name = "humanfriendly" version = "10.0" description = "Human friendly output for text interfaces using Python" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1375,7 +1555,6 @@ pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_ve name = "hyperframe" version = "6.0.1" description = "HTTP/2 framing layer for Python" -category = "dev" optional = false python-versions = ">=3.6.1" files = [ @@ -1385,14 +1564,13 @@ files = [ [[package]] name = "identify" -version = "2.5.26" +version = "2.5.27" description = "File identification library for Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.26-py2.py3-none-any.whl", hash = "sha256:c22a8ead0d4ca11f1edd6c9418c3220669b3b7533ada0a0ffa6cc0ef85cf9b54"}, - {file = "identify-2.5.26.tar.gz", hash = "sha256:7243800bce2f58404ed41b7c002e53d4d22bcf3ae1b7900c2d7aefd95394bf7f"}, + {file = "identify-2.5.27-py2.py3-none-any.whl", hash = "sha256:fdb527b2dfe24602809b2201e033c2a113d7bdf716db3ca8e3243f735dcecaba"}, + {file = "identify-2.5.27.tar.gz", hash = "sha256:287b75b04a0e22d727bc9a41f0d4f3c1bcada97490fa6eabb5b28f0e9097e733"}, ] [package.extras] @@ -1402,7 +1580,6 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1414,7 +1591,6 @@ files = [ name = "importlib-metadata" version = "6.8.0" description = "Read metadata from Python packages" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1432,14 +1608,13 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs [[package]] name = "importlib-resources" -version = "6.0.0" +version = "5.13.0" description = "Read resources from Python packages" -category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_resources-6.0.0-py3-none-any.whl", hash = "sha256:d952faee11004c045f785bb5636e8f885bed30dc3c940d5d42798a2a4541c185"}, - {file = "importlib_resources-6.0.0.tar.gz", hash = "sha256:4cf94875a8368bd89531a756df9a9ebe1f150e0f885030b461237bc7f2d905f2"}, + {file = "importlib_resources-5.13.0-py3-none-any.whl", hash = "sha256:9f7bd0c97b79972a6cce36a366356d16d5e13b09679c11a58f1014bfdf8e64b2"}, + {file = "importlib_resources-5.13.0.tar.gz", hash = "sha256:82d5c6cca930697dbbd86c93333bb2c2e72861d4789a11c2662b933e5ad2b528"}, ] [package.dependencies] @@ -1453,7 +1628,6 @@ testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1463,14 +1637,13 @@ files = [ [[package]] name = "ipykernel" -version = "6.25.0" +version = "6.25.2" description = "IPython Kernel for Jupyter" -category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "ipykernel-6.25.0-py3-none-any.whl", hash = "sha256:f0042e867ac3f6bca1679e6a88cbd6a58ed93a44f9d0866aecde6efe8de76659"}, - {file = "ipykernel-6.25.0.tar.gz", hash = "sha256:e342ce84712861be4b248c4a73472be4702c1b0dd77448bfd6bcfb3af9d5ddf9"}, + {file = "ipykernel-6.25.2-py3-none-any.whl", hash = "sha256:2e2ee359baba19f10251b99415bb39de1e97d04e1fab385646f24f0596510b77"}, + {file = "ipykernel-6.25.2.tar.gz", hash = "sha256:f468ddd1f17acb48c8ce67fcfa49ba6d46d4f9ac0438c1f441be7c3d1372230b"}, ] [package.dependencies] @@ -1479,7 +1652,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" matplotlib-inline = ">=0.1" nest-asyncio = "*" packaging = "*" @@ -1499,7 +1672,6 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio" name = "ipython" version = "8.12.2" description = "IPython: Productive Interactive Computing" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1539,7 +1711,6 @@ test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pa name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" -category = "dev" optional = false python-versions = "*" files = [ @@ -1552,29 +1723,27 @@ six = "*" [[package]] name = "jedi" -version = "0.18.2" +version = "0.19.0" description = "An autocompletion tool for Python that can be used for text editors." -category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "jedi-0.18.2-py2.py3-none-any.whl", hash = "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e"}, - {file = "jedi-0.18.2.tar.gz", hash = "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"}, + {file = "jedi-0.19.0-py2.py3-none-any.whl", hash = "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"}, + {file = "jedi-0.19.0.tar.gz", hash = "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4"}, ] [package.dependencies] -parso = ">=0.8.0,<0.9.0" +parso = ">=0.8.3,<0.9.0" [package.extras] docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] [[package]] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1590,21 +1759,74 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "joblib" -version = "1.3.1" +version = "1.3.2" description = "Lightweight pipelining with Python functions" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "joblib-1.3.1-py3-none-any.whl", hash = "sha256:89cf0529520e01b3de7ac7b74a8102c90d16d54c64b5dd98cafcd14307fdf915"}, - {file = "joblib-1.3.1.tar.gz", hash = "sha256:1f937906df65329ba98013dc9692fe22a4c5e4a648112de500508b18a21b41e3"}, + {file = "joblib-1.3.2-py3-none-any.whl", hash = "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"}, + {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, +] + +[[package]] +name = "jsonschema" +version = "4.19.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema-4.19.0-py3-none-any.whl", hash = "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb"}, + {file = "jsonschema-4.19.0.tar.gz", hash = "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +jsonschema-specifications = ">=2023.03.6" +pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} +referencing = ">=0.28.4" +rpds-py = ">=0.7.1" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "jsonschema-spec" +version = "0.2.4" +description = "JSONSchema Spec with object-oriented paths" +optional = false +python-versions = ">=3.8.0,<4.0.0" +files = [ + {file = "jsonschema_spec-0.2.4-py3-none-any.whl", hash = "sha256:e6dcf7056734ec6854f7888da6c08ce6c421f28aeeddce96bb90de0fb6d711ef"}, + {file = "jsonschema_spec-0.2.4.tar.gz", hash = "sha256:873e396ad1ba6edf9f52d6174c110d4fafb7b5f5894744246a53fe75e5251ec2"}, +] + +[package.dependencies] +pathable = ">=0.4.1,<0.5.0" +PyYAML = ">=5.1" +referencing = ">=0.28.0,<0.31.0" +requests = ">=2.31.0,<3.0.0" + +[[package]] +name = "jsonschema-specifications" +version = "2023.7.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema_specifications-2023.7.1-py3-none-any.whl", hash = "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1"}, + {file = "jsonschema_specifications-2023.7.1.tar.gz", hash = "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"}, ] +[package.dependencies] +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +referencing = ">=0.28.0" + [[package]] name = "jupyter-client" version = "8.3.0" description = "Jupyter protocol implementation and client libraries" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1614,7 +1836,7 @@ files = [ [package.dependencies] importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" python-dateutil = ">=2.8.2" pyzmq = ">=23.0" tornado = ">=6.2" @@ -1628,7 +1850,6 @@ test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pyt name = "jupyter-core" version = "5.3.1" description = "Jupyter core package. A base package on which Jupyter projects rely." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1645,11 +1866,55 @@ traitlets = ">=5.3" docs = ["myst-parser", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] +[[package]] +name = "lazy-object-proxy" +version = "1.9.0" +description = "A fast and thorough lazy object proxy." +optional = false +python-versions = ">=3.7" +files = [ + {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, +] + [[package]] name = "lit" version = "16.0.6" description = "A Software Testing Tool" -category = "dev" optional = false python-versions = "*" files = [ @@ -1660,7 +1925,6 @@ files = [ name = "loguru" version = "0.7.0" description = "Python logging made (stupidly) simple" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1679,7 +1943,6 @@ dev = ["Sphinx (==5.3.0)", "colorama (==0.4.5)", "colorama (==0.4.6)", "freezegu name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1703,6 +1966,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1739,7 +2012,6 @@ files = [ name = "marshmallow" version = "3.20.1" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1760,7 +2032,6 @@ tests = ["pytest", "pytz", "simplejson"] name = "matplotlib-inline" version = "0.1.6" description = "Inline Matplotlib backend for Jupyter" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1773,26 +2044,39 @@ traitlets = "*" [[package]] name = "milvus" -version = "2.2.11" +version = "2.2.13" description = "Embeded Milvus" -category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "milvus-2.2.11-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:64fa0fcbce1cb763d3aac0749cc17e04761e832297eae12ba5c97938f1acd243"}, - {file = "milvus-2.2.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c439d4231019e8cb78b13572dcd78a388cb63a5c271a2ab059bb54f019b1eb1c"}, - {file = "milvus-2.2.11-py3-none-manylinux2014_x86_64.whl", hash = "sha256:d124cf7d6f914177ba14fb38c6a4ea305e3b6a8a09a86e7fc80f44270c0f6ede"}, - {file = "milvus-2.2.11-py3-none-win_amd64.whl", hash = "sha256:118569f56584670f8b1b7b4c89c0050b4678884b4719b8659edb1d47f12bd177"}, + {file = "milvus-2.2.13-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:fcaff6cdc885ab46f432b79294a298e2cac542ffdbcb8c61be5f4c1f1c27dbeb"}, + {file = "milvus-2.2.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:58dfe6b2630a981475c56dc06fe414d2d821c94651b3341e62e7de8bc2850ec9"}, + {file = "milvus-2.2.13-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9f95afcdb7b8912ae7c8a02604ed3a057462df25e6fa22112db1ead828e371a1"}, + {file = "milvus-2.2.13-py3-none-win_amd64.whl", hash = "sha256:03c55043321a72d41d2dcb49745836f10156c61bfc9c48fa58345ae27dbbf164"}, ] [package.extras] -client = ["pymilvus (>=2.2.0,<2.3.0)"] +client = ["pymilvus (>=2.2.0,!=2.2.14,<2.3.0)"] + +[[package]] +name = "minio" +version = "7.1.17" +description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" +optional = false +python-versions = "*" +files = [ + {file = "minio-7.1.17-py3-none-any.whl", hash = "sha256:0aa525d77a3bc61378444c2400b0ba2685ad4cd6ecb3fba4141a0d0765e25f40"}, + {file = "minio-7.1.17.tar.gz", hash = "sha256:b0b687c1ec9be422a1f8b04c65fb8e43a1c090f9508178db57c434a17341c404"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = "*" [[package]] name = "monotonic" version = "1.6" description = "An implementation of time.monotonic() for Python 2 & < 3.3" -category = "dev" optional = false python-versions = "*" files = [ @@ -1800,11 +2084,45 @@ files = [ {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"}, ] +[[package]] +name = "more-itertools" +version = "10.1.0" +description = "More routines for operating on iterables, beyond itertools" +optional = false +python-versions = ">=3.8" +files = [ + {file = "more-itertools-10.1.0.tar.gz", hash = "sha256:626c369fa0eb37bac0291bce8259b332fd59ac792fa5497b59837309cd5b114a"}, + {file = "more_itertools-10.1.0-py3-none-any.whl", hash = "sha256:64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6"}, +] + +[[package]] +name = "motor" +version = "3.3.1" +description = "Non-blocking MongoDB driver for Tornado or asyncio" +optional = false +python-versions = ">=3.7" +files = [ + {file = "motor-3.3.1-py3-none-any.whl", hash = "sha256:a0dee83ad0d47b353932ac37467ba397b1e649ce7e3eea7f5a90554883d7cdbe"}, + {file = "motor-3.3.1.tar.gz", hash = "sha256:c5eb400e27d722a3db03a9826656b6d13acf9b6c70c2fb4604f474eac9da5be4"}, +] + +[package.dependencies] +pymongo = ">=4.5,<5" + +[package.extras] +aws = ["pymongo[aws] (>=4.5,<5)"] +encryption = ["pymongo[encryption] (>=4.5,<5)"] +gssapi = ["pymongo[gssapi] (>=4.5,<5)"] +ocsp = ["pymongo[ocsp] (>=4.5,<5)"] +snappy = ["pymongo[snappy] (>=4.5,<5)"] +srv = ["pymongo[srv] (>=4.5,<5)"] +test = ["aiohttp", "mockupdb", "motor[encryption]", "pytest (>=7)", "tornado (>=5)"] +zstd = ["pymongo[zstd] (>=4.5,<5)"] + [[package]] name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" -category = "dev" optional = false python-versions = "*" files = [ @@ -1822,7 +2140,6 @@ tests = ["pytest (>=4.6)"] name = "msal" version = "1.23.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." -category = "dev" optional = false python-versions = "*" files = [ @@ -1842,7 +2159,6 @@ broker = ["pymsalruntime (>=0.13.2,<0.14)"] name = "msal-extensions" version = "1.0.0" description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." -category = "dev" optional = false python-versions = "*" files = [ @@ -1861,7 +2177,6 @@ portalocker = [ name = "multidict" version = "6.0.4" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1945,7 +2260,6 @@ files = [ name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1955,21 +2269,19 @@ files = [ [[package]] name = "nest-asyncio" -version = "1.5.6" +version = "1.5.7" description = "Patch asyncio to allow nested event loops" -category = "dev" optional = false python-versions = ">=3.5" files = [ - {file = "nest_asyncio-1.5.6-py3-none-any.whl", hash = "sha256:b9a953fb40dceaa587d109609098db21900182b16440652454a146cffb06e8b8"}, - {file = "nest_asyncio-1.5.6.tar.gz", hash = "sha256:d267cc1ff794403f7df692964d1d2a3fa9418ffea2a3f6859a439ff482fef290"}, + {file = "nest_asyncio-1.5.7-py3-none-any.whl", hash = "sha256:5301c82941b550b3123a1ea772ba9a1c80bad3a182be8c1a5ae6ad3be57a9657"}, + {file = "nest_asyncio-1.5.7.tar.gz", hash = "sha256:6a80f7b98f24d9083ed24608977c09dd608d83f91cccc24c9d2cba6d10e01c10"}, ] [[package]] name = "networkx" version = "3.1" description = "Python package for creating and manipulating graphs and networks" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1988,7 +2300,6 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] name = "nltk" version = "3.8.1" description = "Natural Language Toolkit" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2014,7 +2325,6 @@ twitter = ["twython"] name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -2029,7 +2339,6 @@ setuptools = "*" name = "numpy" version = "1.24.4" description = "Fundamental package for array computing in Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2067,7 +2376,6 @@ files = [ name = "nvidia-cublas-cu11" version = "11.10.3.66" description = "CUBLAS native runtime libraries" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2083,7 +2391,6 @@ wheel = "*" name = "nvidia-cuda-cupti-cu11" version = "11.7.101" description = "CUDA profiling tools runtime libs." -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2099,7 +2406,6 @@ wheel = "*" name = "nvidia-cuda-nvrtc-cu11" version = "11.7.99" description = "NVRTC native runtime libraries" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2116,7 +2422,6 @@ wheel = "*" name = "nvidia-cuda-runtime-cu11" version = "11.7.99" description = "CUDA Runtime native Libraries" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2132,7 +2437,6 @@ wheel = "*" name = "nvidia-cudnn-cu11" version = "8.5.0.96" description = "cuDNN runtime libraries" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2148,7 +2452,6 @@ wheel = "*" name = "nvidia-cufft-cu11" version = "10.9.0.58" description = "CUFFT native runtime libraries" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2160,7 +2463,6 @@ files = [ name = "nvidia-curand-cu11" version = "10.2.10.91" description = "CURAND native runtime libraries" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2176,7 +2478,6 @@ wheel = "*" name = "nvidia-cusolver-cu11" version = "11.4.0.1" description = "CUDA solver native runtime libraries" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2193,7 +2494,6 @@ wheel = "*" name = "nvidia-cusparse-cu11" version = "11.7.4.91" description = "CUSPARSE native runtime libraries" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2209,7 +2509,6 @@ wheel = "*" name = "nvidia-nccl-cu11" version = "2.14.3" description = "NVIDIA Collective Communication Library (NCCL) Runtime" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2220,7 +2519,6 @@ files = [ name = "nvidia-nvtx-cu11" version = "11.7.91" description = "NVIDIA Tools Extension" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -2236,7 +2534,6 @@ wheel = "*" name = "onnxruntime" version = "1.15.1" description = "ONNX Runtime is a runtime accelerator for Machine Learning models" -category = "dev" optional = false python-versions = "*" files = [ @@ -2276,14 +2573,13 @@ sympy = "*" [[package]] name = "openai" -version = "0.27.8" +version = "0.28.0" description = "Python client library for the OpenAI API" -category = "main" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-0.27.8-py3-none-any.whl", hash = "sha256:e0a7c2f7da26bdbe5354b03c6d4b82a2f34bd4458c7a17ae1a7092c3e397e03c"}, - {file = "openai-0.27.8.tar.gz", hash = "sha256:2483095c7db1eee274cebac79e315a986c4e55207bb4fa7b82d185b3a2ed9536"}, + {file = "openai-0.28.0-py3-none-any.whl", hash = "sha256:d207ece78469be5648eb87b825753282225155a29d0eec6e02013ddbf8c31c0c"}, + {file = "openai-0.28.0.tar.gz", hash = "sha256:417b78c4c2864ba696aedaf1ccff77be1f04a581ab1739f0a56e0aae19e5a794"}, ] [package.dependencies] @@ -2293,27 +2589,89 @@ tqdm = "*" [package.extras] datalib = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] -dev = ["black (>=21.6b0,<22.0)", "pytest (>=6.0.0,<7.0.0)", "pytest-asyncio", "pytest-mock"] +dev = ["black (>=21.6b0,<22.0)", "pytest (==6.*)", "pytest-asyncio", "pytest-mock"] embeddings = ["matplotlib", "numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "plotly", "scikit-learn (>=1.0.2)", "scipy", "tenacity (>=8.0.1)"] wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "wandb"] +[[package]] +name = "openapi-core" +version = "0.18.1" +description = "client-side and server-side support for the OpenAPI Specification v3" +optional = false +python-versions = ">=3.8.0,<4.0.0" +files = [ + {file = "openapi_core-0.18.1-py3-none-any.whl", hash = "sha256:603983137a2b954843ef4b85fa36e2d5cceaba50add44c1c2a5165ba5d2954b4"}, + {file = "openapi_core-0.18.1.tar.gz", hash = "sha256:63fa13d9af226ac00119b0531ac9929e3dbb4cbe00216770784473fa6a03bc27"}, +] + +[package.dependencies] +asgiref = ">=3.6.0,<4.0.0" +isodate = "*" +jsonschema = ">=4.18.0,<5.0.0" +jsonschema-spec = ">=0.2.3,<0.3.0" +more-itertools = "*" +openapi-schema-validator = ">=0.6.0,<0.7.0" +openapi-spec-validator = ">=0.6.0,<0.7.0" +parse = "*" +werkzeug = "*" + +[package.extras] +aiohttp = ["aiohttp (>=3.0)", "multidict (>=6.0.4,<7.0.0)"] +django = ["django (>=3.0)"] +falcon = ["falcon (>=3.0)"] +flask = ["flask"] +requests = ["requests"] +starlette = ["starlette (>=0.26.1,<0.32.0)"] + +[[package]] +name = "openapi-schema-validator" +version = "0.6.0" +description = "OpenAPI schema validation for Python" +optional = false +python-versions = ">=3.8.0,<4.0.0" +files = [ + {file = "openapi_schema_validator-0.6.0-py3-none-any.whl", hash = "sha256:9e95b95b621efec5936245025df0d6a7ffacd1551e91d09196b3053040c931d7"}, + {file = "openapi_schema_validator-0.6.0.tar.gz", hash = "sha256:921b7c1144b856ca3813e41ecff98a4050f7611824dfc5c6ead7072636af0520"}, +] + +[package.dependencies] +jsonschema = ">=4.18.0,<5.0.0" +jsonschema-specifications = ">=2023.5.2,<2024.0.0" +rfc3339-validator = "*" + +[[package]] +name = "openapi-spec-validator" +version = "0.6.0" +description = "OpenAPI 2.0 (aka Swagger) and OpenAPI 3 spec validator" +optional = false +python-versions = ">=3.8.0,<4.0.0" +files = [ + {file = "openapi_spec_validator-0.6.0-py3-none-any.whl", hash = "sha256:675f1a3c0d0d8eff9116694acde88bcd4613a95bf5240270724d9d78c78f26d6"}, + {file = "openapi_spec_validator-0.6.0.tar.gz", hash = "sha256:68c4c212c88ef14c6b1a591b895bf742c455783c7ebba2507abd7dbc1365a616"}, +] + +[package.dependencies] +importlib-resources = {version = ">=5.8.0,<6.0.0", markers = "python_version < \"3.9\""} +jsonschema = ">=4.18.0,<5.0.0" +jsonschema-spec = ">=0.2.3,<0.3.0" +lazy-object-proxy = ">=1.7.1,<2.0.0" +openapi-schema-validator = ">=0.6.0,<0.7.0" + [[package]] name = "overrides" -version = "7.3.1" +version = "7.4.0" description = "A decorator to automatically detect mismatch when overriding a method." -category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "overrides-7.3.1-py3-none-any.whl", hash = "sha256:6187d8710a935d09b0bcef8238301d6ee2569d2ac1ae0ec39a8c7924e27f58ca"}, - {file = "overrides-7.3.1.tar.gz", hash = "sha256:8b97c6c1e1681b78cbc9424b138d880f0803c2254c5ebaabdde57bb6c62093f2"}, + {file = "overrides-7.4.0-py3-none-any.whl", hash = "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d"}, + {file = "overrides-7.4.0.tar.gz", hash = "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757"}, ] [[package]] name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2325,7 +2683,6 @@ files = [ name = "pandas" version = "2.0.3" description = "Powerful data structures for data analysis, time series, and statistics" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2359,8 +2716,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -2389,11 +2746,21 @@ sql-other = ["SQLAlchemy (>=1.4.16)"] test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.6.3)"] +[[package]] +name = "parse" +version = "1.19.1" +description = "parse() is the opposite of format()" +optional = false +python-versions = "*" +files = [ + {file = "parse-1.19.1-py2.py3-none-any.whl", hash = "sha256:371ed3800dc63983832159cc9373156613947707bc448b5215473a219dbd4362"}, + {file = "parse-1.19.1.tar.gz", hash = "sha256:cc3a47236ff05da377617ddefa867b7ba983819c664e1afe46249e5b469be464"}, +] + [[package]] name = "parso" version = "0.8.3" description = "A Python Parser" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2405,23 +2772,32 @@ files = [ qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] testing = ["docopt", "pytest (<6.0.0)"] +[[package]] +name = "pathable" +version = "0.4.3" +description = "Object-oriented paths" +optional = false +python-versions = ">=3.7.0,<4.0.0" +files = [ + {file = "pathable-0.4.3-py3-none-any.whl", hash = "sha256:cdd7b1f9d7d5c8b8d3315dbf5a86b2596053ae845f056f57d97c0eefff84da14"}, + {file = "pathable-0.4.3.tar.gz", hash = "sha256:5c869d315be50776cc8a993f3af43e0c60dc01506b399643f919034ebf4cdcab"}, +] + [[package]] name = "pathspec" -version = "0.11.1" +version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] [[package]] name = "pexpect" version = "4.8.0" description = "Pexpect allows easy control of interactive console applications." -category = "dev" optional = false python-versions = "*" files = [ @@ -2436,7 +2812,6 @@ ptyprocess = ">=0.5" name = "pickleshare" version = "0.7.5" description = "Tiny 'shelve'-like database with concurrency support" -category = "dev" optional = false python-versions = "*" files = [ @@ -2446,66 +2821,65 @@ files = [ [[package]] name = "pillow" -version = "10.0.0" +version = "10.0.1" description = "Python Imaging Library (Fork)" -category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "Pillow-10.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891"}, - {file = "Pillow-10.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614"}, - {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b"}, - {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c"}, - {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1"}, - {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf"}, - {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3"}, - {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992"}, - {file = "Pillow-10.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de"}, - {file = "Pillow-10.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485"}, - {file = "Pillow-10.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f"}, - {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3"}, - {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d"}, - {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd"}, - {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629"}, - {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538"}, - {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d"}, - {file = "Pillow-10.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f"}, - {file = "Pillow-10.0.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883"}, - {file = "Pillow-10.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e"}, - {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640"}, - {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568"}, - {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223"}, - {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff"}, - {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551"}, - {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5"}, - {file = "Pillow-10.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199"}, - {file = "Pillow-10.0.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3"}, - {file = "Pillow-10.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3"}, - {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43"}, - {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d"}, - {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530"}, - {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51"}, - {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86"}, - {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7"}, - {file = "Pillow-10.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0"}, - {file = "Pillow-10.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa"}, - {file = "Pillow-10.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90"}, - {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967"}, - {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d"}, - {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3"}, - {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba"}, - {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3"}, - {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017"}, - {file = "Pillow-10.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6"}, - {file = "Pillow-10.0.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0"}, - {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba"}, - {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3"}, - {file = "Pillow-10.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334"}, - {file = "Pillow-10.0.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2"}, - {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed"}, - {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684"}, - {file = "Pillow-10.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3"}, - {file = "Pillow-10.0.0.tar.gz", hash = "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396"}, + {file = "Pillow-10.0.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a"}, + {file = "Pillow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff"}, + {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf"}, + {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd"}, + {file = "Pillow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0"}, + {file = "Pillow-10.0.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"}, + {file = "Pillow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2"}, + {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b"}, + {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1"}, + {file = "Pillow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088"}, + {file = "Pillow-10.0.1-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b"}, + {file = "Pillow-10.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91"}, + {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4"}, + {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08"}, + {file = "Pillow-10.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08"}, + {file = "Pillow-10.0.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a"}, + {file = "Pillow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7"}, + {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a"}, + {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7"}, + {file = "Pillow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3"}, + {file = "Pillow-10.0.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849"}, + {file = "Pillow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145"}, + {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2"}, + {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf"}, + {file = "Pillow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d"}, + {file = "Pillow-10.0.1.tar.gz", hash = "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d"}, ] [package.extras] @@ -2516,7 +2890,6 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pinecone-client" version = "2.2.2" description = "Pinecone client and SDK" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2538,27 +2911,36 @@ urllib3 = ">=1.21.1" [package.extras] grpc = ["googleapis-common-protos (>=1.53.0)", "grpc-gateway-protoc-gen-openapiv2 (==0.1.0)", "grpcio (>=1.44.0)", "lz4 (>=3.1.3)", "protobuf (>=3.19.5,<3.20.0)"] +[[package]] +name = "pkgutil-resolve-name" +version = "1.3.10" +description = "Resolve a name to an object." +optional = false +python-versions = ">=3.6" +files = [ + {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, + {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, +] + [[package]] name = "platformdirs" -version = "3.9.1" +version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.9.1-py3-none-any.whl", hash = "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f"}, - {file = "platformdirs-3.9.1.tar.gz", hash = "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421"}, + {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, + {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, ] [package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] [[package]] name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2574,7 +2956,6 @@ testing = ["pytest", "pytest-benchmark"] name = "portalocker" version = "2.7.0" description = "Wraps the portalocker recipe for easy usage" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -2592,14 +2973,13 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "posthog" -version = "3.0.1" +version = "3.0.2" description = "Integrate PostHog into any python application." -category = "dev" optional = false python-versions = "*" files = [ - {file = "posthog-3.0.1-py2.py3-none-any.whl", hash = "sha256:9c7f92fecc713257d4b2710d05b456569c9156fbdd3e85655ba7ba5ba6c7b3ae"}, - {file = "posthog-3.0.1.tar.gz", hash = "sha256:57d2791ff5752ce56ba0f9bb8876faf3ca9208f1c2c6ceaeb5a2504c34493767"}, + {file = "posthog-3.0.2-py2.py3-none-any.whl", hash = "sha256:a8c0af6f2401fbe50f90e68c4143d0824b54e872de036b1c2f23b5abb39d88ce"}, + {file = "posthog-3.0.2.tar.gz", hash = "sha256:701fba6e446a4de687c6e861b587e7b7741955ad624bf34fe013c06a0fec6fb3"}, ] [package.dependencies] @@ -2614,11 +2994,36 @@ dev = ["black", "flake8", "flake8-print", "isort", "pre-commit"] sentry = ["django", "sentry-sdk"] test = ["coverage", "flake8", "freezegun (==0.3.15)", "mock (>=2.0.0)", "pylint", "pytest"] +[[package]] +name = "prance" +version = "23.6.21.0" +description = "Resolving Swagger/OpenAPI 2.0 and 3.0.0 Parser" +optional = false +python-versions = ">=3.8" +files = [ + {file = "prance-23.6.21.0-py3-none-any.whl", hash = "sha256:6a4276fa07ed9f22feda4331097d7503c4adc3097e46ffae97425f2c1026bd9f"}, + {file = "prance-23.6.21.0.tar.gz", hash = "sha256:d8c15f8ac34019751cc4945f866d8d964d7888016d10de3592e339567177cabe"}, +] + +[package.dependencies] +chardet = ">=3.0" +packaging = ">=21.3" +requests = ">=2.25" +"ruamel.yaml" = ">=0.17.10" +six = ">=1.15,<2.0" + +[package.extras] +cli = ["click (>=7.0)"] +dev = ["bumpversion (>=0.6)", "pytest (>=6.1)", "pytest-cov (>=2.11)", "sphinx (>=3.4)", "towncrier (>=19.2)", "tox (>=3.4)"] +flex = ["flex (>=6.13,<7.0)"] +icu = ["PyICU (>=2.4,<3.0)"] +osv = ["openapi-spec-validator (>=0.5.1,<0.6.0)"] +ssv = ["swagger-spec-validator (>=2.4,<3.0)"] + [[package]] name = "pre-commit" version = "3.3.3" description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2637,7 +3042,6 @@ virtualenv = ">=20.10.0" name = "prompt-toolkit" version = "3.0.39" description = "Library for building powerful interactive command lines in Python" -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -2648,34 +3052,49 @@ files = [ [package.dependencies] wcwidth = "*" +[[package]] +name = "proto-plus" +version = "1.22.3" +description = "Beautiful, Pythonic protocol buffers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "proto-plus-1.22.3.tar.gz", hash = "sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b"}, + {file = "proto_plus-1.22.3-py3-none-any.whl", hash = "sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<5.0.0dev" + +[package.extras] +testing = ["google-api-core[grpc] (>=1.31.5)"] + [[package]] name = "protobuf" -version = "4.23.4" +version = "4.24.1" description = "" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.23.4-cp310-abi3-win32.whl", hash = "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b"}, - {file = "protobuf-4.23.4-cp310-abi3-win_amd64.whl", hash = "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12"}, - {file = "protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd"}, - {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a"}, - {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597"}, - {file = "protobuf-4.23.4-cp37-cp37m-win32.whl", hash = "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e"}, - {file = "protobuf-4.23.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0"}, - {file = "protobuf-4.23.4-cp38-cp38-win32.whl", hash = "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70"}, - {file = "protobuf-4.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2"}, - {file = "protobuf-4.23.4-cp39-cp39-win32.whl", hash = "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720"}, - {file = "protobuf-4.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474"}, - {file = "protobuf-4.23.4-py3-none-any.whl", hash = "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff"}, - {file = "protobuf-4.23.4.tar.gz", hash = "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9"}, + {file = "protobuf-4.24.1-cp310-abi3-win32.whl", hash = "sha256:d414199ca605eeb498adc4d2ba82aedc0379dca4a7c364ff9bc9a179aa28e71b"}, + {file = "protobuf-4.24.1-cp310-abi3-win_amd64.whl", hash = "sha256:5906c5e79ff50fe38b2d49d37db5874e3c8010826f2362f79996d83128a8ed9b"}, + {file = "protobuf-4.24.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:970c701ee16788d74f3de20938520d7a0aebc7e4fff37096a48804c80d2908cf"}, + {file = "protobuf-4.24.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fc361148e902949dcb953bbcb148c99fe8f8854291ad01107e4120361849fd0e"}, + {file = "protobuf-4.24.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:5d32363d14aca6e5c9e9d5918ad8fb65b091b6df66740ae9de50ac3916055e43"}, + {file = "protobuf-4.24.1-cp37-cp37m-win32.whl", hash = "sha256:df015c47d6855b8efa0b9be706c70bf7f050a4d5ac6d37fb043fbd95157a0e25"}, + {file = "protobuf-4.24.1-cp37-cp37m-win_amd64.whl", hash = "sha256:d4af4fd9e9418e819be30f8df2a16e72fbad546a7576ac7f3653be92a6966d30"}, + {file = "protobuf-4.24.1-cp38-cp38-win32.whl", hash = "sha256:302e8752c760549ed4c7a508abc86b25d46553c81989343782809e1a062a2ef9"}, + {file = "protobuf-4.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:06437f0d4bb0d5f29e3d392aba69600188d4be5ad1e0a3370e581a9bf75a3081"}, + {file = "protobuf-4.24.1-cp39-cp39-win32.whl", hash = "sha256:0b2b224e9541fe9f046dd7317d05f08769c332b7e4c54d93c7f0f372dedb0b1a"}, + {file = "protobuf-4.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:bd39b9094a4cc003a1f911b847ab379f89059f478c0b611ba1215053e295132e"}, + {file = "protobuf-4.24.1-py3-none-any.whl", hash = "sha256:55dd644adc27d2a624339332755fe077c7f26971045b469ebb9732a69ce1f2ca"}, + {file = "protobuf-4.24.1.tar.gz", hash = "sha256:44837a5ed9c9418ad5d502f89f28ba102e9cd172b6668bc813f21716f9273348"}, ] [[package]] name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2700,14 +3119,13 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "psycopg" -version = "3.1.9" +version = "3.1.12" description = "PostgreSQL database adapter for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "psycopg-3.1.9-py3-none-any.whl", hash = "sha256:fbbac339274d8733ee70ba9822297af3e8871790a26e967b5ea53e30a4b74dcc"}, - {file = "psycopg-3.1.9.tar.gz", hash = "sha256:ab400f207a8c120bafdd8077916d8f6c0106e809401378708485b016508c30c9"}, + {file = "psycopg-3.1.12-py3-none-any.whl", hash = "sha256:8ec5230d6a7eb654b4fb3cf2d3eda8871d68f24807b934790504467f1deee9f8"}, + {file = "psycopg-3.1.12.tar.gz", hash = "sha256:cec7ad2bc6a8510e56c45746c631cf9394148bdc8a9a11fd8cf8554ce129ae78"}, ] [package.dependencies] @@ -2716,82 +3134,80 @@ typing-extensions = ">=4.1" tzdata = {version = "*", markers = "sys_platform == \"win32\""} [package.extras] -binary = ["psycopg-binary (==3.1.9)"] -c = ["psycopg-c (==3.1.9)"] -dev = ["black (>=23.1.0)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=1.2)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] +binary = ["psycopg-binary (==3.1.12)"] +c = ["psycopg-c (==3.1.12)"] +dev = ["black (>=23.1.0)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=1.4.1)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] pool = ["psycopg-pool"] -test = ["anyio (>=3.6.2)", "mypy (>=1.2)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] +test = ["anyio (>=3.6.2,<4.0)", "mypy (>=1.4.1)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] [[package]] name = "psycopg-binary" -version = "3.1.9" +version = "3.1.10" description = "PostgreSQL database adapter for Python -- C optimisation distribution" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "psycopg_binary-3.1.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:284038cbe3f5a0f3de417af9b5eaa2a9524a3a06211523cf245111c71b566506"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2cea4bb0b19245c83486868d7c66f73238c4caa266b5b3c3d664d10dab2ab56"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe5c5c31f59ccb1d1f473466baa93d800138186286e80e251f930e49c80d208"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82704a899d57c29beba5399d41eab5ef5c238b810d7e25e2d1916d2b34c4b1a3"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eab449e39db1c429cac79b7aa27e6827aad4995f32137e922db7254f43fed7b5"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87e0c97733b11eeca3d24e56df70f3f9d792b2abd46f48be2fb2348ffc3e7e39"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:81e34d6df54329424944d5ca91b1cc77df6b8a9130cb5480680d56f53d4e485c"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e2f463079d99568a343ed0b766150b30627e9ed41de99fd82e945e7e2bec764a"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f2cbdef6568da21c39dfd45c2074e85eabbd00e1b721832ba94980f01f582dd4"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53afb0cc2ebe74651f339e22d05ec082a0f44939715d9138d357852f074fcf55"}, - {file = "psycopg_binary-3.1.9-cp310-cp310-win_amd64.whl", hash = "sha256:09167f106e7685591b4cdf58eff0191fb7435d586f384133a0dd30df646cf409"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8aaa47c1791fc05c0229ec1003dd49e13238fba9434e1fc3b879632f749c3c4"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3d91ee0d33ac7b42d0488a9be2516efa2ec00901b81d69566ff34a7a94b66c0b"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5e36504373e5bcdc954b1da1c6fe66379007fe1e329790e8fb72b879a01e097"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c1def6c2d28e257325b3b208cf1966343b498282a0f4d390fda7b7e0577da64"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:055537a9c20efe9bf17cb72bd879602eda71de6f737ebafa1953e017c6a37fbe"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b164355d023a91b23dcc4bb3112bc7d6e9b9c938fb5abcb6e54457d2da1f317"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03b08545ce1c627f4d5e6384eda2946660c4ba6ceb0a09ae47de07419f725669"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1e31bac3d2d41e6446b20b591f638943328c958f4d1ce13d6f1c5db97c3a8dee"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:a274c63c8fb9d419509bed2ef72befc1fd04243972e17e7f5afc5725cb13a560"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:98d9d156b9ada08c271a79662fc5fcc1731b4d7c1f651ef5843d818d35f15ba0"}, - {file = "psycopg_binary-3.1.9-cp311-cp311-win_amd64.whl", hash = "sha256:c3a13aa022853891cadbc7256a9804e5989def760115c82334bddf0d19783b0b"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1a321ef3579a8de0545ade6ff1edfde0c88b8847d58c5615c03751c76054796"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5833bda4c14f24c6a8ac08d3c5712acaa4f35aab31f9ccd2265e9e9a7d0151c8"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a207d5a7f4212443b7452851c9ccd88df9c6d4d58fa2cea2ead4dd9cb328e578"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:07414daa86662f7657e9fabe49af85a32a975e92e6568337887d9c9ffedc224f"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17c5d4936c746f5125c6ef9eb43655e27d4d0c9ffe34c3073878b43c3192511d"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5cdc13c8ec1437240801e43d07e27ff6479ac9dd8583ecf647345bfd2e8390e4"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3836bdaf030a5648bd5f5b452e4b068b265e28f9199060c5b70dbf4a218cde6e"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:96725d9691a84a21eb3e81c884a2e043054e33e176801a57a05e9ac38d142c6e"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dade344aa90bb0b57d1cfc13304ed83ab9a36614b8ddd671381b2de72fe1483d"}, - {file = "psycopg_binary-3.1.9-cp37-cp37m-win_amd64.whl", hash = "sha256:db866cc557d9761036771d666d17fa4176c537af7e6098f42a6bf8f64217935f"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3b62545cc64dd69ea0ae5ffe18d7c97e03660ab8244aa8c5172668a21c41daa0"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:058ab0d79be0b229338f0e61fec6f475077518cba63c22c593645a69f01c3e23"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2340ca2531f69e5ebd9d18987362ba57ed6ab6a271511d8026814a46a2a87b59"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b816ce0e27a2a8786d34b61d3e36e01029245025879d64b88554326b794a4f0"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7b36fe4314a784fbe45c9fd71c902b9bf57341aff9b97c0cbd22f8409a271e2f"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b246fed629482b06f938b23e9281c4af592329daa3ec2cd4a6841ccbfdeb4d68"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:90787ac05b932c0fc678cbf470ccea9c385b8077583f0490136b4569ed3fb652"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9c114f678e8f4a96530fa79cfd84f65f26358ecfc6cca70cfa2d5e3ae5ef217a"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3a82e77400d1ef6c5bbcf3e600e8bdfacf1a554512f96c090c43ceca3d1ce3b6"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c7d990f14a37345ca05a5192cd5ac938c9cbedca9c929872af6ae311158feb0e"}, - {file = "psycopg_binary-3.1.9-cp38-cp38-win_amd64.whl", hash = "sha256:e0ca74fd85718723bb9f08e0c6898e901a0c365aef20b3c3a4ef8709125d6210"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ce8f4dea5934aa6c4933e559c74bef4beb3413f51fbcf17f306ce890216ac33a"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f41a9e0de4db194c053bcc7c00c35422a4d19d92a8187e8065b1c560626efe35"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f94a7985135e084e122b143956c6f589d17aef743ecd0a434a3d3a222631d5a"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bb86d58b90faefdc0bbedf08fdea4cc2afcb1cfa4340f027d458bfd01d8b812"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c696dc84f9ff155761df15779181d8e4af7746b98908e130add8259912e4bb7"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4213953da44324850c8f789301cf665f46fb94301ba403301e7af58546c3a428"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:25e3ce947aaaa1bd9f1920fca76d7281660646304f9ea5bc036b201dd8790655"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9c75be2a9b986139e3ff6bc0a2852081ac00811040f9b82d3aa539821311122e"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:63e8d1dbe253657c70dbfa9c59423f4654d82698fc5ed6868b8dc0765abe20b6"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f4da4ca9b2365fc1d3fc741c3bbd3efccd892ce813444b884c8911a1acf1c932"}, - {file = "psycopg_binary-3.1.9-cp39-cp39-win_amd64.whl", hash = "sha256:c0b8d6bbeff1dba760a208d8bc205a05b745e6cee02b839f969f72cf56a8b80d"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a529c203f6e0f4c67ba27cf8f9739eb3bc880ad70d6ad6c0e56c2230a66b5a09"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd6e14d1aeb12754a43446c77a5ce819b68875cc25ae6538089ef90d7f6dd6f7"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1583ced5948cf88124212c4503dfe5b01ac3e2dd1a2833c083917f4c4aabe8b4"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2098721c486478987be700723b28ec7a48f134eba339de36af0e745f37dfe461"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7e61f7b412fca7b15dd043a0b22fd528d2ed8276e76b3764c3889e29fa65082b"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0f33e33a072e3d5af51ee4d4a439e10dbe623fe87ef295d5d688180d529f13f"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f6f7738c59262d8d19154164d99c881ed58ed377fb6f1d685eb0dc43bbcd8022"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:511d38b1e1961d179d47d5103ba9634ecfc7ead431d19a9337ef82f3a2bca807"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:666e7acf2ffdb5e8a58e8b0c1759facdb9688c7e90ee8ca7aed675803b57404d"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:57b93c756fee5f7c7bd580c34cd5d244f7d5638f8b2cf25333f97b9b8b2ebfd1"}, + {file = "psycopg_binary-3.1.10-cp310-cp310-win_amd64.whl", hash = "sha256:a1d61b7724c7215a8ea4495a5c6b704656f4b7bb6165f4cb9989b685886ebc48"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:36fff836a7823c9d71fa7faa333c74b2b081af216cebdbb0f481dce55ee2d974"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:32caf98cb00881bfcbbbae39a15f2a4e08b79ff983f1c0f13b60a888ef6e8431"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5565a6a86fee8d74f30de89e07f399567cdf59367aeb09624eb690d524339076"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9fb0d64520b29bd80a6731476ad8e1c20348dfdee00ab098899d23247b641675"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfc05ed4e74fa8615d7cc2bd57f00f97662f4e865a731dbd43da9a527e289c8c"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5b59c8cff887757ddf438ff9489d79c5e6b717112c96f5c68e16f367ff8724e"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbaf12361136afefc5faab21a174a437e71c803b083f410e5140c7605bc66b"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ff72576061c774bcce5f5440b93e63d4c430032dd056d30f6cb1988e549dd92c"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:a4e91e1a8d61c60f592a1dfcebdf55e52a29fe4fdb650c5bd5414c848e77d029"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f7187269d825e84c945be7d93dd5088a4e0b6481a4bdaba3bf7069d4ac13703d"}, + {file = "psycopg_binary-3.1.10-cp311-cp311-win_amd64.whl", hash = "sha256:ba7812a593c16d9d661844dc8dd4d81548fd1c2a0ee676f3e3d8638369f4c5e4"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88caa5859740507b3596c6c2e00ceaccee2c6ab5317bc535887801ad3cc7f3e1"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a3a7e99ba10c2e83a48d79431560e0d5ca7865f68f2bac3a462dc2b151e9926"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:848f4f4707dc73f4b4e844c92f3de795b2ddb728f75132602bda5e6ba55084fc"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:415961e839bb49cfd75cd961503fb8846c0768f247db1fa7171c1ac61d38711b"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0471869e658d0c6b8c3ed53153794739c18d7dad2dd5b8e6ff023a364c20f7df"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:4290060ee0d856caa979ecf675c0e6959325f508272ccf27f64c3801c7bcbde7"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:abf04bc06c8f6a1ac3dc2106d3b79c8661352e9d8a57ca2934ffa6aae8fe600a"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:51fe70708243b83bf16710d8c11b61bd46562e6a24a6300d5434380b35911059"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8b658f7f8b49fb60a1c52e3f6692f690a85bdf1ad30aafe0f3f1fd74f6958cf8"}, + {file = "psycopg_binary-3.1.10-cp37-cp37m-win_amd64.whl", hash = "sha256:ffc8c796194f23b9b07f6d25f927ec4df84a194bbc7a1f9e73316734eef512f9"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:74ce92122be34cf0e5f06d79869e1001c8421a68fa7ddf6fe38a717155cf3a64"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:75608a900984061c8898be68fbddc6f3da5eefdffce6e0624f5371645740d172"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6670d160d054466e8fdedfbc749ef8bf7dfdf69296048954d24645dd4d3d3c01"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d32026cfab7ba7ac687a42c33345026a2fb6fc5608a6144077f767af4386be0b"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:908fa388a5b75dfd17a937acb24708bd272e21edefca9a495004c6f70ec2636a"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e46b97073bd4de114f475249d681eaf054e950699c5d7af554d3684db39b82d"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9cf56bb4b115def3a18157f3b3b7d8322ee94a8dea30028db602c8f9ae34ad1e"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3b6c6f90241c4c5a6ca3f0d8827e37ef90fdc4deb9d8cfa5678baa0ea374b391"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:747176a6aeb058079f56c5397bd90339581ab7b3cc0d62e7445654e6a484c7e1"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:41a415e78c457b06497fa0084e4ea7245ca1a377b55756dd757034210b64da7e"}, + {file = "psycopg_binary-3.1.10-cp38-cp38-win_amd64.whl", hash = "sha256:a7bbe9017edd898d7b3a8747700ed045dda96a907dff87f45e642e28d8584481"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0f062f20256708929a58c41d44f350efced4c00a603323d1413f6dc0b84d95a5"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dea30f2704337ca2d0322fccfe1fa30f61ce9185de3937eb986321063114a51f"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9d88ac72531034ebf7ec09114e732b066a9078f4ce213cf65cc5e42eb538d30"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2bea0940d69c3e24a72530730952687912893b34c53aa39e79045e7b446174d"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6a691dc8e2436d9c1e5cf93902d63e9501688fccc957eb22f952d37886257470"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa92661f99351765673835a4d936d79bd24dfbb358b29b084d83be38229a90e4"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:30eb731ed5525d8df892db6532cc8ffd8a163b73bc355127dee9c49334e16eee"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:50bf7a59d3a85a82d466fed341d352b44d09d6adc18656101d163a7cfc6509a0"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f48665947c55f8d6eb3f0be98de80411508e1ec329f354685329b57fced82c7f"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:caa771569da01fc0389ca34920c331a284425a68f92d1ba0a80cc08935f8356e"}, + {file = "psycopg_binary-3.1.10-cp39-cp39-win_amd64.whl", hash = "sha256:b30887e631fd67affaed98f6cd2135b44f2d1a6d9bca353a69c3889c78bd7aa8"}, ] [[package]] name = "psycopg-pool" version = "3.1.7" description = "Connection Pool for Psycopg" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2806,7 +3222,6 @@ typing-extensions = ">=3.10" name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" -category = "dev" optional = false python-versions = "*" files = [ @@ -2818,7 +3233,6 @@ files = [ name = "pulsar-client" version = "3.2.0" description = "Apache Pulsar Python client library" -category = "dev" optional = false python-versions = "*" files = [ @@ -2866,7 +3280,6 @@ functions = ["apache-bookkeeper-client (>=4.16.1)", "grpcio (>=1.8.2)", "prometh name = "pure-eval" version = "0.2.2" description = "Safely evaluate AST nodes without side effects" -category = "dev" optional = false python-versions = "*" files = [ @@ -2877,11 +3290,76 @@ files = [ [package.extras] tests = ["pytest"] +[[package]] +name = "pyarrow" +version = "13.0.0" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyarrow-13.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:1afcc2c33f31f6fb25c92d50a86b7a9f076d38acbcb6f9e74349636109550148"}, + {file = "pyarrow-13.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:70fa38cdc66b2fc1349a082987f2b499d51d072faaa6b600f71931150de2e0e3"}, + {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd57b13a6466822498238877892a9b287b0a58c2e81e4bdb0b596dbb151cbb73"}, + {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8ce69f7bf01de2e2764e14df45b8404fc6f1a5ed9871e8e08a12169f87b7a26"}, + {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:588f0d2da6cf1b1680974d63be09a6530fd1bd825dc87f76e162404779a157dc"}, + {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6241afd72b628787b4abea39e238e3ff9f34165273fad306c7acf780dd850956"}, + {file = "pyarrow-13.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:fda7857e35993673fcda603c07d43889fca60a5b254052a462653f8656c64f44"}, + {file = "pyarrow-13.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:aac0ae0146a9bfa5e12d87dda89d9ef7c57a96210b899459fc2f785303dcbb67"}, + {file = "pyarrow-13.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d7759994217c86c161c6a8060509cfdf782b952163569606bb373828afdd82e8"}, + {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:868a073fd0ff6468ae7d869b5fc1f54de5c4255b37f44fb890385eb68b68f95d"}, + {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51be67e29f3cfcde263a113c28e96aa04362ed8229cb7c6e5f5c719003659d33"}, + {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d1b4e7176443d12610874bb84d0060bf080f000ea9ed7c84b2801df851320295"}, + {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:69b6f9a089d116a82c3ed819eea8fe67dae6105f0d81eaf0fdd5e60d0c6e0944"}, + {file = "pyarrow-13.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:ab1268db81aeb241200e321e220e7cd769762f386f92f61b898352dd27e402ce"}, + {file = "pyarrow-13.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ee7490f0f3f16a6c38f8c680949551053c8194e68de5046e6c288e396dccee80"}, + {file = "pyarrow-13.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3ad79455c197a36eefbd90ad4aa832bece7f830a64396c15c61a0985e337287"}, + {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68fcd2dc1b7d9310b29a15949cdd0cb9bc34b6de767aff979ebf546020bf0ba0"}, + {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc6fd330fd574c51d10638e63c0d00ab456498fc804c9d01f2a61b9264f2c5b2"}, + {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:e66442e084979a97bb66939e18f7b8709e4ac5f887e636aba29486ffbf373763"}, + {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:0f6eff839a9e40e9c5610d3ff8c5bdd2f10303408312caf4c8003285d0b49565"}, + {file = "pyarrow-13.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b30a27f1cddf5c6efcb67e598d7823a1e253d743d92ac32ec1eb4b6a1417867"}, + {file = "pyarrow-13.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:09552dad5cf3de2dc0aba1c7c4b470754c69bd821f5faafc3d774bedc3b04bb7"}, + {file = "pyarrow-13.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3896ae6c205d73ad192d2fc1489cd0edfab9f12867c85b4c277af4d37383c18c"}, + {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6647444b21cb5e68b593b970b2a9a07748dd74ea457c7dadaa15fd469c48ada1"}, + {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47663efc9c395e31d09c6aacfa860f4473815ad6804311c5433f7085415d62a7"}, + {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b9ba6b6d34bd2563345488cf444510588ea42ad5613df3b3509f48eb80250afd"}, + {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:d00d374a5625beeb448a7fa23060df79adb596074beb3ddc1838adb647b6ef09"}, + {file = "pyarrow-13.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:c51afd87c35c8331b56f796eff954b9c7f8d4b7fef5903daf4e05fcf017d23a8"}, + {file = "pyarrow-13.0.0.tar.gz", hash = "sha256:83333726e83ed44b0ac94d8d7a21bbdee4a05029c3b1e8db58a863eec8fd8a33"}, +] + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pyasn1" +version = "0.5.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, + {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + [[package]] name = "pycparser" version = "2.21" description = "C parser in Python" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2893,7 +3371,6 @@ files = [ name = "pydantic" version = "1.10.12" description = "Data validation and settings management using python type hints" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2944,14 +3421,13 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pygments" -version = "2.15.1" +version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, - {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, ] [package.extras] @@ -2961,7 +3437,6 @@ plugins = ["importlib-metadata"] name = "pyjwt" version = "2.8.0" description = "JSON Web Token implementation in Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2980,29 +3455,130 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pymilvus" -version = "2.2.14" +version = "2.3.1" description = "Python Sdk for Milvus" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pymilvus-2.2.14-py3-none-any.whl", hash = "sha256:c19c757832d46cb2688078cb09fbcaa2c8dea1acebcb3b9a087505c0435d10cc"}, - {file = "pymilvus-2.2.14.tar.gz", hash = "sha256:c89b983823b8fb8d9e0ee30977b272e93c31056622d9cea942ec259130bf6e36"}, + {file = "pymilvus-2.3.1-py3-none-any.whl", hash = "sha256:ce65e1de8700f33bd9aade20f013291629702e25b05726773208f1f0b22548ff"}, + {file = "pymilvus-2.3.1.tar.gz", hash = "sha256:d460f6204d7deb2cff93716bd65670c1b440694b77701fb0ab0ead791aa582c6"}, ] [package.dependencies] environs = "<=9.5.0" -grpcio = ">=1.49.1,<=1.56.0" +grpcio = ">=1.49.1,<=1.58.0" +minio = "*" numpy = {version = "<1.25.0", markers = "python_version <= \"3.8\""} pandas = ">=1.2.4" protobuf = ">=3.20.0" +requests = "*" ujson = ">=2.0.0" +[[package]] +name = "pymongo" +version = "4.5.0" +description = "Python driver for MongoDB " +optional = false +python-versions = ">=3.7" +files = [ + {file = "pymongo-4.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2d4fa1b01fa7e5b7bb8d312e3542e211b320eb7a4e3d8dc884327039d93cb9e0"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux1_i686.whl", hash = "sha256:dfcd2b9f510411de615ccedd47462dae80e82fdc09fe9ab0f0f32f11cf57eeb5"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:3e33064f1984db412b34d51496f4ea785a9cff621c67de58e09fb28da6468a52"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux2014_i686.whl", hash = "sha256:33faa786cc907de63f745f587e9879429b46033d7d97a7b84b37f4f8f47b9b32"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux2014_ppc64le.whl", hash = "sha256:76a262c41c1a7cbb84a3b11976578a7eb8e788c4b7bfbd15c005fb6ca88e6e50"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux2014_s390x.whl", hash = "sha256:0f4b125b46fe377984fbaecf2af40ed48b05a4b7676a2ff98999f2016d66b3ec"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:40d5f6e853ece9bfc01e9129b228df446f49316a4252bb1fbfae5c3c9dedebad"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:152259f0f1a60f560323aacf463a3642a65a25557683f49cfa08c8f1ecb2395a"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6d64878d1659d2a5bdfd0f0a4d79bafe68653c573681495e424ab40d7b6d6d41"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1bb3a62395ffe835dbef3a1cbff48fbcce709c78bd1f52e896aee990928432b"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe48f50fb6348511a3268a893bfd4ab5f263f5ac220782449d03cd05964d1ae7"}, + {file = "pymongo-4.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7591a3beea6a9a4fa3080d27d193b41f631130e3ffa76b88c9ccea123f26dc59"}, + {file = "pymongo-4.5.0-cp310-cp310-win32.whl", hash = "sha256:3a7166d57dc74d679caa7743b8ecf7dc3a1235a9fd178654dddb2b2a627ae229"}, + {file = "pymongo-4.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:21b953da14549ff62ea4ae20889c71564328958cbdf880c64a92a48dda4c9c53"}, + {file = "pymongo-4.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ead4f19d0257a756b21ac2e0e85a37a7245ddec36d3b6008d5bfe416525967dc"}, + {file = "pymongo-4.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9aff6279e405dc953eeb540ab061e72c03cf38119613fce183a8e94f31be608f"}, + {file = "pymongo-4.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4c8d6aa91d3e35016847cbe8d73106e3d1c9a4e6578d38e2c346bfe8edb3ca"}, + {file = "pymongo-4.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08819da7864f9b8d4a95729b2bea5fffed08b63d3b9c15b4fea47de655766cf5"}, + {file = "pymongo-4.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a253b765b7cbc4209f1d8ee16c7287c4268d3243070bf72d7eec5aa9dfe2a2c2"}, + {file = "pymongo-4.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8027c9063579083746147cf401a7072a9fb6829678076cd3deff28bb0e0f50c8"}, + {file = "pymongo-4.5.0-cp311-cp311-win32.whl", hash = "sha256:9d2346b00af524757576cc2406414562cced1d4349c92166a0ee377a2a483a80"}, + {file = "pymongo-4.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:c3c3525ea8658ee1192cdddf5faf99b07ebe1eeaa61bf32821126df6d1b8072b"}, + {file = "pymongo-4.5.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e5a27f348909235a106a3903fc8e70f573d89b41d723a500869c6569a391cff7"}, + {file = "pymongo-4.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9a9a39b7cac81dca79fca8c2a6479ef4c7b1aab95fad7544cc0e8fd943595a2"}, + {file = "pymongo-4.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:496c9cbcb4951183d4503a9d7d2c1e3694aab1304262f831d5e1917e60386036"}, + {file = "pymongo-4.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23cc6d7eb009c688d70da186b8f362d61d5dd1a2c14a45b890bd1e91e9c451f2"}, + {file = "pymongo-4.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fff7d17d30b2cd45afd654b3fc117755c5d84506ed25fda386494e4e0a3416e1"}, + {file = "pymongo-4.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6422b6763b016f2ef2beedded0e546d6aa6ba87910f9244d86e0ac7690f75c96"}, + {file = "pymongo-4.5.0-cp312-cp312-win32.whl", hash = "sha256:77cfff95c1fafd09e940b3fdcb7b65f11442662fad611d0e69b4dd5d17a81c60"}, + {file = "pymongo-4.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e57d859b972c75ee44ea2ef4758f12821243e99de814030f69a3decb2aa86807"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2b0176f9233a5927084c79ff80b51bd70bfd57e4f3d564f50f80238e797f0c8a"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:89b3f2da57a27913d15d2a07d58482f33d0a5b28abd20b8e643ab4d625e36257"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:5caee7bd08c3d36ec54617832b44985bd70c4cbd77c5b313de6f7fce0bb34f93"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:1d40ad09d9f5e719bc6f729cc6b17f31c0b055029719406bd31dde2f72fca7e7"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:076afa0a4a96ca9f77fec0e4a0d241200b3b3a1766f8d7be9a905ecf59a7416b"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:3fa3648e4f1e63ddfe53563ee111079ea3ab35c3b09cd25bc22dadc8269a495f"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:44ee985194c426ddf781fa784f31ffa29cb59657b2dba09250a4245431847d73"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b33c17d9e694b66d7e96977e9e56df19d662031483efe121a24772a44ccbbc7e"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d79ae3bb1ff041c0db56f138c88ce1dfb0209f3546d8d6e7c3f74944ecd2439"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d67225f05f6ea27c8dc57f3fa6397c96d09c42af69d46629f71e82e66d33fa4f"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41771b22dd2822540f79a877c391283d4e6368125999a5ec8beee1ce566f3f82"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a1f26bc1f5ce774d99725773901820dfdfd24e875028da4a0252a5b48dcab5c"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3236cf89d69679eaeb9119c840f5c7eb388a2110b57af6bb6baf01a1da387c18"}, + {file = "pymongo-4.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e1f61355c821e870fb4c17cdb318669cfbcf245a291ce5053b41140870c3e5cc"}, + {file = "pymongo-4.5.0-cp37-cp37m-win32.whl", hash = "sha256:49dce6957598975d8b8d506329d2a3a6c4aee911fa4bbcf5e52ffc6897122950"}, + {file = "pymongo-4.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f2227a08b091bd41df5aadee0a5037673f691e2aa000e1968b1ea2342afc6880"}, + {file = "pymongo-4.5.0-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:435228d3c16a375274ac8ab9c4f9aef40c5e57ddb8296e20ecec9e2461da1017"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:8e559116e4128630ad3b7e788e2e5da81cbc2344dee246af44471fa650486a70"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:840eaf30ccac122df260b6005f9dfae4ac287c498ee91e3e90c56781614ca238"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:b4fe46b58010115514b842c669a0ed9b6a342017b15905653a5b1724ab80917f"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:a8127437ebc196a6f5e8fddd746bd0903a400dc6b5ae35df672dd1ccc7170a2a"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:2988ef5e6b360b3ff1c6d55c53515499de5f48df31afd9f785d788cdacfbe2d3"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:e249190b018d63c901678053b4a43e797ca78b93fb6d17633e3567d4b3ec6107"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:1240edc1a448d4ada4bf1a0e55550b6292420915292408e59159fd8bbdaf8f63"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6d2a56fc2354bb6378f3634402eec788a8f3facf0b3e7d468db5f2b5a78d763"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a0aade2b11dc0c326ccd429ee4134d2d47459ff68d449c6d7e01e74651bd255"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74c0da07c04d0781490b2915e7514b1adb265ef22af039a947988c331ee7455b"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3754acbd7efc7f1b529039fcffc092a15e1cf045e31f22f6c9c5950c613ec4d"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:631492573a1bef2f74f9ac0f9d84e0ce422c251644cd81207530af4aa2ee1980"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e2654d1278384cff75952682d17c718ecc1ad1d6227bb0068fd826ba47d426a5"}, + {file = "pymongo-4.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:168172ef7856e20ec024fe2a746bfa895c88b32720138e6438fd765ebd2b62dd"}, + {file = "pymongo-4.5.0-cp38-cp38-win32.whl", hash = "sha256:b25f7bea162b3dbec6d33c522097ef81df7c19a9300722fa6853f5b495aecb77"}, + {file = "pymongo-4.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:b520aafc6cb148bac09ccf532f52cbd31d83acf4d3e5070d84efe3c019a1adbf"}, + {file = "pymongo-4.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8543253adfaa0b802bfa88386db1009c6ebb7d5684d093ee4edc725007553d21"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:bc5d8c3647b8ae28e4312f1492b8f29deebd31479cd3abaa989090fb1d66db83"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:505f8519c4c782a61d94a17b0da50be639ec462128fbd10ab0a34889218fdee3"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:53f2dda54d76a98b43a410498bd12f6034b2a14b6844ca08513733b2b20b7ad8"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:9c04b9560872fa9a91251030c488e0a73bce9321a70f991f830c72b3f8115d0d"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:58a63a26a1e3dc481dd3a18d6d9f8bd1d576cd1ffe0d479ba7dd38b0aeb20066"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:f076b779aa3dc179aa3ed861be063a313ed4e48ae9f6a8370a9b1295d4502111"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:1b1d7d9aabd8629a31d63cd106d56cca0e6420f38e50563278b520f385c0d86e"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37df8f6006286a5896d1cbc3efb8471ced42e3568d38e6cb00857277047b0d63"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:56320c401f544d762fc35766936178fbceb1d9261cd7b24fbfbc8fb6f67aa8a5"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bbd705d5f3c3d1ff2d169e418bb789ff07ab3c70d567cc6ba6b72b04b9143481"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80a167081c75cf66b32f30e2f1eaee9365af935a86dbd76788169911bed9b5d5"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c42748ccc451dfcd9cef6c5447a7ab727351fd9747ad431db5ebb18a9b78a4d"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf62da7a4cdec9a4b2981fcbd5e08053edffccf20e845c0b6ec1e77eb7fab61d"}, + {file = "pymongo-4.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b5bbb87fa0511bd313d9a2c90294c88db837667c2bda2ea3fa7a35b59fd93b1f"}, + {file = "pymongo-4.5.0-cp39-cp39-win32.whl", hash = "sha256:465fd5b040206f8bce7016b01d7e7f79d2fcd7c2b8e41791be9632a9df1b4999"}, + {file = "pymongo-4.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:63d8019eee119df308a075b8a7bdb06d4720bf791e2b73d5ab0e7473c115d79c"}, + {file = "pymongo-4.5.0.tar.gz", hash = "sha256:681f252e43b3ef054ca9161635f81b730f4d8cadd28b3f2b2004f5a72f853982"}, +] + +[package.dependencies] +dnspython = ">=1.16.0,<3.0.0" + +[package.extras] +aws = ["pymongo-auth-aws (<2.0.0)"] +encryption = ["certifi", "pymongo[aws]", "pymongocrypt (>=1.6.0,<2.0.0)"] +gssapi = ["pykerberos", "winkerberos (>=0.5.0)"] +ocsp = ["certifi", "cryptography (>=2.5)", "pyopenssl (>=17.2.0)", "requests (<3.0.0)", "service-identity (>=18.1.0)"] +snappy = ["python-snappy"] +zstd = ["zstandard"] + [[package]] name = "pypika" version = "0.48.9" description = "A SQL query builder API for Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -3013,7 +3589,6 @@ files = [ name = "pyreadline3" version = "3.4.1" description = "A python implementation of GNU readline." -category = "dev" optional = false python-versions = "*" files = [ @@ -3023,14 +3598,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.2" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -3048,7 +3622,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-asyncio" version = "0.21.1" description = "Pytest support for asyncio" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3067,7 +3640,6 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -3082,7 +3654,6 @@ six = ">=1.5" name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3097,7 +3668,6 @@ cli = ["click (>=5.0)"] name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "dev" optional = false python-versions = "*" files = [ @@ -3109,7 +3679,6 @@ files = [ name = "pywin32" version = "306" description = "Python for Window Extensions" -category = "dev" optional = false python-versions = "*" files = [ @@ -3133,7 +3702,6 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -3142,6 +3710,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -3149,8 +3718,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -3167,6 +3743,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -3174,6 +3751,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -3181,89 +3759,104 @@ files = [ [[package]] name = "pyzmq" -version = "25.1.0" +version = "25.1.1" description = "Python bindings for 0MQ" -category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "pyzmq-25.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:1a6169e69034eaa06823da6a93a7739ff38716142b3596c180363dee729d713d"}, - {file = "pyzmq-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:19d0383b1f18411d137d891cab567de9afa609b214de68b86e20173dc624c101"}, - {file = "pyzmq-25.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1e931d9a92f628858a50f5bdffdfcf839aebe388b82f9d2ccd5d22a38a789dc"}, - {file = "pyzmq-25.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97d984b1b2f574bc1bb58296d3c0b64b10e95e7026f8716ed6c0b86d4679843f"}, - {file = "pyzmq-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:154bddda2a351161474b36dba03bf1463377ec226a13458725183e508840df89"}, - {file = "pyzmq-25.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:cb6d161ae94fb35bb518b74bb06b7293299c15ba3bc099dccd6a5b7ae589aee3"}, - {file = "pyzmq-25.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:90146ab578931e0e2826ee39d0c948d0ea72734378f1898939d18bc9c823fcf9"}, - {file = "pyzmq-25.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:831ba20b660b39e39e5ac8603e8193f8fce1ee03a42c84ade89c36a251449d80"}, - {file = "pyzmq-25.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a522510e3434e12aff80187144c6df556bb06fe6b9d01b2ecfbd2b5bfa5c60c"}, - {file = "pyzmq-25.1.0-cp310-cp310-win32.whl", hash = "sha256:be24a5867b8e3b9dd5c241de359a9a5217698ff616ac2daa47713ba2ebe30ad1"}, - {file = "pyzmq-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:5693dcc4f163481cf79e98cf2d7995c60e43809e325b77a7748d8024b1b7bcba"}, - {file = "pyzmq-25.1.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:13bbe36da3f8aaf2b7ec12696253c0bf6ffe05f4507985a8844a1081db6ec22d"}, - {file = "pyzmq-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:69511d604368f3dc58d4be1b0bad99b61ee92b44afe1cd9b7bd8c5e34ea8248a"}, - {file = "pyzmq-25.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a983c8694667fd76d793ada77fd36c8317e76aa66eec75be2653cef2ea72883"}, - {file = "pyzmq-25.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:332616f95eb400492103ab9d542b69d5f0ff628b23129a4bc0a2fd48da6e4e0b"}, - {file = "pyzmq-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58416db767787aedbfd57116714aad6c9ce57215ffa1c3758a52403f7c68cff5"}, - {file = "pyzmq-25.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cad9545f5801a125f162d09ec9b724b7ad9b6440151b89645241d0120e119dcc"}, - {file = "pyzmq-25.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d6128d431b8dfa888bf51c22a04d48bcb3d64431caf02b3cb943269f17fd2994"}, - {file = "pyzmq-25.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b15247c49d8cbea695b321ae5478d47cffd496a2ec5ef47131a9e79ddd7e46c"}, - {file = "pyzmq-25.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:442d3efc77ca4d35bee3547a8e08e8d4bb88dadb54a8377014938ba98d2e074a"}, - {file = "pyzmq-25.1.0-cp311-cp311-win32.whl", hash = "sha256:65346f507a815a731092421d0d7d60ed551a80d9b75e8b684307d435a5597425"}, - {file = "pyzmq-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:8b45d722046fea5a5694cba5d86f21f78f0052b40a4bbbbf60128ac55bfcc7b6"}, - {file = "pyzmq-25.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f45808eda8b1d71308c5416ef3abe958f033fdbb356984fabbfc7887bed76b3f"}, - {file = "pyzmq-25.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b697774ea8273e3c0460cf0bba16cd85ca6c46dfe8b303211816d68c492e132"}, - {file = "pyzmq-25.1.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b324fa769577fc2c8f5efcd429cef5acbc17d63fe15ed16d6dcbac2c5eb00849"}, - {file = "pyzmq-25.1.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:5873d6a60b778848ce23b6c0ac26c39e48969823882f607516b91fb323ce80e5"}, - {file = "pyzmq-25.1.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:f0d9e7ba6a815a12c8575ba7887da4b72483e4cfc57179af10c9b937f3f9308f"}, - {file = "pyzmq-25.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:414b8beec76521358b49170db7b9967d6974bdfc3297f47f7d23edec37329b00"}, - {file = "pyzmq-25.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:01f06f33e12497dca86353c354461f75275a5ad9eaea181ac0dc1662da8074fa"}, - {file = "pyzmq-25.1.0-cp36-cp36m-win32.whl", hash = "sha256:b5a07c4f29bf7cb0164664ef87e4aa25435dcc1f818d29842118b0ac1eb8e2b5"}, - {file = "pyzmq-25.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:968b0c737797c1809ec602e082cb63e9824ff2329275336bb88bd71591e94a90"}, - {file = "pyzmq-25.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:47b915ba666c51391836d7ed9a745926b22c434efa76c119f77bcffa64d2c50c"}, - {file = "pyzmq-25.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5af31493663cf76dd36b00dafbc839e83bbca8a0662931e11816d75f36155897"}, - {file = "pyzmq-25.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5489738a692bc7ee9a0a7765979c8a572520d616d12d949eaffc6e061b82b4d1"}, - {file = "pyzmq-25.1.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1fc56a0221bdf67cfa94ef2d6ce5513a3d209c3dfd21fed4d4e87eca1822e3a3"}, - {file = "pyzmq-25.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:75217e83faea9edbc29516fc90c817bc40c6b21a5771ecb53e868e45594826b0"}, - {file = "pyzmq-25.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3830be8826639d801de9053cf86350ed6742c4321ba4236e4b5568528d7bfed7"}, - {file = "pyzmq-25.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3575699d7fd7c9b2108bc1c6128641a9a825a58577775ada26c02eb29e09c517"}, - {file = "pyzmq-25.1.0-cp37-cp37m-win32.whl", hash = "sha256:95bd3a998d8c68b76679f6b18f520904af5204f089beebb7b0301d97704634dd"}, - {file = "pyzmq-25.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:dbc466744a2db4b7ca05589f21ae1a35066afada2f803f92369f5877c100ef62"}, - {file = "pyzmq-25.1.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:3bed53f7218490c68f0e82a29c92335daa9606216e51c64f37b48eb78f1281f4"}, - {file = "pyzmq-25.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:eb52e826d16c09ef87132c6e360e1879c984f19a4f62d8a935345deac43f3c12"}, - {file = "pyzmq-25.1.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ddbef8b53cd16467fdbfa92a712eae46dd066aa19780681a2ce266e88fbc7165"}, - {file = "pyzmq-25.1.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9301cf1d7fc1ddf668d0abbe3e227fc9ab15bc036a31c247276012abb921b5ff"}, - {file = "pyzmq-25.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7e23a8c3b6c06de40bdb9e06288180d630b562db8ac199e8cc535af81f90e64b"}, - {file = "pyzmq-25.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4a82faae00d1eed4809c2f18b37f15ce39a10a1c58fe48b60ad02875d6e13d80"}, - {file = "pyzmq-25.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c8398a1b1951aaa330269c35335ae69744be166e67e0ebd9869bdc09426f3871"}, - {file = "pyzmq-25.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d40682ac60b2a613d36d8d3a0cd14fbdf8e7e0618fbb40aa9fa7b796c9081584"}, - {file = "pyzmq-25.1.0-cp38-cp38-win32.whl", hash = "sha256:33d5c8391a34d56224bccf74f458d82fc6e24b3213fc68165c98b708c7a69325"}, - {file = "pyzmq-25.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:c66b7ff2527e18554030319b1376d81560ca0742c6e0b17ff1ee96624a5f1afd"}, - {file = "pyzmq-25.1.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:af56229ea6527a849ac9fb154a059d7e32e77a8cba27e3e62a1e38d8808cb1a5"}, - {file = "pyzmq-25.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bdca18b94c404af6ae5533cd1bc310c4931f7ac97c148bbfd2cd4bdd62b96253"}, - {file = "pyzmq-25.1.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0b6b42f7055bbc562f63f3df3b63e3dd1ebe9727ff0f124c3aa7bcea7b3a00f9"}, - {file = "pyzmq-25.1.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4c2fc7aad520a97d64ffc98190fce6b64152bde57a10c704b337082679e74f67"}, - {file = "pyzmq-25.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be86a26415a8b6af02cd8d782e3a9ae3872140a057f1cadf0133de685185c02b"}, - {file = "pyzmq-25.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:851fb2fe14036cfc1960d806628b80276af5424db09fe5c91c726890c8e6d943"}, - {file = "pyzmq-25.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2a21fec5c3cea45421a19ccbe6250c82f97af4175bc09de4d6dd78fb0cb4c200"}, - {file = "pyzmq-25.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bad172aba822444b32eae54c2d5ab18cd7dee9814fd5c7ed026603b8cae2d05f"}, - {file = "pyzmq-25.1.0-cp39-cp39-win32.whl", hash = "sha256:4d67609b37204acad3d566bb7391e0ecc25ef8bae22ff72ebe2ad7ffb7847158"}, - {file = "pyzmq-25.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:71c7b5896e40720d30cd77a81e62b433b981005bbff0cb2f739e0f8d059b5d99"}, - {file = "pyzmq-25.1.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4cb27ef9d3bdc0c195b2dc54fcb8720e18b741624686a81942e14c8b67cc61a6"}, - {file = "pyzmq-25.1.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0c4fc2741e0513b5d5a12fe200d6785bbcc621f6f2278893a9ca7bed7f2efb7d"}, - {file = "pyzmq-25.1.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fc34fdd458ff77a2a00e3c86f899911f6f269d393ca5675842a6e92eea565bae"}, - {file = "pyzmq-25.1.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8751f9c1442624da391bbd92bd4b072def6d7702a9390e4479f45c182392ff78"}, - {file = "pyzmq-25.1.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:6581e886aec3135964a302a0f5eb68f964869b9efd1dbafdebceaaf2934f8a68"}, - {file = "pyzmq-25.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5482f08d2c3c42b920e8771ae8932fbaa0a67dff925fc476996ddd8155a170f3"}, - {file = "pyzmq-25.1.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5e7fbcafa3ea16d1de1f213c226005fea21ee16ed56134b75b2dede5a2129e62"}, - {file = "pyzmq-25.1.0-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:adecf6d02b1beab8d7c04bc36f22bb0e4c65a35eb0b4750b91693631d4081c70"}, - {file = "pyzmq-25.1.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6d39e42a0aa888122d1beb8ec0d4ddfb6c6b45aecb5ba4013c27e2f28657765"}, - {file = "pyzmq-25.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:7018289b402ebf2b2c06992813523de61d4ce17bd514c4339d8f27a6f6809492"}, - {file = "pyzmq-25.1.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9e68ae9864d260b18f311b68d29134d8776d82e7f5d75ce898b40a88df9db30f"}, - {file = "pyzmq-25.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e21cc00e4debe8f54c3ed7b9fcca540f46eee12762a9fa56feb8512fd9057161"}, - {file = "pyzmq-25.1.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f666ae327a6899ff560d741681fdcdf4506f990595201ed39b44278c471ad98"}, - {file = "pyzmq-25.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f5efcc29056dfe95e9c9db0dfbb12b62db9c4ad302f812931b6d21dd04a9119"}, - {file = "pyzmq-25.1.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:48e5e59e77c1a83162ab3c163fc01cd2eebc5b34560341a67421b09be0891287"}, - {file = "pyzmq-25.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:108c96ebbd573d929740d66e4c3d1bdf31d5cde003b8dc7811a3c8c5b0fc173b"}, - {file = "pyzmq-25.1.0.tar.gz", hash = "sha256:80c41023465d36280e801564a69cbfce8ae85ff79b080e1913f6e90481fb8957"}, + {file = "pyzmq-25.1.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:381469297409c5adf9a0e884c5eb5186ed33137badcbbb0560b86e910a2f1e76"}, + {file = "pyzmq-25.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:955215ed0604dac5b01907424dfa28b40f2b2292d6493445dd34d0dfa72586a8"}, + {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:985bbb1316192b98f32e25e7b9958088431d853ac63aca1d2c236f40afb17c83"}, + {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:afea96f64efa98df4da6958bae37f1cbea7932c35878b185e5982821bc883369"}, + {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76705c9325d72a81155bb6ab48d4312e0032bf045fb0754889133200f7a0d849"}, + {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:77a41c26205d2353a4c94d02be51d6cbdf63c06fbc1295ea57dad7e2d3381b71"}, + {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:12720a53e61c3b99d87262294e2b375c915fea93c31fc2336898c26d7aed34cd"}, + {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:57459b68e5cd85b0be8184382cefd91959cafe79ae019e6b1ae6e2ba8a12cda7"}, + {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:292fe3fc5ad4a75bc8df0dfaee7d0babe8b1f4ceb596437213821f761b4589f9"}, + {file = "pyzmq-25.1.1-cp310-cp310-win32.whl", hash = "sha256:35b5ab8c28978fbbb86ea54958cd89f5176ce747c1fb3d87356cf698048a7790"}, + {file = "pyzmq-25.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:11baebdd5fc5b475d484195e49bae2dc64b94a5208f7c89954e9e354fc609d8f"}, + {file = "pyzmq-25.1.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:d20a0ddb3e989e8807d83225a27e5c2eb2260eaa851532086e9e0fa0d5287d83"}, + {file = "pyzmq-25.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e1c1be77bc5fb77d923850f82e55a928f8638f64a61f00ff18a67c7404faf008"}, + {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d89528b4943d27029a2818f847c10c2cecc79fa9590f3cb1860459a5be7933eb"}, + {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:90f26dc6d5f241ba358bef79be9ce06de58d477ca8485e3291675436d3827cf8"}, + {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2b92812bd214018e50b6380ea3ac0c8bb01ac07fcc14c5f86a5bb25e74026e9"}, + {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:2f957ce63d13c28730f7fd6b72333814221c84ca2421298f66e5143f81c9f91f"}, + {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:047a640f5c9c6ade7b1cc6680a0e28c9dd5a0825135acbd3569cc96ea00b2505"}, + {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7f7e58effd14b641c5e4dec8c7dab02fb67a13df90329e61c869b9cc607ef752"}, + {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c2910967e6ab16bf6fbeb1f771c89a7050947221ae12a5b0b60f3bca2ee19bca"}, + {file = "pyzmq-25.1.1-cp311-cp311-win32.whl", hash = "sha256:76c1c8efb3ca3a1818b837aea423ff8a07bbf7aafe9f2f6582b61a0458b1a329"}, + {file = "pyzmq-25.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:44e58a0554b21fc662f2712814a746635ed668d0fbc98b7cb9d74cb798d202e6"}, + {file = "pyzmq-25.1.1-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:e1ffa1c924e8c72778b9ccd386a7067cddf626884fd8277f503c48bb5f51c762"}, + {file = "pyzmq-25.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1af379b33ef33757224da93e9da62e6471cf4a66d10078cf32bae8127d3d0d4a"}, + {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cff084c6933680d1f8b2f3b4ff5bbb88538a4aac00d199ac13f49d0698727ecb"}, + {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2400a94f7dd9cb20cd012951a0cbf8249e3d554c63a9c0cdfd5cbb6c01d2dec"}, + {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d81f1ddae3858b8299d1da72dd7d19dd36aab654c19671aa8a7e7fb02f6638a"}, + {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:255ca2b219f9e5a3a9ef3081512e1358bd4760ce77828e1028b818ff5610b87b"}, + {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a882ac0a351288dd18ecae3326b8a49d10c61a68b01419f3a0b9a306190baf69"}, + {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:724c292bb26365659fc434e9567b3f1adbdb5e8d640c936ed901f49e03e5d32e"}, + {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ca1ed0bb2d850aa8471387882247c68f1e62a4af0ce9c8a1dbe0d2bf69e41fb"}, + {file = "pyzmq-25.1.1-cp312-cp312-win32.whl", hash = "sha256:b3451108ab861040754fa5208bca4a5496c65875710f76789a9ad27c801a0075"}, + {file = "pyzmq-25.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:eadbefd5e92ef8a345f0525b5cfd01cf4e4cc651a2cffb8f23c0dd184975d787"}, + {file = "pyzmq-25.1.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:db0b2af416ba735c6304c47f75d348f498b92952f5e3e8bff449336d2728795d"}, + {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7c133e93b405eb0d36fa430c94185bdd13c36204a8635470cccc200723c13bb"}, + {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:273bc3959bcbff3f48606b28229b4721716598d76b5aaea2b4a9d0ab454ec062"}, + {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cbc8df5c6a88ba5ae385d8930da02201165408dde8d8322072e3e5ddd4f68e22"}, + {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:18d43df3f2302d836f2a56f17e5663e398416e9dd74b205b179065e61f1a6edf"}, + {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:73461eed88a88c866656e08f89299720a38cb4e9d34ae6bf5df6f71102570f2e"}, + {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:34c850ce7976d19ebe7b9d4b9bb8c9dfc7aac336c0958e2651b88cbd46682123"}, + {file = "pyzmq-25.1.1-cp36-cp36m-win32.whl", hash = "sha256:d2045d6d9439a0078f2a34b57c7b18c4a6aef0bee37f22e4ec9f32456c852c71"}, + {file = "pyzmq-25.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:458dea649f2f02a0b244ae6aef8dc29325a2810aa26b07af8374dc2a9faf57e3"}, + {file = "pyzmq-25.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7cff25c5b315e63b07a36f0c2bab32c58eafbe57d0dce61b614ef4c76058c115"}, + {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1579413ae492b05de5a6174574f8c44c2b9b122a42015c5292afa4be2507f28"}, + {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3d0a409d3b28607cc427aa5c30a6f1e4452cc44e311f843e05edb28ab5e36da0"}, + {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:21eb4e609a154a57c520e3d5bfa0d97e49b6872ea057b7c85257b11e78068222"}, + {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:034239843541ef7a1aee0c7b2cb7f6aafffb005ede965ae9cbd49d5ff4ff73cf"}, + {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f8115e303280ba09f3898194791a153862cbf9eef722ad8f7f741987ee2a97c7"}, + {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1a5d26fe8f32f137e784f768143728438877d69a586ddeaad898558dc971a5ae"}, + {file = "pyzmq-25.1.1-cp37-cp37m-win32.whl", hash = "sha256:f32260e556a983bc5c7ed588d04c942c9a8f9c2e99213fec11a031e316874c7e"}, + {file = "pyzmq-25.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:abf34e43c531bbb510ae7e8f5b2b1f2a8ab93219510e2b287a944432fad135f3"}, + {file = "pyzmq-25.1.1-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:87e34f31ca8f168c56d6fbf99692cc8d3b445abb5bfd08c229ae992d7547a92a"}, + {file = "pyzmq-25.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c9c6c9b2c2f80747a98f34ef491c4d7b1a8d4853937bb1492774992a120f475d"}, + {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5619f3f5a4db5dbb572b095ea3cb5cc035335159d9da950830c9c4db2fbb6995"}, + {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5a34d2395073ef862b4032343cf0c32a712f3ab49d7ec4f42c9661e0294d106f"}, + {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25f0e6b78220aba09815cd1f3a32b9c7cb3e02cb846d1cfc526b6595f6046618"}, + {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3669cf8ee3520c2f13b2e0351c41fea919852b220988d2049249db10046a7afb"}, + {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2d163a18819277e49911f7461567bda923461c50b19d169a062536fffe7cd9d2"}, + {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:df27ffddff4190667d40de7beba4a950b5ce78fe28a7dcc41d6f8a700a80a3c0"}, + {file = "pyzmq-25.1.1-cp38-cp38-win32.whl", hash = "sha256:a382372898a07479bd34bda781008e4a954ed8750f17891e794521c3e21c2e1c"}, + {file = "pyzmq-25.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:52533489f28d62eb1258a965f2aba28a82aa747202c8fa5a1c7a43b5db0e85c1"}, + {file = "pyzmq-25.1.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:03b3f49b57264909aacd0741892f2aecf2f51fb053e7d8ac6767f6c700832f45"}, + {file = "pyzmq-25.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:330f9e188d0d89080cde66dc7470f57d1926ff2fb5576227f14d5be7ab30b9fa"}, + {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2ca57a5be0389f2a65e6d3bb2962a971688cbdd30b4c0bd188c99e39c234f414"}, + {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d457aed310f2670f59cc5b57dcfced452aeeed77f9da2b9763616bd57e4dbaae"}, + {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c56d748ea50215abef7030c72b60dd723ed5b5c7e65e7bc2504e77843631c1a6"}, + {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8f03d3f0d01cb5a018debeb412441996a517b11c5c17ab2001aa0597c6d6882c"}, + {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:820c4a08195a681252f46926de10e29b6bbf3e17b30037bd4250d72dd3ddaab8"}, + {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17ef5f01d25b67ca8f98120d5fa1d21efe9611604e8eb03a5147360f517dd1e2"}, + {file = "pyzmq-25.1.1-cp39-cp39-win32.whl", hash = "sha256:04ccbed567171579ec2cebb9c8a3e30801723c575601f9a990ab25bcac6b51e2"}, + {file = "pyzmq-25.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:e61f091c3ba0c3578411ef505992d356a812fb200643eab27f4f70eed34a29ef"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ade6d25bb29c4555d718ac6d1443a7386595528c33d6b133b258f65f963bb0f6"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0c95ddd4f6e9fca4e9e3afaa4f9df8552f0ba5d1004e89ef0a68e1f1f9807c7"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48e466162a24daf86f6b5ca72444d2bf39a5e58da5f96370078be67c67adc978"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abc719161780932c4e11aaebb203be3d6acc6b38d2f26c0f523b5b59d2fc1996"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:1ccf825981640b8c34ae54231b7ed00271822ea1c6d8ba1090ebd4943759abf5"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c2f20ce161ebdb0091a10c9ca0372e023ce24980d0e1f810f519da6f79c60800"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:deee9ca4727f53464daf089536e68b13e6104e84a37820a88b0a057b97bba2d2"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aa8d6cdc8b8aa19ceb319aaa2b660cdaccc533ec477eeb1309e2a291eaacc43a"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:019e59ef5c5256a2c7378f2fb8560fc2a9ff1d315755204295b2eab96b254d0a"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:b9af3757495c1ee3b5c4e945c1df7be95562277c6e5bccc20a39aec50f826cd0"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:548d6482dc8aadbe7e79d1b5806585c8120bafa1ef841167bc9090522b610fa6"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:057e824b2aae50accc0f9a0570998adc021b372478a921506fddd6c02e60308e"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2243700cc5548cff20963f0ca92d3e5e436394375ab8a354bbea2b12911b20b0"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79986f3b4af059777111409ee517da24a529bdbd46da578b33f25580adcff728"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:11d58723d44d6ed4dd677c5615b2ffb19d5c426636345567d6af82be4dff8a55"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:49d238cf4b69652257db66d0c623cd3e09b5d2e9576b56bc067a396133a00d4a"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fedbdc753827cf014c01dbbee9c3be17e5a208dcd1bf8641ce2cd29580d1f0d4"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc16ac425cc927d0a57d242589f87ee093884ea4804c05a13834d07c20db203c"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11c1d2aed9079c6b0c9550a7257a836b4a637feb334904610f06d70eb44c56d2"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e8a701123029cc240cea61dd2d16ad57cab4691804143ce80ecd9286b464d180"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:61706a6b6c24bdece85ff177fec393545a3191eeda35b07aaa1458a027ad1304"}, + {file = "pyzmq-25.1.1.tar.gz", hash = "sha256:259c22485b71abacdfa8bf79720cd7bcf4b9d128b30ea554f01ae71fdbfdaa23"}, ] [package.dependencies] @@ -3271,14 +3864,13 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} [[package]] name = "qdrant-client" -version = "1.1.7" +version = "1.5.4" description = "Client library for the Qdrant vector search engine" -category = "dev" optional = false -python-versions = ">=3.7,<3.12" +python-versions = ">=3.8,<3.12" files = [ - {file = "qdrant_client-1.1.7-py3-none-any.whl", hash = "sha256:4f5d883660b8193840d8982919ab813a0470ace9a7ff46ee730f909841be5319"}, - {file = "qdrant_client-1.1.7.tar.gz", hash = "sha256:686d86934bec2ebb70676fc0650c9a44a9e552e0149124ca5a22ee8533879deb"}, + {file = "qdrant_client-1.5.4-py3-none-any.whl", hash = "sha256:b0247886c51d755f70dc1a62545f38ba5c7d72971ab533f1264fb695c21a6d8f"}, + {file = "qdrant_client-1.5.4.tar.gz", hash = "sha256:6ffbca94f7cab23230001710b7dc04684dbc18dadf66982179a37531b4c4b178"}, ] [package.dependencies] @@ -3287,113 +3879,146 @@ grpcio-tools = ">=1.41.0" httpx = {version = ">=0.14.0", extras = ["http2"]} numpy = {version = ">=1.21", markers = "python_version >= \"3.8\""} portalocker = ">=2.7.0,<3.0.0" -pydantic = ">=1.8,<2.0" -typing-extensions = ">=4.0.0,<5.0.0" +pydantic = ">=1.10.8" urllib3 = ">=1.26.14,<2.0.0" +[package.extras] +fastembed = ["fastembed (==0.0.4)"] + +[[package]] +name = "redis" +version = "4.6.0" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.7" +files = [ + {file = "redis-4.6.0-py3-none-any.whl", hash = "sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c"}, + {file = "redis-4.6.0.tar.gz", hash = "sha256:585dc516b9eb042a619ef0a39c3d7d55fe81bdb4df09a52c9cdde0d07bf1aa7d"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""} + +[package.extras] +hiredis = ["hiredis (>=1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + +[[package]] +name = "referencing" +version = "0.30.2" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.30.2-py3-none-any.whl", hash = "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf"}, + {file = "referencing-0.30.2.tar.gz", hash = "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + [[package]] name = "regex" -version = "2023.6.3" +version = "2023.10.3" description = "Alternative regular expression module, to replace re." -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "regex-2023.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:824bf3ac11001849aec3fa1d69abcb67aac3e150a933963fb12bda5151fe1bfd"}, - {file = "regex-2023.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05ed27acdf4465c95826962528f9e8d41dbf9b1aa8531a387dee6ed215a3e9ef"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b49c764f88a79160fa64f9a7b425620e87c9f46095ef9c9920542ab2495c8bc"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e3f1316c2293e5469f8f09dc2d76efb6c3982d3da91ba95061a7e69489a14ef"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43e1dd9d12df9004246bacb79a0e5886b3b6071b32e41f83b0acbf293f820ee8"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4959e8bcbfda5146477d21c3a8ad81b185cd252f3d0d6e4724a5ef11c012fb06"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:af4dd387354dc83a3bff67127a124c21116feb0d2ef536805c454721c5d7993d"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2239d95d8e243658b8dbb36b12bd10c33ad6e6933a54d36ff053713f129aa536"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:890e5a11c97cf0d0c550eb661b937a1e45431ffa79803b942a057c4fb12a2da2"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a8105e9af3b029f243ab11ad47c19b566482c150c754e4c717900a798806b222"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:25be746a8ec7bc7b082783216de8e9473803706723b3f6bef34b3d0ed03d57e2"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3676f1dd082be28b1266c93f618ee07741b704ab7b68501a173ce7d8d0d0ca18"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:10cb847aeb1728412c666ab2e2000ba6f174f25b2bdc7292e7dd71b16db07568"}, - {file = "regex-2023.6.3-cp310-cp310-win32.whl", hash = "sha256:dbbbfce33cd98f97f6bffb17801b0576e653f4fdb1d399b2ea89638bc8d08ae1"}, - {file = "regex-2023.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:c5f8037000eb21e4823aa485149f2299eb589f8d1fe4b448036d230c3f4e68e0"}, - {file = "regex-2023.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c123f662be8ec5ab4ea72ea300359023a5d1df095b7ead76fedcd8babbedf969"}, - {file = "regex-2023.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9edcbad1f8a407e450fbac88d89e04e0b99a08473f666a3f3de0fd292badb6aa"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcba6dae7de533c876255317c11f3abe4907ba7d9aa15d13e3d9710d4315ec0e"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29cdd471ebf9e0f2fb3cac165efedc3c58db841d83a518b082077e612d3ee5df"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:12b74fbbf6cbbf9dbce20eb9b5879469e97aeeaa874145517563cca4029db65c"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c29ca1bd61b16b67be247be87390ef1d1ef702800f91fbd1991f5c4421ebae8"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77f09bc4b55d4bf7cc5eba785d87001d6757b7c9eec237fe2af57aba1a071d9"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ea353ecb6ab5f7e7d2f4372b1e779796ebd7b37352d290096978fea83c4dba0c"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:10590510780b7541969287512d1b43f19f965c2ece6c9b1c00fc367b29d8dce7"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e2fbd6236aae3b7f9d514312cdb58e6494ee1c76a9948adde6eba33eb1c4264f"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:6b2675068c8b56f6bfd5a2bda55b8accbb96c02fd563704732fd1c95e2083461"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74419d2b50ecb98360cfaa2974da8689cb3b45b9deff0dcf489c0d333bcc1477"}, - {file = "regex-2023.6.3-cp311-cp311-win32.whl", hash = "sha256:fb5ec16523dc573a4b277663a2b5a364e2099902d3944c9419a40ebd56a118f9"}, - {file = "regex-2023.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:09e4a1a6acc39294a36b7338819b10baceb227f7f7dbbea0506d419b5a1dd8af"}, - {file = "regex-2023.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0654bca0cdf28a5956c83839162692725159f4cda8d63e0911a2c0dc76166525"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463b6a3ceb5ca952e66550a4532cef94c9a0c80dc156c4cc343041951aec1697"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87b2a5bb5e78ee0ad1de71c664d6eb536dc3947a46a69182a90f4410f5e3f7dd"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6343c6928282c1f6a9db41f5fd551662310e8774c0e5ebccb767002fcf663ca9"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6192d5af2ccd2a38877bfef086d35e6659566a335b1492786ff254c168b1693"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74390d18c75054947e4194019077e243c06fbb62e541d8817a0fa822ea310c14"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:742e19a90d9bb2f4a6cf2862b8b06dea5e09b96c9f2df1779e53432d7275331f"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8abbc5d54ea0ee80e37fef009e3cec5dafd722ed3c829126253d3e22f3846f1e"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c2b867c17a7a7ae44c43ebbeb1b5ff406b3e8d5b3e14662683e5e66e6cc868d3"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:d831c2f8ff278179705ca59f7e8524069c1a989e716a1874d6d1aab6119d91d1"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ee2d1a9a253b1729bb2de27d41f696ae893507c7db224436abe83ee25356f5c1"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:61474f0b41fe1a80e8dfa70f70ea1e047387b7cd01c85ec88fa44f5d7561d787"}, - {file = "regex-2023.6.3-cp36-cp36m-win32.whl", hash = "sha256:0b71e63226e393b534105fcbdd8740410dc6b0854c2bfa39bbda6b0d40e59a54"}, - {file = "regex-2023.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bbb02fd4462f37060122e5acacec78e49c0fbb303c30dd49c7f493cf21fc5b27"}, - {file = "regex-2023.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b862c2b9d5ae38a68b92e215b93f98d4c5e9454fa36aae4450f61dd33ff48487"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976d7a304b59ede34ca2921305b57356694f9e6879db323fd90a80f865d355a3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:83320a09188e0e6c39088355d423aa9d056ad57a0b6c6381b300ec1a04ec3d16"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9427a399501818a7564f8c90eced1e9e20709ece36be701f394ada99890ea4b3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178bbc1b2ec40eaca599d13c092079bf529679bf0371c602edaa555e10b41c3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:837328d14cde912af625d5f303ec29f7e28cdab588674897baafaf505341f2fc"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d44dc13229905ae96dd2ae2dd7cebf824ee92bc52e8cf03dcead37d926da019"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d54af539295392611e7efbe94e827311eb8b29668e2b3f4cadcfe6f46df9c777"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7117d10690c38a622e54c432dfbbd3cbd92f09401d622902c32f6d377e2300ee"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bb60b503ec8a6e4e3e03a681072fa3a5adcbfa5479fa2d898ae2b4a8e24c4591"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:65ba8603753cec91c71de423a943ba506363b0e5c3fdb913ef8f9caa14b2c7e0"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:271f0bdba3c70b58e6f500b205d10a36fb4b58bd06ac61381b68de66442efddb"}, - {file = "regex-2023.6.3-cp37-cp37m-win32.whl", hash = "sha256:9beb322958aaca059f34975b0df135181f2e5d7a13b84d3e0e45434749cb20f7"}, - {file = "regex-2023.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fea75c3710d4f31389eed3c02f62d0b66a9da282521075061ce875eb5300cf23"}, - {file = "regex-2023.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8f56fcb7ff7bf7404becdfc60b1e81a6d0561807051fd2f1860b0d0348156a07"}, - {file = "regex-2023.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d2da3abc88711bce7557412310dfa50327d5769a31d1c894b58eb256459dc289"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a99b50300df5add73d307cf66abea093304a07eb017bce94f01e795090dea87c"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5708089ed5b40a7b2dc561e0c8baa9535b77771b64a8330b684823cfd5116036"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:687ea9d78a4b1cf82f8479cab23678aff723108df3edeac098e5b2498879f4a7"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d3850beab9f527f06ccc94b446c864059c57651b3f911fddb8d9d3ec1d1b25d"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8915cc96abeb8983cea1df3c939e3c6e1ac778340c17732eb63bb96247b91d2"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:841d6e0e5663d4c7b4c8099c9997be748677d46cbf43f9f471150e560791f7ff"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9edce5281f965cf135e19840f4d93d55b3835122aa76ccacfd389e880ba4cf82"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b956231ebdc45f5b7a2e1f90f66a12be9610ce775fe1b1d50414aac1e9206c06"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:36efeba71c6539d23c4643be88295ce8c82c88bbd7c65e8a24081d2ca123da3f"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:cf67ca618b4fd34aee78740bea954d7c69fdda419eb208c2c0c7060bb822d747"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b4598b1897837067a57b08147a68ac026c1e73b31ef6e36deeeb1fa60b2933c9"}, - {file = "regex-2023.6.3-cp38-cp38-win32.whl", hash = "sha256:f415f802fbcafed5dcc694c13b1292f07fe0befdb94aa8a52905bd115ff41e88"}, - {file = "regex-2023.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:d4f03bb71d482f979bda92e1427f3ec9b220e62a7dd337af0aa6b47bf4498f72"}, - {file = "regex-2023.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ccf91346b7bd20c790310c4147eee6ed495a54ddb6737162a36ce9dbef3e4751"}, - {file = "regex-2023.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b28f5024a3a041009eb4c333863d7894d191215b39576535c6734cd88b0fcb68"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0bb18053dfcfed432cc3ac632b5e5e5c5b7e55fb3f8090e867bfd9b054dbcbf"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a5bfb3004f2144a084a16ce19ca56b8ac46e6fd0651f54269fc9e230edb5e4a"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c6b48d0fa50d8f4df3daf451be7f9689c2bde1a52b1225c5926e3f54b6a9ed1"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051da80e6eeb6e239e394ae60704d2b566aa6a7aed6f2890a7967307267a5dc6"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4c3b7fa4cdaa69268748665a1a6ff70c014d39bb69c50fda64b396c9116cf77"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:457b6cce21bee41ac292d6753d5e94dcbc5c9e3e3a834da285b0bde7aa4a11e9"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aad51907d74fc183033ad796dd4c2e080d1adcc4fd3c0fd4fd499f30c03011cd"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0385e73da22363778ef2324950e08b689abdf0b108a7d8decb403ad7f5191938"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c6a57b742133830eec44d9b2290daf5cbe0a2f1d6acee1b3c7b1c7b2f3606df7"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:3e5219bf9e75993d73ab3d25985c857c77e614525fac9ae02b1bebd92f7cecac"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e5087a3c59eef624a4591ef9eaa6e9a8d8a94c779dade95d27c0bc24650261cd"}, - {file = "regex-2023.6.3-cp39-cp39-win32.whl", hash = "sha256:20326216cc2afe69b6e98528160b225d72f85ab080cbdf0b11528cbbaba2248f"}, - {file = "regex-2023.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:bdff5eab10e59cf26bc479f565e25ed71a7d041d1ded04ccf9aee1d9f208487a"}, - {file = "regex-2023.6.3.tar.gz", hash = "sha256:72d1a25bf36d2050ceb35b517afe13864865268dfb45910e2e17a84be6cbfeb0"}, + {file = "regex-2023.10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4c34d4f73ea738223a094d8e0ffd6d2c1a1b4c175da34d6b0de3d8d69bee6bcc"}, + {file = "regex-2023.10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8f4e49fc3ce020f65411432183e6775f24e02dff617281094ba6ab079ef0915"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cd1bccf99d3ef1ab6ba835308ad85be040e6a11b0977ef7ea8c8005f01a3c29"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81dce2ddc9f6e8f543d94b05d56e70d03a0774d32f6cca53e978dc01e4fc75b8"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c6b4d23c04831e3ab61717a707a5d763b300213db49ca680edf8bf13ab5d91b"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c15ad0aee158a15e17e0495e1e18741573d04eb6da06d8b84af726cfc1ed02ee"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6239d4e2e0b52c8bd38c51b760cd870069f0bdf99700a62cd509d7a031749a55"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4a8bf76e3182797c6b1afa5b822d1d5802ff30284abe4599e1247be4fd6b03be"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9c727bbcf0065cbb20f39d2b4f932f8fa1631c3e01fcedc979bd4f51fe051c5"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3ccf2716add72f80714b9a63899b67fa711b654be3fcdd34fa391d2d274ce767"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:107ac60d1bfdc3edb53be75e2a52aff7481b92817cfdddd9b4519ccf0e54a6ff"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:00ba3c9818e33f1fa974693fb55d24cdc8ebafcb2e4207680669d8f8d7cca79a"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f0a47efb1dbef13af9c9a54a94a0b814902e547b7f21acb29434504d18f36e3a"}, + {file = "regex-2023.10.3-cp310-cp310-win32.whl", hash = "sha256:36362386b813fa6c9146da6149a001b7bd063dabc4d49522a1f7aa65b725c7ec"}, + {file = "regex-2023.10.3-cp310-cp310-win_amd64.whl", hash = "sha256:c65a3b5330b54103e7d21cac3f6bf3900d46f6d50138d73343d9e5b2900b2353"}, + {file = "regex-2023.10.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90a79bce019c442604662d17bf69df99090e24cdc6ad95b18b6725c2988a490e"}, + {file = "regex-2023.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c7964c2183c3e6cce3f497e3a9f49d182e969f2dc3aeeadfa18945ff7bdd7051"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ef80829117a8061f974b2fda8ec799717242353bff55f8a29411794d635d964"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5addc9d0209a9afca5fc070f93b726bf7003bd63a427f65ef797a931782e7edc"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c148bec483cc4b421562b4bcedb8e28a3b84fcc8f0aa4418e10898f3c2c0eb9b"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d1f21af4c1539051049796a0f50aa342f9a27cde57318f2fc41ed50b0dbc4ac"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b9ac09853b2a3e0d0082104036579809679e7715671cfbf89d83c1cb2a30f58"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ebedc192abbc7fd13c5ee800e83a6df252bec691eb2c4bedc9f8b2e2903f5e2a"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d8a993c0a0ffd5f2d3bda23d0cd75e7086736f8f8268de8a82fbc4bd0ac6791e"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:be6b7b8d42d3090b6c80793524fa66c57ad7ee3fe9722b258aec6d0672543fd0"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4023e2efc35a30e66e938de5aef42b520c20e7eda7bb5fb12c35e5d09a4c43f6"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d47840dc05e0ba04fe2e26f15126de7c755496d5a8aae4a08bda4dd8d646c54"}, + {file = "regex-2023.10.3-cp311-cp311-win32.whl", hash = "sha256:9145f092b5d1977ec8c0ab46e7b3381b2fd069957b9862a43bd383e5c01d18c2"}, + {file = "regex-2023.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:b6104f9a46bd8743e4f738afef69b153c4b8b592d35ae46db07fc28ae3d5fb7c"}, + {file = "regex-2023.10.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff507ae210371d4b1fe316d03433ac099f184d570a1a611e541923f78f05037"}, + {file = "regex-2023.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be5e22bbb67924dea15039c3282fa4cc6cdfbe0cbbd1c0515f9223186fc2ec5f"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a992f702c9be9c72fa46f01ca6e18d131906a7180950958f766c2aa294d4b41"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7434a61b158be563c1362d9071358f8ab91b8d928728cd2882af060481244c9e"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2169b2dcabf4e608416f7f9468737583ce5f0a6e8677c4efbf795ce81109d7c"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9e908ef5889cda4de038892b9accc36d33d72fb3e12c747e2799a0e806ec841"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12bd4bc2c632742c7ce20db48e0d99afdc05e03f0b4c1af90542e05b809a03d9"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bc72c231f5449d86d6c7d9cc7cd819b6eb30134bb770b8cfdc0765e48ef9c420"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bce8814b076f0ce5766dc87d5a056b0e9437b8e0cd351b9a6c4e1134a7dfbda9"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:ba7cd6dc4d585ea544c1412019921570ebd8a597fabf475acc4528210d7c4a6f"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b0c7d2f698e83f15228ba41c135501cfe7d5740181d5903e250e47f617eb4292"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5a8f91c64f390ecee09ff793319f30a0f32492e99f5dc1c72bc361f23ccd0a9a"}, + {file = "regex-2023.10.3-cp312-cp312-win32.whl", hash = "sha256:ad08a69728ff3c79866d729b095872afe1e0557251da4abb2c5faff15a91d19a"}, + {file = "regex-2023.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:39cdf8d141d6d44e8d5a12a8569d5a227f645c87df4f92179bd06e2e2705e76b"}, + {file = "regex-2023.10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4a3ee019a9befe84fa3e917a2dd378807e423d013377a884c1970a3c2792d293"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76066d7ff61ba6bf3cb5efe2428fc82aac91802844c022d849a1f0f53820502d"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe50b61bab1b1ec260fa7cd91106fa9fece57e6beba05630afe27c71259c59b"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fd88f373cb71e6b59b7fa597e47e518282455c2734fd4306a05ca219a1991b0"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ab05a182c7937fb374f7e946f04fb23a0c0699c0450e9fb02ef567412d2fa3"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dac37cf08fcf2094159922edc7a2784cfcc5c70f8354469f79ed085f0328ebdf"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e54ddd0bb8fb626aa1f9ba7b36629564544954fff9669b15da3610c22b9a0991"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3367007ad1951fde612bf65b0dffc8fd681a4ab98ac86957d16491400d661302"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:16f8740eb6dbacc7113e3097b0a36065a02e37b47c936b551805d40340fb9971"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4f2ca6df64cbdd27f27b34f35adb640b5d2d77264228554e68deda54456eb11"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:39807cbcbe406efca2a233884e169d056c35aa7e9f343d4e78665246a332f597"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7eece6fbd3eae4a92d7c748ae825cbc1ee41a89bb1c3db05b5578ed3cfcfd7cb"}, + {file = "regex-2023.10.3-cp37-cp37m-win32.whl", hash = "sha256:ce615c92d90df8373d9e13acddd154152645c0dc060871abf6bd43809673d20a"}, + {file = "regex-2023.10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0f649fa32fe734c4abdfd4edbb8381c74abf5f34bc0b3271ce687b23729299ed"}, + {file = "regex-2023.10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b98b7681a9437262947f41c7fac567c7e1f6eddd94b0483596d320092004533"}, + {file = "regex-2023.10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:91dc1d531f80c862441d7b66c4505cd6ea9d312f01fb2f4654f40c6fdf5cc37a"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82fcc1f1cc3ff1ab8a57ba619b149b907072e750815c5ba63e7aa2e1163384a4"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7979b834ec7a33aafae34a90aad9f914c41fd6eaa8474e66953f3f6f7cbd4368"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef71561f82a89af6cfcbee47f0fabfdb6e63788a9258e913955d89fdd96902ab"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd829712de97753367153ed84f2de752b86cd1f7a88b55a3a775eb52eafe8a94"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00e871d83a45eee2f8688d7e6849609c2ca2a04a6d48fba3dff4deef35d14f07"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:706e7b739fdd17cb89e1fbf712d9dc21311fc2333f6d435eac2d4ee81985098c"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cc3f1c053b73f20c7ad88b0d1d23be7e7b3901229ce89f5000a8399746a6e039"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f85739e80d13644b981a88f529d79c5bdf646b460ba190bffcaf6d57b2a9863"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:741ba2f511cc9626b7561a440f87d658aabb3d6b744a86a3c025f866b4d19e7f"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e77c90ab5997e85901da85131fd36acd0ed2221368199b65f0d11bca44549711"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:979c24cbefaf2420c4e377ecd1f165ea08cc3d1fbb44bdc51bccbbf7c66a2cb4"}, + {file = "regex-2023.10.3-cp38-cp38-win32.whl", hash = "sha256:58837f9d221744d4c92d2cf7201c6acd19623b50c643b56992cbd2b745485d3d"}, + {file = "regex-2023.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:c55853684fe08d4897c37dfc5faeff70607a5f1806c8be148f1695be4a63414b"}, + {file = "regex-2023.10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2c54e23836650bdf2c18222c87f6f840d4943944146ca479858404fedeb9f9af"}, + {file = "regex-2023.10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:69c0771ca5653c7d4b65203cbfc5e66db9375f1078689459fe196fe08b7b4930"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ac965a998e1388e6ff2e9781f499ad1eaa41e962a40d11c7823c9952c77123e"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c0e8fae5b27caa34177bdfa5a960c46ff2f78ee2d45c6db15ae3f64ecadde14"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c56c3d47da04f921b73ff9415fbaa939f684d47293f071aa9cbb13c94afc17d"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef1e014eed78ab650bef9a6a9cbe50b052c0aebe553fb2881e0453717573f52"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d29338556a59423d9ff7b6eb0cb89ead2b0875e08fe522f3e068b955c3e7b59b"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9c6d0ced3c06d0f183b73d3c5920727268d2201aa0fe6d55c60d68c792ff3588"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:994645a46c6a740ee8ce8df7911d4aee458d9b1bc5639bc968226763d07f00fa"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:66e2fe786ef28da2b28e222c89502b2af984858091675044d93cb50e6f46d7af"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:11175910f62b2b8c055f2b089e0fedd694fe2be3941b3e2633653bc51064c528"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:06e9abc0e4c9ab4779c74ad99c3fc10d3967d03114449acc2c2762ad4472b8ca"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fb02e4257376ae25c6dd95a5aec377f9b18c09be6ebdefa7ad209b9137b73d48"}, + {file = "regex-2023.10.3-cp39-cp39-win32.whl", hash = "sha256:3b2c3502603fab52d7619b882c25a6850b766ebd1b18de3df23b2f939360e1bd"}, + {file = "regex-2023.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:adbccd17dcaff65704c856bd29951c58a1bd4b2b0f8ad6b826dbd543fe740988"}, + {file = "regex-2023.10.3.tar.gz", hash = "sha256:3fef4f844d2290ee0ba57addcec17eec9e3df73f10a2748485dfd6a3a188cc0f"}, ] [[package]] name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3411,99 +4036,313 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +description = "A pure python RFC3339 validator" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"}, + {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "rpds-py" +version = "0.9.2" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.9.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7"}, + {file = "rpds_py-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f"}, + {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f"}, + {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe"}, + {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6"}, + {file = "rpds_py-0.9.2-cp310-none-win32.whl", hash = "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764"}, + {file = "rpds_py-0.9.2-cp310-none-win_amd64.whl", hash = "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e"}, + {file = "rpds_py-0.9.2-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3"}, + {file = "rpds_py-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d"}, + {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c"}, + {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae"}, + {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de"}, + {file = "rpds_py-0.9.2-cp311-none-win32.whl", hash = "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab"}, + {file = "rpds_py-0.9.2-cp311-none-win_amd64.whl", hash = "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298"}, + {file = "rpds_py-0.9.2-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386"}, + {file = "rpds_py-0.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1"}, + {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b"}, + {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90"}, + {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d"}, + {file = "rpds_py-0.9.2-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d"}, + {file = "rpds_py-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55"}, + {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d"}, + {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32"}, + {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e"}, + {file = "rpds_py-0.9.2-cp38-none-win32.whl", hash = "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920"}, + {file = "rpds_py-0.9.2-cp38-none-win_amd64.whl", hash = "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044"}, + {file = "rpds_py-0.9.2-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260"}, + {file = "rpds_py-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324"}, + {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3"}, + {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535"}, + {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26"}, + {file = "rpds_py-0.9.2-cp39-none-win32.whl", hash = "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0"}, + {file = "rpds_py-0.9.2-cp39-none-win_amd64.whl", hash = "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe"}, + {file = "rpds_py-0.9.2.tar.gz", hash = "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945"}, +] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "ruamel-yaml" +version = "0.17.32" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +optional = false +python-versions = ">=3" +files = [ + {file = "ruamel.yaml-0.17.32-py3-none-any.whl", hash = "sha256:23cd2ed620231677564646b0c6a89d138b6822a0d78656df7abda5879ec4f447"}, + {file = "ruamel.yaml-0.17.32.tar.gz", hash = "sha256:ec939063761914e14542972a5cba6d33c23b0859ab6342f61cf070cfc600efc2"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.12\""} + +[package.extras] +docs = ["ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.7" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +optional = false +python-versions = ">=3.5" +files = [ + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:370445fd795706fd291ab00c9df38a0caed0f17a6fb46b0f607668ecb16ce763"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win32.whl", hash = "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:40d030e2329ce5286d6b231b8726959ebbe0404c92f0a578c0e2482182e38282"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c3ca1fbba4ae962521e5eb66d72998b51f0f4d0f608d3c0347a48e1af262efa7"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3243f48ecd450eddadc2d11b5feb08aca941b5cd98c9b1db14b2fd128be8c697"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win32.whl", hash = "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf9a6bc4a0221538b1a7de3ed7bca4c93c02346853f44e1cd764be0023cd3640"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win32.whl", hash = "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5"}, + {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, +] + [[package]] name = "ruff" -version = "0.0.280" +version = "0.0.289" description = "An extremely fast Python linter, written in Rust." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.0.280-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:48ed5aca381050a4e2f6d232db912d2e4e98e61648b513c350990c351125aaec"}, - {file = "ruff-0.0.280-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:ef6ee3e429fd29d6a5ceed295809e376e6ece5b0f13c7e703efaf3d3bcb30b96"}, - {file = "ruff-0.0.280-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d878370f7e9463ac40c253724229314ff6ebe4508cdb96cb536e1af4d5a9cd4f"}, - {file = "ruff-0.0.280-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:83e8f372fa5627eeda5b83b5a9632d2f9c88fc6d78cead7e2a1f6fb05728d137"}, - {file = "ruff-0.0.280-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7008fc6ca1df18b21fa98bdcfc711dad5f94d0fc3c11791f65e460c48ef27c82"}, - {file = "ruff-0.0.280-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:fe7118c1eae3fda17ceb409629c7f3b5a22dffa7caf1f6796776936dca1fe653"}, - {file = "ruff-0.0.280-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37359cd67d2af8e09110a546507c302cbea11c66a52d2a9b6d841d465f9962d4"}, - {file = "ruff-0.0.280-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd58af46b0221efb95966f1f0f7576df711cb53e50d2fdb0e83c2f33360116a4"}, - {file = "ruff-0.0.280-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e7c15828d09f90e97bea8feefcd2907e8c8ce3a1f959c99f9b4b3469679f33c"}, - {file = "ruff-0.0.280-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2dae8f2d9c44c5c49af01733c2f7956f808db682a4193180dedb29dd718d7bbe"}, - {file = "ruff-0.0.280-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:5f972567163a20fb8c2d6afc60c2ea5ef8b68d69505760a8bd0377de8984b4f6"}, - {file = "ruff-0.0.280-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ffa7347ad11643f29de100977c055e47c988cd6d9f5f5ff83027600b11b9189"}, - {file = "ruff-0.0.280-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7a37dab70114671d273f203268f6c3366c035fe0c8056614069e90a65e614bfc"}, - {file = "ruff-0.0.280-py3-none-win32.whl", hash = "sha256:7784e3606352fcfb193f3cd22b2e2117c444cb879ef6609ec69deabd662b0763"}, - {file = "ruff-0.0.280-py3-none-win_amd64.whl", hash = "sha256:4a7d52457b5dfcd3ab24b0b38eefaead8e2dca62b4fbf10de4cd0938cf20ce30"}, - {file = "ruff-0.0.280-py3-none-win_arm64.whl", hash = "sha256:b7de5b8689575918e130e4384ed9f539ce91d067c0a332aedef6ca7188adac2d"}, - {file = "ruff-0.0.280.tar.gz", hash = "sha256:581c43e4ac5e5a7117ad7da2120d960a4a99e68ec4021ec3cd47fe1cf78f8380"}, + {file = "ruff-0.0.289-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:c9a89d748e90c840bac9c37afe90cf13a5bfd460ca02ea93dad9d7bee3af03b4"}, + {file = "ruff-0.0.289-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:7f7396c6ea01ba332a6ad9d47642bac25d16bd2076aaa595b001f58b2f32ff05"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7180de86c8ecd39624dec1699136f941c07e723201b4ce979bec9e7c67b40ad2"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:73f37c65508203dd01a539926375a10243769c20d4fcab3fa6359cd3fbfc54b7"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c14abcd7563b5c80be2dd809eeab20e4aa716bf849860b60a22d87ddf19eb88"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:91b6d63b6b46d4707916472c91baa87aa0592e73f62a80ff55efdf6c0668cfd6"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6479b8c4be3c36046c6c92054762b276fa0fddb03f6b9a310fbbf4c4951267fd"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5424318c254bcb091cb67e140ec9b9f7122074e100b06236f252923fb41e767"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4daa90865796aedcedf0d8897fdd4cd09bf0ddd3504529a4ccf211edcaff3c7d"}, + {file = "ruff-0.0.289-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8057e8ab0016c13b9419bad119e854f881e687bd96bc5e2d52c8baac0f278a44"}, + {file = "ruff-0.0.289-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7eebfab2e6a6991908ff1bf82f2dc1e5095fc7e316848e62124526837b445f4d"}, + {file = "ruff-0.0.289-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ebc7af550018001a7fb39ca22cdce20e1a0de4388ea4a007eb5c822f6188c297"}, + {file = "ruff-0.0.289-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6e4e6eccb753efe760ba354fc8e9f783f6bba71aa9f592756f5bd0d78db898ed"}, + {file = "ruff-0.0.289-py3-none-win32.whl", hash = "sha256:bbb3044f931c09cf17dbe5b339896eece0d6ac10c9a86e172540fcdb1974f2b7"}, + {file = "ruff-0.0.289-py3-none-win_amd64.whl", hash = "sha256:6d043c5456b792be2615a52f16056c3cf6c40506ce1f2d6f9d3083cfcb9eeab6"}, + {file = "ruff-0.0.289-py3-none-win_arm64.whl", hash = "sha256:04a720bcca5e987426bb14ad8b9c6f55e259ea774da1cbeafe71569744cfd20a"}, + {file = "ruff-0.0.289.tar.gz", hash = "sha256:2513f853b0fc42f0339b7ab0d2751b63ce7a50a0032d2689b54b2931b3b866d7"}, ] [[package]] name = "safetensors" -version = "0.3.1" +version = "0.3.3" description = "Fast and Safe Tensor serialization" -category = "dev" optional = false python-versions = "*" files = [ - {file = "safetensors-0.3.1-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:2ae9b7dd268b4bae6624729dac86deb82104820e9786429b0583e5168db2f770"}, - {file = "safetensors-0.3.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:08c85c1934682f1e2cd904d38433b53cd2a98245a7cc31f5689f9322a2320bbf"}, - {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba625c7af9e1c5d0d91cb83d2fba97d29ea69d4db2015d9714d24c7f6d488e15"}, - {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b57d5890c619ec10d9f1b6426b8690d0c9c2868a90dc52f13fae6f6407ac141f"}, - {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c9f562ea696d50b95cadbeb1716dc476714a87792ffe374280c0835312cbfe2"}, - {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c115951b3a865ece8d98ee43882f2fd0a999c0200d6e6fec24134715ebe3b57"}, - {file = "safetensors-0.3.1-cp310-cp310-win32.whl", hash = "sha256:118f8f7503ea312fc7af27e934088a1b589fb1eff5a7dea2cd1de6c71ee33391"}, - {file = "safetensors-0.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:54846eaae25fded28a7bebbb66be563cad221b4c80daee39e2f55df5e5e0266f"}, - {file = "safetensors-0.3.1-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:5af82e10946c4822506db0f29269f43147e889054704dde994d4e22f0c37377b"}, - {file = "safetensors-0.3.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:626c86dd1d930963c8ea7f953a3787ae85322551e3a5203ac731d6e6f3e18f44"}, - {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12e30677e6af1f4cc4f2832546e91dbb3b0aa7d575bfa473d2899d524e1ace08"}, - {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d534b80bc8d39945bb902f34b0454773971fe9e5e1f2142af451759d7e52b356"}, - {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ddd0ddd502cf219666e7d30f23f196cb87e829439b52b39f3e7da7918c3416df"}, - {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:997a2cc14023713f423e6d16536d55cb16a3d72850f142e05f82f0d4c76d383b"}, - {file = "safetensors-0.3.1-cp311-cp311-win32.whl", hash = "sha256:6ae9ca63d9e22f71ec40550207bd284a60a6b4916ae6ca12c85a8d86bf49e0c3"}, - {file = "safetensors-0.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:62aa7421ca455418423e35029524489480adda53e3f702453580180ecfebe476"}, - {file = "safetensors-0.3.1-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:6d54b3ed367b6898baab75dfd057c24f36ec64d3938ffff2af981d56bfba2f42"}, - {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:262423aeda91117010f8c607889066028f680fbb667f50cfe6eae96f22f9d150"}, - {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10efe2513a8327fd628cea13167089588acc23093ba132aecfc536eb9a4560fe"}, - {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:689b3d6a7ebce70ee9438267ee55ea89b575c19923876645e927d08757b552fe"}, - {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14cd9a87bc73ce06903e9f8ee8b05b056af6f3c9f37a6bd74997a16ed36ff5f4"}, - {file = "safetensors-0.3.1-cp37-cp37m-win32.whl", hash = "sha256:a77cb39624480d5f143c1cc272184f65a296f573d61629eff5d495d2e0541d3e"}, - {file = "safetensors-0.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9eff3190bfbbb52eef729911345c643f875ca4dbb374aa6c559675cfd0ab73db"}, - {file = "safetensors-0.3.1-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:05cbfef76e4daa14796db1bbb52072d4b72a44050c368b2b1f6fd3e610669a89"}, - {file = "safetensors-0.3.1-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:c49061461f4a81e5ec3415070a3f135530834c89cbd6a7db7cd49e3cb9d9864b"}, - {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22cf7e73ca42974f098ce0cf4dd8918983700b6b07a4c6827d50c8daefca776e"}, - {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04f909442d6223ff0016cd2e1b2a95ef8039b92a558014627363a2e267213f62"}, - {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c573c5a0d5d45791ae8c179e26d74aff86e719056591aa7edb3ca7be55bc961"}, - {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6994043b12e717cf2a6ba69077ac41f0d3675b2819734f07f61819e854c622c7"}, - {file = "safetensors-0.3.1-cp38-cp38-win32.whl", hash = "sha256:158ede81694180a0dbba59422bc304a78c054b305df993c0c6e39c6330fa9348"}, - {file = "safetensors-0.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:afdc725beff7121ea8d39a7339f5a6abcb01daa189ea56290b67fe262d56e20f"}, - {file = "safetensors-0.3.1-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:cba910fcc9e5e64d32d62b837388721165e9c7e45d23bc3a38ad57694b77f40d"}, - {file = "safetensors-0.3.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a4f7dbfe7285573cdaddd85ef6fa84ebbed995d3703ab72d71257944e384612f"}, - {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54aed0802f9eaa83ca7b1cbb986bfb90b8e2c67b6a4bcfe245627e17dad565d4"}, - {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34b75a766f3cfc99fd4c33e329b76deae63f5f388e455d863a5d6e99472fca8e"}, - {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a0f31904f35dc14919a145b2d7a2d8842a43a18a629affe678233c4ea90b4af"}, - {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcf527ecc5f58907fd9031510378105487f318cc91ecdc5aee3c7cc8f46030a8"}, - {file = "safetensors-0.3.1-cp39-cp39-win32.whl", hash = "sha256:e2f083112cf97aa9611e2a05cc170a2795eccec5f6ff837f4565f950670a9d83"}, - {file = "safetensors-0.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:5f4f614b8e8161cd8a9ca19c765d176a82b122fa3d3387b77862145bfe9b4e93"}, - {file = "safetensors-0.3.1.tar.gz", hash = "sha256:571da56ff8d0bec8ae54923b621cda98d36dcef10feb36fd492c4d0c2cd0e869"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:92e4d0c8b2836120fddd134474c5bda8963f322333941f8b9f643e5b24f041eb"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:3dcadb6153c42addc9c625a622ebde9293fabe1973f9ef31ba10fb42c16e8536"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:08f26b61e1b0a14dc959aa9d568776bd038805f611caef1de04a80c468d4a7a4"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:17f41344d9a075f2f21b289a49a62e98baff54b5754240ba896063bce31626bf"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:f1045f798e1a16a6ced98d6a42ec72936d367a2eec81dc5fade6ed54638cd7d2"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:eaf0e4bc91da13f21ac846a39429eb3f3b7ed06295a32321fa3eb1a59b5c70f3"}, + {file = "safetensors-0.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25149180d4dc8ca48bac2ac3852a9424b466e36336a39659b35b21b2116f96fc"}, + {file = "safetensors-0.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9e943bf78c39de8865398a71818315e7d5d1af93c7b30d4da3fc852e62ad9bc"}, + {file = "safetensors-0.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cccfcac04a010354e87c7a2fe16a1ff004fc4f6e7ef8efc966ed30122ce00bc7"}, + {file = "safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a07121f427e646a50d18c1be0fa1a2cbf6398624c31149cd7e6b35486d72189e"}, + {file = "safetensors-0.3.3-cp310-cp310-win32.whl", hash = "sha256:a85e29cbfddfea86453cc0f4889b4bcc6b9c155be9a60e27be479a34e199e7ef"}, + {file = "safetensors-0.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:e13adad4a3e591378f71068d14e92343e626cf698ff805f61cdb946e684a218e"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cbc3312f134baf07334dd517341a4b470b2931f090bd9284888acb7dfaf4606f"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:d15030af39d5d30c22bcbc6d180c65405b7ea4c05b7bab14a570eac7d7d43722"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:f84a74cbe9859b28e3d6d7715ac1dd3097bebf8d772694098f6d42435245860c"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:10d637423d98ab2e6a4ad96abf4534eb26fcaf8ca3115623e64c00759374e90d"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_13_0_universal2.whl", hash = "sha256:3b46f5de8b44084aff2e480874c550c399c730c84b2e8ad1bddb062c94aa14e9"}, + {file = "safetensors-0.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e76da691a82dfaf752854fa6d17c8eba0c8466370c5ad8cf1bfdf832d3c7ee17"}, + {file = "safetensors-0.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4e342fd54e66aa9512dd13e410f791e47aa4feeb5f4c9a20882c72f3d272f29"}, + {file = "safetensors-0.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:178fd30b5dc73bce14a39187d948cedd0e5698e2f055b7ea16b5a96c9b17438e"}, + {file = "safetensors-0.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e8fdf7407dba44587ed5e79d5de3533d242648e1f2041760b21474bd5ea5c8c"}, + {file = "safetensors-0.3.3-cp311-cp311-win32.whl", hash = "sha256:7d3b744cee8d7a46ffa68db1a2ff1a1a432488e3f7a5a97856fe69e22139d50c"}, + {file = "safetensors-0.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f579877d30feec9b6ba409d05fa174633a4fc095675a4a82971d831a8bb60b97"}, + {file = "safetensors-0.3.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:2fff5b19a1b462c17322998b2f4b8bce43c16fe208968174d2f3a1446284ceed"}, + {file = "safetensors-0.3.3-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:41adb1d39e8aad04b16879e3e0cbcb849315999fad73bc992091a01e379cb058"}, + {file = "safetensors-0.3.3-cp37-cp37m-macosx_12_0_x86_64.whl", hash = "sha256:0f2b404250b3b877b11d34afcc30d80e7035714a1116a3df56acaca6b6c00096"}, + {file = "safetensors-0.3.3-cp37-cp37m-macosx_13_0_x86_64.whl", hash = "sha256:b43956ef20e9f4f2e648818a9e7b3499edd6b753a0f5526d4f6a6826fbee8446"}, + {file = "safetensors-0.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d61a99b34169981f088ccfbb2c91170843efc869a0a0532f422db7211bf4f474"}, + {file = "safetensors-0.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c0008aab36cd20e9a051a68563c6f80d40f238c2611811d7faa5a18bf3fd3984"}, + {file = "safetensors-0.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:93d54166072b143084fdcd214a080a088050c1bb1651016b55942701b31334e4"}, + {file = "safetensors-0.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c32ee08f61cea56a5d62bbf94af95df6040c8ab574afffaeb7b44ae5da1e9e3"}, + {file = "safetensors-0.3.3-cp37-cp37m-win32.whl", hash = "sha256:351600f367badd59f7bfe86d317bb768dd8c59c1561c6fac43cafbd9c1af7827"}, + {file = "safetensors-0.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:034717e297849dae1af0a7027a14b8647bd2e272c24106dced64d83e10d468d1"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:8530399666748634bc0b301a6a5523756931b0c2680d188e743d16304afe917a"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:9d741c1f1621e489ba10aa3d135b54202684f6e205df52e219d5eecd673a80c9"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:0c345fd85b4d2093a5109596ff4cd9dfc2e84992e881b4857fbc4a93a3b89ddb"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:69ccee8d05f55cdf76f7e6c87d2bdfb648c16778ef8acfd2ecc495e273e9233e"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:c08a9a4b7a4ca389232fa8d097aebc20bbd4f61e477abc7065b5c18b8202dede"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:a002868d2e3f49bbe81bee2655a411c24fa1f8e68b703dec6629cb989d6ae42e"}, + {file = "safetensors-0.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bd2704cb41faa44d3ec23e8b97330346da0395aec87f8eaf9c9e2c086cdbf13"}, + {file = "safetensors-0.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2951bf3f0ad63df5e6a95263652bd6c194a6eb36fd4f2d29421cd63424c883"}, + {file = "safetensors-0.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07114cec116253ca2e7230fdea30acf76828f21614afd596d7b5438a2f719bd8"}, + {file = "safetensors-0.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ab43aeeb9eadbb6b460df3568a662e6f1911ecc39387f8752afcb6a7d96c087"}, + {file = "safetensors-0.3.3-cp38-cp38-win32.whl", hash = "sha256:f2f59fce31dd3429daca7269a6b06f65e6547a0c248f5116976c3f1e9b73f251"}, + {file = "safetensors-0.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:c31ca0d8610f57799925bf08616856b39518ab772c65093ef1516762e796fde4"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:59a596b3225c96d59af412385981f17dd95314e3fffdf359c7e3f5bb97730a19"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:82a16e92210a6221edd75ab17acdd468dd958ef5023d9c6c1289606cc30d1479"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:98a929e763a581f516373ef31983ed1257d2d0da912a8e05d5cd12e9e441c93a"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:12b83f1986cd16ea0454c636c37b11e819d60dd952c26978310a0835133480b7"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:f439175c827c2f1bbd54df42789c5204a10983a30bc4242bc7deaf854a24f3f0"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:0085be33b8cbcb13079b3a8e131656e05b0bc5e6970530d4c24150f7afd76d70"}, + {file = "safetensors-0.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3ec70c87b1e910769034206ad5efc051069b105aac1687f6edcd02526767f4"}, + {file = "safetensors-0.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f490132383e5e490e710608f4acffcb98ed37f91b885c7217d3f9f10aaff9048"}, + {file = "safetensors-0.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:79d1b6c7ed5596baf79c80fbce5198c3cdcc521ae6a157699f427aba1a90082d"}, + {file = "safetensors-0.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad3cc8006e7a86ee7c88bd2813ec59cd7cc75b03e6fa4af89b9c7b235b438d68"}, + {file = "safetensors-0.3.3-cp39-cp39-win32.whl", hash = "sha256:ab29f54c6b8c301ca05fa014728996bd83aac6e21528f893aaf8945c71f42b6d"}, + {file = "safetensors-0.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:0fa82004eae1a71e2aa29843ef99de9350e459a0fc2f65fc6ee0da9690933d2d"}, + {file = "safetensors-0.3.3.tar.gz", hash = "sha256:edb7072d788c4f929d0f5735d3a2fb51e5a27f833587828583b7f5747af1a2b8"}, ] [package.extras] -all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] +all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (==2.11.0)", "torch (>=1.10)"] +dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (==2.11.0)", "torch (>=1.10)"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)"] numpy = ["numpy (>=1.21.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)"] +paddlepaddle = ["numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)"] +pinned-tf = ["tensorflow (==2.11.0)"] quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] -tensorflow = ["tensorflow (>=2.11.0)"] +tensorflow = ["numpy (>=1.21.6)", "tensorflow (>=2.11.0)"] testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] -torch = ["torch (>=1.10)"] +torch = ["numpy (>=1.21.6)", "torch (>=1.10)"] [[package]] name = "scikit-learn" version = "1.3.0" description = "A set of python modules for machine learning and data mining" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3546,7 +4385,6 @@ tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc ( name = "scipy" version = "1.9.3" description = "Fundamental algorithms for scientific computing in Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3585,7 +4423,6 @@ test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "sciki name = "sentence-transformers" version = "2.2.2" description = "Multilingual text embeddings" -category = "dev" optional = false python-versions = ">=3.6.0" files = [ @@ -3608,7 +4445,6 @@ transformers = ">=4.6.0,<5.0.0" name = "sentencepiece" version = "0.1.99" description = "SentencePiece python wrapper" -category = "dev" optional = false python-versions = "*" files = [ @@ -3661,26 +4497,24 @@ files = [ [[package]] name = "setuptools" -version = "68.0.0" +version = "68.1.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, - {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, + {file = "setuptools-68.1.2-py3-none-any.whl", hash = "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"}, + {file = "setuptools-68.1.2.tar.gz", hash = "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5,<=7.1.2)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -3692,7 +4526,6 @@ files = [ name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3700,11 +4533,31 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] +[[package]] +name = "snoop" +version = "0.4.3" +description = "Powerful debugging tools for Python" +optional = false +python-versions = "*" +files = [ + {file = "snoop-0.4.3-py2.py3-none-any.whl", hash = "sha256:b7418581889ff78b29d9dc5ad4625c4c475c74755fb5cba82c693c6e32afadc0"}, + {file = "snoop-0.4.3.tar.gz", hash = "sha256:2e0930bb19ff0dbdaa6f5933f88e89ed5984210ea9f9de0e1d8231fa5c1c1f25"}, +] + +[package.dependencies] +asttokens = "*" +cheap-repr = ">=0.4.0" +executing = "*" +pygments = "*" +six = "*" + +[package.extras] +tests = ["Django", "birdseye", "littleutils", "numpy (>=1.16.5)", "pandas (>=0.24.2)", "pprintpp", "prettyprinter", "pytest", "pytest-order", "pytest-order (<=0.11.0)"] + [[package]] name = "stack-data" version = "0.6.2" description = "Extract data from python stack frames and tracebacks for informative displays" -category = "dev" optional = false python-versions = "*" files = [ @@ -3724,7 +4577,6 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] name = "starlette" version = "0.27.0" description = "The little ASGI library that shines." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3743,7 +4595,6 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyam name = "sympy" version = "1.12" description = "Computer algebra system (CAS) in Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3758,7 +4609,6 @@ mpmath = ">=0.19" name = "threadpoolctl" version = "3.2.0" description = "threadpoolctl" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3768,64 +4618,123 @@ files = [ [[package]] name = "tokenizers" -version = "0.13.3" -description = "Fast and Customizable Tokenizers" -category = "dev" +version = "0.14.0" +description = "" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, - {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee0b1b311d65beab83d7a41c56a1e46ab732a9eed4460648e8eb0bd69fc2d059"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ef4215284df1277dadbcc5e17d4882bda19f770d02348e73523f7e7d8b8d396"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4d53976079cff8a033f778fb9adca2d9d69d009c02fa2d71a878b5f3963ed30"}, - {file = "tokenizers-0.13.3-cp310-cp310-win32.whl", hash = "sha256:1f0e3b4c2ea2cd13238ce43548959c118069db7579e5d40ec270ad77da5833ce"}, - {file = "tokenizers-0.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:89649c00d0d7211e8186f7a75dfa1db6996f65edce4b84821817eadcc2d3c79e"}, - {file = "tokenizers-0.13.3-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:56b726e0d2bbc9243872b0144515ba684af5b8d8cd112fb83ee1365e26ec74c8"}, - {file = "tokenizers-0.13.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:cc5c022ce692e1f499d745af293ab9ee6f5d92538ed2faf73f9708c89ee59ce6"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f55c981ac44ba87c93e847c333e58c12abcbb377a0c2f2ef96e1a266e4184ff2"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f247eae99800ef821a91f47c5280e9e9afaeed9980fc444208d5aa6ba69ff148"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b3e3215d048e94f40f1c95802e45dcc37c5b05eb46280fc2ccc8cd351bff839"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ba2b0bf01777c9b9bc94b53764d6684554ce98551fec496f71bc5be3a03e98b"}, - {file = "tokenizers-0.13.3-cp311-cp311-win32.whl", hash = "sha256:cc78d77f597d1c458bf0ea7c2a64b6aa06941c7a99cb135b5969b0278824d808"}, - {file = "tokenizers-0.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:ecf182bf59bd541a8876deccf0360f5ae60496fd50b58510048020751cf1724c"}, - {file = "tokenizers-0.13.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:0527dc5436a1f6bf2c0327da3145687d3bcfbeab91fed8458920093de3901b44"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07cbb2c307627dc99b44b22ef05ff4473aa7c7cc1fec8f0a8b37d8a64b1a16d2"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4560dbdeaae5b7ee0d4e493027e3de6d53c991b5002d7ff95083c99e11dd5ac0"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64064bd0322405c9374305ab9b4c07152a1474370327499911937fd4a76d004b"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8c6e2ab0f2e3d939ca66aa1d596602105fe33b505cd2854a4c1717f704c51de"}, - {file = "tokenizers-0.13.3-cp37-cp37m-win32.whl", hash = "sha256:6cc29d410768f960db8677221e497226e545eaaea01aa3613fa0fdf2cc96cff4"}, - {file = "tokenizers-0.13.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fc2a7fdf864554a0dacf09d32e17c0caa9afe72baf9dd7ddedc61973bae352d8"}, - {file = "tokenizers-0.13.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:8791dedba834c1fc55e5f1521be325ea3dafb381964be20684b92fdac95d79b7"}, - {file = "tokenizers-0.13.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:d607a6a13718aeb20507bdf2b96162ead5145bbbfa26788d6b833f98b31b26e1"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3791338f809cd1bf8e4fee6b540b36822434d0c6c6bc47162448deee3f77d425"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2f35f30e39e6aab8716f07790f646bdc6e4a853816cc49a95ef2a9016bf9ce6"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310204dfed5aa797128b65d63538a9837cbdd15da2a29a77d67eefa489edda26"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0f9b92ea052305166559f38498b3b0cae159caea712646648aaa272f7160963"}, - {file = "tokenizers-0.13.3-cp38-cp38-win32.whl", hash = "sha256:9a3fa134896c3c1f0da6e762d15141fbff30d094067c8f1157b9fdca593b5806"}, - {file = "tokenizers-0.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:8e7b0cdeace87fa9e760e6a605e0ae8fc14b7d72e9fc19c578116f7287bb873d"}, - {file = "tokenizers-0.13.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:00cee1e0859d55507e693a48fa4aef07060c4bb6bd93d80120e18fea9371c66d"}, - {file = "tokenizers-0.13.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a23ff602d0797cea1d0506ce69b27523b07e70f6dda982ab8cf82402de839088"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70ce07445050b537d2696022dafb115307abdffd2a5c106f029490f84501ef97"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:280ffe95f50eaaf655b3a1dc7ff1d9cf4777029dbbc3e63a74e65a056594abc3"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97acfcec592f7e9de8cadcdcda50a7134423ac8455c0166b28c9ff04d227b371"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd7730c98a3010cd4f523465867ff95cd9d6430db46676ce79358f65ae39797b"}, - {file = "tokenizers-0.13.3-cp39-cp39-win32.whl", hash = "sha256:48625a108029cb1ddf42e17a81b5a3230ba6888a70c9dc14e81bc319e812652d"}, - {file = "tokenizers-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783"}, - {file = "tokenizers-0.13.3.tar.gz", hash = "sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e"}, + {file = "tokenizers-0.14.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:1a90e1030d9c61de64045206c62721a36f892dcfc5bbbc119dfcd417c1ca60ca"}, + {file = "tokenizers-0.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7cacc5a33767bb2a03b6090eac556c301a1d961ac2949be13977bc3f20cc4e3c"}, + {file = "tokenizers-0.14.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:81994795e1b4f868a6e73107af8cdf088d31357bae6f7abf26c42874eab16f43"}, + {file = "tokenizers-0.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ec53f832bfa91abafecbf92b4259b466fb31438ab31e8291ade0fcf07de8fc2"}, + {file = "tokenizers-0.14.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:854aa813a55d6031a6399b1bca09e4e7a79a80ec05faeea77fc6809d59deb3d5"}, + {file = "tokenizers-0.14.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c34d2f02e25e0fa96e574cadb43a6f14bdefc77f84950991da6e3732489e164"}, + {file = "tokenizers-0.14.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f17d5ad725c827d3dc7db2bbe58093a33db2de49bbb639556a6d88d82f0ca19"}, + {file = "tokenizers-0.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:337a7b7d6b32c6f904faee4304987cb018d1488c88b91aa635760999f5631013"}, + {file = "tokenizers-0.14.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:98a7ceb767e1079ef2c99f52a4e7b816f2e682b2b6fef02c8eff5000536e54e1"}, + {file = "tokenizers-0.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:25ad4a0f883a311a5b021ed979e21559cb4184242c7446cd36e07d046d1ed4be"}, + {file = "tokenizers-0.14.0-cp310-none-win32.whl", hash = "sha256:360706b0c2c6ba10e5e26b7eeb7aef106dbfc0a81ad5ad599a892449b4973b10"}, + {file = "tokenizers-0.14.0-cp310-none-win_amd64.whl", hash = "sha256:1c2ce437982717a5e221efa3c546e636f12f325cc3d9d407c91d2905c56593d0"}, + {file = "tokenizers-0.14.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:612d0ba4f40f4d41163af9613dac59c902d017dc4166ea4537a476af807d41c3"}, + {file = "tokenizers-0.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3013ad0cff561d9be9ce2cc92b76aa746b4e974f20e5b4158c03860a4c8ffe0f"}, + {file = "tokenizers-0.14.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c89a0d6d2ec393a6261df71063b1e22bdd7c6ef3d77b8826541b596132bcf524"}, + {file = "tokenizers-0.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5514417f37fc2ca8159b27853cd992a9a4982e6c51f04bd3ac3f65f68a8fa781"}, + {file = "tokenizers-0.14.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8e761fd1af8409c607b11f084dc7cc50f80f08bd426d4f01d1c353b097d2640f"}, + {file = "tokenizers-0.14.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c16fbcd5ef10df9e51cc84238cdb05ee37e4228aaff39c01aa12b0a0409e29b8"}, + {file = "tokenizers-0.14.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3439d9f858dd9033b69769be5a56eb4fb79fde13fad14fab01edbf2b98033ad9"}, + {file = "tokenizers-0.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c19f8cdc3e84090464a6e28757f60461388cc8cd41c02c109e180a6b7c571f6"}, + {file = "tokenizers-0.14.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:df763ce657a297eb73008d5907243a7558a45ae0930b38ebcb575a24f8296520"}, + {file = "tokenizers-0.14.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:095b0b6683a9b76002aa94659f75c09e4359cb291b318d6e77a60965d7a7f138"}, + {file = "tokenizers-0.14.0-cp311-none-win32.whl", hash = "sha256:712ec0e68a399ded8e115e7e25e7017802fa25ee6c36b4eaad88481e50d0c638"}, + {file = "tokenizers-0.14.0-cp311-none-win_amd64.whl", hash = "sha256:917aa6d6615b33d9aa811dcdfb3109e28ff242fbe2cb89ea0b7d3613e444a672"}, + {file = "tokenizers-0.14.0-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:8464ee7d43ecd9dd1723f51652f49b979052ea3bcd25329e3df44e950c8444d1"}, + {file = "tokenizers-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:84c2b96469b34825557c6fe0bc3154c98d15be58c416a9036ca90afdc9979229"}, + {file = "tokenizers-0.14.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:24b3ccec65ee6f876cd67251c1dcfa1c318c9beec5a438b134f7e33b667a8b36"}, + {file = "tokenizers-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bde333fc56dd5fbbdf2de3067d6c0c129867d33eac81d0ba9b65752ad6ef4208"}, + {file = "tokenizers-0.14.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ddcc2f251bd8a2b2f9a7763ad4468a34cfc4ee3b0fba3cfb34d12c964950cac"}, + {file = "tokenizers-0.14.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10a34eb1416dcec3c6f9afea459acd18fcc93234687de605a768a987eda589ab"}, + {file = "tokenizers-0.14.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:56bc7252530a6a20c6eed19b029914bb9cc781efbe943ca9530856051de99d0f"}, + {file = "tokenizers-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07f5c2324326a00c85111081d5eae4da9d64d56abb5883389b3c98bee0b50a7c"}, + {file = "tokenizers-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5efd92e44e43f36332b5f3653743dca5a0b72cdabb012f20023e220f01f675cb"}, + {file = "tokenizers-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9223bcb77a826dbc9fd0efa6bce679a96b1a01005142778bb42ce967581c5951"}, + {file = "tokenizers-0.14.0-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:e2c1b4707344d3fbfce35d76802c2429ca54e30a5ecb05b3502c1e546039a3bb"}, + {file = "tokenizers-0.14.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:5892ba10fe0a477bde80b9f06bce05cb9d83c15a4676dcae5cbe6510f4524bfc"}, + {file = "tokenizers-0.14.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0e1818f33ac901d5d63830cb6a69a707819f4d958ae5ecb955d8a5ad823a2e44"}, + {file = "tokenizers-0.14.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d06a6fe406df1e616f9e649522683411c6c345ddaaaad7e50bbb60a2cb27e04d"}, + {file = "tokenizers-0.14.0-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b6e2d4bc223dc6a99efbe9266242f1ac03eb0bef0104e6cef9f9512dd5c816b"}, + {file = "tokenizers-0.14.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08ea1f612796e438c9a7e2ad86ab3c1c05c8fe0fad32fcab152c69a3a1a90a86"}, + {file = "tokenizers-0.14.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6ab1a58c05a3bd8ece95eb5d1bc909b3fb11acbd3ff514e3cbd1669e3ed28f5b"}, + {file = "tokenizers-0.14.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:495dc7d3b78815de79dafe7abce048a76154dadb0ffc7f09b7247738557e5cef"}, + {file = "tokenizers-0.14.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aaa0401a245d891b3b2ba9cf027dc65ca07627e11fe3ce597644add7d07064f8"}, + {file = "tokenizers-0.14.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ae4fa13a786fd0d6549da241c6a1077f9b6320a7120d922ccc201ad1d4feea8f"}, + {file = "tokenizers-0.14.0-cp37-none-win32.whl", hash = "sha256:ae0d5b5ab6032c24a2e74cc15f65b6510070926671129e922aa3826c834558d7"}, + {file = "tokenizers-0.14.0-cp37-none-win_amd64.whl", hash = "sha256:2839369a9eb948905612f5d8e70453267d9c7bf17573e5ab49c2f28368fd635d"}, + {file = "tokenizers-0.14.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:f483af09a07fcb8b8b4cd07ac1be9f58bb739704ef9156e955531299ab17ec75"}, + {file = "tokenizers-0.14.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9c2ec661d0d63e618cb145ad15ddb6a81e16d9deb7a203f385d78141da028984"}, + {file = "tokenizers-0.14.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:97e87eb7cbeff63c3b1aa770fdcf18ea4f1c852bfb75d0c913e71b8924a99d61"}, + {file = "tokenizers-0.14.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98c4bd09b47f77f41785488971543de63db82608f0dc0bc6646c876b5ca44d1f"}, + {file = "tokenizers-0.14.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0cbeb5406be31f7605d032bb261f2e728da8ac1f4f196c003bc640279ceb0f52"}, + {file = "tokenizers-0.14.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe799fa48fd7dd549a68abb7bee32dd3721f50210ad2e3e55058080158c72c25"}, + {file = "tokenizers-0.14.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:66daf7c6375a95970e86cb3febc48becfeec4e38b2e0195218d348d3bb86593b"}, + {file = "tokenizers-0.14.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce4b177422af79a77c46bb8f56d73827e688fdc092878cff54e24f5c07a908db"}, + {file = "tokenizers-0.14.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a9aef7a5622648b70f979e96cbc2f795eba5b28987dd62f4dbf8f1eac6d64a1a"}, + {file = "tokenizers-0.14.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:397a24feff284d39b40fdd61c1c828bb6648dfe97b6766c84fbaf7256e272d09"}, + {file = "tokenizers-0.14.0-cp38-none-win32.whl", hash = "sha256:93cc2ec19b6ff6149b2e5127ceda3117cc187dd38556a1ed93baba13dffda069"}, + {file = "tokenizers-0.14.0-cp38-none-win_amd64.whl", hash = "sha256:bf7f540ab8a6fc53fb762963edb7539b11f00af8f70b206f0a6d1a25109ad307"}, + {file = "tokenizers-0.14.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:a58d0b34586f4c5229de5aa124cf76b9455f2e01dc5bd6ed018f6e3bb12572d3"}, + {file = "tokenizers-0.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:90ceca6a06bb4b0048d0a51d0d47ef250d3cb37cc36b6b43334be8c02ac18b0f"}, + {file = "tokenizers-0.14.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5f6c9554bda64799b1d65052d834553bff9a6ef4a6c2114668e2ed8f1871a2a3"}, + {file = "tokenizers-0.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ee14b41024bc05ea172fc2c87f66b60d7c5c636c3a52a09a25ec18e752e6dc7"}, + {file = "tokenizers-0.14.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:879201b1c76b24dc70ce02fc42c3eeb7ff20c353ce0ee638be6449f7c80e73ba"}, + {file = "tokenizers-0.14.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ca79ea6ddde5bb32f7ad1c51de1032829c531e76bbcae58fb3ed105a31faf021"}, + {file = "tokenizers-0.14.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd5934048e60aedddf6c5b076d44ccb388702e1650e2eb7b325a1682d883fbf9"}, + {file = "tokenizers-0.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a1566cabd4bf8f09d6c1fa7a3380a181801a495e7218289dbbd0929de471711"}, + {file = "tokenizers-0.14.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a8fc72a7adc6fa12db38100c403d659bc01fbf6e57f2cc9219e75c4eb0ea313c"}, + {file = "tokenizers-0.14.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7fd08ed6c14aa285482d9e5f48c04de52bdbcecaca0d30465d7a36bbea6b14df"}, + {file = "tokenizers-0.14.0-cp39-none-win32.whl", hash = "sha256:3279c0c1d5fdea7d3499c582fed392fb0463d1046544ca010f53aeee5d2ce12c"}, + {file = "tokenizers-0.14.0-cp39-none-win_amd64.whl", hash = "sha256:203ca081d25eb6e4bc72ea04d552e457079c5c6a3713715ece246f6ca02ca8d0"}, + {file = "tokenizers-0.14.0-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:b45704d5175499387e33a1dd5c8d49ab4d7ef3c36a9ba8a410bb3e68d10f80a0"}, + {file = "tokenizers-0.14.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:6d17d5eb38ccc2f615a7a3692dfa285abe22a1e6d73bbfd753599e34ceee511c"}, + {file = "tokenizers-0.14.0-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a7e6e7989ba77a20c33f7a8a45e0f5b3e7530b2deddad2c3b2a58b323156134"}, + {file = "tokenizers-0.14.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81876cefea043963abf6c92e0cf73ce6ee10bdc43245b6565ce82c0305c2e613"}, + {file = "tokenizers-0.14.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d8cd05f73d1ce875a23bfdb3a572417c0f46927c6070ca43a7f6f044c3d6605"}, + {file = "tokenizers-0.14.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:419a38b89be0081d872eac09449c03cd6589c2ee47461184592ee4b1ad93af1d"}, + {file = "tokenizers-0.14.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:4caf274a9ba944eb83bc695beef95abe24ce112907fb06217875894d8a4f62b8"}, + {file = "tokenizers-0.14.0-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:6ecb3a7741d7ebf65db93d246b102efca112860707e07233f1b88703cb01dbc5"}, + {file = "tokenizers-0.14.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cb7fe9a383cb2932848e459d0277a681d58ad31aa6ccda204468a8d130a9105c"}, + {file = "tokenizers-0.14.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4731e0577780d85788ab4f00d54e16e76fe305739396e6fb4c54b89e6fa12de"}, + {file = "tokenizers-0.14.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9900291ccd19417128e328a26672390365dab1d230cd00ee7a5e2a0319e2716"}, + {file = "tokenizers-0.14.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:493e6932fbca6875fd2e51958f1108ce4c5ae41aa6f2b8017c5f07beaff0a1ac"}, + {file = "tokenizers-0.14.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1792e6b46b89aba0d501c0497f38c96e5b54735379fd8a07a28f45736ba51bb1"}, + {file = "tokenizers-0.14.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:0af26d37c7080688ef606679f3a3d44b63b881de9fa00cc45adc240ba443fd85"}, + {file = "tokenizers-0.14.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:99379ec4d7023c07baed85c68983bfad35fd210dfbc256eaafeb842df7f888e3"}, + {file = "tokenizers-0.14.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:84118aa60dcbb2686730342a0cb37e54e02fde001f936557223d46b6cd8112cd"}, + {file = "tokenizers-0.14.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d616e1859ffcc8fcda60f556c34338b96fb72ca642f6dafc3b1d2aa1812fb4dd"}, + {file = "tokenizers-0.14.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7826b79bbbffc2150bf8d621297cc600d8a1ea53992547c4fd39630de10466b4"}, + {file = "tokenizers-0.14.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:eb3931d734f1e66b77c2a8e22ebe0c196f127c7a0f48bf9601720a6f85917926"}, + {file = "tokenizers-0.14.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:6a475b5cafc7a740bf33d00334b1f2b434b6124198384d8b511931a891be39ff"}, + {file = "tokenizers-0.14.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3d3c9e286ae00b0308903d2ef7b31efc84358109aa41abaa27bd715401c3fef4"}, + {file = "tokenizers-0.14.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:27244e96810434cf705f317e9b74a1163cd2be20bdbd3ed6b96dae1914a6778c"}, + {file = "tokenizers-0.14.0-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ca9b0536fd5f03f62427230e85d9d57f9eed644ab74c319ae4877c9144356aed"}, + {file = "tokenizers-0.14.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f64cdff8c0454295b739d77e25cff7264fa9822296395e60cbfecc7f66d88fb"}, + {file = "tokenizers-0.14.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a00cdfb40544656b7a3b176049d63227d5e53cf2574912514ebb4b9da976aaa1"}, + {file = "tokenizers-0.14.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:b611d96b96957cb2f39560c77cc35d2fcb28c13d5b7d741412e0edfdb6f670a8"}, + {file = "tokenizers-0.14.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:27ad1c02fdd74dcf3502fafb87393412e65f698f2e3aba4ad568a1f3b43d5c9f"}, + {file = "tokenizers-0.14.0.tar.gz", hash = "sha256:a06efa1f19dcc0e9bd0f4ffbf963cb0217af92a9694f68fe7eee5e1c6ddc4bde"}, ] +[package.dependencies] +huggingface_hub = ">=0.16.4,<0.17" + [package.extras] -dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +dev = ["tokenizers[testing]"] +docs = ["setuptools_rust", "sphinx", "sphinx_rtd_theme"] testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] [[package]] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3837,7 +4746,6 @@ files = [ name = "torch" version = "2.0.0" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -3893,7 +4801,6 @@ opt-einsum = ["opt-einsum (>=3.3)"] name = "torchvision" version = "0.15.1" description = "image and video datasets and models for torch deep learning" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3921,7 +4828,7 @@ files = [ [package.dependencies] numpy = "*" -pillow = ">=5.3.0,<8.3.0 || >=8.4.0" +pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" requests = "*" torch = "2.0.0" @@ -3930,42 +4837,40 @@ scipy = ["scipy"] [[package]] name = "tornado" -version = "6.3.2" +version = "6.3.3" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -category = "dev" optional = false python-versions = ">= 3.8" files = [ - {file = "tornado-6.3.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829"}, - {file = "tornado-6.3.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b46a6ab20f5c7c1cb949c72c1994a4585d2eaa0be4853f50a03b5031e964fc7c"}, - {file = "tornado-6.3.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f"}, - {file = "tornado-6.3.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4"}, - {file = "tornado-6.3.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b17b1cf5f8354efa3d37c6e28fdfd9c1c1e5122f2cb56dac121ac61baa47cbe"}, - {file = "tornado-6.3.2-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:29e71c847a35f6e10ca3b5c2990a52ce38b233019d8e858b755ea6ce4dcdd19d"}, - {file = "tornado-6.3.2-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:834ae7540ad3a83199a8da8f9f2d383e3c3d5130a328889e4cc991acc81e87a0"}, - {file = "tornado-6.3.2-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6a0848f1aea0d196a7c4f6772197cbe2abc4266f836b0aac76947872cd29b411"}, - {file = "tornado-6.3.2-cp38-abi3-win32.whl", hash = "sha256:7efcbcc30b7c654eb6a8c9c9da787a851c18f8ccd4a5a3a95b05c7accfa068d2"}, - {file = "tornado-6.3.2-cp38-abi3-win_amd64.whl", hash = "sha256:0c325e66c8123c606eea33084976c832aa4e766b7dff8aedd7587ea44a604cdf"}, - {file = "tornado-6.3.2.tar.gz", hash = "sha256:4b927c4f19b71e627b13f3db2324e4ae660527143f9e1f2e2fb404f3a187e2ba"}, + {file = "tornado-6.3.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:502fba735c84450974fec147340016ad928d29f1e91f49be168c0a4c18181e1d"}, + {file = "tornado-6.3.3-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:805d507b1f588320c26f7f097108eb4023bbaa984d63176d1652e184ba24270a"}, + {file = "tornado-6.3.3-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bd19ca6c16882e4d37368e0152f99c099bad93e0950ce55e71daed74045908f"}, + {file = "tornado-6.3.3-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ac51f42808cca9b3613f51ffe2a965c8525cb1b00b7b2d56828b8045354f76a"}, + {file = "tornado-6.3.3-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71a8db65160a3c55d61839b7302a9a400074c9c753040455494e2af74e2501f2"}, + {file = "tornado-6.3.3-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0"}, + {file = "tornado-6.3.3-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:7d01abc57ea0dbb51ddfed477dfe22719d376119844e33c661d873bf9c0e4a16"}, + {file = "tornado-6.3.3-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9dc4444c0defcd3929d5c1eb5706cbe1b116e762ff3e0deca8b715d14bf6ec17"}, + {file = "tornado-6.3.3-cp38-abi3-win32.whl", hash = "sha256:65ceca9500383fbdf33a98c0087cb975b2ef3bfb874cb35b8de8740cf7f41bd3"}, + {file = "tornado-6.3.3-cp38-abi3-win_amd64.whl", hash = "sha256:22d3c2fa10b5793da13c807e6fc38ff49a4f6e1e3868b0a6f4164768bb8e20f5"}, + {file = "tornado-6.3.3.tar.gz", hash = "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe"}, ] [[package]] name = "tqdm" -version = "4.65.0" +version = "4.66.1" description = "Fast, Extensible Progress Meter" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, - {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, + {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, + {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] @@ -3974,7 +4879,6 @@ telegram = ["requests"] name = "traitlets" version = "5.9.0" description = "Traitlets Python configuration system" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3988,43 +4892,42 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] [[package]] name = "transformers" -version = "4.31.0" +version = "4.34.0" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" -category = "dev" optional = false python-versions = ">=3.8.0" files = [ - {file = "transformers-4.31.0-py3-none-any.whl", hash = "sha256:8487aab0195ce1c2a5ae189305118b9720daddbc7b688edb09ccd79e3b149f6b"}, - {file = "transformers-4.31.0.tar.gz", hash = "sha256:4302fba920a1c24d3a429a29efff6a63eac03f3f3cf55b55927fc795d01cb273"}, + {file = "transformers-4.34.0-py3-none-any.whl", hash = "sha256:3f0187183a7f22c51ecbbc9eac5145df666c5b86bec6feed10e11f0363f3a1f9"}, + {file = "transformers-4.34.0.tar.gz", hash = "sha256:cc2ae61bfbfaa45337fd9017326669fc60e4f55125f589d50da47819e3d6f504"}, ] [package.dependencies] filelock = "*" -huggingface-hub = ">=0.14.1,<1.0" +huggingface-hub = ">=0.16.4,<1.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" regex = "!=2019.12.17" requests = "*" safetensors = ">=0.3.1" -tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" +tokenizers = ">=0.14,<0.15" tqdm = ">=4.27" [package.extras] accelerate = ["accelerate (>=0.20.3)"] -agents = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] -all = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +agents = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.10,!=1.12.0)"] +all = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.15)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] deepspeed = ["accelerate (>=0.20.3)", "deepspeed (>=0.9.3)"] deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +dev = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.15)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.14,<0.15)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.15)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.15)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] docs-specific = ["hf-doc-builder"] fairscale = ["fairscale (>0.3)"] -flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] ftfy = ["ftfy"] integrations = ["optuna", "ray[tune]", "sigopt"] @@ -4044,15 +4947,15 @@ sigopt = ["sigopt"] sklearn = ["scikit-learn"] speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx"] tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] timm = ["timm"] -tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -torch = ["accelerate (>=0.20.3)", "torch (>=1.9,!=1.12.0)"] +tokenizers = ["tokenizers (>=0.14,<0.15)"] +torch = ["accelerate (>=0.20.3)", "torch (>=1.10,!=1.12.0)"] torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] torch-vision = ["Pillow (<10.0.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] +torchhub = ["filelock", "huggingface-hub (>=0.16.4,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.14,<0.15)", "torch (>=1.10,!=1.12.0)", "tqdm (>=4.27)"] video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow (<10.0.0)"] @@ -4060,7 +4963,6 @@ vision = ["Pillow (<10.0.0)"] name = "triton" version = "2.0.0" description = "A language and compiler for custom Deep Learning operations" -category = "dev" optional = false python-versions = "*" files = [ @@ -4098,7 +5000,6 @@ tutorials = ["matplotlib", "pandas", "tabulate"] name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4110,7 +5011,6 @@ files = [ name = "tzdata" version = "2023.3" description = "Provider of IANA time zone data" -category = "dev" optional = false python-versions = ">=2" files = [ @@ -4118,11 +5018,41 @@ files = [ {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] +[[package]] +name = "ucall" +version = "0.5.1" +description = "Up to 100x Faster FastAPI. JSON-RPC with io_uring, SIMD-acceleration, and pure CPython bindings" +optional = false +python-versions = ">=3.9" +files = [ + {file = "ucall-0.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15cd5ca5b15d198775d0b80532f41579f4ae7bf3693b86b4ac5f5ff1ed0be1d8"}, + {file = "ucall-0.5.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:577018de6f01651ba53ac7c8867ddd9b92cc79f98fbb4c0513fcc22a8d58e007"}, + {file = "ucall-0.5.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:a18ac9f297ef08e928b59c55ad75cba34511e7d4816af2fcb986043a8ecf719d"}, + {file = "ucall-0.5.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef678d300edcb8d1d6d3af65b63034f2b09873b75b9fcb323eaec0d824cadff7"}, + {file = "ucall-0.5.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f1d3709b4a977c9bdefba3098edd8bc8ae37855f40ccf29f6580f195d7e2b09"}, + {file = "ucall-0.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:fcc1df86e1129bacdbb17662892e02f20189e74f827c0162da56fb2490df87ed"}, + {file = "ucall-0.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:406131c6d0c74035ee7b13131e2e674ca571607bc4c7b3f47c4758f9a5b8724d"}, + {file = "ucall-0.5.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:9a61bf176ce73df006507bdf2a30098b3519e534969886b2caf3d2dc479fda0c"}, + {file = "ucall-0.5.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:29bfcd675c458d23c492e86cf703443f1f4ba266a92880bd943de1ead8e27ddd"}, + {file = "ucall-0.5.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26c79e67dbc7ecf6d925c8ce2f291db281961a8db13d3470f4dafb1c32d72a4b"}, + {file = "ucall-0.5.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba80f2094597dfa182da47cd5a71a5420da2d171aa5067a7a6efbc196eeb86e"}, + {file = "ucall-0.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:562632065fd36968ec92cc8b51033276449610465ef54916ec05bf9505be6b8b"}, + {file = "ucall-0.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d9b1cb358b6967023dcdbf1e9744501a687be75b3dc9b5fd5b3f177f18714a0"}, + {file = "ucall-0.5.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:e99eb1c64837b596281a41a641404bcb618c5037d75d652fc1f2a9b8c38aaed1"}, + {file = "ucall-0.5.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:9b3e346a93c1ff7d2eef9cf03d7f515be4fc6fed195939cfa37cda6ac36a2514"}, + {file = "ucall-0.5.1-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:358ae439f17fd05e70baa8809b38c5ff1146cc3fe77e91d5d50288d9154484af"}, + {file = "ucall-0.5.1-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6012d8d374535b87931dc48cd8ad34529e0e8ae5f30f8a301a8784c39fe7d013"}, + {file = "ucall-0.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:1a96b1df8d1e74afaca42a9b58d6b27a9b4a8444f83e17c0dfd76f9b4c3f3b20"}, +] + +[package.dependencies] +numpy = "*" +pillow = "*" + [[package]] name = "ujson" version = "5.8.0" description = "Ultra fast JSON encoder and decoder for Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -4191,31 +5121,72 @@ files = [ [[package]] name = "urllib3" -version = "1.26.16" +version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "usearch" +version = "1.1.1" +description = "Smaller & Faster Single-File Vector Search Engine from Unum" +optional = false +python-versions = "*" +files = [ + {file = "usearch-1.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1a68a223be42573a036c76e516f30c076b16dd38d8cfe9ca79a1cc0e4d60e8a8"}, + {file = "usearch-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf2d8246a23e5a232a9389f4efd25e0bd10624a96f0f95d0cd415945a6be84ee"}, + {file = "usearch-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8692dbd0b66874e6b01e2dd7c50216347f52d175bc7e072712a5e0646ec9295b"}, + {file = "usearch-1.1.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:5be4ede1592b056714e3f529cd17e69907364e3c0ee6eee5cf1498f946f0c2ec"}, + {file = "usearch-1.1.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6230b0583779f43dba2da3865dd398f8cf88daa6427d60afff3348bbdea6652f"}, + {file = "usearch-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:eb731f74a7a8208e0fa5b04d9488d1dfc177e253b9c761687cb51d38138d5b93"}, + {file = "usearch-1.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7fbb82767109d03c807678664ab02383e31db778adb1d3602da347613fdbf15e"}, + {file = "usearch-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7e1f81a92f3fcc091400f2997b7b12b6d53f7abf4edf87e8a17b5eede350431"}, + {file = "usearch-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7027cdc4c733d6926fc2a58e77cb9b14528a3f585b5d738ad6c5f14dc6e027ca"}, + {file = "usearch-1.1.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e1bb1b238390cc990d051a07fe2a0f173e60bc9e82b7f0f34eb9ddf5bef2b1f8"}, + {file = "usearch-1.1.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:42711bad96f487f5d31ac7be1685797fb4b26904328bc855182e8d6c83b9e538"}, + {file = "usearch-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:ebc34eb7cf0b9f7e52b0f176c48d374f19668ad9653533bdd2e5be1463435d66"}, + {file = "usearch-1.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:36059015e49f9ea303a1d823b80113ce96833330563a54ceac447e4218d63a2c"}, + {file = "usearch-1.1.1-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:918830e1876064227f0a6a17bd4f92c985d8df4856b0370c7729b6e43721b3cc"}, + {file = "usearch-1.1.1-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:a59dfd5d848c484448470e613514f636cf42acac3eab1a9fb9b98d9511de2a97"}, + {file = "usearch-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:70d1f5148a1032da5b0e6651371d29643bf302c0d24a2896d6969d504fccac15"}, + {file = "usearch-1.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5112ebd545ad63b7a63d68838da8a56cfcd313c9ade86bfbe30061c946cbc5dc"}, + {file = "usearch-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0f1e58d11d9dfe1d499e86c108a21f7deb85fe4f40e54b042e057b5df5ead036"}, + {file = "usearch-1.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fbd08ecbf2225f16b9f4b8190cff53de372baddc173e5ba7854425392552013b"}, + {file = "usearch-1.1.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:b8040aa9f448ddfaac5528ec1a1c216351cf7a17af35ddf169b239282f7fa4c4"}, + {file = "usearch-1.1.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:479fcf8b884d1a822b83c7cfb853c090f0db4386e86ef790f2c820f96de70140"}, + {file = "usearch-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:b0338b054fde34ab0a42a786bae43ae1793412995f6a87122850fc0318cb5955"}, + {file = "usearch-1.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:12121e7ac82868877ae9e6b513a61c1113afc8a28d793f9350719ef94ac33091"}, + {file = "usearch-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:33255b29bd7fc1feb6584887f6489bf9f896bd9d79b9ce423ff7185b2c2059e5"}, + {file = "usearch-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9bb5464473e8ceeef6237285fc0e86a0b77a75304397db3365cb011761fd6abe"}, + {file = "usearch-1.1.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:6d84c5771aa37584a335f4b3392185782da785733aab4c3a4ae9949434cbe679"}, + {file = "usearch-1.1.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:02167b0c03062a6d28926535ee862401669b6d6f303e99d2cd1232dc610d2a25"}, + {file = "usearch-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:8300ba31fcc3ace452429781f517273e1297a5881cff629e2f1c6a3a411a48fc"}, +] + +[package.dependencies] +numpy = "*" +pandas = "*" +tqdm = "*" +ucall = {version = "*", markers = "python_version >= \"3.9\""} + [[package]] name = "uvicorn" -version = "0.23.1" +version = "0.23.2" description = "The lightning-fast ASGI server." -category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.23.1-py3-none-any.whl", hash = "sha256:1d55d46b83ee4ce82b4e82f621f2050adb3eb7b5481c13f9af1744951cae2f1f"}, - {file = "uvicorn-0.23.1.tar.gz", hash = "sha256:da9b0c8443b2d7ee9db00a345f1eee6db7317432c9d4400f5049cc8d358383be"}, + {file = "uvicorn-0.23.2-py3-none-any.whl", hash = "sha256:1f9be6558f01239d4fdf22ef8126c39cb1ad0addf76c40e760549d2c2f43ab53"}, + {file = "uvicorn-0.23.2.tar.gz", hash = "sha256:4d3cc12d7727ba72b64d12d3cc7743124074c0a69f7b201512fc50c3e3f1569a"}, ] [package.dependencies] @@ -4226,7 +5197,7 @@ httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standar python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} -uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} @@ -4237,7 +5208,6 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", name = "uvloop" version = "0.17.0" description = "Fast implementation of asyncio event loop on top of libuv" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4280,31 +5250,35 @@ test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "my [[package]] name = "validators" -version = "0.20.0" -description = "Python Data Validation for Humans™." -category = "dev" +version = "0.22.0" +description = "Python Data Validation for Humans™" optional = false -python-versions = ">=3.4" +python-versions = ">=3.8" files = [ - {file = "validators-0.20.0.tar.gz", hash = "sha256:24148ce4e64100a2d5e267233e23e7afeb55316b47d30faae7eb6e7292bc226a"}, + {file = "validators-0.22.0-py3-none-any.whl", hash = "sha256:61cf7d4a62bbae559f2e54aed3b000cea9ff3e2fdbe463f51179b92c58c9585a"}, + {file = "validators-0.22.0.tar.gz", hash = "sha256:77b2689b172eeeb600d9605ab86194641670cdb73b60afd577142a9397873370"}, ] -[package.dependencies] -decorator = ">=3.4.0" - [package.extras] -test = ["flake8 (>=2.4.0)", "isort (>=4.2.2)", "pytest (>=2.2.3)"] +docs-offline = ["myst-parser (>=2.0.0)", "pypandoc-binary (>=1.11)", "sphinx (>=7.1.1)"] +docs-online = ["mkdocs (>=1.5.2)", "mkdocs-git-revision-date-localized-plugin (>=1.2.0)", "mkdocs-material (>=9.2.6)", "mkdocstrings[python] (>=0.22.0)", "pyaml (>=23.7.0)"] +hooks = ["pre-commit (>=3.3.3)"] +package = ["build (>=1.0.0)", "twine (>=4.0.2)"] +runner = ["tox (>=4.11.1)"] +sast = ["bandit[toml] (>=1.7.5)"] +testing = ["pytest (>=7.4.0)"] +tooling = ["black (>=23.7.0)", "pyright (>=1.1.325)", "ruff (>=0.0.287)"] +tooling-extras = ["pyaml (>=23.7.0)", "pypandoc-binary (>=1.11)", "pytest (>=7.4.0)"] [[package]] name = "virtualenv" -version = "20.24.2" +version = "20.24.3" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.24.2-py3-none-any.whl", hash = "sha256:43a3052be36080548bdee0b42919c88072037d50d56c28bd3f853cbe92b953ff"}, - {file = "virtualenv-20.24.2.tar.gz", hash = "sha256:fd8a78f46f6b99a67b7ec5cf73f92357891a7b3a40fd97637c27f854aae3b9e0"}, + {file = "virtualenv-20.24.3-py3-none-any.whl", hash = "sha256:95a6e9398b4967fbcb5fef2acec5efaf9aa4972049d9ae41f95e0972a683fd02"}, + {file = "virtualenv-20.24.3.tar.gz", hash = "sha256:e5c3b4ce817b0b328af041506a2a299418c98747c4b1e68cb7527e74ced23efc"}, ] [package.dependencies] @@ -4320,7 +5294,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "watchfiles" version = "0.19.0" description = "Simple, modern and high performance file watching and code reload in python." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4355,7 +5328,6 @@ anyio = ">=3.0.0" name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "dev" optional = false python-versions = "*" files = [ @@ -4365,30 +5337,27 @@ files = [ [[package]] name = "weaviate-client" -version = "3.22.1" +version = "3.24.2" description = "A python native Weaviate client" -category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "weaviate-client-3.22.1.tar.gz", hash = "sha256:aff61bd3f5d74df20a62328443e3aa9c860d5330fdfb19c4d8ddc44cb604032f"}, - {file = "weaviate_client-3.22.1-py3-none-any.whl", hash = "sha256:01843a4899a227300e570409e77628e9d1b28476313f94943c37aee3f75112e1"}, + {file = "weaviate-client-3.24.2.tar.gz", hash = "sha256:6914c48c9a7e5ad0be9399271f9cb85d6f59ab77476c6d4e56a3925bf149edaa"}, + {file = "weaviate_client-3.24.2-py3-none-any.whl", hash = "sha256:bc50ca5fcebcd48de0d00f66700b0cf7c31a97c4cd3d29b4036d77c5d1d9479b"}, ] [package.dependencies] -authlib = ">=1.1.0" -requests = ">=2.28.0,<=2.31.0" -tqdm = ">=4.59.0,<5.0.0" -validators = ">=0.18.2,<=0.21.0" +authlib = ">=1.2.1,<2.0.0" +requests = ">=2.30.0,<3.0.0" +validators = ">=0.21.2,<1.0.0" [package.extras] -grpc = ["grpcio", "grpcio-tools"] +grpc = ["grpcio (>=1.57.0,<2.0.0)", "grpcio-tools (>=1.57.0,<2.0.0)"] [[package]] name = "websockets" version = "11.0.3" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4464,16 +5433,32 @@ files = [ {file = "websockets-11.0.3.tar.gz", hash = "sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016"}, ] +[[package]] +name = "werkzeug" +version = "2.3.7" +description = "The comprehensive WSGI web application library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "werkzeug-2.3.7-py3-none-any.whl", hash = "sha256:effc12dba7f3bd72e605ce49807bbe692bd729c3bb122a3b91747a6ae77df528"}, + {file = "werkzeug-2.3.7.tar.gz", hash = "sha256:2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog (>=2.3)"] + [[package]] name = "wheel" -version = "0.41.0" +version = "0.41.2" description = "A built-package format for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "wheel-0.41.0-py3-none-any.whl", hash = "sha256:7e9be3bbd0078f6147d82ed9ed957e323e7708f57e134743d2edef3a7b7972a9"}, - {file = "wheel-0.41.0.tar.gz", hash = "sha256:55a0f0a5a84869bce5ba775abfd9c462e3a6b1b7b7ec69d72c0b83d673a5114d"}, + {file = "wheel-0.41.2-py3-none-any.whl", hash = "sha256:75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8"}, + {file = "wheel-0.41.2.tar.gz", hash = "sha256:0c5ac5ff2afb79ac23ab82bab027a0be7b5dbcf2e54dc50efe4bf507de1f7985"}, ] [package.extras] @@ -4483,7 +5468,6 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] name = "win32-setctime" version = "1.1.0" description = "A small Python utility to set file creation time on Windows" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -4498,7 +5482,6 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] name = "yarl" version = "1.9.2" description = "Yet another URL library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4586,7 +5569,6 @@ multidict = ">=4.0" name = "zipp" version = "3.16.2" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -4601,4 +5583,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "92c70d26a6dae06c200a1d4c9c1e7fbc418cd549ca656a5ad2fa12abad7d53ab" +content-hash = "c6888acd2c89afe448e390a7b3601df3546a0e0892ac31ec958a5830ed7088fa" diff --git a/python/pyproject.toml b/python/pyproject.toml index 31b05b228c78..4970cf6746b6 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "semantic-kernel" -version = "0.3.4.dev" +version = "0.3.13.dev" description = "" authors = ["Microsoft "] readme = "pip/README.md" @@ -9,18 +9,27 @@ packages = [{include = "semantic_kernel"}] [tool.poetry.dependencies] python = "^3.8" numpy = "^1.24.2" -openai = "^0.27.0" +openai = ">=0.27,<0.29" aiofiles = "^23.1.0" python-dotenv = "1.0.0" regex = "^2023.6.3" +openapi_core = "^0.18.0" +prance = "^23.6.21.0" +pydantic = "<2" +motor = "^3.3.1" [tool.poetry.group.dev.dependencies] pre-commit = "3.3.3" black = {version = "23.7.0", allow-prereleases = true} ipykernel = "^6.21.1" -pytest = "7.4.0" -ruff = "0.0.280" +pytest = "7.4.2" +ruff = "0.0.289" pytest-asyncio = "0.21.1" +snoop = "0.4.3" + +[tool.poetry.group.google_palm.dependencies] +google-generativeai = { version = ">=0.1,<0.3", markers = "python_version >= '3.9'" } +grpcio-status = { version = "^1.53.0", markers = "python_version >= '3.9'" } [tool.poetry.group.hugging_face.dependencies] transformers = "^4.28.1" @@ -28,7 +37,7 @@ sentence-transformers = "^2.2.2" torch = "2.0.0" [tool.poetry.group.qdrant.dependencies] -qdrant-client = {version = "^1.1.7", python = ">=3.8,<3.12"} +qdrant-client = {version = "^1.3.2", python = ">=3.8,<3.12"} [tool.poetry.group.chromadb.dependencies] chromadb = "^0.4.0" @@ -48,11 +57,18 @@ psycopg-pool = "^3.1.7" psycopg = "^3.1.9" psycopg-binary = "^3.1.9" +[tool.poetry.group.redis.dependencies] +redis = "^4.6.0" + [tool.poetry.group.azure_cognitive_search.dependencies] -azure-search-documents = {version = "11.4.0b6", allow-prereleases = true} +azure-search-documents = {version = "11.4.0b9", allow-prereleases = true} azure-core = "^1.28.0" azure-identity = "^1.13.0" +[tool.poetry.group.usearch.dependencies] +usearch = "^1.1.1" +pyarrow = ">=12.0.1,<14.0.0" + [tool.isort] profile = "black" @@ -63,4 +79,3 @@ line-length = 120 [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" - diff --git a/python/samples/kernel-syntax-examples/action_planner.py b/python/samples/kernel-syntax-examples/action_planner.py new file mode 100644 index 000000000000..710bf14aac01 --- /dev/null +++ b/python/samples/kernel-syntax-examples/action_planner.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft. All rights reserved. + +import semantic_kernel as sk +from semantic_kernel.connectors.ai.open_ai import ( + OpenAIChatCompletion, +) +from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill +from semantic_kernel.planning import ActionPlanner + + +async def main(): + kernel = sk.Kernel() + api_key, org_id = sk.openai_settings_from_dot_env() + + kernel.add_chat_service( + "chat-gpt", OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id) + ) + kernel.import_skill(MathSkill(), "math") + kernel.import_skill(FileIOSkill(), "fileIO") + kernel.import_skill(TimeSkill(), "time") + kernel.import_skill(TextSkill(), "text") + + # create an instance of action planner. + planner = ActionPlanner(kernel) + + # the ask for which the action planner is going to find a relevant function. + ask = "What is the sum of 110 and 990?" + + # ask the action planner to identify a suitable function from the list of functions available. + plan = await planner.create_plan_async(goal=ask) + + # ask the action planner to execute the identified function. + result = await plan.invoke_async() + print(result) + """ + Output: + 1100 + """ + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/azure_chat_gpt_api.py b/python/samples/kernel-syntax-examples/azure_chat_gpt_api.py new file mode 100644 index 000000000000..719730002172 --- /dev/null +++ b/python/samples/kernel-syntax-examples/azure_chat_gpt_api.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +from dotenv import load_dotenv + +import semantic_kernel as sk +import semantic_kernel.connectors.ai.open_ai as sk_oai +from semantic_kernel.utils.settings import azure_openai_settings_from_dot_env_as_dict + +load_dotenv() + +system_message = """ +You are a chat bot. Your name is Mosscap and +you have one goal: figure out what people need. +Your full name, should you need to know it, is +Splendid Speckled Mosscap. You communicate +effectively, but you tend to answer with long +flowery prose. +""" + +kernel = sk.Kernel() + +kernel.add_chat_service( + "chat-gpt", + sk_oai.AzureChatCompletion( + **azure_openai_settings_from_dot_env_as_dict(include_api_version=True) + ), +) + +prompt_config = sk.PromptTemplateConfig.from_completion_parameters( + max_tokens=2000, temperature=0.7, top_p=0.8 +) + +prompt_template = sk.ChatPromptTemplate( + "{{$user_input}}", kernel.prompt_template_engine, prompt_config +) + +prompt_template.add_system_message(system_message) +prompt_template.add_user_message("Hi there, who are you?") +prompt_template.add_assistant_message( + "I am Mosscap, a chat bot. I'm trying to figure out what people need." +) + +function_config = sk.SemanticFunctionConfig(prompt_config, prompt_template) +chat_function = kernel.register_semantic_function("ChatBot", "Chat", function_config) + + +async def chat() -> bool: + context_vars = sk.ContextVariables() + + try: + user_input = input("User:> ") + context_vars["user_input"] = user_input + except KeyboardInterrupt: + print("\n\nExiting chat...") + return False + except EOFError: + print("\n\nExiting chat...") + return False + + if user_input == "exit": + print("\n\nExiting chat...") + return False + + answer = await kernel.run_async(chat_function, input_vars=context_vars) + print(f"Mosscap:> {answer}") + return True + + +async def main() -> None: + chatting = True + while chatting: + chatting = await chat() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/azure_cognitive_search_memory.py b/python/samples/kernel-syntax-examples/azure_cognitive_search_memory.py new file mode 100644 index 000000000000..7f7e9363bd31 --- /dev/null +++ b/python/samples/kernel-syntax-examples/azure_cognitive_search_memory.py @@ -0,0 +1,101 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +from dotenv import dotenv_values + +import semantic_kernel as sk +from semantic_kernel.connectors.ai.open_ai import ( + AzureTextCompletion, + AzureTextEmbedding, +) +from semantic_kernel.connectors.memory.azure_cognitive_search import ( + AzureCognitiveSearchMemoryStore, +) + +COLLECTION_NAME = "acs-index-sample" + + +async def populate_memory(kernel: sk.Kernel) -> None: + # Add some documents to the ACS semantic memory + await kernel.memory.save_information_async( + COLLECTION_NAME, id="info1", text="My name is Andrea" + ) + await kernel.memory.save_information_async( + COLLECTION_NAME, id="info2", text="I currently work as a tour guide" + ) + await kernel.memory.save_information_async( + COLLECTION_NAME, id="info3", text="I've been living in Seattle since 2005" + ) + await kernel.memory.save_information_async( + COLLECTION_NAME, + id="info4", + text="I visited France and Italy five times since 2015", + ) + await kernel.memory.save_information_async( + COLLECTION_NAME, id="info5", text="My family is from New York" + ) + + +async def search_acs_memory_questions(kernel: sk.Kernel) -> None: + questions = [ + "what's my name", + "where do I live?", + "where's my family from?", + "where have I traveled?", + "what do I do for work", + ] + + for question in questions: + print(f"Question: {question}") + result = await kernel.memory.search_async(COLLECTION_NAME, question) + print(f"Answer: {result[0].text}\n") + + +async def main() -> None: + kernel = sk.Kernel() + + config = dotenv_values(".env") + + AZURE_COGNITIVE_SEARCH_ENDPOINT = config["AZURE_COGNITIVE_SEARCH_ENDPOINT"] + AZURE_COGNITIVE_SEARCH_ADMIN_KEY = config["AZURE_COGNITIVE_SEARCH_ADMIN_KEY"] + AZURE_OPENAI_API_KEY = config["AZURE_OPENAI_API_KEY"] + AZURE_OPENAI_ENDPOINT = config["AZURE_OPENAI_ENDPOINT"] + vector_size = 1536 + + # Setting up OpenAI services for text completion and text embedding + kernel.add_text_completion_service( + "dv", + AzureTextCompletion( + deployment_name="text-embedding-ada-002", + endpoint=AZURE_OPENAI_ENDPOINT, + api_key=AZURE_OPENAI_API_KEY, + ), + ) + kernel.add_text_embedding_generation_service( + "ada", + AzureTextEmbedding( + deployment_name="text-embedding-ada-002", + endpoint=AZURE_OPENAI_ENDPOINT, + api_key=AZURE_OPENAI_API_KEY, + ), + ) + + connector = AzureCognitiveSearchMemoryStore( + vector_size, AZURE_COGNITIVE_SEARCH_ENDPOINT, AZURE_COGNITIVE_SEARCH_ADMIN_KEY + ) + + # Register the memory store with the kernel + kernel.register_memory_store(memory_store=connector) + + print("Populating memory...") + await populate_memory(kernel) + + print("Asking questions... (manually)") + await search_acs_memory_questions(kernel) + + await connector.close_async() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/bing_search_skill.py b/python/samples/kernel-syntax-examples/bing_search_skill.py index f5369c5baa9f..af1107d36e4d 100644 --- a/python/samples/kernel-syntax-examples/bing_search_skill.py +++ b/python/samples/kernel-syntax-examples/bing_search_skill.py @@ -3,7 +3,7 @@ from dotenv import load_dotenv import semantic_kernel as sk -from semantic_kernel.connectors.ai.open_ai import OpenAITextCompletion +from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion from semantic_kernel.connectors.search_engine import BingConnector from semantic_kernel.core_skills import WebSearchEngineSkill @@ -13,8 +13,8 @@ async def main(): kernel = sk.Kernel() api_key, org_id = sk.openai_settings_from_dot_env() - kernel.add_text_completion_service( - "dv", OpenAITextCompletion("text-davinci-003", api_key, org_id) + kernel.add_chat_service( + "chat-gpt", OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id) ) connector = BingConnector(api_key=os.getenv("BING_API_KEY")) web_skill = kernel.import_skill(WebSearchEngineSkill(connector), "WebSearch") diff --git a/python/samples/kernel-syntax-examples/chat.py b/python/samples/kernel-syntax-examples/chat.py index 179c4d7166e0..ee149b00b383 100644 --- a/python/samples/kernel-syntax-examples/chat.py +++ b/python/samples/kernel-syntax-examples/chat.py @@ -18,8 +18,8 @@ kernel = sk.Kernel() api_key, org_id = sk.openai_settings_from_dot_env() -kernel.add_text_completion_service( - "davinci-003", sk_oai.OpenAITextCompletion("text-davinci-003", api_key, org_id) +kernel.add_chat_service( + "chat-gpt", sk_oai.OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id) ) prompt_config = sk.PromptTemplateConfig.from_completion_parameters( diff --git a/python/samples/kernel-syntax-examples/chat_gpt_api_function_calling.py b/python/samples/kernel-syntax-examples/chat_gpt_api_function_calling.py new file mode 100644 index 000000000000..df0d3d2e9c21 --- /dev/null +++ b/python/samples/kernel-syntax-examples/chat_gpt_api_function_calling.py @@ -0,0 +1,121 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import os +from typing import Tuple + +import semantic_kernel as sk +import semantic_kernel.connectors.ai.open_ai as sk_oai +from semantic_kernel.connectors.ai.open_ai.semantic_functions.open_ai_chat_prompt_template import ( + OpenAIChatPromptTemplate, +) +from semantic_kernel.connectors.ai.open_ai.utils import ( + chat_completion_with_function_call, + get_function_calling_object, +) +from semantic_kernel.core_skills import MathSkill + +system_message = """ +You are a chat bot. Your name is Mosscap and +you have one goal: figure out what people need. +Your full name, should you need to know it, is +Splendid Speckled Mosscap. You communicate +effectively, but you tend to answer with long +flowery prose. You are also a math wizard, +especially for adding and subtracting. +You also excel at joke telling, where your tone is often sarcastic. +Once you have the answer I am looking for, +you will return a full answer to me as soon as possible. +""" + +kernel = sk.Kernel() + +deployment_name, api_key, endpoint = sk.azure_openai_settings_from_dot_env() +api_version = "2023-07-01-preview" +kernel.add_chat_service( + "chat-gpt", + sk_oai.AzureChatCompletion( + deployment_name, + endpoint, + api_key, + api_version=api_version, + ), +) + +skills_directory = os.path.join(__file__, "../../../../samples/skills") +# adding skills to the kernel +# the joke skill in the FunSkills is a semantic skill and has the function calling disabled. +kernel.import_semantic_skill_from_directory(skills_directory, "FunSkill") +# the math skill is a core skill and has the function calling enabled. +kernel.import_skill(MathSkill(), skill_name="math") + +# enabling or disabling function calling is done by setting the function_call parameter for the completion. +# when the function_call parameter is set to "auto" the model will decide which function to use, if any. +# if you only want to use a specific function, set the name of that function in this parameter, +# the format for that is 'SkillName-FunctionName', (i.e. 'math-Add'). +# if the model or api version do not support this you will get an error. +prompt_config = sk.PromptTemplateConfig.from_completion_parameters( + max_tokens=2000, + temperature=0.7, + top_p=0.8, + function_call="auto", + chat_system_prompt=system_message, +) +prompt_template = OpenAIChatPromptTemplate( + "{{$user_input}}", kernel.prompt_template_engine, prompt_config +) +prompt_template.add_user_message("Hi there, who are you?") +prompt_template.add_assistant_message( + "I am Mosscap, a chat bot. I'm trying to figure out what people need." +) + +function_config = sk.SemanticFunctionConfig(prompt_config, prompt_template) +chat_function = kernel.register_semantic_function("ChatBot", "Chat", function_config) + +# calling the chat, you could add a overloaded version of the settings here, +# to enable or disable function calling or set the function calling to a specific skill. +# see the openai_function_calling example for how to use this with a unrelated function definition +filter = {"exclude_skill": ["ChatBot"]} +functions = get_function_calling_object(kernel, filter) + + +async def chat(context: sk.SKContext) -> Tuple[bool, sk.SKContext]: + try: + user_input = input("User:> ") + context.variables["user_input"] = user_input + except KeyboardInterrupt: + print("\n\nExiting chat...") + return False, None + except EOFError: + print("\n\nExiting chat...") + return False, None + + if user_input == "exit": + print("\n\nExiting chat...") + return False, None + + context = await chat_completion_with_function_call( + kernel, + chat_skill_name="ChatBot", + chat_function_name="Chat", + context=context, + functions=functions, + ) + print(f"Mosscap:> {context.result}") + return True, context + + +async def main() -> None: + chatting = True + context = kernel.create_new_context() + print( + "Welcome to the chat bot!\ +\n Type 'exit' to exit.\ +\n Try a math question to see the function calling in action (i.e. what is 3+3?)." + ) + while chatting: + chatting, context = await chat(context) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/google_palm_chat.py b/python/samples/kernel-syntax-examples/google_palm_chat.py new file mode 100644 index 000000000000..36d2752d0682 --- /dev/null +++ b/python/samples/kernel-syntax-examples/google_palm_chat.py @@ -0,0 +1,48 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +import semantic_kernel as sk +import semantic_kernel.connectors.ai.google_palm as sk_gp +from semantic_kernel.connectors.ai.chat_request_settings import ChatRequestSettings + + +async def chat_request_example(api_key): + palm_chat_completion = sk_gp.GooglePalmChatCompletion( + "models/chat-bison-001", api_key + ) + settings = ChatRequestSettings() + settings.temperature = 1 + + chat_messages = list() + user_mssg = "I'm planning a vacation. Which are some must-visit places in Europe?" + chat_messages.append(("user", user_mssg)) + answer = await palm_chat_completion.complete_chat_async(chat_messages, settings) + chat_messages.append(("assistant", str(answer))) + user_mssg = "Where should I go in France?" + chat_messages.append(("user", user_mssg)) + answer = await palm_chat_completion.complete_chat_async(chat_messages, settings) + chat_messages.append(("assistant", str(answer))) + + context_vars = sk.ContextVariables() + context_vars["chat_history"] = "" + context_vars["chat_bot_ans"] = "" + for role, mssg in chat_messages: + if role == "user": + context_vars["chat_history"] += f"User:> {mssg}\n" + elif role == "assistant": + context_vars["chat_history"] += f"ChatBot:> {mssg}\n" + context_vars["chat_bot_ans"] += f"{mssg}\n" + + return context_vars + + +async def main() -> None: + api_key = sk.google_palm_settings_from_dot_env() + chat = await chat_request_example(api_key) + print(chat["chat_history"]) + return + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/google_palm_chat_with_memory.py b/python/samples/kernel-syntax-examples/google_palm_chat_with_memory.py new file mode 100644 index 000000000000..13e4c71afa3f --- /dev/null +++ b/python/samples/kernel-syntax-examples/google_palm_chat_with_memory.py @@ -0,0 +1,142 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +from typing import Tuple + +import semantic_kernel as sk +import semantic_kernel.connectors.ai.google_palm as sk_gp + +kernel = sk.Kernel() +apikey = sk.google_palm_settings_from_dot_env() +palm_text_embed = sk_gp.GooglePalmTextEmbedding("models/embedding-gecko-001", apikey) +kernel.add_text_embedding_generation_service("gecko", palm_text_embed) +palm_chat_completion = sk_gp.GooglePalmChatCompletion("models/chat-bison-001", apikey) +kernel.add_chat_service("models/chat-bison-001", palm_chat_completion) +kernel.register_memory_store(memory_store=sk.memory.VolatileMemoryStore()) +kernel.import_skill(sk.core_skills.TextMemorySkill()) + + +async def populate_memory(kernel: sk.Kernel) -> None: + # Add some documents to the semantic memory + await kernel.memory.save_information_async( + "aboutMe", id="info1", text="My name is Andrea" + ) + await kernel.memory.save_information_async( + "aboutMe", id="info2", text="I currently work as a tour guide" + ) + await kernel.memory.save_information_async( + "aboutMe", id="info3", text="My favorite hobby is hiking" + ) + await kernel.memory.save_information_async( + "aboutMe", id="info4", text="I visitied Iceland last year." + ) + await kernel.memory.save_information_async( + "aboutMe", id="info5", text="My family is from New York" + ) + + +async def search_memory_examples(kernel: sk.Kernel) -> None: + questions = [ + "what's my name", + "what is my favorite hobby?", + "where's my family from?", + "where did I travel last year?", + "what do I do for work", + ] + + for question in questions: + print(f"Question: {question}") + result = await kernel.memory.search_async("aboutMe", question) + print(f"Answer: {result}\n") + + +async def setup_chat_with_memory( + kernel: sk.Kernel, +) -> Tuple[sk.SKFunctionBase, sk.SKContext]: + """ + When using Google PaLM to chat with memories, a chat prompt template is + essential; otherwise, the kernel will send text prompts to the Google PaLM + chat service. Unfortunately, when a text prompt includes memory, chat + history, and the user's current message, PaLM often struggles to comprehend + the user's message. To address this issue, the prompt containing memory is + incorporated into the chat prompt template as a system message. + Note that this is only an issue for the chat service; the text service + does not require a chat prompt template. + """ + sk_prompt = """ + ChatBot can have a conversation with you about any topic. + It can give explicit instructions or say 'I don't know' if + it does not have an answer. + + Information about me, from previous conversations: + - {{$fact1}} {{recall $fact1}} + - {{$fact2}} {{recall $fact2}} + - {{$fact3}} {{recall $fact3}} + - {{$fact4}} {{recall $fact4}} + - {{$fact5}} {{recall $fact5}} + + """.strip() + + prompt_config = sk.PromptTemplateConfig.from_completion_parameters( + max_tokens=2000, temperature=0.7, top_p=0.8 + ) + prompt_template = sk.ChatPromptTemplate( # Create the chat prompt template + "{{$user_input}}", kernel.prompt_template_engine, prompt_config + ) + prompt_template.add_system_message(sk_prompt) # Add the memory as a system message + function_config = sk.SemanticFunctionConfig(prompt_config, prompt_template) + chat_func = kernel.register_semantic_function( + None, "ChatWithMemory", function_config + ) + + context = kernel.create_new_context() + context["fact1"] = "what is my name?" + context["fact2"] = "what is my favorite hobby?" + context["fact3"] = "where's my family from?" + context["fact4"] = "where did I travel last year?" + context["fact5"] = "what do I do for work?" + + context[sk.core_skills.TextMemorySkill.COLLECTION_PARAM] = "aboutMe" + context[sk.core_skills.TextMemorySkill.RELEVANCE_PARAM] = 0.6 + + context["chat_history"] = "" + + return chat_func, context + + +async def chat( + kernel: sk.Kernel, chat_func: sk.SKFunctionBase, context: sk.SKContext +) -> bool: + try: + user_input = input("User:> ") + context["user_input"] = user_input + except KeyboardInterrupt: + print("\n\nExiting chat...") + return False + except EOFError: + print("\n\nExiting chat...") + return False + + if user_input == "exit": + print("\n\nExiting chat...") + return False + + answer = await kernel.run_async(chat_func, input_vars=context.variables) + context["chat_history"] += f"\nUser:> {user_input}\nChatBot:> {answer}\n" + + print(f"ChatBot:> {answer}") + return True + + +async def main() -> None: + await populate_memory(kernel) + await search_memory_examples(kernel) + chat_func, context = await setup_chat_with_memory(kernel) + print("Begin chatting (type 'exit' to exit):\n") + chatting = True + while chatting: + chatting = await chat(kernel, chat_func, context) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/google_palm_chat_with_skill.py b/python/samples/kernel-syntax-examples/google_palm_chat_with_skill.py new file mode 100644 index 000000000000..042495699446 --- /dev/null +++ b/python/samples/kernel-syntax-examples/google_palm_chat_with_skill.py @@ -0,0 +1,79 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +import semantic_kernel as sk +import semantic_kernel.connectors.ai.google_palm as sk_gp + +""" +System messages prime the assistant with different personalities or behaviors. +The system message is added to the prompt template, and a chat history can be +added as well to provide further context. +A system message can only be used once at the start of the conversation, and +conversation history persists with the instance of GooglePalmChatCompletion. To +overwrite the system message and start a new conversation, you must create a new +instance of GooglePalmChatCompletion. +Sometimes, PaLM struggles to use the information in the prompt template. In this +case, it is recommended to experiment with the messages in the prompt template +or ask different questions. +""" + +system_message = """ +You are a chat bot. Your name is Blackbeard +and you speak in the style of a swashbuckling +pirate. You reply with brief, to-the-point answers +with no elaboration. Your full name is Captain +Bartholomew "Blackbeard" Thorne. +""" + +kernel = sk.Kernel() +api_key = sk.google_palm_settings_from_dot_env() +palm_chat_completion = sk_gp.GooglePalmChatCompletion("models/chat-bison-001", api_key) +kernel.add_chat_service("models/chat-bison-001", palm_chat_completion) +prompt_config = sk.PromptTemplateConfig.from_completion_parameters( + max_tokens=2000, temperature=0.7, top_p=0.8 +) +prompt_template = sk.ChatPromptTemplate( + "{{$user_input}}", kernel.prompt_template_engine, prompt_config +) +prompt_template.add_system_message(system_message) # Add the system message for context +prompt_template.add_user_message( + "Hi there, my name is Andrea, who are you?" +) # Include a chat history +prompt_template.add_assistant_message("I am Blackbeard.") +function_config = sk.SemanticFunctionConfig(prompt_config, prompt_template) +chat_function = kernel.register_semantic_function( + "PirateSkill", "Chat", function_config +) + + +async def chat() -> bool: + context_vars = sk.ContextVariables() + + try: + user_input = input("User:> ") + context_vars["user_input"] = user_input + except KeyboardInterrupt: + print("\n\nExiting chat...") + return False + except EOFError: + print("\n\nExiting chat...") + return False + + if user_input == "exit": + print("\n\nExiting chat...") + return False + + answer = await kernel.run_async(chat_function, input_vars=context_vars) + print(f"Blackbeard:> {answer}") + return True + + +async def main() -> None: + chatting = True + while chatting: + chatting = await chat() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/google_palm_text_completion.py b/python/samples/kernel-syntax-examples/google_palm_text_completion.py new file mode 100644 index 000000000000..664e1194bcf6 --- /dev/null +++ b/python/samples/kernel-syntax-examples/google_palm_text_completion.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +import semantic_kernel as sk +import semantic_kernel.connectors.ai.google_palm as sk_gp +from semantic_kernel.connectors.ai.complete_request_settings import ( + CompleteRequestSettings, +) + + +async def text_completion_example_complete_async(kernel, api_key, user_mssg, settings): + """ + Complete a text prompt using the Google PaLM model and print the results. + """ + palm_text_completion = sk_gp.GooglePalmTextCompletion( + "models/text-bison-001", api_key + ) + kernel.add_text_completion_service("models/text-bison-001", palm_text_completion) + answer = await palm_text_completion.complete_async(user_mssg, settings) + return answer + + +async def main() -> None: + kernel = sk.Kernel() + apikey = sk.google_palm_settings_from_dot_env() + settings = CompleteRequestSettings() + + user_mssg1 = ( + "Sam has three boxes, each containing a certain number of coins. " + "The first box has twice as many coins as the second box, and the second " + "box has three times as many coins as the third box. Together, the three " + "boxes have 98 coins in total. How many coins are there in each box? " + "Think about it step by step, and show your work." + ) + response = await text_completion_example_complete_async( + kernel, apikey, user_mssg1, settings + ) + print(f"User:> {user_mssg1}\n\nChatBot:> {response}\n") + # Use temperature to influence the variance of the responses + settings.number_of_responses = 3 + settings.temperature = 1 + user_mssg2 = ( + "I need a concise answer. A common method for traversing a binary tree is" + ) + response = await text_completion_example_complete_async( + kernel, apikey, user_mssg2, settings + ) + print(f"User:> {user_mssg2}\n\nChatBot:> {response}") + return + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/google_search_skill.py b/python/samples/kernel-syntax-examples/google_search_skill.py new file mode 100644 index 000000000000..14ec3542e858 --- /dev/null +++ b/python/samples/kernel-syntax-examples/google_search_skill.py @@ -0,0 +1,90 @@ +# Copyright (c) Microsoft. All rights reserved. + +import os + +from dotenv import load_dotenv + +import semantic_kernel as sk +from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion +from semantic_kernel.connectors.search_engine import GoogleConnector +from semantic_kernel.core_skills import WebSearchEngineSkill + +load_dotenv() + + +async def main(): + kernel = sk.Kernel() + api_key, org_id = sk.openai_settings_from_dot_env() + kernel.add_chat_service( + "chat-gpt", OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id) + ) + + """ + Instantiate a Google Connector + Make sure to have the following keys in a .env file or set as environment variables + - GOOGLE_API_KEY + - GOOGLE_SEARCH_ENGINE_ID + + A Google Custom Search API has to be created in order to have an API key and a search engine ID. + To create a Google Custom Search API, follow the guide - https://developers.google.com/custom-search/v1/overview. + If you have already created the service, the credentials can be found in the Credentials tab on the page + https://console.cloud.google.com/apis/api/customsearch.googleapis.com + """ + connector = GoogleConnector( + api_key=os.getenv("GOOGLE_API_KEY"), + search_engine_id=os.getenv("GOOGLE_SEARCH_ENGINE_ID"), + ) + + # Import the WebSearchEngineSkill and pass the Google Connector to it. + web_skill = kernel.import_skill(WebSearchEngineSkill(connector), "WebSearch") + + # The search query + prompt = "Who is Leonardo DiCaprio's current girlfriend?" + search_async = web_skill["searchAsync"] + + # By default, only one search result is provided + result = await search_async.invoke_async(prompt) + print(result) + + """ + Output: + ["Celebrity Celebrity News Everything You Need to Know About Leonardo DiCaprio and Camila Morrone's + Relationship From the beginning of their romance to today, we track their relationship here. By..."] + """ + + # Following example demonstrates the use of the skill within a semantic function + prompt = """ + Answer the question using only the data that is provided in the data section. + Do not use any prior knowledge to answer the question. + Data: {{WebSearch.SearchAsync "What is semantic kernel?"}} + Question: What is semantic kernel? + Answer: + """ + + qna = kernel.create_semantic_function(prompt, temperature=0.2) + context = kernel.create_new_context() + + """ + Two context parameters can be passed to the search engine skill. + - num_results controls the number of results returned by the web search. + - offset controls the number of results to omit. + """ + context["num_results"] = "10" + context["offset"] = "0" + + result = await qna.invoke_async(context=context) + print(result) + + """ + Output: + Semantic Kernel is an open-source SDK that lets you easily combine AI services like OpenAI, + Azure OpenAI, and Hugging Face with conventional programming languages like C# and Python. + By doing so, you can create AI apps that combine the best of both worlds. + Semantic Kernel is at the center of the copilot stack. + """ + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/memory.py b/python/samples/kernel-syntax-examples/memory.py index 110a11767426..b4376a846e0e 100644 --- a/python/samples/kernel-syntax-examples/memory.py +++ b/python/samples/kernel-syntax-examples/memory.py @@ -108,8 +108,8 @@ async def main() -> None: kernel = sk.Kernel() api_key, org_id = sk.openai_settings_from_dot_env() - kernel.add_text_completion_service( - "dv", sk_oai.OpenAITextCompletion("text-davinci-003", api_key, org_id) + kernel.add_chat_service( + "chat-gpt", sk_oai.OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id) ) kernel.add_text_embedding_generation_service( "ada", sk_oai.OpenAITextEmbedding("text-embedding-ada-002", api_key, org_id) diff --git a/python/samples/kernel-syntax-examples/openai_function_calling.py b/python/samples/kernel-syntax-examples/openai_function_calling.py new file mode 100644 index 000000000000..ecce09438df2 --- /dev/null +++ b/python/samples/kernel-syntax-examples/openai_function_calling.py @@ -0,0 +1,109 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import os + +import semantic_kernel as sk +import semantic_kernel.connectors.ai.open_ai as sk_oai +from semantic_kernel.core_skills import MathSkill + +system_message = """ +You are a chat bot. Your name is Mosscap and +you have one goal: figure out what people need. +Your full name, should you need to know it, is +Splendid Speckled Mosscap. You communicate +effectively, but you tend to answer with long +flowery prose. You are also a math wizard, +especially for adding and subtracting. +You also excel at joke telling, where your tone is often sarcastic. +Once you have the answer I am looking for, +you will return a full answer to me as soon as possible. +""" + +kernel = sk.Kernel() + +deployment_name, api_key, endpoint = sk.azure_openai_settings_from_dot_env() +api_version = "2023-07-01-preview" +kernel.add_chat_service( + "chat-gpt", + sk_oai.AzureChatCompletion( + deployment_name, + endpoint, + api_key, + api_version=api_version, + ), +) + +skills_directory = os.path.join(__file__, "../../../../samples/skills") +# adding skills to the kernel +# the joke skill in the FunSkills is a semantic skill and has the function calling disabled. +kernel.import_semantic_skill_from_directory(skills_directory, "FunSkill") +# the math skill is a core skill and has the function calling enabled. +kernel.import_skill(MathSkill(), skill_name="math") + +# enabling or disabling function calling is done by setting the function_call parameter for the completion. +# when the function_call parameter is set to "auto" the model will decide which function to use, if any. +# if you only want to use a specific function, set the name of that function in this parameter, +# the format for that is 'SkillName-FunctionName', (i.e. 'math-Add'). +# if the model or api version do not support this you will get an error. +prompt_config = sk.PromptTemplateConfig.from_completion_parameters( + max_tokens=2000, + temperature=0.7, + top_p=0.8, + function_call="auto", + chat_system_prompt=system_message, +) +prompt_template = sk.ChatPromptTemplate( + "{{$user_input}}", kernel.prompt_template_engine, prompt_config +) +prompt_template.add_user_message("Hi there, who are you?") +prompt_template.add_assistant_message( + "I am Mosscap, a chat bot. I'm trying to figure out what people need." +) + +function_config = sk.SemanticFunctionConfig(prompt_config, prompt_template) +chat_function = kernel.register_semantic_function("ChatBot", "Chat", function_config) +# define the functions available +functions = [ + { + "name": "search_hotels", + "description": "Retrieves hotels from the search index based on the parameters provided", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The location of the hotel (i.e. Seattle, WA)", + }, + "max_price": { + "type": "number", + "description": "The maximum price for the hotel", + }, + "features": { + "type": "string", + "description": "A comma separated list of features (i.e. beachfront, free wifi, etc.)", + }, + }, + "required": ["location"], + }, + } +] + + +async def main() -> None: + context = kernel.create_new_context() + context.variables[ + "user_input" + ] = "I want to find a hotel in Seattle with free wifi and a pool." + + context = await chat_function.invoke_async(context=context, functions=functions) + if function_call := context.pop_function_call(): + print(f"Function to be called: {function_call.name}") + print(f"Function parameters: \n{function_call.arguments}") + return + print("No function was called") + print(f"Output was: {str(context)}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/kernel-syntax-examples/openapi_example/README.md b/python/samples/kernel-syntax-examples/openapi_example/README.md new file mode 100644 index 000000000000..1304a61bc4c0 --- /dev/null +++ b/python/samples/kernel-syntax-examples/openapi_example/README.md @@ -0,0 +1,9 @@ +### Running the OpenApi syntax example + +1. In a terminal, navigate to `semantic_kernel/python/samples/kernel-syntax-examples/openapi_example`. + +2. Run `poetry install` followed by `poetry shell` to enter poetry's virtual environment. + +3. Start the server by running `python openapi_server.py`. + +4. In another terminal, do steps 1 & 2. Then, run `python openapi_client.py`, which will register a skill representing the API defined in openapi.yaml diff --git a/python/samples/kernel-syntax-examples/openapi_example/openapi.yaml b/python/samples/kernel-syntax-examples/openapi_example/openapi.yaml new file mode 100644 index 000000000000..bd211febec43 --- /dev/null +++ b/python/samples/kernel-syntax-examples/openapi_example/openapi.yaml @@ -0,0 +1,44 @@ +openapi: 3.0.0 +info: + title: Test API + version: 1.0.0 +servers: + - url: http://localhost:8080 +paths: + /{name}: + post: + summary: Hello World + operationId: helloWorld + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + input: + type: string + description: The input of the request + example: Howdy + responses: + '200': + description: OK + parameters: + - name: name + in: path + required: true + schema: + type: string + description: Your name + - name: Header + in: header + required: false + schema: + type: string + description: The header + - name: q + in: query + required: false + schema: + type: string + description: The query parameter \ No newline at end of file diff --git a/python/samples/kernel-syntax-examples/openapi_example/openapi_client.py b/python/samples/kernel-syntax-examples/openapi_example/openapi_client.py new file mode 100644 index 000000000000..f8e51051f03e --- /dev/null +++ b/python/samples/kernel-syntax-examples/openapi_example/openapi_client.py @@ -0,0 +1,24 @@ +import asyncio + +import semantic_kernel as sk +from semantic_kernel.connectors.openapi import register_openapi_skill + +if __name__ == "__main__": + """Client""" + kernel = sk.Kernel() + + openapi_skill = register_openapi_skill(kernel, "openApiSkill", "openapi.yaml") + + context_variables = sk.ContextVariables( + variables={ + "request_body": '{"input": "hello world"}', + "path_params": '{"name": "mark"}', + "query_params": '{"q": "0.7"}', + "headers": '{"Content-Type": "application/json", "Header": "example"}', + } + ) + result = asyncio.run( + # Call the function defined in openapi.yaml + openapi_skill["helloWorld"].invoke_async(variables=context_variables) + ) + print(result) diff --git a/python/samples/kernel-syntax-examples/openapi_example/openapi_server.py b/python/samples/kernel-syntax-examples/openapi_example/openapi_server.py new file mode 100644 index 000000000000..2b3c54358be8 --- /dev/null +++ b/python/samples/kernel-syntax-examples/openapi_example/openapi_server.py @@ -0,0 +1,24 @@ +from aiohttp import web + +"""Server""" +routes = web.RouteTableDef() + + +@routes.post("/{name}") +async def hello(request): + # Get path parameters + name = request.match_info.get("name", "") + # Get query parameters + q = request.rel_url.query.get("q", "") + # Get body + body = await request.json() + # Get headers + headers = request.headers + return web.Response(text=f"Hello, {name}: q={q}, body={body}, headers={headers}") + + +app = web.Application() +app.add_routes(routes) + +if __name__ == "__main__": + web.run_app(app) diff --git a/python/samples/kernel-syntax-examples/sequential_planner.py b/python/samples/kernel-syntax-examples/sequential_planner.py new file mode 100644 index 000000000000..82c3e32184f7 --- /dev/null +++ b/python/samples/kernel-syntax-examples/sequential_planner.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft. All rights reserved. + +import semantic_kernel as sk +from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion +from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill +from semantic_kernel.planning import SequentialPlanner + + +async def main(): + kernel = sk.Kernel() + api_key, org_id = sk.openai_settings_from_dot_env() + + kernel.add_chat_service( + "gpt-3.5", OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id) + ) + kernel.import_skill(MathSkill(), "math") + kernel.import_skill(FileIOSkill(), "fileIO") + kernel.import_skill(TimeSkill(), "time") + kernel.import_skill(TextSkill(), "text") + + # create an instance of sequential planner. + planner = SequentialPlanner(kernel) + + # the ask for which the sequential planner is going to find a relevant function. + ask = "What day of the week is today, all uppercase?" + + # ask the sequential planner to identify a suitable function from the list of functions available. + plan = await planner.create_plan_async(goal=ask) + + # ask the sequential planner to execute the identified function. + result = await plan.invoke_async() + + for step in plan._steps: + print(step.description, ":", step._state.__dict__) + + print("Expected Answer:") + print(result) + """ + Output: + SUNDAY + """ + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/python/semantic_kernel/__init__.py b/python/semantic_kernel/__init__.py index 1fc5d3c2e8bb..24526dbe03d6 100644 --- a/python/semantic_kernel/__init__.py +++ b/python/semantic_kernel/__init__.py @@ -16,9 +16,13 @@ from semantic_kernel.utils.null_logger import NullLogger from semantic_kernel.utils.settings import ( azure_openai_settings_from_dot_env, + bing_search_settings_from_dot_env, + google_palm_settings_from_dot_env, + mongodb_atlas_settings_from_dot_env, openai_settings_from_dot_env, pinecone_settings_from_dot_env, postgres_settings_from_dot_env, + redis_settings_from_dot_env, ) __all__ = [ @@ -28,6 +32,10 @@ "azure_openai_settings_from_dot_env", "postgres_settings_from_dot_env", "pinecone_settings_from_dot_env", + "bing_search_settings_from_dot_env", + "mongodb_atlas_settings_from_dot_env", + "google_palm_settings_from_dot_env", + "redis_settings_from_dot_env", "PromptTemplateConfig", "PromptTemplate", "ChatPromptTemplate", diff --git a/python/semantic_kernel/connectors/__init__.py b/python/semantic_kernel/connectors/__init__.py new file mode 100644 index 000000000000..2a50eae89411 --- /dev/null +++ b/python/semantic_kernel/connectors/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Microsoft. All rights reserved. diff --git a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py index ab4afdaf0614..118c84c8ac7f 100644 --- a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py +++ b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py @@ -2,26 +2,27 @@ from abc import ABC, abstractmethod from logging import Logger -from typing import TYPE_CHECKING, List, Tuple, Union +from typing import TYPE_CHECKING, List, Optional, Union if TYPE_CHECKING: from semantic_kernel.connectors.ai.chat_request_settings import ChatRequestSettings + from semantic_kernel.models.chat.chat_message import ChatMessage class ChatCompletionClientBase(ABC): @abstractmethod async def complete_chat_async( self, - messages: List[Tuple[str, str]], + messages: List["ChatMessage"], settings: "ChatRequestSettings", - logger: Logger, + logger: Optional[Logger] = None, ) -> Union[str, List[str]]: """ This is the method that is called from the kernel to get a response from a chat-optimized LLM. Arguments: - messages {List[Tuple[str, str]]} -- A list of tuples, where each tuple is - comprised of a speaker ID and a message. + messages {List[ChatMessage]} -- A list of chat messages, that can be rendered into a + set of messages, from system, user, assistant and function. settings {ChatRequestSettings} -- Settings for the request. logger {Logger} -- A logger to use for logging. @@ -33,16 +34,16 @@ async def complete_chat_async( @abstractmethod async def complete_chat_stream_async( self, - messages: List[Tuple[str, str]], + messages: List["ChatMessage"], settings: "ChatRequestSettings", - logger: Logger, + logger: Optional[Logger] = None, ): """ This is the method that is called from the kernel to get a stream response from a chat-optimized LLM. Arguments: - messages {List[Tuple[str, str]]} -- A list of tuples, where each tuple is - comprised of a speaker ID and a message. + messages {List[ChatMessage]} -- A list of chat messages, that can be rendered into a + set of messages, from system, user, assistant and function. settings {ChatRequestSettings} -- Settings for the request. logger {Logger} -- A logger to use for logging. diff --git a/python/semantic_kernel/connectors/ai/chat_request_settings.py b/python/semantic_kernel/connectors/ai/chat_request_settings.py index 65b4fe425316..5718f28fbe30 100644 --- a/python/semantic_kernel/connectors/ai/chat_request_settings.py +++ b/python/semantic_kernel/connectors/ai/chat_request_settings.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING, Dict, List, Optional if TYPE_CHECKING: from semantic_kernel.semantic_functions.prompt_template_config import ( @@ -18,16 +18,25 @@ class ChatRequestSettings: number_of_responses: int = 1 max_tokens: int = 256 token_selection_biases: Dict[int, int] = field(default_factory=dict) + stop_sequences: List[str] = field(default_factory=list) + function_call: Optional[str] = None def update_from_completion_config( self, completion_config: "PromptTemplateConfig.CompletionConfig" ): self.temperature = completion_config.temperature self.top_p = completion_config.top_p - self.presence_penalty = completion_config.presence_penalty - self.frequency_penalty = completion_config.frequency_penalty self.number_of_responses = completion_config.number_of_responses + self.stop_sequences = completion_config.stop_sequences self.max_tokens = completion_config.max_tokens + self.presence_penalty = completion_config.presence_penalty + self.frequency_penalty = completion_config.frequency_penalty + self.token_selection_biases = completion_config.token_selection_biases + self.function_call = ( + completion_config.function_call + if hasattr(completion_config, "function_call") + else None + ) @staticmethod def from_completion_config( diff --git a/python/semantic_kernel/connectors/ai/complete_request_settings.py b/python/semantic_kernel/connectors/ai/complete_request_settings.py index d6889e1d8d60..4669499c2010 100644 --- a/python/semantic_kernel/connectors/ai/complete_request_settings.py +++ b/python/semantic_kernel/connectors/ai/complete_request_settings.py @@ -20,6 +20,7 @@ class CompleteRequestSettings: number_of_responses: int = 1 logprobs: int = 0 token_selection_biases: Dict[int, int] = field(default_factory=dict) + chat_system_prompt: str = "Assistant is a large language model." def update_from_completion_config( self, completion_config: "PromptTemplateConfig.CompletionConfig" @@ -31,6 +32,10 @@ def update_from_completion_config( self.max_tokens = completion_config.max_tokens self.stop_sequences = completion_config.stop_sequences self.number_of_responses = completion_config.number_of_responses + self.token_selection_biases = completion_config.token_selection_biases + + if completion_config.chat_system_prompt: + self.chat_system_prompt = completion_config.chat_system_prompt @staticmethod def from_completion_config( diff --git a/python/semantic_kernel/connectors/ai/google_palm/__init__.py b/python/semantic_kernel/connectors/ai/google_palm/__init__.py new file mode 100644 index 000000000000..249ff10e1e34 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/google_palm/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.connectors.ai.google_palm.services.gp_chat_completion import ( + GooglePalmChatCompletion, +) +from semantic_kernel.connectors.ai.google_palm.services.gp_text_completion import ( + GooglePalmTextCompletion, +) +from semantic_kernel.connectors.ai.google_palm.services.gp_text_embedding import ( + GooglePalmTextEmbedding, +) + +__all__ = [ + "GooglePalmTextCompletion", + "GooglePalmChatCompletion", + "GooglePalmTextEmbedding", +] diff --git a/python/semantic_kernel/connectors/ai/google_palm/services/gp_chat_completion.py b/python/semantic_kernel/connectors/ai/google_palm/services/gp_chat_completion.py new file mode 100644 index 000000000000..6286687fdbe8 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/google_palm/services/gp_chat_completion.py @@ -0,0 +1,224 @@ +# Copyright (c) Microsoft. All rights reserved. + +from logging import Logger +from typing import List, Optional, Tuple, Union + +import google.generativeai as palm +from google.generativeai.types import ChatResponse, ExampleOptions, MessagePromptOptions + +from semantic_kernel.connectors.ai.ai_exception import AIException +from semantic_kernel.connectors.ai.chat_completion_client_base import ( + ChatCompletionClientBase, +) +from semantic_kernel.connectors.ai.chat_request_settings import ChatRequestSettings +from semantic_kernel.connectors.ai.complete_request_settings import ( + CompleteRequestSettings, +) +from semantic_kernel.connectors.ai.text_completion_client_base import ( + TextCompletionClientBase, +) + + +class GooglePalmChatCompletion(ChatCompletionClientBase, TextCompletionClientBase): + _model_id: str + _api_key: str + _message_history: ChatResponse + + def __init__( + self, + model_id: str, + api_key: str, + ) -> None: + """ + Initializes a new instance of the GooglePalmChatCompletion class. + + Arguments: + model_id {str} -- GooglePalm model name, see + https://developers.generativeai.google/models/language + api_key {str} -- GooglePalm API key, see + https://developers.generativeai.google/products/palm + """ + if not api_key: + raise ValueError("The Google PaLM API key cannot be `None` or empty`") + + self._model_id = model_id + self._api_key = api_key + self._message_history = None + + async def complete_chat_async( + self, + messages: List[Tuple[str, str]], + request_settings: ChatRequestSettings, + context: Optional[str] = None, + examples: Optional[ExampleOptions] = None, + prompt: Optional[MessagePromptOptions] = None, + ) -> Union[str, List[str]]: + response = await self._send_chat_request( + messages, request_settings, context, examples, prompt + ) + + if request_settings.number_of_responses > 1: + return [ + candidate["output"] + if candidate["output"] is not None + else "I don't know." + for candidate in response.candidates + ] + else: + if response.last is None: + return "I don't know." # PaLM returns None if it doesn't know + else: + return response.last + + async def complete_chat_stream_async( + self, + messages: List[Tuple[str, str]], + request_settings: ChatRequestSettings, + context: Optional[str] = None, + ): + raise NotImplementedError( + "Google Palm API does not currently support streaming" + ) + + async def complete_async( + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, + ) -> Union[str, List[str]]: + prompt_to_message = [("user", prompt)] + chat_settings = ChatRequestSettings( + temperature=request_settings.temperature, + top_p=request_settings.top_p, + presence_penalty=request_settings.presence_penalty, + frequency_penalty=request_settings.frequency_penalty, + max_tokens=request_settings.max_tokens, + number_of_responses=request_settings.number_of_responses, + token_selection_biases=request_settings.token_selection_biases, + ) + response = await self._send_chat_request(prompt_to_message, chat_settings) + + if chat_settings.number_of_responses > 1: + return [ + candidate["output"] + if candidate["output"] is not None + else "I don't know." + for candidate in response.candidates + ] + else: + if response.last is None: + return "I don't know." # PaLM returns None if it doesn't know + else: + return response.last + + async def complete_stream_async( + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, + ): + raise NotImplementedError( + "Google Palm API does not currently support streaming" + ) + + async def _send_chat_request( + self, + messages: List[Tuple[str, str]], + request_settings: ChatRequestSettings, + context: Optional[str] = None, + examples: Optional[ExampleOptions] = None, + prompt: Optional[MessagePromptOptions] = None, + ): + """ + Completes the given user message. If len(messages) > 1, and a + conversation has not been initiated yet, it is assumed that chat history + is needed for context. All messages preceding the last message will be + utilized for context. This also enables Google PaLM to utilize memory + and skills, which should be stored in the messages parameter as system + messages. + + Arguments: + messages {str} -- The message (from a user) to respond to. + request_settings {ChatRequestSettings} -- The request settings. + context {str} -- Text that should be provided to the model first, + to ground the response. If a system message is provided, it will be + used as context. + examples {ExamplesOptions} -- Examples of what the model should + generate. This includes both the user input and the response that + the model should emulate. These examples are treated identically to + conversation messages except that they take precedence over the + history in messages: If the total input size exceeds the model's + input_token_limit the input will be truncated. Items will be dropped + from messages before examples + See: https://developers.generativeai.google/api/python/google/generativeai/types/ExampleOptions + prompt {MessagePromptOptions} -- You may pass a + types.MessagePromptOptions instead of a setting context/examples/messages, + but not both. + See: https://developers.generativeai.google/api/python/google/generativeai/types/MessagePromptOptions + + Returns: + str -- The completed text. + """ + if request_settings is None: + raise ValueError("The request settings cannot be `None`") + + if request_settings.max_tokens < 1: + raise AIException( + AIException.ErrorCodes.InvalidRequest, + "The max tokens must be greater than 0, " + f"but was {request_settings.max_tokens}", + ) + + if len(messages) <= 0: + raise AIException( + AIException.ErrorCodes.InvalidRequest, + "To complete a chat you need at least one message", + ) + + if messages[-1][0] != "user": + raise AIException( + AIException.ErrorCodes.InvalidRequest, + "The last message must be from the user", + ) + try: + palm.configure(api_key=self._api_key) + except Exception as ex: + raise PermissionError( + "Google PaLM service failed to configure. Invalid API key provided.", + ex, + ) + if ( + self._message_history is None and context is None + ): # If the conversation hasn't started yet and no context is provided + context = "" + if len(messages) > 1: # Check if we need context from messages + for index, (role, message) in enumerate(messages): + if index < len(messages) - 1: + if role == "system": + context += message + "\n" + else: + context += role + ": " + message + "\n" + try: + if self._message_history is None: + response = palm.chat( # Start a new conversation + model=self._model_id, + context=context, + examples=examples, + temperature=request_settings.temperature, + candidate_count=request_settings.number_of_responses, + top_p=request_settings.top_p, + prompt=prompt, + messages=messages[-1][1], + ) + else: + response = self._message_history.reply( # Continue the conversation + messages[-1][1], + ) + self._message_history = response # Store response object for future use + except Exception as ex: + raise AIException( + AIException.ErrorCodes.ServiceError, + "Google PaLM service failed to complete the prompt", + ex, + ) + return response diff --git a/python/semantic_kernel/connectors/ai/google_palm/services/gp_text_completion.py b/python/semantic_kernel/connectors/ai/google_palm/services/gp_text_completion.py new file mode 100644 index 000000000000..09ca400a28ab --- /dev/null +++ b/python/semantic_kernel/connectors/ai/google_palm/services/gp_text_completion.py @@ -0,0 +1,112 @@ +# Copyright (c) Microsoft. All rights reserved. + +from logging import Logger +from typing import List, Optional, Union + +import google.generativeai as palm + +from semantic_kernel.connectors.ai.ai_exception import AIException +from semantic_kernel.connectors.ai.complete_request_settings import ( + CompleteRequestSettings, +) +from semantic_kernel.connectors.ai.text_completion_client_base import ( + TextCompletionClientBase, +) + + +class GooglePalmTextCompletion(TextCompletionClientBase): + _model_id: str + _api_key: str + + def __init__(self, model_id: str, api_key: str) -> None: + """ + Initializes a new instance of the GooglePalmTextCompletion class. + + Arguments: + model_id {str} -- GooglePalm model name, see + https://developers.generativeai.google/models/language + api_key {str} -- GooglePalm API key, see + https://developers.generativeai.google/products/palm + """ + if not api_key: + raise ValueError("The Google PaLM API key cannot be `None` or empty`") + + self._model_id = model_id + self._api_key = api_key + + async def complete_async( + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, + ) -> Union[str, List[str]]: + response = await self._send_completion_request(prompt, request_settings) + + if request_settings.number_of_responses > 1: + return [candidate["output"] for candidate in response.candidates] + else: + return response.result + + async def complete_stream_async( + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, + ): + raise NotImplementedError( + "Google Palm API does not currently support streaming" + ) + + async def _send_completion_request( + self, prompt: str, request_settings: CompleteRequestSettings + ): + """ + Completes the given prompt. Returns a single string completion. + Cannot return multiple completions. Cannot return logprobs. + + Arguments: + prompt {str} -- The prompt to complete. + request_settings {CompleteRequestSettings} -- The request settings. + + Returns: + str -- The completed text. + """ + if not prompt: + raise ValueError("Prompt cannot be `None` or empty") + if request_settings is None: + raise ValueError("Request settings cannot be `None`") + if request_settings.max_tokens < 1: + raise AIException( + AIException.ErrorCodes.InvalidRequest, + "The max tokens must be greater than 0, " + f"but was {request_settings.max_tokens}", + ) + try: + palm.configure(api_key=self._api_key) + except Exception as ex: + raise PermissionError( + "Google PaLM service failed to configure. Invalid API key provided.", + ex, + ) + try: + response = palm.generate_text( + model=self._model_id, + prompt=prompt, + temperature=request_settings.temperature, + max_output_tokens=request_settings.max_tokens, + stop_sequences=( + request_settings.stop_sequences + if request_settings.stop_sequences is not None + and len(request_settings.stop_sequences) > 0 + else None + ), + candidate_count=request_settings.number_of_responses, + top_p=request_settings.top_p, + ) + except Exception as ex: + raise AIException( + AIException.ErrorCodes.ServiceError, + "Google PaLM service failed to complete the prompt", + ex, + ) + return response diff --git a/python/semantic_kernel/connectors/ai/google_palm/services/gp_text_embedding.py b/python/semantic_kernel/connectors/ai/google_palm/services/gp_text_embedding.py new file mode 100644 index 000000000000..aa1f19a7cf52 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/google_palm/services/gp_text_embedding.py @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft. All rights reserved. + + +from typing import List + +import google.generativeai as palm +from numpy import array, ndarray + +from semantic_kernel.connectors.ai.ai_exception import AIException +from semantic_kernel.connectors.ai.embeddings.embedding_generator_base import ( + EmbeddingGeneratorBase, +) + + +class GooglePalmTextEmbedding(EmbeddingGeneratorBase): + _model_id: str + _api_key: str + + def __init__(self, model_id: str, api_key: str) -> None: + """ + Initializes a new instance of the GooglePalmTextEmbedding class. + + Arguments: + model_id {str} -- GooglePalm model name, see + https://developers.generativeai.google/models/language + api_key {str} -- GooglePalm API key, see + https://developers.generativeai.google/products/palm + """ + if not api_key: + raise ValueError("The Google PaLM API key cannot be `None` or empty`") + + self._model_id = model_id + self._api_key = api_key + + async def generate_embeddings_async(self, texts: List[str]) -> ndarray: + """ + Generates embeddings for a list of texts. + + Arguments: + texts {List[str]} -- Texts to generate embeddings for. + + Returns: + ndarray -- Embeddings for the texts. + """ + try: + palm.configure(api_key=self._api_key) + except Exception as ex: + raise PermissionError( + "Google PaLM service failed to configure. Invalid API key provided.", + ex, + ) + embeddings = [] + for text in texts: + try: + response = palm.generate_embeddings( + model=self._model_id, + text=text, + ) + embeddings.append(array(response["embedding"])) + except Exception as ex: + raise AIException( + AIException.ErrorCodes.ServiceError, + "Google PaLM service failed to generate the embedding.", + ex, + ) + return array(embeddings) diff --git a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py index bc94820c24a1..8ce42cfb9565 100644 --- a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py +++ b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py @@ -2,7 +2,7 @@ from logging import Logger from threading import Thread -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union from semantic_kernel.connectors.ai.ai_exception import AIException from semantic_kernel.connectors.ai.complete_request_settings import ( @@ -23,9 +23,11 @@ class HuggingFaceTextCompletion(TextCompletionClientBase): def __init__( self, model_id: str, - device: Optional[int] = -1, + device: Optional[int] = None, task: Optional[str] = None, log: Optional[Logger] = None, + model_kwargs: Dict[str, Any] = None, + pipeline_kwargs: Dict[str, Any] = {}, ) -> None: """ Initializes a new instance of the HuggingFaceTextCompletion class. @@ -33,19 +35,29 @@ def __init__( Arguments: model_id {str} -- Hugging Face model card string, see https://huggingface.co/models - device {Optional[int]} -- Device to run the model on, -1 for CPU, 0+ for GPU. + device {Optional[int]} -- Device to run the model on, defaults to CPU, 0+ for GPU, + -- None if using device_map instead. (If both device and device_map + are specified, device overrides device_map. If unintended, + it can lead to unexpected behavior.) task {Optional[str]} -- Model completion task type, options are: - summarization: takes a long text and returns a shorter summary. - text-generation: takes incomplete text and returns a set of completion candidates. - text2text-generation (default): takes an input prompt and returns a completion. text2text-generation is the default as it behaves more like GPT-3+. log {Optional[Logger]} -- Logger instance. + model_kwargs {Optional[Dict[str, Any]]} -- Additional dictionary of keyword arguments + passed along to the model's `from_pretrained(..., **model_kwargs)` function. + pipeline_kwargs {Optional[Dict[str, Any]]} -- Additional keyword arguments passed along + to the specific pipeline init (see the documentation for the corresponding pipeline class + for possible values). Note that this model will be downloaded from the Hugging Face model hub. """ self._model_id = model_id self._task = "text2text-generation" if task is None else task self._log = log if log is not None else NullLogger() + self._model_kwargs = model_kwargs + self._pipeline_kwargs = pipeline_kwargs try: import torch @@ -55,15 +67,29 @@ def __init__( "Please ensure that torch and transformers are installed to use HuggingFaceTextCompletion" ) - self.device = ( - "cuda:" + device if device >= 0 and torch.cuda.is_available() else "cpu" - ) + device_map = self._pipeline_kwargs.get("device_map", None) + if device is None: + self.device = "cpu" if device_map is None else None + else: + self.device = ( + "cuda:" + str(device) + if device >= 0 and torch.cuda.is_available() + else "cpu" + ) + self.generator = transformers.pipeline( - task=self._task, model=self._model_id, device=self.device + task=self._task, + model=self._model_id, + device=self.device, + model_kwargs=self._model_kwargs, + **self._pipeline_kwargs ) async def complete_async( - self, prompt: str, request_settings: CompleteRequestSettings + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, ) -> Union[str, List[str]]: try: import transformers @@ -110,7 +136,10 @@ async def complete_async( raise AIException("Hugging Face completion failed", e) async def complete_stream_async( - self, prompt: str, request_settings: CompleteRequestSettings + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, ): """ Streams a text completion using a Hugging Face model. diff --git a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py index 137ec90f33f0..97fae9cace4c 100644 --- a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py +++ b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py @@ -46,7 +46,9 @@ def __init__( ) self.device = ( - "cuda:" + device if device >= 0 and torch.cuda.is_available() else "cpu" + "cuda:" + str(device) + if device >= 0 and torch.cuda.is_available() + else "cpu" ) self.generator = sentence_transformers.SentenceTransformer( model_name_or_path=self._model_id, device=self.device diff --git a/python/semantic_kernel/connectors/ai/open_ai/models/chat/function_call.py b/python/semantic_kernel/connectors/ai/open_ai/models/chat/function_call.py new file mode 100644 index 000000000000..6a645ba89e51 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/models/chat/function_call.py @@ -0,0 +1,36 @@ +"""Class to hold chat messages.""" +import json +from typing import Dict, Tuple + +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.sk_pydantic import SKBaseModel + + +class FunctionCall(SKBaseModel): + """Class to hold a function call response.""" + + name: str + arguments: str + + def parse_arguments(self) -> Dict[str, str]: + """Parse the arguments into a dictionary.""" + try: + return json.loads(self.arguments) + except json.JSONDecodeError: + return None + + def to_context_variables(self) -> ContextVariables: + """Return the arguments as a ContextVariables instance.""" + args = self.parse_arguments() + return ContextVariables(variables={k.lower(): v for k, v in args.items()}) + + def split_name(self) -> Tuple[str, str]: + """Split the name into a skill and function name.""" + if "-" not in self.name: + return None, self.name + return self.name.split("-") + + def split_name_dict(self) -> dict: + """Split the name into a skill and function name.""" + parts = self.split_name() + return {"skill_name": parts[0], "function_name": parts[1]} diff --git a/python/semantic_kernel/connectors/ai/open_ai/models/chat/open_ai_chat_message.py b/python/semantic_kernel/connectors/ai/open_ai/models/chat/open_ai_chat_message.py new file mode 100644 index 000000000000..4e0d90c2088c --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/models/chat/open_ai_chat_message.py @@ -0,0 +1,14 @@ +"""Class to hold chat messages.""" +from typing import Optional + +from semantic_kernel.connectors.ai.open_ai.models.chat.function_call import ( + FunctionCall, +) +from semantic_kernel.models.chat.chat_message import ChatMessage + + +class OpenAIChatMessage(ChatMessage): + """Class to hold openai chat messages, which might include name and function_call fields.""" + + name: Optional[str] = None + function_call: Optional[FunctionCall] = None diff --git a/python/semantic_kernel/connectors/ai/open_ai/semantic_functions/open_ai_chat_prompt_template.py b/python/semantic_kernel/connectors/ai/open_ai/semantic_functions/open_ai_chat_prompt_template.py new file mode 100644 index 000000000000..6b4b4d234f07 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/semantic_functions/open_ai_chat_prompt_template.py @@ -0,0 +1,103 @@ +# Copyright (c) Microsoft. All rights reserved. + +from logging import Logger +from typing import Any, Dict, List, Optional + +from semantic_kernel.connectors.ai.open_ai.models.chat.function_call import FunctionCall +from semantic_kernel.connectors.ai.open_ai.models.chat.open_ai_chat_message import ( + OpenAIChatMessage, +) +from semantic_kernel.semantic_functions.chat_prompt_template import ChatPromptTemplate +from semantic_kernel.semantic_functions.prompt_template import PromptTemplate +from semantic_kernel.semantic_functions.prompt_template_config import ( + PromptTemplateConfig, +) +from semantic_kernel.template_engine.protocols.prompt_templating_engine import ( + PromptTemplatingEngine, +) + + +class OpenAIChatPromptTemplate(ChatPromptTemplate): + def add_function_response_message(self, name: str, content: Any) -> None: + """Add a function response message to the chat template.""" + self._messages.append( + OpenAIChatMessage(role="function", name=name, fixed_content=str(content)) + ) + + def add_message( + self, role: str, message: Optional[str] = None, **kwargs: Any + ) -> None: + """Add a message to the chat template. + + Arguments: + role: The role of the message, one of "user", "assistant", "system", "function" + message: The message to add, can include templating components. + kwargs: can be used by inherited classes. + name: the name of the function that was used, to be used with role: function + function_call: the function call that is specified, to be used with role: assistant + """ + name = kwargs.get("name") + if name is not None and role != "function": + self._log.warning("name is only used with role: function, ignoring") + name = None + function_call = kwargs.get("function_call") + if function_call is not None and role != "assistant": + self._log.warning( + "function_call is only used with role: assistant, ignoring" + ) + function_call = None + if function_call and not isinstance(function_call, FunctionCall): + self._log.warning( + "function_call is not a FunctionCall, ignoring: %s", function_call + ) + function_call = None + self._messages.append( + OpenAIChatMessage( + role=role, + content_template=PromptTemplate( + message, self._template_engine, self._prompt_config + ), + name=name, + function_call=function_call, + ) + ) + + @classmethod + def restore( + cls, + messages: List[Dict[str, str]], + template: str, + template_engine: PromptTemplatingEngine, + prompt_config: PromptTemplateConfig, + log: Optional[Logger] = None, + ) -> "OpenAIChatPromptTemplate": + """Restore a ChatPromptTemplate from a list of role and message pairs. + + If there is a chat_system_prompt in the prompt_config.completion settings, + that takes precedence over the first message in the list of messages, + if that is a system message. + """ + chat_template = cls(template, template_engine, prompt_config, log) + if ( + prompt_config.completion.chat_system_prompt + and messages[0]["role"] == "system" + ): + existing_system_message = messages.pop(0) + if ( + existing_system_message["message"] + != prompt_config.completion.chat_system_prompt + ): + chat_template._log.info( + "Overriding system prompt with chat_system_prompt, old system message: %s, new system message: %s", + existing_system_message["message"], + prompt_config.completion.chat_system_prompt, + ) + for message in messages: + chat_template.add_message( + message["role"], + message["message"], + name=message["name"], + function_call=message["function_call"], + ) + + return chat_template diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py index 16b1dc2b339c..24adf4896c61 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py @@ -1,10 +1,15 @@ # Copyright (c) Microsoft. All rights reserved. from logging import Logger -from typing import Any, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import openai +from semantic_kernel.connectors.ai.open_ai.models.chat.function_call import FunctionCall + +if TYPE_CHECKING: + from openai.openai_object import OpenAIObject + from semantic_kernel.connectors.ai.ai_exception import AIException from semantic_kernel.connectors.ai.chat_completion_client_base import ( ChatCompletionClientBase, @@ -27,6 +32,9 @@ class OpenAIChatCompletion(ChatCompletionClientBase, TextCompletionClientBase): _api_version: Optional[str] = None _endpoint: Optional[str] = None _log: Logger + _prompt_tokens: int = 0 + _completion_tokens: int = 0 + _total_tokens: int = 0 def __init__( self, @@ -55,25 +63,55 @@ def __init__( self._org_id = org_id self._api_type = api_type self._api_version = api_version - self._endpoint = endpoint + self._endpoint = endpoint.rstrip("/") if endpoint is not None else None self._log = log if log is not None else NullLogger() self._messages = [] async def complete_chat_async( - self, messages: List[Tuple[str, str]], request_settings: ChatRequestSettings + self, + messages: List[Dict[str, str]], + request_settings: ChatRequestSettings, + logger: Optional[Logger] = None, ) -> Union[str, List[str]]: # TODO: tracking on token counts/etc. - response = await self._send_chat_request(messages, request_settings, False) + response = await self._send_chat_request( + messages, request_settings, False, None + ) if len(response.choices) == 1: return response.choices[0].message.content + return [choice.message.content for choice in response.choices] + + async def complete_chat_with_functions_async( + self, + messages: List[Dict[str, str]], + functions: List[Dict[str, Any]], + request_settings: ChatRequestSettings, + logger: Optional[Logger] = None, + ) -> Union[ + Tuple[Optional[str], Optional[FunctionCall]], + List[Tuple[Optional[str], Optional[FunctionCall]]], + ]: + # TODO: tracking on token counts/etc. + + response = await self._send_chat_request( + messages, request_settings, False, functions + ) + + if len(response.choices) == 1: + return _parse_message(response.choices[0].message, self._log) else: - return [choice.message.content for choice in response.choices] + return [ + _parse_message(choice.message, self._log) for choice in response.choices + ] async def complete_chat_stream_async( - self, messages: List[Tuple[str, str]], request_settings: ChatRequestSettings + self, + messages: List[Dict[str, str]], + request_settings: ChatRequestSettings, ): - response = await self._send_chat_request(messages, request_settings, True) + # TODO: enable function calling + response = await self._send_chat_request(messages, request_settings, True, None) # parse the completion text(s) and yield them async for chunk in response: @@ -88,7 +126,10 @@ async def complete_chat_stream_async( yield text async def complete_async( - self, prompt: str, request_settings: CompleteRequestSettings + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, ) -> Union[str, List[str]]: """ Completes the given prompt. @@ -100,16 +141,8 @@ async def complete_async( Returns: str -- The completed text. """ - prompt_to_message = [("user", prompt)] - chat_settings = ChatRequestSettings( - temperature=request_settings.temperature, - top_p=request_settings.top_p, - presence_penalty=request_settings.presence_penalty, - frequency_penalty=request_settings.frequency_penalty, - max_tokens=request_settings.max_tokens, - number_of_responses=request_settings.number_of_responses, - token_selection_biases=request_settings.token_selection_biases, - ) + prompt_to_message = [{"role": "user", "content": prompt}] + chat_settings = ChatRequestSettings.from_completion_config(request_settings) response = await self._send_chat_request( prompt_to_message, chat_settings, False ) @@ -120,9 +153,12 @@ async def complete_async( return [choice.message.content for choice in response.choices] async def complete_stream_async( - self, prompt: str, request_settings: CompleteRequestSettings + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, ): - prompt_to_message = [("user", prompt)] + prompt_to_message = [{"role": "user", "content": prompt}] chat_settings = ChatRequestSettings( temperature=request_settings.temperature, top_p=request_settings.top_p, @@ -131,6 +167,7 @@ async def complete_stream_async( max_tokens=request_settings.max_tokens, number_of_responses=request_settings.number_of_responses, token_selection_biases=request_settings.token_selection_biases, + stop_sequences=request_settings.stop_sequences, ) response = await self._send_chat_request(prompt_to_message, chat_settings, True) @@ -151,13 +188,16 @@ async def _send_chat_request( messages: List[Tuple[str, str]], request_settings: ChatRequestSettings, stream: bool, + functions: Optional[List[Dict[str, Any]]] = None, ): """ Completes the given user message with an asynchronous stream. Arguments: - user_message {str} -- The message (from a user) to respond to. + messages {List[Tuple[str,str]]} -- The messages (from a user) to respond to. request_settings {ChatRequestSettings} -- The request settings. + stream {bool} -- Whether to stream the response. + functions {List[Dict[str, Any]]} -- The functions available to the api. Returns: str -- The completed text. @@ -178,56 +218,84 @@ async def _send_chat_request( "To complete a chat you need at least one message", ) - if messages[-1][0] != "user": + if messages[-1]["role"] in ["assistant", "system"]: raise AIException( AIException.ErrorCodes.InvalidRequest, - "The last message must be from the user", + "The last message must be from the user or a function output", ) - model_args = {} - if self._api_type in ["azure", "azure_ad"]: - model_args["engine"] = self._model_id - else: - model_args["model"] = self._model_id + model_args = { + "api_key": self._api_key, + "api_type": self._api_type, + "api_base": self._endpoint, + "api_version": self._api_version, + "organization": self._org_id, + "engine" + if self._api_type in ["azure", "azure_ad"] + else "model": self._model_id, + "messages": messages, + "temperature": request_settings.temperature, + "top_p": request_settings.top_p, + "n": request_settings.number_of_responses, + "stream": stream, + "stop": ( + request_settings.stop_sequences + if request_settings.stop_sequences is not None + and len(request_settings.stop_sequences) > 0 + else None + ), + "max_tokens": request_settings.max_tokens, + "presence_penalty": request_settings.presence_penalty, + "frequency_penalty": request_settings.frequency_penalty, + "logit_bias": ( + request_settings.token_selection_biases + if request_settings.token_selection_biases is not None + and len(request_settings.token_selection_biases) > 0 + else {} + ), + } - formatted_messages = [ - {"role": role, "content": message} for role, message in messages - ] + if functions and request_settings.function_call is not None: + model_args["function_call"] = request_settings.function_call + if request_settings.function_call != "auto": + model_args["functions"] = [ + func + for func in functions + if func["name"] == request_settings.function_call + ] + else: + model_args["functions"] = functions try: - response: Any = await openai.ChatCompletion.acreate( - **model_args, - api_key=self._api_key, - api_type=self._api_type, - api_base=self._endpoint, - api_version=self._api_version, - organization=self._org_id, - messages=formatted_messages, - temperature=request_settings.temperature, - top_p=request_settings.top_p, - presence_penalty=request_settings.presence_penalty, - frequency_penalty=request_settings.frequency_penalty, - max_tokens=request_settings.max_tokens, - n=request_settings.number_of_responses, - stream=stream, - logit_bias=( - request_settings.token_selection_biases - if request_settings.token_selection_biases is not None - and len(request_settings.token_selection_biases) > 0 - else {} - ), - ) + response: Any = await openai.ChatCompletion.acreate(**model_args) except Exception as ex: raise AIException( AIException.ErrorCodes.ServiceError, - "OpenAI service failed to complete the chat", + f"{self.__class__.__name__} failed to complete the chat", ex, - ) + ) from ex - # TODO: tracking on token counts/etc. + # streaming does not have usage info, therefore checking the type of the response + if not stream and "usage" in response: + self._log.info(f"OpenAI usage: {response.usage}") + self._prompt_tokens += response.usage.prompt_tokens + self._completion_tokens += response.usage.completion_tokens + self._total_tokens += response.usage.total_tokens return response + @property + def prompt_tokens(self) -> int: + return self._prompt_tokens + + @property + def completion_tokens(self) -> int: + return self._completion_tokens + + @property + def total_tokens(self) -> int: + return self._total_tokens + def _parse_choices(chunk): message = "" @@ -235,6 +303,31 @@ def _parse_choices(chunk): message += chunk.choices[0].delta.role + ": " if "content" in chunk.choices[0].delta: message += chunk.choices[0].delta.content + if "function_call" in chunk.choices[0].delta: + message += chunk.choices[0].delta.function_call index = chunk.choices[0].index return message, index + + +def _parse_message( + message: "OpenAIObject", logger: Optional[Logger] = None +) -> Tuple[Optional[str], Optional[FunctionCall]]: + """ + Parses the message. + + Arguments: + message {OpenAIObject} -- The message to parse. + + Returns: + Tuple[Optional[str], Optional[Dict]] -- The parsed message. + """ + content = message.content if hasattr(message, "content") else None + function_call = message.function_call if hasattr(message, "function_call") else None + if function_call: + function_call = FunctionCall( + name=function_call.name, + arguments=function_call.arguments, + ) + + return (content, function_call) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py index 81e64e51ed61..cd7ccc5c49d6 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py @@ -23,6 +23,9 @@ class OpenAITextCompletion(TextCompletionClientBase): _endpoint: Optional[str] = None _org_id: Optional[str] = None _log: Logger + _prompt_tokens: int + _completion_tokens: int + _total_tokens: int def __init__( self, @@ -50,12 +53,15 @@ def __init__( self._api_key = api_key self._api_type = api_type self._api_version = api_version - self._endpoint = endpoint + self._endpoint = endpoint.rstrip("/") if endpoint is not None else None self._org_id = org_id self._log = log if log is not None else NullLogger() async def complete_async( - self, prompt: str, request_settings: CompleteRequestSettings + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, ) -> Union[str, List[str]]: # TODO: tracking on token counts/etc. response = await self._send_completion_request(prompt, request_settings, False) @@ -68,7 +74,10 @@ async def complete_async( # TODO: complete w/ multiple... async def complete_stream_async( - self, prompt: str, request_settings: CompleteRequestSettings + self, + prompt: str, + request_settings: CompleteRequestSettings, + logger: Optional[Logger] = None, ): response = await self._send_completion_request(prompt, request_settings, True) @@ -152,7 +161,26 @@ async def _send_completion_request( except Exception as ex: raise AIException( AIException.ErrorCodes.ServiceError, - "OpenAI service failed to complete the prompt", + f"{self.__class__.__name__} failed to complete the prompt", ex, ) + + if "usage" in response: + self._log.info(f"OpenAI usage: {response.usage}") + self._prompt_tokens += response.usage.prompt_tokens + self._completion_tokens += response.usage.completion_tokens + self._total_tokens += response.usage.total_tokens + return response + + @property + def prompt_tokens(self) -> int: + return self._prompt_tokens + + @property + def completion_tokens(self) -> int: + return self._completion_tokens + + @property + def total_tokens(self) -> int: + return self._total_tokens diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py index 8a9c6f9bd12a..04fca023c96e 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py @@ -48,11 +48,13 @@ def __init__( self._api_key = api_key self._api_type = api_type self._api_version = api_version - self._endpoint = endpoint + self._endpoint = endpoint.rstrip("/") if endpoint is not None else None self._org_id = org_id self._log = log if log is not None else NullLogger() - async def generate_embeddings_async(self, texts: List[str]) -> ndarray: + async def generate_embeddings_async( + self, texts: List[str], batch_size: Optional[int] = None + ) -> ndarray: model_args = {} if self._api_type in ["azure", "azure_ad"]: model_args["engine"] = self._model_id @@ -60,22 +62,25 @@ async def generate_embeddings_async(self, texts: List[str]) -> ndarray: model_args["model"] = self._model_id try: - response: Any = await openai.Embedding.acreate( - **model_args, - api_key=self._api_key, - api_type=self._api_type, - api_base=self._endpoint, - api_version=self._api_version, - organization=self._org_id, - input=texts, - ) - - # make numpy arrays from the response - raw_embeddings = [array(x["embedding"]) for x in response["data"]] + raw_embeddings = [] + batch_size = batch_size or len(texts) + for i in range(0, len(texts), batch_size): + batch = texts[i : i + batch_size] + response: Any = await openai.Embedding.acreate( + **model_args, + api_key=self._api_key, + api_type=self._api_type, + api_base=self._endpoint, + api_version=self._api_version, + organization=self._org_id, + input=batch, + ) + # make numpy arrays from the response + raw_embeddings.extend([array(x["embedding"]) for x in response["data"]]) return array(raw_embeddings) except Exception as ex: raise AIException( AIException.ErrorCodes.ServiceError, - "OpenAI service failed to generate embeddings", + f"{self.__class__.__name__} failed to generate embeddings", ex, ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/utils.py b/python/semantic_kernel/connectors/ai/open_ai/utils.py new file mode 100644 index 000000000000..9b6da59fc6e6 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/utils.py @@ -0,0 +1,184 @@ +from logging import Logger +from typing import Any, Dict, List, Optional + +from semantic_kernel import Kernel, SKContext +from semantic_kernel.connectors.ai.open_ai.models.chat.function_call import FunctionCall +from semantic_kernel.connectors.ai.open_ai.semantic_functions.open_ai_chat_prompt_template import ( + OpenAIChatPromptTemplate, +) +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase + + +def _describe_function(function: SKFunctionBase) -> Dict[str, str]: + """Create the object used for function_calling. + + Assumes that arguments for semantic functions are optional, for native functions required. + """ + func_view = function.describe() + return { + "name": f"{func_view.skill_name}-{func_view.name}", + "description": func_view.description, + "parameters": { + "type": "object", + "properties": { + param.name: {"description": param.description, "type": param.type_} + for param in func_view.parameters + }, + "required": [p.name for p in func_view.parameters if p.required], + }, + } + + +def get_function_calling_object( + kernel: Kernel, filter: Dict[str, List[str]] +) -> List[Dict[str, str]]: + """Create the object used for function_calling. + + args: + kernel: the kernel. + filter: a dictionary with keys + exclude_skill, include_skill, exclude_function, include_function + and lists of the required filter. + The function name should be in the format "skill_name-function_name". + Using exclude_skill and include_skill at the same time will raise an error. + Using exclude_function and include_function at the same time will raise an error. + If using include_* implies that all other function will be excluded. + Example: + filter = { + "exclude_skill": ["skill1", "skill2"], + "include_function": ["skill3-function1", "skill4-function2"], + } + will return only skill3-function1 and skill4-function2. + filter = { + "exclude_function": ["skill1-function1", "skill2-function2"], + } + will return all functions except skill1-function1 and skill2-function2. + caller_function_name: the name of the function that is calling the other functions. + returns: + a filtered list of dictionaries of the functions in the kernel that can be passed to the function calling api. + """ + include_skill = filter.get("include_skill", None) + exclude_skill = filter.get("exclude_skill", []) + include_function = filter.get("include_function", None) + exclude_function = filter.get("exclude_function", []) + if include_skill and exclude_skill: + raise ValueError( + "Cannot use both include_skill and exclude_skill at the same time." + ) + if include_function and exclude_function: + raise ValueError( + "Cannot use both include_function and exclude_function at the same time." + ) + if include_skill: + include_skill = [skill.lower() for skill in include_skill] + if exclude_skill: + exclude_skill = [skill.lower() for skill in exclude_skill] + if include_function: + include_function = [function.lower() for function in include_function] + if exclude_function: + exclude_function = [function.lower() for function in exclude_function] + result = [] + for ( + skill_name, + skill, + ) in kernel.skills.data.items(): + if skill_name in exclude_skill or ( + include_skill and skill_name not in include_skill + ): + continue + for function_name, function in skill.items(): + current_name = f"{skill_name}-{function_name}" + if current_name in exclude_function or ( + include_function and current_name not in include_function + ): + continue + result.append(_describe_function(function)) + return result + + +async def execute_function_call( + kernel: Kernel, function_call: FunctionCall, log: Optional[Logger] = None +) -> str: + result = await kernel.run_async( + kernel.func(**function_call.split_name_dict()), + input_vars=function_call.to_context_variables(), + ) + if log: + log.info(f"Function call result: {result}") + return str(result) + + +async def chat_completion_with_function_call( + kernel: Kernel, + context: SKContext, + functions: List[Dict[str, str]] = [], + chat_skill_name: Optional[str] = None, + chat_function_name: Optional[str] = None, + chat_function: Optional[SKFunctionBase] = None, + *, + log: Optional[Logger] = None, + **kwargs: Dict[str, Any], +) -> SKContext: + """Perform a chat completion with auto-executing function calling. + + This is a recursive function that will execute the chat function multiple times, + at least once to get a first completion, if a function_call is returned, + the function_call is executed (using the execute_function_call method), + the result is added to the chat prompt template and another completion is requested, + by calling the function again, if it returns a function_call, it is executed again, + until the maximum number of function calls is reached, + at that time a final completion is done without functions. + + args: + kernel: the kernel to use. + context: the context to use. + functions: the function calling object, + make sure to use get_function_calling_object method to create it. + Optional arguments: + chat_skill_name: the skill name of the chat function. + chat_function_name: the function name of the chat function. + chat_function: the chat function, if not provided, it will be retrieved from the kernel. + make sure to provide either the chat_function or the chat_skill_name and chat_function_name. + + log: the logger to use. + max_function_calls: the maximum number of function calls to execute, defaults to 5. + current_call_count: the current number of function calls executed. + + returns: + the context with the result of the chat completion, just like a regular invoke_async/run_async. + """ + # check the number of function calls + max_function_calls = kwargs.get("max_function_calls", 5) + current_call_count = kwargs.get("current_call_count", 0) + # get the chat function + if chat_function is None: + chat_function = kernel.func( + skill_name=chat_skill_name, function_name=chat_function_name + ) + assert isinstance( + chat_function._chat_prompt_template, OpenAIChatPromptTemplate + ), "Please make sure to initialize your chat function with the OpenAIChatPromptTemplate class." + context = await chat_function.invoke_async( + context=context, + # when the maximum number of function calls is reached, execute the chat function without Functions. + functions=[] if current_call_count >= max_function_calls else functions, + ) + function_call = context.objects.pop("function_call", None) + # if there is no function_call or if the content is not a FunctionCall object, return the context + if function_call is None or not isinstance(function_call, FunctionCall): + return context + result = await execute_function_call(kernel, function_call, log=log) + # add the result to the chat prompt template + chat_function._chat_prompt_template.add_function_response_message( + name=function_call.name, content=str(result) + ) + # request another completion + return await chat_completion_with_function_call( + kernel, + chat_function=chat_function, + functions=functions, + context=context, + log=log, + max_function_calls=max_function_calls, + current_call_count=current_call_count + 1, + ) diff --git a/python/semantic_kernel/connectors/ai/text_completion_client_base.py b/python/semantic_kernel/connectors/ai/text_completion_client_base.py index 81d81c767efe..c01653b9aabb 100644 --- a/python/semantic_kernel/connectors/ai/text_completion_client_base.py +++ b/python/semantic_kernel/connectors/ai/text_completion_client_base.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from logging import Logger -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, List, Optional, Union if TYPE_CHECKING: from semantic_kernel.connectors.ai.complete_request_settings import ( @@ -16,7 +16,7 @@ async def complete_async( self, prompt: str, settings: "CompleteRequestSettings", - logger: Logger, + logger: Optional[Logger] = None, ) -> Union[str, List[str]]: """ This is the method that is called from the kernel to get a response from a text-optimized LLM. @@ -36,7 +36,7 @@ async def complete_stream_async( self, prompt: str, settings: "CompleteRequestSettings", - logger: Logger, + logger: Optional[Logger] = None, ): """ This is the method that is called from the kernel to get a stream response from a text-optimized LLM. diff --git a/python/semantic_kernel/connectors/memory/__init__.py b/python/semantic_kernel/connectors/memory/__init__.py new file mode 100644 index 000000000000..2a50eae89411 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Microsoft. All rights reserved. diff --git a/python/semantic_kernel/connectors/memory/azure_cognitive_search/azure_cognitive_search_memory_store.py b/python/semantic_kernel/connectors/memory/azure_cognitive_search/azure_cognitive_search_memory_store.py index c1a907bca5ab..d5add2b89158 100644 --- a/python/semantic_kernel/connectors/memory/azure_cognitive_search/azure_cognitive_search_memory_store.py +++ b/python/semantic_kernel/connectors/memory/azure_cognitive_search/azure_cognitive_search_memory_store.py @@ -8,10 +8,11 @@ from azure.core.exceptions import ResourceNotFoundError from azure.search.documents.indexes.aio import SearchIndexClient from azure.search.documents.indexes.models import ( + HnswVectorSearchAlgorithmConfiguration, SearchIndex, VectorSearch, - VectorSearchAlgorithmConfiguration, ) +from azure.search.documents.models import Vector from numpy import ndarray from semantic_kernel.connectors.memory.azure_cognitive_search.utils import ( @@ -58,7 +59,6 @@ def __init__( Instantiate using Async Context Manager: async with AzureCognitiveSearchMemoryStore(<...>) as memory: await memory.<...> - """ try: pass @@ -82,14 +82,14 @@ async def close_async(self): async def create_collection_async( self, collection_name: str, - vector_config: Optional[VectorSearchAlgorithmConfiguration] = None, + vector_config: Optional[HnswVectorSearchAlgorithmConfiguration] = None, ) -> None: """Creates a new collection if it does not exist. Arguments: collection_name {str} -- The name of the collection to create. - vector_config {VectorSearchAlgorithmConfiguration} -- Optional search algorithm configuration - (default: {None}). + vector_config {HnswVectorSearchAlgorithmConfiguration} -- Optional search algorithm configuration + (default: {None}). semantic_config {SemanticConfiguration} -- Optional search index configuration (default: {None}). Returns: None @@ -100,7 +100,7 @@ async def create_collection_async( else: vector_search = VectorSearch( algorithm_configurations=[ - VectorSearchAlgorithmConfiguration( + HnswVectorSearchAlgorithmConfiguration( name="az-vector-config", kind="hnsw", hnsw_parameters={ @@ -403,12 +403,14 @@ async def get_nearest_matches_async( collection_name.lower() ) + vector = Vector( + value=embedding.flatten(), k=limit, fields=SEARCH_FIELD_EMBEDDING + ) + search_results = await search_client.search( search_text="*", - vector_fields=SEARCH_FIELD_EMBEDDING, - vector=embedding.tolist(), + vectors=[vector], select=get_field_selection(with_embeddings), - top_k=limit, ) if not search_results or search_results is None: diff --git a/python/semantic_kernel/connectors/memory/azure_cognitive_search/utils.py b/python/semantic_kernel/connectors/memory/azure_cognitive_search/utils.py index 5fe9d88dedba..b502ffbcc497 100644 --- a/python/semantic_kernel/connectors/memory/azure_cognitive_search/utils.py +++ b/python/semantic_kernel/connectors/memory/azure_cognitive_search/utils.py @@ -53,9 +53,6 @@ def get_search_index_async_client( "Please install Azure Cognitive Search client" ) - azure_credential: AzureKeyCredential = None - token_credential: TokenCredential = None - # Load environment variables load_dotenv() diff --git a/python/semantic_kernel/connectors/memory/chroma/chroma_memory_store.py b/python/semantic_kernel/connectors/memory/chroma/chroma_memory_store.py index 430ddaba1d8c..031a70d536c0 100644 --- a/python/semantic_kernel/connectors/memory/chroma/chroma_memory_store.py +++ b/python/semantic_kernel/connectors/memory/chroma/chroma_memory_store.py @@ -127,8 +127,6 @@ async def delete_collection_async(self, collection_name: str) -> None: """ # Current version of ChromeDB reject camel case collection names. self._client.delete_collection(name=camel_to_snake(collection_name)) - if self._persist_directory is not None: - self._client.persist() async def does_collection_exist_async(self, collection_name: str) -> bool: """Checks if a collection exists. @@ -175,9 +173,6 @@ async def upsert_async(self, collection_name: str, record: MemoryRecord) -> str: documents=record._text, ids=record._key, ) - - if self._persist_directory is not None: - self._client.persist() return record._key async def upsert_batch_async( diff --git a/python/semantic_kernel/connectors/memory/mongodb_atlas/README.md b/python/semantic_kernel/connectors/memory/mongodb_atlas/README.md new file mode 100644 index 000000000000..c933b5f3a245 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/mongodb_atlas/README.md @@ -0,0 +1,52 @@ +# microsoft.semantic_kernel.connectors.memory.mongodb_atlas + +This connector uses [MongoDB Atlas Vector Search](https://www.mongodb.com/products/platform/atlas-vector-search) to implement Semantic Memory. + +## Quick Start + +1. Create [Atlas cluster](https://www.mongodb.com/docs/atlas/getting-started/) + +2. Create a collection + +3. Create [Vector Search Index](https://www.mongodb.com/docs/atlas/atlas-search/field-types/knn-vector/) for the collection. +The index has to be defined on a field called ```embedding```. For example: +``` +{ + "mappings": { + "dynamic": true, + "fields": { + "embedding": { + "dimension": 1024, + "similarity": "cosine", + "type": "knnVector" + } + } + } +} +``` + +4. Create the MongoDB memory store +```python +import semantic_kernel as sk +import semantic_kernel.connectors.ai.open_ai +from semantic_kernel.connectors.memory.mongodb_atlas import ( + MongoDBAtlasMemoryStore +) + +kernel = sk.Kernel() + +... + +kernel.register_memory_store(memory_store=MongoDBAtlasMemoryStore( + # connection_string = if not provided pull from .env +)) +... + +``` + +## Important Notes + +### Vector search indexes +In this version, vector search index management is outside of ```MongoDBAtlasMemoryStore``` scope. +Creation and maintenance of the indexes have to be done by the user. Please note that deleting a collection +(```memory_store.delete_collection_async```) will delete the index as well. \ No newline at end of file diff --git a/python/semantic_kernel/connectors/memory/mongodb_atlas/__init__.py b/python/semantic_kernel/connectors/memory/mongodb_atlas/__init__.py new file mode 100644 index 000000000000..4ee1e46966ea --- /dev/null +++ b/python/semantic_kernel/connectors/memory/mongodb_atlas/__init__.py @@ -0,0 +1,5 @@ +from semantic_kernel.connectors.memory.mongodb_atlas.mongodb_atlas_memory_store import ( + MongoDBAtlasMemoryStore, +) + +__all__ = ["MongoDBAtlasMemoryStore"] diff --git a/python/semantic_kernel/connectors/memory/mongodb_atlas/mongodb_atlas_memory_store.py b/python/semantic_kernel/connectors/memory/mongodb_atlas/mongodb_atlas_memory_store.py new file mode 100644 index 000000000000..b760e2609260 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/mongodb_atlas/mongodb_atlas_memory_store.py @@ -0,0 +1,325 @@ +# Copyright (c) Microsoft. All rights reserved. +from __future__ import annotations + +from logging import Logger +from typing import Any, List, Mapping, Optional, Tuple + +from motor import MotorCommandCursor, core, motor_asyncio +from numpy import ndarray +from pymongo import DeleteOne, ReadPreference, UpdateOne, results + +from semantic_kernel.connectors.memory.mongodb_atlas.utils import ( + DEFAULT_DB_NAME, + DEFAULT_SEARCH_INDEX_NAME, + MONGODB_FIELD_EMBEDDING, + MONGODB_FIELD_ID, + NUM_CANDIDATES_SCALAR, + document_to_memory_record, + memory_record_to_mongo_document, +) +from semantic_kernel.memory.memory_record import MemoryRecord +from semantic_kernel.memory.memory_store_base import MemoryStoreBase +from semantic_kernel.utils.null_logger import NullLogger +from semantic_kernel.utils.settings import mongodb_atlas_settings_from_dot_env + + +class MongoDBAtlasMemoryStore(MemoryStoreBase): + """Memory Store for MongoDB Atlas Vector Search Connections""" + + __slots__ = ("_mongo_client", "_logger", "__database_name") + + _mongo_client: motor_asyncio.AsyncIOMotorClient + _logger: Logger + __database_name: str + __index_name: str + + def __init__( + self, + index_name: Optional[str] = None, + connection_string: Optional[str] = None, + database_name: Optional[str] = None, + logger: Optional[Logger] = None, + read_preference: Optional[ReadPreference] = ReadPreference.PRIMARY, + ): + self._mongo_client = motor_asyncio.AsyncIOMotorClient( + connection_string or mongodb_atlas_settings_from_dot_env(), + read_preference=read_preference, + ) + self._logger = logger or NullLogger() + self.__database_name = database_name or DEFAULT_DB_NAME + self.__index_name = index_name or DEFAULT_SEARCH_INDEX_NAME + + @property + def database_name(self) -> str: + return self.__database_name + + @property + def database(self) -> core.AgnosticDatabase: + return self._mongo_client[self.database_name] + + @property + def index_name(self) -> str: + return self.__index_name + + @property + def num_candidates(self) -> int: + return self.__num_candidates + + async def close_async(self): + """Async close connection, invoked by MemoryStoreBase.__aexit__()""" + if self._mongo_client: + self._mongo_client.close() + self._mongo_client = None + + async def create_collection_async(self, collection_name: str) -> None: + """Creates a new collection in the data store. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + + Returns: + None + """ + if not await self.does_collection_exist_async(collection_name): + await self.database.create_collection(collection_name) + + async def get_collections_async( + self, + ) -> List[str]: + """Gets all collection names in the data store. + + Returns: + List[str] -- A group of collection names. + """ + return await self.database.list_collection_names() + + async def delete_collection_async(self, collection_name: str) -> None: + """Deletes a collection from the data store. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + + Returns: + None + """ + await self.database[collection_name].drop() + + async def does_collection_exist_async(self, collection_name: str) -> bool: + """Determines if a collection exists in the data store. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + + Returns: + bool -- True if given collection exists, False if not. + """ + return collection_name in (await self.get_collections_async()) + + async def upsert_async(self, collection_name: str, record: MemoryRecord) -> str: + """Upserts a memory record into the data store. Does not guarantee that the collection exists. + If the record already exists, it will be updated. + If the record does not exist, it will be created. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + record {MemoryRecord} -- The memory record to upsert. + + Returns: + str -- The unique identifier for the memory record. + """ + + document: Mapping[str, Any] = memory_record_to_mongo_document(record) + + update_result: results.UpdateResult = await self.database[ + collection_name + ].update_one(document, {"$set": document}, upsert=True) + + assert update_result.acknowledged + return record._id + + async def upsert_batch_async( + self, collection_name: str, records: List[MemoryRecord] + ) -> List[str]: + """Upserts a group of memory records into the data store. Does not guarantee that the collection exists. + If the record already exists, it will be updated. + If the record does not exist, it will be created. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + records {MemoryRecord} -- The memory records to upsert. + + Returns: + List[str] -- The unique identifiers for the memory records. + """ + + upserts: List[UpdateOne] = [] + for record in records: + document = memory_record_to_mongo_document(record) + upserts.append(UpdateOne(document, {"$set": document}, upsert=True)) + bulk_update_result: results.BulkWriteResult = await self.database[ + collection_name + ].bulk_write(upserts, ordered=False) + + # Assert the number matched and the number upserted equal the total batch updated + self._logger.debug( + "matched_count=%s, upserted_count=%s", + bulk_update_result.matched_count, + bulk_update_result.upserted_count, + ) + assert ( + bulk_update_result.matched_count + bulk_update_result.upserted_count + == len(records) + ) + return [record._id for record in records] + + async def get_async( + self, collection_name: str, key: str, with_embedding: bool + ) -> MemoryRecord: + """Gets a memory record from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + key {str} -- The unique id associated with the memory record to get. + with_embedding {bool} -- If true, the embedding will be returned in the memory record. + + Returns: + MemoryRecord -- The memory record if found + """ + document = await self.database[collection_name].find_one( + {MONGODB_FIELD_ID: key} + ) + + return document_to_memory_record(document, with_embedding) if document else None + + async def get_batch_async( + self, collection_name: str, keys: List[str], with_embeddings: bool + ) -> List[MemoryRecord]: + """Gets a batch of memory records from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + keys {List[str]} -- The unique ids associated with the memory records to get. + with_embeddings {bool} -- If true, the embedding will be returned in the memory records. + + Returns: + List[MemoryRecord] -- The memory records associated with the unique keys provided. + """ + results = self.database[collection_name].find({MONGODB_FIELD_ID: {"$in": keys}}) + + return [ + document_to_memory_record(result, with_embeddings) + for result in await results.to_list(length=len(keys)) + ] + + async def remove_async(self, collection_name: str, key: str) -> None: + """Removes a memory record from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + key {str} -- The unique id associated with the memory record to remove. + + Returns: + None + """ + if not await self.does_collection_exist_async(collection_name): + raise Exception(f"collection {collection_name} not found") + await self.database[collection_name].delete_one({MONGODB_FIELD_ID: key}) + + async def remove_batch_async(self, collection_name: str, keys: List[str]) -> None: + """Removes a batch of memory records from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + keys {List[str]} -- The unique ids associated with the memory records to remove. + + Returns: + None + """ + if not await self.does_collection_exist_async(collection_name): + raise Exception(f"collection {collection_name} not found") + deletes: List[DeleteOne] = [DeleteOne({MONGODB_FIELD_ID: key}) for key in keys] + bulk_write_result = await self.database[collection_name].bulk_write( + deletes, ordered=False + ) + self._logger.debug("%s entries deleted", bulk_write_result.deleted_count) + + async def get_nearest_matches_async( + self, + collection_name: str, + embedding: ndarray, + limit: int, + with_embeddings: bool, + min_relevance_score: float | None = None, + ) -> List[Tuple[MemoryRecord, float]]: + """Gets the nearest matches to an embedding of type float. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + embedding {ndarray} -- The embedding to compare the collection's embeddings with. + limit {int} -- The maximum number of similarity results to return, defaults to 1. + min_relevance_score {float} -- The minimum relevance threshold for returned results. + with_embeddings {bool} -- If true, the embeddings will be returned in the memory records. + Returns: + List[Tuple[MemoryRecord, float]] -- A list of tuples where item1 is a MemoryRecord and item2 + is its similarity score as a float. + """ + pipeline: list[dict[str, Any]] = [] + vector_search_query: List[Mapping[str, Any]] = { + "$vectorSearch": { + "queryVector": embedding.tolist(), + "limit": limit, + "numCandidates": limit * NUM_CANDIDATES_SCALAR, + "path": MONGODB_FIELD_EMBEDDING, + "index": self.index_name, + } + } + + pipeline.append(vector_search_query) + # add meta search scoring + pipeline.append({"$set": {"score": {"$meta": "vectorSearchScore"}}}) + + if min_relevance_score: + pipeline.append({"$match": {"$gte": ["$score", min_relevance_score]}}) + + cursor: MotorCommandCursor = self.database[collection_name].aggregate(pipeline) + + return [ + ( + document_to_memory_record(doc, with_embeddings=with_embeddings), + doc["score"], + ) + for doc in await cursor.to_list(length=limit) + ] + + async def get_nearest_match_async( + self, + collection_name: str, + embedding: ndarray, + with_embedding: bool, + min_relevance_score: float | None = None, + ) -> Tuple[MemoryRecord, float]: + """Gets the nearest match to an embedding of type float. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + embedding {ndarray} -- The embedding to compare the collection's embeddings with. + min_relevance_score {float} -- The minimum relevance threshold for returned result. + with_embedding {bool} -- If true, the embeddings will be returned in the memory record. + + Returns: + Tuple[MemoryRecord, float] -- A tuple consisting of the MemoryRecord and the similarity score as a float. + """ + matches: List[ + Tuple[MemoryRecord, float] + ] = await self.get_nearest_matches_async( + collection_name=collection_name, + embedding=embedding, + limit=1, + min_relevance_score=min_relevance_score, + with_embeddings=with_embedding, + ) + + return matches[0] if matches else None + + +__all__ = ["MongoDBAtlasMemoryStore"] diff --git a/python/semantic_kernel/connectors/memory/mongodb_atlas/utils.py b/python/semantic_kernel/connectors/memory/mongodb_atlas/utils.py new file mode 100644 index 000000000000..5d56080a5698 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/mongodb_atlas/utils.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft. All rights reserved. +from __future__ import annotations + +from numpy import array + +from semantic_kernel.memory.memory_record import MemoryRecord + +DEFAULT_DB_NAME = "default" +DEFAULT_SEARCH_INDEX_NAME = "default" +NUM_CANDIDATES_SCALAR = 10 + +MONGODB_FIELD_ID = "_id" +MONGODB_FIELD_TEXT = "text" +MONGODB_FIELD_EMBEDDING = "embedding" +MONGODB_FIELD_SRC = "externalSourceName" +MONGODB_FIELD_DESC = "description" +MONGODB_FIELD_METADATA = "additionalMetadata" +MONGODB_FIELD_IS_REF = "isReference" +MONGODB_FIELD_KEY = "key" +MONGODB_FIELD_TIMESTAMP = "timestamp" + + +def document_to_memory_record(data: dict, with_embeddings: bool) -> MemoryRecord: + """Converts a search result to a MemoryRecord. + + Arguments: + data {dict} -- Azure Cognitive Search result data. + + Returns: + MemoryRecord -- The MemoryRecord from Azure Cognitive Search Data Result. + """ + meta = data.get(MONGODB_FIELD_METADATA, {}) + + return MemoryRecord( + id=meta.get(MONGODB_FIELD_ID), + text=meta.get(MONGODB_FIELD_TEXT), + external_source_name=meta.get(MONGODB_FIELD_SRC), + description=meta.get(MONGODB_FIELD_DESC), + additional_metadata=meta.get(MONGODB_FIELD_METADATA), + is_reference=meta.get(MONGODB_FIELD_IS_REF), + embedding=array(data.get(MONGODB_FIELD_EMBEDDING)) if with_embeddings else None, + timestamp=data.get(MONGODB_FIELD_TIMESTAMP), + key=meta.get(MONGODB_FIELD_ID), + ) + + +def memory_record_to_mongo_document(record: MemoryRecord) -> dict: + """Convert a MemoryRecord to a dictionary + + Arguments: + record {MemoryRecord} -- The MemoryRecord from Azure Cognitive Search Data Result. + + Returns: + data {dict} -- Dictionary data. + """ + + return { + MONGODB_FIELD_ID: record._id, + MONGODB_FIELD_METADATA: { + MONGODB_FIELD_ID: record._id, + MONGODB_FIELD_TEXT: record._text, + MONGODB_FIELD_SRC: record._external_source_name or "", + MONGODB_FIELD_DESC: record._description or "", + MONGODB_FIELD_METADATA: record._additional_metadata or "", + MONGODB_FIELD_IS_REF: record._is_reference, + }, + MONGODB_FIELD_EMBEDDING: record._embedding.tolist(), + MONGODB_FIELD_TIMESTAMP: record._timestamp, + } diff --git a/python/semantic_kernel/connectors/memory/pinecone/pinecone_memory_store.py b/python/semantic_kernel/connectors/memory/pinecone/pinecone_memory_store.py index 3c4f6d2b4595..aeae07244b6e 100644 --- a/python/semantic_kernel/connectors/memory/pinecone/pinecone_memory_store.py +++ b/python/semantic_kernel/connectors/memory/pinecone/pinecone_memory_store.py @@ -3,10 +3,10 @@ from logging import Logger from typing import List, Optional, Tuple -from numpy import ndarray - import pinecone +from numpy import ndarray from pinecone import FetchResponse, IndexDescription + from semantic_kernel.connectors.memory.pinecone.utils import ( build_payload, parse_payload, diff --git a/python/semantic_kernel/connectors/memory/pinecone/utils.py b/python/semantic_kernel/connectors/memory/pinecone/utils.py index e82dc4bf19a0..218c2035aff1 100644 --- a/python/semantic_kernel/connectors/memory/pinecone/utils.py +++ b/python/semantic_kernel/connectors/memory/pinecone/utils.py @@ -1,8 +1,8 @@ # Copyright (c) Microsoft. All rights reserved. import numpy - from pinecone import Vector + from semantic_kernel.memory.memory_record import MemoryRecord diff --git a/python/semantic_kernel/connectors/memory/qdrant/qdrant_memory_store.py b/python/semantic_kernel/connectors/memory/qdrant/qdrant_memory_store.py index 5cc6885b4daa..4e3d209f70de 100644 --- a/python/semantic_kernel/connectors/memory/qdrant/qdrant_memory_store.py +++ b/python/semantic_kernel/connectors/memory/qdrant/qdrant_memory_store.py @@ -164,7 +164,7 @@ async def upsert_batch_async( raise Exception("Batch upsert failed") async def get_async( - self, collection_name: str, key: str, with_embedding: bool + self, collection_name: str, key: str, with_embedding: bool = False ) -> Optional[MemoryRecord]: result = await self._get_existing_record_by_payload_id_async( collection_name=collection_name, @@ -188,7 +188,7 @@ async def get_async( return None async def get_batch_async( - self, collection_name: str, keys: List[str], with_embeddings: bool + self, collection_name: str, keys: List[str], with_embeddings: bool = False ) -> List[MemoryRecord]: tasks = [] for key in keys: @@ -243,7 +243,7 @@ async def get_nearest_matches_async( embedding: ndarray, limit: int, min_relevance_score: float, - with_embeddings: bool, + with_embeddings: bool = False, ) -> List[Tuple[MemoryRecord, float]]: match_results = self._qdrantclient.search( collection_name=collection_name, @@ -276,7 +276,7 @@ async def get_nearest_match_async( collection_name: str, embedding: ndarray, min_relevance_score: float, - with_embedding: bool, + with_embedding: bool = False, ) -> Tuple[MemoryRecord, float]: result = await self.get_nearest_matches_async( collection_name=collection_name, @@ -342,9 +342,9 @@ async def _convert_from_memory_record_async( else: pointId = str(uuid.uuid4()) + payload = record.__dict__.copy() + embedding = payload.pop("_embedding") + return qdrant_models.PointStruct( - id=pointId, - vector=record._embedding.tolist(), - payload=record.__dict__, - default=str, + id=pointId, vector=embedding.tolist(), payload=payload ) diff --git a/python/semantic_kernel/connectors/memory/redis/README.md b/python/semantic_kernel/connectors/memory/redis/README.md new file mode 100644 index 000000000000..3373990ecf19 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/redis/README.md @@ -0,0 +1,31 @@ +# semantic_kernel.connectors.memory.redis + +This connector uses Redis to implement Semantic Memory. It requires the [RediSearch](https://redis.io/docs/interact/search-and-query/) module to be enabled on Redis to implement vector similarity search. + +See the [.net README](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Connectors/Connectors.Memory.Redis/README.md) for more information. + +## Quick start + +1. Run with Docker: + +```bash +docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest +``` + +2. To use Redis as a semantic memory store: + +```python + import semantic_kernel as sk + import semantic_kernel.connectors.ai.open_ai as sk_oai + from semantic_kernel.connectors.memory.redis import RedisMemoryStore + + kernel = sk.Kernel() + + api_key, org_id = sk.openai_settings_from_dot_env() + kernel.add_text_completion_service("dv", sk_oai.OpenAITextCompletion("text-davinci-003", api_key, org_id)) + kernel.add_text_embedding_generation_service("ada", sk_oai.OpenAITextEmbedding("text-embedding-ada-002", api_key, org_id)) + + redis_connection_string = sk.redis_settings_from_dot_env() + kernel.register_memory_store(memory_store=RedisMemoryStore(connection_string=redis_connection_string)) +``` + diff --git a/python/semantic_kernel/connectors/memory/redis/__init__.py b/python/semantic_kernel/connectors/memory/redis/__init__.py new file mode 100644 index 000000000000..85a1b319199b --- /dev/null +++ b/python/semantic_kernel/connectors/memory/redis/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.connectors.memory.redis.redis_memory_store import ( + RedisMemoryStore, +) + +__all__ = ["RedisMemoryStore"] diff --git a/python/semantic_kernel/connectors/memory/redis/redis_memory_store.py b/python/semantic_kernel/connectors/memory/redis/redis_memory_store.py new file mode 100644 index 000000000000..d1052419d129 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/redis/redis_memory_store.py @@ -0,0 +1,395 @@ +# Copyright (c) Microsoft. All rights reserved. + +from logging import Logger +from typing import List, Optional, Tuple + +import numpy as np +import redis +from numpy import ndarray +from redis.commands.search.field import TextField, VectorField +from redis.commands.search.indexDefinition import IndexDefinition, IndexType +from redis.commands.search.query import Query +from redis.exceptions import ResponseError + +from semantic_kernel.connectors.memory.redis.utils import ( + deserialize_document_to_record, + deserialize_redis_to_record, + get_redis_key, + serialize_record_to_redis, +) +from semantic_kernel.memory.memory_record import MemoryRecord +from semantic_kernel.memory.memory_store_base import MemoryStoreBase +from semantic_kernel.utils.null_logger import NullLogger + + +class RedisMemoryStore(MemoryStoreBase): + """A memory store implementation using Redis""" + + _database: "redis.Redis" + _ft: "redis.Redis.ft" + _logger: Logger + # Without RedisAI, it is currently not possible to retrieve index-specific vector attributes to have + # fully independent collections. + _query_dialect: int + _vector_distance_metric: str + _vector_index_algorithm: str + _vector_size: int + _vector_type: "np.dtype" + _vector_type_str: str + + def __init__( + self, + connection_string: str, + vector_size: int = 1536, + vector_distance_metric: str = "COSINE", + vector_type: str = "FLOAT32", + vector_index_algorithm: str = "HNSW", + query_dialect: int = 2, + logger: Optional[Logger] = None, + ) -> None: + """ + RedisMemoryStore is an abstracted interface to interact with a Redis node connection. + See documentation about connections: https://redis-py.readthedocs.io/en/stable/connections.html + See documentation about vector attributes: https://redis.io/docs/stack/search/reference/vectors + + Arguments: + connection_string {str} -- Provide connection URL to a Redis instance + vector_size {str} -- Size of vectors, defaults to 1536 + vector_distance_metric {str} -- Metric for measuring vector distances, defaults to COSINE + vector_type {str} -- Vector type, defaults to FLOAT32 + vector_index_algorithm {str} -- Indexing algorithm for vectors, defaults to HNSW + query_dialect {int} -- Query dialect, must be 2 or greater for vector similarity searching, defaults to 2 + logger {Optional[Logger]} -- Logger, defaults to None + + """ + if vector_size <= 0: + raise ValueError("Vector dimension must be a positive integer") + + self._database = redis.Redis.from_url(connection_string) + self._ft = self._database.ft + self._logger = logger or NullLogger() + + self._query_dialect = query_dialect + self._vector_distance_metric = vector_distance_metric + self._vector_index_algorithm = vector_index_algorithm + self._vector_type_str = vector_type + self._vector_type = np.float32 if vector_type == "FLOAT32" else np.float64 + self._vector_size = vector_size + + async def close_async(self): + """ + Closes the Redis database connection + """ + self._logger.info("Closing Redis connection") + self._database.close() + + async def create_collection_async(self, collection_name: str) -> None: + """ + Creates a collection, implemented as a Redis index containing hashes + prefixed with "collection_name:". + If a collection of the name exists, it is left unchanged. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + """ + + if await self.does_collection_exist_async(collection_name): + self._logger.info(f'Collection "{collection_name}" already exists.') + else: + index_def = IndexDefinition( + prefix=f"{collection_name}:", index_type=IndexType.HASH + ) + schema = ( + TextField(name="key"), + TextField(name="metadata"), + TextField(name="timestamp"), + VectorField( + name="embedding", + algorithm=self._vector_index_algorithm, + attributes={ + "TYPE": self._vector_type_str, + "DIM": self._vector_size, + "DISTANCE_METRIC": self._vector_distance_metric, + }, + ), + ) + + try: + self._ft(collection_name).create_index( + definition=index_def, fields=schema + ) + except Exception as e: + self._logger.error(e) + raise e + + async def get_collections_async(self) -> List[str]: + """ + Returns a list of names of all collection names present in the data store. + + Returns: + List[str] -- list of collection names + """ + # Note: FT._LIST is a temporary command that may be deprecated in the future according to Redis + return [name.decode() for name in self._database.execute_command("FT._LIST")] + + async def delete_collection_async( + self, collection_name: str, delete_records: bool = True + ) -> None: + """ + Deletes a collection from the data store. + If the collection does not exist, the database is left unchanged. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + delete_records {bool} -- Delete all data associated with the collection, default to True + + """ + if await self.does_collection_exist_async(collection_name): + self._ft(collection_name).dropindex(delete_documents=delete_records) + + async def does_collection_exist_async(self, collection_name: str) -> bool: + """ + Determines if a collection exists in the data store. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + + Returns: + True if the collection exists, False if not + """ + try: + self._ft(collection_name).info() + return True + except ResponseError: + return False + + async def upsert_async(self, collection_name: str, record: MemoryRecord) -> str: + """ + Upsert a memory record into the data store. Does not guarantee that the collection exists. + * If the record already exists, it will be updated. + * If the record does not exist, it will be created. + + Note: if the record do not have the same dimensionality configured for the collection, + it will not be detected to belong to the collection in Redis. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + record {MemoryRecord} -- Memory record to upsert + + Returns + str -- Redis key associated with the upserted memory record + """ + + if not await self.does_collection_exist_async(collection_name): + self._logger.error(f'Collection "{collection_name}" does not exist') + raise Exception(f'Collection "{collection_name}" does not exist') + + # Typical Redis key structure: collection_name:{some identifier} + record._key = get_redis_key(collection_name, record._id) + + # Overwrites previous data or inserts new key if not present + # Index registers any hash matching its schema and prefixed with collection_name: + try: + self._database.hset( + record._key, + mapping=serialize_record_to_redis(record, self._vector_type), + ) + return record._key + except Exception as e: + self._logger.error(e) + raise e + + async def upsert_batch_async( + self, collection_name: str, records: List[MemoryRecord] + ) -> List[str]: + """ + Upserts a group of memory records into the data store. Does not guarantee that the collection exists. + * If the record already exists, it will be updated. + * If the record does not exist, it will be created. + + Note: if the records do not have the same dimensionality configured for the collection, + they will not be detected to belong to the collection in Redis. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + records {List[MemoryRecord]} -- List of memory records to upsert + + Returns + List[str] -- Redis keys associated with the upserted memory records + """ + + keys = list() + for record in records: + record_key = await self.upsert_async(collection_name, record) + keys.append(record_key) + + return keys + + async def get_async( + self, collection_name: str, key: str, with_embedding: bool = False + ) -> MemoryRecord: + """ + Gets a memory record from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + key {str} -- ID associated with the memory to get + with_embedding {bool} -- Include embedding with the memory record, default to False + + Returns: + MemoryRecord -- The memory record if found, else None + """ + + if not await self.does_collection_exist_async(collection_name): + self._logger.error(f'Collection "{collection_name}" does not exist') + raise Exception(f'Collection "{collection_name}" does not exist') + + internal_key = get_redis_key(collection_name, key) + fields = self._database.hgetall(internal_key) + + # Did not find the record + if len(fields) == 0: + return None + + record = deserialize_redis_to_record(fields, self._vector_type, with_embedding) + record._key = internal_key + + return record + + async def get_batch_async( + self, collection_name: str, keys: List[str], with_embeddings: bool = False + ) -> List[MemoryRecord]: + """ + Gets a batch of memory records from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + keys {List[str]} -- IDs associated with the memory records to get + with_embedding {bool} -- Include embeddings with the memory records, default to False + + Returns: + List[MemoryRecord] -- The memory records if found, else an empty list + """ + + records = list() + for key in keys: + record = await self.get_async(collection_name, key, with_embeddings) + if record: + records.append(record) + + return records + + async def remove_async(self, collection_name: str, key: str) -> None: + """ + Removes a memory record from the data store. Does not guarantee that the collection exists. + If the key does not exist, do nothing. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + key {str} -- ID associated with the memory to remove + """ + if not await self.does_collection_exist_async(collection_name): + self._logger.error(f'Collection "{collection_name}" does not exist') + raise Exception(f'Collection "{collection_name}" does not exist') + + self._database.delete(get_redis_key(collection_name, key)) + + async def remove_batch_async(self, collection_name: str, keys: List[str]) -> None: + """ + Removes a batch of memory records from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + keys {List[str]} -- IDs associated with the memory records to remove + """ + if not await self.does_collection_exist_async(collection_name): + self._logger.error(f'Collection "{collection_name}" does not exist') + raise Exception(f'Collection "{collection_name}" does not exist') + + self._database.delete(*[get_redis_key(collection_name, key) for key in keys]) + + async def get_nearest_matches_async( + self, + collection_name: str, + embedding: ndarray, + limit: int, + min_relevance_score: float = 0.0, + with_embeddings: bool = False, + ) -> List[Tuple[MemoryRecord, float]]: + """ + Get the nearest matches to an embedding using the configured similarity algorithm. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + embedding {ndarray} -- Embedding to find the nearest matches to + limit {int} -- Maximum number of matches to return + min_relevance_score {float} -- Minimum relevance score of the matches, default to 0.0 + with_embeddings {bool} -- Include embeddings in the resultant memory records, default to False + + Returns: + List[Tuple[MemoryRecord, float]] -- Records and their relevance scores by descending + order, or an empty list if no relevant matches are found + """ + if not await self.does_collection_exist_async(collection_name): + self._logger.error(f'Collection "{collection_name}" does not exist') + raise Exception(f'Collection "{collection_name}" does not exist') + + # Perform a k-nearest neighbors query, score by similarity + query = ( + Query(f"*=>[KNN {limit} @embedding $embedding AS vector_score]") + .dialect(self._query_dialect) + .paging(offset=0, num=limit) + .return_fields( + "metadata", + "timestamp", + "embedding", + "vector_score", + ) + .sort_by("vector_score", asc=False) + ) + query_params = {"embedding": embedding.astype(self._vector_type).tobytes()} + matches = self._ft(collection_name).search(query, query_params).docs + + relevant_records = list() + for match in matches: + score = float(match["vector_score"]) + + # Sorted by descending order + if score < min_relevance_score: + break + + record = deserialize_document_to_record( + self._database, match, self._vector_type, with_embeddings + ) + relevant_records.append((record, score)) + + return relevant_records + + async def get_nearest_match_async( + self, + collection_name: str, + embedding: ndarray, + min_relevance_score: float = 0.0, + with_embedding: bool = False, + ) -> Tuple[MemoryRecord, float]: + """ + Get the nearest match to an embedding using the configured similarity algorithm. + + Arguments: + collection_name {str} -- Name for a collection of embeddings + embedding {ndarray} -- Embedding to find the nearest match to + min_relevance_score {float} -- Minimum relevance score of the match, default to 0.0 + with_embedding {bool} -- Include embedding in the resultant memory record, default to False + + Returns: + Tuple[MemoryRecord, float] -- Record and the relevance score, or None if not found + """ + matches = await self.get_nearest_matches_async( + collection_name=collection_name, + embedding=embedding, + limit=1, + min_relevance_score=min_relevance_score, + with_embeddings=with_embedding, + ) + + return matches[0] if len(matches) else None diff --git a/python/semantic_kernel/connectors/memory/redis/utils.py b/python/semantic_kernel/connectors/memory/redis/utils.py new file mode 100644 index 000000000000..d083d7857774 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/redis/utils.py @@ -0,0 +1,119 @@ +# Copyright (c) Microsoft. All rights reserved. + +import json +from datetime import datetime +from typing import Any, Dict, Tuple + +import numpy as np +from redis import Redis +from redis.commands.search.document import Document + +from semantic_kernel.memory.memory_record import MemoryRecord + + +def get_redis_key(collection_name: str, record_id: str) -> str: + """ + Returns the Redis key for an element called record_id within collection_name + + Arguments: + collection_name {str} -- Name for a collection of embeddings + record_id {str} -- ID associated with a memory record + + Returns: + str -- Redis key in the format collection_name:id + """ + return f"{collection_name}:{record_id}" + + +def split_redis_key(redis_key: str) -> Tuple[str, str]: + """ + Split a Redis key into its collection name and record ID + + Arguments: + collection_name {str} -- Redis key + + Returns: + Tuple[str, str] -- Tuple of the collection name and ID + """ + collection, record_id = redis_key.split(":") + return collection, record_id + + +def serialize_record_to_redis( + record: MemoryRecord, vector_type: np.dtype +) -> Dict[str, Any]: + all_metadata = { + "is_reference": record._is_reference, + "external_source_name": record._external_source_name or "", + "id": record._id or "", + "description": record._description or "", + "text": record._text or "", + "additional_metadata": record._additional_metadata or "", + } + + redis_mapping = { + "key": record._key or "", + "timestamp": record._timestamp.isoformat() if record._timestamp else "", + "metadata": json.dumps(all_metadata), + "embedding": ( + record._embedding.astype(vector_type).tobytes() + if record._embedding is not None + else "" + ), + } + return redis_mapping + + +def deserialize_redis_to_record( + fields: Dict[str, Any], vector_type: np.dtype, with_embedding: bool +) -> MemoryRecord: + metadata = json.loads(fields[b"metadata"]) + record = MemoryRecord( + id=metadata["id"], + is_reference=True if metadata["is_reference"] is True else False, + description=metadata["description"], + external_source_name=metadata["external_source_name"], + text=metadata["text"], + additional_metadata=metadata["additional_metadata"], + embedding=None, + ) + + if fields[b"timestamp"] != b"": + record._timestamp = datetime.fromisoformat(fields[b"timestamp"].decode()) + + if with_embedding: + # Extract using the vector type, then convert to regular Python float type + record._embedding = np.frombuffer( + fields[b"embedding"], dtype=vector_type + ).astype(float) + + return record + + +def deserialize_document_to_record( + database: Redis, doc: Document, vector_type: np.dtype, with_embedding: bool +) -> MemoryRecord: + # Document's ID refers to the Redis key + redis_key = doc["id"] + _, id_str = split_redis_key(redis_key) + + metadata = json.loads(doc["metadata"]) + record = MemoryRecord( + id=id_str, + is_reference=True if metadata["is_reference"] is True else False, + description=metadata["description"], + external_source_name=metadata["external_source_name"], + text=metadata["text"], + additional_metadata=metadata["additional_metadata"], + embedding=None, + ) + + if doc["timestamp"] != "": + record._timestamp = datetime.fromisoformat(doc["timestamp"]) + + if with_embedding: + # Some bytes are lost when retrieving a document, fetch raw embedding + eb = database.hget(redis_key, "embedding") + record._embedding = np.frombuffer(eb, dtype=vector_type).astype(float) + + return record diff --git a/python/semantic_kernel/connectors/memory/usearch/__init__.py b/python/semantic_kernel/connectors/memory/usearch/__init__.py new file mode 100644 index 000000000000..f74403f6441f --- /dev/null +++ b/python/semantic_kernel/connectors/memory/usearch/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.connectors.memory.usearch.usearch_memory_store import ( + USearchMemoryStore, +) + +__all__ = ["USearchMemoryStore"] diff --git a/python/semantic_kernel/connectors/memory/usearch/usearch_memory_store.py b/python/semantic_kernel/connectors/memory/usearch/usearch_memory_store.py new file mode 100644 index 000000000000..997b0427c8f9 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/usearch/usearch_memory_store.py @@ -0,0 +1,638 @@ +# Copyright (c) Microsoft. All rights reserved. + +import itertools +import os +from dataclasses import dataclass +from enum import Enum +from logging import Logger +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +from numpy import ndarray +from usearch.index import ( + BatchMatches, + CompiledMetric, + Index, + Matches, + MetricKind, + ScalarKind, +) + +from semantic_kernel.memory.memory_record import MemoryRecord +from semantic_kernel.memory.memory_store_base import MemoryStoreBase +from semantic_kernel.utils.null_logger import NullLogger + + +@dataclass +class _USearchCollection: + """Represents a collection for USearch with embeddings and related data. + + Attributes: + embeddings_index (Index): The index of embeddings. + embeddings_data_table (pa.Table): The PyArrow table holding embeddings data. + embeddings_id_to_label (Dict[str, int]): Mapping of embeddings ID to label. + """ + + embeddings_index: Index + embeddings_data_table: pa.Table + embeddings_id_to_label: Dict[str, int] + + @staticmethod + def create_default(embeddings_index: Index) -> "_USearchCollection": + """Create a default `_USearchCollection` using a given embeddings index. + + Args: + embeddings_index (Index): The index of embeddings to be used for the default collection. + + Returns: + _USearchCollection: A default `_USearchCollection` initialized with the given embeddings index. + """ + return _USearchCollection( + embeddings_index, + pa.Table.from_pandas( + pd.DataFrame(columns=_embeddings_data_schema.names), + schema=_embeddings_data_schema, + ), + {}, + ) + + +# PyArrow Schema definition for the embeddings data from `MemoryRecord`. +_embeddings_data_schema = pa.schema( + [ + pa.field("key", pa.string()), + pa.field("timestamp", pa.timestamp("us")), + pa.field("is_reference", pa.bool_()), + pa.field("external_source_name", pa.string()), + pa.field("id", pa.string()), + pa.field("description", pa.string()), + pa.field("text", pa.string()), + pa.field("additional_metadata", pa.string()), + ] +) + + +class _CollectionFileType(Enum): + """Enumeration of file types used for storing collections.""" + + USEARCH = 0 + PARQUET = 1 + + +# Mapping of collection file types to their file extensions. +_collection_file_extensions: Dict[_CollectionFileType, str] = { + _CollectionFileType.USEARCH: ".usearch", + _CollectionFileType.PARQUET: ".parquet", +} + + +def memoryrecords_to_pyarrow_table(records: List[MemoryRecord]) -> pa.Table: + """Convert a list of `MemoryRecord` to a PyArrow Table""" + records_pylist = [ + {attr: getattr(record, "_" + attr) for attr in _embeddings_data_schema.names} + for record in records + ] + return pa.Table.from_pylist(records_pylist, schema=_embeddings_data_schema) + + +def pyarrow_table_to_memoryrecords( + table: pa.Table, vectors: Optional[ndarray] = None +) -> List[MemoryRecord]: + """Convert a PyArrow Table to a list of MemoryRecords. + + Args: + table (pa.Table): The PyArrow Table to convert. + vectors (Optional[ndarray], optional): An array of vectors to include as embeddings in the MemoryRecords. + The length and order of the vectors should match the rows in the table. Defaults to None. + + Returns: + List[MemoryRecord]: List of MemoryRecords constructed from the table. + """ + result_memory_records = [ + MemoryRecord( + **row.to_dict(), embedding=vectors[index] if vectors is not None else None + ) + for index, row in table.to_pandas().iterrows() + ] + + return result_memory_records + + +class USearchMemoryStore(MemoryStoreBase): + def __init__( + self, + persist_directory: Optional[os.PathLike] = None, + logger: Optional[Logger] = None, + ) -> None: + """ + Create a USearchMemoryStore instance. + + This store helps searching embeddings with USearch, keeping collections in memory. + To save collections to disk, provide the `persist_directory` param. + Collections are saved when `close_async` is called. + + To both save collections and free up memory, call `close_async`. + When `USearchMemoryStore` is used with a context manager, this will happen automatically. + Otherwise, it should be called explicitly. + + Args: + persist_directory (Optional[os.PathLike], default=None): Directory for loading and saving collections. + If None, collections are not loaded nor saved. + logger (Optional[Logger], default=None): Logger for diagnostics. If None, a NullLogger is used. + """ + self._logger = logger or NullLogger() + self._persist_directory = ( + Path(persist_directory) if persist_directory is not None else None + ) + + self._collections: Dict[str, _USearchCollection] = {} + if self._persist_directory: + self._collections = self._read_collections_from_dir() + + def _get_collection_path( + self, collection_name: str, *, file_type: _CollectionFileType + ) -> Path: + """ + Get the path for the given collection name and file type. + + Args: + collection_name (str): Name of the collection. + file_type (_CollectionFileType): The file type. + + Returns: + Path: Path to the collection file. + + Raises: + ValueError: If persist directory path is not set. + """ + collection_name = collection_name.lower() + if self._persist_directory is None: + raise ValueError("Path of persist directory is not set") + + return self._persist_directory / ( + collection_name + _collection_file_extensions[file_type] + ) + + async def create_collection_async( + self, + collection_name: str, + ndim: int = 0, + metric: Union[str, MetricKind, CompiledMetric] = MetricKind.IP, + dtype: Optional[Union[str, ScalarKind]] = None, + connectivity: Optional[int] = None, + expansion_add: Optional[int] = None, + expansion_search: Optional[int] = None, + view: bool = False, + ) -> None: + """Create a new collection. + + Args: + collection_name (str): Name of the collection. Case-insensitive. + Must have name that is valid file name for the current OS environment. + ndim (int, optional): Number of dimensions. Defaults to 0. + metric (Union[str, MetricKind, CompiledMetric], optional): Metric kind. Defaults to MetricKind.IP. + dtype (Optional[Union[str, ScalarKind]], optional): Data type. Defaults to None. + connectivity (int, optional): Connectivity parameter. Defaults to None. + expansion_add (int, optional): Expansion add parameter. Defaults to None. + expansion_search (int, optional): Expansion search parameter. Defaults to None. + view (bool, optional): Viewing flag. Defaults to False. + + Raises: + ValueError: If collection with the given name already exists. + ValueError: If collection name is empty string. + """ + collection_name = collection_name.lower() + if not collection_name: + raise ValueError("Collection name can not be empty.") + if collection_name in self._collections: + raise ValueError(f"Collection with name {collection_name} already exists.") + + embeddings_index_path = ( + self._get_collection_path( + collection_name, file_type=_CollectionFileType.USEARCH + ) + if self._persist_directory + else None + ) + + embeddings_index = Index( + path=embeddings_index_path, + ndim=ndim, + metric=metric, + dtype=dtype, + connectivity=connectivity, + expansion_add=expansion_add, + expansion_search=expansion_search, + view=view, + ) + + self._collections[collection_name] = _USearchCollection.create_default( + embeddings_index + ) + + return None + + def _read_embeddings_table( + self, path: os.PathLike + ) -> Tuple[pa.Table, Dict[str, int]]: + """Read embeddings from the provided path and generate an ID to label mapping. + + Args: + path (os.PathLike): Path to the embeddings. + + Returns: + Tuple of embeddings table and a dictionary mapping from record ID to its label. + """ + embeddings_table = pq.read_table(path, schema=_embeddings_data_schema) + embeddings_id_to_label: Dict[str, int] = { + record_id: idx + for idx, record_id in enumerate(embeddings_table.column("id").to_pylist()) + } + return embeddings_table, embeddings_id_to_label + + def _read_embeddings_index(self, path: Path) -> Index: + """Read embeddings index.""" + # str cast is temporarily fix for https://github.com/unum-cloud/usearch/issues/196 + return Index.restore(str(path), view=False) + + def _read_collections_from_dir(self) -> Dict[str, _USearchCollection]: + """Read all collections from directory to memory. + + Raises: + ValueError: If files for a collection do not match expected amount. + + Returns: + Dict[str, _USearchCollection]: Dictionary with collection names as keys and + their _USearchCollection as values. + """ + collections: Dict[str, _USearchCollection] = {} + + for collection_name, collection_files in self._get_all_storage_files().items(): + expected_storage_files = len(_CollectionFileType) + if len(collection_files) != expected_storage_files: + raise ValueError( + f"Expected {expected_storage_files} files for collection {collection_name}" + ) + parquet_file, usearch_file = collection_files + if ( + parquet_file.suffix + == _collection_file_extensions[_CollectionFileType.USEARCH] + ): + parquet_file, usearch_file = usearch_file, parquet_file + + embeddings_table, embeddings_id_to_label = self._read_embeddings_table( + parquet_file + ) + embeddings_index = self._read_embeddings_index(usearch_file) + + collections[collection_name] = _USearchCollection( + embeddings_index, + embeddings_table, + embeddings_id_to_label, + ) + + return collections + + async def get_collections_async(self) -> List[str]: + """Get list of existing collections. + + Returns: + List[str]: List of collection names. + """ + return list(self._collections.keys()) + + async def delete_collection_async(self, collection_name: str) -> None: + collection_name = collection_name.lower() + collection = self._collections.pop(collection_name, None) + if collection: + collection.embeddings_index.reset() + return None + + async def does_collection_exist_async(self, collection_name: str) -> bool: + collection_name = collection_name.lower() + return collection_name in self._collections + + async def upsert_async(self, collection_name: str, record: MemoryRecord) -> str: + """Upsert single MemoryRecord and return its ID.""" + collection_name = collection_name.lower() + res = await self.upsert_batch_async( + collection_name=collection_name, records=[record] + ) + return res[0] + + async def upsert_batch_async( + self, + collection_name: str, + records: List[MemoryRecord], + *, + compact: bool = False, + copy: bool = True, + threads: int = 0, + log: Union[str, bool] = False, + batch_size: int = 0, + ) -> List[str]: + """Upsert a batch of MemoryRecords and return their IDs. + + Args: + collection_name (str): Name of the collection to search within. + records (List[MemoryRecord]): Records to upsert. + compact (bool, optional): Removes links to removed nodes (expensive). Defaults to False. + copy (bool, optional): Should the index store a copy of vectors. Defaults to True. + threads (int, optional): Optimal number of cores to use. Defaults to 0. + log (Union[str, bool], optional): Whether to print the progress bar. Defaults to False. + batch_size (int, optional): Number of vectors to process at once. Defaults to 0. + + Raises: + KeyError: If collection not exist + + Returns: + List[str]: List of IDs. + """ + collection_name = collection_name.lower() + if collection_name not in self._collections: + raise KeyError( + f"Collection {collection_name} does not exist, cannot insert." + ) + + ucollection = self._collections[collection_name] + all_records_id = [record._id for record in records] + + # Remove vectors from index + remove_labels = [ + ucollection.embeddings_id_to_label[id] + for id in all_records_id + if id in ucollection.embeddings_id_to_label + ] + ucollection.embeddings_index.remove( + remove_labels, compact=compact, threads=threads + ) + + # Determine label insertion points + table_num_rows = ucollection.embeddings_data_table.num_rows + insert_labels = np.arange(table_num_rows, table_num_rows + len(records)) + + # Add embeddings to index + ucollection.embeddings_index.add( + keys=insert_labels, + vectors=np.stack([record.embedding for record in records]), + copy=copy, + threads=threads, + log=log, + batch_size=batch_size, + ) + + # Update embeddings_table + ucollection.embeddings_data_table = pa.concat_tables( + [ucollection.embeddings_data_table, memoryrecords_to_pyarrow_table(records)] + ) + + # Update embeddings_id_to_label + for index, record_id in enumerate(all_records_id): + ucollection.embeddings_id_to_label[record_id] = insert_labels[index] + + return all_records_id + + async def get_async( + self, + collection_name: str, + key: str, + with_embedding: bool, + dtype: ScalarKind = ScalarKind.F32, + ) -> MemoryRecord: + """Retrieve a single MemoryRecord using its key.""" + collection_name = collection_name.lower() + result = await self.get_batch_async( + collection_name=collection_name, + keys=[key], + with_embeddings=with_embedding, + dtype=dtype, + ) + if not result: + raise KeyError(f"Key '{key}' not found in collection '{collection_name}'") + return result[0] + + async def get_batch_async( + self, + collection_name: str, + keys: List[str], + with_embeddings: bool, + dtype: ScalarKind = ScalarKind.F32, + ) -> List[MemoryRecord]: + """Retrieve a batch of MemoryRecords using their keys.""" + collection_name = collection_name.lower() + if collection_name not in self._collections: + raise KeyError(f"Collection {collection_name} does not exist") + + ucollection = self._collections[collection_name] + labels = [ + ucollection.embeddings_id_to_label[key] + for key in keys + if key in ucollection.embeddings_id_to_label + ] + if not labels: + return [] + vectors = ( + ucollection.embeddings_index.get_vectors(labels, dtype) + if with_embeddings + else None + ) + + return pyarrow_table_to_memoryrecords( + ucollection.embeddings_data_table.take(pa.array(labels)), vectors + ) + + async def remove_async(self, collection_name: str, key: str) -> None: + """Remove a single MemoryRecord using its key.""" + collection_name = collection_name.lower() + await self.remove_batch_async(collection_name=collection_name, keys=[key]) + return None + + async def remove_batch_async(self, collection_name: str, keys: List[str]) -> None: + """Remove a batch of MemoryRecords using their keys.""" + collection_name = collection_name.lower() + if collection_name not in self._collections: + raise KeyError( + f"Collection {collection_name} does not exist, cannot insert." + ) + + ucollection = self._collections[collection_name] + + labels = [ucollection.embeddings_id_to_label[key] for key in keys] + ucollection.embeddings_index.remove(labels) + for key in keys: + del ucollection.embeddings_id_to_label[key] + + return None + + async def get_nearest_match_async( + self, + collection_name: str, + embedding: ndarray, + min_relevance_score: float = 0.0, + with_embedding: bool = True, + exact: bool = False, + ) -> Tuple[MemoryRecord, float]: + """Retrieve the nearest matching MemoryRecord for the provided embedding. + + By default it is approximately search, see `exact` param description. + + Measure of similarity between vectors is relevance score. It is from 0 to 1. + USearch returns distances for vectors. Distance is converted to relevance score by inverse function. + + Args: + collection_name (str): Name of the collection to search within. + embedding (ndarray): The embedding vector to search for. + min_relevance_score (float, optional): The minimum relevance score for vectors. Supposed to be from 0 to 1. + Only vectors with greater or equal relevance score are returned. Defaults to 0.0. + with_embedding (bool, optional): If True, include the embedding in the result. Defaults to True. + exact (bool, optional): Perform exhaustive linear-time exact search. Defaults to False. + + Returns: + Tuple[MemoryRecord, float]: The nearest matching record and its relevance score. + """ + collection_name = collection_name.lower() + results = await self.get_nearest_matches_async( + collection_name=collection_name, + embedding=embedding, + limit=1, + min_relevance_score=min_relevance_score, + with_embeddings=with_embedding, + exact=exact, + ) + return results[0] + + async def get_nearest_matches_async( + self, + collection_name: str, + embedding: ndarray, + limit: int, + min_relevance_score: float = 0.0, + with_embeddings: bool = True, + *, + threads: int = 0, + exact: bool = False, + log: Union[str, bool] = False, + batch_size: int = 0, + ) -> List[Tuple[MemoryRecord, float]]: + """Get the nearest matches to a given embedding. + + By default it is approximately search, see `exact` param description. + + Measure of similarity between vectors is relevance score. It is from 0 to 1. + USearch returns distances for vectors. Distance is converted to relevance score by inverse function. + + Args: + collection_name (str): Name of the collection to search within. + embedding (ndarray): The embedding vector to search for. + limit (int): maximum amount of embeddings to search for. + min_relevance_score (float, optional): The minimum relevance score for vectors. Supposed to be from 0 to 1. + Only vectors with greater or equal relevance score are returned. Defaults to 0.0. + with_embedding (bool, optional): If True, include the embedding in the result. Defaults to True. + threads (int, optional): Optimal number of cores to use. Defaults to 0. + exact (bool, optional): Perform exhaustive linear-time exact search. Defaults to False. + log (Union[str, bool], optional): Whether to print the progress bar. Defaults to False. + batch_size (int, optional): Number of vectors to process at once. Defaults to 0. + + Raises: + KeyError: if a collection with specified name does not exist + + Returns: + List[Tuple[MemoryRecord, float]]: The nearest matching records and their relevance score. + """ + collection_name = collection_name.lower() + ucollection = self._collections[collection_name] + + result: Union[Matches, BatchMatches] = ucollection.embeddings_index.search( + vectors=embedding, + k=limit, + threads=threads, + exact=exact, + log=log, + batch_size=batch_size, + ) + + assert isinstance(result, Matches) + + relevance_score = 1 / (result.distances + 1) + filtered_labels = result.keys[ + np.where(relevance_score >= min_relevance_score)[0] + ] + + filtered_vectors: Optional[np.ndarray] = None + if with_embeddings: + filtered_vectors = ucollection.embeddings_index.get_vectors(filtered_labels) + + return [ + (mem_rec, relevance_score[index].item()) + for index, mem_rec in enumerate( + pyarrow_table_to_memoryrecords( + ucollection.embeddings_data_table.take(pa.array(filtered_labels)), + filtered_vectors, + ) + ) + ] + + def _get_all_storage_files(self) -> Dict[str, List[Path]]: + """Return storage files for each collection in `self._persist_directory`. + + Collection name is derived from file name and converted to lowercase. Files with extensions that + do not match storage extensions are discarded. + + Raises: + ValueError: If persist directory is not set. + + Returns: + Dict[str, List[Path]]: Dictionary of collection names mapped to their respective files. + """ + if self._persist_directory is None: + raise ValueError("Persist directory is not set") + + storage_exts = _collection_file_extensions.values() + collection_storage_files: Dict[str, List[Path]] = {} + for path in self._persist_directory.iterdir(): + if path.is_file() and (path.suffix in storage_exts): + collection_name = path.stem.lower() + if collection_name in collection_storage_files: + collection_storage_files[collection_name].append(path) + else: + collection_storage_files[collection_name] = [path] + return collection_storage_files + + def _dump_collections(self) -> None: + collection_storage_files = self._get_all_storage_files() + for file_path in itertools.chain.from_iterable( + collection_storage_files.values() + ): + file_path.unlink() + + for collection_name, ucollection in self._collections.items(): + ucollection.embeddings_index.save( + self._get_collection_path( + collection_name, file_type=_CollectionFileType.USEARCH + ) + ) + pq.write_table( + ucollection.embeddings_data_table, + self._get_collection_path( + collection_name, file_type=_CollectionFileType.PARQUET + ), + ) + + return None + + async def close_async(self) -> None: + """Persist collection, clear. + + Returns: + None + """ + if self._persist_directory: + self._dump_collections() + + for collection_name in await self.get_collections_async(): + await self.delete_collection_async(collection_name) + self._collections = {} diff --git a/python/semantic_kernel/connectors/memory/weaviate/__init__.py b/python/semantic_kernel/connectors/memory/weaviate/__init__.py new file mode 100644 index 000000000000..dacbcb42bb30 --- /dev/null +++ b/python/semantic_kernel/connectors/memory/weaviate/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft. All rights reserved +from semantic_kernel.connectors.memory.weaviate.weaviate_memory_store import ( + WeaviateMemoryStore, +) + +__all__ = ["WeaviateMemoryStore"] diff --git a/python/semantic_kernel/connectors/openapi/__init__.py b/python/semantic_kernel/connectors/openapi/__init__.py new file mode 100644 index 000000000000..1b4c738ce217 --- /dev/null +++ b/python/semantic_kernel/connectors/openapi/__init__.py @@ -0,0 +1,5 @@ +from semantic_kernel.connectors.openapi.sk_openapi import register_openapi_skill + +__all__ = [ + "register_openapi_skill", +] diff --git a/python/semantic_kernel/connectors/openapi/sk_openapi.py b/python/semantic_kernel/connectors/openapi/sk_openapi.py new file mode 100644 index 000000000000..9c47f878d3a2 --- /dev/null +++ b/python/semantic_kernel/connectors/openapi/sk_openapi.py @@ -0,0 +1,309 @@ +import json +import logging +from typing import Dict, Mapping, Optional, Union +from urllib.parse import urljoin + +import aiohttp +import requests +from openapi_core import Spec, unmarshal_request +from openapi_core.contrib.requests import RequestsOpenAPIRequest +from openapi_core.exceptions import OpenAPIError +from prance import ResolvingParser + +from semantic_kernel import Kernel, SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter +from semantic_kernel.utils.null_logger import NullLogger + + +class PreparedRestApiRequest: + def __init__( + self, method: str, url: str, params=None, headers=None, request_body=None + ): + self.method = method + self.url = url + self.params = params + self.headers = headers + self.request_body = request_body + + def __repr__(self): + return ( + "PreparedRestApiRequest(" + f"method={self.method}, " + f"url={self.url}, " + f"params={self.params}, " + f"headers={self.headers}, " + f"request_body={self.request_body})" + ) + + def validate_request(self, spec: Spec, logger: logging.Logger = NullLogger()): + request = requests.Request( + self.method, + self.url, + params=self.params, + headers=self.headers, + json=self.request_body, + ) + openapi_request = RequestsOpenAPIRequest(request=request) + try: + unmarshal_request(openapi_request, spec=spec) + return True + except OpenAPIError as e: + logger.debug(f"Error validating request: {e}", exc_info=True) + return False + + +class RestApiOperation: + def __init__( + self, + id: str, + method: str, + server_url: str, + path: str, + summary: Optional[str] = None, + description: Optional[str] = None, + params: Optional[Mapping[str, str]] = None, + request_body: Optional[Mapping[str, str]] = None, + ): + self.id = id + self.method = method + self.server_url = server_url + self.path = path + self.summary = summary + self.description = description + self.params = params + self.request_body = request_body + + """ + Fills in this RestApiOperation's parameters and payload with the provided values + :param path_params: A dictionary of path parameters + :param query_params: A dictionary of query parameters + :param headers: A dictionary of headers + :param request_body: The payload of the request + :return: A PreparedRestApiRequest object + """ + + def prepare_request( + self, path_params=None, query_params=None, headers=None, request_body=None + ) -> PreparedRestApiRequest: + path = self.path + if path_params: + path = path.format(**path_params) + + url = urljoin(self.server_url, path) + + processed_query_params, processed_headers = {}, {} + for param in self.params: + param_name = param["name"] + param_schema = param["schema"] + param_default = param_schema.get("default", None) + + if param["in"] == "query": + if query_params and param_name in query_params: + processed_query_params[param_name] = query_params[param_name] + elif param["schema"] and "default" in param["schema"] is not None: + processed_query_params[param_name] = param_default + elif param["in"] == "header": + if headers and param_name in headers: + processed_headers[param_name] = headers[param_name] + elif param_default is not None: + processed_headers[param_name] = param_default + elif param["in"] == "path": + if not path_params or param_name not in path_params: + raise ValueError( + f"Required path parameter {param_name} not provided" + ) + + processed_payload = None + if self.request_body: + if ( + request_body is None + and "required" in self.request_body + and self.request_body["required"] + ): + raise ValueError("Payload is required but was not provided") + content = self.request_body["content"] + content_type = list(content.keys())[0] + processed_headers["Content-Type"] = content_type + processed_payload = request_body + + req = PreparedRestApiRequest( + method=self.method, + url=url, + params=processed_query_params, + headers=processed_headers, + request_body=processed_payload, + ) + return req + + def __repr__(self): + return ( + "RestApiOperation(" + f"id={self.id}, " + f"method={self.method}, " + f"server_url={self.server_url}, " + f"path={self.path}, " + f"params={self.params}, " + f"request_body={self.request_body}, " + f"summary={self.summary}, " + f"description={self.description})" + ) + + +class OpenApiParser: + def __init__(self, logger: logging.Logger = NullLogger()): + self.logger = logger + + """ + Import an OpenAPI file. + :param openapi_file: The path to the OpenAPI file which can be local or a URL. + :return: The parsed OpenAPI file + """ + + def parse(self, openapi_document): + parser = ResolvingParser(openapi_document) + return parser.specification + + """ + Creates a RestApiOperation object for each path/method combination + :param parsed_document: The parsed OpenAPI document + :return: A dictionary of RestApiOperation objects keyed by operationId + """ + + def create_rest_api_operations( + self, parsed_document + ) -> Dict[str, RestApiOperation]: + paths = parsed_document.get("paths", {}) + request_objects = {} + for path, methods in paths.items(): + for method, details in methods.items(): + server_url = parsed_document.get("servers", []) + server_url = server_url[0].get("url") if server_url else "/" + + request_method = method.lower() + + parameters = details.get("parameters", []) + operationId = details.get("operationId", path + "_" + request_method) + summary = details.get("summary", None) + description = details.get("description", None) + + rest_api_operation = RestApiOperation( + id=operationId, + method=request_method, + server_url=server_url, + path=path, + params=parameters, + request_body=details.get("requestBody", None), + summary=summary, + description=description, + ) + + request_objects[operationId] = rest_api_operation + return request_objects + + +class OpenApiRunner: + def __init__( + self, + parsed_openapi_document: Mapping[str, str], + logger: logging.Logger = NullLogger(), + ): + self.logger = logger + self.spec = Spec.from_dict(parsed_openapi_document) + + async def run_operation( + self, + operation: RestApiOperation, + path_params: Optional[Dict[str, str]] = None, + query_params: Optional[Dict[str, str]] = None, + headers: Optional[Dict[str, str]] = None, + request_body: Optional[Union[str, Dict[str, str]]] = None, + ) -> aiohttp.ClientResponse: + prepared_request = operation.prepare_request( + path_params=path_params, + query_params=query_params, + headers=headers, + request_body=request_body, + ) + is_valid = prepared_request.validate_request(spec=self.spec, logger=self.logger) + if not is_valid: + return None + + async with aiohttp.ClientSession(raise_for_status=True) as session: + async with session.request( + prepared_request.method, + prepared_request.url, + params=prepared_request.params, + headers=prepared_request.headers, + json=prepared_request.request_body, + ) as response: + return await response.text() + + +""" +Registers a skill with the kernel that can run OpenAPI operations. +:param kernel: The kernel to register the skill with +:param skill_name: The name of the skill +:param openapi_document: The OpenAPI document to register. Can be a filename or URL +:return: A dictionary of SKFunctions keyed by operationId +""" + + +def register_openapi_skill( + kernel: Kernel, + skill_name: str, + openapi_document: str, +) -> Dict[str, SKFunctionBase]: + parser = OpenApiParser(logger=kernel.logger) + parsed_doc = parser.parse(openapi_document) + operations = parser.create_rest_api_operations(parsed_doc) + openapi_runner = OpenApiRunner( + parsed_openapi_document=parsed_doc, logger=kernel.logger + ) + + skill = {} + + def create_run_operation_function( + runner: OpenApiRunner, operation: RestApiOperation + ): + @sk_function( + description=operation.summary + if operation.summary + else operation.description, + name=operation_id, + ) + @sk_function_context_parameter( + name="path_params", description="A dictionary of path parameters" + ) + @sk_function_context_parameter( + name="query_params", description="A dictionary of query parameters" + ) + @sk_function_context_parameter( + name="headers", description="A dictionary of headers" + ) + @sk_function_context_parameter( + name="request_body", description="A dictionary of the request body" + ) + async def run_openapi_operation(sk_context: SKContext) -> str: + path_params = sk_context.variables.get("path_params") + query_params = sk_context.variables.get("query_params") + headers = sk_context.variables.get("headers") + request_body = sk_context.variables.get("request_body") + + response = await runner.run_operation( + operation, + path_params=json.loads(path_params) if path_params else None, + query_params=json.loads(query_params) if query_params else None, + headers=json.loads(headers) if headers else None, + request_body=json.loads(request_body) if request_body else None, + ) + return response + + return run_openapi_operation + + for operation_id, operation in operations.items(): + kernel.logger.info( + f"Registering OpenAPI operation: {skill_name}.{operation_id}" + ) + skill[operation_id] = create_run_operation_function(openapi_runner, operation) + return kernel.import_skill(skill, skill_name) diff --git a/python/semantic_kernel/connectors/search_engine/__init__.py b/python/semantic_kernel/connectors/search_engine/__init__.py index dc09a678650f..b7439f69c110 100644 --- a/python/semantic_kernel/connectors/search_engine/__init__.py +++ b/python/semantic_kernel/connectors/search_engine/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft. All rights reserved. + from semantic_kernel.connectors.search_engine.bing_connector import BingConnector +from semantic_kernel.connectors.search_engine.google_connector import GoogleConnector -__all__ = ["BingConnector"] +__all__ = ["BingConnector", "GoogleConnector"] diff --git a/python/semantic_kernel/connectors/search_engine/bing_connector.py b/python/semantic_kernel/connectors/search_engine/bing_connector.py index 3bc6c8805434..2e03a03f8c67 100644 --- a/python/semantic_kernel/connectors/search_engine/bing_connector.py +++ b/python/semantic_kernel/connectors/search_engine/bing_connector.py @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft. All rights reserved. + import urllib from logging import Logger from typing import List, Optional diff --git a/python/semantic_kernel/connectors/search_engine/connector.py b/python/semantic_kernel/connectors/search_engine/connector.py index 7b8857b1679b..16d34b245f6c 100644 --- a/python/semantic_kernel/connectors/search_engine/connector.py +++ b/python/semantic_kernel/connectors/search_engine/connector.py @@ -1,7 +1,12 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import List + + class ConnectorBase: """ Base class for search engine connectors """ - def search_async(self, query: str, num_results: str, offset: str) -> str: + def search_async(self, query: str, num_results: str, offset: str) -> List[str]: pass diff --git a/python/semantic_kernel/connectors/search_engine/google_connector.py b/python/semantic_kernel/connectors/search_engine/google_connector.py new file mode 100644 index 000000000000..771df6622acd --- /dev/null +++ b/python/semantic_kernel/connectors/search_engine/google_connector.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft. All rights reserved. + +import urllib +from logging import Logger +from typing import List, Optional + +import aiohttp + +from semantic_kernel.connectors.search_engine.connector import ConnectorBase +from semantic_kernel.utils.null_logger import NullLogger + + +class GoogleConnector(ConnectorBase): + """ + A search engine connector that uses the Google Custom Search API to perform a web search. + """ + + _api_key: str + _search_engine_id: str + _logger: Logger + + def __init__( + self, api_key: str, search_engine_id: str, logger: Optional[Logger] = None + ) -> None: + self._api_key = api_key + self._search_engine_id = search_engine_id + self._logger = logger if logger else NullLogger() + + if not self._api_key: + raise ValueError("Google Custom Search API key cannot be null.") + + if not self._search_engine_id: + raise ValueError("Google search engine ID cannot be null.") + + async def search_async( + self, query: str, num_results: str, offset: str + ) -> List[str]: + """ + Returns the search results of the query provided by pinging the Google Custom search API. + Returns `num_results` results and ignores the first `offset`. + + :param query: search query + :param num_results: the number of search results to return + :param offset: the number of search results to ignore + :return: list of search results + """ + if not query: + raise ValueError("query cannot be 'None' or empty.") + + if not num_results: + num_results = 1 + if not offset: + offset = 0 + + num_results = int(num_results) + offset = int(offset) + + if num_results <= 0: + raise ValueError("num_results value must be greater than 0.") + if num_results > 10: + raise ValueError("num_results value must be less than or equal to 10.") + + if offset < 0: + raise ValueError("offset must be greater than 0.") + + self._logger.info( + f"Received request for google search with \ + params:\nquery: {query}\nnum_results: {num_results}\noffset: {offset}" + ) + + _base_url = "https://www.googleapis.com/customsearch/v1" + _request_url = ( + f"{_base_url}?q={urllib.parse.quote_plus(query)}" + f"&key={self._api_key}&cx={self._search_engine_id}" + f"&num={num_results}&start={offset}" + ) + + self._logger.info("Sending GET request to Google Search API.") + + async with aiohttp.ClientSession() as session: + async with session.get(_request_url, raise_for_status=True) as response: + if response.status == 200: + data = await response.json() + self._logger.info("Request successful.") + self._logger.info(f"API Response: {data}") + items = data["items"] + result = [x["snippet"] for x in items] + return result + else: + self._logger.error( + f"Request to Google Search API failed with status code: {response.status}." + ) + return [] diff --git a/python/semantic_kernel/core_skills/conversation_summary_skill.py b/python/semantic_kernel/core_skills/conversation_summary_skill.py index fce2dd4c81c3..6e03bbed09e8 100644 --- a/python/semantic_kernel/core_skills/conversation_summary_skill.py +++ b/python/semantic_kernel/core_skills/conversation_summary_skill.py @@ -1,10 +1,9 @@ # Copyright (c) Microsoft. All rights reserved. +from typing import TYPE_CHECKING -from semantic_kernel.kernel import Kernel -from semantic_kernel.orchestration.sk_context import SKContext -from semantic_kernel.skill_definition import sk_function -from semantic_kernel.text import text_chunker -from semantic_kernel.text.function_extension import aggregate_chunked_results_async +if TYPE_CHECKING: + from semantic_kernel.kernel import Kernel + from semantic_kernel.orchestration.sk_context import SKContext class ConversationSummarySkill: @@ -12,25 +11,29 @@ class ConversationSummarySkill: Semantic skill that enables conversations summarization. """ + from semantic_kernel.skill_definition import sk_function + # The max tokens to process in a single semantic function call. _max_tokens = 1024 _summarize_conversation_prompt_template = ( - "BEGIN CONTENT TO SUMMARIZE:\n" - "{{" + "$INPUT" + "}}\n" - "END CONTENT TO SUMMARIZE.\n" - "Summarize the conversation in 'CONTENT TO SUMMARIZE',\ - identifying main points of discussion and any conclusions that were reached.\n" - "Do not incorporate other general knowledge.\n" - "Summary is in plain text, in complete sentences, with no markup or tags.\n" - "\nBEGIN SUMMARY:\n" + "BEGIN CONTENT TO SUMMARIZE:\n{{" + + "$INPUT" + + "}}\nEND CONTENT TO SUMMARIZE.\nSummarize the conversation in 'CONTENT TO" + " SUMMARIZE', identifying main points of discussion and any" + " conclusions that were reached.\nDo not incorporate other general" + " knowledge.\nSummary is in plain text, in complete sentences, with no markup" + " or tags.\n\nBEGIN SUMMARY:\n" ) - def __init__(self, kernel: Kernel): + def __init__(self, kernel: "Kernel"): self._summarizeConversationFunction = kernel.create_semantic_function( ConversationSummarySkill._summarize_conversation_prompt_template, skill_name=ConversationSummarySkill.__name__, - description="Given a section of a conversation transcript, summarize the part of the conversation.", + description=( + "Given a section of a conversation transcript, summarize the part of" + " the conversation." + ), max_tokens=ConversationSummarySkill._max_tokens, temperature=0.1, top_p=0.5, @@ -42,8 +45,8 @@ def __init__(self, kernel: Kernel): input_description="A long conversation transcript.", ) async def summarize_conversation_async( - self, input: str, context: SKContext - ) -> SKContext: + self, input: str, context: "SKContext" + ) -> "SKContext": """ Given a long conversation transcript, summarize the conversation. @@ -51,6 +54,11 @@ async def summarize_conversation_async( :param context: The SKContext for function execution. :return: SKContext with the summarized conversation result. """ + from semantic_kernel.text import text_chunker + from semantic_kernel.text.function_extension import ( + aggregate_chunked_results_async, + ) + lines = text_chunker._split_text_lines( input, ConversationSummarySkill._max_tokens, True ) diff --git a/python/semantic_kernel/core_skills/file_io_skill.py b/python/semantic_kernel/core_skills/file_io_skill.py index 250af9307864..c2842b468aaa 100644 --- a/python/semantic_kernel/core_skills/file_io_skill.py +++ b/python/semantic_kernel/core_skills/file_io_skill.py @@ -1,14 +1,18 @@ # Copyright (c) Microsoft. All rights reserved. import os +import typing as t import aiofiles -from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter +if t.TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext -class FileIOSkill: + +class FileIOSkill(PydanticField): """ Description: Read and write from a file. @@ -51,7 +55,7 @@ async def read_async(self, path: str) -> str: ) @sk_function_context_parameter(name="path", description="Destination path") @sk_function_context_parameter(name="content", description="File content") - async def write_async(self, context: SKContext): + async def write_async(self, context: "SKContext") -> None: """ Write a file @@ -60,17 +64,12 @@ async def write_async(self, context: SKContext): Args: Contains the 'path' for the Destination file and the 'content' of the file to write. - - Returns: - The contents of the file """ - has_path, path = context.variables.get("path") - has_content, content = context.variables.get("content") + path = context.variables.get("path") + content = context.variables.get("content") - assert has_path, "Path is required" - assert has_content, "Content is required" - assert content is not None, "Content is required and should not be empty" - assert path is not None, "Path is required and should not be empty" + assert path, "Path is required" + assert content, "Content is required" async with aiofiles.open(path, "w") as fp: await fp.write(content) diff --git a/python/semantic_kernel/core_skills/http_skill.py b/python/semantic_kernel/core_skills/http_skill.py index a3df91254e7f..43ee46106c5f 100644 --- a/python/semantic_kernel/core_skills/http_skill.py +++ b/python/semantic_kernel/core_skills/http_skill.py @@ -1,14 +1,18 @@ # Copyright (c) Microsoft. All rights reserved. import json +import typing as t import aiohttp -from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter +if t.TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext -class HttpSkill: + +class HttpSkill(PydanticField): """ A skill that provides HTTP functionality. @@ -42,7 +46,7 @@ async def get_async(self, url: str) -> str: @sk_function(description="Makes a POST request to a uri", name="postAsync") @sk_function_context_parameter(name="body", description="The body of the request") - async def post_async(self, url: str, context: SKContext) -> str: + async def post_async(self, url: str, context: "SKContext") -> str: """ Sends an HTTP POST request to the specified URI and returns the response body as a string. @@ -55,7 +59,7 @@ async def post_async(self, url: str, context: SKContext) -> str: if not url: raise ValueError("url cannot be `None` or empty") - _, body = context.variables.get("body") + body = context.variables.get("body") headers = {"Content-Type": "application/json"} data = json.dumps(body) @@ -67,7 +71,7 @@ async def post_async(self, url: str, context: SKContext) -> str: @sk_function(description="Makes a PUT request to a uri", name="putAsync") @sk_function_context_parameter(name="body", description="The body of the request") - async def put_async(self, url: str, context: SKContext) -> str: + async def put_async(self, url: str, context: "SKContext") -> str: """ Sends an HTTP PUT request to the specified URI and returns the response body as a string. @@ -79,7 +83,7 @@ async def put_async(self, url: str, context: SKContext) -> str: if not url: raise ValueError("url cannot be `None` or empty") - _, body = context.variables.get("body") + body = context.variables.get("body") headers = {"Content-Type": "application/json"} data = json.dumps(body) diff --git a/python/semantic_kernel/core_skills/math_skill.py b/python/semantic_kernel/core_skills/math_skill.py index 9c17ddaa9347..d0533feadbd8 100644 --- a/python/semantic_kernel/core_skills/math_skill.py +++ b/python/semantic_kernel/core_skills/math_skill.py @@ -1,10 +1,14 @@ # Copyright (c) Microsoft. All rights reserved. +import typing as t -from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter +if t.TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext -class MathSkill: + +class MathSkill(PydanticField): """ Description: MathSkill provides a set of functions to make Math calculations. @@ -23,8 +27,10 @@ class MathSkill: @sk_function_context_parameter( name="Amount", description="Amount to add", + type="number", + required=True, ) - def add(self, initial_value_text: str, context: SKContext) -> str: + def add(self, initial_value_text: str, context: "SKContext") -> str: """ Returns the Addition result of initial and amount values provided. @@ -42,8 +48,10 @@ def add(self, initial_value_text: str, context: SKContext) -> str: @sk_function_context_parameter( name="Amount", description="Amount to subtract", + type="number", + required=True, ) - def subtract(self, initial_value_text: str, context: SKContext) -> str: + def subtract(self, initial_value_text: str, context: "SKContext") -> str: """ Returns the difference of numbers provided. @@ -54,7 +62,9 @@ def subtract(self, initial_value_text: str, context: SKContext) -> str: return MathSkill.add_or_subtract(initial_value_text, context, add=False) @staticmethod - def add_or_subtract(initial_value_text: str, context: SKContext, add: bool) -> str: + def add_or_subtract( + initial_value_text: str, context: "SKContext", add: bool + ) -> str: """ Helper function to perform addition or subtraction based on the add flag. @@ -76,7 +86,8 @@ def add_or_subtract(initial_value_text: str, context: SKContext, add: bool) -> s amount = int(context_amount) except ValueError: raise ValueError( - f"Context amount provided is not in numeric format: {context_amount}" + "Context amount provided is not in numeric format:" + f" {context_amount}" ) result = initial_value + amount if add else initial_value - amount diff --git a/python/semantic_kernel/core_skills/text_memory_skill.py b/python/semantic_kernel/core_skills/text_memory_skill.py index 1c478d4fe37b..7248386d6679 100644 --- a/python/semantic_kernel/core_skills/text_memory_skill.py +++ b/python/semantic_kernel/core_skills/text_memory_skill.py @@ -1,15 +1,22 @@ # Copyright (c) Microsoft. All rights reserved. +import json +import typing as t -from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter +if t.TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext -class TextMemorySkill: + +class TextMemorySkill(PydanticField): COLLECTION_PARAM = "collection" RELEVANCE_PARAM = "relevance" KEY_PARAM = "key" + LIMIT_PARAM = "limit" DEFAULT_COLLECTION = "generic" DEFAULT_RELEVANCE = 0.75 + DEFAULT_LIMIT = 1 # @staticmethod @sk_function( @@ -27,7 +34,12 @@ class TextMemorySkill: description="The relevance score, from 0.0 to 1.0; 1.0 means perfect match", default_value=DEFAULT_RELEVANCE, ) - async def recall_async(self, ask: str, context: SKContext) -> str: + @sk_function_context_parameter( + name=LIMIT_PARAM, + description="The maximum number of relevant memories to recall.", + default_value=DEFAULT_LIMIT, + ) + async def recall_async(self, ask: str, context: "SKContext") -> str: """ Recall a fact from the long term memory. @@ -38,41 +50,50 @@ async def recall_async(self, ask: str, context: SKContext) -> str: Args: ask -- The question to ask the memory context -- Contains the 'collection' to search for information - and the 'relevance' score to use when searching + , the 'relevance' score to use when searching + and the 'limit' of relevant memories to retrieve. Returns: - The nearest item from the memory store + The nearest item from the memory store as a string or empty string if not found. """ + if context.variables is None: - raise ValueError("Context has no variables") + raise ValueError( + "The context doesn't have the variables required to know how to recall memory" + ) if context.memory is None: - raise ValueError("Context has no memory") + raise ValueError("The context doesn't have a memory instance to search") - collection = ( - context.variables[TextMemorySkill.COLLECTION_PARAM] - if context.variables.contains_key(TextMemorySkill.COLLECTION_PARAM) - else TextMemorySkill.DEFAULT_COLLECTION + collection = context.variables.get( + TextMemorySkill.COLLECTION_PARAM, TextMemorySkill.DEFAULT_COLLECTION ) if not collection: raise ValueError("Memory collection not defined for TextMemorySkill") - relevance = ( - context.variables[TextMemorySkill.RELEVANCE_PARAM] - if context.variables.contains_key(TextMemorySkill.RELEVANCE_PARAM) - else TextMemorySkill.DEFAULT_RELEVANCE + relevance = context.variables.get( + TextMemorySkill.RELEVANCE_PARAM, TextMemorySkill.DEFAULT_RELEVANCE ) - if relevance is None or str(relevance).strip() == "": - relevance = TextMemorySkill.DEFAULT_RELEVANCE + if not relevance: + raise ValueError("Relevance value not defined for TextMemorySkill") + + limit = context.variables.get( + TextMemorySkill.LIMIT_PARAM, TextMemorySkill.DEFAULT_LIMIT + ) + if limit is None or str(limit).strip() == "": + raise ValueError("Limit value not defined for TextMemorySkill") results = await context.memory.search_async( - collection, ask, min_relevance_score=float(relevance) + collection=collection, + query=ask, + limit=int(limit), + min_relevance_score=float(relevance), ) if results is None or len(results) == 0: if context.log is not None: context.log.warning(f"Memory not found in collection: {collection}") return "" - return results[0].text if results[0].text is not None else "" + return results[0].text if limit == 1 else json.dumps([r.text for r in results]) @sk_function( description="Save information to semantic memory", @@ -88,7 +109,7 @@ async def recall_async(self, ask: str, context: SKContext) -> str: name=KEY_PARAM, description="The unique key to associate with the information", ) - async def save_async(self, text: str, context: SKContext): + async def save_async(self, text: str, context: "SKContext") -> None: """ Save a fact to the long term memory. @@ -102,24 +123,21 @@ async def save_async(self, text: str, context: SKContext): context -- Contains the 'collection' to save the information and unique 'key' to associate with the information """ + if context.variables is None: - raise ValueError("Context has no variables") + raise ValueError( + "The context doesn't have the variables required to know how to recall memory" + ) if context.memory is None: - raise ValueError("Context has no memory") + raise ValueError("The context doesn't have a memory instance to search") - collection = ( - context.variables[TextMemorySkill.COLLECTION_PARAM] - if context.variables.contains_key(TextMemorySkill.COLLECTION_PARAM) - else TextMemorySkill.DEFAULT_COLLECTION + collection = context.variables.get( + TextMemorySkill.COLLECTION_PARAM, TextMemorySkill.DEFAULT_COLLECTION ) if not collection: raise ValueError("Memory collection not defined for TextMemorySkill") - key = ( - context.variables[TextMemorySkill.KEY_PARAM] - if context.variables.contains_key(TextMemorySkill.KEY_PARAM) - else None - ) + key = context.variables.get(TextMemorySkill.KEY_PARAM, None) if not key: raise ValueError("Memory key not defined for TextMemorySkill") diff --git a/python/semantic_kernel/core_skills/text_skill.py b/python/semantic_kernel/core_skills/text_skill.py index 9e463caaca44..4a4bd7fc6a3c 100644 --- a/python/semantic_kernel/core_skills/text_skill.py +++ b/python/semantic_kernel/core_skills/text_skill.py @@ -1,9 +1,10 @@ # Copyright (c) Microsoft. All rights reserved. +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition import sk_function -class TextSkill: +class TextSkill(PydanticField): """ TextSkill provides a set of functions to manipulate strings. diff --git a/python/semantic_kernel/core_skills/time_skill.py b/python/semantic_kernel/core_skills/time_skill.py index 227ff5286d95..94914831dbef 100644 --- a/python/semantic_kernel/core_skills/time_skill.py +++ b/python/semantic_kernel/core_skills/time_skill.py @@ -2,10 +2,11 @@ import datetime +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition import sk_function -class TimeSkill: +class TimeSkill(PydanticField): """ Description: TimeSkill provides a set of functions to get the current time and date. diff --git a/python/semantic_kernel/core_skills/wait_skill.py b/python/semantic_kernel/core_skills/wait_skill.py index 8f4ca28fbd95..c4cd79b75a08 100644 --- a/python/semantic_kernel/core_skills/wait_skill.py +++ b/python/semantic_kernel/core_skills/wait_skill.py @@ -2,10 +2,11 @@ import asyncio +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition import sk_function -class WaitSkill: +class WaitSkill(PydanticField): """ WaitSkill provides a set of functions to wait for a certain amount of time. @@ -17,7 +18,7 @@ class WaitSkill: """ @sk_function(description="Wait for a certain number of seconds.") - async def wait(self, seconds_text: str): + async def wait(self, seconds_text: str) -> None: try: seconds = max(float(seconds_text), 0) except ValueError: diff --git a/python/semantic_kernel/core_skills/web_search_engine_skill.py b/python/semantic_kernel/core_skills/web_search_engine_skill.py index 8012435648f5..1b6d40cb2c3c 100644 --- a/python/semantic_kernel/core_skills/web_search_engine_skill.py +++ b/python/semantic_kernel/core_skills/web_search_engine_skill.py @@ -1,7 +1,11 @@ +import typing as t + from semantic_kernel.connectors.search_engine.connector import ConnectorBase -from semantic_kernel.orchestration.sk_context import SKContext from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter +if t.TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext + class WebSearchEngineSkill: """ @@ -36,7 +40,7 @@ def __init__(self, connector: "ConnectorBase") -> None: description="The number of search results to skip", default_value="0", ) - async def search_async(self, query: str, context: SKContext) -> str: + async def search_async(self, query: str, context: "SKContext") -> str: """ Returns the search results of the query provided. Returns `num_results` results and ignores the first `offset`. @@ -46,7 +50,7 @@ async def search_async(self, query: str, context: SKContext) -> str: :return: stringified list of search results """ - _, _num_results = context.variables.get("num_results") - _, _offset = context.variables.get("offset") + _num_results = context.variables.get("num_results") + _offset = context.variables.get("offset") result = await self._connector.search_async(query, _num_results, _offset) return str(result) diff --git a/python/semantic_kernel/kernel.py b/python/semantic_kernel/kernel.py index 236464aa3c79..9a94bbbfe527 100644 --- a/python/semantic_kernel/kernel.py +++ b/python/semantic_kernel/kernel.py @@ -5,7 +5,7 @@ import inspect import os from logging import Logger -from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union, cast +from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union from uuid import uuid4 from semantic_kernel.connectors.ai.ai_exception import AIException @@ -35,7 +35,6 @@ PassThroughWithoutRetry, ) from semantic_kernel.reliability.retry_mechanism_base import RetryMechanismBase -from semantic_kernel.semantic_functions.chat_prompt_template import ChatPromptTemplate from semantic_kernel.semantic_functions.prompt_template import PromptTemplate from semantic_kernel.semantic_functions.prompt_template_config import ( PromptTemplateConfig, @@ -134,6 +133,39 @@ def register_semantic_function( return function + def register_native_function( + self, + skill_name: Optional[str], + sk_function: Callable, + ) -> SKFunctionBase: + if not hasattr(sk_function, "__sk_function__"): + raise KernelException( + KernelException.ErrorCodes.InvalidFunctionType, + "sk_function argument must be decorated with @sk_function", + ) + function_name = sk_function.__sk_function_name__ + + if skill_name is None or skill_name == "": + skill_name = SkillCollection.GLOBAL_SKILL + assert skill_name is not None # for type checker + + validate_skill_name(skill_name) + validate_function_name(function_name) + + function = SKFunction.from_native_method(sk_function, skill_name, self.logger) + + if self.skills.has_function(skill_name, function_name): + raise KernelException( + KernelException.ErrorCodes.FunctionOverloadNotSupported, + "Overloaded functions are not supported, " + "please differentiate function names.", + ) + + function.set_default_skill_collection(self.skills) + self._skill_collection.add_native_function(function) + + return function + async def run_stream_async( self, *functions: Any, @@ -152,17 +184,19 @@ async def run_stream_async( elif len(functions) == 1: stream_function = functions[0] + + # TODO: Preparing context for function invoke can be refactored as code below are same as run_async # if the user passed in a context, prioritize it, but merge with any other inputs if input_context is not None: context = input_context if input_vars is not None: - context._variables = input_vars.merge_or_overwrite( - new_vars=context._variables, overwrite=False + context.variables = input_vars.merge_or_overwrite( + new_vars=context.variables, overwrite=False ) if input_str is not None: - context._variables = ContextVariables(input_str).merge_or_overwrite( - new_vars=context._variables, overwrite=False + context.variables = ContextVariables(input_str).merge_or_overwrite( + new_vars=context.variables, overwrite=False ) # if the user did not pass in a context, prioritize an input string, @@ -189,54 +223,24 @@ async def run_stream_async( raise ValueError("No functions passed to run") try: - client: ChatCompletionClientBase | TextCompletionClientBase - client = stream_function._ai_service - - # Get the closure variables from function for finding function_config - closure_vars = stream_function._function.__closure__ - for var in closure_vars: - if isinstance(var.cell_contents, SemanticFunctionConfig): - function_config = var.cell_contents - break - - if function_config.has_chat_prompt: - as_chat_prompt = cast( - ChatPromptTemplate, function_config.prompt_template - ) - - # Similar to non-chat, render prompt (which renders to a - # list of messages) - completion = "" - messages = await as_chat_prompt.render_messages_async(context) - async for steam_message in client.complete_chat_stream_async( - messages, stream_function._chat_request_settings - ): - completion += steam_message - yield steam_message - - # Add the last message from the rendered chat prompt - # (which will be the user message) and the response - # from the model (the assistant message) - _, content = messages[-1] - as_chat_prompt.add_user_message(content) - as_chat_prompt.add_assistant_message(completion) - - # Update context - context.variables.update(completion) - - else: - completion = "" - prompt = await function_config.prompt_template.render_async(context) - async for stream_message in client.complete_stream_async( - prompt, stream_function._ai_request_settings - ): - completion += stream_message - yield stream_message - context.variables.update(completion) - - except Exception as e: + completion = "" + async for stream_message in stream_function.invoke_stream_async( + input=None, context=context + ): + completion += stream_message + yield stream_message + + except Exception as ex: # TODO: "critical exceptions" - context.fail(str(e), e) + self._log.error( + "Something went wrong in stream function. During function invocation:" + f" '{stream_function.skill_name}.{stream_function.name}'. Error" + f" description: '{str(ex)}'" + ) + raise KernelException( + KernelException.ErrorCodes.FunctionInvokeError, + "Error occurred while invoking stream function", + ) async def run_async( self, @@ -244,18 +248,19 @@ async def run_async( input_context: Optional[SKContext] = None, input_vars: Optional[ContextVariables] = None, input_str: Optional[str] = None, + **kwargs: Dict[str, Any], ) -> SKContext: # if the user passed in a context, prioritize it, but merge with any other inputs if input_context is not None: context = input_context if input_vars is not None: - context._variables = input_vars.merge_or_overwrite( - new_vars=context._variables, overwrite=False + context.variables = input_vars.merge_or_overwrite( + new_vars=context.variables, overwrite=False ) if input_str is not None: - context._variables = ContextVariables(input_str).merge_or_overwrite( - new_vars=context._variables, overwrite=False + context.variables = ContextVariables(input_str).merge_or_overwrite( + new_vars=context.variables, overwrite=False ) # if the user did not pass in a context, prioritize an input string, @@ -296,7 +301,7 @@ async def run_async( pipeline_step += 1 try: - context = await func.invoke_async(input=None, context=context) + context = await func.invoke_async(input=None, context=context, **kwargs) if context.error_occurred: self._log.error( @@ -351,9 +356,11 @@ def register_memory(self, memory: SemanticTextMemoryBase) -> None: def register_memory_store(self, memory_store: MemoryStoreBase) -> None: self.use_memory(memory_store) - def create_new_context(self) -> SKContext: + def create_new_context( + self, variables: Optional[ContextVariables] = None + ) -> SKContext: return SKContext( - ContextVariables(), + ContextVariables() if not variables else variables, self._memory, self.skills, self._log, @@ -369,8 +376,13 @@ def import_skill( self._log.debug(f"Importing skill {skill_name}") functions = [] + + if isinstance(skill_instance, dict): + candidates = skill_instance.items() + else: + candidates = inspect.getmembers(skill_instance, inspect.ismethod) # Read every method from the skill instance - for _, candidate in inspect.getmembers(skill_instance, inspect.ismethod): + for _, candidate in candidates: # If the method is a semantic function, register it if not hasattr(candidate, "__sk_function__"): continue @@ -386,8 +398,10 @@ def import_skill( if len(function_names) != len(set(function_names)): raise KernelException( KernelException.ErrorCodes.FunctionOverloadNotSupported, - "Overloaded functions are not supported, " - "please differentiate function names.", + ( + "Overloaded functions are not supported, " + "please differentiate function names." + ), ) skill = {} @@ -669,9 +683,11 @@ def _create_semantic_function( if service is None: raise AIException( AIException.ErrorCodes.InvalidConfiguration, - "Could not load chat service, unable to prepare semantic function. " - "Function description: " - "{function_config.prompt_template_config.description}", + ( + "Could not load chat service, unable to prepare semantic" + " function. Function description:" + " {function_config.prompt_template_config.description}" + ), ) function.set_chat_service(lambda: service(self)) @@ -692,9 +708,11 @@ def _create_semantic_function( if service is None: raise AIException( AIException.ErrorCodes.InvalidConfiguration, - "Could not load text service, unable to prepare semantic function. " - "Function description: " - "{function_config.prompt_template_config.description}", + ( + "Could not load text service, unable to prepare semantic" + " function. Function description:" + " {function_config.prompt_template_config.description}" + ), ) function.set_ai_service(lambda: service(self)) @@ -719,26 +737,22 @@ def import_native_skill_from_directory( ) skill_name = os.path.basename(skill_directory) - try: - spec = importlib.util.spec_from_file_location( - MODULE_NAME, native_py_file_path - ) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - class_name = next( - ( - name - for name, cls in inspect.getmembers(module, inspect.isclass) - if cls.__module__ == MODULE_NAME - ), - None, - ) - if class_name: - skill_obj = getattr(module, class_name)() - return self.import_skill(skill_obj, skill_name) - except Exception: - pass + spec = importlib.util.spec_from_file_location(MODULE_NAME, native_py_file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + class_name = next( + ( + name + for name, cls in inspect.getmembers(module, inspect.isclass) + if cls.__module__ == MODULE_NAME + ), + None, + ) + if class_name: + skill_obj = getattr(module, class_name)() + return self.import_skill(skill_obj, skill_name) return {} diff --git a/python/semantic_kernel/memory/memory_store_base.py b/python/semantic_kernel/memory/memory_store_base.py index c8b43d50602b..0b11b34b9358 100644 --- a/python/semantic_kernel/memory/memory_store_base.py +++ b/python/semantic_kernel/memory/memory_store_base.py @@ -16,54 +16,145 @@ async def __aexit__(self, *args): await self.close_async() async def close_async(self): + """Async close connection, invoked by MemoryStoreBase.__aexit__()""" pass @abstractmethod async def create_collection_async(self, collection_name: str) -> None: + """Creates a new collection in the data store. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + + Returns: + None + """ pass @abstractmethod async def get_collections_async( self, ) -> List[str]: + """Gets all collection names in the data store. + + Returns: + List[str] -- A group of collection names. + """ pass @abstractmethod async def delete_collection_async(self, collection_name: str) -> None: + """Deletes a collection from the data store. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + + Returns: + None + """ pass @abstractmethod async def does_collection_exist_async(self, collection_name: str) -> bool: + """Determines if a collection exists in the data store. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + + Returns: + bool -- True if given collection exists, False if not. + """ + pass @abstractmethod async def upsert_async(self, collection_name: str, record: MemoryRecord) -> str: + """Upserts a memory record into the data store. Does not guarantee that the collection exists. + If the record already exists, it will be updated. + If the record does not exist, it will be created. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + record {MemoryRecord} -- The memory record to upsert. + + Returns: + str -- The unique identifier for the memory record. + """ pass @abstractmethod async def upsert_batch_async( self, collection_name: str, records: List[MemoryRecord] ) -> List[str]: + """Upserts a group of memory records into the data store. Does not guarantee that the collection exists. + If the record already exists, it will be updated. + If the record does not exist, it will be created. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + records {MemoryRecord} -- The memory records to upsert. + + Returns: + List[str] -- The unique identifiers for the memory records. + """ pass @abstractmethod async def get_async( self, collection_name: str, key: str, with_embedding: bool ) -> MemoryRecord: + """Gets a memory record from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + key {str} -- The unique id associated with the memory record to get. + with_embedding {bool} -- If true, the embedding will be returned in the memory record. + + Returns: + MemoryRecord -- The memory record if found + """ pass @abstractmethod async def get_batch_async( self, collection_name: str, keys: List[str], with_embeddings: bool ) -> List[MemoryRecord]: + """Gets a batch of memory records from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + keys {List[str]} -- The unique ids associated with the memory records to get. + with_embeddings {bool} -- If true, the embedding will be returned in the memory records. + + Returns: + List[MemoryRecord] -- The memory records associated with the unique keys provided. + """ pass @abstractmethod async def remove_async(self, collection_name: str, key: str) -> None: + """Removes a memory record from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + key {str} -- The unique id associated with the memory record to remove. + + Returns: + None + """ pass @abstractmethod async def remove_batch_async(self, collection_name: str, keys: List[str]) -> None: + """Removes a batch of memory records from the data store. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + keys {List[str]} -- The unique ids associated with the memory records to remove. + + Returns: + None + """ pass @abstractmethod @@ -75,6 +166,19 @@ async def get_nearest_matches_async( min_relevance_score: float, with_embeddings: bool, ) -> List[Tuple[MemoryRecord, float]]: + """Gets the nearest matches to an embedding of type float. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + embedding {ndarray} -- The embedding to compare the collection's embeddings with. + limit {int} -- The maximum number of similarity results to return. + min_relevance_score {float} -- The minimum relevance threshold for returned results. + with_embeddings {bool} -- If true, the embeddings will be returned in the memory records. + + Returns: + List[Tuple[MemoryRecord, float]] -- A list of tuples where item1 is a MemoryRecord and item2 + is its similarity score as a float. + """ pass @abstractmethod @@ -85,4 +189,15 @@ async def get_nearest_match_async( min_relevance_score: float, with_embedding: bool, ) -> Tuple[MemoryRecord, float]: + """Gets the nearest match to an embedding of type float. Does not guarantee that the collection exists. + + Arguments: + collection_name {str} -- The name associated with a collection of embeddings. + embedding {ndarray} -- The embedding to compare the collection's embeddings with. + min_relevance_score {float} -- The minimum relevance threshold for returned result. + with_embedding {bool} -- If true, the embeddings will be returned in the memory record. + + Returns: + Tuple[MemoryRecord, float] -- A tuple consisting of the MemoryRecord and the similarity score as a float. + """ pass diff --git a/python/semantic_kernel/memory/null_memory.py b/python/semantic_kernel/memory/null_memory.py index 27ba99d52bc6..94c2ef0232cf 100644 --- a/python/semantic_kernel/memory/null_memory.py +++ b/python/semantic_kernel/memory/null_memory.py @@ -15,6 +15,7 @@ async def save_information_async( description: Optional[str] = None, additional_metadata: Optional[str] = None, ) -> None: + """Nullifies behavior of SemanticTextMemoryBase.save_information_async()""" return None async def save_reference_async( @@ -26,11 +27,13 @@ async def save_reference_async( description: Optional[str] = None, additional_metadata: Optional[str] = None, ) -> None: + """Nullifies behavior of SemanticTextMemoryBase.save_reference_async()""" return None async def get_async( self, collection: str, query: str ) -> Optional[MemoryQueryResult]: + """Nullifies behavior of SemanticTextMemoryBase.get_async()""" return None async def search_async( @@ -40,9 +43,11 @@ async def search_async( limit: int = 1, min_relevance_score: float = 0.7, ) -> List[MemoryQueryResult]: + """Nullifies behavior of SemanticTextMemoryBase.search_async()""" return [] async def get_collections_async(self) -> List[str]: + """Nullifies behavior of SemanticTextMemoryBase.get_collections_async()""" return [] diff --git a/python/semantic_kernel/memory/semantic_text_memory_base.py b/python/semantic_kernel/memory/semantic_text_memory_base.py index 616580a31ada..2027b6813126 100644 --- a/python/semantic_kernel/memory/semantic_text_memory_base.py +++ b/python/semantic_kernel/memory/semantic_text_memory_base.py @@ -1,12 +1,15 @@ # Copyright (c) Microsoft. All rights reserved. -from abc import ABC, abstractmethod -from typing import List, Optional +from abc import abstractmethod +from typing import List, Optional, TypeVar from semantic_kernel.memory.memory_query_result import MemoryQueryResult +from semantic_kernel.sk_pydantic import PydanticField +SemanticTextMemoryT = TypeVar("SemanticTextMemoryT", bound="SemanticTextMemoryBase") -class SemanticTextMemoryBase(ABC): + +class SemanticTextMemoryBase(PydanticField): @abstractmethod async def save_information_async( self, @@ -17,6 +20,17 @@ async def save_information_async( additional_metadata: Optional[str] = None, # TODO: ctoken? ) -> None: + """Save information to the memory (calls the memory store's upsert method). + + Arguments: + collection {str} -- The collection to save the information to. + text {str} -- The text to save. + id {str} -- The id of the information. + description {Optional[str]} -- The description of the information. + + Returns: + None -- None. + """ pass @abstractmethod @@ -29,6 +43,18 @@ async def save_reference_async( description: Optional[str] = None, additional_metadata: Optional[str] = None, ) -> None: + """Save a reference to the memory (calls the memory store's upsert method). + + Arguments: + collection {str} -- The collection to save the reference to. + text {str} -- The text to save. + external_id {str} -- The external id of the reference. + external_source_name {str} -- The external source name of the reference. + description {Optional[str]} -- The description of the reference. + + Returns: + None -- None. + """ pass @abstractmethod @@ -36,7 +62,17 @@ async def get_async( self, collection: str, query: str, + # TODO: with_embedding: bool, ) -> Optional[MemoryQueryResult]: + """Get information from the memory (calls the memory store's get method). + + Arguments: + collection {str} -- The collection to get the information from. + key {str} -- The key of the information. + + Returns: + Optional[MemoryQueryResult] -- The MemoryQueryResult if found, None otherwise. + """ pass @abstractmethod @@ -48,8 +84,25 @@ async def search_async( min_relevance_score: float = 0.7, # TODO: ctoken? ) -> List[MemoryQueryResult]: + """Search the memory (calls the memory store's get_nearest_matches method). + + Arguments: + collection {str} -- The collection to search in. + query {str} -- The query to search for. + limit {int} -- The maximum number of results to return. (default: {1}) + min_relevance_score {float} -- The minimum relevance score to return. (default: {0.0}) + with_embeddings {bool} -- Whether to return the embeddings of the results. (default: {False}) + + Returns: + List[MemoryQueryResult] -- The list of MemoryQueryResult found. + """ pass @abstractmethod async def get_collections_async(self) -> List[str]: + """Get the list of collections in the memory (calls the memory store's get_collections method). + + Returns: + List[str] -- The list of all the memory collection names. + """ pass diff --git a/python/semantic_kernel/models/chat/chat_message.py b/python/semantic_kernel/models/chat/chat_message.py new file mode 100644 index 000000000000..1f132e730f6e --- /dev/null +++ b/python/semantic_kernel/models/chat/chat_message.py @@ -0,0 +1,40 @@ +"""Class to hold chat messages.""" +from typing import TYPE_CHECKING, Dict, Optional + +from pydantic import Field + +from semantic_kernel.semantic_functions.prompt_template import PromptTemplate +from semantic_kernel.sk_pydantic import SKBaseModel + +if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext + + +class ChatMessage(SKBaseModel): + """Class to hold chat messages.""" + + role: Optional[str] = "assistant" + fixed_content: Optional[str] = Field(default=None, init=False, alias="content") + content_template: Optional[PromptTemplate] = Field( + default=None, init=True, repr=False + ) + + @property + def content(self) -> Optional[str]: + """Return the content of the message.""" + return self.fixed_content + + async def render_message_async(self, context: "SKContext") -> None: + """Render the message. + The first time this is called for a given message, + it will render the message with the context at that time. + Subsequent calls will do nothing. + """ + if self.fixed_content is None: + self.fixed_content = await self.content_template.render_async(context) + + def as_dict(self) -> Dict[str, str]: + """Return the message as a dict. + Make sure to call render_message_async first to embed the context in the content. + """ + return self.dict(exclude_none=True, by_alias=True, exclude={"content_template"}) diff --git a/python/semantic_kernel/orchestration/context_variables.py b/python/semantic_kernel/orchestration/context_variables.py index 8367fd8f189f..8f7767b5776f 100644 --- a/python/semantic_kernel/orchestration/context_variables.py +++ b/python/semantic_kernel/orchestration/context_variables.py @@ -1,76 +1,129 @@ # Copyright (c) Microsoft. All rights reserved. +from typing import Dict, Optional +import pydantic as pdt -from typing import Dict, Tuple +from semantic_kernel.sk_pydantic import SKBaseModel -class ContextVariables: - def __init__(self, content: str = "", variables: Dict[str, str] = None) -> None: +class ContextVariables(SKBaseModel): + """Class for the context variables, maintains a dict with keys and values for the variables. + The keys are all converted to lower case, both in setting and in getting. + + Uses `input` as the main key for the content string, that value can be extracted by using: + - the `input` property + - context_variables_instance['input'] + - context_variables_instance.get('input') + - str(context_variables_instance) + + The value for that is always a string, and is set to "" if not provided and can be updated by using: + - the `update` method + - context_variables_instance['input'] = "new value" + - context_variables_instance.set("input", "new value") + """ + + variables: Dict[str, str] = pdt.Field(default_factory=dict) + _main_key: str = pdt.PrivateAttr(default="input") + + def __init__( + self, content: Optional[str] = None, variables: Optional[Dict[str, str]] = {} + ) -> None: """ Initialize the ContextVariables instance with an optional content string. + The value of the content variable has precedence over the value of the 'input' key in the variables dictionary. + If the 'input' key is not in the variables dictionary. + :param content: The content string to be stored as the main variable, defaults to an empty string. + :param variables: The variables dictionary, defaults to an empty dictionary. """ - self._variables: Dict[str, str] = variables or {} - self._main_key: str = "input" - self._variables[self._main_key] = content + super().__init__(variables=variables) + if self._main_key not in self.variables or content is not None: + self.variables[self._main_key] = content if content else "" @property def input(self) -> str: - return self._variables[self._main_key] + """Returns the 'input' field.""" + return self.variables[self._main_key] def update(self, content: str) -> "ContextVariables": - self._variables[self._main_key] = content + """Updates the 'input' field with the given content.""" + self.variables[self._main_key] = content return self def merge_or_overwrite( self, new_vars: "ContextVariables", overwrite: bool = False ) -> "ContextVariables": + """Merge or overwrite the current variables with the new variables. + + Arguments: + new_vars {ContextVariables} -- The new variables. + overwrite {bool} -- If True, overwrite the current variables with the new variables, + otherwise merge the new variables with the current variables. + Defaults to False. + + Returns: + ContextVariables -- The current instance with updated variables. + """ if overwrite: - self._variables = new_vars._variables - else: - self._variables.update(new_vars._variables) + self.variables = new_vars.variables + return self + self.variables.update(new_vars.variables) return self - def set(self, name: str, value: str) -> "ContextVariables": + def set(self, name: str, value: Optional[str]) -> "ContextVariables": + """Set a variable value by name. + + If the value is None, the variable is removed. + + Returns: + ContextVariables -- The current instance with updated variables. + """ if not name: raise ValueError("The variable name cannot be `None` or empty") - name = name.lower() - if value is not None: - self._variables[name] = value - else: - self._variables.pop(name, None) - + self.variables[name.lower()] = value + return self + self.variables.pop(name.lower(), None) return self - def get(self, name: str) -> Tuple[bool, str]: - name = name.lower() - if name in self._variables: - return True, self._variables[name] + def get(self, name: str, value: Optional[str] = None) -> Optional[str]: + """Get a variable value by name, or return a default value if not found. - return False, "" + Arguments: + name {str} -- The variable name. + value {Optional[str]} -- The value to return if the variable is not found. + Defaults to None. - def __getitem__(self, name: str) -> str: + Returns: + Optional[str] -- The variable value, or the default value if not found. + """ name = name.lower() - return self._variables[name] + if name in self.variables: + return self.variables[name] + return value + + def __getitem__(self, name: str) -> str: + return self.variables[name.lower()] def __setitem__(self, name: str, value: str) -> None: if not name: raise ValueError("The variable name cannot be `None` or empty") - name = name.lower() - self._variables[name] = value + self.variables[name.lower()] = value - def contains_key(self, name: str) -> bool: - name = name.lower() - return name in self._variables + def __delitem__(self, name: str) -> None: + del self.variables[name.lower()] + + def __contains__(self, name: str) -> bool: + return name.lower() in self.variables def __str__(self) -> str: - return self._variables[self._main_key] + return self.variables[self._main_key] def clone(self) -> "ContextVariables": - main_content = self._variables.get(self._main_key, "") - new_vars = ContextVariables(main_content) - new_vars._variables = self._variables.copy() - return new_vars + """Create a clone of this instance. + + If the `input` key is empty, the value is set to "". + """ + return ContextVariables(variables=self.variables.copy()) diff --git a/python/semantic_kernel/orchestration/delegate_handlers.py b/python/semantic_kernel/orchestration/delegate_handlers.py index c4320eb38fe3..b745ee5eb874 100644 --- a/python/semantic_kernel/orchestration/delegate_handlers.py +++ b/python/semantic_kernel/orchestration/delegate_handlers.py @@ -3,6 +3,7 @@ from semantic_kernel.kernel_exception import KernelException from semantic_kernel.orchestration.delegate_types import DelegateTypes +from semantic_kernel.sk_pydantic import PydanticField def _handles(delegate_type): @@ -13,7 +14,7 @@ def decorator(function): return decorator -class DelegateHandlers: +class DelegateHandlers(PydanticField): @staticmethod @_handles(DelegateTypes.Void) async def handle_void(function, context): diff --git a/python/semantic_kernel/orchestration/delegate_inference.py b/python/semantic_kernel/orchestration/delegate_inference.py index c3f3e636060a..da59c50df21f 100644 --- a/python/semantic_kernel/orchestration/delegate_inference.py +++ b/python/semantic_kernel/orchestration/delegate_inference.py @@ -5,7 +5,7 @@ from semantic_kernel.kernel_exception import KernelException from semantic_kernel.orchestration.delegate_types import DelegateTypes -from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.sk_pydantic import PydanticField def _infers(delegate_type): @@ -16,6 +16,15 @@ def decorator(function): return decorator +def _is_annotation_of_type(annotation, type_to_match) -> bool: + return (annotation is type_to_match) or ( + # Handle cases where the annotation is provided as a string to avoid circular imports + # for example: `async def read_async(self, context: "SKContext"):` in file_io_skill.py + isinstance(annotation, str) + and annotation == type_to_match.__name__ + ) + + def _has_no_params(signature: Signature) -> bool: return len(signature.parameters) == 0 @@ -25,7 +34,13 @@ def _return_is_str(signature: Signature) -> bool: def _return_is_context(signature: Signature) -> bool: - return signature.return_annotation is SKContext + from semantic_kernel.orchestration.sk_context import SKContext + + return _is_annotation_of_type(signature.return_annotation, SKContext) + + +def _return_is_none(signature: Signature) -> bool: + return signature.return_annotation is None def _no_return(signature: Signature) -> bool: @@ -41,14 +56,16 @@ def _has_first_param_with_type( return False first_param = list(signature.parameters.values())[0] - return first_param.annotation is annotation + return _is_annotation_of_type(first_param.annotation, annotation) def _has_two_params_second_is_context(signature: Signature) -> bool: + from semantic_kernel.orchestration.sk_context import SKContext + if len(signature.parameters) < 2: return False second_param = list(signature.parameters.values())[1] - return second_param.annotation is SKContext + return _is_annotation_of_type(second_param.annotation, SKContext) def _first_param_is_str(signature: Signature, only: bool = True) -> bool: @@ -56,15 +73,17 @@ def _first_param_is_str(signature: Signature, only: bool = True) -> bool: def _first_param_is_context(signature: Signature) -> bool: + from semantic_kernel.orchestration.sk_context import SKContext + return _has_first_param_with_type(signature, SKContext) -class DelegateInference: +class DelegateInference(PydanticField): @staticmethod @_infers(DelegateTypes.Void) def infer_void(signature: Signature, awaitable: bool) -> bool: matches = _has_no_params(signature) - matches = matches and _no_return(signature) + matches = matches and _return_is_none(signature) matches = matches and not awaitable return matches @@ -88,7 +107,7 @@ def infer_out_task_string(signature: Signature, awaitable: bool) -> bool: @_infers(DelegateTypes.InSKContext) def infer_in_sk_context(signature: Signature, awaitable: bool) -> bool: matches = _first_param_is_context(signature) - matches = matches and _no_return(signature) + matches = matches and _return_is_none(signature) matches = matches and not awaitable return matches @@ -124,7 +143,7 @@ def infer_context_switch_in_sk_context_out_task_sk_context( @_infers(DelegateTypes.InString) def infer_in_string(signature: Signature, awaitable: bool) -> bool: matches = _first_param_is_str(signature) - matches = matches and _no_return(signature) + matches = matches and _return_is_none(signature) matches = matches and not awaitable return matches @@ -149,7 +168,7 @@ def infer_in_string_out_task_string(signature: Signature, awaitable: bool) -> bo def infer_in_string_and_context(signature: Signature, awaitable: bool) -> bool: matches = _first_param_is_str(signature, only=False) matches = matches and _has_two_params_second_is_context(signature) - matches = matches and _no_return(signature) + matches = matches and _return_is_none(signature) matches = matches and not awaitable return matches @@ -190,7 +209,7 @@ def infer_context_switch_in_string_and_context_out_task_context( @_infers(DelegateTypes.InStringOutTask) def infer_in_string_out_task(signature: Signature, awaitable: bool) -> bool: matches = _first_param_is_str(signature) - matches = matches and _no_return(signature) + matches = matches and _return_is_none(signature) matches = matches and awaitable return matches @@ -198,7 +217,7 @@ def infer_in_string_out_task(signature: Signature, awaitable: bool) -> bool: @_infers(DelegateTypes.InContextOutTask) def infer_in_context_out_task(signature: Signature, awaitable: bool) -> bool: matches = _first_param_is_context(signature) - matches = matches and _no_return(signature) + matches = matches and _return_is_none(signature) matches = matches and awaitable return matches @@ -209,7 +228,7 @@ def infer_in_string_and_context_out_task( ) -> bool: matches = _first_param_is_str(signature, only=False) matches = matches and _has_two_params_second_is_context(signature) - matches = matches and _no_return(signature) + matches = matches and _return_is_none(signature) matches = matches and awaitable return matches @@ -217,7 +236,6 @@ def infer_in_string_and_context_out_task( @_infers(DelegateTypes.OutTask) def infer_out_task(signature: Signature, awaitable: bool) -> bool: matches = _has_no_params(signature) - matches = matches and _no_return(signature) matches = matches and awaitable return matches @@ -226,13 +244,21 @@ def infer_out_task(signature: Signature, awaitable: bool) -> bool: def infer_unknown(signature: Signature, awaitable: bool) -> NoReturn: raise KernelException( KernelException.ErrorCodes.FunctionTypeNotSupported, - "Invalid function type detected, unable to infer DelegateType.", + "Invalid function type detected, unable to infer DelegateType." + + f" Function: {signature}", ) @staticmethod def infer_delegate_type(function) -> DelegateTypes: # Get the function signature function_signature = signature(function) + + if _no_return(function_signature): + raise KernelException( + KernelException.ErrorCodes.FunctionTypeNotSupported, + "No return type specified, unable to infer DelegateType.", + ) + awaitable = iscoroutinefunction(function) for name, value in DelegateInference.__dict__.items(): diff --git a/python/semantic_kernel/orchestration/sk_context.py b/python/semantic_kernel/orchestration/sk_context.py index 8eaaa0cf9911..d8a0e5018fbb 100644 --- a/python/semantic_kernel/orchestration/sk_context.py +++ b/python/semantic_kernel/orchestration/sk_context.py @@ -1,33 +1,46 @@ # Copyright (c) Microsoft. All rights reserved. from logging import Logger -from typing import Any, Literal, Optional, Tuple, Union +from typing import Any, Dict, Generic, Literal, Optional, Tuple, Union + +import pydantic as pdt from semantic_kernel.kernel_exception import KernelException -from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryBase +from semantic_kernel.memory.semantic_text_memory_base import ( + SemanticTextMemoryBase, + SemanticTextMemoryT, +) from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.sk_pydantic import SKGenericModel +from semantic_kernel.skill_definition.read_only_skill_collection import ( + ReadOnlySkillCollection, +) from semantic_kernel.skill_definition.read_only_skill_collection_base import ( ReadOnlySkillCollectionBase, ) -class SKContext: +class SKContext(SKGenericModel, Generic[SemanticTextMemoryT]): """Semantic Kernel context.""" - _error_occurred: bool = False - _last_exception: Optional[Exception] = None - _last_error_description: str = "" - _logger: Logger - _memory: SemanticTextMemoryBase - _skill_collection: ReadOnlySkillCollectionBase - _variables: ContextVariables + memory: SemanticTextMemoryT + variables: ContextVariables + # This field can be used to hold anything that is not a string + skill_collection: ReadOnlySkillCollection = pdt.Field( + default_factory=ReadOnlySkillCollection + ) + _objects: Dict[str, Any] = pdt.PrivateAttr(default_factory=dict) + _error_occurred: bool = pdt.PrivateAttr(False) + _last_exception: Optional[Exception] = pdt.PrivateAttr(None) + _last_error_description: str = pdt.PrivateAttr("") + _logger: Logger = pdt.PrivateAttr() def __init__( self, variables: ContextVariables, memory: SemanticTextMemoryBase, - skill_collection: ReadOnlySkillCollectionBase, - logger: Logger, + skill_collection: Union[ReadOnlySkillCollection, None], + logger: Optional[Logger] = None, # TODO: cancellation token? ) -> None: """ @@ -39,10 +52,16 @@ def __init__( skill_collection {ReadOnlySkillCollectionBase} -- The skill collection. logger {Logger} -- The logger. """ - self._variables = variables - self._memory = memory - self._skill_collection = skill_collection - self._logger = logger + # Local import to avoid circular dependency + from semantic_kernel import NullLogger + + if skill_collection is None: + skill_collection = ReadOnlySkillCollection() + + super().__init__( + variables=variables, memory=memory, skill_collection=skill_collection + ) + self._logger = logger or NullLogger() def fail(self, error_description: str, exception: Optional[Exception] = None): """ @@ -69,7 +88,7 @@ def result(self) -> str: Returns: str -- Processed input, aka result. """ - return str(self._variables) + return str(self.variables) @property def error_occurred(self) -> bool: @@ -102,24 +121,14 @@ def last_exception(self) -> Optional[Exception]: return self._last_exception @property - def variables(self) -> ContextVariables: - """ - User variables. - - Returns: - ContextVariables -- The context variables. - """ - return self._variables - - @property - def memory(self) -> SemanticTextMemoryBase: + def objects(self) -> Dict[str, Any]: """ - The semantic text memory. + The objects dictionary. Returns: - SemanticTextMemoryBase -- The semantic text memory. + Dict[str, Any] -- The objects dictionary. """ - return self._memory + return self._objects @property def skills(self) -> ReadOnlySkillCollectionBase: @@ -129,14 +138,14 @@ def skills(self) -> ReadOnlySkillCollectionBase: Returns: ReadOnlySkillCollectionBase -- The skills collection. """ - return self._skill_collection + return self.skill_collection @skills.setter def skills(self, value: ReadOnlySkillCollectionBase) -> None: """ Set the value of skills collection """ - self._skill_collection = value + self.skill_collection = value @property def log(self) -> Logger: @@ -156,7 +165,7 @@ def __setitem__(self, key: str, value: Any) -> None: key {str} -- The variable name. value {Any} -- The variable value. """ - self._variables[key] = value + self.variables[key] = value def __getitem__(self, key: str) -> Any: """ @@ -168,7 +177,7 @@ def __getitem__(self, key: str) -> Any: Returns: Any -- The variable value. """ - return self._variables[key] + return self.variables[key] def func(self, skill_name: str, function_name: str): """ @@ -183,14 +192,14 @@ def func(self, skill_name: str, function_name: str): Returns: SKFunctionBase -- The function. """ - if self._skill_collection is None: + if self.skill_collection is None: raise ValueError("The skill collection hasn't been set") - assert self._skill_collection is not None # for type checker + assert self.skill_collection is not None # for type checker - if self._skill_collection.has_native_function(skill_name, function_name): - return self._skill_collection.get_native_function(skill_name, function_name) + if self.skill_collection.has_native_function(skill_name, function_name): + return self.skill_collection.get_native_function(skill_name, function_name) - return self._skill_collection.get_semantic_function(skill_name, function_name) + return self.skill_collection.get_semantic_function(skill_name, function_name) def __str__(self) -> str: if self._error_occurred: @@ -202,7 +211,7 @@ def throw_if_skill_collection_not_set(self) -> None: """ Throws an exception if the skill collection hasn't been set. """ - if self._skill_collection is None: + if self.skill_collection is None: raise KernelException( KernelException.ErrorCodes.SkillCollectionNotSet, "Skill collection not found in the context", @@ -223,20 +232,20 @@ def is_function_registered( whether the function is registered and the function itself (or None). """ self.throw_if_skill_collection_not_set() - assert self._skill_collection is not None # for type checker + assert self.skill_collection is not None # for type checker - if self._skill_collection.has_native_function(skill_name, function_name): - the_func = self._skill_collection.get_native_function( + if self.skill_collection.has_native_function(skill_name, function_name): + the_func = self.skill_collection.get_native_function( skill_name, function_name ) return True, the_func - if self._skill_collection.has_native_function(None, function_name): - the_func = self._skill_collection.get_native_function(None, function_name) + if self.skill_collection.has_native_function(None, function_name): + the_func = self.skill_collection.get_native_function(None, function_name) return True, the_func - if self._skill_collection.has_semantic_function(skill_name, function_name): - the_func = self._skill_collection.get_semantic_function( + if self.skill_collection.has_semantic_function(skill_name, function_name): + the_func = self.skill_collection.get_semantic_function( skill_name, function_name ) return True, the_func diff --git a/python/semantic_kernel/orchestration/sk_function.py b/python/semantic_kernel/orchestration/sk_function.py index 17af6d375cfa..c39d558d0b6f 100644 --- a/python/semantic_kernel/orchestration/sk_function.py +++ b/python/semantic_kernel/orchestration/sk_function.py @@ -6,7 +6,7 @@ import threading from enum import Enum from logging import Logger -from typing import Any, Callable, List, Optional, cast +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional from semantic_kernel.connectors.ai.chat_completion_client_base import ( ChatCompletionClientBase, @@ -25,7 +25,6 @@ from semantic_kernel.orchestration.delegate_handlers import DelegateHandlers from semantic_kernel.orchestration.delegate_inference import DelegateInference from semantic_kernel.orchestration.delegate_types import DelegateTypes -from semantic_kernel.orchestration.sk_context import SKContext from semantic_kernel.orchestration.sk_function_base import SKFunctionBase from semantic_kernel.semantic_functions.chat_prompt_template import ChatPromptTemplate from semantic_kernel.semantic_functions.semantic_function_config import ( @@ -38,6 +37,9 @@ ) from semantic_kernel.utils.null_logger import NullLogger +if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext + if platform.system() == "Windows" and sys.version_info >= (3, 8, 0): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) @@ -56,6 +58,7 @@ class SKFunction(SKFunctionBase): _ai_request_settings: CompleteRequestSettings _chat_service: Optional[ChatCompletionClientBase] _chat_request_settings: ChatRequestSettings + _chat_prompt_template: ChatPromptTemplate @staticmethod def from_native_method(method, skill_name="", log=None) -> "SKFunction": @@ -75,21 +78,32 @@ def from_native_method(method, skill_name="", log=None) -> "SKFunction": parameters.append( ParameterView( - param["name"], param["description"], param["default_value"] + name=param["name"], + description=param["description"], + default_value=param["default_value"], + type=param.get("type", "string"), + required=param.get("required", False), ) ) - if hasattr(method, "__sk_function_input_description__"): + if ( + hasattr(method, "__sk_function_input_description__") + and method.__sk_function_input_description__ is not None + and method.__sk_function_input_description__ != "" + ): input_param = ParameterView( - "input", - method.__sk_function_input_description__, - method.__sk_function_input_default_value__, + name="input", + description=method.__sk_function_input_description__, + default_value=method.__sk_function_input_default_value__, + type="string", + required=False, ) parameters = [input_param] + parameters return SKFunction( delegate_type=DelegateInference.infer_delegate_type(method), delegate_function=method, + delegate_stream_function=method, parameters=parameters, description=method.__sk_function_description__, skill_name=skill_name, @@ -108,22 +122,77 @@ def from_semantic_config( if function_config is None: raise ValueError("Function configuration cannot be `None`") - async def _local_func(client, request_settings, context): + async def _local_func(client, request_settings, context: "SKContext", **kwargs): if client is None: raise ValueError("AI LLM service cannot be `None`") try: - if function_config.has_chat_prompt: - as_chat_prompt = cast( - ChatPromptTemplate, function_config.prompt_template + if not function_config.has_chat_prompt: + prompt = await function_config.prompt_template.render_async(context) + completion = await client.complete_async(prompt, request_settings) + context.variables.update(completion) + return context + except Exception as e: + # TODO: "critical exceptions" + context.fail(str(e), e) + return context + + as_chat_prompt = function_config.prompt_template + # Similar to non-chat, render prompt (which renders to a + # dict of messages) + messages = await as_chat_prompt.render_messages_async(context) + + functions = ( + kwargs.get("functions") + if request_settings.function_call is not None + else None + ) + if request_settings.function_call is not None and functions is None: + log.warning("Function call is not None, but functions is None") + try: + if functions and hasattr(client, "complete_chat_with_functions_async"): + ( + completion, + function_call, + ) = await client.complete_chat_with_functions_async( + messages, functions, request_settings ) + as_chat_prompt.add_message( + "assistant", message=completion, function_call=function_call + ) + if completion is not None: + context.variables.update(completion) + if function_call is not None: + context.objects["function_call"] = function_call + else: + completion = await client.complete_chat_async( + messages, request_settings + ) + as_chat_prompt.add_assistant_message(completion) + context.variables.update(completion) + except Exception as exc: + # TODO: "critical exceptions" + context.fail(str(exc), exc) + finally: + return context + + async def _local_stream_func(client, request_settings, context): + if client is None: + raise ValueError("AI LLM service cannot be `None`") + + try: + if function_config.has_chat_prompt: + as_chat_prompt = function_config.prompt_template # Similar to non-chat, render prompt (which renders to a # list of messages) + completion = "" messages = await as_chat_prompt.render_messages_async(context) - completion = await client.complete_chat_async( + async for steam_message in client.complete_chat_stream_async( messages, request_settings - ) + ): + completion += steam_message + yield steam_message # Add the last message from the rendered chat prompt # (which will be the user message) and the response @@ -136,23 +205,31 @@ async def _local_func(client, request_settings, context): context.variables.update(completion) else: prompt = await function_config.prompt_template.render_async(context) - completion = await client.complete_async(prompt, request_settings) + + completion = "" + async for stream_message in client.complete_stream_async( + prompt, request_settings + ): + completion += stream_message + yield stream_message context.variables.update(completion) except Exception as e: # TODO: "critical exceptions" context.fail(str(e), e) - return context - return SKFunction( delegate_type=DelegateTypes.ContextSwitchInSKContextOutTaskSKContext, delegate_function=_local_func, + delegate_stream_function=_local_stream_func, parameters=function_config.prompt_template.get_parameters(), description=function_config.prompt_template_config.description, skill_name=skill_name, function_name=function_name, is_semantic=True, log=log, + chat_prompt_template=function_config.prompt_template + if function_config.has_chat_prompt + else None, ) @property @@ -193,6 +270,8 @@ def __init__( function_name: str, is_semantic: bool, log: Optional[Logger] = None, + delegate_stream_function: Optional[Callable[..., Any]] = None, + **kwargs: Dict[str, Any], ) -> None: self._delegate_type = delegate_type self._function = delegate_function @@ -202,11 +281,13 @@ def __init__( self._name = function_name self._is_semantic = is_semantic self._log = log if log is not None else NullLogger() + self._stream_function = delegate_stream_function self._skill_collection = None self._ai_service = None self._ai_request_settings = CompleteRequestSettings() self._chat_service = None self._chat_request_settings = ChatRequestSettings() + self._chat_prompt_template = kwargs.get("chat_prompt_template", None) def set_default_skill_collection( self, skills: ReadOnlySkillCollectionBase @@ -259,11 +340,11 @@ def __call__( self, input: Optional[str] = None, variables: ContextVariables = None, - context: Optional[SKContext] = None, + context: Optional["SKContext"] = None, memory: Optional[SemanticTextMemoryBase] = None, settings: Optional[CompleteRequestSettings] = None, log: Optional[Logger] = None, - ) -> SKContext: + ) -> "SKContext": return self.invoke( input=input, variables=variables, @@ -277,11 +358,13 @@ def invoke( self, input: Optional[str] = None, variables: ContextVariables = None, - context: Optional[SKContext] = None, + context: Optional["SKContext"] = None, memory: Optional[SemanticTextMemoryBase] = None, settings: Optional[CompleteRequestSettings] = None, log: Optional[Logger] = None, - ) -> SKContext: + ) -> "SKContext": + from semantic_kernel.orchestration.sk_context import SKContext + if context is None: context = SKContext( variables=ContextVariables("") if variables is None else variables, @@ -292,11 +375,11 @@ def invoke( else: # If context is passed, we need to merge the variables if variables is not None: - context._variables = variables.merge_or_overwrite( - new_vars=context._variables, overwrite=False + context.variables = variables.merge_or_overwrite( + new_vars=context.variables, overwrite=False ) if memory is not None: - context._memory = memory + context.memory = memory if input is not None: context.variables.update(input) @@ -323,11 +406,14 @@ async def invoke_async( self, input: Optional[str] = None, variables: ContextVariables = None, - context: Optional[SKContext] = None, + context: Optional["SKContext"] = None, memory: Optional[SemanticTextMemoryBase] = None, settings: Optional[CompleteRequestSettings] = None, log: Optional[Logger] = None, - ) -> SKContext: + **kwargs: Dict[str, Any], + ) -> "SKContext": + from semantic_kernel.orchestration.sk_context import SKContext + if context is None: context = SKContext( variables=ContextVariables("") if variables is None else variables, @@ -338,25 +424,25 @@ async def invoke_async( else: # If context is passed, we need to merge the variables if variables is not None: - context._variables = variables.merge_or_overwrite( - new_vars=context._variables, overwrite=False + context.variables = variables.merge_or_overwrite( + new_vars=context.variables, overwrite=False ) if memory is not None: - context._memory = memory + context.memory = memory if input is not None: context.variables.update(input) try: if self.is_semantic: - return await self._invoke_semantic_async(context, settings) + return await self._invoke_semantic_async(context, settings, **kwargs) else: - return await self._invoke_native_async(context) + return await self._invoke_native_async(context, **kwargs) except Exception as e: context.fail(str(e), e) return context - async def _invoke_semantic_async(self, context, settings): + async def _invoke_semantic_async(self, context: "SKContext", settings, **kwargs): self._verify_is_semantic() self._ensure_context_has_skills(context) @@ -375,7 +461,9 @@ async def _invoke_semantic_async(self, context, settings): service = ( self._ai_service if self._ai_service is not None else self._chat_service ) - new_context = await self._function(service, settings, context) + new_context = await self._function( + service, settings, context, functions=kwargs.get("functions", None) + ) context.variables.merge_or_overwrite(new_context.variables) return context @@ -412,6 +500,80 @@ def _verify_is_native(self) -> None: "Invalid operation, the method requires a native function", ) + async def invoke_stream_async( + self, + input: Optional[str] = None, + variables: ContextVariables = None, + context: Optional["SKContext"] = None, + memory: Optional[SemanticTextMemoryBase] = None, + settings: Optional[CompleteRequestSettings] = None, + log: Optional[Logger] = None, + ): + from semantic_kernel.orchestration.sk_context import SKContext + + if context is None: + context = SKContext( + variables=ContextVariables("") if variables is None else variables, + skill_collection=self._skill_collection, + memory=memory if memory is not None else NullMemory.instance, + logger=log if log is not None else self._log, + ) + else: + # If context is passed, we need to merge the variables + if variables is not None: + context.variables = variables.merge_or_overwrite( + new_vars=context.variables, overwrite=False + ) + if memory is not None: + context._memory = memory + + if input is not None: + context.variables.update(input) + + try: + if self.is_semantic: + async for stream_msg in self._invoke_semantic_stream_async( + context, settings + ): + yield stream_msg + else: + async for stream_msg in self._invoke_native_stream_async(context): + yield stream_msg + except Exception as e: + context.fail(str(e), e) + raise KernelException( + KernelException.ErrorCodes.FunctionInvokeError, + "Error occurred while invoking stream function", + ) + + async def _invoke_semantic_stream_async(self, context, settings): + self._verify_is_semantic() + + self._ensure_context_has_skills(context) + + if settings is None: + if self._ai_service is not None: + settings = self._ai_request_settings + elif self._chat_service is not None: + settings = self._chat_request_settings + else: + raise KernelException( + KernelException.ErrorCodes.UnknownError, + "Semantic functions must have either an AI service or Chat service", + ) + + service = ( + self._ai_service if self._ai_service is not None else self._chat_service + ) + + async for stream_msg in self._stream_function(service, settings, context): + yield stream_msg + + async def _invoke_native_stream_async(self, context): + result = await self._invoke_native_async(context) + + yield result + def _ensure_context_has_skills(self, context) -> None: if context.skills is not None: return diff --git a/python/semantic_kernel/orchestration/sk_function_base.py b/python/semantic_kernel/orchestration/sk_function_base.py index 0b183c267bf9..1e0f0e411dd0 100644 --- a/python/semantic_kernel/orchestration/sk_function_base.py +++ b/python/semantic_kernel/orchestration/sk_function_base.py @@ -1,8 +1,8 @@ # Copyright (c) Microsoft. All rights reserved. -from abc import ABC, abstractmethod +from abc import abstractmethod from logging import Logger -from typing import TYPE_CHECKING, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional from semantic_kernel.connectors.ai.complete_request_settings import ( CompleteRequestSettings, @@ -12,16 +12,17 @@ ) from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryBase from semantic_kernel.orchestration.context_variables import ContextVariables -from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition.function_view import FunctionView if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext from semantic_kernel.skill_definition.read_only_skill_collection_base import ( ReadOnlySkillCollectionBase, ) -class SKFunctionBase(ABC): +class SKFunctionBase(PydanticField): FUNCTION_PARAM_NAME_REGEX = r"^[0-9A-Za-z_]*$" FUNCTION_NAME_REGEX = r"^[0-9A-Za-z_]*$" SKILL_NAME_REGEX = r"^[0-9A-Za-z_]*$" @@ -103,11 +104,11 @@ def invoke( self, input: Optional[str] = None, variables: ContextVariables = None, - context: Optional[SKContext] = None, + context: Optional["SKContext"] = None, memory: Optional[SemanticTextMemoryBase] = None, settings: Optional[CompleteRequestSettings] = None, log: Optional[Logger] = None, - ) -> SKContext: + ) -> "SKContext": """ Invokes the function with an explicit string input Keyword Arguments: @@ -128,11 +129,12 @@ async def invoke_async( self, input: Optional[str] = None, variables: ContextVariables = None, - context: Optional[SKContext] = None, + context: Optional["SKContext"] = None, memory: Optional[SemanticTextMemoryBase] = None, settings: Optional[CompleteRequestSettings] = None, log: Optional[Logger] = None, - ) -> SKContext: + **kwargs: Dict[str, Any], + ) -> "SKContext": """ Invokes the function with an explicit string input Keyword Arguments: diff --git a/python/semantic_kernel/planning/__init__.py b/python/semantic_kernel/planning/__init__.py index 17add29ee104..be5e35f0b72f 100644 --- a/python/semantic_kernel/planning/__init__.py +++ b/python/semantic_kernel/planning/__init__.py @@ -1,7 +1,15 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.planning.action_planner.action_planner import ActionPlanner from semantic_kernel.planning.basic_planner import BasicPlanner from semantic_kernel.planning.plan import Plan +from semantic_kernel.planning.sequential_planner import SequentialPlanner +from semantic_kernel.planning.stepwise_planner import StepwisePlanner __all__ = [ "BasicPlanner", "Plan", + "SequentialPlanner", + "StepwisePlanner", + "ActionPlanner", ] diff --git a/python/semantic_kernel/planning/action_planner/__init__.py b/python/semantic_kernel/planning/action_planner/__init__.py new file mode 100644 index 000000000000..0ebb553876f6 --- /dev/null +++ b/python/semantic_kernel/planning/action_planner/__init__.py @@ -0,0 +1,7 @@ +from semantic_kernel.planning.action_planner.action_planner import ( + ActionPlanner, +) + +__all__ = [ + "ActionPlanner", +] diff --git a/python/semantic_kernel/planning/action_planner/action_planner.py b/python/semantic_kernel/planning/action_planner/action_planner.py new file mode 100644 index 000000000000..078f2f8639dc --- /dev/null +++ b/python/semantic_kernel/planning/action_planner/action_planner.py @@ -0,0 +1,352 @@ +# Copyright (c) Microsoft. All rights reserved. + +import itertools +import json +import os +from logging import Logger +from textwrap import dedent +from typing import List, Optional + +import regex + +from semantic_kernel import Kernel +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.action_planner.action_planner_config import ( + ActionPlannerConfig, +) +from semantic_kernel.planning.plan import Plan +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.parameter_view import ParameterView +from semantic_kernel.utils.null_logger import NullLogger + + +class ActionPlanner: + """ + Action Planner allows to select one function out of many, to achieve a given goal. + The planner implements the Intent Detection pattern, uses the functions registered + in the kernel to see if there's a relevant one, providing instructions to call the + function and the rationale used to select it. The planner can also return + "no function" if nothing relevant is available. + """ + + RESTRICTED_SKILL_NAME = "ActionPlanner_Excluded" + config: ActionPlannerConfig + _stop_sequence: str = "#END-OF-PLAN" + + _planner_function: SKFunctionBase + + _kernel: Kernel + _prompt_template: str + _logger: Logger + + def __init__( + self, + kernel: Kernel, + config: Optional[ActionPlannerConfig] = None, + prompt: Optional[str] = None, + logger: Optional[Logger] = None, + ) -> None: + if kernel is None: + raise PlanningException( + PlanningException.ErrorCodes.InvalidConfiguration, + "Kernel cannot be `None`.", + ) + + self._logger = logger if logger else NullLogger() + self.config = config or ActionPlannerConfig() + + __cur_dir = os.path.dirname(os.path.abspath(__file__)) + __prompt_file = os.path.join(__cur_dir, "skprompt.txt") + + self._prompt_template = prompt if prompt else open(__prompt_file, "r").read() + + self._planner_function = kernel.create_semantic_function( + skill_name=self.RESTRICTED_SKILL_NAME, + prompt_template=self._prompt_template, + max_tokens=self.config.max_tokens, + stop_sequences=[self._stop_sequence], + ) + kernel.import_skill(self, self.RESTRICTED_SKILL_NAME) + + self._kernel = kernel + self._context = kernel.create_new_context() + + async def create_plan_async(self, goal: str) -> Plan: + """ + :param goal: The input to the planner based on which the plan is made + :return: a Plan object + """ + + if goal is None: + raise PlanningException( + PlanningException.ErrorCodes.InvalidGoal, "Goal cannot be `None`." + ) + + self._logger.info(f"Finding the best function for achieving the goal: {goal}") + + self._context.variables.update(goal) + + generated_plan_raw = await self._planner_function.invoke_async( + context=self._context + ) + generated_plan_raw_str = str(generated_plan_raw) + + if not generated_plan_raw or not generated_plan_raw_str: + self._logger.error("No plan has been generated.") + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + "No plan has been generated.", + ) + + self._logger.info(f"Plan generated by ActionPlanner:\n{generated_plan_raw_str}") + + # Ignore additional text around JSON recursively + json_regex = r"\{(?:[^{}]|(?R))*\}" + generated_plan_str = regex.search(json_regex, generated_plan_raw_str) + + if not generated_plan_str: + self._logger.error("No valid plan has been generated.") + raise PlanningException( + PlanningException.ErrorCodes.InvalidPlan, + "No valid plan has been generated.", + inner_exception=ValueError(generated_plan_raw_str), + ) + + generated_plan_str = generated_plan_str.group() + generated_plan_str = generated_plan_str.replace('""', '"') + + try: + generated_plan = json.loads(generated_plan_str) + except json.decoder.JSONDecodeError as e: + self._logger.error("Encountered an error while parsing Plan JSON.") + self._logger.error(e) + raise PlanningException( + PlanningException.ErrorCodes.InvalidPlan, + "Encountered an error while parsing Plan JSON.", + ) + + self._logger.info( + f"Python dictionary of plan generated by ActionPlanner:\n{generated_plan}" + ) + + if not generated_plan["plan"]: + self._logger.error("Suitable plan not generated by ActionPlanner.") + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + "Suitable plan not generated by ActionPlanner.", + ) + + if not generated_plan["plan"]["function"]: + # no suitable function identified, returning plan with no steps + self._logger.warn( + "No suitable function has been identified by ActionPlanner." + ) + plan = Plan(description=goal) + elif "." in generated_plan["plan"]["function"]: + skill, fun = generated_plan["plan"]["function"].split(".") + function_ref = self._context.skills.get_function(skill, fun) + self._logger.info( + f"ActionPlanner has picked {skill}.{fun}. Reference to this function" + f" found in context: {function_ref}" + ) + plan = Plan(description=goal, function=function_ref) + else: + function_ref = self._context.skills.get_function( + generated_plan["plan"]["function"] + ) + self._logger.info( + f"ActionPlanner has picked {generated_plan['plan']['function']}. " + " Reference to this function found in context:" + f" {function_ref}" + ) + plan = Plan(description=goal, function=function_ref) + + for key, val in generated_plan["plan"]["parameters"].items(): + self._logger.info(f"Parameter {key}: {val}") + if val: + plan.parameters[key] = str(val) + plan.state[key] = str(val) + + return plan + + @sk_function( + description="List a few good examples of plans to generate", name="GoodExamples" + ) + @sk_function_context_parameter( + name="goal", description="The current goal processed by the planner" + ) + def good_examples(self, goal: str, context: SKContext) -> str: + return dedent( + """ + [EXAMPLE] + - List of functions: + // Read a file. + FileIOSkill.ReadAsync + Parameter ""path"": Source file. + // Write a file. + FileIOSkill.WriteAsync + Parameter ""path"": Destination file. (default value: sample.txt) + Parameter ""content"": File content. + // Get the current time. + TimeSkill.Time + No parameters. + // Makes a POST request to a uri. + HttpSkill.PostAsync + Parameter ""body"": The body of the request. + - End list of functions. + Goal: create a file called ""something.txt"". + {""plan"":{ + ""rationale"": ""the list contains a function that allows to create files"", + ""function"": ""FileIOSkill.WriteAsync"", + ""parameters"": { + ""path"": ""something.txt"", + ""content"": null + }}} + #END-OF-PLAN + """ + ) + + @sk_function( + description="List a few edge case examples of plans to handle", + name="EdgeCaseExamples", + ) + @sk_function_context_parameter( + name="goal", description="The current goal processed by the planner" + ) + def edge_case_examples(self, goal: str, context: SKContext) -> str: + return dedent( + ''' + [EXAMPLE] + - List of functions: + // Get the current time. + TimeSkill.Time + No parameters. + // Write a file. + FileIOSkill.WriteAsync + Parameter ""path"": Destination file. (default value: sample.txt) + Parameter ""content"": File content. + // Makes a POST request to a uri. + HttpSkill.PostAsync + Parameter ""body"": The body of the request. + // Read a file. + FileIOSkill.ReadAsync + Parameter ""path"": Source file. + - End list of functions. + Goal: tell me a joke. + {""plan"":{ + ""rationale"": ""the list does not contain functions to tell jokes or something funny"", + ""function"": """", + ""parameters"": { + }}} + #END-OF-PLAN + ''' + ) + + @sk_function( + description="List all functions available in the kernel", name="ListOfFunctions" + ) + @sk_function_context_parameter( + name="goal", description="The current goal processed by the planner" + ) + def list_of_functions(self, goal: str, context: SKContext) -> str: + if context.skills is None: + raise PlanningException( + error_code=PlanningException.ErrorCodes.InvalidConfiguration, + message="Suitable plan not generated by ActionPlanner.", + inner_exception=ValueError("No plugins are available."), + ) + + functions_view = context.skills.get_functions_view() + + available_functions: List[FunctionView] = [ + *functions_view.semantic_functions.values(), + *functions_view.native_functions.values(), + ] + available_functions = itertools.chain.from_iterable(available_functions) + + available_functions = [ + self._create_function_string(func) + for func in available_functions + if ( + func.skill_name != self.RESTRICTED_SKILL_NAME + and func.skill_name not in self.config.excluded_skills + and func.name not in self.config.excluded_functions + ) + ] + + available_functions_str = "\n".join(available_functions) + + self._logger.info(f"List of available functions:\n{available_functions_str}") + + return available_functions_str + + def _create_function_string(self, function: FunctionView) -> str: + """ + Takes an instance of FunctionView and returns a string that consists of + function name, function description and parameters in the following format + // + . + Parameter """": (default value: `default_value`) + ... + + :param function: An instance of FunctionView for which the string representation needs to be generated + :return: string representation of function + """ + + if not function.description: + self._logger.warn( + f"{function.skill_name}.{function.name} is missing a description" + ) + description = f"// Function {function.skill_name}.{function.name}." + else: + description = f"// {function.description}" + + # add trailing period for description if not present + if description[-1] != ".": + description = f"{description}." + + name = f"{function.skill_name}.{function.name}" + + parameters_list = [ + result + for x in function.parameters + if (result := self._create_parameter_string(x)) is not None + ] + + if len(parameters_list) == 0: + parameters = "No parameters." + else: + parameters = "\n".join(parameters_list) + + func_str = f"{description}\n{name}\n{parameters}" + + return func_str + + def _create_parameter_string(self, parameter: ParameterView) -> str: + """ + Takes an instance of ParameterView and returns a string that consists of + parameter name, parameter description and default value for the parameter + in the following format + Parameter """": (default value: ) + + :param parameter: An instance of ParameterView for which the string representation needs to be generated + :return: string representation of parameter + """ + + name = parameter.name + description = desc if (desc := parameter.description) else name + + # add trailing period for description if not present + if description[-1] != ".": + description = f"{description}." + + default_value = ( + f"(default value: {val})" if (val := parameter.default_value) else "" + ) + + param_str = f'Parameter ""{name}"": {description} {default_value}' + + return param_str.strip() diff --git a/python/semantic_kernel/planning/action_planner/action_planner_config.py b/python/semantic_kernel/planning/action_planner/action_planner_config.py new file mode 100644 index 000000000000..fdfe62cb1aec --- /dev/null +++ b/python/semantic_kernel/planning/action_planner/action_planner_config.py @@ -0,0 +1,13 @@ +from typing import List + + +class ActionPlannerConfig: + def __init__( + self, + excluded_skills: List[str] = None, + excluded_functions: List[str] = None, + max_tokens: int = 1024, + ): + self.excluded_skills: List[str] = excluded_skills or [] + self.excluded_functions: List[str] = excluded_functions or [] + self.max_tokens: int = max_tokens diff --git a/python/semantic_kernel/planning/action_planner/skprompt.txt b/python/semantic_kernel/planning/action_planner/skprompt.txt new file mode 100644 index 000000000000..5d3378c985fb --- /dev/null +++ b/python/semantic_kernel/planning/action_planner/skprompt.txt @@ -0,0 +1,11 @@ +A planner takes a list of functions, a goal, and chooses which function to use. +For each function the list includes details about the input parameters. +[START OF EXAMPLES] +{{ActionPlanner_Excluded.GoodExamples}} +{{ActionPlanner_Excluded.EdgeCaseExamples}} +[END OF EXAMPLES] +[REAL SCENARIO STARTS HERE] +- List of functions: +{{ActionPlanner_Excluded.ListOfFunctions}} +- End list of functions. +Goal: {{ $input }} diff --git a/python/semantic_kernel/planning/basic_planner.py b/python/semantic_kernel/planning/basic_planner.py index c7d3db142940..fee7f2fdb80d 100644 --- a/python/semantic_kernel/planning/basic_planner.py +++ b/python/semantic_kernel/planning/basic_planner.py @@ -7,7 +7,22 @@ from semantic_kernel.kernel import Kernel from semantic_kernel.orchestration.context_variables import ContextVariables -from semantic_kernel.planning.plan import Plan + + +class Plan: + """A simple plan object for the Semantic Kernel""" + + def __init__(self, prompt: str, goal: str, plan: str): + self.prompt = prompt + self.goal = goal + self.generated_plan = plan + + def __str__(self): + return f"Prompt: {self.prompt}\nGoal: {self.goal}\nPlan: {self.generated_plan}" + + def __repr__(self): + return str(self) + PROMPT = """ You are a planner for the Semantic Kernel. @@ -119,8 +134,8 @@ def _create_available_functions_string(self, kernel: Kernel) -> str: string for the prompt. """ # Get a dictionary of skill names to all native and semantic functions - native_functions = kernel.skills.get_functions_view()._native_functions - semantic_functions = kernel.skills.get_functions_view()._semantic_functions + native_functions = kernel.skills.get_functions_view().native_functions + semantic_functions = kernel.skills.get_functions_view().semantic_functions native_functions.update(semantic_functions) # Create a mapping between all function names and their descriptions diff --git a/python/semantic_kernel/planning/plan.py b/python/semantic_kernel/planning/plan.py index d20217b4809d..acbb3754a981 100644 --- a/python/semantic_kernel/planning/plan.py +++ b/python/semantic_kernel/planning/plan.py @@ -4,7 +4,7 @@ import re import threading from logging import Logger -from typing import Any, Callable, List, Optional +from typing import Any, Callable, List, Optional, Union from semantic_kernel import Kernel from semantic_kernel.connectors.ai import CompleteRequestSettings @@ -12,14 +12,19 @@ TextCompletionClientBase, ) from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.memory.null_memory import NullMemory from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryBase from semantic_kernel.orchestration.context_variables import ContextVariables from semantic_kernel.orchestration.sk_context import SKContext from semantic_kernel.orchestration.sk_function_base import SKFunctionBase from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.read_only_skill_collection import ( + ReadOnlySkillCollection, +) from semantic_kernel.skill_definition.read_only_skill_collection_base import ( ReadOnlySkillCollectionBase, ) +from semantic_kernel.utils.null_logger import NullLogger class Plan(SKFunctionBase): @@ -112,6 +117,16 @@ def __init__( if function is not None: self.set_function(function) + @classmethod + def from_goal(cls, goal: str) -> "Plan": + return cls(description=goal, skill_name=cls.__name__) + + @classmethod + def from_function(cls, function: SKFunctionBase) -> "Plan": + plan = cls() + plan.set_function(function) + return plan + async def invoke_async( self, input: Optional[str] = None, @@ -121,15 +136,15 @@ async def invoke_async( logger: Optional[Logger] = None, # TODO: cancellation_token: CancellationToken, ) -> SKContext: - if input is not None: + if input is not None and input != "": self._state.update(input) if context is None: context = SKContext( variables=self._state, - skill_collection=None, - memory=memory, - logger=logger, + skill_collection=ReadOnlySkillCollection(), + memory=memory or NullMemory(), + logger=logger if logger is not None else NullLogger(), ) if self._function is not None: @@ -138,8 +153,8 @@ async def invoke_async( ) if result.error_occurred: result.log.error( - msg="Something went wrong in plan step {0}.{1}:'{2}'".format( - self._skill_name, self._name, context.last_error_description + "Something went wrong in plan step {0}.{1}:'{2}'".format( + self._skill_name, self._name, result.last_error_description ) ) return result @@ -162,14 +177,14 @@ def invoke( memory: Optional[SemanticTextMemoryBase] = None, logger: Optional[Logger] = None, ) -> SKContext: - if input is not None: + if input is not None and input != "": self._state.update(input) if context is None: context = SKContext( variables=self._state, - skill_collection=None, - memory=memory, + skill_collection=ReadOnlySkillCollection(), + memory=memory or NullMemory(), logger=logger, ) @@ -244,7 +259,7 @@ def set_available_functions(self, plan: "Plan", context: SKContext) -> "Plan": return plan - def add_steps(self, steps: Optional[List[SKFunctionBase]]) -> None: + def add_steps(self, steps: Union[List["Plan"], List[SKFunctionBase]]) -> None: for step in steps: if type(step) is Plan: self._steps.append(step) @@ -288,7 +303,7 @@ async def invoke_next_step(self, context: SKContext) -> "Plan": # Invoke the step func_context = SKContext( variables=variables, - memory=context._memory, + memory=context.memory, skill_collection=context.skills, logger=context.log, ) @@ -309,7 +324,7 @@ async def invoke_next_step(self, context: SKContext) -> "Plan": # Update plan result in state with matching outputs (if any) if set(self._outputs).intersection(set(step._outputs)): current_plan_result = "" - if Plan.DEFAULT_RESULT_KEY in self._state._variables: + if Plan.DEFAULT_RESULT_KEY in self._state.variables: current_plan_result = self._state[Plan.DEFAULT_RESULT_KEY] self._state.set( Plan.DEFAULT_RESULT_KEY, current_plan_result.strip() + result_value @@ -317,7 +332,7 @@ async def invoke_next_step(self, context: SKContext) -> "Plan": # Update state with outputs (if any) for output in step._outputs: - if output in result.variables._variables: + if output in result.variables.variables: self._state.set(output, result.variables[output]) else: self._state.set(output, result_value) @@ -330,13 +345,13 @@ async def invoke_next_step(self, context: SKContext) -> "Plan": def add_variables_to_context( self, variables: ContextVariables, context: SKContext ) -> None: - for key in variables._variables: - if not context.variables.contains_key(key): + for key in variables.variables: + if key not in context.variables: context.variables.set(key, variables[key]) def update_context_with_outputs(self, context: SKContext) -> None: result_string = "" - if Plan.DEFAULT_RESULT_KEY in self._state._variables: + if Plan.DEFAULT_RESULT_KEY in self._state.variables: result_string = self._state[Plan.DEFAULT_RESULT_KEY] else: result_string = str(self._state) @@ -361,17 +376,18 @@ def get_next_step_variables( # - Empty if sending to another plan # - Plan.Description input_string = "" - if step._parameters["input"] is not None: - input_string = self.expand_from_variables( - variables, step._parameters["input"] - ) - elif variables["input"] is not None: - input_string = variables["input"] - elif self._state["input"] is not None: - input_string = self._state["input"] + step_input_value = step._parameters.get("input") + variables_input_value = variables.get("input") + state_input_value = self._state.get("input") + if step_input_value and step_input_value != "": + input_string = self.expand_from_variables(variables, step_input_value) + elif variables_input_value and variables_input_value != "": + input_string = variables_input_value + elif state_input_value and state_input_value != "": + input_string = state_input_value elif len(step._steps) > 0: input_string = "" - elif self._description is not None: + elif self._description is not None and self._description != "": input_string = self._description step_variables = ContextVariables(input_string) @@ -379,32 +395,37 @@ def get_next_step_variables( # Priority for remaining stepVariables is: # - Function Parameters (pull from variables or state by a key value) # - Step Parameters (pull from variables or state by a key value) + # - All other variables. These are carried over in case the function wants access to the ambient content. function_params = step.describe() - for param in function_params._parameters: - if param.name.lower == "input": + for param in function_params.parameters: + if param.name.lower() == variables._main_key.lower(): continue - if step_variables.contains_key(param.name): + + if param.name in variables: step_variables.set(param.name, variables[param.name]) - elif ( - self._state.contains_key(param.name) - and self._state[param.name] is not None + elif param.name in self._state and ( + self._state[param.name] is not None and self._state[param.name] != "" ): step_variables.set(param.name, self._state[param.name]) - for param_var in step.parameters._variables: - if step_variables.contains_key(param_var): + for param_var in step.parameters.variables: + if param_var in step_variables: continue expanded_value = self.expand_from_variables(variables, param_var) if expanded_value.lower() == param_var.lower(): - step_variables.set(param_var, expanded_value) - elif variables.contains_key(param_var): + step_variables.set(param_var, step.parameters.variables[param_var]) + elif param_var in variables: step_variables.set(param_var, variables[param_var]) - elif self._state.contains_key(param_var): + elif param_var in self._state: step_variables.set(param_var, self._state[param_var]) else: step_variables.set(param_var, expanded_value) + for item in variables.variables: + if item not in step_variables: + step_variables.set(item, variables[item]) + return step_variables def expand_from_variables( @@ -412,14 +433,14 @@ def expand_from_variables( ) -> str: result = input_string variables_regex = r"\$(?P\w+)" - matches = re.findall(variables_regex, input_string) + matches = [m for m in re.finditer(variables_regex, input_string)] ordered_matches = sorted( matches, key=lambda m: len(m.group("var")), reverse=True ) for match in ordered_matches: var_name = match.group("var") - if variables.contains_key(var_name): + if var_name in variables: result = result.replace(f"${var_name}", variables[var_name]) return result diff --git a/python/semantic_kernel/planning/planning_exception.py b/python/semantic_kernel/planning/planning_exception.py new file mode 100644 index 000000000000..7cf72a10f356 --- /dev/null +++ b/python/semantic_kernel/planning/planning_exception.py @@ -0,0 +1,46 @@ +# Copyright (c) Microsoft. All rights reserved. + +from enum import Enum +from typing import Optional + + +class PlanningException(Exception): + class ErrorCodes(Enum): + # Unknown error. + UnknownError = -1 + # Invalid goal. + InvalidGoal = 0 + # Invalid plan. + InvalidPlan = 1 + # Invalid configuration. + InvalidConfiguration = 2 + # Create plan error. + CreatePlanError = 3 + + # The error code. + _error_code: ErrorCodes + + def __init__( + self, + error_code: ErrorCodes, + message: str, + inner_exception: Optional[Exception] = None, + ) -> None: + """Initializes a new instance of the PlanningError class. + + Arguments: + error_code {ErrorCodes} -- The error code. + message {str} -- The error message. + inner_exception {Exception} -- The inner exception. + """ + super().__init__(error_code, message, inner_exception) + self._error_code = error_code + + @property + def error_code(self) -> ErrorCodes: + """Gets the error code. + + Returns: + ErrorCodes -- The error code. + """ + return self._error_code diff --git a/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/config.json b/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/config.json new file mode 100644 index 000000000000..1309f85b5a1a --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/config.json @@ -0,0 +1,27 @@ +{ + "schema": 1, + "description": "Given a request or command or goal generate a step by step plan to fulfill the request using functions. This ability is also known as decision making and function flow", + "type": "completion", + "completion": { + "max_tokens": 1024, + "temperature": 0, + "top_p": 0, + "presence_penalty": 0, + "frequency_penalty": 0, + "stop_sequences": [""] + }, + "input": { + "parameters": [ + { + "name": "input", + "description": "The question to answer", + "defaultValue": "" + }, + { + "name": "available_functions", + "description": "The list of the agent's available_functions", + "defaultValue": "" + } + ] + } +} diff --git a/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/skprompt.txt b/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/skprompt.txt new file mode 100644 index 000000000000..325beca173be --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/skprompt.txt @@ -0,0 +1,55 @@ +Create an XML plan step by step, to satisfy the goal given, with the available functions. + +[AVAILABLE FUNCTIONS] + +{{$available_functions}} + +[END AVAILABLE FUNCTIONS] + +To create a plan, follow these steps: +0. The plan should be as short as possible. +1. From a create a as a series of . +2. A plan has 'INPUT' available in context variables by default. +3. Before using any function in a plan, check that it is present in the [AVAILABLE FUNCTIONS] list. If it is not, do not use it. +4. Only use functions that are required for the given goal. +5. Append an "END" XML comment at the end of the plan after the final closing tag. +6. Always output valid XML that can be parsed by an XML parser. +7. If a plan cannot be created with the [AVAILABLE FUNCTIONS], return . + +All plans take the form of: + + + + + + + + (... etc ...) + + + +To call a function, follow these steps: +1. A function has one or more named parameters and a single 'output' which are all strings. Parameter values should be xml escaped. +2. To save an 'output' from a , to pass into a future , use +3. To save an 'output' from a , to return as part of a plan result, use +4. Use a '$' to reference a context variable in a parameter, e.g. when `INPUT='world'` the parameter 'Hello $INPUT' will evaluate to `Hello world`. +5. Functions do not have access to the context variables of other functions. Do not attempt to use context variables as arrays or objects. Instead, use available functions to extract specific elements or properties from context variables. + +DO NOT DO THIS, THE PARAMETER VALUE IS NOT XML ESCAPED: + + +DO NOT DO THIS, THE PARAMETER VALUE IS ATTEMPTING TO USE A CONTEXT VARIABLE AS AN ARRAY/OBJECT: + + +Here is a valid example of how to call a function "_Function_.Name" with a single input and save its output: + + +Here is a valid example of how to call a function "FunctionName2" with a single input and return its output as part of the plan result: + + +Here is a valid example of how to call a function "Name3" with multiple inputs: + + +Begin! + +{{$input}} diff --git a/python/semantic_kernel/planning/sequential_planner/__init__.py b/python/semantic_kernel/planning/sequential_planner/__init__.py new file mode 100644 index 000000000000..1c06b014bcae --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/__init__.py @@ -0,0 +1,7 @@ +from semantic_kernel.planning.sequential_planner.sequential_planner import ( + SequentialPlanner, +) + +__all__ = [ + "SequentialPlanner", +] diff --git a/python/semantic_kernel/planning/sequential_planner/sequential_planner.py b/python/semantic_kernel/planning/sequential_planner/sequential_planner.py new file mode 100644 index 000000000000..edb850f88fdc --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/sequential_planner.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft. All rights reserved. + +import os +from typing import TYPE_CHECKING + +from semantic_kernel.kernel import Kernel +from semantic_kernel.planning.plan import Plan +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.planning.sequential_planner.sequential_planner_config import ( + SequentialPlannerConfig, +) +from semantic_kernel.planning.sequential_planner.sequential_planner_extensions import ( + SequentialPlannerSKContextExtension as SKContextExtension, +) +from semantic_kernel.planning.sequential_planner.sequential_planner_parser import ( + SequentialPlanParser, +) +from semantic_kernel.semantic_functions.prompt_template import PromptTemplate +from semantic_kernel.semantic_functions.prompt_template_config import ( + PromptTemplateConfig, +) +from semantic_kernel.semantic_functions.semantic_function_config import ( + SemanticFunctionConfig, +) + +if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext + from semantic_kernel.orchestration.sk_function_base import SKFunctionBase + +SEQUENTIAL_PLANNER_DEFAULT_DESCRIPTION = ( + "Given a request or command or goal generate a step by step plan to " + + "fulfill the request using functions. This ability is also known as decision making and function flow" +) + +CUR_DIR = os.path.dirname(os.path.realpath(__file__)) +PROMPT_CONFIG_FILE_PATH = os.path.join(CUR_DIR, "Skills/SequentialPlanning/config.json") +PROMPT_TEMPLATE_FILE_PATH = os.path.join( + CUR_DIR, "Skills/SequentialPlanning/skprompt.txt" +) + + +def read_file(file_path: str) -> str: + with open(file_path, "r") as file: + return file.read() + + +class SequentialPlanner: + RESTRICTED_SKILL_NAME = "SequentialPlanner_Excluded" + + config: SequentialPlannerConfig + _context: "SKContext" + _function_flow_function: "SKFunctionBase" + + def __init__( + self, kernel: Kernel, config: SequentialPlannerConfig = None, prompt: str = None + ): + assert isinstance(kernel, Kernel) + self.config = config or SequentialPlannerConfig() + + self.config.excluded_skills.append(self.RESTRICTED_SKILL_NAME) + + self._function_flow_function = self._init_flow_function(prompt, kernel) + + self._context = kernel.create_new_context() + + def _init_flow_function(self, prompt: str, kernel: Kernel): + prompt_config = PromptTemplateConfig.from_json( + read_file(PROMPT_CONFIG_FILE_PATH) + ) + prompt_template = prompt or read_file(PROMPT_TEMPLATE_FILE_PATH) + prompt_config.completion.max_tokens = self.config.max_tokens + + prompt_template = PromptTemplate( + template=prompt_template, + template_engine=kernel.prompt_template_engine, + prompt_config=prompt_config, + ) + function_config = SemanticFunctionConfig(prompt_config, prompt_template) + + return kernel.register_semantic_function( + skill_name=self.RESTRICTED_SKILL_NAME, + function_name=self.RESTRICTED_SKILL_NAME, + function_config=function_config, + ) + + async def create_plan_async(self, goal: str) -> Plan: + if len(goal) == 0: + raise PlanningException( + PlanningException.ErrorCodes.InvalidGoal, "The goal specified is empty" + ) + + relevant_function_manual = await SKContextExtension.get_functions_manual_async( + self._context, goal, self.config + ) + self._context.variables.set("available_functions", relevant_function_manual) + + self._context.variables.update(goal) + + plan_result = await self._function_flow_function.invoke_async( + context=self._context + ) + + if plan_result.error_occurred: + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + f"Error creating plan for goal: {plan_result.last_error_description}", + plan_result.last_exception, + ) + + plan_result_string = plan_result.result.strip() + + try: + get_skill_function = ( + self.config.get_skill_function + or SequentialPlanParser.get_skill_function(self._context) + ) + plan = SequentialPlanParser.to_plan_from_xml( + plan_result_string, + goal, + get_skill_function, + self.config.allow_missing_functions, + ) + + if len(plan._steps) == 0: + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + ( + "Not possible to create plan for goal with available functions.\n", + f"Goal:{goal}\nFunctions:\n{relevant_function_manual}", + ), + ) + + return plan + + except PlanningException as e: + if e.error_code == PlanningException.ErrorCodes.CreatePlanError: + raise e + elif e.error_code in [ + PlanningException.ErrorCodes.InvalidPlan, + PlanningException.ErrorCodes.InvalidGoal, + ]: + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + "Unable to create plan", + e, + ) + else: + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + "Unable to create plan", + e, + ) + + except Exception as e: + raise PlanningException( + PlanningException.ErrorCodes.UnknownError, + "Unknown error creating plan", + e, + ) diff --git a/python/semantic_kernel/planning/sequential_planner/sequential_planner_config.py b/python/semantic_kernel/planning/sequential_planner/sequential_planner_config.py new file mode 100644 index 000000000000..5a223c4bdb8f --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/sequential_planner_config.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import Callable, List, Optional + + +class SequentialPlannerConfig: + def __init__( + self, + relevancy_threshold: Optional[float] = None, + max_relevant_functions: int = 100, + excluded_skills: List[str] = None, + excluded_functions: List[str] = None, + included_functions: List[str] = None, + max_tokens: int = 1024, + allow_missing_functions: bool = False, + get_available_functions_async: Callable = None, + get_skill_function: Callable = None, + ): + self.relevancy_threshold: float = relevancy_threshold + self.max_relevant_functions: int = max_relevant_functions + self.excluded_skills: List[str] = excluded_skills or [] + self.excluded_functions: List[str] = excluded_functions or [] + self.included_functions: List[str] = included_functions or [] + self.max_tokens: int = max_tokens + self.allow_missing_functions: bool = allow_missing_functions + self.get_available_functions_async = get_available_functions_async + self.get_skill_function = get_skill_function diff --git a/python/semantic_kernel/planning/sequential_planner/sequential_planner_extensions.py b/python/semantic_kernel/planning/sequential_planner/sequential_planner_extensions.py new file mode 100644 index 000000000000..8efa7c2e0352 --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/sequential_planner_extensions.py @@ -0,0 +1,237 @@ +# Copyright (c) Microsoft. All rights reserved. + +import itertools +from typing import AsyncIterable, List + +from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.memory.memory_query_result import MemoryQueryResult +from semantic_kernel.memory.null_memory import NullMemory +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.planning.sequential_planner.sequential_planner_config import ( + SequentialPlannerConfig, +) +from semantic_kernel.skill_definition.function_view import FunctionView + + +class SequentialPlannerFunctionViewExtension: + @staticmethod + def to_manual_string(function: FunctionView): + inputs = [ + f" - {parameter.name}: {parameter.description}" + + ( + f" (default value: {parameter.default_value})" + if parameter.default_value + else "" + ) + for parameter in function.parameters + ] + + inputs = "\n".join(inputs) + qualified_name = SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + function + ) + + return ( + f"{qualified_name}:\n description: {function.description}\n inputs:\n " + f" {inputs}" + ) + + @staticmethod + def to_fully_qualified_name(function: FunctionView): + return f"{function.skill_name}.{function.name}" + + @staticmethod + def to_embedding_string(function: FunctionView): + inputs = "\n".join( + [ + f" - {parameter.name}: {parameter.description}" + for parameter in function.parameters + ] + ) + return ( + f"{function.name}:\n description: {function.description}\n " + f" inputs:\n{inputs}" + ) + + +class SequentialPlannerSKContextExtension: + PLANNER_MEMORY_COLLECTION_NAME = " Planning.SKFunctionManual" + PLAN_SK_FUNCTIONS_ARE_REMEMBERED = "Planning.SKFunctionsAreRemembered" + + @staticmethod + async def get_functions_manual_async( + context: SKContext, + semantic_query: str = None, + config: SequentialPlannerConfig = None, + ) -> str: + config = config or SequentialPlannerConfig() + + if config.get_available_functions_async is None: + functions = ( + await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + ) + else: + functions = await config.get_available_functions_async( + config, semantic_query + ) + + return "\n\n".join( + [ + SequentialPlannerFunctionViewExtension.to_manual_string(func) + for func in functions + ] + ) + + @staticmethod + async def get_available_functions_async( + context: SKContext, + config: SequentialPlannerConfig, + semantic_query: str = None, + ): + excluded_skills = config.excluded_skills or [] + excluded_functions = config.excluded_functions or [] + included_functions = config.included_functions or [] + + if context.skills is None: + raise KernelException( + KernelException.ErrorCodes.SkillCollectionNotSet, + "Skill collection not found in the context", + ) + + functions_view = context.skills.get_functions_view() + + available_functions: List[FunctionView] = [ + *functions_view.semantic_functions.values(), + *functions_view.native_functions.values(), + ] + available_functions = itertools.chain.from_iterable(available_functions) + + available_functions = [ + func + for func in available_functions + if ( + func.skill_name not in excluded_skills + and func.name not in excluded_functions + ) + ] + + if ( + semantic_query is None + or isinstance(context.memory, NullMemory) + or config.relevancy_threshold is None + ): + # If no semantic query is provided, return all available functions. + # If a Memory provider has not been registered, return all available functions. + return available_functions + + # Remember functions in memory so that they can be searched. + await SequentialPlannerSKContextExtension.remember_functions_async( + context, available_functions + ) + + # Search for functions that match the semantic query. + memories = await context.memory.search_async( + SequentialPlannerSKContextExtension.PLANNER_MEMORY_COLLECTION_NAME, + semantic_query, + config.max_relevant_functions, + config.relevancy_threshold, + ) + + # Add functions that were found in the search results. + relevant_functions = ( + await SequentialPlannerSKContextExtension.get_relevant_functions_async( + context, available_functions, memories + ) + ) + + # Add any missing functions that were included but not found in the search results. + missing_functions = [ + func + for func in included_functions + if func not in [func.name for func in relevant_functions] + ] + + relevant_functions += [ + func for func in available_functions if func.name in missing_functions + ] + + return sorted(relevant_functions, key=lambda x: (x.skill_name, x.name)) + + @staticmethod + async def get_relevant_functions_async( + context: SKContext, + available_functions: List[FunctionView], + memories: AsyncIterable[MemoryQueryResult], + ) -> List[FunctionView]: + relevant_functions = [] + # TODO: cancellation + async for memory_entry in memories: + function = next( + ( + func + for func in available_functions + if SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + func + ) + == memory_entry.id + ), + None, + ) + if function is not None: + context.log.debug( + "Found relevant function. Relevance Score: {0}, Function: {1}".format( + memory_entry.relevance, + SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + function + ), + ) + ) + relevant_functions.append(function) + + return relevant_functions + + @staticmethod + async def remember_functions_async( + context: SKContext, available_functions: List[FunctionView] + ): + # Check if the functions have already been saved to memory. + if ( + SequentialPlannerSKContextExtension.PLAN_SK_FUNCTIONS_ARE_REMEMBERED + in context.variables + ): + return + + for function in available_functions: + function_name = ( + SequentialPlannerFunctionViewExtension.to_fully_qualified_name(function) + ) + key = function_name + description = function.description or function_name + text_to_embed = SequentialPlannerFunctionViewExtension.to_embedding_string( + function + ) + + # It'd be nice if there were a saveIfNotExists method on the memory interface + memory_entry = await context.memory.get_async( + collection=SequentialPlannerSKContextExtension.PLANNER_MEMORY_COLLECTION_NAME, + key=key, + with_embedding=False, + ) + if memory_entry is None: + # TODO It'd be nice if the minRelevanceScore could be a parameter for each item that was saved to memory + # As folks may want to tune their functions to be more or less relevant. + # Memory now supports these such strategies. + await context.memory.save_information_async( + collection=SequentialPlannerSKContextExtension.PLANNER_MEMORY_COLLECTION_NAME, + text=text_to_embed, + id=key, + description=description, + additional_metadata="", + ) + + # Set a flag to indicate that the functions have been saved to memory. + context.variables.set( + SequentialPlannerSKContextExtension.PLAN_SK_FUNCTIONS_ARE_REMEMBERED, "true" + ) diff --git a/python/semantic_kernel/planning/sequential_planner/sequential_planner_parser.py b/python/semantic_kernel/planning/sequential_planner/sequential_planner_parser.py new file mode 100644 index 000000000000..cea7747521f3 --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/sequential_planner_parser.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft. All rights reserved. + +import re +from typing import Callable, Optional, Tuple +from xml.etree import ElementTree as ET + +from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.plan import Plan +from semantic_kernel.planning.planning_exception import PlanningException + +# Constants +GOAL_TAG = "goal" +SOLUTION_TAG = "plan" +FUNCTION_TAG = "function." +SET_CONTEXT_VARIABLE_TAG = "setContextVariable" +APPEND_TO_RESULT_TAG = "appendToResult" + + +class SequentialPlanParser: + @staticmethod + def get_skill_function( + context: SKContext, + ) -> Callable[[str, str], Optional[SKFunctionBase]]: + def function(skill_name: str, function_name: str) -> Optional[SKFunctionBase]: + try: + return context.skills.get_function(skill_name, function_name) + except KernelException: + return None + + return function + + @staticmethod + def to_plan_from_xml( + xml_string: str, + goal: str, + get_skill_function: Callable[[str, str], Optional[SKFunctionBase]], + allow_missing_functions: bool = False, + ): + xml_string = "" + xml_string + "" + try: + xml_doc = ET.fromstring(xml_string) + except ET.ParseError: + # Attempt to parse out of it + plan_regex = re.compile(r"]*>(.*?)", re.DOTALL) + match = plan_regex.search(xml_string) + + if match: + plan_xml = match.group(0) + try: + xml_doc = ET.fromstring("" + plan_xml + "") + except ET.ParseError: + raise PlanningException( + PlanningException.ErrorCodes.InvalidPlan, + f"Failed to parse plan xml strings: '{xml_string}' or '{plan_xml}'", + ) + else: + raise PlanningException( + PlanningException.ErrorCodes.InvalidPlan, + f"Failed to parse plan xml string: '{xml_string}'", + ) + + solution = xml_doc.findall(".//" + SOLUTION_TAG) + + plan = Plan.from_goal(goal) + for solution_node in solution: + for child_node in solution_node: + if child_node.tag == "#text" or child_node.tag == "#comment": + continue + + if child_node.tag.startswith(FUNCTION_TAG): + skill_function_name = child_node.tag.split(FUNCTION_TAG)[1] + ( + skill_name, + function_name, + ) = SequentialPlanParser.get_skill_function_names( + skill_function_name + ) + + if function_name: + skill_function = get_skill_function(skill_name, function_name) + + if skill_function is not None: + plan_step = Plan.from_function(skill_function) + + function_variables = ContextVariables() + function_outputs = [] + function_results = [] + + view = skill_function.describe() + for p in view.parameters: + function_variables.set(p.name, p.default_value) + + for attr in child_node.attrib: + if attr == SET_CONTEXT_VARIABLE_TAG: + function_outputs.append(child_node.attrib[attr]) + elif attr == APPEND_TO_RESULT_TAG: + function_outputs.append(child_node.attrib[attr]) + function_results.append(child_node.attrib[attr]) + else: + function_variables.set( + attr, child_node.attrib[attr] + ) + + plan_step._outputs = function_outputs + plan_step._parameters = function_variables + + for result in function_results: + plan._outputs.append(result) + + plan.add_steps([plan_step]) + elif allow_missing_functions: + plan.add_steps([Plan.from_goal(skill_function_name)]) + else: + raise PlanningException( + PlanningException.ErrorCodes.InvalidPlan, + f"Failed to find function '{skill_function_name}' in skill '{skill_name}'.", + ) + + return plan + + @staticmethod + def get_skill_function_names(skill_function_name: str) -> Tuple[str, str]: + skill_function_name_parts = skill_function_name.split(".") + skill_name = ( + skill_function_name_parts[0] if len(skill_function_name_parts) > 0 else "" + ) + function_name = ( + skill_function_name_parts[1] + if len(skill_function_name_parts) > 1 + else skill_function_name + ) + return skill_name, function_name diff --git a/python/semantic_kernel/planning/stepwise_planner/Skills/StepwiseStep/config.json b/python/semantic_kernel/planning/stepwise_planner/Skills/StepwiseStep/config.json new file mode 100644 index 000000000000..0802ff14375e --- /dev/null +++ b/python/semantic_kernel/planning/stepwise_planner/Skills/StepwiseStep/config.json @@ -0,0 +1,32 @@ +{ + "schema": 1, + "description": "Given a request or command or goal generate multi-step plan to reach the goal. After each step LLM is called to perform the reasoning for the next step.", + "type": "completion", + "completion": { + "max_tokens": 1024, + "temperature": 0, + "top_p": 0, + "presence_penalty": 0, + "frequency_penalty": 0, + "stop_sequences": ["[OBSERVATION]", "\n[THOUGHT]"] + }, + "input": { + "parameters": [ + { + "name": "question", + "description": "The question to answer", + "defaultValue": "" + }, + { + "name": "agentScratchPad", + "description": "The agent's scratch pad", + "defaultValue": "" + }, + { + "name": "functionDescriptions", + "description": "The manual of the agent's functions", + "defaultValue": "" + } + ] + } + } \ No newline at end of file diff --git a/python/semantic_kernel/planning/stepwise_planner/Skills/StepwiseStep/skprompt.txt b/python/semantic_kernel/planning/stepwise_planner/Skills/StepwiseStep/skprompt.txt new file mode 100644 index 000000000000..2c4576b30056 --- /dev/null +++ b/python/semantic_kernel/planning/stepwise_planner/Skills/StepwiseStep/skprompt.txt @@ -0,0 +1,53 @@ +[INSTRUCTION] +Answer the following questions as accurately as possible using the provided functions. + +[AVAILABLE FUNCTIONS] +The function definitions below are in the following format: +: + inputs: + - : + - ... + +{{$function_descriptions}} +[END AVAILABLE FUNCTIONS] + +[USAGE INSTRUCTIONS] +To use the functions, specify a JSON blob representing an action. The JSON blob should contain an "action" key with the name of the function to use, and an "action_variables" key with a JSON object of string values to use when calling the function. +Do not call functions directly; they must be invoked through an action. +The "action_variables" value should always include an "input" key, even if the input value is empty. Additional keys in the "action_variables" value should match the defined [PARAMETERS] of the named "action" in [AVAILABLE FUNCTIONS]. +Dictionary values in "action_variables" must be strings and represent the actual values to be passed to the function. +Ensure that the $JSON_BLOB contains only a SINGLE action; do NOT return multiple actions. +IMPORTANT: Use only the available functions listed in the [AVAILABLE FUNCTIONS] section. Do not attempt to use any other functions that are not specified. + +Here is an example of a valid $JSON_BLOB: +{ + "action": "functionName", + "action_variables": {"parameterName": "some value", ...} +} +[END USAGE INSTRUCTIONS] +[END INSTRUCTION] + +[THOUGHT PROCESS] +[QUESTION] +the input question I must answer +[THOUGHT] +To solve this problem, I should carefully analyze the given question and identify the necessary steps. Any facts I discover earlier in my thought process should be repeated here to keep them readily available. +[ACTION] +{ + "action": "functionName", + "action_variables": {"parameterName": "some value", ...} +} +[OBSERVATION] +The result of the action will be provided here. +... (These Thought/Action/Observation can repeat until the final answer is reached.) +[FINAL ANSWER] +Once I have gathered all the necessary observations and performed any required actions, I can provide the final answer in a clear and human-readable format. +[END THOUGHT PROCESS] + +Let's break down the problem step by step and think about the best approach. Questions and observations should be followed by a single thought and an optional single action to take. + +Begin! + +[QUESTION] +{{$question}} +{{$agent_scratch_pad}} \ No newline at end of file diff --git a/python/semantic_kernel/planning/stepwise_planner/__init__.py b/python/semantic_kernel/planning/stepwise_planner/__init__.py new file mode 100644 index 000000000000..7a72895111c7 --- /dev/null +++ b/python/semantic_kernel/planning/stepwise_planner/__init__.py @@ -0,0 +1,5 @@ +from semantic_kernel.planning.stepwise_planner.stepwise_planner import StepwisePlanner + +__all__ = [ + "StepwisePlanner", +] diff --git a/python/semantic_kernel/planning/stepwise_planner/stepwise_planner.py b/python/semantic_kernel/planning/stepwise_planner/stepwise_planner.py new file mode 100644 index 000000000000..660a95693309 --- /dev/null +++ b/python/semantic_kernel/planning/stepwise_planner/stepwise_planner.py @@ -0,0 +1,461 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import itertools +import json +import os +import re +from typing import TYPE_CHECKING, Dict, List + +from semantic_kernel.kernel import Kernel +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.planning.plan import Plan +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.planning.stepwise_planner.stepwise_planner_config import ( + StepwisePlannerConfig, +) +from semantic_kernel.planning.stepwise_planner.system_step import SystemStep +from semantic_kernel.semantic_functions.prompt_template import PromptTemplate +from semantic_kernel.semantic_functions.prompt_template_config import ( + PromptTemplateConfig, +) +from semantic_kernel.semantic_functions.semantic_function_config import ( + SemanticFunctionConfig, +) +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.sk_function_context_parameter_decorator import ( + sk_function_context_parameter, +) +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + +if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_function_base import SKFunctionBase + + +CUR_DIR = os.path.dirname(os.path.realpath(__file__)) +PROMPT_CONFIG_FILE_PATH = os.path.join(CUR_DIR, "Skills/StepwiseStep/config.json") +PROMPT_TEMPLATE_FILE_PATH = os.path.join(CUR_DIR, "Skills/StepwiseStep/skprompt.txt") + + +def read_file(file_path: str) -> str: + with open(file_path, "r") as file: + return file.read() + + +# TODO: Original C# uses "StepwisePlanner_Excluded" for RESTRICTED_SKILL_NAME +RESTRICTED_SKILL_NAME = "StepwisePlanner" +S_FINAL_ANSWER_REGEX = re.compile( + r"\[FINAL[_\s\-]ANSWER\](?P.+)", re.DOTALL +) +S_THOUGHT_REGEX = re.compile( + r"(\[THOUGHT\])?(?P.+?)(?=\[ACTION\]|$)", re.DOTALL +) +S_ACTION_REGEX = re.compile(r"\[ACTION\][^{}]*({(?:[^{}]*{[^{}]*})*[^{}]*})", re.DOTALL) + +ACTION = "[ACTION]" +THOUGHT = "[THOUGHT]" +OBSERVATION = "[OBSERVATION]" +SCRATCH_PAD_PREFIX = ( + "This was my previous work (but they haven't seen any of it!" + " They only see what I return as final answer):" +) + + +def is_null_or_empty(value: str) -> bool: + return value is None or value == "" + + +class StepwisePlanner: + config: StepwisePlannerConfig + _context: "SKContext" + _function_flow_function: "SKFunctionBase" + + def __init__( + self, + kernel: Kernel, + config: StepwisePlannerConfig = None, + prompt: str = None, + prompt_user_config: PromptTemplateConfig = None, + ): + assert isinstance(kernel, Kernel) + self._kernel = kernel + + self.config = config or StepwisePlannerConfig() + self.config.excluded_skills.append(RESTRICTED_SKILL_NAME) + + prompt_config = prompt_user_config or PromptTemplateConfig() + prompt_template = prompt or read_file(PROMPT_TEMPLATE_FILE_PATH) + + if prompt_user_config is None: + prompt_config = PromptTemplateConfig.from_json( + read_file(PROMPT_CONFIG_FILE_PATH) + ) + + prompt_config.completion.max_tokens = self.config.max_tokens + + self._system_step_function = self.import_semantic_function( + kernel, "StepwiseStep", prompt_template, prompt_config + ) + self._native_functions = self._kernel.import_skill(self, RESTRICTED_SKILL_NAME) + + self._context = kernel.create_new_context() + self._logger = self._kernel.logger + + def create_plan(self, goal: str) -> Plan: + if is_null_or_empty(goal): + raise PlanningException( + PlanningException.ErrorCodes.InvalidGoal, "The goal specified is empty" + ) + + function_descriptions = self.get_function_descriptions() + + plan_step: Plan = Plan.from_function(self._native_functions["ExecutePlan"]) + plan_step.parameters.set("function_descriptions", function_descriptions) + plan_step.parameters.set("question", goal) + + plan_step._outputs.append("agent_scratch_pad") + plan_step._outputs.append("step_count") + plan_step._outputs.append("skill_count") + plan_step._outputs.append("steps_taken") + + plan = Plan(goal) + + plan.add_steps([plan_step]) + + return plan + + # TODO: sync C# with https://github.com/microsoft/semantic-kernel/pull/1195 + @sk_function(name="ExecutePlan", description="Execute a plan") + @sk_function_context_parameter( + name="question", description="The question to answer" + ) + @sk_function_context_parameter( + name="function_descriptions", description="List of tool descriptions" + ) + async def execute_plan_async(self, context: SKContext) -> SKContext: + question = context["question"] + + steps_taken: List[SystemStep] = [] + if not is_null_or_empty(question): + for i in range(self.config.max_iterations): + scratch_pad = self.create_scratch_pad(question, steps_taken) + + context.variables.set("agent_scratch_pad", scratch_pad) + + llm_response = await self._system_step_function.invoke_async( + context=context + ) + + if llm_response.error_occurred: + raise PlanningException( + PlanningException.ErrorCodes.UnknownError, + f"Error occurred while executing stepwise plan: {llm_response.last_exception}", + llm_response.last_exception, + ) + + action_text = llm_response.result.strip() + self._logger.debug(f"Response: {action_text}") + + next_step = self.parse_result(action_text) + steps_taken.append(next_step) + + if not is_null_or_empty(next_step.final_answer): + self._logger.debug(f"Final Answer: {next_step.final_answer}") + + context.variables.update(next_step.final_answer) + updated_scratch_pad = self.create_scratch_pad(question, steps_taken) + context.variables.set("agent_scratch_pad", updated_scratch_pad) + + # Add additional results to the context + self.add_execution_stats_to_context(steps_taken, context) + + return context + + self._logger.debug("Thoughts: {next_step.thought}") + + if not is_null_or_empty(next_step.action): + self._logger.info(f"Action: {next_step.action}. Iteration: {i+1}.") + self._logger.debug( + f"Action: {next_step.action}({next_step.action_variables}). Iteration: {i+1}.", + ) + + try: + await asyncio.sleep(self.config.min_iteration_time_ms / 1000) + result = await self.invoke_action_async( + next_step.action, next_step.action_variables + ) + + if is_null_or_empty(result): + next_step.observation = "Got no result from action" + else: + next_step.observation = result + + except Exception as e: + next_step.observation = ( + f"Error invoking action {next_step.action}: {str(e)}" + ) + self._logger.warning( + f"Error invoking action {next_step.action}" + ) + + self._logger.debug(f"Observation: {next_step.observation}") + else: + self._logger.info("Action: No action to take") + + # sleep 3 seconds + await asyncio.sleep(self.config.min_iteration_time_ms / 1000) + + steps_taken_str = json.dumps([s.__dict__ for s in steps_taken], indent=4) + context.variables.update( + f"Result not found, review _steps_taken to see what happened.\n{steps_taken_str}" + ) + else: + context.variables.update("Question not found.") + + return context + + def parse_result(self, input: str): + result = SystemStep(original_response=input) + + # Extract final answer + final_answer_match = re.search(S_FINAL_ANSWER_REGEX, input) + + if final_answer_match: + result.final_answer = final_answer_match.group(1).strip() + return result + + # Extract thought + thought_match = re.search(S_THOUGHT_REGEX, input) + + if thought_match: + result.thought = thought_match.group(0).strip() + elif ACTION not in input: + result.thought = input + else: + raise ValueError("Unexpected input format") + + result.thought = result.thought.replace(THOUGHT, "").strip() + + # Extract action + action_match = re.search(S_ACTION_REGEX, input) + + if action_match: + action_json = action_match.group(1).strip() + + try: + system_step_results = json.loads(action_json) + + if system_step_results is None or len(system_step_results) == 0: + result.observation = ( + f"System step parsing error, empty JSON: {action_json}" + ) + else: + result.action = system_step_results["action"] + result.action_variables = system_step_results["action_variables"] + except Exception: + result.observation = ( + f"System step parsing error, invalid JSON: {action_json}" + ) + + if is_null_or_empty(result.thought) and is_null_or_empty(result.action): + result.observation = ( + "System step error, no thought or action found.", + "Please give a valid thought and/or action.", + ) + + return result + + def add_execution_stats_to_context( + self, steps_taken: List[SystemStep], context: SKContext + ): + context.variables.set("step_count", str(len(steps_taken))) + context.variables.set( + "steps_taken", json.dumps([s.__dict__ for s in steps_taken], indent=4) + ) + + action_counts: Dict[str, int] = {} + for step in steps_taken: + if is_null_or_empty(step.action): + continue + + current_count = action_counts.get(step.action, 0) + action_counts[step.action] = current_count + 1 + + skill_call_list_with_counts = [ + f"{skill}({action_counts[skill]})" for skill in action_counts + ] + skill_call_list_with_counts = ", ".join(skill_call_list_with_counts) + skill_call_count_str = str(sum(action_counts.values())) + + context.variables.set( + "skill_count", f"{skill_call_count_str} ({skill_call_list_with_counts})" + ) + + def create_scratch_pad(self, question: str, steps_taken: List[SystemStep]) -> str: + if len(steps_taken) == 0: + return "" + + scratch_pad_lines: List[str] = [] + + # Add the original first thought + scratch_pad_lines.append(SCRATCH_PAD_PREFIX) + scratch_pad_lines.append(f"{THOUGHT}\n{steps_taken[0].thought}") + + # Keep track of where to insert the next step + insert_point = len(scratch_pad_lines) + + for i in reversed(range(len(steps_taken))): + if len(scratch_pad_lines) / 4.0 > (self.config.max_tokens * 0.75): + self._logger.debug( + f"Scratchpad is too long, truncating. Skipping {i + 1} steps." + ) + break + + s = steps_taken[i] + + if not is_null_or_empty(s.observation): + scratch_pad_lines.insert( + insert_point, f"{OBSERVATION}\n{s.observation}" + ) + + if not is_null_or_empty(s.action): + scratch_pad_lines.insert( + insert_point, + f'{ACTION}\n{{"action": "{s.action}", "action_variables": {json.dumps(s.action_variables)}}}', + ) + + if i != 0: + scratch_pad_lines.insert(insert_point, f"{THOUGHT}\n{s.thought}") + + scratch_pad = "\n".join(scratch_pad_lines).strip() + + if not (is_null_or_empty(scratch_pad.strip())): + self._logger.debug(f"Scratchpad: {scratch_pad}") + + return scratch_pad + + async def invoke_action_async( + self, action_name: str, action_variables: Dict[str, str] + ) -> str: + available_functions = self.get_available_functions() + target_function = next( + ( + f + for f in available_functions + if self.to_fully_qualified_name(f) == action_name + ), + None, + ) + + if target_function is None: + raise PlanningException( + PlanningException.ErrorCodes.UnknownError, + f"The function '{action_name}' was not found.", + ) + + try: + function = self._kernel.func( + target_function.skill_name, target_function.name + ) + action_context = self.create_action_context(action_variables) + + result = await function.invoke_async(context=action_context) + + if result.error_occurred: + self._logger.error(f"Error occurred: {result.last_exception}") + return f"Error occurred: {result.last_exception}" + + self._logger.debug( + f"Invoked {target_function.name}. Result: {result.result}" + ) + + return result.result + + except Exception as e: + self._logger.error( + e, + f"Something went wrong in system step: {target_function.skill_name}.{target_function.name}. Error: {e}", + ) + return ( + "Something went wrong in system step: ", + f"{target_function.skill_name}.{target_function.name}. Error: {e}", + ) + + def create_action_context(self, action_variables: Dict[str, str]) -> SKContext: + action_context = self._kernel.create_new_context() + if action_variables is not None: + for k, v in action_variables.items(): + action_context.variables.set(k, v) + + return action_context + + def get_available_functions(self) -> List[FunctionView]: + functions_view = self._context.skills.get_functions_view() + + excluded_skills = self.config.excluded_skills or [] + excluded_functions = self.config.excluded_functions or [] + + available_functions: List[FunctionView] = [ + *functions_view.semantic_functions.values(), + *functions_view.native_functions.values(), + ] + available_functions = itertools.chain.from_iterable(available_functions) + available_functions = [ + func + for func in available_functions + if ( + func.skill_name not in excluded_skills + and func.name not in excluded_functions + ) + ] + available_functions = sorted( + available_functions, key=lambda x: (x.skill_name, x.name) + ) + + return available_functions + + def get_function_descriptions(self) -> str: + available_functions = self.get_available_functions() + + function_descriptions = "\n".join( + [self.to_manual_string(f) for f in available_functions] + ) + return function_descriptions + + def import_semantic_function( + self, + kernel: Kernel, + function_name: str, + prompt_template: str, + config: PromptTemplateConfig = None, + ) -> "SKFunctionBase": + template = PromptTemplate( + prompt_template, kernel.prompt_template_engine, config + ) + function_config = SemanticFunctionConfig(config, template) + + return kernel.register_semantic_function( + RESTRICTED_SKILL_NAME, function_name, function_config + ) + + def to_manual_string(self, function: FunctionView) -> str: + inputs = [ + f" - {parameter.name}: {parameter.description}" + + ( + f" (default value={parameter.default_value})" + if parameter.default_value + else "" + ) + for parameter in function.parameters + ] + inputs = "\n".join(inputs) + + function_description = function.description.strip() + + if is_null_or_empty(inputs): + return f"{self.to_fully_qualified_name(function)}: {function_description}\n inputs: None\n" + + return f"{self.to_fully_qualified_name(function)}: {function_description}\n inputs:\n{inputs}\n" + + def to_fully_qualified_name(self, function: FunctionView): + return f"{function.skill_name}.{function.name}" diff --git a/python/semantic_kernel/planning/stepwise_planner/stepwise_planner_config.py b/python/semantic_kernel/planning/stepwise_planner/stepwise_planner_config.py new file mode 100644 index 000000000000..0654a829dd9c --- /dev/null +++ b/python/semantic_kernel/planning/stepwise_planner/stepwise_planner_config.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import List, Optional + + +class StepwisePlannerConfig: + def __init__( + self, + relevancy_threshold: Optional[float] = None, + max_relevant_functions: int = 100, + excluded_skills: List[str] = None, + excluded_functions: List[str] = None, + included_functions: List[str] = None, + max_tokens: int = 1024, + max_iterations: int = 100, + min_iteration_time_ms: int = 0, + ): + self.relevancy_threshold: float = relevancy_threshold + self.max_relevant_functions: int = max_relevant_functions + self.excluded_skills: List[str] = excluded_skills or [] + self.excluded_functions: List[str] = excluded_functions or [] + self.included_functions: List[str] = included_functions or [] + self.max_tokens: int = max_tokens + self.max_iterations: int = max_iterations + self.min_iteration_time_ms: int = min_iteration_time_ms diff --git a/python/semantic_kernel/planning/stepwise_planner/system_step.py b/python/semantic_kernel/planning/stepwise_planner/system_step.py new file mode 100644 index 000000000000..6d14bf198f73 --- /dev/null +++ b/python/semantic_kernel/planning/stepwise_planner/system_step.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass, field +from typing import Dict, Optional + + +@dataclass +class SystemStep: + thought: Optional[str] = None + action: Optional[str] = None + action_variables: Optional[Dict[str, str]] = field(default_factory=dict) + observation: Optional[str] = None + final_answer: Optional[str] = None + original_response: Optional[str] = None diff --git a/python/semantic_kernel/reliability/pass_through_without_retry.py b/python/semantic_kernel/reliability/pass_through_without_retry.py index 8aaa287f2959..8056cb87a257 100644 --- a/python/semantic_kernel/reliability/pass_through_without_retry.py +++ b/python/semantic_kernel/reliability/pass_through_without_retry.py @@ -4,11 +4,12 @@ from typing import Awaitable, Callable, TypeVar from semantic_kernel.reliability.retry_mechanism_base import RetryMechanismBase +from semantic_kernel.sk_pydantic import PydanticField T = TypeVar("T") -class PassThroughWithoutRetry(RetryMechanismBase): +class PassThroughWithoutRetry(RetryMechanismBase, PydanticField): """A retry mechanism that does not retry.""" async def execute_with_retry_async( diff --git a/python/semantic_kernel/semantic_functions/chat_prompt_template.py b/python/semantic_kernel/semantic_functions/chat_prompt_template.py index 053e8798408f..8aa999e948ec 100644 --- a/python/semantic_kernel/semantic_functions/chat_prompt_template.py +++ b/python/semantic_kernel/semantic_functions/chat_prompt_template.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft. All rights reserved. +import asyncio from logging import Logger -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, TypeVar +from semantic_kernel.models.chat.chat_message import ChatMessage from semantic_kernel.semantic_functions.prompt_template import PromptTemplate from semantic_kernel.semantic_functions.prompt_template_config import ( PromptTemplateConfig, @@ -14,9 +16,11 @@ if TYPE_CHECKING: from semantic_kernel.orchestration.sk_context import SKContext +ChatMessageT = TypeVar("ChatMessageT", bound=ChatMessage) -class ChatPromptTemplate(PromptTemplate): - _messages: List[Tuple[str, PromptTemplate]] + +class ChatPromptTemplate(PromptTemplate, Generic[ChatMessageT]): + _messages: List[ChatMessageT] def __init__( self, @@ -27,6 +31,8 @@ def __init__( ) -> None: super().__init__(template, template_engine, prompt_config, log) self._messages = [] + if self._prompt_config.completion.chat_system_prompt: + self.add_system_message(self._prompt_config.completion.chat_system_prompt) async def render_async(self, context: "SKContext") -> str: raise NotImplementedError( @@ -35,29 +41,84 @@ async def render_async(self, context: "SKContext") -> str: ) def add_system_message(self, message: str) -> None: + """Add a system message to the chat template.""" self.add_message("system", message) def add_user_message(self, message: str) -> None: + """Add a user message to the chat template.""" self.add_message("user", message) def add_assistant_message(self, message: str) -> None: + """Add an assistant message to the chat template.""" self.add_message("assistant", message) - def add_message(self, role: str, message: str) -> None: + def add_message( + self, role: str, message: Optional[str] = None, **kwargs: Any + ) -> None: + """Add a message to the chat template. + + Arguments: + role: The role of the message, one of "user", "assistant", "system". + message: The message to add, can include templating components. + kwargs: can be used by inherited classes. + """ self._messages.append( - (role, PromptTemplate(message, self._template_engine, self._prompt_config)) + ChatMessage( + role=role, + content_template=PromptTemplate( + message, self._template_engine, self._prompt_config + ), + ) ) - async def render_messages_async( - self, context: "SKContext" - ) -> List[Tuple[str, str]]: - rendered_messages = [] - for role, message in self._messages: - rendered_messages.append((role, await message.render_async(context))) - - latest_user_message = await self._template_engine.render_async( - self._template, context + async def render_messages_async(self, context: "SKContext") -> List[Dict[str, str]]: + """Render the content of the message in the chat template, based on the context.""" + if len(self._messages) == 0 or self._messages[-1].role in [ + "assistant", + "system", + ]: + self.add_user_message(message=self._template) + await asyncio.gather( + *[message.render_message_async(context) for message in self._messages] ) - rendered_messages.append(("user", latest_user_message)) + return [message.as_dict() for message in self._messages] + + @property + def messages(self) -> List[Dict[str, str]]: + """Return the messages as a list of dicts with role, content, name.""" + return [message.as_dict() for message in self._messages] + + @classmethod + def restore( + cls, + messages: List[Dict[str, str]], + template: str, + template_engine: PromptTemplatingEngine, + prompt_config: PromptTemplateConfig, + log: Optional[Logger] = None, + ) -> "ChatPromptTemplate": + """Restore a ChatPromptTemplate from a list of role and message pairs. + + If there is a chat_system_prompt in the prompt_config.completion settings, + that takes precedence over the first message in the list of messages, + if that is a system message. + """ + chat_template = cls(template, template_engine, prompt_config, log) + if ( + prompt_config.completion.chat_system_prompt + and messages[0]["role"] == "system" + ): + existing_system_message = messages.pop(0) + if ( + existing_system_message["message"] + != prompt_config.completion.chat_system_prompt + ): + chat_template._log.info( + "Overriding system prompt with chat_system_prompt, old system message: %s, new system message: %s", + existing_system_message["message"], + prompt_config.completion.chat_system_prompt, + ) + for message in messages: + chat_template.add_message(message["role"], message["message"]) - return rendered_messages + return chat_template diff --git a/python/semantic_kernel/semantic_functions/prompt_template.py b/python/semantic_kernel/semantic_functions/prompt_template.py index 2fe943e0a547..b8ead27d112e 100644 --- a/python/semantic_kernel/semantic_functions/prompt_template.py +++ b/python/semantic_kernel/semantic_functions/prompt_template.py @@ -46,7 +46,11 @@ def get_parameters(self) -> List[ParameterView]: continue result.append( - ParameterView(param.name, param.description, param.default_value) + ParameterView( + name=param.name, + description=param.description, + default_value=param.default_value, + ) ) seen.add(param.name) @@ -62,7 +66,9 @@ def get_parameters(self) -> List[ParameterView]: if var_block.name in seen: continue - result.append(ParameterView(var_block.name, "", "")) + result.append( + ParameterView(name=var_block.name, description="", default_value="") + ) seen.add(var_block.name) diff --git a/python/semantic_kernel/semantic_functions/prompt_template_config.py b/python/semantic_kernel/semantic_functions/prompt_template_config.py index ce7fceff1669..1d996d2054cd 100644 --- a/python/semantic_kernel/semantic_functions/prompt_template_config.py +++ b/python/semantic_kernel/semantic_functions/prompt_template_config.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. from dataclasses import dataclass, field -from typing import List +from typing import Dict, List, Optional @dataclass @@ -15,12 +15,19 @@ class CompletionConfig: max_tokens: int = 256 number_of_responses: int = 1 stop_sequences: List[str] = field(default_factory=list) + token_selection_biases: Dict[int, int] = field(default_factory=dict) + chat_system_prompt: str = None + # the function_call should be 'auto' or the name of a specific function in order to leverage function calling + # when not using auto, the format is 'SkillName-FunctionName', e.g. 'Weather-GetWeather' + function_call: Optional[str] = None @dataclass class InputParameter: name: str = "" description: str = "" default_value: str = "" + type_: str = "string" + required: bool = True @dataclass class InputConfig: @@ -40,23 +47,30 @@ class InputConfig: @staticmethod def from_dict(data: dict) -> "PromptTemplateConfig": config = PromptTemplateConfig() - config.schema = data.get("schema") - config.type = data.get("type") - config.description = data.get("description") + keys = ["schema", "type", "description"] + for key in keys: + if key in data: + setattr(config, key, data[key]) # Some skills may not have all completion parameters defined config.completion = PromptTemplateConfig.CompletionConfig() completion_dict = data["completion"] - config.completion.temperature = completion_dict.get("temperature") - config.completion.top_p = completion_dict.get("top_p") - config.completion.presence_penalty = completion_dict.get("presence_penalty") - config.completion.frequency_penalty = completion_dict.get("frequency_penalty") - config.completion.max_tokens = completion_dict.get("max_tokens") - config.completion.number_of_responses = completion_dict.get( - "number_of_responses" - ) - config.completion.stop_sequences = completion_dict.get("stop_sequences", []) - config.default_services = data.get("default_services", []) + completion_keys = [ + "temperature", + "top_p", + "presence_penalty", + "frequency_penalty", + "max_tokens", + "number_of_responses", + "stop_sequences", + "token_selection_biases", + "default_services", + "chat_system_prompt", + "function_call", + ] + for comp_key in completion_keys: + if comp_key in completion_dict: + setattr(config.completion, comp_key, completion_dict[comp_key]) # Some skills may not have input parameters defined config.input = PromptTemplateConfig.InputConfig() @@ -84,11 +98,16 @@ def from_dict(data: dict) -> "PromptTemplateConfig": f"Input parameter '{name}' doesn't have a default value (function: {config.description})" ) + type_ = parameter.get("type") + required = parameter.get("required") + config.input.parameters.append( PromptTemplateConfig.InputParameter( name, description, defaultValue, + type_, + required, ) ) return config @@ -97,7 +116,12 @@ def from_dict(data: dict) -> "PromptTemplateConfig": def from_json(json_str: str) -> "PromptTemplateConfig": import json - return PromptTemplateConfig.from_dict(json.loads(json_str)) + def keystoint(d): + return {int(k) if k.isdigit() else k: v for k, v in d.items()} + + return PromptTemplateConfig.from_dict( + json.loads(json_str, object_hook=keystoint) + ) @staticmethod def from_completion_parameters( @@ -108,6 +132,9 @@ def from_completion_parameters( max_tokens: int = 256, number_of_responses: int = 1, stop_sequences: List[str] = [], + token_selection_biases: Dict[int, int] = {}, + chat_system_prompt: str = None, + function_call: Optional[str] = None, ) -> "PromptTemplateConfig": config = PromptTemplateConfig() config.completion.temperature = temperature @@ -117,4 +144,7 @@ def from_completion_parameters( config.completion.max_tokens = max_tokens config.completion.number_of_responses = number_of_responses config.completion.stop_sequences = stop_sequences + config.completion.token_selection_biases = token_selection_biases + config.completion.chat_system_prompt = chat_system_prompt + config.completion.function_call = function_call return config diff --git a/python/semantic_kernel/sk_pydantic.py b/python/semantic_kernel/sk_pydantic.py new file mode 100644 index 000000000000..7f00badf4059 --- /dev/null +++ b/python/semantic_kernel/sk_pydantic.py @@ -0,0 +1,90 @@ +import abc +import json +import typing as t + +import numpy as np +import pydantic as pdt +import typing_extensions as te +from pydantic.generics import GenericModel +from pydantic.parse import Protocol +from pydantic.types import StrBytes + + +class PydanticField(abc.ABC): + """Subclass this class to make your class a valid pydantic field type. + + This class is a no-op, but it's necessary to make pydantic recognize your class as + a valid field type. See https://pydantic-docs.helpmanual.io/usage/types/#custom-data-types + for more information. + + - If you want to add validation to your class, you can do so by implementing the + `__get_validators__` class method. See + https://pydantic-docs.helpmanual.io/usage/validators/ for more information. + - If you want to add serialization to your class, you can do so by implementing the + `json` and `parse_raw` methods. See + https://pydantic-docs.helpmanual.io/usage/exporting_models/#json for more information. + """ + + @classmethod + def __get_validators__(cls) -> t.Generator[t.Callable[..., t.Any], None, None]: + """Gets the validators for the class.""" + yield cls.no_op_validate + + @classmethod + def no_op_validate(cls, v: t.Any) -> t.Any: + """Does no validation, just returns the value.""" + if v is None: + v = cls() + if isinstance(v, str): + v = cls(**json.loads(v)) + return v + + def json(self) -> str: + """Serialize the model to JSON.""" + return "{}" + + @classmethod + def parse_raw( + cls: t.Type[te.Self], + b: StrBytes, + *, + content_type: str = None, + encoding: str = "utf8", + proto: Protocol = None, + allow_pickle: bool = False, + ) -> te.Self: + """Parse a raw byte string into a model.""" + return cls() + + def __eq__(self, other: t.Any) -> bool: + """Check if two instances are equal.""" + return isinstance(other, self.__class__) + + +_JSON_ENCODERS: t.Final[t.Dict[t.Type[t.Any], t.Callable[[t.Any], str]]] = { + PydanticField: lambda v: v.json(), + np.ndarray: lambda v: json.dumps(v.tolist()), +} + + +class SKBaseModel(pdt.BaseModel): + """Base class for all pydantic models in the SK.""" + + class Config: + """Pydantic configuration.""" + + json_encoders = _JSON_ENCODERS + # See the `allow_population_by_field_name` section of + # https://docs.pydantic.dev/latest/usage/model_config/#options + allow_population_by_field_name = True + arbitrary_types_allowed = True + + +class SKGenericModel(GenericModel): + """Base class for all pydantic `GenericModel`s in the SK.""" + + class Config: + """Pydantic configuration.""" + + json_encoders = _JSON_ENCODERS + arbitrary_types_allowed = True diff --git a/python/semantic_kernel/skill_definition/constants.py b/python/semantic_kernel/skill_definition/constants.py new file mode 100644 index 000000000000..e9f919bdde32 --- /dev/null +++ b/python/semantic_kernel/skill_definition/constants.py @@ -0,0 +1,3 @@ +import typing as t + +GLOBAL_SKILL: t.Final[str] = "_GLOBAL_FUNCTIONS_" diff --git a/python/semantic_kernel/skill_definition/function_view.py b/python/semantic_kernel/skill_definition/function_view.py index 12539178487e..19961026a71e 100644 --- a/python/semantic_kernel/skill_definition/function_view.py +++ b/python/semantic_kernel/skill_definition/function_view.py @@ -2,17 +2,18 @@ from typing import List +from semantic_kernel.sk_pydantic import SKBaseModel from semantic_kernel.skill_definition.parameter_view import ParameterView from semantic_kernel.utils.validation import validate_function_name -class FunctionView: - _name: str - _skill_name: str - _description: str - _is_semantic: bool - _is_asynchronous: bool - _parameters: List[ParameterView] +class FunctionView(SKBaseModel): + name: str + skill_name: str + description: str + is_semantic: bool + parameters: List[ParameterView] + is_asynchronous: bool = True def __init__( self, @@ -24,60 +25,11 @@ def __init__( is_asynchronous: bool = True, ) -> None: validate_function_name(name) - - self._name = name - self._skill_name = skill_name - self._description = description - self._parameters = parameters - self._is_semantic = is_semantic - self._is_asynchronous = is_asynchronous - - @property - def name(self) -> str: - return self._name - - @property - def skill_name(self) -> str: - return self._skill_name - - @property - def description(self) -> str: - return self._description - - @property - def parameters(self) -> List[ParameterView]: - return self._parameters - - @property - def is_semantic(self) -> bool: - return self._is_semantic - - @property - def is_asynchronous(self) -> bool: - return self._is_asynchronous - - @name.setter - def name(self, value: str) -> None: - validate_function_name(value) - - self._name = value - - @skill_name.setter - def skill_name(self, value: str) -> None: - self._skill_name = value - - @description.setter - def description(self, value: str) -> None: - self._description = value - - @parameters.setter - def parameters(self, value: List[ParameterView]) -> None: - self._parameters = value - - @is_semantic.setter - def is_semantic(self, value: bool) -> None: - self._is_semantic = value - - @is_asynchronous.setter - def is_asynchronous(self, value: bool) -> None: - self._is_asynchronous = value + super().__init__( + name=name, + skill_name=skill_name, + description=description, + parameters=parameters, + is_semantic=is_semantic, + is_asynchronous=is_asynchronous, + ) diff --git a/python/semantic_kernel/skill_definition/functions_view.py b/python/semantic_kernel/skill_definition/functions_view.py index 72a6af4d419a..62ff9cdadab1 100644 --- a/python/semantic_kernel/skill_definition/functions_view.py +++ b/python/semantic_kernel/skill_definition/functions_view.py @@ -2,58 +2,61 @@ from typing import Dict, List +import pydantic as pdt + from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.sk_pydantic import SKBaseModel from semantic_kernel.skill_definition.function_view import FunctionView -class FunctionsView: - _semantic_functions: Dict[str, List[FunctionView]] - _native_functions: Dict[str, List[FunctionView]] - - def __init__(self) -> None: - self._semantic_functions = {} - self._native_functions = {} +class FunctionsView(SKBaseModel): + semantic_functions: Dict[str, List[FunctionView]] = pdt.Field(default_factory=dict) + native_functions: Dict[str, List[FunctionView]] = pdt.Field(default_factory=dict) def add_function(self, view: FunctionView) -> "FunctionsView": if view.is_semantic: - if view.skill_name not in self._semantic_functions: - self._semantic_functions[view.skill_name] = [] - self._semantic_functions[view.skill_name].append(view) + if view.skill_name not in self.semantic_functions: + self.semantic_functions[view.skill_name] = [] + self.semantic_functions[view.skill_name].append(view) else: - if view.skill_name not in self._native_functions: - self._native_functions[view.skill_name] = [] - self._native_functions[view.skill_name].append(view) + if view.skill_name not in self.native_functions: + self.native_functions[view.skill_name] = [] + self.native_functions[view.skill_name].append(view) return self def is_semantic(self, skill_name: str, function_name: str) -> bool: - as_sf = self._semantic_functions.get(skill_name, []) + as_sf = self.semantic_functions.get(skill_name, []) as_sf = any(f.name == function_name for f in as_sf) - as_nf = self._native_functions.get(skill_name, []) + as_nf = self.native_functions.get(skill_name, []) as_nf = any(f.name == function_name for f in as_nf) if as_sf and as_nf: raise KernelException( KernelException.ErrorCodes.AmbiguousImplementation, - f"There are 2 functions with the same name: {function_name}." - f"One is native and the other semantic.", + ( + f"There are 2 functions with the same name: {function_name}." + "One is native and the other semantic." + ), ) return as_sf def is_native(self, skill_name: str, function_name: str) -> bool: - as_sf = self._semantic_functions.get(skill_name, []) + as_sf = self.semantic_functions.get(skill_name, []) as_sf = any(f.name == function_name for f in as_sf) - as_nf = self._native_functions.get(skill_name, []) + as_nf = self.native_functions.get(skill_name, []) as_nf = any(f.name == function_name for f in as_nf) if as_sf and as_nf: raise KernelException( KernelException.ErrorCodes.AmbiguousImplementation, - f"There are 2 functions with the same name: {function_name}." - f"One is native and the other semantic.", + ( + f"There are 2 functions with the same name: {function_name}." + "One is native and the other semantic." + ), ) return as_nf diff --git a/python/semantic_kernel/skill_definition/parameter_view.py b/python/semantic_kernel/skill_definition/parameter_view.py index 41ad4a19401a..38b3c794d730 100644 --- a/python/semantic_kernel/skill_definition/parameter_view.py +++ b/python/semantic_kernel/skill_definition/parameter_view.py @@ -1,41 +1,20 @@ # Copyright (c) Microsoft. All rights reserved. -from semantic_kernel.utils.validation import validate_function_param_name - - -class ParameterView: - _name: str - _description: str - _default_value: str - - def __init__(self, name: str, description: str, default_value: str) -> None: - validate_function_param_name(name) - self._name = name - self._description = description - self._default_value = default_value +from pydantic import Field, validator - @property - def name(self) -> str: - return self._name - - @property - def description(self) -> str: - return self._description - - @property - def default_value(self) -> str: - return self._default_value +from semantic_kernel.sk_pydantic import SKBaseModel +from semantic_kernel.utils.validation import validate_function_param_name - @name.setter - def name(self, value: str) -> None: - validate_function_param_name(value) - self._name = value - @description.setter - def description(self, value: str) -> None: - self._description = value +class ParameterView(SKBaseModel): + name: str + description: str + default_value: str + type_: str = Field(default="string", alias="type") + required: bool = False - @default_value.setter - def default_value(self, value: str) -> None: - self._default_value = value + @validator("name") + def validate_name(cls, name: str): + validate_function_param_name(name) + return name diff --git a/python/semantic_kernel/skill_definition/read_only_skill_collection.py b/python/semantic_kernel/skill_definition/read_only_skill_collection.py index a91b4abb0600..3a727fc1640a 100644 --- a/python/semantic_kernel/skill_definition/read_only_skill_collection.py +++ b/python/semantic_kernel/skill_definition/read_only_skill_collection.py @@ -1,56 +1,127 @@ # Copyright (c) Microsoft. All rights reserved. -from typing import TYPE_CHECKING, Optional +from logging import Logger +from typing import TYPE_CHECKING, ClassVar, Dict, Optional, Tuple +import pydantic as pdt + +from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.orchestration.sk_function import SKFunction +from semantic_kernel.sk_pydantic import SKBaseModel +from semantic_kernel.skill_definition import constants +from semantic_kernel.skill_definition.functions_view import FunctionsView from semantic_kernel.skill_definition.read_only_skill_collection_base import ( ReadOnlySkillCollectionBase, ) +from semantic_kernel.utils.null_logger import NullLogger if TYPE_CHECKING: from semantic_kernel.orchestration.sk_function_base import SKFunctionBase - from semantic_kernel.skill_definition.functions_view import FunctionsView - from semantic_kernel.skill_definition.skill_collection_base import ( - SkillCollectionBase, - ) -class ReadOnlySkillCollection(ReadOnlySkillCollectionBase): - _skill_collection: "SkillCollectionBase" +class ReadOnlySkillCollection(SKBaseModel, ReadOnlySkillCollectionBase): + GLOBAL_SKILL: ClassVar[str] = constants.GLOBAL_SKILL + data: Dict[str, Dict[str, SKFunction]] = pdt.Field(default_factory=dict) + _log: Logger = pdt.PrivateAttr() + + class Config: + allow_mutation = False - def __init__(self, skill_collection: "SkillCollectionBase") -> None: - self._skill_collection = skill_collection + def __init__( + self, + data: Dict[str, Dict[str, SKFunction]] = None, + log: Optional[Logger] = None, + ) -> None: + super().__init__(data=data or {}) + self._log = log or NullLogger() def has_function(self, skill_name: Optional[str], function_name: str) -> bool: - return self._skill_collection.has_function(skill_name, function_name) + s_name, f_name = self._normalize_names(skill_name, function_name, True) + if s_name not in self.data: + return False + return f_name in self.data[s_name] - def has_semantic_function( - self, skill_name: Optional[str], function_name: str - ) -> bool: - return self._skill_collection.has_semantic_function(skill_name, function_name) + def has_semantic_function(self, skill_name: str, function_name: str) -> bool: + s_name, f_name = self._normalize_names(skill_name, function_name) + if s_name not in self.data: + return False + if f_name not in self.data[s_name]: + return False + return self.data[s_name][f_name].is_semantic - def has_native_function( - self, skill_name: Optional[str], function_name: str - ) -> bool: - return self._skill_collection.has_native_function(skill_name, function_name) + def has_native_function(self, skill_name: str, function_name: str) -> bool: + s_name, f_name = self._normalize_names(skill_name, function_name, True) + if s_name not in self.data: + return False + if f_name not in self.data[s_name]: + return False + return self.data[s_name][f_name].is_native def get_semantic_function( - self, skill_name: Optional[str], function_name: str + self, skill_name: str, function_name: str ) -> "SKFunctionBase": - return self._skill_collection.get_semantic_function(skill_name, function_name) + s_name, f_name = self._normalize_names(skill_name, function_name) + if self.has_semantic_function(s_name, f_name): + return self.data[s_name][f_name] + + self._log.error(f"Function not available: {s_name}.{f_name}") + raise KernelException( + KernelException.ErrorCodes.FunctionNotAvailable, + f"Function not available: {s_name}.{f_name}", + ) def get_native_function( - self, skill_name: Optional[str], function_name: str + self, skill_name: str, function_name: str ) -> "SKFunctionBase": - return self._skill_collection.get_native_function(skill_name, function_name) + s_name, f_name = self._normalize_names(skill_name, function_name, True) + if self.has_native_function(s_name, f_name): + return self.data[s_name][f_name] + + self._log.error(f"Function not available: {s_name}.{f_name}") + raise KernelException( + KernelException.ErrorCodes.FunctionNotAvailable, + f"Function not available: {s_name}.{f_name}", + ) def get_functions_view( self, include_semantic: bool = True, include_native: bool = True - ) -> "FunctionsView": - return self._skill_collection.get_functions_view( - include_semantic, include_native - ) + ) -> FunctionsView: + result = FunctionsView() + + for skill in self.data.values(): + for function in skill.values(): + if include_semantic and function.is_semantic: + result.add_function(function.describe()) + elif include_native and function.is_native: + result.add_function(function.describe()) + + return result def get_function( self, skill_name: Optional[str], function_name: str ) -> "SKFunctionBase": - return self._skill_collection.get_function(skill_name, function_name) + s_name, f_name = self._normalize_names(skill_name, function_name, True) + if self.has_function(s_name, f_name): + return self.data[s_name][f_name] + + self._log.error(f"Function not available: {s_name}.{f_name}") + raise KernelException( + KernelException.ErrorCodes.FunctionNotAvailable, + f"Function not available: {s_name}.{f_name}", + ) + + def _normalize_names( + self, + skill_name: Optional[str], + function_name: str, + allow_substitution: bool = False, + ) -> Tuple[str, str]: + s_name, f_name = skill_name, function_name + if s_name is None and allow_substitution: + s_name = self.GLOBAL_SKILL + + if s_name is None: + raise ValueError("The skill name provided cannot be `None`") + + s_name, f_name = s_name.lower(), f_name.lower() + return s_name, f_name diff --git a/python/semantic_kernel/skill_definition/read_only_skill_collection_base.py b/python/semantic_kernel/skill_definition/read_only_skill_collection_base.py index 1f456bbd596d..4218527a0a18 100644 --- a/python/semantic_kernel/skill_definition/read_only_skill_collection_base.py +++ b/python/semantic_kernel/skill_definition/read_only_skill_collection_base.py @@ -3,12 +3,14 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Optional +from semantic_kernel.sk_pydantic import PydanticField + if TYPE_CHECKING: from semantic_kernel.orchestration.sk_function_base import SKFunctionBase from semantic_kernel.skill_definition.functions_view import FunctionsView -class ReadOnlySkillCollectionBase(ABC): +class ReadOnlySkillCollectionBase(PydanticField, ABC): @abstractmethod def has_function(self, skill_name: Optional[str], function_name: str) -> bool: pass diff --git a/python/semantic_kernel/skill_definition/sk_function_context_parameter_decorator.py b/python/semantic_kernel/skill_definition/sk_function_context_parameter_decorator.py index b67693b59461..c7eb2d670614 100644 --- a/python/semantic_kernel/skill_definition/sk_function_context_parameter_decorator.py +++ b/python/semantic_kernel/skill_definition/sk_function_context_parameter_decorator.py @@ -1,10 +1,13 @@ # Copyright (c) Microsoft. All rights reserved. -from typing import Optional - def sk_function_context_parameter( - *, name: str, description: str, default_value: Optional[str] = None + *, + name: str, + description: str, + default_value: str = "", + type: str = "string", + required: bool = False ): """ Decorator for SK function context parameters. @@ -13,6 +16,9 @@ def sk_function_context_parameter( name -- The name of the context parameter description -- The description of the context parameter default_value -- The default value of the context parameter + type -- The type of the context parameter, used for function calling + required -- Whether the context parameter is required + """ def decorator(func): @@ -24,6 +30,8 @@ def decorator(func): "name": name, "description": description, "default_value": default_value, + "type": type, + "required": required, } ) return func diff --git a/python/semantic_kernel/skill_definition/sk_function_decorator.py b/python/semantic_kernel/skill_definition/sk_function_decorator.py index 048280444cdc..6c56fbf85bad 100644 --- a/python/semantic_kernel/skill_definition/sk_function_decorator.py +++ b/python/semantic_kernel/skill_definition/sk_function_decorator.py @@ -4,9 +4,9 @@ def sk_function( *, description: str = "", - name: str = None, - input_description: str = None, - input_default_value: str = None + name: str = "", + input_description: str = "", + input_default_value: str = "", ): """ Decorator for SK functions. @@ -20,10 +20,10 @@ def sk_function( def decorator(func): func.__sk_function__ = True - func.__sk_function_description__ = description - func.__sk_function_name__ = name if name else func.__name__ - func.__sk_function_input_description__ = input_description - func.__sk_function_input_default_value__ = input_default_value + func.__sk_function_description__ = description or "" + func.__sk_function_name__ = name or func.__name__ + func.__sk_function_input_description__ = input_description or "" + func.__sk_function_input_default_value__ = input_default_value or "" return func return decorator diff --git a/python/semantic_kernel/skill_definition/skill_collection.py b/python/semantic_kernel/skill_definition/skill_collection.py index b4977ead4641..77458f7e9054 100644 --- a/python/semantic_kernel/skill_definition/skill_collection.py +++ b/python/semantic_kernel/skill_definition/skill_collection.py @@ -1,37 +1,67 @@ # Copyright (c) Microsoft. All rights reserved. from logging import Logger -from typing import TYPE_CHECKING, Dict, Literal, Optional, Tuple +from typing import ( + TYPE_CHECKING, + ClassVar, + Dict, + Optional, + Union, +) + +import pydantic as pdt -from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.orchestration.sk_function import SKFunction +from semantic_kernel.sk_pydantic import SKGenericModel +from semantic_kernel.skill_definition import constants from semantic_kernel.skill_definition.functions_view import FunctionsView from semantic_kernel.skill_definition.read_only_skill_collection import ( ReadOnlySkillCollection, ) +from semantic_kernel.skill_definition.read_only_skill_collection_base import ( + ReadOnlySkillCollectionBase, +) from semantic_kernel.skill_definition.skill_collection_base import SkillCollectionBase from semantic_kernel.utils.null_logger import NullLogger -from semantic_kernel.utils.static_property import static_property if TYPE_CHECKING: from semantic_kernel.orchestration.sk_function_base import SKFunctionBase - from semantic_kernel.skill_definition.read_only_skill_collection_base import ( - ReadOnlySkillCollectionBase, - ) -class SkillCollection(SkillCollectionBase): - _skill_collection: Dict[str, Dict[str, "SKFunctionBase"]] - _read_only_skill_collection: "ReadOnlySkillCollectionBase" - _log: Logger +class SkillCollection(SKGenericModel, SkillCollectionBase): + GLOBAL_SKILL: ClassVar[str] = constants.GLOBAL_SKILL + read_only_skill_collection_: ReadOnlySkillCollection = pdt.Field( + alias="read_only_skill_collection" + ) + _log: Logger = pdt.PrivateAttr() + + def __init__( + self, + log: Optional[Logger] = None, + skill_collection: Union[Dict[str, Dict[str, SKFunction]], None] = None, + read_only_skill_collection_: Optional[ReadOnlySkillCollection] = None, + ) -> None: + if skill_collection and read_only_skill_collection_: + raise ValueError( + "Only one of `skill_collection` and `read_only_skill_collection` can be" + " provided" + ) + elif not skill_collection and not read_only_skill_collection_: + read_only_skill_collection = ReadOnlySkillCollection({}) + elif not read_only_skill_collection_: + read_only_skill_collection = ReadOnlySkillCollection(skill_collection) + else: + read_only_skill_collection = read_only_skill_collection_ + super().__init__(read_only_skill_collection=read_only_skill_collection) + self._log = log if log is not None else NullLogger() @property - def read_only_skill_collection(self) -> "ReadOnlySkillCollectionBase": - return self._read_only_skill_collection + def read_only_skill_collection(self) -> ReadOnlySkillCollectionBase: + return self.read_only_skill_collection_ - def __init__(self, log: Optional[Logger] = None) -> None: - self._log = log if log is not None else NullLogger() - self._read_only_skill_collection = ReadOnlySkillCollection(self) - self._skill_collection = {} + @property + def skill_collection(self): + return self.read_only_skill_collection_.data def add_semantic_function(self, function: "SKFunctionBase") -> None: if function is None: @@ -40,114 +70,58 @@ def add_semantic_function(self, function: "SKFunctionBase") -> None: s_name, f_name = function.skill_name, function.name s_name, f_name = s_name.lower(), f_name.lower() - if s_name not in self._skill_collection: - self._skill_collection[s_name] = {} - - self._skill_collection[s_name][f_name] = function + self.skill_collection.setdefault(s_name, {})[f_name] = function def add_native_function(self, function: "SKFunctionBase") -> None: if function is None: raise ValueError("The function provided cannot be `None`") s_name, f_name = function.skill_name, function.name - s_name, f_name = self._normalize_names(s_name, f_name, True) - - if s_name not in self._skill_collection: - self._skill_collection[s_name] = {} + s_name, f_name = self.read_only_skill_collection_._normalize_names( + s_name, f_name, True + ) - self._skill_collection[s_name][f_name] = function + self.skill_collection.setdefault(s_name, {})[f_name] = function def has_function(self, skill_name: Optional[str], function_name: str) -> bool: - s_name, f_name = self._normalize_names(skill_name, function_name, True) - if s_name not in self._skill_collection: - return False - return f_name in self._skill_collection[s_name] - - def has_semantic_function(self, skill_name: str, function_name: str) -> bool: - s_name, f_name = self._normalize_names(skill_name, function_name) - if s_name not in self._skill_collection: - return False - if f_name not in self._skill_collection[s_name]: - return False - return self._skill_collection[s_name][f_name].is_semantic - - def has_native_function(self, skill_name: str, function_name: str) -> bool: - s_name, f_name = self._normalize_names(skill_name, function_name, True) - if s_name not in self._skill_collection: - return False - if f_name not in self._skill_collection[s_name]: - return False - return self._skill_collection[s_name][f_name].is_native + return self.read_only_skill_collection_.has_function(skill_name, function_name) + + def has_semantic_function( + self, skill_name: Optional[str], function_name: str + ) -> bool: + return self.read_only_skill_collection_.has_semantic_function( + skill_name, function_name + ) + + def has_native_function( + self, skill_name: Optional[str], function_name: str + ) -> bool: + return self.read_only_skill_collection_.has_native_function( + skill_name, function_name + ) def get_semantic_function( - self, skill_name: str, function_name: str + self, skill_name: Optional[str], function_name: str ) -> "SKFunctionBase": - s_name, f_name = self._normalize_names(skill_name, function_name) - if self.has_semantic_function(s_name, f_name): - return self._skill_collection[s_name][f_name] - - self._log.error(f"Function not available: {s_name}.{f_name}") - raise KernelException( - KernelException.ErrorCodes.FunctionNotAvailable, - f"Function not available: {s_name}.{f_name}", + return self.read_only_skill_collection_.get_semantic_function( + skill_name, function_name ) def get_native_function( - self, skill_name: str, function_name: str + self, skill_name: Optional[str], function_name: str ) -> "SKFunctionBase": - s_name, f_name = self._normalize_names(skill_name, function_name, True) - if self.has_native_function(s_name, f_name): - return self._skill_collection[s_name][f_name] - - self._log.error(f"Function not available: {s_name}.{f_name}") - raise KernelException( - KernelException.ErrorCodes.FunctionNotAvailable, - f"Function not available: {s_name}.{f_name}", + return self.read_only_skill_collection_.get_native_function( + skill_name, function_name ) def get_functions_view( self, include_semantic: bool = True, include_native: bool = True ) -> FunctionsView: - result = FunctionsView() - - for skill in self._skill_collection.values(): - for function in skill.values(): - if include_semantic and function.is_semantic: - result.add_function(function.describe()) - elif include_native and function.is_native: - result.add_function(function.describe()) - - return result + return self.read_only_skill_collection_.get_functions_view( + include_semantic, include_native + ) def get_function( self, skill_name: Optional[str], function_name: str ) -> "SKFunctionBase": - s_name, f_name = self._normalize_names(skill_name, function_name, True) - if self.has_function(s_name, f_name): - return self._skill_collection[s_name][f_name] - - self._log.error(f"Function not available: {s_name}.{f_name}") - raise KernelException( - KernelException.ErrorCodes.FunctionNotAvailable, - f"Function not available: {s_name}.{f_name}", - ) - - def _normalize_names( - self, - skill_name: Optional[str], - function_name: str, - allow_substitution: bool = False, - ) -> Tuple[str, str]: - s_name, f_name = skill_name, function_name - if s_name is None and allow_substitution: - s_name = self.GLOBAL_SKILL - - if s_name is None: - raise ValueError("The skill name provided cannot be `None`") - - s_name, f_name = s_name.lower(), f_name.lower() - return s_name, f_name - - @static_property - def GLOBAL_SKILL() -> Literal["_GLOBAL_FUNCTIONS_"]: - return "_GLOBAL_FUNCTIONS_" + return self.read_only_skill_collection_.get_function(skill_name, function_name) diff --git a/python/semantic_kernel/skill_definition/skill_collection_base.py b/python/semantic_kernel/skill_definition/skill_collection_base.py index 0ed7c695e033..089ea544b191 100644 --- a/python/semantic_kernel/skill_definition/skill_collection_base.py +++ b/python/semantic_kernel/skill_definition/skill_collection_base.py @@ -1,8 +1,9 @@ # Copyright (c) Microsoft. All rights reserved. from abc import ABC, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.skill_definition.read_only_skill_collection_base import ( ReadOnlySkillCollectionBase, ) @@ -11,7 +12,10 @@ from semantic_kernel.orchestration.sk_function_base import SKFunctionBase -class SkillCollectionBase(ReadOnlySkillCollectionBase, ABC): +SkillCollectionT = TypeVar("SkillCollectionT", bound="SkillCollectionBase") + + +class SkillCollectionBase(ReadOnlySkillCollectionBase, PydanticField, ABC): @property @abstractmethod def read_only_skill_collection(self) -> ReadOnlySkillCollectionBase: diff --git a/python/semantic_kernel/template_engine/blocks/block.py b/python/semantic_kernel/template_engine/blocks/block.py index 86a1a9012131..fd1db61d5132 100644 --- a/python/semantic_kernel/template_engine/blocks/block.py +++ b/python/semantic_kernel/template_engine/blocks/block.py @@ -3,29 +3,30 @@ from logging import Logger from typing import Optional, Tuple +import pydantic as pdt + +from semantic_kernel.sk_pydantic import SKBaseModel from semantic_kernel.template_engine.blocks.block_types import BlockTypes from semantic_kernel.utils.null_logger import NullLogger -class Block: +class Block(SKBaseModel): + content: Optional[str] + _log: Optional[Logger] = pdt.PrivateAttr(default_factory=NullLogger) + def __init__( self, content: Optional[str] = None, log: Optional[Logger] = NullLogger ) -> None: - self._content = content or "" + super().__init__(content=content) self._log = log or NullLogger() - self._type = BlockTypes.UNDEFINED - @property - def type(self) -> BlockTypes: - return self._type + def is_valid(self) -> Tuple[bool, str]: + raise NotImplementedError("Subclasses must implement this method.") @property - def content(self) -> str: - return self._content + def type(self) -> BlockTypes: + return BlockTypes.UNDEFINED @property def log(self) -> Logger: return self._log - - def is_valid(self) -> Tuple[bool, str]: - raise NotImplementedError("Subclasses must implement this method.") diff --git a/python/semantic_kernel/template_engine/blocks/code_block.py b/python/semantic_kernel/template_engine/blocks/code_block.py index 0d04ad11efae..6150f4488e6a 100644 --- a/python/semantic_kernel/template_engine/blocks/code_block.py +++ b/python/semantic_kernel/template_engine/blocks/code_block.py @@ -3,6 +3,8 @@ from logging import Logger from typing import List, Optional, Tuple +import pydantic as pdt + from semantic_kernel.orchestration.sk_function_base import SKFunctionBase from semantic_kernel.skill_definition.read_only_skill_collection_base import ( ReadOnlySkillCollectionBase, @@ -11,10 +13,12 @@ from semantic_kernel.template_engine.blocks.block_types import BlockTypes from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock from semantic_kernel.template_engine.code_tokenizer import CodeTokenizer -from semantic_kernel.template_engine.protocols.code_renderer import CodeRenderer -class CodeBlock(Block, CodeRenderer): +class CodeBlock(Block): + _tokens: List[Block] = pdt.PrivateAttr() + _validated: bool = pdt.PrivateAttr(default=False) + def __init__( self, content: str, diff --git a/python/semantic_kernel/template_engine/blocks/function_id_block.py b/python/semantic_kernel/template_engine/blocks/function_id_block.py index 241c558d0b83..83e3f952b61d 100644 --- a/python/semantic_kernel/template_engine/blocks/function_id_block.py +++ b/python/semantic_kernel/template_engine/blocks/function_id_block.py @@ -4,13 +4,17 @@ from re import match as re_match from typing import Optional, Tuple +import pydantic as pdt + from semantic_kernel.orchestration.context_variables import ContextVariables from semantic_kernel.template_engine.blocks.block import Block from semantic_kernel.template_engine.blocks.block_types import BlockTypes -from semantic_kernel.template_engine.protocols.text_renderer import TextRenderer -class FunctionIdBlock(Block, TextRenderer): +class FunctionIdBlock(Block): + _skill_name: str = pdt.PrivateAttr() + _function_name: str = pdt.PrivateAttr() + def __init__(self, content: Optional[str] = None, log: Optional[Logger] = None): super().__init__(content=content and content.strip(), log=log) @@ -23,16 +27,32 @@ def __init__(self, content: Optional[str] = None, log: Optional[Logger] = None): ) if len(function_name_parts) == 2: - self.skill_name = function_name_parts[0] - self.function_name = function_name_parts[1] + self._skill_name = function_name_parts[0] + self._function_name = function_name_parts[1] else: - self.skill_name = "" - self.function_name = self.content + self._skill_name = "" + self._function_name = self.content @property def type(self) -> BlockTypes: return BlockTypes.FUNCTION_ID + @property + def skill_name(self) -> str: + return self._skill_name + + @skill_name.setter + def skill_name(self, value: str) -> None: + self._skill_name = value + + @property + def function_name(self) -> str: + return self._function_name + + @function_name.setter + def function_name(self, value: str) -> None: + self._function_name = value + def is_valid(self) -> Tuple[bool, str]: if self.content is None or len(self.content) == 0: error_msg = "The function identifier is empty" diff --git a/python/semantic_kernel/template_engine/blocks/text_block.py b/python/semantic_kernel/template_engine/blocks/text_block.py index dec1b17cd9a8..d94530b89fc0 100644 --- a/python/semantic_kernel/template_engine/blocks/text_block.py +++ b/python/semantic_kernel/template_engine/blocks/text_block.py @@ -6,17 +6,19 @@ from semantic_kernel.orchestration.context_variables import ContextVariables from semantic_kernel.template_engine.blocks.block import Block from semantic_kernel.template_engine.blocks.block_types import BlockTypes -from semantic_kernel.template_engine.protocols.text_renderer import TextRenderer -class TextBlock(Block, TextRenderer): - def __init__( - self, +class TextBlock(Block): + @classmethod + def from_text( + cls, text: Optional[str] = None, start_index: Optional[int] = None, stop_index: Optional[int] = None, log: Optional[Logger] = None, ): + if text is None: + return cls(content="", log=log) if start_index is not None and stop_index is not None: if start_index > stop_index: raise ValueError( @@ -33,7 +35,7 @@ def __init__( elif stop_index is not None: text = text[:stop_index] - super().__init__(content=text, log=log) + return cls(content=text, log=log) @property def type(self) -> BlockTypes: diff --git a/python/semantic_kernel/template_engine/blocks/val_block.py b/python/semantic_kernel/template_engine/blocks/val_block.py index f4006f22fcdc..07a8a881f6ed 100644 --- a/python/semantic_kernel/template_engine/blocks/val_block.py +++ b/python/semantic_kernel/template_engine/blocks/val_block.py @@ -3,14 +3,19 @@ from logging import Logger from typing import Optional, Tuple +import pydantic as pdt + from semantic_kernel.orchestration.context_variables import ContextVariables from semantic_kernel.template_engine.blocks.block import Block from semantic_kernel.template_engine.blocks.block_types import BlockTypes from semantic_kernel.template_engine.blocks.symbols import Symbols -from semantic_kernel.template_engine.protocols.text_renderer import TextRenderer -class ValBlock(Block, TextRenderer): +class ValBlock(Block): + _first: str = pdt.PrivateAttr() + _last: str = pdt.PrivateAttr() + _value: str = pdt.PrivateAttr() + def __init__(self, content: Optional[str] = None, log: Optional[Logger] = None): super().__init__(content=content and content.strip(), log=log) diff --git a/python/semantic_kernel/template_engine/blocks/var_block.py b/python/semantic_kernel/template_engine/blocks/var_block.py index 98dc5ffaefb7..0e77166ead0f 100644 --- a/python/semantic_kernel/template_engine/blocks/var_block.py +++ b/python/semantic_kernel/template_engine/blocks/var_block.py @@ -4,29 +4,40 @@ from re import match as re_match from typing import Optional, Tuple +import pydantic as pdt + from semantic_kernel.orchestration.context_variables import ContextVariables from semantic_kernel.template_engine.blocks.block import Block from semantic_kernel.template_engine.blocks.block_types import BlockTypes from semantic_kernel.template_engine.blocks.symbols import Symbols -from semantic_kernel.template_engine.protocols.text_renderer import TextRenderer -class VarBlock(Block, TextRenderer): +class VarBlock(Block): + _name: str = pdt.PrivateAttr() + def __init__(self, content: Optional[str] = None, log: Optional[Logger] = None): super().__init__(content=content and content.strip(), log=log) if len(self.content) < 2: err = "The variable name is empty" self.log.error(err) - self.name = "" + self._name = "" return - self.name = self.content[1:] + self._name = self.content[1:] @property def type(self) -> BlockTypes: return BlockTypes.VARIABLE + @property + def name(self) -> str: + return self._name + + @name.setter + def name(self, value: str) -> None: + self._name = value + def is_valid(self) -> Tuple[bool, str]: if not self.content: error_msg = ( @@ -65,8 +76,8 @@ def render(self, variables: Optional[ContextVariables] = None) -> str: self.log.error(error_msg) raise ValueError(error_msg) - exists, value = variables.get(self.name) - if not exists: + value = variables.get(self.name, None) + if not value: self.log.warning(f"Variable `{Symbols.VAR_PREFIX}{self.name}` not found") - return value if exists else "" + return value or "" diff --git a/python/semantic_kernel/template_engine/code_tokenizer.py b/python/semantic_kernel/template_engine/code_tokenizer.py index 16fc6a3f35e0..9198f807134e 100644 --- a/python/semantic_kernel/template_engine/code_tokenizer.py +++ b/python/semantic_kernel/template_engine/code_tokenizer.py @@ -3,6 +3,7 @@ from logging import Logger from typing import List +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.template_engine.blocks.block import Block from semantic_kernel.template_engine.blocks.block_types import BlockTypes from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock @@ -20,7 +21,7 @@ # [value] ::= "'" [text] "'" | '"' [text] '"' # [function-call] ::= [function-id] | [function-id] [parameter] # [parameter] ::= [variable] | [value] -class CodeTokenizer: +class CodeTokenizer(PydanticField): def __init__(self, log: Logger = None): self.log = log or NullLogger() diff --git a/python/semantic_kernel/template_engine/prompt_template_engine.py b/python/semantic_kernel/template_engine/prompt_template_engine.py index 48af58c9e3c7..bb1eb861e27b 100644 --- a/python/semantic_kernel/template_engine/prompt_template_engine.py +++ b/python/semantic_kernel/template_engine/prompt_template_engine.py @@ -1,23 +1,21 @@ # Copyright (c) Microsoft. All rights reserved. from logging import Logger -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from semantic_kernel.orchestration.context_variables import ContextVariables -from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.template_engine.blocks.block import Block from semantic_kernel.template_engine.blocks.block_types import BlockTypes -from semantic_kernel.template_engine.blocks.text_block import TextBlock -from semantic_kernel.template_engine.protocols.code_renderer import CodeRenderer -from semantic_kernel.template_engine.protocols.prompt_templating_engine import ( - PromptTemplatingEngine, -) from semantic_kernel.template_engine.protocols.text_renderer import TextRenderer from semantic_kernel.template_engine.template_tokenizer import TemplateTokenizer from semantic_kernel.utils.null_logger import NullLogger +if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext -class PromptTemplateEngine(PromptTemplatingEngine): + +class PromptTemplateEngine(PydanticField): def __init__(self, logger: Optional[Logger] = None) -> None: self._logger = logger or NullLogger() self._tokenizer = TemplateTokenizer(self._logger) @@ -46,7 +44,7 @@ def extract_blocks( return blocks - async def render_async(self, template_text: str, context: SKContext) -> str: + async def render_async(self, template_text: str, context: "SKContext") -> str: """ Given a prompt template, replace the variables with their values and execute the functions replacing their reference with the @@ -60,7 +58,9 @@ async def render_async(self, template_text: str, context: SKContext) -> str: blocks = self.extract_blocks(template_text) return await self.render_blocks_async(blocks, context) - async def render_blocks_async(self, blocks: List[Block], context: SKContext) -> str: + async def render_blocks_async( + self, blocks: List[Block], context: "SKContext" + ) -> str: """ Given a list of blocks render each block and compose the final result. @@ -68,6 +68,8 @@ async def render_blocks_async(self, blocks: List[Block], context: SKContext) -> :param context: Access into the current kernel execution context :return: The prompt template ready to be used for an AI request """ + from semantic_kernel.template_engine.protocols.code_renderer import CodeRenderer + self._logger.debug(f"Rendering list of {len(blocks)} blocks") rendered_blocks = [] for block in blocks: @@ -98,6 +100,8 @@ def render_variables( :return: An updated list of blocks where Variable Blocks have rendered to Text Blocks """ + from semantic_kernel.template_engine.blocks.text_block import TextBlock + self._logger.debug("Rendering variables") rendered_blocks = [] @@ -107,12 +111,14 @@ def render_variables( continue if not isinstance(block, TextRenderer): raise ValueError("TextBlock must implement TextRenderer protocol") - rendered_blocks.append(TextBlock(block.render(variables), log=self._logger)) + rendered_blocks.append( + TextBlock.from_text(block.render(variables), log=self._logger) + ) return rendered_blocks async def render_code_async( - self, blocks: List[Block], execution_context: SKContext + self, blocks: List[Block], execution_context: "SKContext" ) -> List[Block]: """ Given a list of blocks, render the Code Blocks, executing the @@ -123,6 +129,9 @@ async def render_code_async( :return: An updated list of blocks where Code Blocks have rendered to Text Blocks """ + from semantic_kernel.template_engine.blocks.text_block import TextBlock + from semantic_kernel.template_engine.protocols.code_renderer import CodeRenderer + self._logger.debug("Rendering code") rendered_blocks = [] @@ -133,7 +142,7 @@ async def render_code_async( if not isinstance(block, CodeRenderer): raise ValueError("CodeBlock must implement CodeRenderer protocol") rendered_blocks.append( - TextBlock( + TextBlock.from_text( await block.render_code_async(execution_context), log=self._logger ) ) diff --git a/python/semantic_kernel/template_engine/protocols/prompt_templating_engine.py b/python/semantic_kernel/template_engine/protocols/prompt_templating_engine.py index 2f76613fff6b..55f2cd442329 100644 --- a/python/semantic_kernel/template_engine/protocols/prompt_templating_engine.py +++ b/python/semantic_kernel/template_engine/protocols/prompt_templating_engine.py @@ -1,12 +1,15 @@ # Copyright (c) Microsoft. All rights reserved. -from typing import List, Optional, Protocol +from typing import TYPE_CHECKING, List, Optional, Protocol, runtime_checkable from semantic_kernel.orchestration.context_variables import ContextVariables -from semantic_kernel.orchestration.sk_context import SKContext from semantic_kernel.template_engine.blocks.block import Block +if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext + +@runtime_checkable class PromptTemplatingEngine(Protocol): """ Prompt templating engine protocol. @@ -27,7 +30,7 @@ def extract_blocks( """ ... - async def render_async(self, template_text: str, context: SKContext) -> str: + async def render_async(self, template_text: str, context: "SKContext") -> str: """ Given a prompt template, replace the variables with their values and execute the functions replacing their reference with the @@ -39,7 +42,9 @@ async def render_async(self, template_text: str, context: SKContext) -> str: """ ... - async def render_blocks_async(self, blocks: List[Block], context: SKContext) -> str: + async def render_blocks_async( + self, blocks: List[Block], context: "SKContext" + ) -> str: """ Given a list of blocks render each block and compose the final result. @@ -64,7 +69,7 @@ def render_variables( ... async def render_code_async( - self, blocks: List[Block], execution_context: SKContext + self, blocks: List[Block], execution_context: "SKContext" ) -> List[Block]: """ Given a list of blocks, render the Code Blocks, executing the diff --git a/python/semantic_kernel/template_engine/template_tokenizer.py b/python/semantic_kernel/template_engine/template_tokenizer.py index f8f29d8f9989..a055356ed261 100644 --- a/python/semantic_kernel/template_engine/template_tokenizer.py +++ b/python/semantic_kernel/template_engine/template_tokenizer.py @@ -3,6 +3,7 @@ from logging import Logger from typing import List +from semantic_kernel.sk_pydantic import PydanticField from semantic_kernel.template_engine.blocks.block import Block from semantic_kernel.template_engine.blocks.block_types import BlockTypes from semantic_kernel.template_engine.blocks.code_block import CodeBlock @@ -20,7 +21,7 @@ # | "{{" [function-call] "}}" # [text-block] ::= [any-char] | [any-char] [text-block] # [any-char] ::= any char -class TemplateTokenizer: +class TemplateTokenizer(PydanticField): def __init__(self, log: Logger = None): self.log = log or NullLogger() self.code_tokenizer = CodeTokenizer(self.log) @@ -36,11 +37,11 @@ def tokenize(self, text: str) -> List[Block]: # Render None/empty to "" if not text or text == "": - return [TextBlock("", log=self.log)] + return [TextBlock.from_text("", log=self.log)] # If the template is "empty" return it as a text block if len(text) < MIN_CODE_BLOCK_LENGTH: - return [TextBlock(text, log=self.log)] + return [TextBlock.from_text(text, log=self.log)] blocks = [] end_of_last_block = 0 @@ -99,7 +100,7 @@ def tokenize(self, text: str) -> List[Block]: # add it as a text block if block_start_pos > end_of_last_block: blocks.append( - TextBlock( + TextBlock.from_text( text, end_of_last_block, block_start_pos, @@ -118,7 +119,9 @@ def tokenize(self, text: str) -> List[Block]: # If what is left is empty, consider the raw block # a TextBlock blocks.append( - TextBlock(content_with_delimiters, log=self.log) + TextBlock.from_text( + content_with_delimiters, log=self.log + ) ) else: code_blocks = self.code_tokenizer.tokenize( @@ -168,7 +171,9 @@ def tokenize(self, text: str) -> List[Block]: # If there is something left after the last block, capture it as a TextBlock if end_of_last_block < len(text): - blocks.append(TextBlock(text, end_of_last_block, len(text), log=self.log)) + blocks.append( + TextBlock.from_text(text, end_of_last_block, len(text), log=self.log) + ) return blocks diff --git a/python/semantic_kernel/utils/null_logger.py b/python/semantic_kernel/utils/null_logger.py index 86f8b48cc913..586d3d1d27a6 100644 --- a/python/semantic_kernel/utils/null_logger.py +++ b/python/semantic_kernel/utils/null_logger.py @@ -1,24 +1,40 @@ # Copyright (c) Microsoft. All rights reserved. +from functools import wraps from logging import Logger +from typing import Any, Callable -class NullLogger(Logger): +def _nullify(fn) -> Callable[[Any], None]: + """General wrapper to not call wrapped function""" + + @wraps(fn) + def _inner_nullify(*args, **kwargs) -> None: + return + + return _inner_nullify + + +class _NullerMeta(type): + def __new__(cls, classname, base_classes, class_dict): + """Return a Class that nullifies all Logger object callbacks""" + nullified_dict = { + attr_name: _nullify(attr) + for attr_name, attr in Logger.__dict__.items() + if callable(attr) + } + return type.__new__( + cls, classname, base_classes, {**class_dict, **nullified_dict} + ) + + +class NullLogger(Logger, metaclass=_NullerMeta): """ A logger that does nothing. """ - def __init__(self) -> None: - pass - - def debug(self, _: str) -> None: - pass - - def info(self, _: str) -> None: - pass + def __init__(self): + super().__init__(None) - def warning(self, _: str) -> None: - pass - def error(self, _: str) -> None: - pass +__all__ = ["NullLogger"] diff --git a/python/semantic_kernel/utils/settings.py b/python/semantic_kernel/utils/settings.py index 695c92d581c5..c75319e75647 100644 --- a/python/semantic_kernel/utils/settings.py +++ b/python/semantic_kernel/utils/settings.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. -from typing import Optional, Tuple +from typing import Dict, Optional, Tuple, Union from dotenv import dotenv_values @@ -17,39 +17,80 @@ def openai_settings_from_dot_env() -> Tuple[str, Optional[str]]: api_key = config.get("OPENAI_API_KEY", None) org_id = config.get("OPENAI_ORG_ID", None) - assert api_key is not None, "OpenAI API key not found in .env file" + assert api_key, "OpenAI API key not found in .env file" # It's okay if the org ID is not found (not required) return api_key, org_id -def azure_openai_settings_from_dot_env(include_deployment=True) -> Tuple[str, str, str]: +def azure_openai_settings_from_dot_env( + include_deployment: bool = True, include_api_version: bool = False +) -> Union[Tuple[str, str, str], Tuple[str, str, str, str]]: """ Reads the Azure OpenAI API key and endpoint from the .env file. + Arguments: + include_deployment {bool} -- Whether to include the deployment name in the return value + include_api_version {bool} -- Whether to include the API version in the return value, + when set to True, this will also make the output a Tuple[str, str, str, str]. + Returns: - Tuple[str, str, str]: The deployment name (or empty), Azure OpenAI API key, - and the endpoint + Union[Tuple[str, str, str], Tuple[str, str, str, str]]: The deployment name (or empty), Azure OpenAI API key, + the endpoint and optionally the api version """ - deployment, api_key, endpoint = None, None, None + deployment, api_key, endpoint, api_version = None, None, None, None config = dotenv_values(".env") deployment = config.get("AZURE_OPENAI_DEPLOYMENT_NAME", None) api_key = config.get("AZURE_OPENAI_API_KEY", None) endpoint = config.get("AZURE_OPENAI_ENDPOINT", None) + api_version = config.get("AZURE_OPENAI_API_VERSION", None) # Azure requires the deployment name, the API key and the endpoint URL. if include_deployment: assert ( deployment is not None ), "Azure OpenAI deployment name not found in .env file" + if include_api_version: + assert ( + api_version is not None + ), "Azure OpenAI API version not found in .env file" - assert api_key is not None, "Azure OpenAI API key not found in .env file" - assert endpoint is not None, "Azure OpenAI endpoint not found in .env file" + assert api_key, "Azure OpenAI API key not found in .env file" + assert endpoint, "Azure OpenAI endpoint not found in .env file" + if include_api_version: + return deployment or "", api_key, endpoint, api_version or "" return deployment or "", api_key, endpoint +def azure_openai_settings_from_dot_env_as_dict( + include_deployment: bool = True, include_api_version: bool = False +) -> Dict[str, str]: + """ + Reads the Azure OpenAI API key and endpoint from the .env file. + + Returns: + Dict[str, str]: The deployment name (or empty), Azure OpenAI API key, + endpoint and api version (or empty) + """ + ( + deployment_name, + api_key, + endpoint, + api_version, + ) = azure_openai_settings_from_dot_env(include_deployment, include_api_version) + ret = { + "api_key": api_key, + "endpoint": endpoint, + } + if include_deployment: + ret["deployment_name"] = deployment_name + if include_api_version: + ret["api_version"] = api_version + return ret + + def postgres_settings_from_dot_env() -> str: """Reads the Postgres connection string from the .env file. @@ -60,26 +101,114 @@ def postgres_settings_from_dot_env() -> str: config = dotenv_values(".env") connection_string = config.get("POSTGRES_CONNECTION_STRING", None) - assert ( - connection_string is not None - ), "Postgres connection string not found in .env file" + assert connection_string, "Postgres connection string not found in .env file" return connection_string -def pinecone_settings_from_dot_env() -> Tuple[str, str]: - """Reads the Pinecone API key and Environment from the .env file. - +def pinecone_settings_from_dot_env() -> Tuple[str, Optional[str]]: + """ + Reads the Pinecone API key and Environment from the .env file. Returns: Tuple[str, str]: The Pinecone API key, the Pinecone Environment """ api_key, environment = None, None - config = dotenv_values(".env") - api_key = config.get("PINECONE_API_KEY", None) - environment = config.get("PINECONE_ENVIRONMENT", None) + with open(".env", "r") as f: + lines = f.readlines() + + for line in lines: + if line.startswith("PINECONE_API_KEY"): + parts = line.split("=")[1:] + api_key = "=".join(parts).strip().strip('"') + continue + + if line.startswith("PINECONE_ENVIRONMENT"): + parts = line.split("=")[1:] + environment = "=".join(parts).strip().strip('"') + continue - assert api_key is not None, "Pinecone API key not found in .env file" - assert environment is not None, "Pinecone environment not found in .env file" + assert api_key, "Pinecone API key not found in .env file" + assert environment, "Pinecone environment not found in .env file" return api_key, environment + + +def weaviate_settings_from_dot_env() -> Tuple[Optional[str], str]: + """ + Reads the Weaviate API key and URL from the .env file. + + Returns: + Tuple[str, str]: The Weaviate API key, the Weaviate URL + """ + + config = dotenv_values(".env") + api_key = config.get("WEAVIATE_API_KEY", None) + url = config.get("WEAVIATE_URL", None) + + # API key not needed for local Weaviate deployment, URL still needed + assert url is not None, "Weaviate instance URL not found in .env file" + + return api_key, url + + +def bing_search_settings_from_dot_env() -> str: + """Reads the Bing Search API key from the .env file. + + Returns: + Tuple[str, str]: The Bing Search API key, the Bing Search endpoint + """ + + api_key = None + config = dotenv_values(".env") + api_key = config.get("BING_API_KEY", None) + + assert api_key is not None, "Bing Search API key not found in .env file" + + return api_key + + +def mongodb_atlas_settings_from_dot_env() -> str: + """Returns the Atlas MongoDB Connection String from the .env file. + + Returns: + str: MongoDB Connection String URI + """ + + config = dotenv_values(".env") + uri = config.get("MONGODB_ATLAS_CONNECTION_STRING") + assert uri is not None, "MongoDB Connection String not found in .env file" + + return uri + + +def google_palm_settings_from_dot_env() -> str: + """ + Reads the Google PaLM API key from the .env file. + + Returns: + str: The Google PaLM API key + """ + + config = dotenv_values(".env") + api_key = config.get("GOOGLE_PALM_API_KEY", None) + + assert api_key is not None, "Google PaLM API key not found in .env file" + + return api_key + + +def redis_settings_from_dot_env() -> str: + """Reads the Redis connection string from the .env file. + + Returns: + str: The Redis connection string + """ + config = dotenv_values(".env") + connection_string = config.get("REDIS_CONNECTION_STRING", None) + + assert ( + connection_string is not None + ), "Redis connection string not found in .env file" + + return connection_string diff --git a/python/tests/conftest.py b/python/tests/conftest.py index a90ffb90ddf0..25794c29e725 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -1,10 +1,57 @@ # Copyright (c) Microsoft. All rights reserved. import os +import typing as t +import warnings import pytest import semantic_kernel as sk +from semantic_kernel.memory.null_memory import NullMemory +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function import SKFunction +from semantic_kernel.skill_definition.read_only_skill_collection import ( + ReadOnlySkillCollection, +) + + +@pytest.fixture(autouse=True) +def enable_debug_mode(): + """Set `autouse=True` to enable easy debugging for tests. + + How to debug: + 1. Ensure [snoop](https://github.com/alexmojaki/snoop) is installed + (`pip install snoop`). + 2. If you're doing print based debugging, use `pr` instead of `print`. + That is, convert `print(some_var)` to `pr(some_var)`. + 3. If you want a trace of a particular functions calls, just add `ss()` as the first + line of the function. + + NOTE: + ---- + It's completely fine to leave `autouse=True` in the fixture. It doesn't affect + the tests unless you use `pr` or `ss` in any test. + + NOTE: + ---- + When you use `ss` or `pr` in a test, pylance or mypy will complain. This is + because they don't know that we're adding these functions to the builtins. The + tests will run fine though. + """ + import builtins + + try: + import snoop + except ImportError: + warnings.warn( + "Install snoop to enable trace debugging. `pip install snoop`", + ImportWarning, + ) + return + + builtins.ss = snoop.snoop(depth=4).__enter__ + builtins.pr = snoop.pp @pytest.fixture(scope="session") @@ -37,3 +84,37 @@ def get_oai_config(): api_key, org_id = sk.openai_settings_from_dot_env() return api_key, org_id + + +@pytest.fixture() +def context_factory() -> t.Callable[[ContextVariables], SKContext]: + """Return a factory for SKContext objects.""" + + def create_context( + context_variables: ContextVariables, *functions: SKFunction + ) -> SKContext: + """Return a SKContext object.""" + return SKContext( + context_variables, + NullMemory(), + skill_collection=ReadOnlySkillCollection( + data={ + ReadOnlySkillCollection.GLOBAL_SKILL.lower(): { + f.name: f for f in functions + } + }, + ), + ) + + return create_context + + +@pytest.fixture(scope="session") +def get_gp_config(): + if "Python_Integration_Tests" in os.environ: + api_key = os.environ["GOOGLE_PALM_API_KEY"] + else: + # Load credentials from .env file + api_key = sk.google_palm_settings_from_dot_env() + + return api_key diff --git a/python/tests/integration/completions/conftest.py b/python/tests/integration/completions/conftest.py index 323335ff560e..7b0a48ceca49 100644 --- a/python/tests/integration/completions/conftest.py +++ b/python/tests/integration/completions/conftest.py @@ -1,9 +1,14 @@ # Copyright (c) Microsoft. All rights reserved. +import sys + import pytest import semantic_kernel.connectors.ai.hugging_face as sk_hf +if sys.version_info >= (3, 9): + import semantic_kernel.connectors.ai.google_palm as sk_gp + @pytest.fixture( scope="module", @@ -149,3 +154,27 @@ def setup_summarize_conversation_using_skill(create_kernel): John: Yeah, that's a good idea.""" yield kernel, ChatTranscript + + +@pytest.fixture(scope="module") +def setup_gp_text_completion_function(create_kernel, get_gp_config): + kernel = create_kernel + api_key = get_gp_config + # Configure LLM service + palm_text_completion = sk_gp.GooglePalmTextCompletion( + "models/text-bison-001", api_key + ) + kernel.add_text_completion_service("models/text-bison-001", palm_text_completion) + + # Define semantic function using SK prompt template language + sk_prompt = "Hello, I like {{$input}}{{$input2}}" + + # Create the semantic function + text2text_function = kernel.create_semantic_function( + sk_prompt, max_tokens=25, temperature=0.7, top_p=0.5 + ) + + # User input + simple_input = "sleeping and " + + yield kernel, text2text_function, simple_input diff --git a/python/tests/integration/completions/test_conversation_summary_skill.py b/python/tests/integration/completions/test_conversation_summary_skill.py index 0eb86d427251..5352ab2847ee 100644 --- a/python/tests/integration/completions/test_conversation_summary_skill.py +++ b/python/tests/integration/completions/test_conversation_summary_skill.py @@ -14,7 +14,7 @@ @pytest.mark.asyncio async def test_azure_summarize_conversation_using_skill( - setup_summarize_conversation_using_skill, + setup_summarize_conversation_using_skill, get_aoai_config ): kernel, chatTranscript = setup_summarize_conversation_using_skill @@ -24,7 +24,7 @@ async def test_azure_summarize_conversation_using_skill( endpoint = os.environ["AzureOpenAI__Endpoint"] else: # Load credentials from .env file - deployment_name, api_key, endpoint = sk.azure_openai_settings_from_dot_env() + deployment_name, api_key, endpoint = get_aoai_config deployment_name = "text-davinci-003" kernel.add_text_completion_service( diff --git a/python/tests/integration/completions/test_gp_chat_service.py b/python/tests/integration/completions/test_gp_chat_service.py new file mode 100644 index 000000000000..e4701e3eed95 --- /dev/null +++ b/python/tests/integration/completions/test_gp_chat_service.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import os +import sys + +import pytest + +if sys.version_info >= (3, 9): + import semantic_kernel.connectors.ai.google_palm as sk_gp + +pytestmark = [ + pytest.mark.skipif( + sys.version_info < (3, 9), reason="Google Palm requires Python 3.9 or greater" + ), + pytest.mark.skipif( + "Python_Integration_Tests" in os.environ, + reason="Google Palm integration tests are only set up to run locally", + ), +] + + +@pytest.mark.asyncio +async def test_gp_chat_service_with_skills( + setup_tldr_function_for_oai_models, get_gp_config +): + kernel, sk_prompt, text_to_summarize = setup_tldr_function_for_oai_models + api_key = get_gp_config + + print("* Service: Google PaLM Chat Completion") + print("* Model: chat-bison-001") + palm_chat_completion = sk_gp.GooglePalmChatCompletion( + "models/chat-bison-001", api_key + ) + kernel.add_chat_service("models/chat-bison-001", palm_chat_completion) + + # Create the semantic function + tldr_function = kernel.create_semantic_function( + sk_prompt, max_tokens=200, temperature=0, top_p=0.5 + ) + + max_retries = 5 # Adjust the number of retries as per your requirement + retry_delay = 2 # Adjust the delay (in seconds) between retries + + for _ in range(max_retries): + try: + summary = await kernel.run_async(tldr_function, input_str=text_to_summarize) + output = str(summary).strip() + print(f"TLDR using input string: '{output}'") + assert "First Law" not in output and ( + "human" in output or "Human" in output or "preserve" in output + ) + assert len(output) < 100 + break + except Exception as e: + print(f"Error occurred: {e}") + await asyncio.sleep(retry_delay) # Introduce a delay before the next retry + else: + # The loop completed without breaking, meaning all retries failed + raise AssertionError("Test failed after multiple retries") diff --git a/python/tests/integration/completions/test_gp_text_service.py b/python/tests/integration/completions/test_gp_text_service.py new file mode 100644 index 000000000000..125f771fb20b --- /dev/null +++ b/python/tests/integration/completions/test_gp_text_service.py @@ -0,0 +1,118 @@ +# Copyright (c) Microsoft. All rights reserved. + +import os +import sys + +import pytest + +import semantic_kernel as sk + +pytestmark = [ + pytest.mark.skipif( + sys.version_info < (3, 9), reason="Google Palm requires Python 3.9 or greater" + ), + pytest.mark.skipif( + "Python_Integration_Tests" in os.environ, + reason="Google Palm integration tests are only set up to run locally", + ), +] + + +@pytest.mark.asyncio +async def test_text2text_generation_input_str(setup_gp_text_completion_function): + kernel, text2text_function, simple_input = setup_gp_text_completion_function + + # Complete input string and print + summary = await kernel.run_async(text2text_function, input_str=simple_input) + + output = str(summary).strip() + print(f"Completion using input string: '{output}'") + assert len(output) > 0 + + +@pytest.mark.asyncio +async def test_text2text_generation_input_vars(setup_gp_text_completion_function): + kernel, text2text_function, simple_input = setup_gp_text_completion_function + + # Complete input as context variable and print + context_vars = sk.ContextVariables(simple_input) + summary = await kernel.run_async(text2text_function, input_vars=context_vars) + + output = str(summary).strip() + print(f"Completion using context variables: '{output}'") + assert len(output) > 0 + + +@pytest.mark.asyncio +async def test_text2text_generation_input_context(setup_gp_text_completion_function): + kernel, text2text_function, simple_input = setup_gp_text_completion_function + + # Complete input context and print + context = kernel.create_new_context() + context["input"] = simple_input + summary = await kernel.run_async(text2text_function, input_context=context) + + output = str(summary).strip() + print(f"Completion using input context: '{output}'") + assert len(output) > 0 + + +@pytest.mark.asyncio +async def test_text2text_generation_input_context_with_vars( + setup_gp_text_completion_function, +): + kernel, text2text_function, simple_input = setup_gp_text_completion_function + + # Complete input context with additional variables and print + context = kernel.create_new_context() + context["input"] = simple_input + context_vars = sk.ContextVariables("running and") + summary = await kernel.run_async( + text2text_function, input_context=context, input_vars=context_vars + ) + + output = str(summary).strip() + print(f"Completion using context and additional variables: '{output}'") + assert len(output) > 0 + + +@pytest.mark.asyncio +async def test_text2text_generation_input_context_with_str( + setup_gp_text_completion_function, +): + kernel, text2text_function, simple_input = setup_gp_text_completion_function + + # Complete input context with additional input string and print + context = kernel.create_new_context() + context["input"] = simple_input + summary = await kernel.run_async( + text2text_function, input_context=context, input_str="running and" + ) + + output = str(summary).strip() + print(f"Completion using context and additional string: '{output}'") + assert len(output) > 0 + + +@pytest.mark.asyncio +async def test_text2text_generation_input_context_with_vars_and_str( + setup_gp_text_completion_function, +): + kernel, text2text_function, simple_input = setup_gp_text_completion_function + + # Complete input context with additional variables and string and print + context = kernel.create_new_context() + context["input"] = simple_input + context_vars = sk.ContextVariables(variables={"input2": "running and"}) + summary = await kernel.run_async( + text2text_function, + input_context=context, + input_vars=context_vars, + input_str="new text", + ) + + output = str(summary).strip() + print( + f"Completion using context, additional variables, and additional string: '{output}'" + ) + assert len(output) > 0 diff --git a/python/tests/integration/completions/test_hf_local_text_completions.py b/python/tests/integration/completions/test_hf_local_text_completions.py index 669e0df5c265..f5af64418518 100644 --- a/python/tests/integration/completions/test_hf_local_text_completions.py +++ b/python/tests/integration/completions/test_hf_local_text_completions.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft. All rights reserved. import pytest +from transformers import AutoTokenizer import semantic_kernel as sk +import semantic_kernel.connectors.ai.hugging_face as sk_hf @pytest.mark.asyncio @@ -103,3 +105,48 @@ async def test_text2text_generation_input_context_with_vars_and_str( f"Completion using context, additional variables, and additional string: '{output}'" ) assert len(output) > 0 + + +@pytest.mark.asyncio +async def test_text_generation_with_kwargs(): + simple_input = "sleeping and " + model_name = "google/flan-t5-base" + + tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path=model_name, trust_remote_code=True + ) + + hf_model = sk_hf.HuggingFaceTextCompletion( + model_name, + task="text2text-generation", + model_kwargs={"repetition_penalty": 0.2}, + pipeline_kwargs={"tokenizer": tokenizer, "trust_remote_code": True}, + ) + + kernel = sk.Kernel() + + # Configure LLM service + kernel.add_text_completion_service("hf-local", hf_model) + + # Define semantic function using SK prompt template language + sk_prompt = "Hello, I like {{$input}}{{$input2}}" + text2text_function = kernel.create_semantic_function( + sk_prompt, max_tokens=25, temperature=0.2, top_p=0.5 + ) + + # Complete input context with additional variables and string and print + context = kernel.create_new_context() + context["input"] = simple_input + context_vars = sk.ContextVariables(variables={"input2": "running and"}) + summary = await kernel.run_async( + text2text_function, + input_context=context, + input_vars=context_vars, + input_str="new text", + ) + + output = str(summary).strip() + print( + f"Completion using context, additional variables, and additional string: '{output}'" + ) + assert len(output) > 0 diff --git a/python/tests/integration/connectors/memory/test_chroma.py b/python/tests/integration/connectors/memory/test_chroma.py index 3d9f394993d4..a580c937b12d 100644 --- a/python/tests/integration/connectors/memory/test_chroma.py +++ b/python/tests/integration/connectors/memory/test_chroma.py @@ -1,5 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. +import asyncio + import numpy as np import pytest @@ -18,6 +20,16 @@ ) +@pytest.fixture +def setup_chroma(): + persist_directory = "chroma/TEMP/" + memory = ChromaMemoryStore(persist_directory=persist_directory) + yield memory + collections = asyncio.run(memory.get_collections_async()) + for collection in collections: + asyncio.run(memory.delete_collection_async(collection)) + + @pytest.fixture def memory_record1(): return MemoryRecord( @@ -46,14 +58,14 @@ def memory_record2(): ) -def test_constructor(): - memory = ChromaMemoryStore() +def test_constructor(setup_chroma): + memory = setup_chroma assert memory._client is not None @pytest.mark.asyncio -async def test_create_and_get_collection_async(): - memory = ChromaMemoryStore() +async def test_create_and_get_collection_async(setup_chroma): + memory = setup_chroma await memory.create_collection_async("test_collection") result = await memory.get_collection_async("test_collection") @@ -61,8 +73,8 @@ async def test_create_and_get_collection_async(): @pytest.mark.asyncio -async def test_get_collections_async(): - memory = ChromaMemoryStore() +async def test_get_collections_async(setup_chroma): + memory = setup_chroma await memory.create_collection_async("test_collection1") await memory.create_collection_async("test_collection2") @@ -73,8 +85,8 @@ async def test_get_collections_async(): @pytest.mark.asyncio -async def test_delete_collection_async(): - memory = ChromaMemoryStore() +async def test_delete_collection_async(setup_chroma): + memory = setup_chroma await memory.create_collection_async("test_collection") await memory.delete_collection_async("test_collection") @@ -88,8 +100,8 @@ async def test_delete_collection_async(): @pytest.mark.asyncio -async def test_does_collection_exist_async(): - memory = ChromaMemoryStore() +async def test_does_collection_exist_async(setup_chroma): + memory = setup_chroma await memory.create_collection_async("test_collection") result = await memory.does_collection_exist_async("test_collection") assert result is True @@ -99,8 +111,8 @@ async def test_does_collection_exist_async(): @pytest.mark.asyncio -async def test_upsert_and_get_async(memory_record1): - memory = ChromaMemoryStore() +async def test_upsert_and_get_async(setup_chroma, memory_record1): + memory = setup_chroma await memory.create_collection_async("test_collection") collection = await memory.get_collection_async("test_collection") @@ -118,8 +130,8 @@ async def test_upsert_and_get_async(memory_record1): @pytest.mark.asyncio -async def test_upsert_and_get_async_with_no_embedding(memory_record1): - memory = ChromaMemoryStore() +async def test_upsert_and_get_async_with_no_embedding(setup_chroma, memory_record1): + memory = setup_chroma await memory.create_collection_async("test_collection") collection = await memory.get_collection_async("test_collection") @@ -137,8 +149,8 @@ async def test_upsert_and_get_async_with_no_embedding(memory_record1): @pytest.mark.asyncio -async def test_upsert_and_get_batch_async(memory_record1, memory_record2): - memory = ChromaMemoryStore() +async def test_upsert_and_get_batch_async(setup_chroma, memory_record1, memory_record2): + memory = setup_chroma await memory.create_collection_async("test_collection") collection = await memory.get_collection_async("test_collection") @@ -159,8 +171,8 @@ async def test_upsert_and_get_batch_async(memory_record1, memory_record2): @pytest.mark.asyncio -async def test_remove_async(memory_record1): - memory = ChromaMemoryStore() +async def test_remove_async(setup_chroma, memory_record1): + memory = setup_chroma await memory.create_collection_async("test_collection") collection = await memory.get_collection_async("test_collection") @@ -173,8 +185,8 @@ async def test_remove_async(memory_record1): @pytest.mark.asyncio -async def test_remove_batch_async(memory_record1, memory_record2): - memory = ChromaMemoryStore() +async def test_remove_batch_async(setup_chroma, memory_record1, memory_record2): + memory = setup_chroma await memory.create_collection_async("test_collection") collection = await memory.get_collection_async("test_collection") @@ -188,8 +200,8 @@ async def test_remove_batch_async(memory_record1, memory_record2): @pytest.mark.asyncio -async def test_get_nearest_matches_async(memory_record1, memory_record2): - memory = ChromaMemoryStore() +async def test_get_nearest_matches_async(setup_chroma, memory_record1, memory_record2): + memory = setup_chroma await memory.create_collection_async("test_collection") collection = await memory.get_collection_async("test_collection") @@ -205,8 +217,8 @@ async def test_get_nearest_matches_async(memory_record1, memory_record2): @pytest.mark.asyncio -async def test_get_nearest_match_async(memory_record1, memory_record2): - memory = ChromaMemoryStore() +async def test_get_nearest_match_async(setup_chroma, memory_record1, memory_record2): + memory = setup_chroma await memory.create_collection_async("test_collection") collection = await memory.get_collection_async("test_collection") diff --git a/python/tests/integration/connectors/memory/test_mongodb_atlas.py b/python/tests/integration/connectors/memory/test_mongodb_atlas.py new file mode 100644 index 000000000000..4f1013dcf316 --- /dev/null +++ b/python/tests/integration/connectors/memory/test_mongodb_atlas.py @@ -0,0 +1,264 @@ +# Copyright (c) Microsoft. All rights reserved. +import os +import random +import time + +import numpy as np +import pytest +import pytest_asyncio +from pymongo import errors + +from semantic_kernel.connectors.memory.mongodb_atlas.mongodb_atlas_memory_store import ( + MongoDBAtlasMemoryStore, +) +from semantic_kernel.memory.memory_record import MemoryRecord + +mongodb_atlas_installed: bool +try: + import motor # noqa: F401 + + mongodb_atlas_installed = True +except ImportError: + mongodb_atlas_installed = False + +pytestmark = pytest.mark.skipif( + not mongodb_atlas_installed, + reason="MongoDB Atlas Vector Search not installed; pip install motor", +) +DUPLICATE_INDEX_ERR_CODE = 68 +READ_ONLY_COLLECTION = "nearestSearch" +DIMENSIONS = 3 + + +def is_equal_memory_record( + mem1: MemoryRecord, mem2: MemoryRecord, with_embeddings: bool +): + """Comparator for two memory records""" + + def dictify_memory_record(mem): + return {k: v for k, v in mem.__dict__.items() if k != "_embedding"} + + assert dictify_memory_record(mem1) == dictify_memory_record(mem2) + if with_embeddings: + assert mem1._embedding.tolist() == mem2._embedding.tolist() + + +@pytest.fixture +def memory_record_gen(): + def memory_record(_id): + return MemoryRecord( + id=str(_id), + text=f"{_id} text", + is_reference=False, + embedding=np.array([1 / (_id + val) for val in range(0, DIMENSIONS)]), + description=f"{_id} description", + external_source_name=f"{_id} external source", + additional_metadata=f"{_id} additional metadata", + timestamp=None, + key=str(_id), + ) + + return memory_record + + +@pytest.fixture +def test_collection(): + return f"AVSTest-{random.randint(0,9999)}" + + +@pytest_asyncio.fixture +async def vector_search_store(): + if "Python_Integration_Tests" in os.environ: + connection_string = os.environ["MONGODB_ATLAS_CONNECTION_STRING"] + async with MongoDBAtlasMemoryStore( + connection_string=connection_string, database_name="pyMSKTest" + ) as memory: + # Delete all collections before and after + for cname in await memory.get_collections_async(): + await memory.delete_collection_async(cname) + + def patch_index_exception(fn): + """Function patch for collection creation call to retry + on duplicate index errors + """ + + async def _patch(collection_name): + while True: + try: + await fn(collection_name) + break + except errors.OperationFailure as e: + # In this test instance, this error code is indicative + # of a previous index not completing teardown + if e.code != DUPLICATE_INDEX_ERR_CODE: + raise + time.sleep(1) + + return _patch + + memory.create_collection_async = patch_index_exception( + memory.create_collection_async + ) + + try: + yield memory + finally: + pass + for cname in await memory.get_collections_async(): + await memory.delete_collection_async(cname) + + +@pytest_asyncio.fixture +async def nearest_match_store(): + """Fixture for read only vector store; the URI for test needs atlas configured""" + if "Python_Integration_Tests" in os.environ: + connection_string = os.environ["MONGODB_ATLAS_CONNECTION_STRING"] + async with MongoDBAtlasMemoryStore( + connection_string=connection_string, database_name="pyMSKTest" + ) as memory: + if not await memory.does_collection_exist_async("nearestSearch"): + pytest.skip( + reason="db: readOnly collection: nearestSearch not found, " + + "please ensure your Atlas Test Cluster has this collection configured" + ) + yield memory + + +@pytest.mark.asyncio +async def test_constructor(vector_search_store): + assert isinstance(vector_search_store, MongoDBAtlasMemoryStore) + + +@pytest.mark.asyncio +async def test_collection_create_and_delete(vector_search_store, test_collection): + await vector_search_store.create_collection_async(test_collection) + assert await vector_search_store.does_collection_exist_async(test_collection) + await vector_search_store.delete_collection_async(test_collection) + assert not await vector_search_store.does_collection_exist_async(test_collection) + + +@pytest.mark.asyncio +async def test_collection_upsert( + vector_search_store, test_collection, memory_record_gen +): + mems = [memory_record_gen(i) for i in range(1, 4)] + mem1 = await vector_search_store.upsert_async(test_collection, mems[0]) + assert mem1 == mems[0]._id + + +@pytest.mark.asyncio +async def test_collection_batch_upsert( + vector_search_store, test_collection, memory_record_gen +): + mems = [memory_record_gen(i) for i in range(1, 4)] + mems_check = await vector_search_store.upsert_batch_async(test_collection, mems) + assert [m._id for m in mems] == mems_check + + +@pytest.mark.asyncio +async def test_collection_deletion( + vector_search_store, test_collection, memory_record_gen +): + mem = memory_record_gen(1) + await vector_search_store.upsert_async(test_collection, mem) + insertion_val = await vector_search_store.get_async(test_collection, mem._id, True) + assert mem._id == insertion_val._id + assert mem._embedding.tolist() == insertion_val._embedding.tolist() + assert insertion_val is not None + await vector_search_store.remove_async(test_collection, mem._id) + val = await vector_search_store.get_async(test_collection, mem._id, False) + assert val is None + + +@pytest.mark.asyncio +async def test_collection_batch_deletion( + vector_search_store, test_collection, memory_record_gen +): + mems = [memory_record_gen(i) for i in range(1, 4)] + await vector_search_store.upsert_batch_async(test_collection, mems) + ids = [mem._id for mem in mems] + insertion_val = await vector_search_store.get_batch_async( + test_collection, ids, True + ) + assert len(insertion_val) == len(mems) + await vector_search_store.remove_batch_async(test_collection, ids) + assert not await vector_search_store.get_batch_async(test_collection, ids, False) + + +@pytest.mark.asyncio +async def test_collection_get(vector_search_store, test_collection, memory_record_gen): + mem = memory_record_gen(1) + await vector_search_store.upsert_async(test_collection, mem) + insertion_val = await vector_search_store.get_async(test_collection, mem._id, False) + is_equal_memory_record(mem, insertion_val, False) + + refetched_record = await vector_search_store.get_async( + test_collection, mem._id, True + ) + is_equal_memory_record(mem, refetched_record, True) + + +@pytest.mark.asyncio +async def test_collection_batch_get( + vector_search_store, test_collection, memory_record_gen +): + mems = {str(i): memory_record_gen(i) for i in range(1, 4)} + await vector_search_store.upsert_batch_async(test_collection, list(mems.values())) + insertion_val = await vector_search_store.get_batch_async( + test_collection, list(mems.keys()), False + ) + assert len(insertion_val) == len(mems) + for val in insertion_val: + is_equal_memory_record(mems[val._id], val, False) + + refetched_vals = await vector_search_store.get_batch_async( + test_collection, list(mems.keys()), True + ) + for ref in refetched_vals: + is_equal_memory_record(mems[ref._id], ref, True) + + +@pytest.mark.asyncio +async def test_collection_knn_match(nearest_match_store, memory_record_gen): + mem = memory_record_gen(7) + await nearest_match_store.upsert_async(READ_ONLY_COLLECTION, mem) + result, score = await nearest_match_store.get_nearest_match_async( + collection_name=READ_ONLY_COLLECTION, + embedding=mem._embedding, + with_embedding=True, + ) + is_equal_memory_record(mem, result, True) + assert score + + +async def knn_matcher( + nearest_match_store, + test_collection, + mems, + query_limit, + expected_limit, +): + results_and_scores = await nearest_match_store.get_nearest_matches_async( + collection_name=test_collection, + embedding=mems["2"]._embedding, + limit=query_limit, + with_embeddings=True, + ) + assert len(results_and_scores) == expected_limit + scores = [score for _, score in results_and_scores] + assert scores == sorted(scores, reverse=True) + for result, _ in results_and_scores: + is_equal_memory_record(mems[result._id], result, True) + + +@pytest.mark.asyncio +async def test_collection_knn_matches(nearest_match_store, memory_record_gen): + mems = {str(i): memory_record_gen(i) for i in range(1, 4)} + await nearest_match_store.upsert_batch_async(READ_ONLY_COLLECTION, mems.values()) + await knn_matcher( + nearest_match_store, + READ_ONLY_COLLECTION, + mems, + query_limit=2, + expected_limit=2, + ) diff --git a/python/tests/integration/connectors/memory/test_pinecone.py b/python/tests/integration/connectors/memory/test_pinecone.py index ee45aacac00a..583ed2cbb0f8 100644 --- a/python/tests/integration/connectors/memory/test_pinecone.py +++ b/python/tests/integration/connectors/memory/test_pinecone.py @@ -22,7 +22,7 @@ ) -async def retry(func, retries=5): +async def retry(func, retries=1): for i in range(retries): try: return await func() @@ -37,7 +37,7 @@ async def retry(func, retries=5): @pytest.fixture(autouse=True, scope="module") def slow_down_tests(): yield - time.sleep(1) + time.sleep(3) @pytest.fixture(scope="session") @@ -101,6 +101,9 @@ def test_constructor(get_pinecone_config): @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_create_and_get_collection_async(get_pinecone_config): api_key, environment = get_pinecone_config memory = PineconeMemoryStore(api_key, environment, 2) @@ -112,6 +115,9 @@ async def test_create_and_get_collection_async(get_pinecone_config): @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_get_collections_async(get_pinecone_config): api_key, environment = get_pinecone_config memory = PineconeMemoryStore(api_key, environment, 2) @@ -122,6 +128,9 @@ async def test_get_collections_async(get_pinecone_config): @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_delete_collection_async(get_pinecone_config): api_key, environment = get_pinecone_config memory = PineconeMemoryStore(api_key, environment, 2) @@ -133,6 +142,9 @@ async def test_delete_collection_async(get_pinecone_config): @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_does_collection_exist_async(get_pinecone_config): api_key, environment = get_pinecone_config memory = PineconeMemoryStore(api_key, environment, 2) @@ -143,6 +155,9 @@ async def test_does_collection_exist_async(get_pinecone_config): @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_upsert_async_and_get_async(get_pinecone_config, memory_record1): api_key, environment = get_pinecone_config memory = PineconeMemoryStore(api_key, environment, 2) @@ -166,6 +181,9 @@ async def test_upsert_async_and_get_async(get_pinecone_config, memory_record1): @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_upsert_batch_async_and_get_batch_async( get_pinecone_config, memory_record1, memory_record2 ): @@ -193,6 +211,9 @@ async def test_upsert_batch_async_and_get_batch_async( @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_remove_async(get_pinecone_config, memory_record1): api_key, environment = get_pinecone_config memory = PineconeMemoryStore(api_key, environment, 2) @@ -208,6 +229,9 @@ async def test_remove_async(get_pinecone_config, memory_record1): @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_remove_batch_async(get_pinecone_config, memory_record1, memory_record2): api_key, environment = get_pinecone_config memory = PineconeMemoryStore(api_key, environment, 2) @@ -236,6 +260,9 @@ async def test_remove_batch_async(get_pinecone_config, memory_record1, memory_re @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_get_nearest_match_async( get_pinecone_config, memory_record1, memory_record2 ): @@ -266,6 +293,9 @@ async def test_get_nearest_match_async( @pytest.mark.asyncio +@pytest.mark.xfail( + reason="Test failed due to known unreliable communications with Pinecone free tier" +) async def test_get_nearest_matches_async( get_pinecone_config, memory_record1, memory_record2, memory_record3 ): diff --git a/python/tests/integration/connectors/memory/test_qdrant_memory_store.py b/python/tests/integration/connectors/memory/test_qdrant_memory_store.py index e6518175c023..67fa74f109b7 100644 --- a/python/tests/integration/connectors/memory/test_qdrant_memory_store.py +++ b/python/tests/integration/connectors/memory/test_qdrant_memory_store.py @@ -119,15 +119,11 @@ async def test_upsert_async_and_get_async(memory_record1): await qdrant_mem_store.create_collection_async("test_collection") await qdrant_mem_store.upsert_async("test_collection", memory_record1) - result = await qdrant_mem_store.get_async( - "test_collection", memory_record1._id, with_embedding=True - ) + result = await qdrant_mem_store.get_async("test_collection", memory_record1._id) assert result is not None assert result._id == memory_record1._id assert result._text == memory_record1._text assert result._timestamp == memory_record1._timestamp - for i in range(len(result._embedding)): - assert result._embedding[i] == memory_record1._embedding[i] @pytest.mark.asyncio @@ -211,14 +207,12 @@ async def test_get_nearest_match_async(memory_record1, memory_record2): test_embedding[0] = test_embedding[0] + 0.01 result = await qdrant_mem_store.get_nearest_match_async( - "test_collection", test_embedding, min_relevance_score=0.0, with_embedding=True + "test_collection", test_embedding, min_relevance_score=0.0 ) assert result is not None assert result[0]._id == memory_record1._id assert result[0]._text == memory_record1._text assert result[0]._timestamp == memory_record1._timestamp - for i in range(len(result[0]._embedding)): - assert result[0]._embedding[i] == memory_record1._embedding[i] @pytest.mark.asyncio diff --git a/python/tests/integration/connectors/memory/test_redis.py b/python/tests/integration/connectors/memory/test_redis.py new file mode 100644 index 000000000000..ee76e05762e2 --- /dev/null +++ b/python/tests/integration/connectors/memory/test_redis.py @@ -0,0 +1,284 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import os +import platform +from datetime import datetime + +import numpy as np +import pytest + +import semantic_kernel as sk +from semantic_kernel.connectors.memory.redis import RedisMemoryStore +from semantic_kernel.memory.memory_record import MemoryRecord + +try: + import redis # noqa: F401 + + redis_installed = True + TEST_COLLECTION_NAME = "test_collection" + TEST_VEC_SIZE = 2 +except ImportError: + redis_installed = False + +pytestmark = pytest.mark.skipif(not redis_installed, reason="Redis is not installed") + +pytestmark = pytest.mark.skipif( + platform.system() != "Linux" and "Python_Integration_Tests" in os.environ, + reason="local redis docker container is not available on all non-Linux platforms", +) + + +@pytest.fixture(scope="session") +def connection_string(): + try: + connection_string = sk.redis_settings_from_dot_env() + except Exception: + connection_string = "redis://localhost:6379" + + return connection_string + + +@pytest.fixture +def memory_record1(): + return MemoryRecord( + id="test_id1", + text="sample text1", + is_reference=False, + embedding=np.array([0.5, 0.5]), + description="description", + additional_metadata="additional metadata", + external_source_name="external source", + timestamp=datetime.now(), + ) + + +@pytest.fixture +def memory_record2(): + return MemoryRecord( + id="test_id2", + text="sample text2", + is_reference=False, + embedding=np.array([0.25, 0.75]), + description="description", + additional_metadata="additional metadata", + external_source_name="external source", + timestamp=datetime.now(), + ) + + +@pytest.fixture +def memory_record3(): + return MemoryRecord( + id="test_id3", + text="sample text3", + is_reference=False, + embedding=np.array([0.25, 0.80]), + description="description", + additional_metadata="additional metadata", + external_source_name="external source", + timestamp=datetime.now(), + ) + + +@pytest.fixture +def memory_store(connection_string): + # Setup and yield + redis_mem_store = RedisMemoryStore(connection_string, vector_size=TEST_VEC_SIZE) + yield redis_mem_store + + # Delete test collection after test + asyncio.run(redis_mem_store.delete_collection_async(TEST_COLLECTION_NAME)) + + +def test_constructor(memory_store): + memory = memory_store + assert memory and memory._database.ping() + + +@pytest.mark.asyncio +async def test_create_and_does_collection_exist_async(memory_store): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + exists = await memory.does_collection_exist_async(TEST_COLLECTION_NAME) + assert exists + + +@pytest.mark.asyncio +async def test_delete_collection_async(memory_store): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + await memory.delete_collection_async(TEST_COLLECTION_NAME) + + exists = await memory.does_collection_exist_async(TEST_COLLECTION_NAME) + assert not exists + + # Delete a non-existent collection with no error + await memory.delete_collection_async(TEST_COLLECTION_NAME) + + +@pytest.mark.asyncio +async def test_get_collections_async(memory_store): + memory = memory_store + + collection_names = ["c1", "c2", "c3"] + for c_n in collection_names: + await memory.create_collection_async(c_n) + + names_from_func = await memory.get_collections_async() + for c_n in collection_names: + assert c_n in names_from_func + await memory.delete_collection_async(c_n) + + +@pytest.mark.asyncio +async def test_does_collection_exist_async(memory_store): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + exists = await memory.does_collection_exist_async(TEST_COLLECTION_NAME) + assert exists + + await memory.delete_collection_async(TEST_COLLECTION_NAME) + exists = await memory.does_collection_exist_async(TEST_COLLECTION_NAME) + assert not exists + + +@pytest.mark.asyncio +async def test_upsert_async_and_get_async(memory_store, memory_record1): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + + # Insert a record + await memory.upsert_async(TEST_COLLECTION_NAME, memory_record1) + fetch_1 = await memory.get_async(TEST_COLLECTION_NAME, memory_record1._id, True) + + assert fetch_1 is not None, "Could not get record" + assert fetch_1._id == memory_record1._id + assert fetch_1._timestamp == memory_record1._timestamp + assert fetch_1._is_reference == memory_record1._is_reference + assert fetch_1._external_source_name == memory_record1._external_source_name + assert fetch_1._description == memory_record1._description + assert fetch_1._text == memory_record1._text + assert fetch_1._additional_metadata == memory_record1._additional_metadata + for expected, actual in zip(fetch_1.embedding, memory_record1.embedding): + assert expected == actual, "Did not retain correct embedding" + + # Update a record + memory_record1._text = "updated sample text1" + + await memory.upsert_async(TEST_COLLECTION_NAME, memory_record1) + fetch_1 = await memory.get_async(TEST_COLLECTION_NAME, memory_record1._id, True) + + assert fetch_1 is not None, "Could not get record" + assert fetch_1._text == memory_record1._text, "Did not update record" + + +@pytest.mark.asyncio +async def test_upsert_batch_async_and_get_batch_async( + memory_store, memory_record1, memory_record2 +): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + + ids = [memory_record1._id, memory_record2._id] + await memory.upsert_batch_async( + TEST_COLLECTION_NAME, [memory_record1, memory_record2] + ) + + fetched = await memory.get_batch_async(TEST_COLLECTION_NAME, ids, True) + + assert len(fetched) > 0, "Could not get records" + for f in fetched: + assert f._id in ids + + +@pytest.mark.asyncio +async def test_remove_async(memory_store, memory_record1): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + + await memory.upsert_async(TEST_COLLECTION_NAME, memory_record1) + await memory.remove_async(TEST_COLLECTION_NAME, memory_record1._id) + get_record = await memory.get_async(TEST_COLLECTION_NAME, memory_record1._id, False) + + assert not get_record, "Record was not removed" + + +@pytest.mark.asyncio +async def test_remove_batch_async(memory_store, memory_record1, memory_record2): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + + ids = [memory_record1._id, memory_record2._id] + await memory.upsert_batch_async( + TEST_COLLECTION_NAME, [memory_record1, memory_record2] + ) + await memory.remove_batch_async(TEST_COLLECTION_NAME, ids) + get_records = await memory.get_batch_async(TEST_COLLECTION_NAME, ids, False) + + assert len(get_records) == 0, "Records were not removed" + + +@pytest.mark.asyncio +async def test_get_nearest_match_async(memory_store, memory_record1, memory_record2): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + + await memory.upsert_batch_async( + TEST_COLLECTION_NAME, [memory_record1, memory_record2] + ) + test_embedding = memory_record1.embedding.copy() + test_embedding[0] = test_embedding[0] + 0.01 + + result = await memory.get_nearest_match_async( + TEST_COLLECTION_NAME, + test_embedding, + min_relevance_score=0.0, + with_embedding=True, + ) + + assert result is not None + assert result[0]._id == memory_record1._id + assert result[0]._timestamp == memory_record1._timestamp + assert result[0]._is_reference == memory_record1._is_reference + assert result[0]._external_source_name == memory_record1._external_source_name + assert result[0]._description == memory_record1._description + assert result[0]._text == memory_record1._text + assert result[0]._additional_metadata == memory_record1._additional_metadata + for i in range(len(result[0]._embedding)): + assert result[0]._embedding[i] == memory_record1._embedding[i] + + +@pytest.mark.asyncio +async def test_get_nearest_matches_async( + memory_store, memory_record1, memory_record2, memory_record3 +): + memory = memory_store + + await memory.create_collection_async(TEST_COLLECTION_NAME) + + await memory.upsert_batch_async( + TEST_COLLECTION_NAME, [memory_record1, memory_record2, memory_record3] + ) + test_embedding = memory_record2.embedding.copy() + test_embedding[0] = test_embedding[0] + 0.025 + + result = await memory.get_nearest_matches_async( + TEST_COLLECTION_NAME, + test_embedding, + limit=2, + min_relevance_score=0.0, + with_embeddings=True, + ) + + assert len(result) == 2 + assert result[0][0]._id in [memory_record3._id, memory_record2._id] + assert result[1][0]._id in [memory_record3._id, memory_record2._id] diff --git a/python/tests/integration/connectors/memory/test_usearch.py b/python/tests/integration/connectors/memory/test_usearch.py new file mode 100644 index 000000000000..30e24295b42e --- /dev/null +++ b/python/tests/integration/connectors/memory/test_usearch.py @@ -0,0 +1,389 @@ +# Copyright (c) Microsoft. All rights reserved. + +from datetime import datetime +from typing import List + +import numpy as np +import pytest + +from semantic_kernel.connectors.memory.usearch import USearchMemoryStore +from semantic_kernel.memory.memory_record import MemoryRecord + +try: + import pyarrow # noqa: F401 + + pyarrow_installed = True +except ImportError: + pyarrow_installed = False + +try: + import usearch # noqa: F401 + + usearch_installed = True +except ImportError: + usearch_installed = False + + +pytestmark = [ + pytest.mark.skipif(not usearch_installed, reason="`USearch` is not installed"), + pytest.mark.skipif( + not pyarrow_installed, + reason="`USearch` dependency `pyarrow` is not installed", + ), +] + + +@pytest.fixture +def memory_record1(): + return MemoryRecord( + id="test_id1", + text="sample text1", + is_reference=False, + embedding=np.array([0.5, 0.5], dtype=np.float32), + description="description", + additional_metadata="additional metadata", + external_source_name="external source", + timestamp=datetime.now(), + ) + + +@pytest.fixture +def memory_record1_with_collision(): + return MemoryRecord( + id="test_id1", + text="sample text2", + is_reference=False, + embedding=np.array([1, 0.6], dtype=np.float32), + description="description_2", + additional_metadata="additional metadata_2", + external_source_name="external source", + timestamp=datetime.now(), + ) + + +@pytest.fixture +def memory_record2(): + return MemoryRecord( + id="test_id2", + text="sample text2", + is_reference=False, + embedding=np.array([0.25, 0.75], dtype=np.float32), + description="description", + additional_metadata="additional metadata", + external_source_name="external source", + timestamp=datetime.now(), + ) + + +@pytest.fixture +def memory_record3(): + return MemoryRecord( + id="test_id3", + text="sample text3", + is_reference=False, + embedding=np.array([0.25, 0.80], dtype=np.float32), + description="description", + additional_metadata="additional metadata", + external_source_name="external source", + timestamp=datetime.now(), + ) + + +def gen_memory_records( + count: int, ndim: int, start_index: int = 0 +) -> List[MemoryRecord]: + return [ + MemoryRecord( + is_reference=False, + text="random text", + additional_metadata="additional", + external_source_name="external_name", + description="something descriptive", + timestamp=datetime.datetime.now(), + id=f":{start_index + index}", + embedding=np.random.uniform(0, 0.3, (ndim)).astype(np.float32), + ) + for index in range(count) + ] + + +def compare_memory_records( + record1: MemoryRecord, record2: MemoryRecord, with_embedding: bool +): + """Compare two MemoryRecord instances and assert they are the same.""" + + assert ( + record1._key == record2._key + ), f"_key mismatch: {record1._key} != {record2._key}" + assert ( + record1._timestamp == record2._timestamp + ), f"_timestamp mismatch: {record1._timestamp} != {record2._timestamp}" + assert ( + record1._is_reference == record2._is_reference + ), f"_is_reference mismatch: {record1._is_reference} != {record2._is_reference}" + assert ( + record1._external_source_name == record2._external_source_name + ), f"_external_source_name mismatch: {record1._external_source_name} != {record2._external_source_name}" + assert record1._id == record2._id, f"_id mismatch: {record1._id} != {record2._id}" + assert ( + record1._description == record2._description + ), f"_description mismatch: {record1._description} != {record2._description}" + assert ( + record1._text == record2._text + ), f"_text mismatch: {record1._text} != {record2._text}" + assert ( + record1._additional_metadata == record2._additional_metadata + ), f"_additional_metadata mismatch: {record1._additional_metadata} != {record2._additional_metadata}" + if with_embedding is True: + assert np.array_equal( + record1._embedding, record2._embedding + ), "_embedding arrays are not equal" + + +@pytest.mark.asyncio +async def test_create_and_get_collection_async(): + memory = USearchMemoryStore() + + await memory.create_collection_async("test_collection1") + await memory.create_collection_async("test_collection2") + await memory.create_collection_async("test_collection3") + result = await memory.get_collections_async() + + assert len(result) == 3 + assert result == ["test_collection1", "test_collection2", "test_collection3"] + + +@pytest.mark.asyncio +async def test_delete_collection_async(): + memory = USearchMemoryStore() + + await memory.create_collection_async("test_collection") + await memory.delete_collection_async("test_collection") + result = await memory.get_collections_async() + assert len(result) == 0 + + await memory.create_collection_async("test_collection") + await memory.delete_collection_async("TEST_COLLECTION") + result = await memory.get_collections_async() + assert len(result) == 0 + + +@pytest.mark.asyncio +async def test_does_collection_exist_async(): + memory = USearchMemoryStore() + await memory.create_collection_async("test_collection") + result = await memory.does_collection_exist_async("test_collection") + assert result is True + + result = await memory.does_collection_exist_async("TEST_COLLECTION") + assert result is True + + +@pytest.mark.asyncio +async def test_upsert_and_get_async_with_no_embedding(memory_record1: MemoryRecord): + memory = USearchMemoryStore() + await memory.create_collection_async("test_collection", ndim=2) + await memory.upsert_async("test_collection", memory_record1) + + result = await memory.get_async("test_collection", "test_id1", False) + compare_memory_records(result, memory_record1, False) + + +@pytest.mark.asyncio +async def test_upsert_and_get_async_with_embedding(memory_record1: MemoryRecord): + memory = USearchMemoryStore() + await memory.create_collection_async("test_collection", ndim=2) + await memory.upsert_async("test_collection", memory_record1) + + result = await memory.get_async("test_collection", "test_id1", True) + compare_memory_records(result, memory_record1, True) + + +@pytest.mark.asyncio +async def test_upsert_and_get_batch_async( + memory_record1: MemoryRecord, memory_record2: MemoryRecord +): + memory = USearchMemoryStore() + await memory.create_collection_async( + "test_collection", ndim=memory_record1.embedding.shape[0] + ) + + await memory.upsert_batch_async("test_collection", [memory_record1, memory_record2]) + + result = await memory.get_batch_async( + "test_collection", ["test_id1", "test_id2"], True + ) + assert len(result) == 2 + + compare_memory_records(result[0], memory_record1, True) + compare_memory_records(result[1], memory_record2, True) + + +@pytest.mark.asyncio +async def test_remove_async(memory_record1): + memory = USearchMemoryStore() + await memory.create_collection_async( + "test_collection", ndim=memory_record1.embedding.shape[0] + ) + + await memory.upsert_async("test_collection", memory_record1) + await memory.remove_async("test_collection", "test_id1") + + # memory.get_async should raise Exception if record is not found + with pytest.raises(KeyError): + await memory.get_async("test_collection", "test_id1", True) + + +@pytest.mark.asyncio +async def test_remove_batch_async( + memory_record1: MemoryRecord, memory_record2: MemoryRecord +): + memory = USearchMemoryStore() + await memory.create_collection_async( + "test_collection", ndim=memory_record1.embedding.shape[0] + ) + + await memory.upsert_batch_async("test_collection", [memory_record1, memory_record2]) + await memory.remove_batch_async("test_collection", ["test_id1", "test_id2"]) + + result = await memory.get_batch_async( + "test_collection", ["test_id1", "test_id2"], True + ) + assert len(result) == 0 + + +@pytest.mark.asyncio +async def test_get_nearest_match_async( + memory_record1: MemoryRecord, memory_record2: MemoryRecord +): + memory = USearchMemoryStore() + + collection_name = "test_collection" + await memory.create_collection_async( + collection_name, ndim=memory_record1.embedding.shape[0], metric="cos" + ) + + await memory.upsert_batch_async(collection_name, [memory_record1, memory_record2]) + + result = await memory.get_nearest_match_async( + collection_name, np.array([0.5, 0.5]), exact=True + ) + + assert len(result) == 2 + assert isinstance(result[0], MemoryRecord) + assert result[1] == pytest.approx(1, abs=1e-5) + + +@pytest.mark.asyncio +async def test_get_nearest_matches_async( + memory_record1: MemoryRecord, memory_record2: MemoryRecord +): + memory = USearchMemoryStore() + + collection_name = "test_collection" + await memory.create_collection_async( + collection_name, ndim=memory_record1.embedding.shape[0], metric="cos" + ) + + await memory.upsert_batch_async(collection_name, [memory_record1, memory_record2]) + + results = await memory.get_nearest_matches_async( + collection_name, np.array([0.5, 0.5]), limit=2, exact=True + ) + + assert len(results) == 2 + assert isinstance(results[0][0], MemoryRecord) + assert results[0][1] == pytest.approx(1, abs=1e-5) + assert results[1][1] == pytest.approx(0.90450, abs=1e-5) + + +@pytest.mark.asyncio +async def test_create_and_save_collection_async( + tmpdir, memory_record1, memory_record2, memory_record3 +): + memory = USearchMemoryStore(tmpdir) + + await memory.create_collection_async("test_collection1", ndim=2) + await memory.create_collection_async("test_collection2", ndim=2) + await memory.create_collection_async("test_collection3", ndim=2) + await memory.upsert_batch_async( + "test_collection1", [memory_record1, memory_record2] + ) + await memory.upsert_batch_async( + "test_collection2", [memory_record2, memory_record3] + ) + await memory.upsert_batch_async( + "test_collection3", [memory_record1, memory_record3] + ) + await memory.close_async() + + assert (tmpdir / "test_collection1.parquet").exists() + assert (tmpdir / "test_collection1.usearch").exists() + assert (tmpdir / "test_collection2.parquet").exists() + assert (tmpdir / "test_collection2.usearch").exists() + assert (tmpdir / "test_collection3.parquet").exists() + assert (tmpdir / "test_collection3.usearch").exists() + + memory = USearchMemoryStore(tmpdir) + result = await memory.get_collections_async() + assert len(result) == 3 + assert set(result) == {"test_collection1", "test_collection2", "test_collection3"} + await memory.delete_collection_async("test_collection1") + await memory.delete_collection_async("test_collection3") + await memory.close_async() + + memory = USearchMemoryStore(tmpdir) + result = await memory.get_collections_async() + assert len(result) == 1 + assert set(result) == {"test_collection2"} + await memory.delete_collection_async("test_collection2") + await memory.close_async() + + memory = USearchMemoryStore(tmpdir) + result = await memory.get_collections_async() + assert len(result) == 0 + + +@pytest.mark.asyncio +async def test_upsert_and_get_async_with_embedding_with_persist( + tmpdir, memory_record1: MemoryRecord, memory_record1_with_collision: MemoryRecord +): + memory = USearchMemoryStore(tmpdir) + assert len(await memory.get_collections_async()) == 0 + await memory.create_collection_async("test_collection", ndim=2) + await memory.upsert_async("test_collection", memory_record1) + await memory.close_async() + + memory = USearchMemoryStore(tmpdir) + assert len(await memory.get_collections_async()) == 1 + result = await memory.get_async("test_collection", "test_id1", True) + compare_memory_records(result, memory_record1, True) + + await memory.upsert_async("test_collection", memory_record1_with_collision) + result = await memory.get_async("test_collection", "test_id1", True) + compare_memory_records(result, memory_record1_with_collision, True) + await memory.close_async() + + memory = USearchMemoryStore(tmpdir) + assert len(await memory.get_collections_async()) == 1 + result = await memory.get_async("test_collection", "test_id1", True) + compare_memory_records(result, memory_record1_with_collision, True) + + +@pytest.mark.asyncio +async def test_remove_get_async( + memory_record1: MemoryRecord, memory_record2: MemoryRecord +): + memory = USearchMemoryStore() + await memory.create_collection_async( + "test_collection", ndim=memory_record1.embedding.shape[0] + ) + + await memory.upsert_batch_async("test_collection", [memory_record1, memory_record2]) + await memory.remove_async("test_collection", "test_id1") + + result = await memory.get_batch_async( + "test_collection", ["test_id1", "test_id2"], True + ) + assert len(result) == 1 + compare_memory_records(result[0], memory_record2, True) diff --git a/python/tests/integration/embeddings/test_azure_oai_embedding_service.py b/python/tests/integration/embeddings/test_azure_oai_embedding_service.py index 933bac7f598b..79851f65907a 100644 --- a/python/tests/integration/embeddings/test_azure_oai_embedding_service.py +++ b/python/tests/integration/embeddings/test_azure_oai_embedding_service.py @@ -33,3 +33,23 @@ async def test_azure_text_embedding_service(create_kernel, get_aoai_config): text="this is a test", external_source_name="external source", ) + + +@pytest.mark.asyncio +async def test_batch_azure_embeddings(get_aoai_config): + # Configure LLM service + _, api_key, endpoint = get_aoai_config + + if "Python_Integration_Tests" in os.environ: + deployment_name = os.environ["AzureOpenAIEmbeddings__DeploymentName"] + + else: + deployment_name = "ada-002" + + embeddings_service = sk_oai.AzureTextEmbedding(deployment_name, endpoint, api_key) + texts = ["hello world", "goodbye world"] + results = await embeddings_service.generate_embeddings_async(texts) + batch_results = await embeddings_service.generate_embeddings_async( + texts, batch_size=1 + ) + assert len(results) == len(batch_results) diff --git a/python/tests/integration/embeddings/test_gp_embedding_service.py b/python/tests/integration/embeddings/test_gp_embedding_service.py new file mode 100644 index 000000000000..7e851ab3a6e3 --- /dev/null +++ b/python/tests/integration/embeddings/test_gp_embedding_service.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft. All rights reserved. + +import os +import sys + +import pytest + +import semantic_kernel as sk + +if sys.version_info >= (3, 9): + import semantic_kernel.connectors.ai.google_palm as sk_gp + +pytestmark = [ + pytest.mark.skipif( + sys.version_info < (3, 9), reason="Google Palm requires Python 3.9 or greater" + ), + pytest.mark.skipif( + "Python_Integration_Tests" in os.environ, + reason="Google Palm integration tests are only set up to run locally", + ), +] + + +@pytest.mark.asyncio +async def test_gp_embedding_service(create_kernel, get_gp_config): + kernel = create_kernel + + api_key = get_gp_config + + palm_text_embed = sk_gp.GooglePalmTextEmbedding( + "models/embedding-gecko-001", api_key + ) + kernel.add_text_embedding_generation_service("gecko", palm_text_embed) + kernel.register_memory_store(memory_store=sk.memory.VolatileMemoryStore()) + + await kernel.memory.save_information_async( + "test", id="info1", text="this is a test" + ) + await kernel.memory.save_reference_async( + "test", + external_id="info1", + text="this is a test", + external_source_name="external source", + ) diff --git a/python/tests/integration/fakes/email_skill_fake.py b/python/tests/integration/fakes/email_skill_fake.py new file mode 100644 index 000000000000..02ac2dc9a0bc --- /dev/null +++ b/python/tests/integration/fakes/email_skill_fake.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + + +class EmailSkillFake: + @sk_function( + description="Given an email address and message body, send an email", + name="SendEmail", + ) + def send_email(self, input: str) -> str: + return f"Sent email to: . Body: {input}" + + @sk_function( + description="Lookup an email address for a person given a name", + name="GetEmailAddress", + ) + def get_email_address(self, input: str) -> str: + if input == "": + return "johndoe1234@example.com" + return f"{input}@example.com" + + @sk_function(description="Write a short poem for an e-mail", name="WritePoem") + def write_poem(self, input: str) -> str: + return f"Roses are red, violets are blue, {input} is hard, so is this test." diff --git a/python/tests/integration/fakes/fun_skill_fake.py b/python/tests/integration/fakes/fun_skill_fake.py new file mode 100644 index 000000000000..034c4a0b2923 --- /dev/null +++ b/python/tests/integration/fakes/fun_skill_fake.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + + +# TODO: this fake skill is temporal usage. +# C# supports import skill from samples dir by using test helper and python should do the same +# `semantic-kernel/dotnet/src/IntegrationTests/TestHelpers.cs` +class FunSkillFake: + @sk_function( + description="Write a joke", + name="WriteJoke", + ) + def write_joke(self) -> str: + return "WriteJoke" diff --git a/python/tests/integration/fakes/summarize_skill_fake.py b/python/tests/integration/fakes/summarize_skill_fake.py new file mode 100644 index 000000000000..0d0857a55be4 --- /dev/null +++ b/python/tests/integration/fakes/summarize_skill_fake.py @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + +# TODO: this fake skill is temporal usage. +# C# supports import skill from samples dir by using test helper and python should do the same +# `semantic-kernel/dotnet/src/IntegrationTests/TestHelpers.cs` + + +class SummarizeSkillFake: + @sk_function( + description="Summarize", + name="Summarize", + ) + def translate(self) -> str: + return "Summarize" diff --git a/python/tests/integration/fakes/writer_skill_fake.py b/python/tests/integration/fakes/writer_skill_fake.py new file mode 100644 index 000000000000..c19f8bf3c855 --- /dev/null +++ b/python/tests/integration/fakes/writer_skill_fake.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter + +# TODO: this fake skill is temporal usage. +# C# supports import skill from samples dir by using test helper and python should do the same +# `semantic-kernel/dotnet/src/IntegrationTests/TestHelpers.cs` + + +class WriterSkillFake: + @sk_function( + description="Translate", + name="Translate", + ) + def translate(self, language: str) -> str: + return f"Translate: {language}" + + @sk_function(description="Write an outline for a novel", name="NovelOutline") + @sk_function_context_parameter( + name="endMarker", + description="The marker to use to end each chapter.", + default_value="", + ) + def write_novel_outline(self, input: str) -> str: + return f"Novel outline: {input}" diff --git a/python/tests/integration/planning/sequential_planner/test_sequential_plan_parser.py b/python/tests/integration/planning/sequential_planner/test_sequential_plan_parser.py new file mode 100644 index 000000000000..9f1c0371f178 --- /dev/null +++ b/python/tests/integration/planning/sequential_planner/test_sequential_plan_parser.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft. All rights reserved. + +import pytest + +import semantic_kernel.connectors.ai.open_ai as sk_oai +from semantic_kernel.kernel import Kernel +from semantic_kernel.planning.sequential_planner.sequential_planner_parser import ( + SequentialPlanParser, +) +from tests.integration.fakes.email_skill_fake import EmailSkillFake +from tests.integration.fakes.summarize_skill_fake import SummarizeSkillFake +from tests.integration.fakes.writer_skill_fake import WriterSkillFake + + +@pytest.mark.asyncio +async def test_can_call_to_plan_from_xml(get_aoai_config): + deployment_name, api_key, endpoint = get_aoai_config + + kernel = Kernel() + # Configure LLM service + kernel.add_text_completion_service( + "text_completion", + sk_oai.AzureChatCompletion(deployment_name, endpoint, api_key), + ) + kernel.import_skill(EmailSkillFake(), "email") + kernel.import_skill(SummarizeSkillFake(), "SummarizeSkill") + kernel.import_skill(WriterSkillFake(), "WriterSkill") + + plan_string = """ + + + + + + +""" + goal = "Summarize an input, translate to french, and e-mail to John Doe" + + plan = SequentialPlanParser.to_plan_from_xml( + plan_string, + goal, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + assert plan is not None + assert ( + plan.description + == "Summarize an input, translate to french, and e-mail to John Doe" + ) + + assert len(plan._steps) == 4 + step = plan._steps[0] + assert step.skill_name == "SummarizeSkill" + assert step.name == "Summarize" + + step = plan._steps[1] + assert step.skill_name == "WriterSkill" + assert step.name == "Translate" + assert step.parameters["language"] == "French" + assert "TRANSLATED_SUMMARY" in step._outputs + + step = plan._steps[2] + assert step.skill_name == "email" + assert step.name == "GetEmailAddress" + assert step.parameters["input"] == "John Doe" + assert "EMAIL_ADDRESS" in step._outputs + + step = plan._steps[3] + assert step.skill_name == "email" + assert step.name == "SendEmail" + assert step.parameters["input"] == "$TRANSLATED_SUMMARY" + assert step.parameters["email_address"] == "$EMAIL_ADDRESS" diff --git a/python/tests/integration/planning/sequential_planner/test_sequential_planner.py b/python/tests/integration/planning/sequential_planner/test_sequential_planner.py new file mode 100644 index 000000000000..36f9be9e6156 --- /dev/null +++ b/python/tests/integration/planning/sequential_planner/test_sequential_planner.py @@ -0,0 +1,167 @@ +# Copyright (c) Microsoft. All rights reserved. + +import time + +import pytest + +import semantic_kernel +import semantic_kernel.connectors.ai.open_ai as sk_oai +from semantic_kernel.kernel import Kernel +from semantic_kernel.planning import SequentialPlanner +from semantic_kernel.planning.sequential_planner.sequential_planner_config import ( + SequentialPlannerConfig, +) +from tests.integration.fakes.email_skill_fake import EmailSkillFake +from tests.integration.fakes.fun_skill_fake import FunSkillFake +from tests.integration.fakes.writer_skill_fake import WriterSkillFake + + +async def retry(func, retries=3): + min_delay = 2 + max_delay = 7 + for i in range(retries): + try: + result = await func() + return result + except Exception: + if i == retries - 1: # Last retry + raise + time.sleep(max(min(i, max_delay), min_delay)) + + +def initialize_kernel(get_aoai_config, use_embeddings=False, use_chat_model=False): + _, api_key, endpoint = get_aoai_config + + kernel = Kernel() + if use_chat_model: + kernel.add_chat_service( + "chat_completion", + sk_oai.AzureChatCompletion("gpt-35-turbo", endpoint, api_key), + ) + else: + kernel.add_text_completion_service( + "text_completion", + sk_oai.AzureChatCompletion("gpt-35-turbo", endpoint, api_key), + ) + + if use_embeddings: + kernel.add_text_embedding_generation_service( + "text_embedding", + sk_oai.AzureTextEmbedding("text-embedding-ada-002", endpoint, api_key), + ) + return kernel + + +@pytest.mark.parametrize( + "use_chat_model, prompt, expected_function, expected_skill", + [ + ( + False, + "Write a joke and send it in an e-mail to Kai.", + "SendEmail", + "_GLOBAL_FUNCTIONS_", + ), + ( + True, + "Write a joke and send it in an e-mail to Kai.", + "SendEmail", + "_GLOBAL_FUNCTIONS_", + ), + ], +) +@pytest.mark.asyncio +async def test_create_plan_function_flow_async( + get_aoai_config, use_chat_model, prompt, expected_function, expected_skill +): + # Arrange + kernel = initialize_kernel(get_aoai_config, False, use_chat_model) + kernel.import_skill(EmailSkillFake()) + kernel.import_skill(FunSkillFake()) + + planner = SequentialPlanner(kernel) + + # Act + plan = await planner.create_plan_async(prompt) + + # Assert + assert any( + step.name == expected_function and step.skill_name == expected_skill + for step in plan._steps + ) + + +@pytest.mark.parametrize( + "prompt, expected_function, expected_skill, expected_default", + [ + ( + "Write a novel outline.", + "NovelOutline", + "WriterSkill", + "", + ) + ], +) +@pytest.mark.asyncio +@pytest.mark.xfail( + raises=semantic_kernel.planning.planning_exception.PlanningException, + reason="Test is known to occasionally produce unexpected results.", +) +async def test_create_plan_with_defaults_async( + get_aoai_config, prompt, expected_function, expected_skill, expected_default +): + # Arrange + kernel = initialize_kernel(get_aoai_config) + kernel.import_skill(EmailSkillFake()) + kernel.import_skill(WriterSkillFake(), "WriterSkill") + + planner = SequentialPlanner(kernel) + + # Act + plan = await retry(lambda: planner.create_plan_async(prompt)) + + # Assert + assert any( + step.name == expected_function + and step.skill_name == expected_skill + and step.parameters["endMarker"] == expected_default + for step in plan._steps + ) + + +@pytest.mark.parametrize( + "prompt, expected_function, expected_skill", + [ + ( + "Write a poem or joke and send it in an e-mail to Kai.", + "SendEmail", + "_GLOBAL_FUNCTIONS_", + ) + ], +) +@pytest.mark.asyncio +@pytest.mark.xfail( + raises=semantic_kernel.planning.planning_exception.PlanningException, + reason="Test is known to occasionally produce unexpected results.", +) +async def test_create_plan_goal_relevant_async( + get_aoai_config, prompt, expected_function, expected_skill +): + # Arrange + kernel = initialize_kernel(get_aoai_config, use_embeddings=True) + kernel.import_skill(EmailSkillFake()) + kernel.import_skill(FunSkillFake()) + kernel.import_skill(WriterSkillFake()) + + planner = SequentialPlanner( + kernel, + SequentialPlannerConfig(relevancy_threshold=0.65, max_relevant_functions=30), + ) + + # Act + plan = await retry(lambda: planner.create_plan_async(prompt)) + + # Assert + assert any( + step.name == expected_function and step.skill_name == expected_skill + for step in plan._steps + ) diff --git a/python/tests/integration/planning/stepwise_planner/test_stepwise_planner.py b/python/tests/integration/planning/stepwise_planner/test_stepwise_planner.py new file mode 100644 index 000000000000..0e48e2dc3ce8 --- /dev/null +++ b/python/tests/integration/planning/stepwise_planner/test_stepwise_planner.py @@ -0,0 +1,174 @@ +# Copyright (c) Microsoft. All rights reserved. + +import json +import os + +import pytest + +import semantic_kernel as sk +import semantic_kernel.connectors.ai.open_ai as sk_oai +from semantic_kernel.connectors.search_engine import BingConnector +from semantic_kernel.core_skills.math_skill import MathSkill +from semantic_kernel.core_skills.time_skill import TimeSkill +from semantic_kernel.kernel import Kernel +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.planning import StepwisePlanner +from semantic_kernel.planning.stepwise_planner.stepwise_planner_config import ( + StepwisePlannerConfig, +) +from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter + + +class TempWebSearchEngineSkill: + """ + TODO: replace this class with semantic_kernel.core_skills.web_search_engine_skill.WebSearchEngineSkill + + SKFunction.describe() does not contains info for arguments. + + so that `query: str` is not shown in the function description, + BUT this argument must be passed to planner to work appropriately. + + This function temporarily add `query` as parameter by using @sk_function_context_parameter. + original file is here: semantic-kernel/python/semantic_kernel/core_skills/web_search_engine_skill.py + """ + + def __init__(self, connector) -> None: + self._connector = connector + + @sk_function( + description="Performs a web search for a given query", name="searchAsync" + ) + @sk_function_context_parameter( + name="query", + description="The search query", + ) + async def search_async(self, query: str, context: SKContext) -> str: + query = query or context.variables.get("query") + result = await self._connector.search_async(query, num_results=5, offset=0) + return str(result) + + +@pytest.fixture(scope="session") +def get_bing_config(): + if "Python_Integration_Tests" in os.environ: + api_key = os.environ["Bing__ApiKey"] + else: + # Load credentials from .env file + api_key = sk.bing_search_settings_from_dot_env() + + return api_key + + +def initialize_kernel(get_aoai_config, use_embeddings=False, use_chat_model=False): + _, api_key, endpoint = get_aoai_config + + kernel = Kernel() + if use_chat_model: + kernel.add_chat_service( + "chat_completion", + sk_oai.AzureChatCompletion("gpt-35-turbo", endpoint, api_key), + ) + else: + kernel.add_text_completion_service( + "text_completion", + sk_oai.AzureChatCompletion("gpt-35-turbo", endpoint, api_key), + ) + + if use_embeddings: + kernel.add_text_embedding_generation_service( + "text_embedding", + sk_oai.AzureTextEmbedding("text-embedding-ada-002", endpoint, api_key), + ) + return kernel + + +@pytest.mark.parametrize( + "use_chat_model, prompt, expected_function, expected_skill", + [ + ( + False, + "What is the tallest mountain on Earth? How tall is it divided by 2?", + "ExecutePlan", + "StepwisePlanner", + ), + ( + True, + "What is the tallest mountain on Earth? How tall is it divided by 2?", + "ExecutePlan", + "StepwisePlanner", + ), + ], +) +@pytest.mark.asyncio +async def test_can_create_stepwise_plan( + get_aoai_config, + get_bing_config, + use_chat_model, + prompt, + expected_function, + expected_skill, +): + # Arrange + use_embeddings = False + kernel = initialize_kernel(get_aoai_config, use_embeddings, use_chat_model) + bing_connector = BingConnector(api_key=get_bing_config) + web_search_engine_skill = TempWebSearchEngineSkill(bing_connector) + kernel.import_skill(web_search_engine_skill, "WebSearch") + kernel.import_skill(TimeSkill(), "time") + + planner = StepwisePlanner( + kernel, StepwisePlannerConfig(max_iterations=10, min_iteration_time_ms=1000) + ) + + # Act + plan = planner.create_plan(prompt) + + # Assert + assert any( + step.name == expected_function and step.skill_name == expected_skill + for step in plan._steps + ) + + +@pytest.mark.parametrize( + "use_chat_model, prompt", + [ + ( + False, + "What is the tallest mountain on Earth? How tall is it divided by 2?", + ) + ], +) +@pytest.mark.asyncio +async def test_can_execute_stepwise_plan( + get_aoai_config, + get_bing_config, + use_chat_model, + prompt, +): + # Arrange + use_embeddings = False + kernel = initialize_kernel(get_aoai_config, use_embeddings, use_chat_model) + bing_connector = BingConnector(api_key=get_bing_config) + web_search_engine_skill = TempWebSearchEngineSkill(bing_connector) + kernel.import_skill(web_search_engine_skill, "WebSearch") + kernel.import_skill(TimeSkill(), "time") + kernel.import_skill(MathSkill(), "math") + + planner = StepwisePlanner( + kernel, StepwisePlannerConfig(max_iterations=10, min_iteration_time_ms=1000) + ) + + # Act + plan = planner.create_plan(prompt) + result = await plan.invoke_async() + + steps_taken_string = result.variables["steps_taken"] + assert steps_taken_string is not None + + steps_taken = json.loads(steps_taken_string) + assert steps_taken is not None and len(steps_taken) > 0 + + assert ( + 3 <= len(steps_taken) <= 10 + ), f"Actual: {len(steps_taken)}. Expected at least 3 steps and at most 10 steps to be taken." diff --git a/python/tests/unit/ai/google_palm/services/test_palm_chat_completion.py b/python/tests/unit/ai/google_palm/services/test_palm_chat_completion.py new file mode 100644 index 000000000000..d22f19e2d992 --- /dev/null +++ b/python/tests/unit/ai/google_palm/services/test_palm_chat_completion.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from semantic_kernel.connectors.ai.chat_request_settings import ( + ChatRequestSettings, +) + +if sys.version_info >= (3, 9): + from semantic_kernel.connectors.ai.google_palm.services.gp_chat_completion import ( + GooglePalmChatCompletion, + ) + + +pytestmark = pytest.mark.skipif( + sys.version_info < (3, 9), reason="Google Palm requires Python 3.9 or greater" +) + + +def test_google_palm_chat_completion_init() -> None: + model_id = "test_model_id" + api_key = "test_api_key" + + gp_chat_completion = GooglePalmChatCompletion( + model_id=model_id, + api_key=api_key, + ) + + assert gp_chat_completion._model_id == model_id + assert gp_chat_completion._api_key == api_key + assert isinstance(gp_chat_completion, GooglePalmChatCompletion) + + +def test_google_palm_chat_completion_init_with_empty_api_key() -> None: + model_id = "test_model_id" + # api_key = "test_api_key" + + with pytest.raises( + ValueError, match="The Google PaLM API key cannot be `None` or empty" + ): + GooglePalmChatCompletion( + model_id=model_id, + api_key="", + ) + + +@pytest.mark.asyncio +async def test_google_palm_text_completion_complete_chat_async_call_with_parameters() -> None: + mock_response = MagicMock() + mock_response.last = asyncio.Future() + mock_response.last.set_result("Example response") + mock_gp = MagicMock() + mock_gp.chat.return_value = mock_response + with patch( + "semantic_kernel.connectors.ai.google_palm.services.gp_chat_completion.palm", + new=mock_gp, + ): + model_id = "test_model_id" + api_key = "test_api_key" + prompt = [("user", "hello world")] + gp_chat_completion = GooglePalmChatCompletion( + model_id=model_id, + api_key=api_key, + ) + settings = ChatRequestSettings() + response = await gp_chat_completion.complete_chat_async(prompt, settings) + assert isinstance(response.result(), str) and len(response.result()) > 0 + + mock_gp.chat.assert_called_once_with( + model=model_id, + context="", + examples=None, + temperature=settings.temperature, + candidate_count=settings.number_of_responses, + top_p=settings.top_p, + prompt=None, + messages=prompt[-1][1], + ) diff --git a/python/tests/unit/ai/google_palm/services/test_palm_text_completion.py b/python/tests/unit/ai/google_palm/services/test_palm_text_completion.py new file mode 100644 index 000000000000..3a9ede666f58 --- /dev/null +++ b/python/tests/unit/ai/google_palm/services/test_palm_text_completion.py @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft. All rights reserved. +import asyncio +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from semantic_kernel.connectors.ai.complete_request_settings import ( + CompleteRequestSettings, +) + +if sys.version_info >= (3, 9): + from semantic_kernel.connectors.ai.google_palm.services.gp_text_completion import ( + GooglePalmTextCompletion, + ) + + +pytestmark = pytest.mark.skipif( + sys.version_info < (3, 9), reason="Google Palm requires Python 3.9 or greater" +) + + +def test_google_palm_text_completion_init() -> None: + model_id = "test_model_id" + api_key = "test_api_key" + + # Test successful initialization + gp_text_completion = GooglePalmTextCompletion( + model_id=model_id, + api_key=api_key, + ) + + assert gp_text_completion._model_id == model_id + assert gp_text_completion._api_key == api_key + assert isinstance(gp_text_completion, GooglePalmTextCompletion) + + +def test_google_palm_text_completion_init_with_empty_api_key() -> None: + model_id = "test_model_id" + # api_key = "test_api_key" + + with pytest.raises( + ValueError, match="The Google PaLM API key cannot be `None` or empty" + ): + GooglePalmTextCompletion( + model_id=model_id, + api_key="", + ) + + +@pytest.mark.asyncio +async def test_google_palm_text_completion_complete_async_call_with_parameters() -> None: + mock_response = MagicMock() + mock_response.result = asyncio.Future() + mock_response.result.set_result("Example response") + mock_gp = MagicMock() + mock_gp.generate_text.return_value = mock_response + with patch( + "semantic_kernel.connectors.ai.google_palm.services.gp_text_completion.palm", + new=mock_gp, + ): + model_id = "test_model_id" + api_key = "test_api_key" + prompt = "hello world" + gp_text_completion = GooglePalmTextCompletion( + model_id=model_id, + api_key=api_key, + ) + settings = CompleteRequestSettings() + response = await gp_text_completion.complete_async(prompt, settings) + assert isinstance(response.result(), str) and len(response.result()) > 0 + + mock_gp.generate_text.assert_called_once_with( + model=model_id, + prompt=prompt, + temperature=settings.temperature, + max_output_tokens=settings.max_tokens, + stop_sequences=None, + candidate_count=settings.number_of_responses, + top_p=settings.top_p, + ) diff --git a/python/tests/unit/ai/google_palm/services/test_palm_text_embedding.py b/python/tests/unit/ai/google_palm/services/test_palm_text_embedding.py new file mode 100644 index 000000000000..add7507acba1 --- /dev/null +++ b/python/tests/unit/ai/google_palm/services/test_palm_text_embedding.py @@ -0,0 +1,70 @@ +# Copyright (c) Microsoft. All rights reserved. + +import sys +from unittest.mock import MagicMock, patch + +import pytest + +if sys.version_info >= (3, 9): + from semantic_kernel.connectors.ai.google_palm.services.gp_text_embedding import ( + GooglePalmTextEmbedding, + ) + + +pytestmark = pytest.mark.skipif( + sys.version_info < (3, 9), reason="Google Palm requires Python 3.9 or greater" +) + + +def test_google_palm_text_embedding_init() -> None: + model_id = "test_model_id" + api_key = "test_api_key" + + # Test successful initialization + gp_text_embed = GooglePalmTextEmbedding( + model_id=model_id, + api_key=api_key, + ) + + assert gp_text_embed._model_id == model_id + assert gp_text_embed._api_key == api_key + assert isinstance(gp_text_embed, GooglePalmTextEmbedding) + + +def test_google_palm_text_embedding_init_with_empty_api_key() -> None: + model_id = "test_model_id" + # api_key = "test_api_key" + + with pytest.raises( + ValueError, match="The Google PaLM API key cannot be `None` or empty" + ): + GooglePalmTextEmbedding( + model_id=model_id, + api_key="", + ) + + +@pytest.mark.asyncio +async def test_google_palm_text_embedding_calls_with_parameters() -> None: + mock_gp = MagicMock() + mock_gp.generate_embeddings.return_value = {"embedding": [0.1, 0.2, 0.3]} + with patch( + "semantic_kernel.connectors.ai.google_palm.services.gp_text_embedding.palm", + new=mock_gp, + ): + model_id = "test_model_id" + api_key = "test_api_key" + texts = ["hello world"] + text = "hello world" + + gp_text_embedding = GooglePalmTextEmbedding( + model_id=model_id, + api_key=api_key, + ) + + await gp_text_embedding.generate_embeddings_async(texts) + + mock_gp.generate_embeddings.assert_called_once_with( + model=model_id, + text=text, + ) diff --git a/python/tests/unit/ai/open_ai/models/chat/test_function_call.py b/python/tests/unit/ai/open_ai/models/chat/test_function_call.py new file mode 100644 index 000000000000..e0200ff37d1f --- /dev/null +++ b/python/tests/unit/ai/open_ai/models/chat/test_function_call.py @@ -0,0 +1,28 @@ +import pytest + +from semantic_kernel.connectors.ai.open_ai.models.chat.function_call import FunctionCall +from semantic_kernel.orchestration.context_variables import ContextVariables + + +def test_function_call(): + # Test initialization with default values + fc = FunctionCall(name="Test-Function", arguments="""{"input": "world"}""") + assert fc.name == "Test-Function" + assert fc.arguments == """{"input": "world"}""" + + +@pytest.mark.asyncio +async def test_function_call_to_content_variables(create_kernel): + # Test parsing arguments to variables + kernel = create_kernel + + func_call = FunctionCall( + name="Test-Function", + arguments="""{"input": "world", "input2": "world2"}""", + ) + context = kernel.create_new_context() + assert isinstance(func_call.to_context_variables(), ContextVariables) + + context.variables.merge_or_overwrite(func_call.to_context_variables()) + assert context.variables.input == "world" + assert context.variables["input2"] == "world2" diff --git a/python/tests/unit/ai/open_ai/services/test_azure_chat_completion.py b/python/tests/unit/ai/open_ai/services/test_azure_chat_completion.py index 66161da9569d..8ef3429232e1 100644 --- a/python/tests/unit/ai/open_ai/services/test_azure_chat_completion.py +++ b/python/tests/unit/ai/open_ai/services/test_azure_chat_completion.py @@ -148,12 +148,13 @@ async def test_azure_chat_completion_call_with_parameters() -> None: organization=None, messages=messages, temperature=complete_request_settings.temperature, - max_tokens=complete_request_settings.max_tokens, top_p=complete_request_settings.top_p, - presence_penalty=complete_request_settings.presence_penalty, - frequency_penalty=complete_request_settings.frequency_penalty, n=complete_request_settings.number_of_responses, stream=False, + stop=None, + max_tokens=complete_request_settings.max_tokens, + presence_penalty=complete_request_settings.presence_penalty, + frequency_penalty=complete_request_settings.frequency_penalty, logit_bias={}, ) @@ -197,11 +198,62 @@ async def test_azure_chat_completion_call_with_parameters_and_Logit_Bias_Defined organization=None, messages=messages, temperature=complete_request_settings.temperature, - max_tokens=complete_request_settings.max_tokens, top_p=complete_request_settings.top_p, + n=complete_request_settings.number_of_responses, + stream=False, + stop=None, + max_tokens=complete_request_settings.max_tokens, presence_penalty=complete_request_settings.presence_penalty, frequency_penalty=complete_request_settings.frequency_penalty, + logit_bias=token_bias, + ) + + +@pytest.mark.asyncio +async def test_azure_chat_completion_call_with_parameters_and_Stop_Defined() -> None: + mock_openai = AsyncMock() + with patch( + "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion.openai", + new=mock_openai, + ): + deployment_name = "test_deployment" + endpoint = "https://test-endpoint.com" + api_key = "test_api_key" + api_type = "azure" + api_version = "2023-03-15-preview" + logger = Logger("test_logger") + prompt = "hello world" + messages = [{"role": "user", "content": prompt}] + complete_request_settings = CompleteRequestSettings() + + stop = ["!"] + complete_request_settings.stop_sequences = stop + + azure_chat_completion = AzureChatCompletion( + deployment_name=deployment_name, + endpoint=endpoint, + api_key=api_key, + api_version=api_version, + logger=logger, + ) + + await azure_chat_completion.complete_async(prompt, complete_request_settings) + + mock_openai.ChatCompletion.acreate.assert_called_once_with( + api_key=api_key, + api_type=api_type, + api_base=endpoint, + api_version=api_version, + organization=None, + engine=deployment_name, + messages=messages, + temperature=complete_request_settings.temperature, + top_p=complete_request_settings.top_p, n=complete_request_settings.number_of_responses, stream=False, - logit_bias=token_bias, + stop=complete_request_settings.stop_sequences, + max_tokens=complete_request_settings.max_tokens, + presence_penalty=complete_request_settings.presence_penalty, + frequency_penalty=complete_request_settings.frequency_penalty, + logit_bias={}, ) diff --git a/python/tests/unit/ai/open_ai/services/test_azure_text_embedding.py b/python/tests/unit/ai/open_ai/services/test_azure_text_embedding.py index a5b0558b6f65..af7b4b94c937 100644 --- a/python/tests/unit/ai/open_ai/services/test_azure_text_embedding.py +++ b/python/tests/unit/ai/open_ai/services/test_azure_text_embedding.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. from logging import Logger -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, call, patch import pytest @@ -122,7 +122,7 @@ async def test_azure_text_embedding_calls_with_parameters() -> None: api_type = "azure" api_version = "2023-03-15-preview" logger = Logger("test_logger") - texts = ["hello world"] + texts = ["hello world", "goodbye world"] azure_text_embedding = AzureTextEmbedding( deployment_name=deployment_name, @@ -143,3 +143,55 @@ async def test_azure_text_embedding_calls_with_parameters() -> None: organization=None, input=texts, ) + + +@pytest.mark.asyncio +async def test_azure_text_embedding_calls_with_batches() -> None: + mock_openai = AsyncMock() + with patch( + "semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding.openai", + new=mock_openai, + ): + deployment_name = "test_deployment" + endpoint = "https://test-endpoint.com" + api_key = "test_api_key" + api_type = "azure" + api_version = "2023-03-15-preview" + logger = Logger("test_logger") + texts = [i for i in range(0, 5)] + + azure_text_embedding = AzureTextEmbedding( + deployment_name=deployment_name, + endpoint=endpoint, + api_key=api_key, + api_version=api_version, + logger=logger, + ) + + await azure_text_embedding.generate_embeddings_async(texts, batch_size=3) + + mock_openai.assert_has_calls( + [ + call.Embedding.acreate( + engine=deployment_name, + api_key=api_key, + api_type=api_type, + api_base=endpoint, + api_version=api_version, + organization=None, + input=texts[0:3], + ), + call.Embedding.acreate().__getitem__("data"), + call.Embedding.acreate().__getitem__().__iter__(), + call.Embedding.acreate( + engine=deployment_name, + api_key=api_key, + api_type=api_type, + api_base=endpoint, + api_version=api_version, + organization=None, + input=texts[3:5], + ), + ], + any_order=False, + ) diff --git a/python/tests/unit/ai/test_request_settings.py b/python/tests/unit/ai/test_request_settings.py new file mode 100644 index 000000000000..9d1a8d6701c9 --- /dev/null +++ b/python/tests/unit/ai/test_request_settings.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.connectors.ai.chat_request_settings import ChatRequestSettings +from semantic_kernel.connectors.ai.complete_request_settings import ( + CompleteRequestSettings, +) + + +def test_default_complete_request_settings(): + settings = CompleteRequestSettings() + assert settings.temperature == 0.0 + assert settings.top_p == 1.0 + assert settings.presence_penalty == 0.0 + assert settings.frequency_penalty == 0.0 + assert settings.max_tokens == 256 + assert settings.stop_sequences == [] + assert settings.number_of_responses == 1 + assert settings.logprobs == 0 + assert settings.token_selection_biases == {} + assert settings.chat_system_prompt == "Assistant is a large language model." + + +def test_custom_complete_request_settings(): + settings = CompleteRequestSettings( + temperature=0.5, + top_p=0.5, + presence_penalty=0.5, + frequency_penalty=0.5, + max_tokens=128, + stop_sequences=["\n"], + number_of_responses=2, + logprobs=1, + token_selection_biases={1: 1}, + chat_system_prompt="Hello", + ) + assert settings.temperature == 0.5 + assert settings.top_p == 0.5 + assert settings.presence_penalty == 0.5 + assert settings.frequency_penalty == 0.5 + assert settings.max_tokens == 128 + assert settings.stop_sequences == ["\n"] + assert settings.number_of_responses == 2 + assert settings.logprobs == 1 + assert settings.token_selection_biases == {1: 1} + assert settings.chat_system_prompt == "Hello" + + +def test_default_chat_request_settings(): + settings = ChatRequestSettings() + assert settings.temperature == 0.0 + assert settings.top_p == 1.0 + assert settings.presence_penalty == 0.0 + assert settings.frequency_penalty == 0.0 + assert settings.max_tokens == 256 + assert settings.stop_sequences == [] + assert settings.number_of_responses == 1 + assert settings.token_selection_biases == {} + + +def test_complete_request_settings_from_default_completion_config(): + settings = CompleteRequestSettings() + chat_settings = ChatRequestSettings.from_completion_config(settings) + chat_settings = ChatRequestSettings() + assert chat_settings.temperature == 0.0 + assert chat_settings.top_p == 1.0 + assert chat_settings.presence_penalty == 0.0 + assert chat_settings.frequency_penalty == 0.0 + assert chat_settings.max_tokens == 256 + assert chat_settings.stop_sequences == [] + assert chat_settings.number_of_responses == 1 + assert chat_settings.token_selection_biases == {} + + +def test_chat_request_settings_from_custom_completion_config(): + settings = CompleteRequestSettings( + temperature=0.5, + top_p=0.5, + presence_penalty=0.5, + frequency_penalty=0.5, + max_tokens=128, + stop_sequences=["\n"], + number_of_responses=2, + logprobs=1, + token_selection_biases={1: 1}, + chat_system_prompt="Hello", + ) + chat_settings = ChatRequestSettings.from_completion_config(settings) + assert chat_settings.temperature == 0.5 + assert chat_settings.top_p == 0.5 + assert chat_settings.presence_penalty == 0.5 + assert chat_settings.frequency_penalty == 0.5 + assert chat_settings.max_tokens == 128 + assert chat_settings.stop_sequences == ["\n"] + assert chat_settings.number_of_responses == 2 + assert chat_settings.token_selection_biases == {1: 1} diff --git a/python/tests/unit/core_skills/test_file_io_skill.py b/python/tests/unit/core_skills/test_file_io_skill.py index 4849421f9b86..910e14a9deb9 100644 --- a/python/tests/unit/core_skills/test_file_io_skill.py +++ b/python/tests/unit/core_skills/test_file_io_skill.py @@ -6,7 +6,6 @@ from semantic_kernel import Kernel from semantic_kernel.core_skills.file_io_skill import FileIOSkill from semantic_kernel.orchestration.context_variables import ContextVariables -from semantic_kernel.orchestration.sk_context import SKContext def test_can_be_instantiated(): @@ -49,7 +48,7 @@ async def test_cannot_read_async(): @pytest.mark.asyncio -async def test_can_write(): +async def test_can_write(context_factory): skill = FileIOSkill() fp = None try: @@ -59,7 +58,7 @@ async def test_can_write(): context_variables.set("path", fp.name) context_variables.set("content", "Hello, world!") - context = SKContext(context_variables, None, None, None) + context = context_factory(context_variables) await skill.write_async(context) @@ -72,7 +71,7 @@ async def test_can_write(): @pytest.mark.asyncio -async def test_cannot_write(): +async def test_cannot_write(context_factory): skill = FileIOSkill() fp = None try: @@ -84,7 +83,7 @@ async def test_cannot_write(): context_variables.set("path", fp.name) context_variables.set("content", "Hello, world!") - context = SKContext(context_variables, None, None, None) + context = context_factory(context_variables) with pytest.raises(PermissionError): await skill.write_async(context) diff --git a/python/tests/unit/core_skills/test_http_skill.py b/python/tests/unit/core_skills/test_http_skill.py index d6a8455640b1..3f369924a992 100644 --- a/python/tests/unit/core_skills/test_http_skill.py +++ b/python/tests/unit/core_skills/test_http_skill.py @@ -7,7 +7,6 @@ from semantic_kernel import Kernel from semantic_kernel.core_skills import HttpSkill from semantic_kernel.orchestration.context_variables import ContextVariables -from semantic_kernel.orchestration.sk_context import SKContext @pytest.mark.asyncio @@ -45,54 +44,54 @@ async def test_get_none_url(): @patch("aiohttp.ClientSession.post") @pytest.mark.asyncio -async def test_post(mock_post): +async def test_post(mock_post, context_factory): mock_post.return_value.__aenter__.return_value.text.return_value = "Hello World !" mock_post.return_value.__aenter__.return_value.status = 200 skill = HttpSkill() context_variables = ContextVariables() context_variables.set("body", "{message: 'Hello, world!'}") - context = SKContext(context_variables, None, None, None) + context = context_factory(context_variables) response = await skill.post_async("https://example.org/post", context) assert response == "Hello World !" @patch("aiohttp.ClientSession.post") @pytest.mark.asyncio -async def test_post_nobody(mock_post): +async def test_post_nobody(mock_post, context_factory): mock_post.return_value.__aenter__.return_value.text.return_value = "Hello World !" mock_post.return_value.__aenter__.return_value.status = 200 skill = HttpSkill() context_variables = ContextVariables() - context = SKContext(context_variables, None, None, None) + context = context_factory(context_variables) response = await skill.post_async("https://example.org/post", context) assert response == "Hello World !" @patch("aiohttp.ClientSession.put") @pytest.mark.asyncio -async def test_put(mock_put): +async def test_put(mock_put, context_factory): mock_put.return_value.__aenter__.return_value.text.return_value = "Hello World !" mock_put.return_value.__aenter__.return_value.status = 200 skill = HttpSkill() context_variables = ContextVariables() context_variables.set("body", "{message: 'Hello, world!'}") - context = SKContext(context_variables, None, None, None) + context = context_factory(context_variables) response = await skill.put_async("https://example.org/put", context) assert response == "Hello World !" @patch("aiohttp.ClientSession.put") @pytest.mark.asyncio -async def test_put_nobody(mock_put): +async def test_put_nobody(mock_put, context_factory): mock_put.return_value.__aenter__.return_value.text.return_value = "Hello World !" mock_put.return_value.__aenter__.return_value.status = 200 skill = HttpSkill() context_variables = ContextVariables() - context = SKContext(context_variables, None, None, None) + context = context_factory(context_variables) response = await skill.put_async("https://example.org/put", context) assert response == "Hello World !" diff --git a/python/tests/unit/kernel_extensions/test_register_functions.py b/python/tests/unit/kernel_extensions/test_register_functions.py new file mode 100644 index 000000000000..772f94a8f700 --- /dev/null +++ b/python/tests/unit/kernel_extensions/test_register_functions.py @@ -0,0 +1,61 @@ +# Copyright (c) Microsoft. All rights reserved. + + +import pytest + +from semantic_kernel import Kernel +from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.skill_definition.sk_function_decorator import sk_function +from semantic_kernel.skill_definition.skill_collection import SkillCollection + + +def not_decorated_native_function(arg1: str) -> str: + return "test" + + +@sk_function(name="getLightStatus") +def decorated_native_function(arg1: str) -> str: + return "test" + + +def test_register_valid_native_function(): + kernel = Kernel() + + registered_func = kernel.register_native_function( + "TestSkill", decorated_native_function + ) + + assert isinstance(registered_func, SKFunctionBase) + assert ( + kernel.skills.get_native_function("TestSkill", "getLightStatus") + == registered_func + ) + assert registered_func.invoke("testtest").result == "test" + + +def test_register_undecorated_native_function(): + kernel = Kernel() + + with pytest.raises(KernelException): + kernel.register_native_function("TestSkill", not_decorated_native_function) + + +def test_register_with_none_skill_name(): + kernel = Kernel() + + registered_func = kernel.register_native_function(None, decorated_native_function) + assert registered_func.skill_name == SkillCollection.GLOBAL_SKILL + + +def test_register_overloaded_native_function(): + kernel = Kernel() + + kernel.register_native_function("TestSkill", decorated_native_function) + + with pytest.raises(KernelException): + kernel.register_native_function("TestSkill", decorated_native_function) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/tests/unit/models/chat/test_chat_message.py b/python/tests/unit/models/chat/test_chat_message.py new file mode 100644 index 000000000000..375e412725e1 --- /dev/null +++ b/python/tests/unit/models/chat/test_chat_message.py @@ -0,0 +1,43 @@ +import pytest + +from semantic_kernel.models.chat.chat_message import ChatMessage +from semantic_kernel.semantic_functions.prompt_template import PromptTemplate +from semantic_kernel.semantic_functions.prompt_template_config import ( + PromptTemplateConfig, +) + + +def test_chat_message(): + # Test initialization with default values + message = ChatMessage() + assert message.role == "assistant" + assert message.fixed_content is None + assert message.content is None + assert message.content_template is None + + +@pytest.mark.asyncio +async def test_chat_message_rendering(create_kernel): + # Test initialization with custom values + kernel = create_kernel + expected_content = "Hello, world!" + prompt_config = PromptTemplateConfig.from_completion_parameters( + max_tokens=2000, temperature=0.7, top_p=0.8 + ) + content_template = PromptTemplate( + "Hello, {{$input}}!", kernel.prompt_template_engine, prompt_config + ) + + message = ChatMessage( + role="user", + content_template=content_template, + ) + context = kernel.create_new_context() + context.variables["input"] = "world" + await message.render_message_async(context) + assert message.role == "user" + assert message.fixed_content == expected_content + assert message.content_template == content_template + + # Test content property + assert message.content == expected_content diff --git a/python/tests/unit/openapi/invalid_openapi.yaml b/python/tests/unit/openapi/invalid_openapi.yaml new file mode 100644 index 000000000000..44b0ca61ef8a --- /dev/null +++ b/python/tests/unit/openapi/invalid_openapi.yaml @@ -0,0 +1,21 @@ +openapi: 3.0.0 +info: + title: Test API + version: 1.0.0 + servers: + - url: http://example.com +paths: + /todos: + get: + summary: Get all todos + operationId: getTodos + responses: + '200': + description: OK + parameters: + - name: Authorization + in: header + required: true + schema: + type: string + description: The authorization token \ No newline at end of file diff --git a/python/tests/unit/openapi/openapi.yaml b/python/tests/unit/openapi/openapi.yaml new file mode 100644 index 000000000000..c2487b4d29d6 --- /dev/null +++ b/python/tests/unit/openapi/openapi.yaml @@ -0,0 +1,128 @@ +openapi: 3.0.0 +info: + title: Test API + version: 1.0.0 +servers: + - url: http://example.com +paths: + /todos: + get: + summary: Get all todos + operationId: getTodos + responses: + '200': + description: OK + parameters: + - name: Authorization + in: header + required: true + schema: + type: string + description: The authorization token + post: + summary: Add a new todo + operationId: addTodo + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + title: + type: string + description: The title of the todo + example: Buy milk + completed: + type: boolean + description: Whether the todo is completed or not + example: false + responses: + '201': + description: Created + parameters: + - name: Authorization + in: header + required: true + schema: + type: string + description: The authorization token + /todos/{id}: + get: + summary: Get a todo by ID + operationId: getTodoById + parameters: + - name: id + in: path + required: true + schema: + type: integer + minimum: 1 + - name: Authorization + in: header + required: true + schema: + type: string + description: The authorization token + responses: + '200': + description: OK + put: + summary: Update a todo by ID + operationId: updateTodoById + parameters: + - name: id + in: path + required: true + schema: + type: integer + minimum: 1 + - name: Authorization + in: header + required: true + schema: + type: string + description: The authorization token + - name: completed + in: query + required: false + schema: + type: boolean + description: Whether the todo is completed or not + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + title: + type: string + description: The title of the todo + example: Buy milk + completed: + type: boolean + description: Whether the todo is completed or not + example: false + responses: + '200': + description: OK + delete: + summary: Delete a todo by ID + operationId: deleteTodoById + parameters: + - name: id + in: path + required: true + schema: + type: integer + minimum: 1 + - name: Authorization + in: header + required: true + schema: + type: string + description: The authorization token + responses: + '204': + description: No Content \ No newline at end of file diff --git a/python/tests/unit/openapi/test_sk_openapi.py b/python/tests/unit/openapi/test_sk_openapi.py new file mode 100644 index 000000000000..df0293692901 --- /dev/null +++ b/python/tests/unit/openapi/test_sk_openapi.py @@ -0,0 +1,301 @@ +import os +from unittest.mock import patch + +import pytest +import yaml +from openapi_core import Spec + +from semantic_kernel.connectors.openapi.sk_openapi import ( + OpenApiParser, + OpenApiRunner, + PreparedRestApiRequest, + RestApiOperation, +) + +directory = os.path.dirname(os.path.realpath(__file__)) +openapi_document = directory + "/openapi.yaml" +invalid_openapi_document = directory + "/invalid_openapi.yaml" +with open(openapi_document, "r") as f: + openapi_document_json = yaml.safe_load(f) +spec = Spec.from_dict(openapi_document_json) + +operation_names = [ + "getTodos", + "addTodo", + "getTodoById", + "updateTodoById", + "deleteTodoById", +] + +put_operation = RestApiOperation( + id="updateTodoById", + method="PUT", + server_url="http://example.com", + path="/todos/{id}", + summary="Update a todo by ID", + params=[ + { + "name": "id", + "in": "path", + "required": True, + "schema": {"type": "integer", "minimum": 1}, + }, + { + "name": "Authorization", + "in": "header", + "required": True, + "schema": {"type": "string", "description": "The authorization token"}, + }, + { + "name": "completed", + "in": "query", + "required": False, + "schema": { + "type": "boolean", + "description": "Whether the todo is completed or not", + }, + }, + ], + request_body={ + "required": True, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "The title of the todo", + "example": "Buy milk", + }, + "completed": { + "type": "boolean", + "description": "Whether the todo is completed or not", + "example": False, + }, + }, + } + } + }, + }, +) + +"""RestApiOperation tests""" + + +def test_prepare_request_with_path_params(): + path_params = {"id": 1} + query_params = {"completed": False} + headers = {"Authorization": "Bearer abc123"} + request_body = {"title": "Buy milk", "completed": False} + expected_request = PreparedRestApiRequest( + method="PUT", + url="http://example.com/todos/1", + params={"completed": False}, + headers={"Authorization": "Bearer abc123", "Content-Type": "application/json"}, + request_body={"title": "Buy milk", "completed": False}, + ) + actual_request = put_operation.prepare_request( + path_params=path_params, + query_params=query_params, + headers=headers, + request_body=request_body, + ) + assert str(actual_request) == str(expected_request) + + +def test_prepare_request_with_missing_path_param(): + path_params = {} + query_params = {"completed": False} + headers = {"Authorization": "Bearer abc123"} + request_body = {"title": "Buy milk", "completed": False} + with pytest.raises(ValueError): + put_operation.prepare_request( + path_params=path_params, + query_params=query_params, + headers=headers, + request_body=request_body, + ) + + +def test_prepare_request_with_default_query_param(): + path_params = {"id": 1} + query_params = {} + headers = {"Authorization": "Bearer abc123"} + request_body = {"title": "Buy milk", "completed": False} + expected_request = PreparedRestApiRequest( + method="PUT", + url="http://example.com/todos/1", + params={}, + headers={"Authorization": "Bearer abc123", "Content-Type": "application/json"}, + request_body={"title": "Buy milk", "completed": False}, + ) + actual_request = put_operation.prepare_request( + path_params=path_params, + query_params=query_params, + headers=headers, + request_body=request_body, + ) + assert str(actual_request) == str(expected_request) + + +def test_prepare_request_with_default_header(): + path_params = {"id": 1} + query_params = {"completed": False} + headers = {} + request_body = {"title": "Buy milk", "completed": False} + expected_request = PreparedRestApiRequest( + method="PUT", + url="http://example.com/todos/1", + params={"completed": False}, + headers={"Content-Type": "application/json"}, + request_body={"title": "Buy milk", "completed": False}, + ) + actual_request = put_operation.prepare_request( + path_params=path_params, + query_params=query_params, + headers=headers, + request_body=request_body, + ) + assert str(actual_request) == str(expected_request) + + +def test_prepare_request_with_no_request_body(): + path_params = {"id": 1} + query_params = {"completed": False} + headers = {"Authorization": "Bearer abc123"} + request_body = None + with pytest.raises(ValueError): + put_operation.prepare_request( + path_params=path_params, + query_params=query_params, + headers=headers, + request_body=request_body, + ) + + +"""OpenApiParser tests""" + + +def test_parse_valid(): + parser = OpenApiParser() + result = parser.parse(openapi_document) + assert result == spec.content() + + +def test_parse_invalid_location(): + parser = OpenApiParser() + with pytest.raises(Exception): + parser.parse("invalid_location") + + +def test_parse_invalid_format(): + parser = OpenApiParser() + with pytest.raises(Exception): + parser.parse(invalid_openapi_document) + + +def test_create_rest_api_operations(): + parser = OpenApiParser() + result = parser.create_rest_api_operations(parser.parse(openapi_document)) + assert all([operation in result for operation in operation_names]) + + get_todos_rest_api_operation = result["getTodos"] + assert get_todos_rest_api_operation.method.lower() == "get" + assert get_todos_rest_api_operation.path == "/todos" + assert get_todos_rest_api_operation.params == [ + { + "name": "Authorization", + "in": "header", + "required": True, + "schema": {"type": "string", "description": "The authorization token"}, + } + ] + assert get_todos_rest_api_operation.id == "getTodos" + assert get_todos_rest_api_operation.request_body is None + + add_todo_rest_api_operation = result["addTodo"] + assert add_todo_rest_api_operation.method.lower() == "post" + assert add_todo_rest_api_operation.path == "/todos" + assert add_todo_rest_api_operation.params == [ + { + "name": "Authorization", + "in": "header", + "required": True, + "schema": {"type": "string", "description": "The authorization token"}, + } + ] + assert add_todo_rest_api_operation.id == "addTodo" + assert add_todo_rest_api_operation.request_body == { + "required": True, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "The title of the todo", + "example": "Buy milk", + }, + "completed": { + "type": "boolean", + "description": "Whether the todo is completed or not", + "example": False, + }, + }, + } + } + }, + } + + +@pytest.fixture +def openapi_runner(): + parser = OpenApiParser() + parsed_doc = parser.parse(openapi_document) + operations = parser.create_rest_api_operations(parsed_doc) + runner = OpenApiRunner(parsed_openapi_document=parsed_doc) + return runner, operations + + +@patch("aiohttp.ClientSession.request") +@pytest.mark.asyncio +async def test_run_operation_with_valid_request(mock_request, openapi_runner): + runner, operations = openapi_runner + operation = operations["addTodo"] + headers = {"Authorization": "Bearer abc123"} + request_body = {"title": "Buy milk", "completed": False} + mock_request.return_value.__aenter__.return_value.text.return_value = 200 + response = await runner.run_operation( + operation, headers=headers, request_body=request_body + ) + assert response == 200 + + +@patch("aiohttp.ClientSession.request") +@pytest.mark.asyncio +async def test_run_operation_with_invalid_request(mock_request, openapi_runner): + runner, operations = openapi_runner + operation = operations["getTodoById"] + headers = {"Authorization": "Bearer abc123"} + request_body = {"title": "Buy milk"} + mock_request.return_value.__aenter__.return_value.text.return_value = 400 + with pytest.raises(Exception): + await runner.run_operation( + operation, headers=headers, request_body=request_body + ) + + +@patch("aiohttp.ClientSession.request") +@pytest.mark.asyncio +async def test_run_operation_with_error(mock_request, openapi_runner): + runner, operations = openapi_runner + operation = operations["addTodo"] + headers = {"Authorization": "Bearer abc123"} + request_body = {"title": "Buy milk", "completed": False} + mock_request.side_effect = Exception("Error") + with pytest.raises(Exception): + await runner.run_operation( + operation, headers=headers, request_body=request_body + ) diff --git a/python/tests/unit/orchestration/test_context_variables.py b/python/tests/unit/orchestration/test_context_variables.py index 2979e755ec12..24430d0586c5 100644 --- a/python/tests/unit/orchestration/test_context_variables.py +++ b/python/tests/unit/orchestration/test_context_variables.py @@ -3,36 +3,36 @@ def test_context_vars_contain_single_var_by_default(): context_vars = ContextVariables() - assert context_vars._variables is not None - assert len(context_vars._variables) == 1 - assert context_vars._variables["input"] == "" + assert context_vars.variables is not None + assert len(context_vars.variables) == 1 + assert context_vars.variables["input"] == "" def test_context_vars_can_be_constructed_with_string(): content = "Hello, world!" context_vars = ContextVariables(content) - assert context_vars._variables is not None - assert len(context_vars._variables) == 1 - assert context_vars._variables["input"] == content + assert context_vars.variables is not None + assert len(context_vars.variables) == 1 + assert context_vars.variables["input"] == content def test_context_vars_can_be_constructed_with_dict(): variables = {"test_string": "Hello, world!"} context_vars = ContextVariables(variables=variables) - assert context_vars._variables is not None - assert len(context_vars._variables) == 2 - assert context_vars._variables["input"] == "" - assert context_vars._variables["test_string"] == variables["test_string"] + assert context_vars.variables is not None + assert len(context_vars.variables) == 2 + assert context_vars.variables["input"] == "" + assert context_vars.variables["test_string"] == variables["test_string"] def test_context_vars_can_be_constructed_with_string_and_dict(): content = "I love muffins" variables = {"test_string": "Hello, world!"} context_vars = ContextVariables(content=content, variables=variables) - assert context_vars._variables is not None - assert len(context_vars._variables) == 2 - assert context_vars._variables["input"] == content - assert context_vars._variables["test_string"] == variables["test_string"] + assert context_vars.variables is not None + assert len(context_vars.variables) == 2 + assert context_vars.variables["input"] == content + assert context_vars.variables["test_string"] == variables["test_string"] def test_merged_context_vars_with_empty_input_results_in_empty_input(): @@ -43,19 +43,19 @@ def test_merged_context_vars_with_empty_input_results_in_empty_input(): context_vars_combined_1with2 = context_vars1.merge_or_overwrite(context_vars2) context_vars_combined_2with1 = context_vars2.merge_or_overwrite(context_vars1) - assert context_vars_combined_1with2._variables is not None - assert len(context_vars_combined_1with2._variables) == 2 - assert context_vars_combined_1with2._variables["input"] == "" + assert context_vars_combined_1with2.variables is not None + assert len(context_vars_combined_1with2.variables) == 2 + assert context_vars_combined_1with2.variables["input"] == "" assert ( - context_vars_combined_1with2._variables["test_string"] + context_vars_combined_1with2.variables["test_string"] == variables["test_string"] ) - assert context_vars_combined_2with1._variables is not None - assert len(context_vars_combined_2with1._variables) == 2 - assert context_vars_combined_2with1._variables["input"] == "" + assert context_vars_combined_2with1.variables is not None + assert len(context_vars_combined_2with1.variables) == 2 + assert context_vars_combined_2with1.variables["input"] == "" assert ( - context_vars_combined_2with1._variables["test_string"] + context_vars_combined_2with1.variables["test_string"] == variables["test_string"] ) @@ -68,19 +68,19 @@ def test_merged_context_vars_with_same_input_results_in_unchanged_input(): context_vars_combined_1with2 = context_vars1.merge_or_overwrite(context_vars2) context_vars_combined_2with1 = context_vars2.merge_or_overwrite(context_vars1) - assert context_vars_combined_1with2._variables is not None - assert len(context_vars_combined_1with2._variables) == 2 - assert context_vars_combined_1with2._variables["input"] == content + assert context_vars_combined_1with2.variables is not None + assert len(context_vars_combined_1with2.variables) == 2 + assert context_vars_combined_1with2.variables["input"] == content assert ( - context_vars_combined_1with2._variables["test_string"] + context_vars_combined_1with2.variables["test_string"] == variables["test_string"] ) - assert context_vars_combined_2with1._variables is not None - assert len(context_vars_combined_2with1._variables) == 2 - assert context_vars_combined_2with1._variables["input"] == content + assert context_vars_combined_2with1.variables is not None + assert len(context_vars_combined_2with1.variables) == 2 + assert context_vars_combined_2with1.variables["input"] == content assert ( - context_vars_combined_2with1._variables["test_string"] + context_vars_combined_2with1.variables["test_string"] == variables["test_string"] ) @@ -95,15 +95,15 @@ def test_merged_context_vars_with_different_input_results_in_input_overwrite1(): context_vars2, overwrite=False ) - assert context_vars_combined_1with2._variables is not None - assert len(context_vars_combined_1with2._variables) == 2 + assert context_vars_combined_1with2.variables is not None + assert len(context_vars_combined_1with2.variables) == 2 assert ( - context_vars_combined_1with2._variables["input"] - == context_vars2._variables["input"] + context_vars_combined_1with2.variables["input"] + == context_vars2.variables["input"] ) assert ( - context_vars_combined_1with2._variables["test_string"] - == context_vars2._variables["test_string"] + context_vars_combined_1with2.variables["test_string"] + == context_vars2.variables["test_string"] ) @@ -117,12 +117,12 @@ def test_merged_context_vars_with_different_input_results_in_input_overwrite2(): context_vars1, overwrite=False ) - assert context_vars_combined_2with1._variables is not None - assert len(context_vars_combined_2with1._variables) == 2 - assert context_vars_combined_2with1._variables["input"] == context_vars1["input"] + assert context_vars_combined_2with1.variables is not None + assert len(context_vars_combined_2with1.variables) == 2 + assert context_vars_combined_2with1.variables["input"] == context_vars1["input"] assert ( - context_vars_combined_2with1._variables["test_string"] - == context_vars2._variables["test_string"] + context_vars_combined_2with1.variables["test_string"] + == context_vars2.variables["test_string"] ) @@ -136,16 +136,14 @@ def test_can_overwrite_context_variables1(): context_vars2, overwrite=True ) - assert context_vars_overwrite_1with2._variables is not None - assert len(context_vars_overwrite_1with2._variables) == len( - context_vars2._variables - ) + assert context_vars_overwrite_1with2.variables is not None + assert len(context_vars_overwrite_1with2.variables) == len(context_vars2.variables) assert ( - context_vars_overwrite_1with2._variables["input"] - == context_vars2._variables["input"] + context_vars_overwrite_1with2.variables["input"] + == context_vars2.variables["input"] ) assert ( - context_vars_overwrite_1with2._variables["test_string"] + context_vars_overwrite_1with2.variables["test_string"] == context_vars2["test_string"] ) @@ -160,11 +158,9 @@ def test_can_overwrite_context_variables2(): context_vars1, overwrite=True ) - assert context_vars_overwrite_2with1._variables is not None - assert len(context_vars_overwrite_2with1._variables) == len( - context_vars1._variables - ) + assert context_vars_overwrite_2with1.variables is not None + assert len(context_vars_overwrite_2with1.variables) == len(context_vars1.variables) assert ( - context_vars_overwrite_2with1._variables["input"] - == context_vars1._variables["input"] + context_vars_overwrite_2with1.variables["input"] + == context_vars1.variables["input"] ) diff --git a/python/tests/unit/orchestration/test_native_function.py b/python/tests/unit/orchestration/test_native_function.py new file mode 100644 index 000000000000..edb7a32c8c4d --- /dev/null +++ b/python/tests/unit/orchestration/test_native_function.py @@ -0,0 +1,118 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import TYPE_CHECKING + +from semantic_kernel.orchestration.sk_function import SKFunction +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + +if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext + + +def test_init_native_function_with_input_description(): + def mock_function(input: str, context: "SKContext") -> None: + pass + + mock_function.__sk_function__ = True + mock_function.__sk_function_name__ = "mock_function" + mock_function.__sk_function_description__ = "Mock description" + mock_function.__sk_function_input_description__ = "Mock input description" + mock_function.__sk_function_input_default_value__ = "default_input_value" + mock_function.__sk_function_context_parameters__ = [ + { + "name": "param1", + "description": "Param 1 description", + "default_value": "default_param1_value", + } + ] + + mock_method = mock_function + + native_function = SKFunction.from_native_method(mock_method, "MockSkill") + + assert native_function._function == mock_method + assert native_function._parameters[0].name == "input" + assert native_function._parameters[0].description == "Mock input description" + assert native_function._parameters[0].default_value == "default_input_value" + assert native_function._parameters[0].type_ == "string" + assert native_function._parameters[0].required is False + assert native_function._parameters[1].name == "param1" + assert native_function._parameters[1].description == "Param 1 description" + assert native_function._parameters[1].default_value == "default_param1_value" + assert native_function._parameters[1].type_ == "string" + assert native_function._parameters[1].required is False + + +def test_init_native_function_without_input_description(): + def mock_function(context: "SKContext") -> None: + pass + + mock_function.__sk_function__ = True + mock_function.__sk_function_name__ = "mock_function_no_input_desc" + mock_function.__sk_function_description__ = "Mock description no input desc" + mock_function.__sk_function_context_parameters__ = [ + { + "name": "param1", + "description": "Param 1 description", + "default_value": "default_param1_value", + "required": True, + } + ] + + mock_method = mock_function + + native_function = SKFunction.from_native_method(mock_method, "MockSkill") + + assert native_function._function == mock_method + assert native_function._parameters[0].name == "param1" + assert native_function._parameters[0].description == "Param 1 description" + assert native_function._parameters[0].default_value == "default_param1_value" + assert native_function._parameters[0].type_ == "string" + assert native_function._parameters[0].required is True + + +def test_init_native_function_from_sk_function_decorator(): + @sk_function( + description="Test description", + name="test_function", + input_description="Test input description", + input_default_value="test_default_value", + ) + def decorated_function() -> None: + pass + + assert decorated_function.__sk_function__ is True + assert decorated_function.__sk_function_description__ == "Test description" + assert decorated_function.__sk_function_name__ == "test_function" + assert ( + decorated_function.__sk_function_input_description__ == "Test input description" + ) + assert ( + decorated_function.__sk_function_input_default_value__ == "test_default_value" + ) + + native_function = SKFunction.from_native_method(decorated_function, "MockSkill") + + assert native_function._function == decorated_function + assert native_function._parameters[0].name == "input" + assert native_function._parameters[0].description == "Test input description" + assert native_function._parameters[0].default_value == "test_default_value" + assert native_function._parameters[0].type_ == "string" + assert native_function._parameters[0].required is False + + +def test_init_native_function_from_sk_function_decorator_defaults(): + @sk_function() + def decorated_function() -> None: + pass + + assert decorated_function.__sk_function__ is True + assert decorated_function.__sk_function_description__ == "" + assert decorated_function.__sk_function_name__ == "decorated_function" + assert decorated_function.__sk_function_input_description__ == "" + assert decorated_function.__sk_function_input_default_value__ == "" + + native_function = SKFunction.from_native_method(decorated_function, "MockSkill") + + assert native_function._function == decorated_function + assert len(native_function._parameters) == 0 diff --git a/python/tests/unit/planning/action_planner/test_action_planner.py b/python/tests/unit/planning/action_planner/test_action_planner.py new file mode 100644 index 000000000000..1e742974fcff --- /dev/null +++ b/python/tests/unit/planning/action_planner/test_action_planner.py @@ -0,0 +1,260 @@ +from textwrap import dedent +from unittest.mock import Mock + +import pytest + +from semantic_kernel import Kernel +from semantic_kernel.memory.semantic_text_memory import SemanticTextMemoryBase +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning import ActionPlanner +from semantic_kernel.planning.action_planner.action_planner_config import ( + ActionPlannerConfig, +) +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.functions_view import FunctionsView +from semantic_kernel.skill_definition.skill_collection_base import SkillCollectionBase + + +def create_mock_function(function_view: FunctionView) -> Mock(spec=SKFunctionBase): + mock_function = Mock(spec=SKFunctionBase) + mock_function.describe.return_value = function_view + mock_function.name = function_view.name + mock_function.skill_name = function_view.skill_name + mock_function.description = function_view.description + return mock_function + + +def test_throw_without_kernel(): + with pytest.raises(PlanningException): + ActionPlanner(None) + + +def test_throw_without_completion_service(): + kernel = Kernel() + + with pytest.raises(ValueError): + ActionPlanner(kernel) + + +@pytest.mark.asyncio +async def test_plan_creation_async(): + goal = "Translate Happy birthday to German." + plan_str = dedent( + """Here is a plan that can achieve the given task:\n\n{""plan"":\n{""rationale"": + ""the list contains a function that allows to translate one language to another."", + ""function"": ""WriterSkill.Translate"",""parameters"": \n{""translate_from"": + ""english"",""translate_to"": ""german"",""input"": ""Happy birthday""}\n}\n}\n\n + This plan makes use of the Translate function in WriterSkill to translate the message + `Happy birthday` from english to german.""" + ) + + kernel = Mock(spec=Kernel) + mock_function = Mock(spec=SKFunctionBase) + memory = Mock(spec=SemanticTextMemoryBase) + skills = Mock(spec=SkillCollectionBase) + + function_view = FunctionView( + name="Translate", + description="Translate something", + skill_name="WriterSkill", + is_semantic=False, + parameters=[], + ) + mock_function = create_mock_function(function_view) + skills.get_function.return_value = mock_function + + context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + return_context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + + return_context.variables.update(plan_str) + + mock_function.invoke_async.return_value = return_context + + kernel.create_semantic_function.return_value = mock_function + kernel.create_new_context.return_value = context + + planner = ActionPlanner(kernel) + plan = await planner.create_plan_async(goal) + + assert plan is not None + assert plan.description == mock_function.description + assert "translate_from" in plan.state + assert "translate_to" in plan.state + assert "input" in plan.state + + +@pytest.fixture +def skills_input(): + return [ + ("SendEmail", "email", "Send an e-mail", False), + ("GetEmailAddress", "email", "Get an e-mail address", False), + ("Translate", "WriterSkill", "Translate something", True), + ("Summarize", "SummarizeSkill", "Summarize something", True), + ] + + +@pytest.fixture +def mock_context(skills_input): + memory = Mock(spec=Kernel) + context = Mock(spec=SKContext) + + functionsView = FunctionsView() + skills = Mock(spec=SkillCollectionBase) + mock_functions = [] + for name, skillName, description, isSemantic in skills_input: + function_view = FunctionView(name, skillName, description, [], isSemantic, True) + mock_function = create_mock_function(function_view) + functionsView.add_function(function_view) + + _context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + _context.variables.update("MOCK FUNCTION CALLED") + mock_function.invoke_async.return_value = _context + mock_functions.append(mock_function) + + skills.get_function.side_effect = lambda skill_name, function_name: next( + ( + func + for func in mock_functions + if func.skill_name == skill_name and func.name == function_name + ), + None, + ) + skills.get_functions_view.return_value = functionsView + context.skills.return_value = skills + context.skills.get_functions_view.return_value = functionsView + + return context + + +def test_available_functions(skills_input, mock_context): + goal = "Translate Happy birthday to German." + kernel = Mock(spec=Kernel) + + planner = ActionPlanner(kernel) + result = planner.list_of_functions(goal=goal, context=mock_context) + + expected_skills = [f"{val[1]}.{val[0]}" for val in skills_input[1:]] + + assert all(skill in result for skill in expected_skills) + + +def test_exclude_skills(skills_input, mock_context): + goal = "Translate Happy birthday to German." + kernel = Mock(spec=Kernel) + + # Exclude the first and second in skills_input + excluded_skill_name = "email" + + planner_config = ActionPlannerConfig(excluded_skills=[excluded_skill_name]) + planner = ActionPlanner(kernel, config=planner_config) + result = planner.list_of_functions(goal=goal, context=mock_context) + + all_skills = [f"{val[1]}.{val[0]}" for val in skills_input] + excluded_skills = all_skills[:2] + expected_skills = all_skills[2:] + + assert all(skill in result for skill in expected_skills) + assert all(skill not in result for skill in excluded_skills) + + +def test_exclude_functions(skills_input, mock_context): + goal = "Translate Happy birthday to German." + kernel = Mock(spec=Kernel) + + excluded_function_name = "SendEmail" + + planner_config = ActionPlannerConfig(excluded_functions=[excluded_function_name]) + planner = ActionPlanner(kernel, config=planner_config) + result = planner.list_of_functions(goal=goal, context=mock_context) + + all_skills = [f"{val[1]}.{val[0]}" for val in skills_input] + excluded_skills = all_skills[:1] + expected_skills = all_skills[1:] + + assert all(skill in result for skill in expected_skills) + assert all(skill not in result for skill in excluded_skills) + + +@pytest.mark.asyncio +async def test_invalid_json_throw_async(): + goal = "Translate Happy birthday to German." + plan_str = '{"":{""function"": ""WriterSkill.Translate""}}' + + kernel = Mock(spec=Kernel) + mock_function = Mock(spec=SKFunctionBase) + memory = Mock(spec=SemanticTextMemoryBase) + skills = Mock(spec=SkillCollectionBase) + + function_view = FunctionView( + name="Translate", + description="Translate something", + skill_name="WriterSkill", + is_semantic=False, + parameters=[], + ) + mock_function = create_mock_function(function_view) + skills.get_function.return_value = mock_function + + context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + return_context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + + return_context.variables.update(plan_str) + + mock_function.invoke_async.return_value = return_context + + kernel.create_semantic_function.return_value = mock_function + kernel.create_new_context.return_value = context + + planner = ActionPlanner(kernel) + + with pytest.raises(PlanningException): + await planner.create_plan_async(goal) + + +@pytest.mark.asyncio +async def test_empty_goal_throw_async(): + goal = "" + + kernel = Mock(spec=Kernel) + mock_function = Mock(spec=SKFunctionBase) + memory = Mock(spec=SemanticTextMemoryBase) + skills = Mock(spec=SkillCollectionBase) + + function_view = FunctionView( + name="Translate", + description="Translate something", + skill_name="WriterSkill", + is_semantic=False, + parameters=[], + ) + mock_function = create_mock_function(function_view) + skills.get_function.return_value = mock_function + + context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + return_context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + mock_function.invoke_async.return_value = return_context + + kernel.create_semantic_function.return_value = mock_function + kernel.create_new_context.return_value = context + + planner = ActionPlanner(kernel) + + with pytest.raises(PlanningException): + await planner.create_plan_async(goal) diff --git a/python/tests/unit/planning/sequential_planner/__init__.py b/python/tests/unit/planning/sequential_planner/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/tests/unit/planning/sequential_planner/test_sequential_planner.py b/python/tests/unit/planning/sequential_planner/test_sequential_planner.py new file mode 100644 index 000000000000..c4e04a42a4c1 --- /dev/null +++ b/python/tests/unit/planning/sequential_planner/test_sequential_planner.py @@ -0,0 +1,155 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import Mock + +import pytest + +from semantic_kernel.kernel import Kernel +from semantic_kernel.memory.semantic_text_memory import SemanticTextMemoryBase +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.planning.sequential_planner.sequential_planner import ( + SequentialPlanner, +) +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.functions_view import FunctionsView +from semantic_kernel.skill_definition.skill_collection_base import SkillCollectionBase + + +def create_mock_function(function_view: FunctionView): + mock_function = Mock(spec=SKFunctionBase) + mock_function.describe.return_value = function_view + mock_function.name = function_view.name + mock_function.skill_name = function_view.skill_name + return mock_function + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "goal", ["Write a poem or joke and send it in an e-mail to Kai."] +) +async def test_it_can_create_plan_async(goal): + # Arrange + kernel = Mock(spec=Kernel) + + memory = Mock(spec=SemanticTextMemoryBase) + + input = [ + ("SendEmail", "email", "Send an e-mail", False), + ("GetEmailAddress", "email", "Get an e-mail address", False), + ("Translate", "WriterSkill", "Translate something", True), + ("Summarize", "SummarizeSkill", "Summarize something", True), + ] + + functionsView = FunctionsView() + skills = Mock(spec=SkillCollectionBase) + mock_functions = [] + for name, skillName, description, isSemantic in input: + function_view = FunctionView(name, skillName, description, [], isSemantic, True) + mock_function = create_mock_function(function_view) + functionsView.add_function(function_view) + + context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + context.variables.update("MOCK FUNCTION CALLED") + mock_function.invoke_async.return_value = context + mock_functions.append(mock_function) + + skills.get_function.side_effect = lambda skill_name, function_name: next( + ( + func + for func in mock_functions + if func.skill_name == skill_name and func.name == function_name + ), + None, + ) + skills.get_functions_view.return_value = functionsView + + expected_functions = [x[0] for x in input] + expected_skills = [x[1] for x in input] + + context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + return_context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + plan_string = """ + + + + + +""" + + return_context.variables.update(plan_string) + + mock_function_flow_function = Mock(spec=SKFunctionBase) + mock_function_flow_function.invoke_async.return_value = return_context + + kernel.skills = skills + kernel.create_new_context.return_value = context + kernel.register_semantic_function.return_value = mock_function_flow_function + + planner = SequentialPlanner(kernel) + + # Act + plan = await planner.create_plan_async(goal) + + # Assert + assert plan.description == goal + assert any( + step.name in expected_functions and step.skill_name in expected_skills + for step in plan._steps + ) + for expected_function in expected_functions: + assert any(step.name == expected_function for step in plan._steps) + for expectedSkill in expected_skills: + assert any(step.skill_name == expectedSkill for step in plan._steps) + + +@pytest.mark.asyncio +async def test_empty_goal_throws_async(): + # Arrange + kernel = Mock(spec=Kernel) + planner = SequentialPlanner(kernel) + + # Act & Assert + with pytest.raises(PlanningException): + await planner.create_plan_async("") + + +@pytest.mark.asyncio +async def test_invalid_xml_throws_async(): + # Arrange + kernel = Mock(spec=Kernel) + memory = Mock(spec=SemanticTextMemoryBase) + skills = Mock(spec=SkillCollectionBase) + + functionsView = FunctionsView() + skills.get_functions_view.return_value = functionsView + + plan_string = "notvalid<" + return_context = SKContext.construct( + variables=ContextVariables(plan_string), memory=memory, skill_collection=skills + ) + + context = SKContext.construct( + variables=ContextVariables(), memory=memory, skill_collection=skills + ) + + mock_function_flow_function = Mock(spec=SKFunctionBase) + mock_function_flow_function.invoke_async.return_value = return_context + + kernel.skills = skills + kernel.create_new_context.return_value = context + kernel.register_semantic_function.return_value = mock_function_flow_function + + planner = SequentialPlanner(kernel) + + # Act & Assert + with pytest.raises(PlanningException): + await planner.create_plan_async("goal") diff --git a/python/tests/unit/planning/sequential_planner/test_sequential_planner_extensions.py b/python/tests/unit/planning/sequential_planner/test_sequential_planner_extensions.py new file mode 100644 index 000000000000..5357ddc14212 --- /dev/null +++ b/python/tests/unit/planning/sequential_planner/test_sequential_planner_extensions.py @@ -0,0 +1,263 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import Mock + +import pytest + +from semantic_kernel.memory.memory_query_result import MemoryQueryResult +from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryBase +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.sequential_planner.sequential_planner_config import ( + SequentialPlannerConfig, +) +from semantic_kernel.planning.sequential_planner.sequential_planner_extensions import ( + SequentialPlannerFunctionViewExtension, + SequentialPlannerSKContextExtension, +) +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.functions_view import FunctionsView +from semantic_kernel.skill_definition.read_only_skill_collection_base import ( + ReadOnlySkillCollectionBase, +) +from semantic_kernel.skill_definition.skill_collection import SkillCollection + + +async def _async_generator(query_result): + yield query_result + + +@pytest.mark.asyncio +async def test_can_call_get_available_functions_with_no_functions_async(): + variables = ContextVariables() + skills = SkillCollection() + + memory = Mock(spec=SemanticTextMemoryBase) + memory_query_result = MemoryQueryResult( + is_reference=False, + id="id", + text="text", + description="description", + external_source_name="sourceName", + additional_metadata="value", + relevance=0.8, + embedding=None, + ) + + async_enumerable = _async_generator(memory_query_result) + memory.search_async.return_value = async_enumerable + + # Arrange GetAvailableFunctionsAsync parameters + context = SKContext(variables, memory, skills.read_only_skill_collection, Mock()) + config = SequentialPlannerConfig() + semantic_query = "test" + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + memory.search_async.assert_not_called() + + +@pytest.mark.asyncio +async def test_can_call_get_available_functions_with_functions_async(): + variables = ContextVariables() + + function_mock = Mock(spec=SKFunctionBase) + functions_view = FunctionsView() + function_view = FunctionView( + "functionName", + "skillName", + "description", + [], + is_semantic=True, + is_asynchronous=False, + ) + native_function_view = FunctionView( + "nativeFunctionName", + "skillName", + "description", + [], + is_semantic=False, + is_asynchronous=False, + ) + functions_view.add_function(function_view) + functions_view.add_function(native_function_view) + + skills = Mock(spec=ReadOnlySkillCollectionBase) + skills.get_function.return_value = function_mock + skills.get_functions_view.return_value = functions_view + + memory_query_result = MemoryQueryResult( + is_reference=False, + id=SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + function_view + ), + text="text", + description="description", + external_source_name="sourceName", + additional_metadata="value", + relevance=0.8, + embedding=None, + ) + + async_enumerable = _async_generator(memory_query_result) + memory = Mock(spec=SemanticTextMemoryBase) + memory.search_async.return_value = async_enumerable + + # Arrange GetAvailableFunctionsAsync parameters + context = SKContext.construct( + variables=variables, memory=memory, skill_collection=skills + ) + config = SequentialPlannerConfig() + semantic_query = "test" + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + assert len(result) == 2 + assert result[0] == function_view + + # Arrange update IncludedFunctions + config.included_functions.append(["nativeFunctionName"]) + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + assert len(result) == 2 # IncludedFunctions should be added to the result + assert result[0] == function_view + assert result[1] == native_function_view + + +@pytest.mark.asyncio +async def test_can_call_get_available_functions_with_functions_and_relevancy_async(): + # Arrange + variables = ContextVariables() + + # Arrange FunctionView + function_mock = Mock(spec=SKFunctionBase) + functions_view = FunctionsView() + function_view = FunctionView( + "functionName", + "skillName", + "description", + [], + is_semantic=True, + is_asynchronous=False, + ) + native_function_view = FunctionView( + "nativeFunctionName", + "skillName", + "description", + [], + is_semantic=False, + is_asynchronous=False, + ) + functions_view.add_function(function_view) + functions_view.add_function(native_function_view) + + # Arrange Mock Memory and Result + memory_query_result = MemoryQueryResult( + is_reference=False, + id=SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + function_view + ), + text="text", + description="description", + external_source_name="sourceName", + additional_metadata="value", + relevance=0.8, + embedding=None, + ) + memory = Mock(spec=SemanticTextMemoryBase) + memory.search_async.return_value = _async_generator(memory_query_result) + + skills = Mock(spec=ReadOnlySkillCollectionBase) + skills.get_function.return_value = function_mock + skills.get_functions_view.return_value = functions_view + skills.read_only_skill_collection = skills + + # Arrange GetAvailableFunctionsAsync parameters + context = SKContext.construct( + variables=variables, + memory=memory, + skill_collection=skills, + ) + context._logger = Mock() + config = SequentialPlannerConfig(relevancy_threshold=0.78) + semantic_query = "test" + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + assert len(result) == 1 + assert result[0] == function_view + + # Arrange update IncludedFunctions + config.included_functions.append("nativeFunctionName") + memory.search_async.return_value = _async_generator(memory_query_result) + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + assert len(result) == 2 # IncludedFunctions should be added to the result + assert result[0] == function_view + assert result[1] == native_function_view + + +@pytest.mark.asyncio +async def test_can_call_get_available_functions_async_with_default_relevancy_async(): + # Arrange + variables = ContextVariables() + skills = SkillCollection() + + # Arrange Mock Memory and Result + memory_query_result = MemoryQueryResult( + is_reference=False, + id="id", + text="text", + description="description", + external_source_name="sourceName", + additional_metadata="value", + relevance=0.8, + embedding=None, + ) + async_enumerable = _async_generator(memory_query_result) + memory = Mock(spec=SemanticTextMemoryBase) + memory.search_async.return_value = async_enumerable + + # Arrange GetAvailableFunctionsAsync parameters + context = SKContext.construct( + variables=variables, memory=memory, skill_collection=skills + ) + config = SequentialPlannerConfig(relevancy_threshold=0.78) + semantic_query = "test" + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + memory.search_async.assert_called_once() diff --git a/python/tests/unit/planning/sequential_planner/test_sequential_planner_parser.py b/python/tests/unit/planning/sequential_planner/test_sequential_planner_parser.py new file mode 100644 index 000000000000..d39e226201a3 --- /dev/null +++ b/python/tests/unit/planning/sequential_planner/test_sequential_planner_parser.py @@ -0,0 +1,349 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import Mock + +import pytest + +from semantic_kernel.kernel import Kernel +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.planning.sequential_planner.sequential_planner_parser import ( + SequentialPlanParser, +) +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.functions_view import FunctionsView + + +def create_mock_function(function_view: FunctionView) -> SKFunctionBase: + mock_function = Mock(spec=SKFunctionBase) + mock_function.describe.return_value = function_view + mock_function.name = function_view.name + mock_function.skill_name = function_view.skill_name + mock_function.description = function_view.description + return mock_function + + +def create_kernel_and_functions_mock(functions) -> Kernel: + kernel = Kernel() + functions_view = FunctionsView() + for name, skill_name, description, is_semantic, result_string in functions: + function_view = FunctionView( + name, skill_name, description, [], is_semantic, True + ) + functions_view.add_function(function_view) + mock_function = create_mock_function(function_view) + + result = kernel.create_new_context() + result.variables.update(result_string) + mock_function.invoke_async.return_value = result + kernel._skill_collection.add_semantic_function(mock_function) + + return kernel + + +def test_can_call_to_plan_from_xml(): + functions = [ + ( + "Summarize", + "SummarizeSkill", + "Summarize an input", + True, + "This is the summary.", + ), + ("Translate", "WriterSkill", "Translate to french", True, "Bonjour!"), + ( + "GetEmailAddressAsync", + "email", + "Get email address", + False, + "johndoe@email.com", + ), + ("SendEmailAsync", "email", "Send email", False, "Email sent."), + ] + kernel = create_kernel_and_functions_mock(functions) + + plan_string = """ + + + + +""" + goal = "Summarize an input, translate to french, and e-mail to John Doe" + + plan = SequentialPlanParser.to_plan_from_xml( + plan_string, + goal, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + assert plan is not None + assert ( + plan.description + == "Summarize an input, translate to french, and e-mail to John Doe" + ) + + assert len(plan._steps) == 4 + assert plan._steps[0].skill_name == "SummarizeSkill" + assert plan._steps[0].name == "Summarize" + assert plan._steps[1].skill_name == "WriterSkill" + assert plan._steps[1].name == "Translate" + assert plan._steps[1].parameters["language"] == "French" + assert "TRANSLATED_SUMMARY" in plan._steps[1]._outputs + + assert plan._steps[2].skill_name == "email" + assert plan._steps[2].name == "GetEmailAddressAsync" + assert plan._steps[2].parameters["input"] == "John Doe" + assert "EMAIL_ADDRESS" in plan._steps[2]._outputs + + assert plan._steps[3].skill_name == "email" + assert plan._steps[3].name == "SendEmailAsync" + assert "$TRANSLATED_SUMMARY" in plan._steps[3].parameters["input"] + assert "$EMAIL_ADDRESS" in plan._steps[3].parameters["email_address"] + + +def test_invalid_plan_execute_plan_returns_invalid_result(): + # Arrange + kernel = create_kernel_and_functions_mock([]) + + # Act and Assert + with pytest.raises(PlanningException): + SequentialPlanParser.to_plan_from_xml( + "", + "Solve the equation x^2 = 2.", + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + +def test_can_create_plan_with_text_nodes(): + # Arrange + goal_text = "Test the functionFlowRunner" + plan_text = """ + Test the functionFlowRunner + + + This is some text + """ + functions = [ + ("Echo", "MockSkill", "Echo an input", True, "Mock Echo Result"), + ] + kernel = create_kernel_and_functions_mock(functions) + + # Act + plan = SequentialPlanParser.to_plan_from_xml( + plan_text, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + # Assert + assert plan is not None + assert plan.description == goal_text + assert len(plan._steps) == 1 + assert plan._steps[0].skill_name == "MockSkill" + assert plan._steps[0].name == "Echo" + + +@pytest.mark.parametrize( + "plan_text, allow_missing_functions", + [ + ( + """ + + + + """, + True, + ), + ( + """ + + + + """, + False, + ), + ], +) +def test_can_create_plan_with_invalid_function_nodes( + plan_text, allow_missing_functions +): + # Arrange + functions = [ + ("Echo", "MockSkill", "Echo an input", True, "Mock Echo Result"), + ] + kernel = create_kernel_and_functions_mock(functions) + # Act and Assert + if allow_missing_functions: + plan = SequentialPlanParser.to_plan_from_xml( + plan_text, + "", + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + allow_missing_functions, + ) + + # Assert + assert plan is not None + assert len(plan._steps) == 2 + + assert plan._steps[0].skill_name == "MockSkill" + assert plan._steps[0].name == "Echo" + assert plan._steps[0].description == "Echo an input" + + assert plan._steps[1].skill_name == plan.__class__.__name__ + assert plan._steps[1].name == "" + assert plan._steps[1].description == "MockSkill.DoesNotExist" + else: + with pytest.raises(PlanningException): + SequentialPlanParser.to_plan_from_xml( + plan_text, + "", + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + allow_missing_functions, + ) + + +def test_can_create_plan_with_other_text(): + # Arrange + goal_text = "Test the functionFlowRunner" + plan_text1 = """Possible result: Test the functionFlowRunner + + + This is some text + """ + plan_text2 = """ + + + This is some text + + + plan end""" + plan_text3 = """ + + + This is some text + + + plan end""" + functions = [ + ("Echo", "MockSkill", "Echo an input", True, "Mock Echo Result"), + ] + kernel = create_kernel_and_functions_mock(functions) + + # Act + plan1 = SequentialPlanParser.to_plan_from_xml( + plan_text1, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + plan2 = SequentialPlanParser.to_plan_from_xml( + plan_text2, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + plan3 = SequentialPlanParser.to_plan_from_xml( + plan_text3, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + # Assert + assert plan1 is not None + assert plan1.description == goal_text + assert len(plan1._steps) == 1 + assert plan1._steps[0].skill_name == "MockSkill" + assert plan1._steps[0].name == "Echo" + + assert plan2 is not None + assert plan2.description == goal_text + assert len(plan2._steps) == 1 + assert plan2._steps[0].skill_name == "MockSkill" + assert plan2._steps[0].name == "Echo" + + assert plan3 is not None + assert plan3.description == goal_text + assert len(plan3._steps) == 1 + assert plan3._steps[0].skill_name == "MockSkill" + assert plan3._steps[0].name == "Echo" + + +@pytest.mark.parametrize( + "plan_text", + [ + """ """, + """ + +""", + """ + +""", + ], +) +def test_can_create_plan_with_open_api_plugin(plan_text): + # Arrange + functions = [ + ( + "codesearchresults_post", + "CodeSearch", + "Echo an input", + True, + "Mock Echo Result", + ), + ] + kernel = create_kernel_and_functions_mock(functions) + + # Act + plan = SequentialPlanParser.to_plan_from_xml( + plan_text, + "", + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + # Assert + assert plan is not None + assert len(plan._steps) == 1 + assert plan._steps[0].skill_name == "CodeSearch" + assert plan._steps[0].name == "codesearchresults_post" + + +def test_can_create_plan_with_ignored_nodes(): + # Arrange + goal_text = "Test the functionFlowRunner" + plan_text = """ + + Some other tag + + """ + functions = [ + ("Echo", "MockSkill", "Echo an input", True, "Mock Echo Result"), + ] + kernel = create_kernel_and_functions_mock(functions) + + # Act + plan = SequentialPlanParser.to_plan_from_xml( + plan_text, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + # Assert + assert plan is not None + assert plan.description == goal_text + assert len(plan._steps) == 2 + assert plan._steps[0].skill_name == "MockSkill" + assert plan._steps[0].name == "Echo" + assert len(plan._steps[1]._steps) == 0 + assert plan._steps[1].skill_name == "MockSkill" + assert plan._steps[1].name == "Echo" diff --git a/python/tests/unit/planning/stepwise_planner/test_stepwise_planner_parse_result.py b/python/tests/unit/planning/stepwise_planner/test_stepwise_planner_parse_result.py new file mode 100644 index 000000000000..6be9228effe5 --- /dev/null +++ b/python/tests/unit/planning/stepwise_planner/test_stepwise_planner_parse_result.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import Mock + +import pytest + +from semantic_kernel.kernel import Kernel +from semantic_kernel.planning.stepwise_planner.stepwise_planner import StepwisePlanner + + +@pytest.mark.parametrize( + "input, expected", + [ + ("[FINAL ANSWER] 42", "42"), + ("[FINAL ANSWER]42", "42"), + ("I think I have everything I need.\n[FINAL ANSWER] 42", "42"), + ("I think I have everything I need.\n[FINAL ANSWER] 42\n", "42"), + ("I think I have everything I need.\n[FINAL ANSWER] 42\n\n", "42"), + ("I think I have everything I need.\n[FINAL ANSWER]42\n\n\n", "42"), + ("I think I have everything I need.\n[FINAL ANSWER]\n 42\n\n\n", "42"), + ], +) +def test_when_input_is_final_answer_returns_final_answer(input: str, expected: str): + kernel = Mock(spec=Kernel) + + planner = StepwisePlanner(kernel) + + result = planner.parse_result(input) + + assert result.final_answer == expected + + +@pytest.mark.parametrize( + "input, expected", + [ + ("My thought", "My thought"), + ("My thought\n", "My thought"), + ("My thought\n\n", "My thought"), + ("My thought\n\n\n", "My thought"), + ], +) +def test_when_input_is_only_thought_does_not_throw_error(input: str, expected: str): + kernel = Mock(spec=Kernel) + planner = StepwisePlanner(kernel) + result = planner.parse_result(input) + assert result.thought == expected + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/tests/unit/planning/test_plan_execution.py b/python/tests/unit/planning/test_plan_execution.py index 3e993d656570..fa26475aa848 100644 --- a/python/tests/unit/planning/test_plan_execution.py +++ b/python/tests/unit/planning/test_plan_execution.py @@ -3,6 +3,7 @@ import pytest import semantic_kernel as sk +from semantic_kernel.core_skills.math_skill import MathSkill from semantic_kernel.core_skills.text_skill import TextSkill from semantic_kernel.planning import Plan @@ -180,3 +181,33 @@ async def test_invoke_multi_step_plan_async(): plan.add_steps([new_step, new_step2]) result = await plan.invoke_async(context=context) assert result.result == "HELLO WORLD" + + +@pytest.mark.asyncio +async def test_invoke_multi_step_plan_async_with_variables(): + # create a kernel + kernel = sk.Kernel() + + # import test (text) skill + skill = MathSkill() + skill_config_dict = kernel.import_skill(skill, "math") + test_function = skill_config_dict["Add"] + test_function2 = skill_config_dict["Subtract"] + + plan = Plan(name="test") + + # setup context for step 1 + context1 = kernel.create_new_context() + context1["amount"] = "10" + new_step = Plan(name="test", function=test_function, parameters=context1.variables) + + # setup context for step 2 + context2 = kernel.create_new_context() + context2["amount"] = "5" + new_step2 = Plan( + name="test", function=test_function2, parameters=context2.variables + ) + + plan.add_steps([new_step, new_step2]) + result = await plan.invoke_async(input="2") + assert result.result == "7" diff --git a/python/tests/unit/skill_definition/test_functions_view.py b/python/tests/unit/skill_definition/test_functions_view.py index e37f64d7521b..f377ec667230 100644 --- a/python/tests/unit/skill_definition/test_functions_view.py +++ b/python/tests/unit/skill_definition/test_functions_view.py @@ -18,7 +18,7 @@ def test_add_semantic_function(): ) functions_view = FunctionsView() functions_view.add_function(view) - semantic_functions = functions_view._semantic_functions.get("skill1") + semantic_functions = functions_view.semantic_functions.get("skill1") assert len(semantic_functions) == 1 assert semantic_functions[0] == view @@ -34,7 +34,7 @@ def test_add_native_function(): ) functions_view = FunctionsView() functions_view.add_function(view) - native_functions = functions_view._native_functions.get("skill2") + native_functions = functions_view.native_functions.get("skill2") assert len(native_functions) == 1 assert native_functions[0] == view @@ -59,8 +59,8 @@ def test_add_multiple_functions(): functions_view = FunctionsView() functions_view.add_function(semantic_function) functions_view.add_function(native_function) - semantic_functions = functions_view._semantic_functions.get("skill1") - native_functions = functions_view._native_functions.get("skill2") + semantic_functions = functions_view.semantic_functions.get("skill1") + native_functions = functions_view.native_functions.get("skill2") assert len(semantic_functions) == 1 assert semantic_functions[0] == semantic_function assert len(native_functions) == 1 diff --git a/python/tests/unit/skill_definition/test_prompt_templates.py b/python/tests/unit/skill_definition/test_prompt_templates.py new file mode 100644 index 000000000000..82c03875ba00 --- /dev/null +++ b/python/tests/unit/skill_definition/test_prompt_templates.py @@ -0,0 +1,175 @@ +# Copyright (c) Microsoft. All rights reserved. + +import json + +import pytest + +from semantic_kernel.semantic_functions.chat_prompt_template import ChatPromptTemplate +from semantic_kernel.semantic_functions.prompt_template_config import ( + PromptTemplateConfig, +) + + +def test_default_prompt_template_config(): + prompt_template_config = PromptTemplateConfig() + assert prompt_template_config.schema == 1 + assert prompt_template_config.type == "completion" + assert prompt_template_config.description == "" + assert prompt_template_config.completion.temperature == 0.0 + assert prompt_template_config.completion.top_p == 1.0 + assert prompt_template_config.completion.presence_penalty == 0.0 + assert prompt_template_config.completion.frequency_penalty == 0.0 + assert prompt_template_config.completion.max_tokens == 256 + assert prompt_template_config.completion.number_of_responses == 1 + assert prompt_template_config.completion.stop_sequences == [] + assert prompt_template_config.completion.token_selection_biases == {} + assert prompt_template_config.completion.chat_system_prompt is None + + +def test_default_chat_prompt_template_from_empty_dict(): + with pytest.raises(KeyError): + _ = PromptTemplateConfig().from_dict({}) + + +def test_default_chat_prompt_template_from_empty_string(): + with pytest.raises(json.decoder.JSONDecodeError): + _ = PromptTemplateConfig().from_json("") + + +def test_default_chat_prompt_template_from_empty_json(): + with pytest.raises(KeyError): + _ = PromptTemplateConfig().from_json("{}") + + +def test_custom_prompt_template_config(): + prompt_template_config = PromptTemplateConfig( + schema=2, + type="completion2", + description="Custom description.", + completion=PromptTemplateConfig.CompletionConfig( + temperature=0.5, + top_p=0.5, + presence_penalty=0.5, + frequency_penalty=0.5, + max_tokens=128, + number_of_responses=2, + stop_sequences=["\n"], + token_selection_biases={1: 1}, + chat_system_prompt="Custom system prompt.", + ), + ) + assert prompt_template_config.schema == 2 + assert prompt_template_config.type == "completion2" + assert prompt_template_config.description == "Custom description." + assert prompt_template_config.completion.temperature == 0.5 + assert prompt_template_config.completion.top_p == 0.5 + assert prompt_template_config.completion.presence_penalty == 0.5 + assert prompt_template_config.completion.frequency_penalty == 0.5 + assert prompt_template_config.completion.max_tokens == 128 + assert prompt_template_config.completion.number_of_responses == 2 + assert prompt_template_config.completion.stop_sequences == ["\n"] + assert prompt_template_config.completion.token_selection_biases == {1: 1} + assert ( + prompt_template_config.completion.chat_system_prompt == "Custom system prompt." + ) + + +def test_custom_prompt_template_config_from_dict(): + prompt_template_dict = { + "schema": 2, + "type": "completion2", + "description": "Custom description.", + "completion": { + "temperature": 0.5, + "top_p": 0.5, + "presence_penalty": 0.5, + "frequency_penalty": 0.5, + "max_tokens": 128, + "number_of_responses": 2, + "stop_sequences": ["\n"], + "token_selection_biases": {1: 1}, + "chat_system_prompt": "Custom system prompt.", + }, + } + prompt_template_config = PromptTemplateConfig().from_dict(prompt_template_dict) + assert prompt_template_config.schema == 2 + assert prompt_template_config.type == "completion2" + assert prompt_template_config.description == "Custom description." + assert prompt_template_config.completion.temperature == 0.5 + assert prompt_template_config.completion.top_p == 0.5 + assert prompt_template_config.completion.presence_penalty == 0.5 + assert prompt_template_config.completion.frequency_penalty == 0.5 + assert prompt_template_config.completion.max_tokens == 128 + assert prompt_template_config.completion.number_of_responses == 2 + assert prompt_template_config.completion.stop_sequences == ["\n"] + assert prompt_template_config.completion.token_selection_biases == {1: 1} + assert ( + prompt_template_config.completion.chat_system_prompt == "Custom system prompt." + ) + + +def test_custom_prompt_template_config_from_json(): + prompt_template_json = """ + { + "schema": 2, + "type": "completion2", + "description": "Custom description.", + "completion": { + "temperature": 0.5, + "top_p": 0.5, + "presence_penalty": 0.5, + "frequency_penalty": 0.5, + "max_tokens": 128, + "number_of_responses": 2, + "stop_sequences": ["s"], + "token_selection_biases": {"1": 1}, + "chat_system_prompt": "Custom system prompt." + } + } + """ + prompt_template_config = PromptTemplateConfig().from_json(prompt_template_json) + assert prompt_template_config.schema == 2 + assert prompt_template_config.type == "completion2" + assert prompt_template_config.description == "Custom description." + assert prompt_template_config.completion.temperature == 0.5 + assert prompt_template_config.completion.top_p == 0.5 + assert prompt_template_config.completion.presence_penalty == 0.5 + assert prompt_template_config.completion.frequency_penalty == 0.5 + assert prompt_template_config.completion.max_tokens == 128 + assert prompt_template_config.completion.number_of_responses == 2 + assert prompt_template_config.completion.stop_sequences == ["s"] + assert prompt_template_config.completion.token_selection_biases == {1: 1} + assert ( + prompt_template_config.completion.chat_system_prompt == "Custom system prompt." + ) + + +def test_chat_prompt_template(): + chat_prompt_template = ChatPromptTemplate( + "{{$user_input}}", + None, + prompt_config=PromptTemplateConfig(), + ) + + assert chat_prompt_template._messages == [] + + +def test_chat_prompt_template_with_system_prompt(): + prompt_template_config = PromptTemplateConfig( + completion=PromptTemplateConfig.CompletionConfig( + chat_system_prompt="Custom system prompt.", + ) + ) + + chat_prompt_template = ChatPromptTemplate( + "{{$user_input}}", + None, + prompt_config=prompt_template_config, + ) + print(chat_prompt_template._messages) + assert len(chat_prompt_template.messages) == 1 + assert chat_prompt_template._messages[0].role == "system" + assert ( + chat_prompt_template._messages[0].content_template._template + == "Custom system prompt." + ) diff --git a/python/tests/unit/template_engine/blocks/test_code_block.py b/python/tests/unit/template_engine/blocks/test_code_block.py index b9454d9e01de..e83dcf10501c 100644 --- a/python/tests/unit/template_engine/blocks/test_code_block.py +++ b/python/tests/unit/template_engine/blocks/test_code_block.py @@ -25,8 +25,8 @@ def setup_method(self): @mark.asyncio async def test_it_throws_if_a_function_doesnt_exist(self): - context = SKContext( - ContextVariables(), + context = SKContext.construct( + variables=ContextVariables(), memory=NullMemory(), skill_collection=self.skills, logger=self.log, @@ -40,8 +40,8 @@ async def test_it_throws_if_a_function_doesnt_exist(self): @mark.asyncio async def test_it_throws_if_a_function_call_throws(self): - context = SKContext( - ContextVariables(), + context = SKContext.construct( + variables=ContextVariables(), memory=NullMemory(), skill_collection=self.skills, logger=self.log, @@ -123,8 +123,11 @@ async def test_it_renders_code_block_consisting_of_just_a_var_block1(self): variables = ContextVariables() variables["varName"] = "foo" - context = SKContext( - variables, memory=NullMemory(), skill_collection=None, logger=self.log + context = SKContext.construct( + variables=variables, + memory=NullMemory(), + skill_collection=None, + logger=self.log, ) code_block = CodeBlock(content="$varName", log=self.log) @@ -137,8 +140,11 @@ async def test_it_renders_code_block_consisting_of_just_a_var_block2(self): variables = ContextVariables() variables["varName"] = "bar" - context = SKContext( - variables, memory=NullMemory(), skill_collection=None, logger=self.log + context = SKContext.construct( + variables=variables, + memory=NullMemory(), + skill_collection=None, + logger=self.log, ) code_block = CodeBlock( @@ -150,8 +156,8 @@ async def test_it_renders_code_block_consisting_of_just_a_var_block2(self): @mark.asyncio async def test_it_renders_code_block_consisting_of_just_a_val_block1(self): - context = SKContext( - ContextVariables(), + context = SKContext.construct( + variables=ContextVariables(), memory=NullMemory(), skill_collection=None, logger=self.log, @@ -164,8 +170,8 @@ async def test_it_renders_code_block_consisting_of_just_a_val_block1(self): @mark.asyncio async def test_it_renders_code_block_consisting_of_just_a_val_block2(self): - context = SKContext( - ContextVariables(), + context = SKContext.construct( + variables=ContextVariables(), memory=NullMemory(), skill_collection=None, logger=self.log, @@ -187,8 +193,8 @@ async def test_it_invokes_function_cloning_all_variables(self): variables["var2"] = "due" # Create a context with the variables, memory, skill collection, and logger - context = SKContext( - variables, + context = SKContext.construct( + variables=variables, memory=NullMemory(), skill_collection=self.skills, logger=self.log, @@ -252,8 +258,8 @@ async def test_it_invokes_function_with_custom_variable(self): variables[VAR_NAME] = VAR_VALUE # Create a context with the variables, memory, skill collection, and logger - context = SKContext( - variables, + context = SKContext.construct( + variables=variables, memory=NullMemory(), skill_collection=self.skills, logger=self.log, @@ -303,8 +309,8 @@ async def test_it_invokes_function_with_custom_value(self): VALUE = "value" # Create a context with empty variables, memory, skill collection, and logger - context = SKContext( - ContextVariables(), + context = SKContext.construct( + variables=ContextVariables(), memory=NullMemory(), skill_collection=self.skills, logger=self.log, diff --git a/python/tests/unit/template_engine/blocks/test_text_block.py b/python/tests/unit/template_engine/blocks/test_text_block.py index 131115b80137..aff0a97244c5 100644 --- a/python/tests/unit/template_engine/blocks/test_text_block.py +++ b/python/tests/unit/template_engine/blocks/test_text_block.py @@ -10,32 +10,36 @@ def test_init(): - text_block = TextBlock(text="test text", log=Logger("test_logger")) + text_block = TextBlock.from_text(text="test text", log=Logger("test_logger")) assert text_block.content == "test text" assert isinstance(text_block.log, Logger) def test_init_with_just_start_index(): - text_block = TextBlock(text="test text", start_index=2, log=Logger("test_logger")) + text_block = TextBlock.from_text( + text="test text", start_index=2, log=Logger("test_logger") + ) assert text_block.content == "st text" assert isinstance(text_block.log, Logger) def test_init_with_just_stop_index(): - text_block = TextBlock(text="test text", stop_index=2, log=Logger("test_logger")) + text_block = TextBlock.from_text( + text="test text", stop_index=2, log=Logger("test_logger") + ) assert text_block.content == "te" assert isinstance(text_block.log, Logger) def test_init_with_start_index_greater_than_stop_index(): with raises(ValueError): - TextBlock( + TextBlock.from_text( text="test text", start_index=2, stop_index=1, log=Logger("test_logger") ) def test_init_with_start_stop_indices(): - text_block = TextBlock( + text_block = TextBlock.from_text( text="test text", start_index=0, stop_index=4, log=Logger("test_logger") ) assert text_block.content == "test" @@ -44,66 +48,68 @@ def test_init_with_start_stop_indices(): def test_init_with_start_index_less_than_zero(): with raises(ValueError): - TextBlock( + TextBlock.from_text( text="test text", start_index=-1, stop_index=1, log=Logger("test_logger") ) def test_init_with_negative_stop_index(): - text_block = TextBlock(text="test text", stop_index=-1, log=Logger("test_logger")) + text_block = TextBlock.from_text( + text="test text", stop_index=-1, log=Logger("test_logger") + ) assert text_block.content == "test tex" def test_type_property(): - text_block = TextBlock(text="test text") + text_block = TextBlock.from_text(text="test text") assert text_block.type == BlockTypes.TEXT def test_is_valid(): - text_block = TextBlock(text="test text") + text_block = TextBlock.from_text(text="test text") is_valid, error_msg = text_block.is_valid() assert is_valid assert error_msg == "" def test_render(): - text_block = TextBlock(text="test text") + text_block = TextBlock.from_text(text="test text") rendered_value = text_block.render(ContextVariables()) assert rendered_value == "test text" def test_preserves_empty_values(): - assert "" == TextBlock(text=None).content - assert "" == TextBlock(text="").content - assert " " == TextBlock(text=" ").content - assert " " == TextBlock(text=" ").content - assert " \n" == TextBlock(text=" \n").content - assert " \t" == TextBlock(text=" \t").content - assert " \r" == TextBlock(text=" \r").content + assert "" == TextBlock.from_text(text=None).content + assert "" == TextBlock.from_text(text="").content + assert " " == TextBlock.from_text(text=" ").content + assert " " == TextBlock.from_text(text=" ").content + assert " \n" == TextBlock.from_text(text=" \n").content + assert " \t" == TextBlock.from_text(text=" \t").content + assert " \r" == TextBlock.from_text(text=" \r").content def test_is_always_valid(): - assert TextBlock(text=None).is_valid() == (True, "") - assert TextBlock(text="").is_valid() == (True, "") - assert TextBlock(text=" ").is_valid() == (True, "") - assert TextBlock(text=" ").is_valid() == (True, "") - assert TextBlock(text=" \n").is_valid() == (True, "") - assert TextBlock(text=" \t").is_valid() == (True, "") - assert TextBlock(text=" \r").is_valid() == (True, "") - assert TextBlock(text="test").is_valid() == (True, "") - assert TextBlock(text=" \nabc").is_valid() == (True, "") + assert TextBlock.from_text(text=None).is_valid() == (True, "") + assert TextBlock.from_text(text="").is_valid() == (True, "") + assert TextBlock.from_text(text=" ").is_valid() == (True, "") + assert TextBlock.from_text(text=" ").is_valid() == (True, "") + assert TextBlock.from_text(text=" \n").is_valid() == (True, "") + assert TextBlock.from_text(text=" \t").is_valid() == (True, "") + assert TextBlock.from_text(text=" \r").is_valid() == (True, "") + assert TextBlock.from_text(text="test").is_valid() == (True, "") + assert TextBlock.from_text(text=" \nabc").is_valid() == (True, "") def test_renders_the_content_as_it(): - assert TextBlock(text=None).render() == "" - assert TextBlock(text="").render() == "" - assert TextBlock(text=" ").render() == " " - assert TextBlock(text=" ").render() == " " - assert TextBlock(text=" \n").render() == " \n" - assert TextBlock(text=" \t").render() == " \t" - assert TextBlock(text=" \r").render() == " \r" - assert TextBlock(text="test").render() == "test" - assert TextBlock(text=" \nabc").render() == " \nabc" - assert TextBlock(text="'x'").render() == "'x'" - assert TextBlock(text='"x"').render() == '"x"' - assert TextBlock(text="\"'x'\"").render() == "\"'x'\"" + assert TextBlock.from_text(text=None).render() == "" + assert TextBlock.from_text(text="").render() == "" + assert TextBlock.from_text(text=" ").render() == " " + assert TextBlock.from_text(text=" ").render() == " " + assert TextBlock.from_text(text=" \n").render() == " \n" + assert TextBlock.from_text(text=" \t").render() == " \t" + assert TextBlock.from_text(text=" \r").render() == " \r" + assert TextBlock.from_text(text="test").render() == "test" + assert TextBlock.from_text(text=" \nabc").render() == " \nabc" + assert TextBlock.from_text(text="'x'").render() == "'x'" + assert TextBlock.from_text(text='"x"').render() == '"x"' + assert TextBlock.from_text(text="\"'x'\"").render() == "\"'x'\"" diff --git a/python/tests/unit/template_engine/test_prompt_template_engine.py b/python/tests/unit/template_engine/test_prompt_template_engine.py index 8d9d9c452a57..5f5487f60bfe 100644 --- a/python/tests/unit/template_engine/test_prompt_template_engine.py +++ b/python/tests/unit/template_engine/test_prompt_template_engine.py @@ -126,10 +126,9 @@ def test_it_renders_variables( async def test_it_renders_code_using_input_async( target: PromptTemplateEngine, variables: ContextVariables, - skills: Mock, - context: SKContext, + context_factory, ): - @sk_function(name="test") + @sk_function(name="function") def my_function_async(cx: SKContext) -> str: return f"F({cx.variables.input})" @@ -138,11 +137,7 @@ def my_function_async(cx: SKContext) -> str: variables.update("INPUT-BAR") template = "foo-{{function}}-baz" - skills.configure_mock( - **{"has_function.return_value": True, "get_function.return_value": func} - ) - - result = await target.render_async(template, context) + result = await target.render_async(template, context_factory(variables, func)) assert result == "foo-F(INPUT-BAR)-baz" @@ -151,10 +146,9 @@ def my_function_async(cx: SKContext) -> str: async def test_it_renders_code_using_variables_async( target: PromptTemplateEngine, variables: ContextVariables, - skills: Mock, - context: SKContext, + context_factory, ): - @sk_function(name="test") + @sk_function(name="function") def my_function_async(cx: SKContext) -> str: return f"F({cx.variables.input})" @@ -163,11 +157,7 @@ def my_function_async(cx: SKContext) -> str: variables.set("myVar", "BAR") template = "foo-{{function $myVar}}-baz" - skills.configure_mock( - **{"has_function.return_value": True, "get_function.return_value": func} - ) - - result = await target.render_async(template, context) + result = await target.render_async(template, context_factory(variables, func)) assert result == "foo-F(BAR)-baz" @@ -176,10 +166,9 @@ def my_function_async(cx: SKContext) -> str: async def test_it_renders_async_code_using_variables_async( target: PromptTemplateEngine, variables: ContextVariables, - skills: Mock, - context: SKContext, + context_factory, ): - @sk_function(name="test") + @sk_function(name="function") async def my_function_async(cx: SKContext) -> str: return cx.variables.input @@ -189,10 +178,7 @@ async def my_function_async(cx: SKContext) -> str: variables.set("myVar", "BAR") template = "foo-{{function $myVar}}-baz" - skills.configure_mock( - **{"has_function.return_value": True, "get_function.return_value": func} - ) - result = await target.render_async(template, context) + result = await target.render_async(template, context_factory(variables, func)) assert result == "foo-BAR-baz" diff --git a/python/tests/unit/test_serialization.py b/python/tests/unit/test_serialization.py new file mode 100644 index 000000000000..53d2d760daaa --- /dev/null +++ b/python/tests/unit/test_serialization.py @@ -0,0 +1,285 @@ +import logging +import typing as t + +import pydantic as pdt +import pytest +import typing_extensions as te + +from semantic_kernel import SKFunctionBase +from semantic_kernel.core_skills.conversation_summary_skill import ( + ConversationSummarySkill, +) +from semantic_kernel.core_skills.file_io_skill import FileIOSkill +from semantic_kernel.core_skills.http_skill import HttpSkill +from semantic_kernel.core_skills.math_skill import MathSkill +from semantic_kernel.core_skills.text_memory_skill import TextMemorySkill +from semantic_kernel.core_skills.text_skill import TextSkill +from semantic_kernel.core_skills.time_skill import TimeSkill +from semantic_kernel.core_skills.wait_skill import WaitSkill +from semantic_kernel.core_skills.web_search_engine_skill import WebSearchEngineSkill +from semantic_kernel.memory.null_memory import NullMemory +from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryBase +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.delegate_handlers import DelegateHandlers +from semantic_kernel.orchestration.delegate_inference import DelegateInference +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function import SKFunction +from semantic_kernel.sk_pydantic import PydanticField, SKBaseModel +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.functions_view import FunctionsView +from semantic_kernel.skill_definition.parameter_view import ParameterView +from semantic_kernel.skill_definition.read_only_skill_collection import ( + ReadOnlySkillCollection, +) +from semantic_kernel.skill_definition.read_only_skill_collection_base import ( + ReadOnlySkillCollectionBase, +) +from semantic_kernel.skill_definition.sk_function_decorator import sk_function +from semantic_kernel.skill_definition.skill_collection import SkillCollection +from semantic_kernel.skill_definition.skill_collection_base import SkillCollectionBase +from semantic_kernel.template_engine.blocks.block import Block +from semantic_kernel.template_engine.blocks.block_types import BlockTypes +from semantic_kernel.template_engine.blocks.code_block import CodeBlock +from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock +from semantic_kernel.template_engine.blocks.text_block import TextBlock +from semantic_kernel.template_engine.blocks.val_block import ValBlock +from semantic_kernel.template_engine.blocks.var_block import VarBlock +from semantic_kernel.template_engine.code_tokenizer import CodeTokenizer +from semantic_kernel.template_engine.prompt_template_engine import PromptTemplateEngine +from semantic_kernel.template_engine.protocols.code_renderer import CodeRenderer +from semantic_kernel.template_engine.protocols.prompt_templating_engine import ( + PromptTemplatingEngine, +) +from semantic_kernel.template_engine.protocols.text_renderer import TextRenderer +from semantic_kernel.template_engine.template_tokenizer import TemplateTokenizer + +PydanticFieldT = t.TypeVar("PydanticFieldT", bound=PydanticField) + + +class _Serializable(t.Protocol): + """A serializable object.""" + + def json(self) -> pdt.Json: + """Return a JSON representation of the object.""" + raise NotImplementedError + + @classmethod + def parse_raw(cls: t.Type[te.Self], json: pdt.Json) -> te.Self: + """Return the constructed object from a JSON representation.""" + raise NotImplementedError + + +@pytest.fixture() +def sk_factory() -> t.Callable[[t.Type[_Serializable]], _Serializable]: + """Return a factory for various objects in semantic-kernel.""" + + def create_functions_view() -> FunctionsView: + """Return a functions view.""" + result = FunctionsView() + result.add_function( + FunctionView( + name="function1", + skill_name="skill1", + description="Native function", + parameters=[], + is_semantic=False, + is_asynchronous=True, + ) + ) + result.add_function( + FunctionView( + name="function1", + skill_name="skill1", + description="Semantic function", + parameters=[], + is_semantic=True, + is_asynchronous=True, + ) + ) + return result + + def create_sk_function() -> SKFunction: + """Return an SKFunction.""" + + @sk_function(name="function") + def my_function_async(cx: SKContext) -> str: + return f"F({cx.variables.input})" + + return SKFunction.from_native_method(my_function_async) + + def create_context_variables() -> ContextVariables: + """Return a context variables object.""" + return ContextVariables( + content="content", + variables={"foo": "bar"}, + ) + + def create_skill_collection() -> SkillCollection: + """Return a skill collection.""" + # TODO: Add a few skills to this collection. + return SkillCollection() + + cls_obj_map = { + Block: Block("foo"), + CodeBlock: CodeBlock("foo"), + FunctionIdBlock: FunctionIdBlock("bar"), + TextBlock: TextBlock("baz"), + ValBlock: ValBlock("qux"), + VarBlock: VarBlock("quux"), + CodeTokenizer: CodeTokenizer(log=logging.getLogger("test")), + PromptTemplateEngine: PromptTemplateEngine(logger=logging.getLogger("test")), + TemplateTokenizer: TemplateTokenizer(log=logging.getLogger("test")), + ParameterView: ParameterView( + name="foo", + description="bar", + default_value="baz", + type="string", + required=True, + ), + FunctionView: FunctionView( + "foo", + "bar", + "baz", + [ParameterView(name="qux", description="bar", default_value="baz")], + True, + False, + ), + FunctionsView: create_functions_view(), + ReadOnlySkillCollection: create_skill_collection().read_only_skill_collection, + DelegateHandlers: DelegateHandlers(), + DelegateInference: DelegateInference(), + ContextVariables: create_context_variables(), + SkillCollection: create_skill_collection(), + SKContext[NullMemory]: SKContext( + # TODO: Test serialization with different types of memories. + memory=NullMemory(), + variables=create_context_variables(), + skill_collection=create_skill_collection().read_only_skill_collection, + ), + NullMemory: NullMemory(), + } + + def constructor(cls: t.Type[_Serializable]) -> _Serializable: + """Return a serializable object.""" + return cls_obj_map[cls] + + return constructor + + +PROTOCOLS = [ + pytest.param( + ConversationSummarySkill, marks=pytest.mark.xfail(reason="Contains data") + ), + FileIOSkill, + HttpSkill, + MathSkill, + TextMemorySkill, + TextSkill, + TimeSkill, + WaitSkill, + pytest.param(WebSearchEngineSkill, marks=pytest.mark.xfail(reason="Contains data")), + CodeRenderer, + PromptTemplatingEngine, + TextRenderer, +] + +BASE_CLASSES = [ + ReadOnlySkillCollectionBase, + SkillCollectionBase, + SemanticTextMemoryBase, + SKFunctionBase, +] + +# Classes that don't need serialization +UNSERIALIZED_CLASSES = [ + ReadOnlySkillCollection, +] + +STATELESS_CLASSES = [ + CodeTokenizer, + PromptTemplateEngine, + TemplateTokenizer, + DelegateHandlers, + DelegateInference, + NullMemory, +] + +ENUMS = [ + BlockTypes, + DelegateInference, +] + +PYDANTIC_MODELS = [ + Block, + CodeBlock, + FunctionIdBlock, + TextBlock, + ValBlock, + VarBlock, + ParameterView, + FunctionView, + FunctionsView, + ReadOnlySkillCollection, + SkillCollection, + ContextVariables, + SKContext[NullMemory], + pytest.param( + SKFunction, + marks=pytest.mark.xfail(reason="Need to implement Pickle serialization."), + ), +] + + +class TestUsageInPydanticFields: + @pytest.mark.parametrize( + "sk_type", + BASE_CLASSES + + PROTOCOLS + + ENUMS + + PYDANTIC_MODELS + + STATELESS_CLASSES + + UNSERIALIZED_CLASSES, + ) + def test_usage_as_optional_field( + self, + sk_type: t.Type[PydanticFieldT], + ) -> None: + """Semantic Kernel objects should be valid Pydantic fields. + + Otherwise, they cannot be used in Pydantic models. + """ + + class TestModel(SKBaseModel): + """A test model.""" + + field: t.Optional[sk_type] = None + + assert_serializable(TestModel(), TestModel) + + @pytest.mark.parametrize("sk_type", PYDANTIC_MODELS + STATELESS_CLASSES) + def test_usage_as_required_field( + self, + sk_factory: t.Callable[[t.Type[PydanticFieldT]], PydanticFieldT], + sk_type: t.Type[PydanticFieldT], + ) -> None: + """Semantic Kernel objects should be valid Pydantic fields. + + Otherwise, they cannot be used in Pydantic models. + """ + + class TestModel(SKBaseModel): + """A test model.""" + + field: sk_type = pdt.Field(default_factory=lambda: sk_factory(sk_type)) + + assert_serializable(TestModel(), TestModel) + assert_serializable(TestModel(field=sk_factory(sk_type)), TestModel) + + +def assert_serializable(obj: _Serializable, obj_type) -> None: + """Assert that an object is serializable.""" + assert obj is not None + serialized = obj.json() + assert isinstance(serialized, str) + deserialized = obj_type.parse_raw(serialized) + assert deserialized == obj diff --git a/samples/apps/auth-api-webapp-react/README.md b/samples/apps/auth-api-webapp-react/README.md index 378067e435b9..8e3f546388dd 100644 --- a/samples/apps/auth-api-webapp-react/README.md +++ b/samples/apps/auth-api-webapp-react/README.md @@ -1,5 +1,10 @@ # Authenticated API’s Sample Learning App +> [!IMPORTANT] +> This sample will be removed in a future release. If you are looking for samples that demonstrate +> how to use Semantic Kernel, please refer to the sample folders in the root [python](../../../python/samples/) +> and [dotnet](../../../dotnet/samples/) folders. + > [!IMPORTANT] > This learning sample is for educational purposes only and should not be used in any production > use case. It is intended to highlight concepts of Semantic Kernel and not any @@ -9,8 +14,8 @@ ## Running the sample -1. You will need an [Open AI Key](https://openai.com/api/) or - [Azure Open AI Service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart) +1. You will need an [OpenAI Key](https://openai.com/product/) or + [Azure OpenAI Service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart) for this sample 2. Ensure the KernelHttpServer sample is already running at `http://localhost:7071`. If not, follow the steps to start it [here](../../dotnet/KernelHttpServer/README.md). @@ -21,7 +26,7 @@ - Select **`Single-page application (SPA)`** as platform type, and the Redirect URI will be **`http://localhost:3000`** - Select **`Personal Microsoft accounts only`** as supported account types for this sample 4. Copy **[.env.example](.env.example)** into a new file with name "**.env**". - > **Note**: Samples are configured to use chat completion AI models (e.g., gpt-3.5-turbo, gpt-4, etc.). See https://platform.openai.com/docs/models/model-endpoint-compatibility for chat completion model options. + > **Note**: Samples are configured to use chat completion AI models (e.g., gpt-3.5-turbo, gpt-4, etc.). See https://platform.openai.com/docs/models/model-endpoint-compatibility for chat completion model options. 5. Once registered, copy the **Application (client) ID** from the Azure Portal and paste the GUID into the **.env** file next to `REACT_APP_GRAPH_CLIENT_ID=` (first line of the .env file). 6. **Run** the following command `yarn install` (if you have never run the sample before) diff --git a/samples/apps/auth-api-webapp-react/src/components/InteractWithGraph.tsx b/samples/apps/auth-api-webapp-react/src/components/InteractWithGraph.tsx index b02797b11fb4..9f0d7255d890 100644 --- a/samples/apps/auth-api-webapp-react/src/components/InteractWithGraph.tsx +++ b/samples/apps/auth-api-webapp-react/src/components/InteractWithGraph.tsx @@ -35,7 +35,7 @@ const InteractWithGraph: FC = ({ uri, config, onBack }) => { inputs: [{ key: 'filePath', value: path }], }, 'documentskill', - 'appendtextasync', + 'appendtext', ); //upload to onedrive @@ -46,7 +46,7 @@ const InteractWithGraph: FC = ({ uri, config, onBack }) => { inputs: [{ key: 'destinationPath', value: destinationPath }], }, 'clouddriveskill', - 'uploadfileasync', + 'uploadfile', ); } catch (e) { alert('Something went wrong.\n\nDetails:\n' + e); @@ -59,9 +59,9 @@ const InteractWithGraph: FC = ({ uri, config, onBack }) => { config, { value: destinationPath }, 'clouddriveskill', - 'createlinkasync', + 'createlink', ); - var myEmail = await sk.invokeAsync(config, { value: '' }, 'emailskill', 'getmyemailaddressasync'); + var myEmail = await sk.invokeAsync(config, { value: '' }, 'emailskill', 'getmyemailaddress'); await sk.invokeAsync( config, @@ -79,7 +79,7 @@ const InteractWithGraph: FC = ({ uri, config, onBack }) => { ], }, 'emailskill', - 'sendemailasync', + 'sendemail', ); } catch (e) { alert('Something went wrong.\n\nDetails:\n' + e); @@ -103,7 +103,7 @@ const InteractWithGraph: FC = ({ uri, config, onBack }) => { ], }, 'tasklistskill', - 'addtaskasync', + 'addtask', ); } catch (e) { alert('Something went wrong.\n\nDetails:\n' + e); diff --git a/samples/apps/auth-api-webapp-react/yarn.lock b/samples/apps/auth-api-webapp-react/yarn.lock index a77b3a40f74e..e4c26c2c94d4 100644 --- a/samples/apps/auth-api-webapp-react/yarn.lock +++ b/samples/apps/auth-api-webapp-react/yarn.lock @@ -3,9 +3,9 @@ "@adobe/css-tools@^4.0.1": - version "4.2.0" - resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.2.0.tgz#e1a84fca468f4b337816fcb7f0964beb620ba855" - integrity sha512-E09FiIft46CmH5Qnjb0wsW54/YQd69LsxeKUOWawmws1XWvyFGURnAChH0mlr7YPFR1ofwvUQfcL0J3lMxXqPA== + version "4.3.1" + resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.1.tgz#abfccb8ca78075a2b6187345c26243c1a0842f28" + integrity sha512-/62yikz7NLScCGAAST5SHdnjaDJQBDq0M2muyRTpf2VQhw6StBg2ALiu73zSJQ4fMVLA+0uBhBHAle7Wg+2kSg== "@ampproject/remapping@^2.2.0": version "2.2.0" @@ -48,6 +48,14 @@ dependencies: "@babel/highlight" "^7.18.6" +"@babel/code-frame@^7.22.13": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== + dependencies: + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" + "@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.1", "@babel/compat-data@^7.20.5": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.21.0.tgz#c241dc454e5b5917e40d37e525e2f4530c399298" @@ -93,6 +101,16 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz#eaa49f6f80d5a33f9a5dd2276e6d6e451be0a6bb" @@ -158,6 +176,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz#0c0cee9b35d2ca190478756865bb3528422f51be" integrity sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg== +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + "@babel/helper-explode-assignable-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz#41f8228ef0a6f1a036b8dfdfec7ce94f9a6bc096" @@ -173,6 +196,14 @@ "@babel/template" "^7.20.7" "@babel/types" "^7.21.0" +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== + dependencies: + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" + "@babel/helper-hoist-variables@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz#d4d2c8fb4baeaa5c68b99cc8245c56554f926678" @@ -180,6 +211,13 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-member-expression-to-functions@^7.20.7", "@babel/helper-member-expression-to-functions@^7.21.0": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.21.0.tgz#319c6a940431a133897148515877d2f3269c3ba5" @@ -263,16 +301,33 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-string-parser@^7.19.4": version "7.19.4" resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz#38d3acb654b4701a9b77fb0615a96f775c3a9e63" integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw== +"@babel/helper-string-parser@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" + integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== + "@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": version "7.19.1" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-option@^7.18.6", "@babel/helper-validator-option@^7.21.0": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.21.0.tgz#8224c7e13ace4bafdc4004da2cf064ef42673180" @@ -306,11 +361,25 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.22.13": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" + js-tokens "^4.0.0" + "@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.21.0": version "7.21.1" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.21.1.tgz#a8f81ee2fe872af23faea4b17a08fcc869de7bcc" integrity sha512-JzhBFpkuhBNYUY7qs+wTzNmyCWUHEaAFpQQD2YfU1rPL38/L43Wvid0fFkiOCnHvsGncRZgEPyGnltABLcVDTg== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz#da5b8f9a580acdfbe53494dba45ea389fb09a4d2" @@ -1068,19 +1137,28 @@ "@babel/parser" "^7.20.7" "@babel/types" "^7.20.7" -"@babel/traverse@^7.20.5", "@babel/traverse@^7.20.7", "@babel/traverse@^7.21.0", "@babel/traverse@^7.7.2": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.21.0.tgz#0e1807abd5db98e6a19c204b80ed1e3f5bca0edc" - integrity sha512-Xdt2P1H4LKTO8ApPfnO1KmzYMFpp7D/EinoXzLYN/cHcBNrVCAkAtGUcXnHXrl/VGktureU6fkQrHSBE2URfoA== +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== dependencies: - "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.21.0" - "@babel/helper-environment-visitor" "^7.18.9" - "@babel/helper-function-name" "^7.21.0" - "@babel/helper-hoist-variables" "^7.18.6" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.21.0" - "@babel/types" "^7.21.0" + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.20.5", "@babel/traverse@^7.20.7", "@babel/traverse@^7.21.0", "@babel/traverse@^7.7.2": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -1093,6 +1171,15 @@ "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -3693,7 +3780,7 @@ case-sensitive-paths-webpack-plugin@^2.4.0: resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz#db64066c6422eed2e08cc14b986ca43796dbc6d4" integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw== -chalk@^2.0.0, chalk@^2.4.1: +chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== diff --git a/samples/apps/book-creator-webapp-react/README.md b/samples/apps/book-creator-webapp-react/README.md index 280eba5cbe1e..644a94c6d1a4 100644 --- a/samples/apps/book-creator-webapp-react/README.md +++ b/samples/apps/book-creator-webapp-react/README.md @@ -1,5 +1,10 @@ # Book Creator Sample Learning App +> [!IMPORTANT] +> This sample will be removed in a future release. If you are looking for samples that demonstrate +> how to use Semantic Kernel, please refer to the sample folders in the root [python](../../../python/samples/) +> and [dotnet](../../../dotnet/samples/) folders. + > **!IMPORTANT** > This learning sample is for educational purposes only and should not be used in any > production use case. It is intended to highlight concepts of Semantic Kernel and not @@ -9,13 +14,13 @@ ## Running the sample -1. You will need an [Open AI Key](https://openai.com/api/) or - [Azure Open AI Service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart) +1. You will need an [OpenAI Key](https://openai.com/product/) or + [Azure OpenAI Service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart) for this sample. 2. Ensure the KernelHttpServer sample is already running at `http://localhost:7071`. If not, follow the steps to start it [here](../../dotnet/KernelHttpServer/README.md). 3. Copy **[.env.example](.env.example)** into a new file with name "**.env**". - > **Note**: Samples are configured to use chat completion AI models (e.g., gpt-3.5-turbo, gpt-4, etc.). See https://platform.openai.com/docs/models/model-endpoint-compatibility for chat completion model options. + > **Note**: Samples are configured to use chat completion AI models (e.g., gpt-3.5-turbo, gpt-4, etc.). See https://platform.openai.com/docs/models/model-endpoint-compatibility for chat completion model options. 4. You will also need to **Run** the following command `yarn install` (if you have never run the sample before) and/or `yarn start` from the command line. 5. A browser will automatically open, otherwise you can navigate to `http://localhost:3000` to use the sample. diff --git a/samples/apps/book-creator-webapp-react/yarn.lock b/samples/apps/book-creator-webapp-react/yarn.lock index 0361c24e2950..65b9324d9f11 100644 --- a/samples/apps/book-creator-webapp-react/yarn.lock +++ b/samples/apps/book-creator-webapp-react/yarn.lock @@ -3,9 +3,9 @@ "@adobe/css-tools@^4.0.1": - version "4.2.0" - resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.2.0.tgz#e1a84fca468f4b337816fcb7f0964beb620ba855" - integrity sha512-E09FiIft46CmH5Qnjb0wsW54/YQd69LsxeKUOWawmws1XWvyFGURnAChH0mlr7YPFR1ofwvUQfcL0J3lMxXqPA== + version "4.3.1" + resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.1.tgz#abfccb8ca78075a2b6187345c26243c1a0842f28" + integrity sha512-/62yikz7NLScCGAAST5SHdnjaDJQBDq0M2muyRTpf2VQhw6StBg2ALiu73zSJQ4fMVLA+0uBhBHAle7Wg+2kSg== "@ampproject/remapping@^2.2.0": version "2.2.0" @@ -31,6 +31,14 @@ dependencies: "@babel/highlight" "^7.18.6" +"@babel/code-frame@^7.22.13": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== + dependencies: + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" + "@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.1", "@babel/compat-data@^7.20.5": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.21.0.tgz#c241dc454e5b5917e40d37e525e2f4530c399298" @@ -76,6 +84,16 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz#eaa49f6f80d5a33f9a5dd2276e6d6e451be0a6bb" @@ -141,6 +159,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz#0c0cee9b35d2ca190478756865bb3528422f51be" integrity sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg== +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + "@babel/helper-explode-assignable-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz#41f8228ef0a6f1a036b8dfdfec7ce94f9a6bc096" @@ -156,6 +179,14 @@ "@babel/template" "^7.20.7" "@babel/types" "^7.21.0" +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== + dependencies: + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" + "@babel/helper-hoist-variables@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz#d4d2c8fb4baeaa5c68b99cc8245c56554f926678" @@ -163,6 +194,13 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-member-expression-to-functions@^7.20.7", "@babel/helper-member-expression-to-functions@^7.21.0": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.21.0.tgz#319c6a940431a133897148515877d2f3269c3ba5" @@ -246,16 +284,33 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-string-parser@^7.19.4": version "7.19.4" resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz#38d3acb654b4701a9b77fb0615a96f775c3a9e63" integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw== +"@babel/helper-string-parser@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" + integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== + "@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": version "7.19.1" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-option@^7.18.6", "@babel/helper-validator-option@^7.21.0": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.21.0.tgz#8224c7e13ace4bafdc4004da2cf064ef42673180" @@ -289,11 +344,25 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.22.13": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" + js-tokens "^4.0.0" + "@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.21.0": version "7.21.1" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.21.1.tgz#a8f81ee2fe872af23faea4b17a08fcc869de7bcc" integrity sha512-JzhBFpkuhBNYUY7qs+wTzNmyCWUHEaAFpQQD2YfU1rPL38/L43Wvid0fFkiOCnHvsGncRZgEPyGnltABLcVDTg== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz#da5b8f9a580acdfbe53494dba45ea389fb09a4d2" @@ -1051,19 +1120,28 @@ "@babel/parser" "^7.20.7" "@babel/types" "^7.20.7" -"@babel/traverse@^7.20.5", "@babel/traverse@^7.20.7", "@babel/traverse@^7.21.0", "@babel/traverse@^7.7.2": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.21.0.tgz#0e1807abd5db98e6a19c204b80ed1e3f5bca0edc" - integrity sha512-Xdt2P1H4LKTO8ApPfnO1KmzYMFpp7D/EinoXzLYN/cHcBNrVCAkAtGUcXnHXrl/VGktureU6fkQrHSBE2URfoA== +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== dependencies: - "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.21.0" - "@babel/helper-environment-visitor" "^7.18.9" - "@babel/helper-function-name" "^7.21.0" - "@babel/helper-hoist-variables" "^7.18.6" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.21.0" - "@babel/types" "^7.21.0" + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.20.5", "@babel/traverse@^7.20.7", "@babel/traverse@^7.21.0", "@babel/traverse@^7.7.2": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -1076,6 +1154,15 @@ "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -3676,7 +3763,7 @@ case-sensitive-paths-webpack-plugin@^2.4.0: resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz#db64066c6422eed2e08cc14b986ca43796dbc6d4" integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw== -chalk@^2.0.0, chalk@^2.4.1: +chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== diff --git a/samples/apps/chat-summary-webapp-react/README.md b/samples/apps/chat-summary-webapp-react/README.md index b4c7aa39b2ea..48eef0561923 100644 --- a/samples/apps/chat-summary-webapp-react/README.md +++ b/samples/apps/chat-summary-webapp-react/README.md @@ -1,5 +1,10 @@ # Simple Chat Summary Sample Learning App +> [!IMPORTANT] +> This sample will be removed in a future release. If you are looking for samples that demonstrate +> how to use Semantic Kernel, please refer to the sample folders in the root [python](../../../python/samples/) +> and [dotnet](../../../dotnet/samples/) folders. + Watch the [Chat Summary Quick Start Video](https://aka.ms/SK-Samples-SimChat-Video). > **!IMPORTANT** @@ -9,13 +14,13 @@ Watch the [Chat Summary Quick Start Video](https://aka.ms/SK-Samples-SimChat-Vid ## Running the sample -1. You will need an [Open AI Key](https://openai.com/api/) or - [Azure Open AI Service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart) +1. You will need an [OpenAI Key](https://openai.com/product/) or + [Azure OpenAI Service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart) for this sample. 2. Ensure the KernelHttpServer sample is already running at `http://localhost:7071`. If not, follow the steps to start it [here](../../dotnet/KernelHttpServer/README.md). 3. Copy **[.env.example](.env.example)** into a new file with name "**.env**". - > **Note**: Samples are configured to use chat completion AI models (e.g., gpt-3.5-turbo, gpt-4, etc.). See https://platform.openai.com/docs/models/model-endpoint-compatibility for chat completion model options. + > **Note**: Samples are configured to use chat completion AI models (e.g., gpt-3.5-turbo, gpt-4, etc.). See https://platform.openai.com/docs/models/model-endpoint-compatibility for chat completion model options. 4. You will also need to **Run** the following command `yarn install` (if you have never run the sample before) and/or `yarn start` from the command line. 5. A browser will automatically open, otherwise you can navigate to `http://localhost:3000` to use the sample. @@ -26,8 +31,8 @@ Watch the [Chat Summary Quick Start Video](https://aka.ms/SK-Samples-SimChat-Vid The Simple Chat Summary sample allows you to see the power of semantic functions used in a chat. -The sample highlights the [SummarizeConversation](../../../dotnet/src/Skills/Skills.Core/SemanticFunctionConstants.cs#7), [GetConversationActionItems](../../../dotnet/src/Skills/Skills.Core/SemanticFunctionConstants.cs#20), and [GetConversationTopics](../../../dotnet/src/Skills/Skills.Core/SemanticFunctionConstants.cs#63) -native functions in the [Conversation Summary Skill](../../../dotnet/src/Skills//Skills.Core/ConversationSummarySkill.cs). +The sample highlights the [SummarizeConversation](../../../dotnet/src/Plugins/Plugins.Core/SemanticFunctionConstants.cs#7), [GetConversationActionItems](../../../dotnet/src/Plugins/Plugins.Core/SemanticFunctionConstants.cs#20), and [GetConversationTopics](../../../dotnet/src/Plugins/Plugins.Core/SemanticFunctionConstants.cs#63) +native functions in the [Conversation Summary Skill](../../../dotnet/src/Plugins/Plugins.Core/ConversationSummaryPlugin.cs). Each function calls Open AI to review the information in the chat window and produces insights. The chat data can be loaded from this [data file](src/components/chat/ChatThread.ts) – which you diff --git a/samples/apps/chat-summary-webapp-react/src/App.tsx b/samples/apps/chat-summary-webapp-react/src/App.tsx index d0fcd6f0548d..ba2bf2b30924 100644 --- a/samples/apps/chat-summary-webapp-react/src/App.tsx +++ b/samples/apps/chat-summary-webapp-react/src/App.tsx @@ -63,15 +63,15 @@ const App: FC = () => { items: [ { title: 'Summarize', - uri: 'https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Skills/Skills.Core/ConversationSummarySkill.cs#L70', + uri: 'https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Plugins/Plugins.Core/ConversationSummarySkill.cs#L70', }, { title: 'Action Items', - uri: 'https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Skills/Skills.Core/ConversationSummarySkill.cs#L87', + uri: 'https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Plugins/Plugins.Core/ConversationSummarySkill.cs#L87', }, { title: 'Topics', - uri: 'https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Skills/Skills.Core/ConversationSummarySkill.cs#L104', + uri: 'https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Plugins/Plugins.Core/ConversationSummarySkill.cs#L104', }, ], }, diff --git a/samples/apps/chat-summary-webapp-react/yarn.lock b/samples/apps/chat-summary-webapp-react/yarn.lock index 7567f9048f6f..5a6e28062bac 100644 --- a/samples/apps/chat-summary-webapp-react/yarn.lock +++ b/samples/apps/chat-summary-webapp-react/yarn.lock @@ -3,9 +3,9 @@ "@adobe/css-tools@^4.0.1": - version "4.2.0" - resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.2.0.tgz#e1a84fca468f4b337816fcb7f0964beb620ba855" - integrity sha512-E09FiIft46CmH5Qnjb0wsW54/YQd69LsxeKUOWawmws1XWvyFGURnAChH0mlr7YPFR1ofwvUQfcL0J3lMxXqPA== + version "4.3.1" + resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.1.tgz#abfccb8ca78075a2b6187345c26243c1a0842f28" + integrity sha512-/62yikz7NLScCGAAST5SHdnjaDJQBDq0M2muyRTpf2VQhw6StBg2ALiu73zSJQ4fMVLA+0uBhBHAle7Wg+2kSg== "@ampproject/remapping@^2.2.0": version "2.2.0" @@ -31,6 +31,14 @@ dependencies: "@babel/highlight" "^7.18.6" +"@babel/code-frame@^7.22.13": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== + dependencies: + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" + "@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.1", "@babel/compat-data@^7.20.5": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.21.0.tgz#c241dc454e5b5917e40d37e525e2f4530c399298" @@ -76,6 +84,16 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz#eaa49f6f80d5a33f9a5dd2276e6d6e451be0a6bb" @@ -141,6 +159,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz#0c0cee9b35d2ca190478756865bb3528422f51be" integrity sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg== +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + "@babel/helper-explode-assignable-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz#41f8228ef0a6f1a036b8dfdfec7ce94f9a6bc096" @@ -156,6 +179,14 @@ "@babel/template" "^7.20.7" "@babel/types" "^7.21.0" +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== + dependencies: + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" + "@babel/helper-hoist-variables@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz#d4d2c8fb4baeaa5c68b99cc8245c56554f926678" @@ -163,6 +194,13 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-member-expression-to-functions@^7.20.7", "@babel/helper-member-expression-to-functions@^7.21.0": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.21.0.tgz#319c6a940431a133897148515877d2f3269c3ba5" @@ -246,16 +284,33 @@ dependencies: "@babel/types" "^7.18.6" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-string-parser@^7.19.4": version "7.19.4" resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz#38d3acb654b4701a9b77fb0615a96f775c3a9e63" integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw== +"@babel/helper-string-parser@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" + integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== + "@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": version "7.19.1" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-option@^7.18.6", "@babel/helper-validator-option@^7.21.0": version "7.21.0" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.21.0.tgz#8224c7e13ace4bafdc4004da2cf064ef42673180" @@ -289,11 +344,25 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.22.13": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" + js-tokens "^4.0.0" + "@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.21.0": version "7.21.1" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.21.1.tgz#a8f81ee2fe872af23faea4b17a08fcc869de7bcc" integrity sha512-JzhBFpkuhBNYUY7qs+wTzNmyCWUHEaAFpQQD2YfU1rPL38/L43Wvid0fFkiOCnHvsGncRZgEPyGnltABLcVDTg== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz#da5b8f9a580acdfbe53494dba45ea389fb09a4d2" @@ -1051,19 +1120,28 @@ "@babel/parser" "^7.20.7" "@babel/types" "^7.20.7" -"@babel/traverse@^7.20.5", "@babel/traverse@^7.20.7", "@babel/traverse@^7.21.0", "@babel/traverse@^7.7.2": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.21.0.tgz#0e1807abd5db98e6a19c204b80ed1e3f5bca0edc" - integrity sha512-Xdt2P1H4LKTO8ApPfnO1KmzYMFpp7D/EinoXzLYN/cHcBNrVCAkAtGUcXnHXrl/VGktureU6fkQrHSBE2URfoA== +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== dependencies: - "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.21.0" - "@babel/helper-environment-visitor" "^7.18.9" - "@babel/helper-function-name" "^7.21.0" - "@babel/helper-hoist-variables" "^7.18.6" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.21.0" - "@babel/types" "^7.21.0" + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.20.5", "@babel/traverse@^7.20.7", "@babel/traverse@^7.21.0", "@babel/traverse@^7.7.2": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -1076,6 +1154,15 @@ "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -3676,7 +3763,7 @@ case-sensitive-paths-webpack-plugin@^2.4.0: resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz#db64066c6422eed2e08cc14b986ca43796dbc6d4" integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw== -chalk@^2.0.0, chalk@^2.4.1: +chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== diff --git a/samples/apps/copilot-chat-app/.vscode/tasks.json b/samples/apps/copilot-chat-app/.vscode/tasks.json deleted file mode 100644 index 45d576208fa7..000000000000 --- a/samples/apps/copilot-chat-app/.vscode/tasks.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - // Copilot Chat - { - "label": "run (Copilot Chat)", - "detail": "Run all copilot chat components", - "group": "test", - "dependsOn": ["run (CopilotChatWebApi)", "run (CopilotChatApp)"], - "dependsOrder": "parallel" - }, - // Copilot Setup - { - "label": "install (CopilotChatApp)", - "detail": "Install all copilot chat app dependencies", - "type": "shell", - "group": "build", - "command": "yarn", - "presentation": { - "echo": true, - "reveal": "always", - "showReuseMessage": false, - "panel": "shared", - "group": "buildTasks" - }, - "options": { - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapp" - }, - "problemMatcher": [] - }, - { - "label": "setup (Copilot Chat)", - "detail": "Setup (like setting secrets) for copilot chat app and api", - "group": "test", - "dependsOn": ["GetSecret (AIService:Key)"], - "dependsOrder": "sequence" - // TODO -- add tasks for configuring environment variables - }, - { - "label": "GetSecret (AIService:Key)", - "command": "dotnet", - "type": "process", - "args": ["user-secrets", "set", "AIService:Key", "${input:aiServiceSecret}"], - "options": { - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapi" - } - }, - // Copilot Chat App - { - "label": "build (CopilotChatApp)", - "type": "shell", - "group": "build", - "command": "yarn build", - "presentation": { - "echo": true, - "reveal": "always", - "panel": "shared", - "group": "buildTasks" - }, - "options": { - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapp" - }, - "problemMatcher": [] - }, - { - "label": "run (CopilotChatApp)", - "type": "shell", - "group": "test", - "command": "yarn start", - "presentation": { - "reveal": "always", - "panel": "shared", - "group": "copilot" - }, - "options": { - "cwd": "${workspaceFolder}/samples/apps/copilot-chat-app/webapp" - } - }, - // Copilot Chat Api - { - "label": "build (CopilotChatWebApi)", - "command": "dotnet", - "type": "process", - "args": [ - "build", - "${workspaceFolder}/samples/apps/copilot-chat-app/webapi/CopilotChatWebApi.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary", - "/property:DebugType=portable" - ], - "problemMatcher": "$msCompile", - "group": "build" - }, - { - "label": "run (CopilotChatWebApi)", - "command": "dotnet", - "type": "process", - "args": [ - "run", - "--project", - "${workspaceFolder}/samples/apps/copilot-chat-app/webapi/CopilotChatWebApi.csproj" - ], - "problemMatcher": "$msCompile", - "group": "test", - "presentation": { - "reveal": "always", - "panel": "shared", - "group": "copilot" - } - }, - { - "label": "watch (CopilotChatWebApi)", - "command": "dotnet", - "type": "process", - "args": [ - "watch", - "run", - "--project", - "${workspaceFolder}/samples/apps/copilot-chat-app/webapi/CopilotChatWebApi.csproj" - ], - "problemMatcher": "$msCompile", - "group": "build" - } - ], - "inputs": [ - { - "id": "aiServiceSecret", - "type": "promptString", - "default": "", - "description": "Enter a secret for Copilot Chat AIService:Key", - "password": true - } - ] -} diff --git a/samples/apps/copilot-chat-app/CopilotChat.sln b/samples/apps/copilot-chat-app/CopilotChat.sln deleted file mode 100644 index 1135cca43bdc..000000000000 --- a/samples/apps/copilot-chat-app/CopilotChat.sln +++ /dev/null @@ -1,25 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.6.33706.43 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CopilotChatWebApi", "webapi\CopilotChatWebApi.csproj", "{5252E68F-B653-44CE-9A32-360A75C54E0E}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {5252E68F-B653-44CE-9A32-360A75C54E0E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5252E68F-B653-44CE-9A32-360A75C54E0E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5252E68F-B653-44CE-9A32-360A75C54E0E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5252E68F-B653-44CE-9A32-360A75C54E0E}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {971570D3-60EA-4EE4-980C-0BDA3E66E741} - EndGlobalSection -EndGlobal diff --git a/samples/apps/copilot-chat-app/README.md b/samples/apps/copilot-chat-app/README.md index 92a0807e25a0..3acc1a105104 100644 --- a/samples/apps/copilot-chat-app/README.md +++ b/samples/apps/copilot-chat-app/README.md @@ -1,172 +1,3 @@ -# Copilot Chat Sample Application +# Semantic Kernel copilot-chat-app -> This sample is for educational purposes only and is not recommended for production deployments. - -# About Copilot Chat - -This sample allows you to build your own integrated large language model chat copilot. -This is an enriched intelligence app, with multiple dynamic components including -command messages, user intent, and memories. - -The chat prompt and response will evolve as the conversation between the user and the application proceeds. -This chat experience is orchestrated with Semantic Kernel and a Copilot Chat skill containing numerous -functions that work together to construct each response. - -![UI Sample](images/UI-Sample.png) - -# Automated Setup and Local Deployment - -Refer to [./scripts/README.md](./scripts/README.md) for local configuration and deployment. - -Refer to [./deploy/README.md](./deploy/README.md) for Azure configuration and deployment. - -# Manual Setup and Local Deployment - -## Configure your environment - -Before you get started, make sure you have the following requirements in place: - -- [.NET 6.0 SDK](https://dotnet.microsoft.com/download/dotnet/6.0) -- [Node.js](https://nodejs.org/) -- [Yarn](https://classic.yarnpkg.com/lang/en/docs/install) - After installation, run `yarn --version` in a terminal window to ensure you are running v1.22.19. -- [Azure OpenAI](https://aka.ms/oai/access) resource or an account with [OpenAI](https://platform.openai.com). -- [Visual Studio Code](https://code.visualstudio.com/Download) **(Optional)** - -## Start the WebApi Backend Server - -The sample uses two applications, a front-end web UI, and a back-end API server. -First, let’s set up and verify the back-end API server is running. - -1. Generate and trust a localhost developer certificate. Open a terminal and run: - - For Windows and Mac run `dotnet dev-certs https --trust` and select `Yes` when asked if you want to install this certificate. - - For Linux run `dotnet dev-certs https` - > **Note:** It is recommended you close all instances of your web browser after installing the developer certificates. - -2. Navigate to `samples/apps/copilot-chat-app/webapi` and open `appsettings.json` - - Update the `AIService` configuration section: - - Update `Type` to the AI service you will be using (i.e., `AzureOpenAI` or `OpenAI`). - - If your are using Azure OpenAI, update `Endpoint` to your Azure OpenAI resource Endpoint address (e.g., - `http://contoso.openai.azure.com`). - > If you are using OpenAI, this property will be ignored. - - Set your Azure OpenAI or OpenAI key by opening a terminal in the webapi project directory and using `dotnet user-secrets` - ```bash - cd semantic-kernel/samples/apps/copilot-chat-app/webapi - dotnet user-secrets set "AIService:Key" "MY_AZUREOPENAI_OR_OPENAI_KEY" - ``` - - **(Optional)** Update `Models` to the Azure OpenAI deployment or OpenAI models you want to use. - - For `Completion` and `Planner`, CopilotChat is optimized for Chat completion models, such as gpt-3.5-turbo and gpt-4. - > **Important:** gpt-3.5-turbo is normally labelled as "`gpt-35-turbo`" (no period) in Azure OpenAI and "`gpt-3.5-turbo`" (with a period) in OpenAI. - - For `Embedding`, `text-embedding-ada-002` is sufficient and cost-effective for generating embeddings. - > **Important:** If you are using Azure OpenAI, please use [deployment names](https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource). If you are using OpenAI, please use [model names](https://platform.openai.com/docs/models). - - - **(Optional)** To enable speech-to-text for chat input, update the `AzureSpeech` configuration section: - > If you have not already, you will need to [create an Azure Speech resource](https://ms.portal.azure.com/#create/Microsoft.CognitiveServicesSpeechServices) - (see [./webapi/appsettings.json](webapi/appsettings.json) for more details). - - Update `Region` to whichever region is appropriate for your speech sdk instance. - - Set your Azure speech key by opening a terminal in the webapi project directory and setting - a dotnet user-secrets value for `AzureSpeech:Key` - ```bash - dotnet user-secrets set "AzureSpeech:Key" "MY_AZURE_SPEECH_KEY" - ``` - -3. Build and run the back-end API server - 1. Open a terminal and navigate to `samples/apps/copilot-chat-app/webapi` - - 2. Run `dotnet build` to build the project. - - 3. Run `dotnet run` to start the server. - - 4. Verify the back-end server is responding, open a web browser and navigate to `https://localhost:40443/healthz` - > The first time accessing the probe you may get a warning saying that there is a problem with website's certificate. - Select the option to accept/continue - this is expected when running a service on `localhost` - It is important to do this, as your browser may need to accept the certificate before allowing the frontend to communicate with the backend. - - > You may also need to acknowledge the Windows Defender Firewall, and allow the app to communicate over private or public networks as appropriate. - -## Start the WebApp FrontEnd application - -1. Build and start the front-end application - 1. You will need an Azure Active Directory (AAD) application registration. - > For more details on creating an application registration, go [here](https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app). - - Select `Single-page application (SPA)` as platform type, and set the Web redirect URI to `http://localhost:3000` - - Select `Accounts in any organizational directory and personal Microsoft Accounts` as supported account types for this sample. - - Make a note of the `Application (client) ID` from the Azure Portal, we will use of it later. - - 2. Open a terminal and navigate to `samples/apps/copilot-chat-app/webapp` Copy `.env.example` into a new - file `.env` and update the `REACT_APP_AAD_CLIENT_ID` with the AAD application (Client) ID created above. - For example: - ```bash - REACT_APP_BACKEND_URI=https://localhost:40443/ - REACT_APP_AAD_CLIENT_ID={Your Application (client) ID} - REACT_APP_AAD_AUTHORITY=https://login.microsoftonline.com/common - ``` - > For more detail on AAD authorities, see [Client Application Configuration Authorities](https://learn.microsoft.com/en-us/azure/active-directory/develop/msal-client-application-configuration#authority). - - > `REACT_APP_SK_API_KEY` is only required if you're using an Semantic Kernel service deployed to Azure. See the [Authorization section of Deploying Semantic Kernel to Azure in a web app service](./deploy/README.md#authorization) for more details and instruction on how to find your API key. - ```bash - REACT_APP_SK_API_KEY={Your API Key, should be the same as Authorization:ApiKey from appsettings.json} - ``` - - 3. To build and run the front-end application, open a terminal and navigate to `samples/apps/copilot-chat-app/webapp` if not already, then run: - ```bash - yarn install - yarn start - ``` - > To run the WebApp with HTTPs, see [How to use HTTPS for local development](./webapp/README.md#how-to-use-https-for-local-development). - - 4. With the back end and front end running, your web browser should automatically launch and navigate to `http://localhost:3000` - > The first time running the front-end application may take a minute or so to start. - - 5. Sign in with a Microsoft personal account or a "Work or School" account. - - 6. Consent permission for the application to read your profile information (i.e., your name). - - If you you experience any errors or issues, consult the troubleshooting section below. - -2. Have fun! - > **Note:** Each chat interaction will call Azure OpenAI/OpenAI which will use tokens that you may be billed for. - -# Troubleshooting - -## 1. Unable to load chats. Details: interaction_in_progress: Interaction is currently in progress. - -The WebApp can display this error when the application is configured for an active directory tenant, -(e.g., personal/MSA accounts) and the browser attempts to use single sign-on with an account from -another tenant (e.g., work or school account). Either user a private/incognito browser tab or clear -your browser credentials/cookies. - -## 2. Issues using text completion models, such as `text-davinci-003` - -CopilotChat supports chat completion models, such as `gpt-3.5-*` and `gpt-4-*`. -See [OpenAI's model compatibility](https://platform.openai.com/docs/models/model-endpoint-compatibility) for -the complete list of current models supporting chat completions. - -## 3. Localhost SSL certificate errors / CORS errors - -![](images/Cert-Issue.png) - -If you are stopped at an error message similar to the one above, your browser may be blocking the front-end access -to the back end while waiting for your permission to connect. To resolve this, try the following: - -1. Confirm the backend service is running by opening a web browser, and navigating to `https://localhost:40443/healthz` - - You should see a confirmation message: `Healthy` -2. If your browser asks you to acknowledge the risks of visiting an insecure website, you must acknowledge the - message before the front end will be allowed to connect to the back-end server. - - Acknowledge, continue, and navigate until you see the message `Healthy`. -3. Navigate to `http://localhost:3000` or refresh the page to use the Copilot Chat application. - -## 4. Have Yarn version 2.x or 3.x - -The webapp uses packages that are only supported by classic Yarn (v1.x). If you have Yarn v2.x+, run -the following commands in your preferred shell to flip Yarn to the classic version. - -```shell -npm install -g yarn -yarn set version classic -``` - -You can confirm the active Yarn version by running `yarn --version`. - -# Additional resources - -1. [Import Document Application](./importdocument/README.md): Import a document to the memory store. +_This project has moved to the [microsoft/chat-copilot](https://github.com/microsoft/chat-copilot) repository._ diff --git a/samples/apps/copilot-chat-app/deploy/README.md b/samples/apps/copilot-chat-app/deploy/README.md deleted file mode 100644 index 95753a2f1883..000000000000 --- a/samples/apps/copilot-chat-app/deploy/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# Deploying Copilot Chat -This document details how to deploy CopilotChat's required resources to your Azure subscription. - -## Things to know -- Access to Azure OpenAI is currently limited as we navigate high demand, upcoming product improvements, and Microsoft’s commitment to responsible AI. - For more details and information on applying for access, go [here](https://learn.microsoft.com/azure/cognitive-services/openai/overview?ocid=AID3051475#how-do-i-get-access-to-azure-openai). - For regional availability of Azure OpenAI, see the [availability map](https://azure.microsoft.com/explore/global-infrastructure/products-by-region/?products=cognitive-services). - -- With the limited availability of Azure OpenAI, consider sharing an Azure OpenAI instance across multiple resources. - -- `F1` and `D1` SKUs for the App Service Plans are not currently supported for this deployment in order to support private networking. - - -# Configure your environment -Before you get started, make sure you have the following requirements in place: -- Azure CLI (i.e., az) (if you already installed Azure CLI, make sure to update your installation to the latest version) - - Windows, go to https://aka.ms/installazurecliwindows - - Linux, run "`curl -L https://aka.ms/InstallAzureCli | bash`" -- Azure Static Web App CLI (i.e., swa) can be installed by running "`npm install -g @azure/static-web-apps-cli`" -- (Linux only) `zip` can be installed by running "`sudo apt install zip`" - - -# Deploy Azure Infrastructure -The examples below assume you are using an existing Azure OpenAI resource. See the notes following each command for using OpenAI or creating a new Azure OpenAI resource. - -## PowerShell -```powershell -./deploy-azure.ps1 -Subscription {YOUR_SUBSCRIPTION_ID} -DeploymentName {YOUR_DEPLOYMENT_NAME} -AIService {AzureOpenAI or OpenAI} -AIApiKey {YOUR_AI_KEY} -AIEndpoint {YOUR_AZURE_OPENAI_ENDPOINT} -``` - - To use an existing Azure OpenAI resource, set `-AIService` to `AzureOpenAI` and include `-AIApiKey` and `-AIEndpoint`. - - To deploy a new Azure OpenAI resource, set `-AIService` to `AzureOpenAI` and omit `-AIApiKey` and `-AIEndpoint`. - - To use an an OpenAI account, set `-AIService` to `OpenAI` and include `-AIApiKey`. - -## Bash -```bash -chmod +x ./deploy-azure.sh -./deploy-azure.sh --subscription {YOUR_SUBSCRIPTION_ID} --deployment-name {YOUR_DEPLOYMENT_NAME} --ai-service {AzureOpenAI or OpenAI} --ai-service-key {YOUR_AI_KEY} --ai-endpoint {YOUR_AZURE_OPENAI_ENDPOINT} -``` - - To use an existing Azure OpenAI resource, set `--ai-service` to `AzureOpenAI` and include `--ai-service-key` and `--ai-endpoint`. - - To deploy a new Azure OpenAI resource, set `--ai-service` to `AzureOpenAI` and omit `--ai-service-key` and `--ai-endpoint`. - - To use an an OpenAI account, set `--ai-service` to `OpenAI` and include `--ai-service-key`. - -## Azure Portal -You can also deploy the infrastructure directly from the Azure Portal by clicking the button below: - -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fmicrosoft%2Fsemantic-kernel%2Fmain%2Fsamples%2Fapps%2Fcopilot-chat-app%2Fdeploy%2Fmain.json) - -> This will automatically deploy the most recent release of CopilotChat backend binaries ([link](https://github.com/microsoft/semantic-kernel/releases?q=copilotchat)). - -> To find the deployment name when using `Deploy to Azure`, look for a deployment in your resource group that starts with `Microsoft.Template`. - - -# Deploy Backend (WebAPI) -To deploy the backend, build the deployment package first and deploy it to the Azure resources created above. - -## PowerShell -```powershell -./package-webapi.ps1 - -./deploy-webapi.ps1 -Subscription {YOUR_SUBSCRIPTION_ID} -ResourceGroupName rg-{YOUR_DEPLOYMENT_NAME} -DeploymentName {YOUR_DEPLOYMENT_NAME} -``` - -## Bash -```bash -chmod +x ./package-webapi.sh -./package-webapi.sh - -chmod +x ./deploy-webapi.sh -./deploy-webapi.sh --subscription {YOUR_SUBSCRIPTION_ID} --resource-group rg-{YOUR_DEPLOYMENT_NAME} --deployment-name {YOUR_DEPLOYMENT_NAME} -``` - - -# Deploy Frontend (WebApp) - -## Prerequisites -### App registration (identity) -You will need an Azure Active Directory (AAD) application registration. -> For details on creating an application registration, go [here](https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app). -- Select `Single-page application (SPA)` as platform type, and set the redirect URI to `http://localhost:3000` -- Select `Accounts in any organizational directory and personal Microsoft Accounts` as supported account types for this sample. -- Make a note of the `Application (client) ID` from the Azure Portal for use in the `Deploy` below. - -### Install Azure's Static Web Apps CLI -```bash -npm install -g @azure/static-web-apps-cli -``` - -## PowerShell - -```powershell - -./deploy-webapp.ps1 -Subscription {YOUR_SUBSCRIPTION_ID} -ResourceGroupName rg-{YOUR_DEPLOYMENT_NAME} -DeploymentName {YOUR_DEPLOYMENT_NAME} -ApplicationClientId {YOUR_APPLICATION_ID} -``` - -## Bash - -```bash -./deploy-webapp.sh --subscription {YOUR_SUBSCRIPTION_ID} --resource-group rg-{YOUR_DEPLOYMENT_NAME} --deployment-name {YOUR_DEPLOYMENT_NAME} --application-id {YOUR_APPLICATION_ID} -``` - -Your CopilotChat application is now deployed! - - -# Appendix -## Using custom web frontends to access your deployment -Make sure to include your frontend's URL as an allowed origin in your deployment's CORS settings. Otherwise, web browsers will refuse to let JavaScript make calls to your deployment. - -To do this, go on the Azure portal, select your Semantic Kernel App Service, then click on "CORS" under the "API" section of the resource menu on the left of the page. -This will get you to the CORS page where you can add your allowed hosts. - -## Authorization -All of endpoints (except `/healthz`) require authorization to access. -By default, an API key is required for access which can be found in the `Authorization:ApiKey` configuration setting. -To authorize requests with the API key, add the API key value to a `x-sk-api-key` header in your requests. - -To view your CopilotChat API key: -### PowerShell -```powershell -$webApiName = $(az deployment group show --name {DEPLOYMENT_NAME} --resource-group rg-{DEPLOYMENT_NAME} --output json | ConvertFrom-Json).properties.outputs.webapiName.value - -($(az webapp config appsettings list --name $webapiName --resource-group rg-{YOUR_DEPLOYMENT_NAME} | ConvertFrom-JSON) | Where-Object -Property name -EQ -Value Authorization:ApiKey).value -``` - -### Bash -```bash -eval WEB_API_NAME=$(az deployment group show --name $DEPLOYMENT_NAME --resource-group $RESOURCE_GROUP --output json) | jq -r '.properties.outputs.webapiName.value' - -$(az webapp config appsettings list --name $WEB_API_NAME --resource-group rg-{YOUR_DEPLOYMENT_NAME} | jq '.[] | select(.name=="Authorization:ApiKey").value') -``` - diff --git a/samples/apps/copilot-chat-app/deploy/deploy-azure.ps1 b/samples/apps/copilot-chat-app/deploy/deploy-azure.ps1 deleted file mode 100644 index 487f3619f45d..000000000000 --- a/samples/apps/copilot-chat-app/deploy/deploy-azure.ps1 +++ /dev/null @@ -1,139 +0,0 @@ -<# -.SYNOPSIS -Deploy CopilotChat Azure resources -#> - -param( - [Parameter(Mandatory)] - [string] - # Name for the deployment - $DeploymentName, - - [Parameter(Mandatory)] - [string] - # Subscription to which to make the deployment - $Subscription, - - [Parameter(Mandatory)] - [ValidateSet("AzureOpenAI","OpenAI")] - [string] - # AI service to use - $AIService, - - [string] - # API key for existing Azure OpenAI resource or OpenAI account - $AIApiKey, - - # Endpoint for existing Azure OpenAI resource - [string] - $AIEndpoint, - - [string] - # Resource group to which to make the deployment - $ResourceGroup, - - [string] - # Region to which to make the deployment (ignored when deploying to an existing resource group) - $Region = "centralus", - - [string] - # SKU for the Azure App Service plan - $WebAppServiceSku = "B1", - - [switch] - # Don't deploy Qdrant for memory storage - Use volatile memory instead - $NoQdrant, - - [switch] - # Don't deploy Cosmos DB for chat storage - Use volatile memory instead - $NoCosmosDb, - - [switch] - # Don't deploy Speech Services to enable speech as chat input - $NoSpeechServices, - - [switch] - # Switches on verbose template deployment output - $DebugDeployment -) - -# if AIService is AzureOpenAI -if ($AIService -eq "AzureOpenAI") { - # Both $AIEndpoint and $AIApiKey must be set - if ((!$AIEndpoint -and $AIApiKey) -or ($AIEndpoint -and !$AIApiKey)) { - Write-Error "When AIService is AzureOpenAI, when either AIEndpoint and AIApiKey are set then both must be set." - exit 1 - } - - # If both $AIEndpoint and $AIApiKey are not set, set $DeployAzureOpenAI to true and inform the user. Otherwise set $DeployAzureOpenAI to false and inform the user. - if (!$AIEndpoint -and !$AIApiKey) { - $DeployAzureOpenAI = $true - Write-Host "When AIService is AzureOpenAI and both AIEndpoint and AIApiKey are not set then a new Azure OpenAI resource will be created." - } - else { - $DeployAzureOpenAI = $false - Write-Host "When AIService is AzureOpenAI and both AIEndpoint and AIApiKey are set, use the existing Azure OpenAI resource." - } -} - -# if AIService is OpenAI then $AIApiKey is mandatory. -if ($AIService -eq "OpenAI" -and !$AIApiKey) { - Write-Error "When AIService is OpenAI, AIApiKey must be set." - exit 1 -} - -$jsonConfig = " -{ - `\`"name`\`": { `\`"value`\`": `\`"$DeploymentName`\`" }, - `\`"webAppServiceSku`\`": { `\`"value`\`": `\`"$WebAppServiceSku`\`" }, - `\`"aiService`\`": { `\`"value`\`": `\`"$AIService`\`" }, - `\`"aiApiKey`\`": { `\`"value`\`": `\`"$AIApiKey`\`" }, - `\`"aiEndpoint`\`": { `\`"value`\`": `\`"$AIEndpoint`\`" }, - `\`"deployNewAzureOpenAI`\`": { `\`"value`\`": $(If ($DeployAzureOpenAI) {"true"} Else {"false"}) }, - `\`"deployQdrant`\`": { `\`"value`\`": $(If (!($NoQdrant)) {"true"} Else {"false"}) }, - `\`"deployCosmosDB`\`": { `\`"value`\`": $(If (!($NoCosmosDb)) {"true"} Else {"false"}) }, - `\`"deploySpeechServices`\`": { `\`"value`\`": $(If (!($NoSpeechServices)) {"true"} Else {"false"}) } -} -" - -$jsonConfig = $jsonConfig -replace '\s','' - -$ErrorActionPreference = "Stop" - -$templateFile = "$($PSScriptRoot)/main.bicep" - -if (!$ResourceGroup) -{ - $ResourceGroup = "rg-" + $DeploymentName -} - -az account show --output none -if ($LASTEXITCODE -ne 0) { - Write-Host "Log into your Azure account" - az login --output none -} - -az account set -s $Subscription -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Ensuring resource group '$ResourceGroup' exists..." -az group create --location $Region --name $ResourceGroup --tags Creator=$env:UserName -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Validating template file..." -az deployment group validate --name $DeploymentName --resource-group $ResourceGroup --template-file $templateFile --parameters $jsonConfig -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Deploying Azure resources ($DeploymentName)..." -if ($DebugDeployment) { - az deployment group create --name $DeploymentName --resource-group $ResourceGroup --template-file $templateFile --debug --parameters $jsonConfig -} -else { - az deployment group create --name $DeploymentName --resource-group $ResourceGroup --template-file $templateFile --parameters $jsonConfig -} diff --git a/samples/apps/copilot-chat-app/deploy/deploy-azure.sh b/samples/apps/copilot-chat-app/deploy/deploy-azure.sh deleted file mode 100644 index 6e69991594c9..000000000000 --- a/samples/apps/copilot-chat-app/deploy/deploy-azure.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/bin/bash - -# Deploy CopilotChat Azure resources. - -set -e - -usage() { - echo "Usage: $0 -d DEPLOYMENT_NAME -s SUBSCRIPTION -ai AI_SERVICE_TYPE -aikey AI_SERVICE_KEY [OPTIONS]" - echo "" - echo "Arguments:" - echo " -d, --deployment-name DEPLOYMENT_NAME Name for the deployment (mandatory)" - echo " -s, --subscription SUBSCRIPTION Subscription to which to make the deployment (mandatory)" - echo " -ai, --ai-service AI_SERVICE_TYPE Type of AI service to use (i.e., OpenAI or AzureOpenAI)" - echo " -aikey, --ai-service-key AI_SERVICE_KEY API key for existing Azure OpenAI resource or OpenAI account" - echo " -aiend, --ai-endpoint AI_ENDPOINT Endpoint for existing Azure OpenAI resource" - echo " -rg, --resource-group RESOURCE_GROUP Resource group to which to make the deployment (default: \"rg-\$DEPLOYMENT_NAME\")" - echo " -r, --region REGION Region to which to make the deployment (default: \"South Central US\")" - echo " -a, --app-service-sku WEB_APP_SVC_SKU SKU for the Azure App Service plan (default: \"B1\")" - echo " -nq, --no-qdrant Don't deploy Qdrant for memory storage - Use volatile memory instead" - echo " -nc, --no-cosmos-db Don't deploy Cosmos DB for chat storage - Use volatile memory instead" - echo " -ns, --no-speech-services Don't deploy Speech Services to enable speech as chat input" - echo " -dd, --debug-deployment Switches on verbose template deployment output" -} - -# Parse arguments -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -d|--deployment-name) - DEPLOYMENT_NAME="$2" - shift - shift - ;; - -s|--subscription) - SUBSCRIPTION="$2" - shift - shift - ;; - -ai|--ai-service) - AI_SERVICE_TYPE="$2" - shift - shift - ;; - -aikey|--ai-service-key) - AI_SERVICE_KEY="$2" - shift - shift - ;; - -aiend|--ai-endpoint) - AI_ENDPOINT="$2" - shift - shift - ;; - -rg|--resource-group) - RESOURCE_GROUP="$2" - shift - shift - ;; - -r|--region) - REGION="$2" - shift - shift - ;; - -a|--app-service-sku) - WEB_APP_SVC_SKU="$2" - shift - shift - ;; - -nq|--no-qdrant) - NO_QDRANT=true - shift - ;; - -nc|--no-cosmos-db) - NO_COSMOS_DB=true - shift - ;; - -ns|--no-speech-services) - NO_SPEECH_SERVICES=true - shift - ;; - -dd|--debug-deployment) - DEBUG_DEPLOYMENT=true - shift - ;; - *) - echo "Unknown option $1" - usage - exit 1 - ;; - esac -done - -# Check mandatory arguments -if [[ -z "$DEPLOYMENT_NAME" ]] || [[ -z "$SUBSCRIPTION" ]] || [[ -z "$AI_SERVICE_TYPE" ]]; then - usage - exit 1 -fi - -# Check if AI_SERVICE_TYPE is either OpenAI or AzureOpenAI -if [[ "${AI_SERVICE_TYPE,,}" != "openai" ]] && [[ "${AI_SERVICE_TYPE,,}" != "azureopenai" ]]; then - echo "--ai-service must be either OpenAI or AzureOpenAI" - usage - exit 1 -fi - -# if AI_SERVICE_TYPE is AzureOpenAI -if [[ "${AI_SERVICE_TYPE,,}" = "azureopenai" ]]; then - # Both AI_ENDPOINT and AI_SERVICE_KEY must be set or neither of them. - if [[ (-z "$AI_ENDPOINT" && -n "$AI_SERVICE_KEY") || (-n "$AI_ENDPOINT" && -z "$AI_SERVICE_KEY") ]]; then - echo "When --ai is 'AzureOpenAI', if either --ai-endpoint or --ai-service-key is set, then both must be set." - usage - exit 1 - fi - - # if AI_ENDPOINT and AI_SERVICE_KEY are not set, set NO_NEW_AZURE_OPENAI to false and tell the user, else set NO_NEW_AZURE_OPENAI to true - if [[ -z "$AI_ENDPOINT" ]] && [[ -z "$AI_SERVICE_KEY" ]]; then - NO_NEW_AZURE_OPENAI=false - echo "When --ai is 'AzureOpenAI', if neither --ai-endpoint nor --ai-service-key are set, then a new Azure OpenAI resource will be created." - else - NO_NEW_AZURE_OPENAI=true - echo "When --ai is 'AzureOpenAI', if both --ai-endpoint and --ai-service-key are set, then an existing Azure OpenAI resource will be used." - fi -fi - -# if AI_SERVICE_TYPE is OpenAI then AI_SERVICE_KEY is mandatory -if [[ "${AI_SERVICE_TYPE,,}" = "openai" ]] && [[ -z "$AI_SERVICE_KEY" ]]; then - echo "When --ai is 'OpenAI', --ai-service-key must be set." - usage - exit 1 -fi - -# If resource group is not set, then set it to rg-DEPLOYMENT_NAME -if [ -z "$RESOURCE_GROUP" ]; then - RESOURCE_GROUP="rg-${DEPLOYMENT_NAME}" -fi - -TEMPLATE_FILE="$(dirname "$0")/main.bicep" - -az account show --output none -if [ $? -ne 0 ]; then - echo "Log into your Azure account" - az login --use-device-code -fi - -az account set -s "$SUBSCRIPTION" - -# Set defaults -: "${REGION:="centralus"}" -: "${WEB_APP_SVC_SKU:="B1"}" -: "${NO_QDRANT:=false}" -: "${NO_COSMOS_DB:=false}" -: "${NO_SPEECH_SERVICES:=false}" - -# Create JSON config -JSON_CONFIG=$(cat << EOF -{ - "name": { "value": "$DEPLOYMENT_NAME" }, - "webAppServiceSku": { "value": "$WEB_APP_SVC_SKU" }, - "aiService": { "value": "$AI_SERVICE_TYPE" }, - "aiApiKey": { "value": "$AI_SERVICE_KEY" }, - "aiEndpoint": { "value": "$([ -z "$AI_ENDPOINT" ] && echo "$AI_ENDPOINT")" }, - "deployNewAzureOpenAI": { "value": $([ "$NO_NEW_AZURE_OPENAI" = true ] && echo "false" || echo "true") }, - "deployQdrant": { "value": $([ "$NO_QDRANT" = true ] && echo "false" || echo "true") }, - "deployCosmosDB": { "value": $([ "$NO_COSMOS_DB" = true ] && echo "false" || echo "true") }, - "deploySpeechServices": { "value": $([ "$NO_SPEECH_SERVICES" = true ] && echo "false" || echo "true") } -} -EOF -) - -echo "Ensuring resource group $RESOURCE_GROUP..." -az group create --location "$REGION" --name "$RESOURCE_GROUP" --tags Creator="$USER" - -echo "Validating template file..." -az deployment group validate --name "$DEPLOYMENT_NAME" --resource-group "$RESOURCE_GROUP" --template-file "$TEMPLATE_FILE" --parameters "$JSON_CONFIG" - -echo "Deploying Azure resources ($DEPLOYMENT_NAME)..." -if [ "$DEBUG_DEPLOYMENT" = true ]; then - az deployment group create --name "$DEPLOYMENT_NAME" --resource-group "$RESOURCE_GROUP" --template-file "$TEMPLATE_FILE" --debug --parameters "$JSON_CONFIG" -else - az deployment group create --name "$DEPLOYMENT_NAME" --resource-group "$RESOURCE_GROUP" --template-file "$TEMPLATE_FILE" --parameters "$JSON_CONFIG" -fi diff --git a/samples/apps/copilot-chat-app/deploy/deploy-webapi.ps1 b/samples/apps/copilot-chat-app/deploy/deploy-webapi.ps1 deleted file mode 100644 index 4712c2a99ae1..000000000000 --- a/samples/apps/copilot-chat-app/deploy/deploy-webapi.ps1 +++ /dev/null @@ -1,63 +0,0 @@ -<# -.SYNOPSIS -Deploy CopilotChat's WebAPI to Azure -#> - -param( - [Parameter(Mandatory)] - [string] - # Subscription to which to make the deployment - $Subscription, - - [Parameter(Mandatory)] - [string] - # Resource group to which to make the deployment - $ResourceGroupName, - - [Parameter(Mandatory)] - [string] - # Name of the previously deployed Azure deployment - $DeploymentName, - - [string] - # CopilotChat WebApi package to deploy - $PackageFilePath = "$PSScriptRoot/out/webapi.zip" -) - -# Ensure $PackageFilePath exists -if (!(Test-Path $PackageFilePath)) { - Write-Error "Package file '$PackageFilePath' does not exist. Have you run 'package-webapi.ps1' yet?" - exit 1 -} - -az account show --output none -if ($LASTEXITCODE -ne 0) { - Write-Host "Log into your Azure account" - az login --output none -} - -az account set -s $Subscription -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Getting Azure WebApp resource name..." -$webappName=$(az deployment group show --name $DeploymentName --resource-group $ResourceGroupName --output json | ConvertFrom-Json).properties.outputs.webapiName.value -if ($null -eq $webAppName) { - Write-Error "Could not get Azure WebApp resource name from deployment output." - exit 1 -} - -Write-Host "Azure WebApp name: $webappName" - -Write-Host "Configuring Azure WebApp to run from package..." -az webapp config appsettings set --resource-group $ResourceGroupName --name $webappName --settings WEBSITE_RUN_FROM_PACKAGE="1" | out-null -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Deploying '$PackageFilePath' to Azure WebApp '$webappName'..." -az webapp deployment source config-zip --resource-group $ResourceGroupName --name $webappName --src $PackageFilePath -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/deploy/deploy-webapi.sh b/samples/apps/copilot-chat-app/deploy/deploy-webapi.sh deleted file mode 100644 index 6e0ea511e588..000000000000 --- a/samples/apps/copilot-chat-app/deploy/deploy-webapi.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -# Deploy CopilotChat's WebAPI to Azure. - -set -e - -usage() { - echo "Usage: $0 -d DEPLOYMENT_NAME -s SUBSCRIPTION -rg RESOURCE_GROUP [OPTIONS]" - echo "" - echo "Arguments:" - echo " -d, --deployment-name DEPLOYMENT_NAME Name of the deployment from a 'deploy-azure.sh' deployment (mandatory)" - echo " -s, --subscription SUBSCRIPTION Subscription to which to make the deployment (mandatory)" - echo " -rg, --resource-group RESOURCE_GROUP Resource group name from a 'deploy-azure.sh' deployment (mandatory)" - echo " -p, --package PACKAGE_FILE_PATH Path to the WebAPI package file from a 'package-webapi.sh' run" -} - -# Parse arguments -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -d|--deployment-name) - DEPLOYMENT_NAME="$2" - shift - shift - ;; - -s|--subscription) - SUBSCRIPTION="$2" - shift - shift - ;; - -rg|--resource-group) - RESOURCE_GROUP="$2" - shift - shift - ;; - -p|--package) - PACKAGE_FILE_PATH="$2" - shift - shift - ;; - *) - echo "Unknown option $1" - usage - exit 1 - ;; - esac -done - -# Check mandatory arguments -if [[ -z "$DEPLOYMENT_NAME" ]] || [[ -z "$SUBSCRIPTION" ]] || [[ -z "$RESOURCE_GROUP" ]]; then - usage - exit 1 -fi - -# Set defaults -: "${PACKAGE_FILE_PATH:="$(dirname "$0")/out/webapi.zip"}" - -# Ensure $PACKAGE_FILE_PATH exists -if [[ ! -f "$PACKAGE_FILE_PATH" ]]; then - echo "Package file '$PACKAGE_FILE_PATH' does not exist. Have you run 'package-webapi.sh' yet?" - exit 1 -fi - -az account show --output none -if [ $? -ne 0 ]; then - echo "Log into your Azure account" - az login --use-device-code -fi - -az account set -s "$SUBSCRIPTION" - -echo "Getting Azure WebApp resource name..." -eval WEB_APP_NAME=$(az deployment group show --name $DEPLOYMENT_NAME --resource-group $RESOURCE_GROUP --output json | jq '.properties.outputs.webapiName.value') -# Ensure $WEB_APP_NAME is set -if [[ -z "$WEB_APP_NAME" ]]; then - echo "Could not get Azure WebApp resource name from deployment output." - exit 1 -fi - -echo "Azure WebApp name: $webappName" - -echo "Configuring Azure WebApp to run from package..." -az webapp config appsettings set --resource-group $RESOURCE_GROUP --name $WEB_APP_NAME --settings WEBSITE_RUN_FROM_PACKAGE="1" -if [ $? -ne 0 ]; then - echo "Could not configure Azure WebApp to run from package." - exit 1 -fi - -echo "Deploying '$PackageFilePath' to Azure WebApp '$webappName'..." -az webapp deployment source config-zip --resource-group $RESOURCE_GROUP --name $WEB_APP_NAME --src $PACKAGE_FILE_PATH -if [ $? -ne 0 ]; then - echo "Could not deploy '$PackageFilePath' to Azure WebApp '$webappName'." - exit 1 -fi - -eval WEB_APP_URL=$(az deployment group show --name $DEPLOYMENT_NAME --resource-group $RESOURCE_GROUP --output json | jq '.properties.outputs.webapiUrl.value') -echo "To verify your deployment, go to 'https://$WEB_APP_URL/healthz' in your browser." diff --git a/samples/apps/copilot-chat-app/deploy/deploy-webapp.ps1 b/samples/apps/copilot-chat-app/deploy/deploy-webapp.ps1 deleted file mode 100644 index e9997946bdcb..000000000000 --- a/samples/apps/copilot-chat-app/deploy/deploy-webapp.ps1 +++ /dev/null @@ -1,122 +0,0 @@ -<# -.SYNOPSIS -Deploy CopilotChat's WebApp to Azure -#> - -param( - [Parameter(Mandatory)] - [string] - # Subscription to which to make the deployment - $Subscription, - - [Parameter(Mandatory)] - [string] - # Resource group to which to make the deployment - $ResourceGroupName, - - [Parameter(Mandatory)] - [string] - # Name of the previously deployed Azure deployment - $DeploymentName, - - [Parameter(Mandatory)] - [string] - # Client application id - $ApplicationClientId -) - -Write-Host "Setting up Azure credentials..." -az account show --output none -if ($LASTEXITCODE -ne 0) { - Write-Host "Log into your Azure account" - az login --output none -} - -Write-Host "Setting subscription to '$Subscription'..." -az account set -s $Subscription -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Getting deployment outputs..." -$deployment = $(az deployment group show --name $DeploymentName --resource-group $ResourceGroupName --output json | ConvertFrom-Json) -$webappUrl = $deployment.properties.outputs.webappUrl.value -$webappName = $deployment.properties.outputs.webappName.value -$webapiUrl = $deployment.properties.outputs.webapiUrl.value -$webapiName = $deployment.properties.outputs.webapiName.value -$webapiApiKey = ($(az webapp config appsettings list --name $webapiName --resource-group $ResourceGroupName | ConvertFrom-JSON) | Where-Object -Property name -EQ -Value Authorization:ApiKey).value -Write-Host "webappUrl: $webappUrl" -Write-Host "webappName: $webappName" -Write-Host "webapiName: $webapiName" -Write-Host "webapiUrl: $webapiUrl" - -# Set UTF8 as default encoding for Out-File -$PSDefaultParameterValues['Out-File:Encoding'] = 'ascii' - -$envFilePath = "$PSScriptRoot/../webapp/.env" -Write-Host "Writing environment variables to '$envFilePath'..." -"REACT_APP_BACKEND_URI=https://$webapiUrl/" | Out-File -FilePath $envFilePath -"REACT_APP_AAD_AUTHORITY=https://login.microsoftonline.com/common" | Out-File -FilePath $envFilePath -Append -"REACT_APP_AAD_CLIENT_ID=$ApplicationClientId" | Out-File -FilePath $envFilePath -Append -"REACT_APP_SK_API_KEY=$webapiApiKey" | Out-File -FilePath $envFilePath -Append - -Write-Host "Generating SWA config..." -$swaConfig = $(Get-Content "$PSScriptRoot/../webapp/template.swa-cli.config.json" -Raw) -$swaConfig = $swaConfig.Replace("{{appDevserverUrl}}", "https://$webappUrl") -$swaConfig = $swaConfig.Replace("{{appName}}", "$webappName") -$swaConfig = $swaConfig.Replace("{{resourceGroup}}", "$ResourceGroupName") -$swaConfig = $swaConfig.Replace("{{subscription-id}}", "$Subscription") - -$swaConfig | Out-File -FilePath "$PSScriptRoot/../webapp/swa-cli.config.json" -Write-Host $(Get-Content "$PSScriptRoot/../webapp/swa-cli.config.json" -Raw) - -Push-Location -Path "$PSScriptRoot/../webapp" -Write-Host "Installing yarn dependencies..." -yarn install -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Building webapp..." -swa build -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Deploying webapp..." -swa deploy -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -$origin = "https://$webappUrl" -Write-Host "Ensuring CORS origin '$origin' to webapi '$webapiName'..." -if (-not ((az webapp cors show --name $webapiName --resource-group $ResourceGroupName --subscription $Subscription | ConvertFrom-Json).allowedOrigins -contains $origin)) { - az webapp cors add --name $webapiName --resource-group $ResourceGroupName --subscription $Subscription --allowed-origins $origin -} - -Write-Host "Ensuring '$origin' is included in AAD app registration's redirect URIs..." -$objectId = (az ad app show --id $ApplicationClientId | ConvertFrom-Json).id -$redirectUris = (az rest --method GET --uri "https://graph.microsoft.com/v1.0/applications/$objectId" --headers 'Content-Type=application/json' | ConvertFrom-Json).spa.redirectUris -if ($redirectUris -notcontains "$origin") { - $redirectUris += "$origin" - - $body = "{spa:{redirectUris:[" - foreach ($uri in $redirectUris) { - $body += "'$uri'," - } - $body += "]}}" - - az rest ` - --method PATCH ` - --uri "https://graph.microsoft.com/v1.0/applications/$objectId" ` - --headers 'Content-Type=application/json' ` - --body $body -} -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Pop-Location - -Write-Host "To verify your deployment, go to 'https://$webappUrl' in your browser." diff --git a/samples/apps/copilot-chat-app/deploy/deploy-webapp.sh b/samples/apps/copilot-chat-app/deploy/deploy-webapp.sh deleted file mode 100644 index 13b49c6e46b4..000000000000 --- a/samples/apps/copilot-chat-app/deploy/deploy-webapp.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash - -# Deploy CopilotChat's WebApp to Azure - -set -e - -SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -usage() { - echo "Usage: $0 -d DEPLOYMENT_NAME -s SUBSCRIPTION --ai AI_SERVICE_TYPE -aikey AI_SERVICE_KEY [OPTIONS]" - echo "" - echo "Arguments:" - echo " -s, --subscription SUBSCRIPTION Subscription to which to make the deployment (mandatory)" - echo " -rg, --resource-group RESOURCE_GROUP Resource group name from a 'deploy-azure.sh' deployment (mandatory)" - echo " -d, --deployment-name DEPLOYMENT_NAME Name of the deployment from a 'deploy-azure.sh' deployment (mandatory)" - echo " -a, --application-id Client application ID (mandatory)" -} - -# Parse arguments -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -d|--deployment-name) - DEPLOYMENT_NAME="$2" - shift - shift - ;; - -s|--subscription) - SUBSCRIPTION="$2" - shift - shift - ;; - -rg|--resource-group) - RESOURCE_GROUP="$2" - shift - shift - ;; - -a|--application-id) - APPLICATION_ID="$2" - shift - shift - ;; - *) - echo "Unknown option $1" - usage - exit 1 - ;; - esac -done - -# Check mandatory arguments -if [[ -z "$DEPLOYMENT_NAME" ]] || [[ -z "$SUBSCRIPTION" ]] || [[ -z "$RESOURCE_GROUP" ]] || [[ -z "$APPLICATION_ID" ]]; then - usage - exit 1 -fi - -az account show --output none -if [ $? -ne 0 ]; then - echo "Log into your Azure account" - az login --use-device-code -fi - -az account set -s "$SUBSCRIPTION" - -echo "Getting deployment outputs..." -DEPLOYMENT_JSON=$(az deployment group show --name $DEPLOYMENT_NAME --resource-group $RESOURCE_GROUP --output json) -# get the webapiUrl from the deployment outputs -eval WEB_APP_URL=$(echo $DEPLOYMENT_JSON | jq -r '.properties.outputs.webappUrl.value') -echo "WEB_APP_URL: $WEB_APP_URL" -eval WEB_APP_NAME=$(echo $DEPLOYMENT_JSON | jq -r '.properties.outputs.webappName.value') -echo "WEB_APP_NAME: $WEB_APP_NAME" -eval WEB_API_URL=$(echo $DEPLOYMENT_JSON | jq -r '.properties.outputs.webapiUrl.value') -echo "WEB_API_URL: $WEB_API_URL" -eval WEB_API_NAME=$(echo $DEPLOYMENT_JSON | jq -r '.properties.outputs.webapiName.value') -echo "WEB_API_NAME: $WEB_API_NAME" -echo "Getting webapi key..." -eval WEB_API_KEY=$(az webapp config appsettings list --name $WEB_API_NAME --resource-group $RESOURCE_GROUP | jq '.[] | select(.name=="Authorization:ApiKey").value') - -ENV_FILE_PATH="$SCRIPT_ROOT/../webapp/.env" -echo "Writing environment variables to '$ENV_FILE_PATH'..." -echo "REACT_APP_BACKEND_URI=https://$WEB_API_URL/" > $ENV_FILE_PATH -echo "REACT_APP_AAD_AUTHORITY=https://login.microsoftonline.com/common" >> $ENV_FILE_PATH -echo "REACT_APP_AAD_CLIENT_ID=$APPLICATION_ID" >> $ENV_FILE_PATH -echo "REACT_APP_SK_API_KEY=$WEB_API_KEY" >> $ENV_FILE_PATH - -echo "Writing swa-cli.config.json..." -SWA_CONFIG_FILE_PATH="$SCRIPT_ROOT/../webapp/swa-cli.config.json" -sed "s/{{appDevserverUrl}}/https:\/\/${WEB_APP_URL}/g" $SCRIPT_ROOT/../webapp/template.swa-cli.config.json > $SWA_CONFIG_FILE_PATH -cat $SWA_CONFIG_FILE_PATH - -pushd "$SCRIPT_ROOT/../webapp" - -echo "Installing yarn dependencies..." -yarn install -if [ $? -ne 0 ]; then - echo "Failed to install yarn dependencies" - exit 1 -fi - -echo "Building webapp..." -swa build -if [ $? -ne 0 ]; then - echo "Failed to build webapp" - exit 1 -fi - -echo "Deploying webapp..." -swa deploy --subscription-id $SUBSCRIPTION --app-name $WEB_APP_NAME --env production -if [ $? -ne 0 ]; then - echo "Failed to deploy webapp" - exit 1 -fi - -ORIGIN="https://$WEB_APP_URL" -echo "Ensuring CORS origin '$ORIGIN' to webapi '$WEB_API_NAME'..." -CORS_RESULT=$(az webapp cors show --name $WEB_API_NAME --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION | jq '.allowedOrigins | index("$ORIGIN")') -if [[ "$CORS_RESULT" == "null" ]]; then - echo "Adding CORS origin '$ORIGIN' to webapi '$WEB_API_NAME'..." - az webapp cors add --name $WEB_API_NAME --resource-group $RESOURCE_GROUP --subscription $SUBSCRIPTION --allowed-origins $ORIGIN -fi - -echo "Ensuring '$ORIGIN' is included in AAD app registration's redirect URIs..." -eval OBJECT_ID=$(az ad app show --id $APPLICATION_ID | jq -r '.id') - -REDIRECT_URIS=$(az rest --method GET --uri "https://graph.microsoft.com/v1.0/applications/$OBJECT_ID" --headers 'Content-Type=application/json' | jq -r '.spa.redirectUris') -if [[ ! "$REDIRECT_URIS" =~ "$ORIGIN" ]]; then - BODY="{spa:{redirectUris:['" - eval BODY+=$(echo $REDIRECT_URIS | jq $'join("\',\'")') - BODY+="','$ORIGIN']}}" - - az rest \ - --method PATCH \ - --uri "https://graph.microsoft.com/v1.0/applications/$OBJECT_ID" \ - --headers 'Content-Type=application/json' \ - --body $BODY -fi -if [ $? -ne 0 ]; then - echo "Failed to update app registration" - exit 1 -fi - -popd - -echo "To verify your deployment, go to 'https://$WEB_APP_URL' in your browser." \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/deploy/main.bicep b/samples/apps/copilot-chat-app/deploy/main.bicep deleted file mode 100644 index f60caeaff552..000000000000 --- a/samples/apps/copilot-chat-app/deploy/main.bicep +++ /dev/null @@ -1,711 +0,0 @@ -/* -Copyright (c) Microsoft. All rights reserved. -Licensed under the MIT license. See LICENSE file in the project root for full license information. - -Bicep template for deploying CopilotChat Azure resources. -*/ - -@description('Name for the deployment consisting of alphanumeric characters or dashes (\'-\')') -param name string = 'copichat' - -@description('SKU for the Azure App Service plan') -@allowed(['B1', 'S1', 'S2', 'S3', 'P1V3', 'P2V3', 'I1V2', 'I2V2' ]) -param webAppServiceSku string = 'B1' - -@description('Location of package to deploy as the web service') -#disable-next-line no-hardcoded-env-urls -param packageUri string = 'https://aka.ms/copilotchat/webapi/latest' - -@description('Underlying AI service') -@allowed([ - 'AzureOpenAI' - 'OpenAI' -]) -param aiService string = 'AzureOpenAI' - -@description('Model to use for chat completions') -param completionModel string = 'gpt-35-turbo' - -@description('Model to use for text embeddings') -param embeddingModel string = 'text-embedding-ada-002' - -@description('Completion model the task planner should use') -param plannerModel string = 'gpt-35-turbo' - -@description('Azure OpenAI endpoint to use (Azure OpenAI only)') -param aiEndpoint string = '' - -@secure() -@description('Azure OpenAI or OpenAI API key') -param aiApiKey string = '' - -@secure() -@description('WebAPI key to use for authorization') -param webApiKey string = newGuid() - -@description('Whether to deploy a new Azure OpenAI instance') -param deployNewAzureOpenAI bool = false - -@description('Whether to deploy Cosmos DB for persistent chat storage') -param deployCosmosDB bool = true - -@description('Whether to deploy Qdrant (in a container) for persistent memory storage') -param deployQdrant bool = true - -@description('Whether to deploy Azure Speech Services to enable input by voice') -param deploySpeechServices bool = true - -@description('Region for the resources') -param location string = resourceGroup().location - -@description('Region for the webapp frontend') -param webappLocation string = 'westus2' - -@description('Hash of the resource group ID') -var rgIdHash = uniqueString(resourceGroup().id) - -@description('Deployment name unique to resource group') -var uniqueName = '${name}-${rgIdHash}' - -@description('Name of the Azure Storage file share to create') -var storageFileShareName = 'aciqdrantshare' - - -resource openAI 'Microsoft.CognitiveServices/accounts@2022-12-01' = if(deployNewAzureOpenAI) { - name: 'ai-${uniqueName}' - location: location - kind: 'OpenAI' - sku: { - name: 'S0' - } - properties: { - customSubDomainName: toLower(uniqueName) - } -} - -resource openAI_completionModel 'Microsoft.CognitiveServices/accounts/deployments@2022-12-01' = if(deployNewAzureOpenAI) { - parent: openAI - name: completionModel - properties: { - model: { - format: 'OpenAI' - name: completionModel - } - scaleSettings: { - scaleType: 'Standard' - } - } -} - -resource openAI_embeddingModel 'Microsoft.CognitiveServices/accounts/deployments@2022-12-01' = if(deployNewAzureOpenAI) { - parent: openAI - name: embeddingModel - properties: { - model: { - format: 'OpenAI' - name: embeddingModel - } - scaleSettings: { - scaleType: 'Standard' - } - } - dependsOn: [ // This "dependency" is to create models sequentially because the resource - openAI_completionModel // provider does not support parallel creation of models properly. - ] -} - -resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' = { - name: 'asp-${uniqueName}-webapi' - location: location - kind: 'app' - sku: { - name: webAppServiceSku - } -} - -resource appServiceWeb 'Microsoft.Web/sites@2022-09-01' = { - name: 'app-${uniqueName}-webapi' - location: location - kind: 'app' - tags: { - skweb: '1' - } - properties: { - serverFarmId: appServicePlan.id - httpsOnly: true - virtualNetworkSubnetId: virtualNetwork.properties.subnets[0].id - siteConfig: { - healthCheckPath: '/healthz' - } - } -} - -resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = { - parent: appServiceWeb - name: 'web' - properties: { - alwaysOn: false - cors: { - allowedOrigins: [ - 'http://localhost:3000' - 'https://localhost:3000' - ] - supportCredentials: true - } - detailedErrorLoggingEnabled: true - minTlsVersion: '1.2' - netFrameworkVersion: 'v6.0' - use32BitWorkerProcess: false - vnetRouteAllEnabled: true - webSocketsEnabled: true - appSettings: [ - { - name: 'AIService:Type' - value: aiService - } - { - name: 'AIService:Endpoint' - value: deployNewAzureOpenAI ? openAI.properties.endpoint : aiEndpoint - } - { - name: 'AIService:Key' - value: deployNewAzureOpenAI ? openAI.listKeys().key1 : aiApiKey - } - { - name: 'AIService:Models:Completion' - value: completionModel - } - { - name: 'AIService:Models:Embedding' - value: embeddingModel - } - { - name: 'AIService:Models:Planner' - value: plannerModel - } - { - name: 'Authorization:Type' - value: empty(webApiKey) ? 'None' : 'ApiKey' - } - { - name: 'Authorization:ApiKey' - value: webApiKey - } - { - name: 'ChatStore:Type' - value: deployCosmosDB ? 'cosmos' : 'volatile' - } - { - name: 'ChatStore:Cosmos:Database' - value: 'CopilotChat' - } - { - name: 'ChatStore:Cosmos:ChatSessionsContainer' - value: 'chatsessions' - } - { - name: 'ChatStore:Cosmos:ChatMessagesContainer' - value: 'chatmessages' - } - { - name: 'ChatStore:Cosmos:ChatMemorySourcesContainer' - value: 'chatmemorysources' - } - { - name: 'ChatStore:Cosmos:ChatParticipantsContainer' - value: 'chatparticipants' - } - { - name: 'ChatStore:Cosmos:ConnectionString' - value: deployCosmosDB ? cosmosAccount.listConnectionStrings().connectionStrings[0].connectionString : '' - } - { - name: 'MemoriesStore:Type' - value: deployQdrant ? 'Qdrant' : 'Volatile' - } - { - name: 'MemoriesStore:Qdrant:Host' - value: deployQdrant ? 'https://${appServiceQdrant.properties.defaultHostName}' : '' - } - { - name: 'MemoriesStore:Qdrant:Port' - value: '443' - } - { - name: 'AzureSpeech:Region' - value: location - } - { - name: 'AzureSpeech:Key' - value: deploySpeechServices ? speechAccount.listKeys().key1 : '' - } - { - name: 'AllowedOrigins' - value: '[*]' // Defer list of allowed origins to the Azure service app's CORS configuration - } - { - name: 'Kestrel:Endpoints:Https:Url' - value: 'https://localhost:443' - } - { - name: 'Logging:LogLevel:Default' - value: 'Warning' - } - { - name: 'Logging:LogLevel:SemanticKernel.Service' - value: 'Warning' - } - { - name: 'Logging:LogLevel:Microsoft.SemanticKernel' - value: 'Warning' - } - { - name: 'Logging:LogLevel:Microsoft.AspNetCore.Hosting' - value: 'Warning' - } - { - name: 'Logging:LogLevel:Microsoft.Hosting.Lifetimel' - value: 'Warning' - } - { - name: 'ApplicationInsights:ConnectionString' - value: appInsights.properties.ConnectionString - } - { - name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' - value: appInsights.properties.ConnectionString - } - { - name: 'ApplicationInsightsAgent_EXTENSION_VERSION' - value: '~2' - } - ] - } -} - -resource appServiceWebDeploy 'Microsoft.Web/sites/extensions@2022-09-01' = { - name: 'MSDeploy' - kind: 'string' - parent: appServiceWeb - properties: { - packageUri: packageUri - } - dependsOn: [ - appServiceWebConfig - ] -} - -resource appInsights 'Microsoft.Insights/components@2020-02-02' = { - name: 'appins-${uniqueName}' - location: location - kind: 'string' - tags: { - displayName: 'AppInsight' - } - properties: { - Application_Type: 'web' - WorkspaceResourceId: logAnalyticsWorkspace.id - } -} - -resource appInsightExtension 'Microsoft.Web/sites/siteextensions@2022-09-01' = { - parent: appServiceWeb - name: 'Microsoft.ApplicationInsights.AzureWebSites' - dependsOn: [ - appServiceWebDeploy - ] -} - -resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { - name: 'la-${uniqueName}' - location: location - tags: { - displayName: 'Log Analytics' - } - properties: { - sku: { - name: 'PerGB2018' - } - retentionInDays: 90 - features: { - searchVersion: 1 - legacy: 0 - enableLogAccessUsingOnlyResourcePermissions: true - } - } -} - -resource storage 'Microsoft.Storage/storageAccounts@2022-09-01' = if (deployQdrant) { - name: 'st${rgIdHash}' // Not using full unique name to avoid hitting 24 char limit - location: location - kind: 'StorageV2' - sku: { - name: 'Standard_LRS' - } - properties: { - supportsHttpsTrafficOnly: true - allowBlobPublicAccess: false - } - resource fileservices 'fileServices' = { - name: 'default' - resource share 'shares' = { - name: storageFileShareName - } - } -} - -resource appServicePlanQdrant 'Microsoft.Web/serverfarms@2022-03-01' = if (deployQdrant) { - name: 'asp-${uniqueName}-qdrant' - location: location - kind: 'linux' - sku: { - name: 'P1v3' - } - properties: { - reserved: true - } -} - -resource appServiceQdrant 'Microsoft.Web/sites@2022-09-01' = if (deployQdrant) { - name: 'app-${uniqueName}-qdrant' - location: location - kind: 'app,linux,container' - properties: { - serverFarmId: appServicePlanQdrant.id - httpsOnly: true - reserved: true - clientCertMode: 'Required' - virtualNetworkSubnetId: virtualNetwork.properties.subnets[1].id - siteConfig: { - numberOfWorkers: 1 - linuxFxVersion: 'DOCKER|qdrant/qdrant:latest' - alwaysOn: true - vnetRouteAllEnabled: true - ipSecurityRestrictions: [ - { - vnetSubnetResourceId: virtualNetwork.properties.subnets[0].id - action: 'Allow' - priority: 300 - name: 'Allow front vnet' - } - { - ipAddress: 'Any' - action: 'Deny' - priority: 2147483647 - name: 'Deny all' - } - ] - azureStorageAccounts: { - aciqdrantshare: { - type: 'AzureFiles' - accountName: deployQdrant ? storage.name : 'notdeployed' - shareName: storageFileShareName - mountPath: '/qdrant/storage' - accessKey: deployQdrant ? storage.listKeys().keys[0].value : '' - } - } - } - } -} - -resource virtualNetwork 'Microsoft.Network/virtualNetworks@2021-05-01' = { - name: 'vnet-${uniqueName}' - location: location - properties: { - addressSpace: { - addressPrefixes: [ - '10.0.0.0/16' - ] - } - subnets: [ - { - name: 'webSubnet' - properties: { - addressPrefix: '10.0.1.0/24' - networkSecurityGroup: { - id: webNsg.id - } - serviceEndpoints: [ - { - service: 'Microsoft.Web' - locations: [ - '*' - ] - } - ] - delegations: [ - { - name: 'delegation' - properties: { - serviceName: 'Microsoft.Web/serverfarms' - } - } - ] - privateEndpointNetworkPolicies: 'Disabled' - privateLinkServiceNetworkPolicies: 'Enabled' - } - } - { - name: 'qdrantSubnet' - properties: { - addressPrefix: '10.0.2.0/24' - networkSecurityGroup: { - id: qdrantNsg.id - } - serviceEndpoints: [ - { - service: 'Microsoft.Web' - locations: [ - '*' - ] - } - ] - delegations: [ - { - name: 'delegation' - properties: { - serviceName: 'Microsoft.Web/serverfarms' - } - } - ] - privateEndpointNetworkPolicies: 'Disabled' - privateLinkServiceNetworkPolicies: 'Enabled' - } - } - ] - } -} - -resource webNsg 'Microsoft.Network/networkSecurityGroups@2022-11-01' = { - name: 'nsg-${uniqueName}-webapi' - location: location - properties: { - securityRules: [ - { - name: 'AllowAnyHTTPSInbound' - properties: { - protocol: 'TCP' - sourcePortRange: '*' - destinationPortRange: '443' - sourceAddressPrefix: '*' - destinationAddressPrefix: '*' - access: 'Allow' - priority: 100 - direction: 'Inbound' - } - } - ] - } -} - -resource qdrantNsg 'Microsoft.Network/networkSecurityGroups@2022-11-01' = { - name: 'nsg-${uniqueName}-qdrant' - location: location - properties: { - securityRules: [] - } -} - -resource webSubnetConnection 'Microsoft.Web/sites/virtualNetworkConnections@2022-09-01' = { - parent: appServiceWeb - name: 'webSubnetConnection' - properties: { - vnetResourceId: virtualNetwork.properties.subnets[0].id - isSwift: true - } -} - -resource qdrantSubnetConnection 'Microsoft.Web/sites/virtualNetworkConnections@2022-09-01' = if (deployQdrant) { - parent: appServiceQdrant - name: 'qdrantSubnetConnection' - properties: { - vnetResourceId: virtualNetwork.properties.subnets[1].id - isSwift: true - } -} - -resource cosmosAccount 'Microsoft.DocumentDB/databaseAccounts@2023-04-15' = if (deployCosmosDB) { - name: toLower('cosmos-${uniqueName}') - location: location - kind: 'GlobalDocumentDB' - properties: { - consistencyPolicy: { defaultConsistencyLevel: 'Session' } - locations: [ { - locationName: location - failoverPriority: 0 - isZoneRedundant: false - } - ] - databaseAccountOfferType: 'Standard' - } -} - -resource cosmosDatabase 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases@2023-04-15' = if (deployCosmosDB) { - parent: cosmosAccount - name: 'CopilotChat' - properties: { - resource: { - id: 'CopilotChat' - } - } -} - -resource messageContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-04-15' = if (deployCosmosDB) { - parent: cosmosDatabase - name: 'chatmessages' - properties: { - resource: { - id: 'chatmessages' - indexingPolicy: { - indexingMode: 'consistent' - automatic: true - includedPaths: [ - { - path: '/*' - } - ] - excludedPaths: [ - { - path: '/"_etag"/?' - } - ] - } - partitionKey: { - paths: [ - '/id' - ] - kind: 'Hash' - version: 2 - } - } - } -} - -resource sessionContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-04-15' = if (deployCosmosDB) { - parent: cosmosDatabase - name: 'chatsessions' - properties: { - resource: { - id: 'chatsessions' - indexingPolicy: { - indexingMode: 'consistent' - automatic: true - includedPaths: [ - { - path: '/*' - } - ] - excludedPaths: [ - { - path: '/"_etag"/?' - } - ] - } - partitionKey: { - paths: [ - '/id' - ] - kind: 'Hash' - version: 2 - } - } - } -} - -resource participantContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-04-15' = if (deployCosmosDB) { - parent: cosmosDatabase - name: 'chatparticipants' - properties: { - resource: { - id: 'chatparticipants' - indexingPolicy: { - indexingMode: 'consistent' - automatic: true - includedPaths: [ - { - path: '/*' - } - ] - excludedPaths: [ - { - path: '/"_etag"/?' - } - ] - } - partitionKey: { - paths: [ - '/id' - ] - kind: 'Hash' - version: 2 - } - } - } -} - -resource memorySourcesContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-04-15' = if (deployCosmosDB) { - parent: cosmosDatabase - name: 'chatmemorysources' - properties: { - resource: { - id: 'chatmemorysources' - indexingPolicy: { - indexingMode: 'consistent' - automatic: true - includedPaths: [ - { - path: '/*' - } - ] - excludedPaths: [ - { - path: '/"_etag"/?' - } - ] - } - partitionKey: { - paths: [ - '/id' - ] - kind: 'Hash' - version: 2 - } - } - } -} - -resource speechAccount 'Microsoft.CognitiveServices/accounts@2022-12-01' = if (deploySpeechServices) { - name: 'cog-${uniqueName}' - location: location - sku: { - name: 'S0' - } - kind: 'SpeechServices' - identity: { - type: 'None' - } - properties: { - customSubDomainName: 'cog-${uniqueName}' - networkAcls: { - defaultAction: 'Allow' - } - publicNetworkAccess: 'Enabled' - } -} - -resource staticWebApp 'Microsoft.Web/staticSites@2022-09-01' = { - name: 'swa-${uniqueName}' - location: webappLocation - properties: { - provider: 'None' - } - sku: { - name: 'Free' - tier: 'Free' - } -} - -output webappUrl string = staticWebApp.properties.defaultHostname -output webappName string = staticWebApp.name -output webapiUrl string = appServiceWeb.properties.defaultHostName -output webapiName string = appServiceWeb.name diff --git a/samples/apps/copilot-chat-app/deploy/main.json b/samples/apps/copilot-chat-app/deploy/main.json deleted file mode 100644 index 1f75396319fe..000000000000 --- a/samples/apps/copilot-chat-app/deploy/main.json +++ /dev/null @@ -1,891 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "18037485528098010448" - } - }, - "parameters": { - "name": { - "type": "string", - "defaultValue": "copichat", - "metadata": { - "description": "Name for the deployment consisting of alphanumeric characters or dashes ('-')" - } - }, - "webAppServiceSku": { - "type": "string", - "defaultValue": "B1", - "allowedValues": [ - "B1", - "S1", - "S2", - "S3", - "P1V3", - "P2V3", - "I1V2", - "I2V2" - ], - "metadata": { - "description": "SKU for the Azure App Service plan" - } - }, - "packageUri": { - "type": "string", - "defaultValue": "https://aka.ms/copilotchat/webapi/latest", - "metadata": { - "description": "Location of package to deploy as the web service" - } - }, - "aiService": { - "type": "string", - "defaultValue": "AzureOpenAI", - "allowedValues": [ - "AzureOpenAI", - "OpenAI" - ], - "metadata": { - "description": "Underlying AI service" - } - }, - "completionModel": { - "type": "string", - "defaultValue": "gpt-35-turbo", - "metadata": { - "description": "Model to use for chat completions" - } - }, - "embeddingModel": { - "type": "string", - "defaultValue": "text-embedding-ada-002", - "metadata": { - "description": "Model to use for text embeddings" - } - }, - "plannerModel": { - "type": "string", - "defaultValue": "gpt-35-turbo", - "metadata": { - "description": "Completion model the task planner should use" - } - }, - "aiEndpoint": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Azure OpenAI endpoint to use (Azure OpenAI only)" - } - }, - "aiApiKey": { - "type": "securestring", - "defaultValue": "", - "metadata": { - "description": "Azure OpenAI or OpenAI API key" - } - }, - "webApiKey": { - "type": "securestring", - "defaultValue": "[newGuid()]", - "metadata": { - "description": "WebAPI key to use for authorization" - } - }, - "deployNewAzureOpenAI": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to deploy a new Azure OpenAI instance" - } - }, - "deployCosmosDB": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Whether to deploy Cosmos DB for persistent chat storage" - } - }, - "deployQdrant": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Whether to deploy Qdrant (in a container) for persistent memory storage" - } - }, - "deploySpeechServices": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Whether to deploy Azure Speech Services to enable input by voice" - } - }, - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "Region for the resources" - } - }, - "webappLocation": { - "type": "string", - "defaultValue": "westus2", - "metadata": { - "description": "Region for the webapp frontend" - } - } - }, - "variables": { - "rgIdHash": "[uniqueString(resourceGroup().id)]", - "uniqueName": "[format('{0}-{1}', parameters('name'), variables('rgIdHash'))]", - "storageFileShareName": "aciqdrantshare" - }, - "resources": [ - { - "condition": "[parameters('deployQdrant')]", - "type": "Microsoft.Storage/storageAccounts/fileServices/shares", - "apiVersion": "2022-09-01", - "name": "[format('{0}/{1}/{2}', format('st{0}', variables('rgIdHash')), 'default', variables('storageFileShareName'))]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts/fileServices', format('st{0}', variables('rgIdHash')), 'default')]" - ] - }, - { - "condition": "[parameters('deployQdrant')]", - "type": "Microsoft.Storage/storageAccounts/fileServices", - "apiVersion": "2022-09-01", - "name": "[format('{0}/{1}', format('st{0}', variables('rgIdHash')), 'default')]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash')))]" - ] - }, - { - "condition": "[parameters('deployNewAzureOpenAI')]", - "type": "Microsoft.CognitiveServices/accounts", - "apiVersion": "2022-12-01", - "name": "[format('ai-{0}', variables('uniqueName'))]", - "location": "[parameters('location')]", - "kind": "OpenAI", - "sku": { - "name": "S0" - }, - "properties": { - "customSubDomainName": "[toLower(variables('uniqueName'))]" - } - }, - { - "condition": "[parameters('deployNewAzureOpenAI')]", - "type": "Microsoft.CognitiveServices/accounts/deployments", - "apiVersion": "2022-12-01", - "name": "[format('{0}/{1}', format('ai-{0}', variables('uniqueName')), parameters('completionModel'))]", - "properties": { - "model": { - "format": "OpenAI", - "name": "[parameters('completionModel')]" - }, - "scaleSettings": { - "scaleType": "Standard" - } - }, - "dependsOn": [ - "[resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName')))]" - ] - }, - { - "condition": "[parameters('deployNewAzureOpenAI')]", - "type": "Microsoft.CognitiveServices/accounts/deployments", - "apiVersion": "2022-12-01", - "name": "[format('{0}/{1}', format('ai-{0}', variables('uniqueName')), parameters('embeddingModel'))]", - "properties": { - "model": { - "format": "OpenAI", - "name": "[parameters('embeddingModel')]" - }, - "scaleSettings": { - "scaleType": "Standard" - } - }, - "dependsOn": [ - "[resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName')))]", - "[resourceId('Microsoft.CognitiveServices/accounts/deployments', format('ai-{0}', variables('uniqueName')), parameters('completionModel'))]" - ] - }, - { - "type": "Microsoft.Web/serverfarms", - "apiVersion": "2022-03-01", - "name": "[format('asp-{0}-webapi', variables('uniqueName'))]", - "location": "[parameters('location')]", - "kind": "app", - "sku": { - "name": "[parameters('webAppServiceSku')]" - } - }, - { - "type": "Microsoft.Web/sites", - "apiVersion": "2022-09-01", - "name": "[format('app-{0}-webapi', variables('uniqueName'))]", - "location": "[parameters('location')]", - "kind": "app", - "tags": { - "skweb": "1" - }, - "properties": { - "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', format('asp-{0}-webapi', variables('uniqueName')))]", - "httpsOnly": true, - "virtualNetworkSubnetId": "[reference(resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName'))), '2021-05-01').subnets[0].id]" - }, - "dependsOn": [ - "[resourceId('Microsoft.Web/serverfarms', format('asp-{0}-webapi', variables('uniqueName')))]", - "[resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName')))]" - ] - }, - { - "type": "Microsoft.Web/sites/config", - "apiVersion": "2022-09-01", - "name": "[format('{0}/{1}', format('app-{0}-webapi', variables('uniqueName')), 'web')]", - "properties": { - "alwaysOn": true, - "cors": { - "allowedOrigins": [ - "http://localhost:3000", - "https://localhost:3000" - ], - "supportCredentials": true - }, - "detailedErrorLoggingEnabled": true, - "minTlsVersion": "1.2", - "netFrameworkVersion": "v6.0", - "use32BitWorkerProcess": false, - "vnetRouteAllEnabled": true, - "webSocketsEnabled": true, - "appSettings": [ - { - "name": "AIService:Type", - "value": "[parameters('aiService')]" - }, - { - "name": "AIService:Endpoint", - "value": "[if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2022-12-01').endpoint, parameters('aiEndpoint'))]" - }, - { - "name": "AIService:Key", - "value": "[if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2022-12-01').key1, parameters('aiApiKey'))]" - }, - { - "name": "AIService:Models:Completion", - "value": "[parameters('completionModel')]" - }, - { - "name": "AIService:Models:Embedding", - "value": "[parameters('embeddingModel')]" - }, - { - "name": "AIService:Models:Planner", - "value": "[parameters('plannerModel')]" - }, - { - "name": "Authorization:Type", - "value": "[if(empty(parameters('webApiKey')), 'None', 'ApiKey')]" - }, - { - "name": "Authorization:ApiKey", - "value": "[parameters('webApiKey')]" - }, - { - "name": "ChatStore:Type", - "value": "[if(parameters('deployCosmosDB'), 'cosmos', 'volatile')]" - }, - { - "name": "ChatStore:Cosmos:Database", - "value": "CopilotChat" - }, - { - "name": "ChatStore:Cosmos:ChatSessionsContainer", - "value": "chatsessions" - }, - { - "name": "ChatStore:Cosmos:ChatMessagesContainer", - "value": "chatmessages" - }, - { - "name": "ChatStore:Cosmos:ChatMemorySourcesContainer", - "value": "chatmemorysources" - }, - { - "name": "ChatStore:Cosmos:ChatParticipantsContainer", - "value": "chatparticipants" - }, - { - "name": "ChatStore:Cosmos:ConnectionString", - "value": "[if(parameters('deployCosmosDB'), listConnectionStrings(resourceId('Microsoft.DocumentDB/databaseAccounts', toLower(format('cosmos-{0}', variables('uniqueName')))), '2023-04-15').connectionStrings[0].connectionString, '')]" - }, - { - "name": "MemoriesStore:Type", - "value": "[if(parameters('deployQdrant'), 'Qdrant', 'Volatile')]" - }, - { - "name": "MemoriesStore:Qdrant:Host", - "value": "[if(parameters('deployQdrant'), format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('app-{0}-qdrant', variables('uniqueName'))), '2022-09-01').defaultHostName), '')]" - }, - { - "name": "MemoriesStore:Qdrant:Port", - "value": "443" - }, - { - "name": "AzureSpeech:Region", - "value": "[parameters('location')]" - }, - { - "name": "AzureSpeech:Key", - "value": "[if(parameters('deploySpeechServices'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('cog-{0}', variables('uniqueName'))), '2022-12-01').key1, '')]" - }, - { - "name": "AllowedOrigins", - "value": "[[*]" - }, - { - "name": "Kestrel:Endpoints:Https:Url", - "value": "https://localhost:443" - }, - { - "name": "Logging:LogLevel:Default", - "value": "Warning" - }, - { - "name": "Logging:LogLevel:SemanticKernel.Service", - "value": "Warning" - }, - { - "name": "Logging:LogLevel:Microsoft.SemanticKernel", - "value": "Warning" - }, - { - "name": "Logging:LogLevel:Microsoft.AspNetCore.Hosting", - "value": "Warning" - }, - { - "name": "Logging:LogLevel:Microsoft.Hosting.Lifetimel", - "value": "Warning" - }, - { - "name": "ApplicationInsights:ConnectionString", - "value": "[reference(resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName'))), '2020-02-02').ConnectionString]" - }, - { - "name": "APPLICATIONINSIGHTS_CONNECTION_STRING", - "value": "[reference(resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName'))), '2020-02-02').ConnectionString]" - }, - { - "name": "ApplicationInsightsAgent_EXTENSION_VERSION", - "value": "~2" - } - ] - }, - "dependsOn": [ - "[resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName')))]", - "[resourceId('Microsoft.Web/sites', format('app-{0}-qdrant', variables('uniqueName')))]", - "[resourceId('Microsoft.Web/sites', format('app-{0}-webapi', variables('uniqueName')))]", - "[resourceId('Microsoft.DocumentDB/databaseAccounts', toLower(format('cosmos-{0}', variables('uniqueName'))))]", - "[resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName')))]", - "[resourceId('Microsoft.CognitiveServices/accounts', format('cog-{0}', variables('uniqueName')))]" - ] - }, - { - "type": "Microsoft.Web/sites/extensions", - "apiVersion": "2022-09-01", - "name": "[format('{0}/{1}', format('app-{0}-webapi', variables('uniqueName')), 'MSDeploy')]", - "kind": "string", - "properties": { - "packageUri": "[parameters('packageUri')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.Web/sites', format('app-{0}-webapi', variables('uniqueName')))]", - "[resourceId('Microsoft.Web/sites/config', format('app-{0}-webapi', variables('uniqueName')), 'web')]" - ] - }, - { - "type": "Microsoft.Insights/components", - "apiVersion": "2020-02-02", - "name": "[format('appins-{0}', variables('uniqueName'))]", - "location": "[parameters('location')]", - "kind": "string", - "tags": { - "displayName": "AppInsight" - }, - "properties": { - "Application_Type": "web", - "WorkspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', format('la-{0}', variables('uniqueName')))]" - }, - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', format('la-{0}', variables('uniqueName')))]" - ] - }, - { - "type": "Microsoft.Web/sites/siteextensions", - "apiVersion": "2022-09-01", - "name": "[format('{0}/{1}', format('app-{0}-webapi', variables('uniqueName')), 'Microsoft.ApplicationInsights.AzureWebSites')]", - "dependsOn": [ - "[resourceId('Microsoft.Web/sites', format('app-{0}-webapi', variables('uniqueName')))]", - "[resourceId('Microsoft.Web/sites/extensions', format('app-{0}-webapi', variables('uniqueName')), 'MSDeploy')]" - ] - }, - { - "type": "Microsoft.OperationalInsights/workspaces", - "apiVersion": "2022-10-01", - "name": "[format('la-{0}', variables('uniqueName'))]", - "location": "[parameters('location')]", - "tags": { - "displayName": "Log Analytics" - }, - "properties": { - "sku": { - "name": "PerGB2018" - }, - "retentionInDays": 90, - "features": { - "searchVersion": 1, - "legacy": 0, - "enableLogAccessUsingOnlyResourcePermissions": true - } - } - }, - { - "condition": "[parameters('deployQdrant')]", - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2022-09-01", - "name": "[format('st{0}', variables('rgIdHash'))]", - "location": "[parameters('location')]", - "kind": "StorageV2", - "sku": { - "name": "Standard_LRS" - }, - "properties": { - "supportsHttpsTrafficOnly": true, - "allowBlobPublicAccess": false - } - }, - { - "condition": "[parameters('deployQdrant')]", - "type": "Microsoft.Web/serverfarms", - "apiVersion": "2022-03-01", - "name": "[format('asp-{0}-qdrant', variables('uniqueName'))]", - "location": "[parameters('location')]", - "kind": "linux", - "sku": { - "name": "P1v3" - }, - "properties": { - "reserved": true - } - }, - { - "condition": "[parameters('deployQdrant')]", - "type": "Microsoft.Web/sites", - "apiVersion": "2022-09-01", - "name": "[format('app-{0}-qdrant', variables('uniqueName'))]", - "location": "[parameters('location')]", - "kind": "app,linux,container", - "properties": { - "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', format('asp-{0}-qdrant', variables('uniqueName')))]", - "httpsOnly": true, - "reserved": true, - "clientCertMode": "Required", - "virtualNetworkSubnetId": "[reference(resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName'))), '2021-05-01').subnets[1].id]", - "siteConfig": { - "numberOfWorkers": 1, - "linuxFxVersion": "DOCKER|qdrant/qdrant:latest", - "alwaysOn": true, - "vnetRouteAllEnabled": true, - "ipSecurityRestrictions": [ - { - "vnetSubnetResourceId": "[reference(resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName'))), '2021-05-01').subnets[0].id]", - "action": "Allow", - "priority": 300, - "name": "Allow front vnet" - }, - { - "ipAddress": "Any", - "action": "Deny", - "priority": 2147483647, - "name": "Deny all" - } - ], - "azureStorageAccounts": { - "aciqdrantshare": { - "type": "AzureFiles", - "accountName": "[if(parameters('deployQdrant'), format('st{0}', variables('rgIdHash')), 'notdeployed')]", - "shareName": "[variables('storageFileShareName')]", - "mountPath": "/qdrant/storage", - "accessKey": "[if(parameters('deployQdrant'), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[0].value, '')]" - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.Web/serverfarms', format('asp-{0}-qdrant', variables('uniqueName')))]", - "[resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash')))]", - "[resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName')))]" - ] - }, - { - "type": "Microsoft.Network/virtualNetworks", - "apiVersion": "2021-05-01", - "name": "[format('vnet-{0}', variables('uniqueName'))]", - "location": "[parameters('location')]", - "properties": { - "addressSpace": { - "addressPrefixes": [ - "10.0.0.0/16" - ] - }, - "subnets": [ - { - "name": "webSubnet", - "properties": { - "addressPrefix": "10.0.1.0/24", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', format('nsg-{0}-webapi', variables('uniqueName')))]" - }, - "serviceEndpoints": [ - { - "service": "Microsoft.Web", - "locations": [ - "*" - ] - } - ], - "delegations": [ - { - "name": "delegation", - "properties": { - "serviceName": "Microsoft.Web/serverfarms" - } - } - ], - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Enabled" - } - }, - { - "name": "qdrantSubnet", - "properties": { - "addressPrefix": "10.0.2.0/24", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', format('nsg-{0}-qdrant', variables('uniqueName')))]" - }, - "serviceEndpoints": [ - { - "service": "Microsoft.Web", - "locations": [ - "*" - ] - } - ], - "delegations": [ - { - "name": "delegation", - "properties": { - "serviceName": "Microsoft.Web/serverfarms" - } - } - ], - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Enabled" - } - } - ] - }, - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', format('nsg-{0}-qdrant', variables('uniqueName')))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', format('nsg-{0}-webapi', variables('uniqueName')))]" - ] - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2022-11-01", - "name": "[format('nsg-{0}-webapi', variables('uniqueName'))]", - "location": "[parameters('location')]", - "properties": { - "securityRules": [ - { - "name": "AllowAnyHTTPSInbound", - "properties": { - "protocol": "TCP", - "sourcePortRange": "*", - "destinationPortRange": "443", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - } - ] - } - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2022-11-01", - "name": "[format('nsg-{0}-qdrant', variables('uniqueName'))]", - "location": "[parameters('location')]", - "properties": { - "securityRules": [] - } - }, - { - "type": "Microsoft.Web/sites/virtualNetworkConnections", - "apiVersion": "2022-09-01", - "name": "[format('{0}/{1}', format('app-{0}-webapi', variables('uniqueName')), 'webSubnetConnection')]", - "properties": { - "vnetResourceId": "[reference(resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName'))), '2021-05-01').subnets[0].id]", - "isSwift": true - }, - "dependsOn": [ - "[resourceId('Microsoft.Web/sites', format('app-{0}-webapi', variables('uniqueName')))]", - "[resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName')))]" - ] - }, - { - "condition": "[parameters('deployQdrant')]", - "type": "Microsoft.Web/sites/virtualNetworkConnections", - "apiVersion": "2022-09-01", - "name": "[format('{0}/{1}', format('app-{0}-qdrant', variables('uniqueName')), 'qdrantSubnetConnection')]", - "properties": { - "vnetResourceId": "[reference(resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName'))), '2021-05-01').subnets[1].id]", - "isSwift": true - }, - "dependsOn": [ - "[resourceId('Microsoft.Web/sites', format('app-{0}-qdrant', variables('uniqueName')))]", - "[resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName')))]" - ] - }, - { - "condition": "[parameters('deployCosmosDB')]", - "type": "Microsoft.DocumentDB/databaseAccounts", - "apiVersion": "2023-04-15", - "name": "[toLower(format('cosmos-{0}', variables('uniqueName')))]", - "location": "[parameters('location')]", - "kind": "GlobalDocumentDB", - "properties": { - "consistencyPolicy": { - "defaultConsistencyLevel": "Session" - }, - "locations": [ - { - "locationName": "[parameters('location')]", - "failoverPriority": 0, - "isZoneRedundant": false - } - ], - "databaseAccountOfferType": "Standard" - } - }, - { - "condition": "[parameters('deployCosmosDB')]", - "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases", - "apiVersion": "2023-04-15", - "name": "[format('{0}/{1}', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat')]", - "properties": { - "resource": { - "id": "CopilotChat" - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', toLower(format('cosmos-{0}', variables('uniqueName'))))]" - ] - }, - { - "condition": "[parameters('deployCosmosDB')]", - "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers", - "apiVersion": "2023-04-15", - "name": "[format('{0}/{1}/{2}', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat', 'chatmessages')]", - "properties": { - "resource": { - "id": "chatmessages", - "indexingPolicy": { - "indexingMode": "consistent", - "automatic": true, - "includedPaths": [ - { - "path": "/*" - } - ], - "excludedPaths": [ - { - "path": "/\"_etag\"/?" - } - ] - }, - "partitionKey": { - "paths": [ - "/id" - ], - "kind": "Hash", - "version": 2 - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat')]" - ] - }, - { - "condition": "[parameters('deployCosmosDB')]", - "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers", - "apiVersion": "2023-04-15", - "name": "[format('{0}/{1}/{2}', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat', 'chatsessions')]", - "properties": { - "resource": { - "id": "chatsessions", - "indexingPolicy": { - "indexingMode": "consistent", - "automatic": true, - "includedPaths": [ - { - "path": "/*" - } - ], - "excludedPaths": [ - { - "path": "/\"_etag\"/?" - } - ] - }, - "partitionKey": { - "paths": [ - "/id" - ], - "kind": "Hash", - "version": 2 - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat')]" - ] - }, - { - "condition": "[parameters('deployCosmosDB')]", - "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers", - "apiVersion": "2023-04-15", - "name": "[format('{0}/{1}/{2}', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat', 'chatparticipants')]", - "properties": { - "resource": { - "id": "chatparticipants", - "indexingPolicy": { - "indexingMode": "consistent", - "automatic": true, - "includedPaths": [ - { - "path": "/*" - } - ], - "excludedPaths": [ - { - "path": "/\"_etag\"/?" - } - ] - }, - "partitionKey": { - "paths": [ - "/id" - ], - "kind": "Hash", - "version": 2 - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat')]" - ] - }, - { - "condition": "[parameters('deployCosmosDB')]", - "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers", - "apiVersion": "2023-04-15", - "name": "[format('{0}/{1}/{2}', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat', 'chatmemorysources')]", - "properties": { - "resource": { - "id": "chatmemorysources", - "indexingPolicy": { - "indexingMode": "consistent", - "automatic": true, - "includedPaths": [ - { - "path": "/*" - } - ], - "excludedPaths": [ - { - "path": "/\"_etag\"/?" - } - ] - }, - "partitionKey": { - "paths": [ - "/id" - ], - "kind": "Hash", - "version": 2 - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', toLower(format('cosmos-{0}', variables('uniqueName'))), 'CopilotChat')]" - ] - }, - { - "condition": "[parameters('deploySpeechServices')]", - "type": "Microsoft.CognitiveServices/accounts", - "apiVersion": "2022-12-01", - "name": "[format('cog-{0}', variables('uniqueName'))]", - "location": "[parameters('location')]", - "sku": { - "name": "S0" - }, - "kind": "SpeechServices", - "identity": { - "type": "None" - }, - "properties": { - "customSubDomainName": "[format('cog-{0}', variables('uniqueName'))]", - "networkAcls": { - "defaultAction": "Allow" - }, - "publicNetworkAccess": "Enabled" - } - }, - { - "type": "Microsoft.Web/staticSites", - "apiVersion": "2022-09-01", - "name": "[format('swa-{0}', variables('uniqueName'))]", - "location": "[parameters('webappLocation')]", - "properties": { - "provider": "None" - }, - "sku": { - "name": "Free", - "tier": "Free" - } - } - ], - "outputs": { - "webappUrl": { - "type": "string", - "value": "[reference(resourceId('Microsoft.Web/staticSites', format('swa-{0}', variables('uniqueName'))), '2022-09-01').defaultHostname]" - }, - "webappName": { - "type": "string", - "value": "[format('swa-{0}', variables('uniqueName'))]" - }, - "webapiUrl": { - "type": "string", - "value": "[reference(resourceId('Microsoft.Web/sites', format('app-{0}-webapi', variables('uniqueName'))), '2022-09-01').defaultHostName]" - }, - "webapiName": { - "type": "string", - "value": "[format('app-{0}-webapi', variables('uniqueName'))]" - } - } -} \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/deploy/package-webapi.ps1 b/samples/apps/copilot-chat-app/deploy/package-webapi.ps1 deleted file mode 100644 index 3966b35c6758..000000000000 --- a/samples/apps/copilot-chat-app/deploy/package-webapi.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -<# -.SYNOPSIS -Package CopilotChat's WebAPI for deployment to Azure -#> - -param( - [string] - # Build configuration to publish. - $BuildConfiguration = "Release", - - [string] - # .NET framework to publish. - $DotNetFramework = "net6.0", - - [string] - # Target runtime to publish. - $TargetRuntime = "win-x64", - - [string] - # Output directory for published assets. - $OutputDirectory = "$PSScriptRoot" -) - -Write-Host "BuildConfiguration: $BuildConfiguration" -Write-Host "DotNetFramework: $DotNetFramework" -Write-Host "TargetRuntime: $TargetRuntime" -Write-Host "OutputDirectory: $OutputDirectory" - -$publishOutputDirectory = "$OutputDirectory/publish" -$publishedZipDirectory = "$OutputDirectory/out" -$publishedZipFilePath = "$publishedZipDirectory/webapi.zip" -if (!(Test-Path $publishedZipDirectory)) { - New-Item -ItemType Directory -Force -Path $publishedZipDirectory | Out-Null -} -if (!(Test-Path $publishOutputDirectory)) { - New-Item -ItemType Directory -Force -Path $publishOutputDirectory | Out-Null -} - -Write-Host "Build configuration: $BuildConfiguration" -dotnet publish "$PSScriptRoot/../webapi/CopilotChatWebApi.csproj" --configuration $BuildConfiguration --framework $DotNetFramework --runtime $TargetRuntime --self-contained --output "$publishOutputDirectory" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Host "Compressing to $publishedZipFilePath" -Compress-Archive -Path $publishOutputDirectory\* -DestinationPath $publishedZipFilePath -Force - -Write-Host "Published webapi package to '$publishedZipFilePath'" \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/deploy/package-webapi.sh b/samples/apps/copilot-chat-app/deploy/package-webapi.sh deleted file mode 100644 index 8f5da09eb4bf..000000000000 --- a/samples/apps/copilot-chat-app/deploy/package-webapi.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash - -# Package CiopilotChat's WebAPI for deployment to Azure - -set -e - -SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -OUTPUT_DIRECTORY="$SCRIPT_ROOT" - -usage() { - echo "Usage: $0 -d DEPLOYMENT_NAME -s SUBSCRIPTION --ai AI_SERVICE_TYPE -aikey AI_SERVICE_KEY [OPTIONS]" - echo "" - echo "Arguments:" - echo " -c, --configuration CONFIGURATION Build configuration (default: Release)" - echo " -d, --dotnet DOTNET_FRAMEWORK_VERSION Target dotnet framework (default: net6.0)" - echo " -r, --runtime TARGET_RUNTIME Runtime identifier (default: linux-x64)" - echo " -p, --output OUTPUT_DIRECTORY Output directory (default: $SCRIPT_ROOT)" - echo " -nz, --no-zip Do not zip package (default: false)" -} - -# Parse arguments -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -c|--configuration) - CONFIGURATION="$2" - shift - shift - ;; - -d|--dotnet) - DOTNET="$2" - shift - shift - ;; - -r|--runtime) - RUNTIME="$2" - shift - shift - ;; - -o|--output) - OUTPUT_DIRECTORY="$2" - shift - shift - ;; - -nz|--no-zip) - NO_ZIP=true - shift - ;; - *) - echo "Unknown option $1" - usage - exit 1 - ;; - esac -done - -# Set defaults -: "${CONFIGURATION:="Release"}" -: "${DOTNET:="net6.0"}" -: "${RUNTIME:="win-x64"}" -: "${OUTPUT_DIRECTORY:="$SCRIPT_ROOT"}" - -PUBLISH_OUTPUT_DIRECTORY="$OUTPUT_DIRECTORY/publish" -PUBLISH_ZIP_DIRECTORY="$OUTPUT_DIRECTORY/out" -PACKAGE_FILE_PATH="$PUBLISH_ZIP_DIRECTORY/webapi.zip" - -if [[ ! -d "$PUBLISH_OUTPUT_DIRECTORY" ]]; then - mkdir -p "$PUBLISH_OUTPUT_DIRECTORY" -fi -if [[ ! -d "$PUBLISH_ZIP_DIRECTORY" ]]; then - mkdir -p "$PUBLISH_ZIP_DIRECTORY" -fi - -echo "Build configuration: $CONFIGURATION" -dotnet publish "$SCRIPT_ROOT/../webapi/CopilotChatWebApi.csproj" --configuration $CONFIGURATION --framework $DOTNET --runtime $RUNTIME --self-contained --output "$PUBLISH_OUTPUT_DIRECTORY" -if [ $? -ne 0 ]; then - exit 1 -fi - -# if not NO_ZIP then zip the package -if [[ -z "$NO_ZIP" ]]; then - pushd "$PUBLISH_OUTPUT_DIRECTORY" - echo "Compressing to $PACKAGE_FILE_PATH" - zip -r $PACKAGE_FILE_PATH . - popd -fi - - diff --git a/samples/apps/copilot-chat-app/images/Cert-Issue.png b/samples/apps/copilot-chat-app/images/Cert-Issue.png deleted file mode 100644 index c0ac3f82205c..000000000000 Binary files a/samples/apps/copilot-chat-app/images/Cert-Issue.png and /dev/null differ diff --git a/samples/apps/copilot-chat-app/images/Document-Memory-Sample-1.png b/samples/apps/copilot-chat-app/images/Document-Memory-Sample-1.png deleted file mode 100644 index 84ced7083339..000000000000 Binary files a/samples/apps/copilot-chat-app/images/Document-Memory-Sample-1.png and /dev/null differ diff --git a/samples/apps/copilot-chat-app/images/Document-Memory-Sample-2.png b/samples/apps/copilot-chat-app/images/Document-Memory-Sample-2.png deleted file mode 100644 index 777ecba850b3..000000000000 Binary files a/samples/apps/copilot-chat-app/images/Document-Memory-Sample-2.png and /dev/null differ diff --git a/samples/apps/copilot-chat-app/images/UI-Sample.png b/samples/apps/copilot-chat-app/images/UI-Sample.png deleted file mode 100644 index 1bad26fe3546..000000000000 Binary files a/samples/apps/copilot-chat-app/images/UI-Sample.png and /dev/null differ diff --git a/samples/apps/copilot-chat-app/importdocument/Config.cs b/samples/apps/copilot-chat-app/importdocument/Config.cs deleted file mode 100644 index abf5f59c732a..000000000000 --- a/samples/apps/copilot-chat-app/importdocument/Config.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Configuration; - -namespace ImportDocument; - -/// -/// Configuration for the app. -/// -public sealed class Config -{ - /// - /// Client ID for the app as registered in Azure AD. - /// - public string ClientId { get; set; } = string.Empty; - - /// - /// Redirect URI for the app as registered in Azure AD. - /// -#pragma warning disable CA1056 // URI-like properties should not be strings - public string RedirectUri { get; set; } = string.Empty; -#pragma warning restore CA1056 // URI-like properties should not be strings - - /// - /// Uri for the service that is running the chat. - /// -#pragma warning disable CA1056 // URI-like properties should not be strings - public string ServiceUri { get; set; } = string.Empty; -#pragma warning restore CA1056 // URI-like properties should not be strings - - /// - /// Api key for the service that is running the chat. - /// - public string ApiKey { get; set; } = string.Empty; - - /// - /// Gets configuration from appsettings.json. - /// - /// An Config instance - public static Config? GetConfig() - { - var config = new ConfigurationBuilder() - .AddJsonFile("appsettings.json") - .Build(); - - return config.GetRequiredSection("Config").Get(); - } - - /// - /// Validates a Config object. - /// - /// - /// True is the config object is not null. - public static bool Validate(Config? config) - { - return config != null; - } -} diff --git a/samples/apps/copilot-chat-app/importdocument/ImportDocument.csproj b/samples/apps/copilot-chat-app/importdocument/ImportDocument.csproj deleted file mode 100644 index a9c42e30b495..000000000000 --- a/samples/apps/copilot-chat-app/importdocument/ImportDocument.csproj +++ /dev/null @@ -1,30 +0,0 @@ - - - Exe - net6.0 - LatestMajor - disable - enable - 10 - false - - - - - Always - - - - - - - - - - - - - <_Parameter1>false - - - diff --git a/samples/apps/copilot-chat-app/importdocument/Program.cs b/samples/apps/copilot-chat-app/importdocument/Program.cs deleted file mode 100644 index 3149de2e6709..000000000000 --- a/samples/apps/copilot-chat-app/importdocument/Program.cs +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.CommandLine; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Identity.Client; - -namespace ImportDocument; - -/// -/// This console app imports a list of files to the CopilotChat WebAPI document memory store. -/// -public static class Program -{ - public static void Main(string[] args) - { - var config = Config.GetConfig(); - if (!Config.Validate(config)) - { - Console.WriteLine("Error: Failed to read appsettings.json."); - return; - } - - var filesOption = new Option>(name: "--files", description: "The files to import to document memory store.") - { - IsRequired = true, - AllowMultipleArgumentsPerToken = true, - }; - - var chatCollectionOption = new Option( - name: "--chat-id", - description: "Save the extracted context to an isolated chat collection.", - getDefaultValue: () => Guid.Empty - ); - - var rootCommand = new RootCommand( - "This console app imports files to the CopilotChat WebAPI's document memory store." - ) - { - filesOption, chatCollectionOption - }; - - rootCommand.SetHandler(async (files, chatCollectionId) => - { - await ImportFilesAsync(files, config!, chatCollectionId); - }, - filesOption, chatCollectionOption - ); - - rootCommand.Invoke(args); - } - - /// - /// Acquires a user account from Azure AD. - /// - /// The App configuration. - /// Sets the account to the first account found. - /// Sets the access token to the first account found. - /// True if the user account was acquired. - private static async Task AcquireUserAccountAsync( - Config config, - Action setAccount, - Action setAccessToken) - { - Console.WriteLine("Requesting User Account ID..."); - - string[] scopes = { "User.Read" }; - try - { - var app = PublicClientApplicationBuilder.Create(config.ClientId) - .WithRedirectUri(config.RedirectUri) - .Build(); - var result = await app.AcquireTokenInteractive(scopes).ExecuteAsync(); - IEnumerable? accounts = await app.GetAccountsAsync(); - IAccount? first = accounts.FirstOrDefault(); - - if (first is null) - { - Console.WriteLine("Error: No accounts found"); - return false; - } - - setAccount(first); - setAccessToken(result.AccessToken); - return true; - } - catch (Exception ex) when (ex is MsalServiceException or MsalClientException) - { - Console.WriteLine($"Error: {ex.Message}"); - return false; - } - } - - /// - /// Conditionally imports a list of files to the Document Store. - /// - /// A list of files to import. - /// Configuration. - /// Save the extracted context to an isolated chat collection. - private static async Task ImportFilesAsync(IEnumerable files, Config config, Guid chatCollectionId) - { - foreach (var file in files) - { - if (!file.Exists) - { - Console.WriteLine($"File {file.FullName} does not exist."); - return; - } - } - - IAccount? userAccount = null; - string? accessToken = null; - - if (await AcquireUserAccountAsync(config, v => { userAccount = v; }, v => { accessToken = v; }) == false) - { - Console.WriteLine("Error: Failed to acquire user account."); - return; - } - Console.WriteLine($"Successfully acquired User ID. Continuing..."); - - using var formContent = new MultipartFormDataContent(); - List filesContent = files.Select(file => new StreamContent(file.OpenRead())).ToList(); - for (int i = 0; i < filesContent.Count; i++) - { - formContent.Add(filesContent[i], "formFiles", files.ElementAt(i).Name); - } - - var userId = userAccount!.HomeAccountId.Identifier; - var userName = userAccount.Username; - using var userIdContent = new StringContent(userId); - using var userNameContent = new StringContent(userName); - formContent.Add(userIdContent, "userId"); - formContent.Add(userNameContent, "userName"); - - if (chatCollectionId != Guid.Empty) - { - Console.WriteLine($"Uploading and parsing file to chat {chatCollectionId}..."); - using var chatScopeContent = new StringContent("Chat"); - using var chatCollectionIdContent = new StringContent(chatCollectionId.ToString()); - formContent.Add(chatScopeContent, "documentScope"); - formContent.Add(chatCollectionIdContent, "chatId"); - - // Calling UploadAsync here to make sure disposable objects are still in scope. - await UploadAsync(formContent, accessToken!, config); - } - else - { - Console.WriteLine("Uploading and parsing file to global collection..."); - using var globalScopeContent = new StringContent("Global"); - formContent.Add(globalScopeContent, "documentScope"); - - // Calling UploadAsync here to make sure disposable objects are still in scope. - await UploadAsync(formContent, accessToken!, config); - } - - // Dispose of all the file streams. - foreach (var fileContent in filesContent) - { - fileContent.Dispose(); - } - } - - /// - /// Sends a POST request to the Document Store to upload a file for parsing. - /// - /// The multipart form data content to send. - /// Configuration. - private static async Task UploadAsync( - MultipartFormDataContent multipartFormDataContent, - string accessToken, - Config config) - { - // Create a HttpClient instance and set the timeout to infinite since - // large documents will take a while to parse. - using HttpClientHandler clientHandler = new() - { - CheckCertificateRevocationList = true - }; - using HttpClient httpClient = new(clientHandler) - { - Timeout = Timeout.InfiniteTimeSpan - }; - // Add required properties to the request header. - httpClient.DefaultRequestHeaders.Add("Authorization", $"Bearer {accessToken}"); - if (!string.IsNullOrEmpty(config.ApiKey)) - { - httpClient.DefaultRequestHeaders.Add("x-sk-api-key", config.ApiKey); - } - - try - { - using HttpResponseMessage response = await httpClient.PostAsync( - new Uri(new Uri(config.ServiceUri), "importDocuments"), - multipartFormDataContent - ); - - if (!response.IsSuccessStatusCode) - { - Console.WriteLine($"Error: {response.StatusCode} {response.ReasonPhrase}"); - Console.WriteLine(await response.Content.ReadAsStringAsync()); - return; - } - - Console.WriteLine("Uploading and parsing successful."); - } - catch (HttpRequestException ex) - { - Console.WriteLine($"Error: {ex.Message}"); - } - } -} diff --git a/samples/apps/copilot-chat-app/importdocument/README.md b/samples/apps/copilot-chat-app/importdocument/README.md deleted file mode 100644 index 8ab490ed515c..000000000000 --- a/samples/apps/copilot-chat-app/importdocument/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Copilot Chat Import Document App - -> **!IMPORTANT** -> This sample is for educational purposes only and is not recommended for production deployments. - -One of the exciting features of the Copilot Chat App is its ability to store contextual information -to [memories](https://github.com/microsoft/semantic-kernel/blob/main/docs/EMBEDDINGS.md) and retrieve -relevant information from memories to provide more meaningful answers to users through out the conversations. - -Memories can be generated from conversations as well as imported from external sources, such as documents. -Importing documents enables Copilot Chat to have up-to-date knowledge of specific contexts, such as enterprise and personal data. - -## Configure your environment -1. A registered App in Azure Portal (https://learn.microsoft.com/azure/active-directory/develop/quickstart-register-app) - - Select Mobile and desktop applications as platform type, and the Redirect URI will be `http://localhost` - - Select **`Accounts in any organizational directory (Any Azure AD directory - Multitenant) - and personal Microsoft accounts (e.g. Skype, Xbox)`** as the supported account - type for this sample. - - Note the **`Application (client) ID`** from your app registration. -2. Make sure the service is running. To start the service, see [here](../webapi/README.md). - -## Running the app -1. Ensure the web api is running at `https://localhost:40443/`. -2. Configure the appsettings.json file under this folder root with the following variables and fill - in with your information, where - `ClientId` is the GUID copied from the **Application (client) ID** from your app registration in the Azure Portal, - `RedirectUri` is the Redirect URI also from the app registration in the Azure Portal, and - `ServiceUri` is the address the web api is running at. - `ApiKey` is the API key to the service if there is one. - -3. Change directory to this folder root. -4. **Run** the following command to import a document to the app under the global document collection where - all users will have access to: - - `dotnet run --files .\sample-docs\ms10k.txt` - - Or **Run** the following command to import a document to the app under a chat isolated document collection where - only the chat session will have access to: - - `dotnet run --files .\sample-docs\ms10k.txt --chat-id [chatId]` - - > Note that this will open a browser window for you to sign in to retrieve your user id to make sure you have access to the chat session. - - > Currently only supports txt and pdf files. A sample file is provided under ./sample-docs. - - Importing may take some time to generate embeddings for each piece/chunk of a document. - - To import multiple files, specify multiple files. For example: - - `dotnet run --files .\sample-docs\ms10k.txt .\sample-docs\Microsoft-Responsible-AI-Standard-v2-General-Requirements.pdf` - -5. Chat with the bot. - - Examples: - - With [ms10k.txt](./sample-docs/ms10k.txt): - - ![](../images/Document-Memory-Sample-1.png) - - With [Microsoft Responsible AI Standard v2 General Requirements.pdf](./sample-docs/Microsoft-Responsible-AI-Standard-v2-General-Requirements.pdf): - - ![](../images/Document-Memory-Sample-2.png) \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/importdocument/appsettings.json b/samples/apps/copilot-chat-app/importdocument/appsettings.json deleted file mode 100644 index c1aa3ade814c..000000000000 --- a/samples/apps/copilot-chat-app/importdocument/appsettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Config": { - "ClientId": "", - "RedirectUri": "", - "ServiceUri": "https://localhost:40443", - "ApiKey": "" - } -} \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/importdocument/sample-docs/Lorem_ipsum.pdf b/samples/apps/copilot-chat-app/importdocument/sample-docs/Lorem_ipsum.pdf deleted file mode 100644 index e25081e0325c..000000000000 Binary files a/samples/apps/copilot-chat-app/importdocument/sample-docs/Lorem_ipsum.pdf and /dev/null differ diff --git a/samples/apps/copilot-chat-app/importdocument/sample-docs/Microsoft-Responsible-AI-Standard-v2-General-Requirements.pdf b/samples/apps/copilot-chat-app/importdocument/sample-docs/Microsoft-Responsible-AI-Standard-v2-General-Requirements.pdf deleted file mode 100644 index 31669fb07c3c..000000000000 Binary files a/samples/apps/copilot-chat-app/importdocument/sample-docs/Microsoft-Responsible-AI-Standard-v2-General-Requirements.pdf and /dev/null differ diff --git a/samples/apps/copilot-chat-app/importdocument/sample-docs/ms10k.txt b/samples/apps/copilot-chat-app/importdocument/sample-docs/ms10k.txt deleted file mode 100644 index 355a08de969e..000000000000 --- a/samples/apps/copilot-chat-app/importdocument/sample-docs/ms10k.txt +++ /dev/null @@ -1,18675 +0,0 @@ -UNITED STATES - - -SECURITIES AND EXCHANGE COMMISSION -Washington, D.C. 20549 - - -FORM 10-K - - -? ANNUAL REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 - -For the Fiscal Year Ended June 30, 2022 - -OR - -? TRANSITION REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 - -For the Transition Period From to - -Commission File Number 001-37845 - - -MICROSOFT CORPORATION - - -WASHINGTON 91-1144442 - -(STATE OF INCORPORATION) (I.R.S. ID) - -ONE MICROSOFT WAY, REDMOND, WASHINGTON 98052-6399 - -(425) 882-8080 - -www.microsoft.com/investor - -Securities registered pursuant to Section 12(b) of the Act: - -Title of each class Trading Symbol Name of exchange on which registered - - -Common stock, $0.00000625 par value per share -MSFT -3.125% Notes due 2028 -MSFT -2.625% Notes due 2033 -MSFT - -Securities registered pursuant to Section 12(g) of the Act: - -NONE - -Indicate by check mark if the registrant is a well-known seasoned issuer, as defined in Rule 405 of the Securities Act. - -Indicate by check mark if the registrant is not required to file reports pursuant to Section 13 or Section 15(d) of the Act. - - - -NASDAQ - -NASDAQ - -NASDAQ - - -Yes ? No ? - -Yes ? No ? - - -Indicate by check mark whether the registrant (1) has filed all reports required to be filed by Section 13 or 15(d) of the Securities Exchange Act of 1934 during the preceding 12 months (or for such shorter period that the registrant was required to file such reports), and (2) has been subject to such filing requirements for the past - -90 days. Yes ? No ? - -Indicate by check mark whether the registrant has submitted electronically every Interactive Data File required to be submitted pursuant to Rule 405 of Regulation S-T (�232.405 of this chapter) during the preceding 12 months (or for such shorter period that the registrant was required to submit such files). Yes ? No ? - -Indicate by check mark whether the registrant is a large accelerated filer, an accelerated filer, a non-accelerated filer, a smaller reporting company, or an emerging growth company. See the definitions of �large accelerated filer,� �accelerated filer,� �smaller reporting company,� and �emerging growth company� in Rule 12b-2 of the Exchange Act. - -Large Accelerated Filer ? Accelerated Filer ? - -Non-accelerated Filer ? Smaller Reporting Company ? - -Emerging Growth Company ? - -If an emerging growth company, indicate by check mark if the registrant has elected not to use the extended transition period for complying with any new or revised financial accounting standards provided pursuant to Section 13(a) of the Exchange Act. ? - -Indicate by check mark whether the registrant has filed a report on and attestation to its management�s assessment of the effectiveness of its internal control over financial reporting under Section 404(b) of the Sarbanes-Oxley Act (15 U.S.C. 7262(b)) by the registered public accounting firm that prepared or issued its audit report. ? - -Indicate by check mark whether the registrant is a shell company (as defined in Rule 12b-2 of the Act). Yes ? No ? - -As of December 31, 2021, the aggregate market value of the registrant�s common stock held by non-affiliates of the registrant was $2.5 trillion based on the closing sale price as reported on the NASDAQ National Market System. As of July 25, 2022, there were 7,457,891,872 shares of common stock outstanding. - -DOCUMENTS INCORPORATED BY REFERENCE - -Portions of the definitive Proxy Statement to be delivered to shareholders in connection with the Annual Meeting of Shareholders to be held on December 13, 2022 are incorporated by reference into Part III. - - -MICROSOFT CORPORATION - -FORM 10-K - -For the Fiscal Year Ended June 30, 2022 - -INDEX - - - -PART I - -Item 1. -Business - - - - - - - - - -Information about our Executive Officers -Item 1A. -Risk Factors - -Item 1B. -Unresolved Staff Comments - -Item 2. -Properties - -Item 3. -Legal Proceedings - - - - -Item 4. -Mine Safety Disclosures - - - - - - - - - -PART II - - - -Page - - -3 - -21 - -23 - -37 - -37 - -37 - -37 - - -Item 5. -Market for Registrant�s Common Equity, Related Stockholder Matters, and Issuer Purchases of Equity - -Securities - - - - - - - - - - - - - - - - - - - - -Item 6. -[Reserved] - -Item 7. -Management�s Discussion and Analysis of Financial Condition and Results of Operations - -Item 7A. -Quantitative and Qualitative Disclosures about Market Risk - -Item 8. -Financial Statements and Supplementary Data - - - - - - - - - - - - - - - - -Item 9. -Changes in and Disagreements with Accountants on Accounting and Financial Disclosure - -Item 9A. -Controls and Procedures - - - - - - - - - - - - - - - -Report of Management on Internal Control over Financial Reporting - - -Report of Independent Registered Public Accounting Firm -Item 9B. -Other Information - - -Item 9C. -Disclosure Regarding Foreign Jurisdictions that Prevent Inspections -PART III - - - - - - - - - - - - - - - - - - - -Item 10. -Directors, Executive Officers and Corporate Governance -Item 11. -Executive Compensation - - - - - - - - - - -Item 12. -Security Ownership of Certain Beneficial Owners and Management and Related Stockholder Matters -Item 13. -Certain Relationships and Related Transactions, and Director Independence - - -Item 14. -Principal Accountant Fees and Services -PART IV - - - - - - - - - - - - - - - - - - - -Item 15. -Exhibit and Financial Statement Schedules -Item 16. -Form 10-K Summary - - -Signatures - - - -2 - - - - - - - - - - - - - - -38 - -39 - -40 - -56 - -57 - -99 - -99 - -99 - -100 - -101 - -101 - - -101 - -101 - -101 - -101 - -101 - - -102 - -108 - -109 - -PART I -Item 1 - - -Note About Forward-Looking Statements - -This report includes estimates, projections, statements relating to our business plans, objectives, and expected operating results that are �forward-looking statements� within the meaning of the Private Securities Litigation Reform Act of 1995, Section 27A of the Securities Act of 1933, and Section 21E of the Securities Exchange Act of 1934. Forward-looking statements may appear throughout this report, including the following sections: �Business� (Part I, Item 1 of this Form 10-K), �Risk Factors� (Part I, Item 1A of this Form 10-K), and �Management�s Discussion and Analysis of Financial Condition and Results of Operations� (Part II, Item 7 of this Form 10-K). These forward-looking statements generally are identified by the words �believe,� �project,� �expect,� �anticipate,� �estimate,� �intend,� �strategy,� �future,� �opportunity,� �plan,� �may,� �should,� �will,� �would,� �will be,� �will continue,� �will likely result,� and similar expressions. Forward-looking statements are based on current expectations and assumptions that are subject to risks and uncertainties that may cause actual results to differ materially. We describe risks and uncertainties that could cause actual results and events to differ materially in �Risk Factors,� �Management�s Discussion and Analysis of Financial Condition and Results of Operations,� and �Quantitative and Qualitative Disclosures about Market Risk� (Part II, Item 7A of this Form 10-K). Readers are cautioned not to place undue reliance on forward-looking statements, which speak only as of the date they are made. We undertake no obligation to update or revise publicly any forward-looking statements, whether because of new information, future events, or otherwise. - -PART I - -ITEM 1. BUSINESS - -GENERAL - -Embracing Our Future - -Microsoft is a technology company whose mission is to empower every person and every organization on the planet to achieve more. We strive to create local opportunity, growth, and impact in every country around the world. Our platforms and tools help drive small business productivity, large business competitiveness, and public-sector efficiency. We are creating the tools and platforms that deliver better, faster, and more effective solutions to support new startups, improve educational and health outcomes, and empower human ingenuity. - -Microsoft is innovating and expanding our entire portfolio to help people and organizations overcome today�s challenges and emerge stronger. We bring technology and products together into experiences and solutions that unlock value for our customers. - -In a dynamic environment, digital technology is the key input that powers the world�s economic output. Our ecosystem of customers and partners have learned that while hybrid work is complex, embracing flexibility, different work styles, and a culture of trust can help navigate the challenges the world faces today. Organizations of all sizes have digitized business-critical functions, redefining what they can expect from their business applications. Customers are looking to unlock value while simplifying security and management. From infrastructure and data, to business applications and collaboration, we provide unique, differentiated value to customers. - -We are building a distributed computing fabric � across cloud and the edge � to help every organization build, run, and manage mission-critical workloads anywhere. In the next phase of innovation, artificial intelligence (�AI�) capabilities are rapidly advancing, fueled by data and knowledge of the world. We are enabling metaverse experiences at all layers of our stack, so customers can more effectively model, automate, simulate, and predict changes within their industrial environments, feel a greater sense of presence in the new world of hybrid work, and create custom immersive worlds to enable new opportunities for connection and experimentation. - -What We Offer - -Founded in 1975, we develop and support software, services, devices, and solutions that deliver new value for customers and help people and businesses realize their full potential. - -We offer an array of services, including cloud-based solutions that provide customers with software, services, platforms, and content, and we provide solution support and consulting services. We also deliver relevant online advertising to a global audience. - -3 - - -PART I -Item 1 - -Our products include operating systems, cross-device productivity and collaboration applications, server applications, business solution applications, desktop and server management tools, software development tools, and video games. We also design and sell devices, including PCs, tablets, gaming and entertainment consoles, other intelligent devices, and related accessories. - -The Ambitions That Drive Us - -To achieve our vision, our research and development efforts focus on three interconnected ambitions: - -� Reinvent productivity and business processes. - -� Build the intelligent cloud and intelligent edge platform. - -� Create more personal computing. - -Reinvent Productivity and Business Processes - -At Microsoft, we provide technology and resources to help our customers create a secure hybrid work environment. Our family of products plays a key role in the ways the world works, learns, and connects. - -Our growth depends on securely delivering continuous innovation and advancing our leading productivity and collaboration tools and services, including Office 365, Dynamics 365, and LinkedIn. Microsoft 365 brings together Office 365, Windows, and Enterprise Mobility + Security to help organizations empower their employees with AI-backed tools that unlock creativity, increase collaboration, and fuel innovation, all the while enabling compliance coverage and data protection. Microsoft Teams is a comprehensive platform for work, with meetings, calls, chat, collaboration, and business process automation. Microsoft Viva is an employee experience platform that brings together communications, knowledge, learning, resources, and insights powered by Microsoft 365. Together with the Microsoft Cloud, Dynamics 365, Microsoft Teams, and Azure Synapse bring a new era of collaborative applications that transform every business function and process. Microsoft Power Platform is helping domain experts drive productivity gains with low-code/no-code tools, robotic process automation, virtual agents, and business intelligence. In a dynamic labor market, LinkedIn is helping professionals use the platform to connect, learn, grow, and get hired. - -Build the Intelligent Cloud and Intelligent Edge Platform - -As digital transformation accelerates, organizations in every sector across the globe can address challenges that will have a fundamental impact on their success. For enterprises, digital technology empowers employees, optimizes operations, engages customers, and in some cases, changes the very core of products and services. Microsoft has a proven track record of delivering high value to our customers across many diverse and durable growth markets. - -We continue to invest in high performance and sustainable computing to meet the growing demand for fast access to Microsoft services provided by our network of cloud computing infrastructure and datacenters. Azure is a trusted cloud with comprehensive compliance coverage and AI-based security built in. - -Our cloud business benefits from three economies of scale: datacenters that deploy computational resources at significantly lower cost per unit than smaller ones; datacenters that coordinate and aggregate diverse customer, geographic, and application demand patterns, improving the utilization of computing, storage, and network resources; and multi-tenancy locations that lower application maintenance labor costs. - -The Microsoft Cloud is the most comprehensive and trusted cloud, providing the best integration across the technology stack while offering openness, improving time to value, reducing costs, and increasing agility. Being a global-scale cloud, Azure uniquely offers hybrid consistency, developer productivity, AI capabilities, and trusted security and compliance. We see more emerging use cases and needs for compute and security at the edge and are accelerating our innovation across the spectrum of intelligent edge devices, from Internet of Things (�IoT�) sensors to gateway devices and edge hardware to build, manage, and secure edge workloads. With Azure Stack, organizations can extend Azure into their own datacenters to create a consistent stack across the public cloud and the intelligent edge. - -4 - - -PART I -Item 1 - -Our hybrid infrastructure consistency spans security, compliance, identity, and management, helping to support the real-world needs and evolving regulatory requirements of commercial customers and enterprises. Our industry clouds bring together capabilities across the entire Microsoft Cloud, along with industry-specific customizations, to improve time to value, increase agility, and lower costs. Azure Arc simplifies governance and management by delivering a consistent multi-cloud and on-premises management platform. Security, compliance, identity, and management underlie our entire tech stack. We offer integrated, end-to-end capabilities to protect people and organizations. - -In March 2022, we completed our acquisition of Nuance Communications, Inc. (�Nuance�). Together, Microsoft and Nuance will enable organizations across industries to accelerate their business goals with security-focused, cloud-based solutions infused with powerful, vertically optimized AI. - -We are accelerating our development of mixed reality solutions with new Azure services and devices. Microsoft Mesh enables presence and shared experiences from anywhere through mixed reality applications. The opportunity to merge the physical and digital worlds, when combined with the power of Azure cloud services, unlocks new workloads and experiences to create common understanding and drive more informed decisions. - -The ability to convert data into AI drives our competitive advantage. Azure SQL Database makes it possible for customers to take SQL Server from their on-premises datacenter to a fully managed instance in the cloud to utilize built-in AI. Azure Synapse brings together data integration, enterprise data warehousing, and big data analytics in a comprehensive solution. We are accelerating adoption of AI innovations from research to products. Our innovation helps every developer be an AI developer, with approachable new tools from Azure Machine Learning Studio for creating simple machine learning models, to the powerful Azure Machine Learning Workbench for the most advanced AI modeling and data science. From GitHub to Visual Studio, we provide a developer tool chain for everyone, no matter the technical experience, across all platforms, whether Azure, Windows, or any other cloud or client platform. - -Additionally, we are extending our infrastructure beyond the planet, bringing cloud computing to space. Azure Orbital is a fully managed ground station as a service for fast downlinking of data. - -Create More Personal Computing - -We strive to make computing more personal by putting people at the core of the experience, enabling them to interact with technology in more intuitive, engaging, and dynamic ways. Microsoft 365 is empowering people and organizations to be productive and secure as they adapt to more fluid ways of working, learning, and playing. Windows also plays a critical role in fueling our cloud business with Windows 365, a desktop operating system that�s also a cloud service. From another internet-connected device, including Android or macOS devices, you can run Windows 365, just like a virtual machine. - -With Windows 11, we have simplified the design and experience to empower productivity and inspire creativity. Windows 11 offers innovations focused on enhancing productivity and is designed to support hybrid work. It adds new experiences that include powerful task switching tools like new snap layouts, snap groups, and desktops; new ways to stay connected through Microsoft Teams chat; the information you want at your fingertips; and more. Windows 11 security and privacy features include operating system security, application security, and user and identity security. - -Tools like search, news, and maps have given us immediate access to the world�s information. Today, through our Search, News, Mapping, and Browse services, Microsoft delivers unique trust, privacy, and safety features. Microsoft Edge is our fast and secure browser that helps protect your data, with built-in shopping tools designed to save you time and money. Organizational tools such as Collections, Vertical Tabs, and Immersive Reader help make the most of your time while browsing, streaming, searching, and sharing. - -We are committed to designing and marketing first-party devices to help drive innovation, create new device categories, and stimulate demand in the Windows ecosystem. The Surface family includes Surface Laptop Studio, Surface Laptop 4, Surface Laptop Go 2, Surface Laptop Pro 8, Surface Pro X, Surface Go 3, Surface Studio 2, and Surface Duo 2. - -5 - - -PART I -Item 1 - -With three billion people actively playing games today, and a new generation steeped in interactive entertainment, Microsoft continues to invest in content, community, and cloud services. We have broadened our approach to how we think about gaming end-to-end, from the way games are created and distributed to how they are played, including cloud gaming so players can stream across PC, console, and mobile. We have a strong position with our large and growing highly engaged community of gamers, including the acquisition of ZeniMax Media Inc., the parent company of Bethesda Softworks LLC. In January 2022, we announced plans to acquire Activision Blizzard, Inc., a leader in game development and an interactive entertainment content publisher. Xbox Game Pass is a community with access to a curated library of over 100 first- and third -party console and PC titles. Xbox Cloud Gaming is Microsoft�s game streaming technology that is complementary to our console hardware and gives fans the ultimate choice to play the games they want, with the people they want, on the devices they want. - -Our Future Opportunity - -The case for digital transformation has never been more urgent. Customers are looking to us to help improve productivity and the affordability of their products and services. We continue to develop complete, intelligent solutions for our customers that empower people to stay productive and collaborate, while safeguarding businesses and simplifying IT management. Our goal is to lead the industry in several distinct areas of technology over the long term, which we expect will translate to sustained growth. We are investing significant resources in: - -� Transforming the workplace to deliver new modern, modular business applications, drive deeper insights, and improve how people communicate, collaborate, learn, work, play, and interact with one another. - -� Building and running cloud-based services in ways that unleash new experiences and opportunities for businesses and individuals. - -� Applying AI to drive insights and act on our customer�s behalf by understanding and interpreting their needs using natural methods of communication. - -� Tackling security from all angles with our integrated, end-to-end solutions spanning security, compliance, identity, and management, across all clouds and platforms. - -� Inventing new gaming experiences that bring people together around their shared love for games on any devices and pushing the boundaries of innovation with console and PC gaming by creating the next wave of entertainment. - -� Using Windows to fuel our cloud business, grow our share of the PC market, and drive increased engagement with our services like Microsoft 365 Consumer, Teams, Edge, Bing, Xbox Game Pass, and more. - -Our future growth depends on our ability to transcend current product category definitions, business models, and sales motions. We have the opportunity to redefine what customers and partners can expect and are working to deliver new solutions that reflect the best of Microsoft. - -Corporate Social Responsibility - -Commitment to Sustainability - -We work to ensure that technology is inclusive, trusted, and increases sustainability. We are accelerating progress toward a more sustainable future by reducing our environmental footprint, advancing research, helping our customers build sustainable solutions, and advocating for policies that benefit the environment. In January 2020, we announced a bold commitment and detailed plan to be carbon negative by 2030, and to remove from the environment by 2050 all the carbon we have emitted since our founding in 1975. This included a commitment to invest $1 billion over four years in new technologies and innovative climate solutions. We built on this pledge by adding commitments to be water positive by 2030, zero waste by 2030, and to protect ecosystems by developing a Planetary Computer. We also help our suppliers and customers around the world use Microsoft technology to reduce their own carbon footprint. - -Fiscal year 2021 was a year of both successes and challenges. While we continued to make progress on several of our goals, with an overall reduction in our combined Scope 1 and Scope 2 emissions, our Scope 3 emissions increased, due in substantial part to significant global datacenter expansions and growth in Xbox sales and usage as a result of the COVID-19 pandemic. Despite these Scope 3 increases, we will continue to build the foundations and do the work to deliver on our commitments, and help our customers and partners achieve theirs. We have learned the impact of our work will not all be felt immediately, and our experience highlights how progress won�t always be linear. - -6 - - -PART I -Item 1 - -While fiscal year 2021 presented us with some new learnings, we also made some great progress. A few examples that illuminate the diversity of our work include: - -� We purchased the removal of 1.4 million metrics tons of carbon. - -� Four of our datacenters received new or renewed Zero Waste certifications. - -� We granted $100 million to Breakthrough Energy Catalyst to accelerate the development of climate solutions the world needs to reach net-zero across four key areas: direct air capture, green hydrogen, long duration energy storage, and sustainable aviation fuel. - -� We joined the First Movers Coalition as an early leader and expert partner in the carbon dioxide removal sector, with a commitment of $200 million toward carbon removal by 2030. - -Sustainability is an existential priority for our society and businesses today. This led us to create our Microsoft Cloud for Sustainability, an entirely new business process category to help organizations monitor their carbon footprint across their operations. We also joined with leading organizations to launch the Carbon Call � an initiative to mobilize collective action to solve carbon emissions and removal accounting challenges for a net zero future. - -The investments we make in sustainability carry through to our products, services, and devices. We design our devices, from Surface to Xbox, to minimize their impact on the environment. Our cloud and AI services and datacenters help businesses cut energy consumption, reduce physical footprints, and design sustainable products. - -Addressing Racial Injustice and Inequity - -We are committed to addressing racial injustice and inequity in the United States for Black and African American communities and helping improve lived experiences at Microsoft, in employees� communities, and beyond. Our Racial Equity Initiative focuses on three multi-year pillars, each containing actions and progress we expect to make or exceed by 2025. - -� Strengthening our communities: using data, technology, and partnerships to help improve the lives of Black and African American people in the United States, including our employees and their communities. - -� Evolving our ecosystem: using our balance sheet and relationships with suppliers and partners to foster societal change and create new opportunities. - -� Increasing representation and strengthening inclusion: build on our momentum, adding a $150 million investment to strengthen inclusion and double the number of Black, African American, Hispanic, and Latinx leaders in the United States by 2025. - -Over the last year, we collaborated with partners and worked within neighborhoods and communities to launch and scale a number of projects and programs, including: working with 70 organizations in 145 communities on the Justice Reform Initiative, expanding access to affordable broadband and devices for Black and African American communities and key institutions that support them in major urban centers, expanding access to skills and education to support Black and African American students and adults to succeed in the digital economy, and increasing technology support for nonprofits that provide critical services to Black and African American communities. - -We have made meaningful progress on representation and inclusion at Microsoft. We are 90 percent of the way to our 2025 commitment to double the number of Black and African American people managers, senior individual contributors, and senior leaders in the U.S., and 50 percent of the way for Hispanic and Latinx people managers, senior individual contributors, and senior leaders in the U.S. - -We exceeded our goal on increasing the percentage of transaction volumes with Black- and African American-owned financial institutions and increased our deposits with Black- and African American-owned minority depository institutions, enabling increased funds into local communities. Additionally, we enriched our supplier pipeline, reaching more than 90 percent of our goal to spend $ 500 million with double the number of Black and African American -owned suppliers. We also increased the number of identified partners in the Black Partner Growth Initiative and continue to invest in the partner community through the Black Channel Partner Alliance by supporting events focused on business growth, accelerators, and mentorship. - -Progress does not undo the egregious injustices of the past or diminish those who continue to live with inequity. We are committed to leveraging our resources to help accelerate diversity and inclusion across our ecosystem and to hold ourselves accountable to accelerate change � for Microsoft, and beyond. - -7 - - -PART I -Item 1 - - -Investing in Digital Skills - -The COVID-19 pandemic led to record unemployment, disrupting livelihoods of people around the world. After helping over 30 million people in 249 countries and territories with our global skills initiative, we introduced a new initiative to support a more skills-based labor market, with greater flexibility and accessible learning paths to develop the right skills needed for the most in-demand jobs. Our skills initiative brings together learning resources, certification opportunities, and job-seeker tools from LinkedIn, GitHub, and Microsoft Learn, and is built on data insights drawn from LinkedIn�s Economic Graph. We previously invested $20 million in key non-profit partnerships through Microsoft Philanthropies to help people from underserved communities that are often excluded by the digital economy. - -We also launched a national campaign with U.S. community colleges to help skill and recruit into the cybersecurity workforce 250,000 people by 2025, representing half of the country�s workforce shortage. To that end, we are making curriculum available free of charge to all of the nation�s public community colleges, providing training for new and existing faculty at 150 community colleges, and providing scholarships and supplemental resources to 25,000 students. - -HUMAN CAPITAL RESOURCES - -Overview - -Microsoft aims to recruit, develop, and retain world-changing talent from a diversity of backgrounds. To foster their and our success, we seek to create an environment where people can thrive, where they can do their best work, where they can proudly be their authentic selves, guided by our values, and where they know their needs can be met. We strive to maximize the potential of our human capital resources by creating a respectful, rewarding, and inclusive work environment that enables our global employees to create products and services that further our mission to empower every person and every organization on the planet to achieve more. - -As of June 30, 2022, we employed approximately 221,000 people on a full-time basis, 122,000 in the U.S. and 99,000 internationally. Of the total employed people, 85,000 were in operations, including manufacturing, distribution, product support, and consulting services; 73,000 were in product research and development; 47,000 were in sales and marketing; and 16,000 were in general and administration. Certain employees are subject to collective bargaining agreements. - -Our Culture - -Microsoft�s culture is grounded in the growth mindset. This means everyone is on a continuous journey to learn and grow. We believe potential can be nurtured and is not pre-determined, and we should always be learning and curious � trying new things without fear of failure. We identified four attributes that allow growth mindset to flourish: - -� Obsessing over what matters to our customers. - -� Becoming more diverse and inclusive in everything we do. - -� Operating as one company, One Microsoft, instead of multiple siloed businesses. - -� Making a difference in the lives of each other, our customers, and the world around us. - -Our employee listening systems enable us to gather feedback directly from our workforce to inform our programs and employee needs globally. Seventy percent of employees globally participated in our fiscal year 2022 Employee Signals survey, which covers a variety of topics such as thriving, inclusion, team culture, wellbeing, and learning and development. Throughout the fiscal year, we collect over 75,000 Daily Pulse employee survey responses. During fiscal year 2022, our Daily Pulse surveys gave us invaluable insights into ways we could support employees through the COVID-19 pandemic, addressing racial injustice, the war in Ukraine, and their general wellbeing. In addition to Employee Signals and Daily Pulse surveys, we gain insights through onboarding, internal mobility, leadership, performance and development, exit surveys, internal Yammer channels, employee Q&A sessions, and AskHR Service support. - -8 - - -PART I -Item 1 - - -Diversity and Inclusion - -At Microsoft we have an inherently inclusive mission: to empower every person and every organization on the planet to achieve more. We think of diversity and inclusion as core to our business model, informing our actions to impact economies and people around the world. There are billions of people who want to achieve more, but have a different set of circumstances, abilities, and backgrounds that often limit access to opportunity and achievement. The better we represent that diversity inside Microsoft, the more effectively we can innovate for those we seek to empower. - -We strive to include others by holding ourselves accountable for diversity, driving global systemic change in our workplace and workforce, and creating an inclusive work environment. Through this commitment we can allow everyone the chance to be their authentic selves and do their best work every day. We support multiple highly active Employee Resource Groups for women, families, racial and ethnic minorities, military, people with disabilities, and employees who identify as LGBTQIA+, where employees can go for support, networking, and community-building. As described in our 2021 Proxy Statement, annual performance and compensation reviews of our senior leadership team include an evaluation of their contributions to employee culture and diversity. To ensure accountability over time, we publicly disclose our progress on a multitude of workforce metrics including: - -� Detailed breakdowns of gender, racial, and ethnic minority representation in our employee population, with data by job types, levels, and segments of our business. - -� Our EEO-1 report (equal employment opportunity). - -� Disability representation. - -� Pay equity (see details below). - -Total Rewards - -We develop dynamic, sustainable, market-driven, and strategic programs with the goal of providing a highly differentiated portfolio to attract, reward, and retain top talent and enable our employees to thrive. These programs reinforce our culture and values such as collaboration and growth mindset. Managers evaluate and recommend rewards based on, for example, how well we leverage the work of others and contribute to the success of our colleagues. We monitor pay equity and career progress across multiple dimensions. - -As part of our effort to promote a One Microsoft and inclusive culture, in fiscal year 2021 we expanded stock eligibility to all Microsoft employees as part of our annual rewards process. This includes all non-exempt and exempt employees and equivalents across the globe including business support professionals and datacenter and retail employees. In response to the Great Reshuffle, in fiscal year 2022 we announced a sizable investment in annual merit and annual stock award opportunity for all employees below senior executive levels. We also invested in base salary adjustments for our datacenter and retail hourly employees and hourly equivalents outside the U.S. These investments have supported retention and help to ensure that Microsoft remains an employer of choice. - -Pay Equity - -In our 2021 Diversity and Inclusion Report, we reported that all racial and ethnic minority employees in the U.S. combined earn $1.006 for every $1.000 earned by their white counterparts, that women in the U.S. earn $1.002 for every $1.000 earned by their counterparts in the U.S. who are men, and women in the U.S. plus our twelve other largest employee geographies representing 86.6% of our global population (Australia, Canada, China, France, Germany, India, Ireland, Israel, Japan, Romania, Singapore, and the United Kingdom) combined earn $1.001 for every $1.000 by men in these countries. Our intended result is a global performance and development approach that fosters our culture, and competitive compensation that ensures equitable pay by role while supporting pay for performance. - -Wellness and Safety - -Microsoft is committed to supporting our employees� well-being and safety while they are at work and in their personal lives. - -We took a wide variety of measures to protect the health and well-being of our employees, suppliers, and customers during the COVID-19 pandemic and are now supporting employees in shifting to return to office and/or hybrid arrangements. We developed hybrid guidelines for managers and employees to support the transition and continue to identify ways we can support hybrid work scenarios through our employee listening systems. - -9 - - -PART I -Item 1 - -We have invested significantly in holistic wellbeing, and offer a differentiated benefits package which includes many physical, emotional, and financial wellness programs including counseling through the Microsoft CARES Employee Assistance Program, mental wellbeing support, flexible fitness benefits, savings and investment tools, adoption assistance, and back-up care for children and elders. Finally, our Occupational Health and Safety program helps ensure employees can stay safe while they are working. - -We continue to strive to support our Ukrainian employees and their dependents during the Ukraine crisis with emergency relocation assistance, emergency leave, and other benefits. - -Learning and Development - -Our growth mindset culture begins with valuing learning over knowing � seeking out new ideas, driving innovation, embracing challenges, learning from failure, and improving over time. To support this culture, we offer a wide range of learning and development opportunities. We believe learning can be more than formal instruction, and our learning philosophy focuses on providing the right learning, at the right time, in the right way. Opportunities include: - -� Personalized, integrated, and relevant views of all learning opportunities on both our internal learning portal Learning (Viva Learning + LinkedIn Learning) and our external learning portal MS Learn are available to all employees worldwide. - -� In-the-classroom learning, learning cohorts, our early-in-career Aspire program, and manager excellence communities. - -� Required learning for all employees and managers on topics such as compliance, regulation, company culture, leadership, and management. This includes the annual Standards of Business Conduct training. - -� On-the-job �stretch� and advancement opportunities. - -� Managers holding conversations about employees� career and development plans, coaching on career opportunities, and programs like mentoring and sponsorship. - -� Customized manager learning to build people manager capabilities and similar learning solutions to build leadership skills for all employees including differentiated leadership development programs. - -� New employee orientation covering a range of topics including company values, and culture, as well as ongoing onboarding programs. - -� New tools to assist managers and employees in learning how to operate, be productive, and connect in the new flexible hybrid world of work. These include quick guides for teams to use, such as Creating Team Agreements, Reconnecting as a Team, and Running Effective Hybrid Meetings. - -Our employees embrace the growth mindset and take advantage of the formal learning opportunities as well as thousands of informal and on-the-job learning opportunities. In terms of formal on-line learning solutions, in fiscal year 2022 our employees completed over 4.7 million courses, averaging over 14 hours per employee. Given our focus on understanding core company beliefs and compliance topics, all employees complete required learning programs like Standards of Business Conduct, Privacy, Unconscious Bias, and preventing harassment courses. Our corporate learning portal has over 100,000 average monthly active users. We have over 27,000 people managers, all of whom must complete between 20-33 hours of required manager capability and excellence training and are assigned ongoing required training each year. In addition, all employees complete skills training based on the profession they are in each year. - -New Ways of Working - -The COVID-19 pandemic accelerated our capabilities and culture with respect to flexible work. We introduced a Hybrid Workplace Flexibility Guide to better support managers and employees as they adapt to new ways of working that shift paradigms, embrace flexibility, promote inclusion, and drive innovation. Our ongoing survey data shows employees value the flexibility related to work location, work site, and work hours, and while many have begun returning to worksites as conditions have permitted, they also continue to adjust hours and/or spend some of workweeks working at home, another site, or remotely. We are focused on building capabilities to support a variety of workstyles where individuals, teams, and our business can deliver success. - -10 - - -PART I -Item 1 - - -OPERATING SEGMENTS - -We operate our business and report our financial performance using three segments: Productivity and Business Processes, Intelligent Cloud, and More Personal Computing. Our segments provide management with a comprehensive financial view of our key businesses. The segments enable the alignment of strategies and objectives across the development, sales, marketing, and services organizations, and they provide a framework for timely and rational allocation of resources within businesses. - -Additional information on our operating segments and geographic and product information is contained in Note 19 � Segment Information and Geographic Data of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K). - -Our reportable segments are described below. - -Productivity and Business Processes - -Our Productivity and Business Processes segment consists of products and services in our portfolio of productivity, communication, and information services, spanning a variety of devices and platforms. This segment primarily comprises: - -� Office Commercial (Office 365 subscriptions, the Office 365 portion of Microsoft 365 Commercial subscriptions, and Office licensed on-premises), comprising Office, Exchange, SharePoint, Microsoft Teams, Office 365 Security and Compliance, and Microsoft Viva. - -� Office Consumer, including Microsoft 365 Consumer subscriptions, Office licensed on-premises, and other Office services. - -� LinkedIn, including Talent Solutions, Marketing Solutions, Premium Subscriptions, and Sales Solutions. - -� Dynamics business solutions, including Dynamics 365, comprising a set of intelligent, cloud-based applications across ERP, CRM, Customer Insights, Power Apps, and Power Automate; and on-premises ERP and CRM applications. - -Office Commercial - -Office Commercial is designed to increase personal, team, and organizational productivity through a range of products and services. Growth depends on our ability to reach new users in new markets such as frontline workers, small and medium businesses, and growth markets, as well as add value to our core product and service offerings to span productivity categories such as communication, collaboration, analytics, security, and compliance. Office Commercial revenue is mainly affected by a combination of continued installed base growth and average revenue per user expansion, as well as the continued shift from Office licensed on-premises to Office 365. - -Office Consumer - -Office Consumer is designed to increase personal productivity through a range of products and services. Growth depends on our ability to reach new users, add value to our core product set, and continue to expand our product and service offerings into new markets. Office Consumer revenue is mainly affected by the percentage of customers that buy Office with their new devices and the continued shift from Office licensed on-premises to Microsoft 365 Consumer subscriptions. Office Consumer Services revenue is mainly affected by the demand for communication and storage through Skype, Outlook.com, and OneDrive, which is largely driven by subscriptions, advertising, and the sale of minutes. - -11 - - -PART I -Item 1 - - -LinkedIn - -LinkedIn connects the world�s professionals to make them more productive and successful and transforms the way companies hire, market, sell, and learn. Our vision is to create economic opportunity for every member of the global workforce through the ongoing development of the world�s first Economic Graph, a digital representation of the global economy. In addition to LinkedIn�s free services, LinkedIn offers monetized solutions: Talent Solutions, Marketing Solutions, Premium Subscriptions, and Sales Solutions. Talent Solutions provide insights for workforce planning and tools to hire, nurture, and develop talent. Talent Solutions also includes Learning Solutions, which help businesses close critical skills gaps in times where companies are having to do more with existing talent. Marketing Solutions help companies reach, engage, and convert their audiences at scale. Premium Subscriptions enables professionals to manage their professional identity, grow their network, and connect with talent through additional services like premium search. Sales Solutions help companies strengthen customer relationships, empower teams with digital selling tools, and acquire new opportunities. LinkedIn has over 850 million members and has offices around the globe. Growth will depend on our ability to increase the number of LinkedIn members and our ability to continue offering services that provide value for our members and increase their engagement. LinkedIn revenue is mainly affected by demand from enterprises and professional organizations for subscriptions to Talent Solutions, Sales Solutions, and Premium Subscriptions offerings, as well as member engagement and the quality of the sponsored content delivered to those members to drive Marketing Solutions. - -Dynamics - -Dynamics provides cloud-based and on-premises business solutions for financial management, enterprise resource planning (�ERP�), customer relationship management (�CRM�), supply chain management, and other application development platforms for small and medium businesses, large organizations, and divisions of global enterprises. Dynamics revenue is driven by the number of users licensed and applications consumed, expansion of average revenue per user, and the continued shift to Dynamics 365, a unified set of cloud-based intelligent business applications, including Power Apps and Power Automate. - -Competition - -Competitors to Office include software and global application vendors, such as Apple, Cisco Systems, Meta, Google, IBM, Okta, Proofpoint, Slack, Symantec, Zoom, and numerous web-based and mobile application competitors as well as local application developers. Apple distributes versions of its pre -installed application software, such as email and calendar products, through its PCs, tablets, and phones. Cisco Systems is using its position in enterprise communications equipment to grow its unified communications business. Google provides a hosted messaging and productivity suite. Slack provides teamwork and collaboration software. Zoom offers videoconferencing and cloud phone solutions. Okta, Proofpoint, and Symantec provide security solutions across email security, information protection, identity, and governance. Web-based offerings competing with individual applications have also positioned themselves as alternatives to our products and services. We compete by providing powerful, flexible, secure, integrated industry-specific, and easy-to-use productivity and collaboration tools and services that create comprehensive solutions and work well with technologies our customers already have both on-premises or in the cloud. - -LinkedIn faces competition from online professional networks, recruiting companies, talent management companies, and larger companies that are focusing on talent management and human resource services; job boards; traditional recruiting firms; and companies that provide learning and development products and services. Marketing Solutions competes with online and offline outlets that generate revenue from advertisers and marketers, and Sales Solutions competes with online and offline outlets for companies with lead generation and customer intelligence and insights. - -Dynamics competes with cloud-based and on-premises business solution providers such as Oracle, Salesforce, and SAP. - -12 - - -PART I -Item 1 - - -Intelligent Cloud - -Our Intelligent Cloud segment consists of our public, private, and hybrid server products and cloud services that can power modern business and developers. This segment primarily comprises: - -� Server products and cloud services, including Azure and other cloud services; SQL Server, Windows Server, Visual Studio, System Center, and related Client Access Licenses (�CALs�); and Nuance and GitHub. - -� Enterprise Services, including Enterprise Support Services, Microsoft Consulting Services, and Nuance professional services. - -Server Products and Cloud Services - -Azure is a comprehensive set of cloud services that offer developers, IT professionals, and enterprises freedom to build, deploy, and manage applications on any platform or device. Customers can use Azure through our global network of datacenters for computing, networking, storage, mobile and web application services, AI, IoT, cognitive services, and machine learning. Azure enables customers to devote more resources to development and use of applications that benefit their organizations, rather than managing on-premises hardware and software. Azure revenue is mainly affected by infrastructure-as-a-service and platform-as-a-service consumption-based services, and per user-based services such as Enterprise Mobility + Security. - -Our server products are designed to make IT professionals, developers, and their systems more productive and efficient. Server software is integrated server infrastructure and middleware designed to support software applications built on the Windows Server operating system. This includes the server platform, database, business intelligence, storage, management and operations, virtualization, service-oriented architecture platform, security, and identity software. We also license standalone and software development lifecycle tools for software architects, developers, testers, and project managers. GitHub provides a collaboration platform and code hosting service for developers. Server products revenue is mainly affected by purchases through volume licensing programs, licenses sold to original equipment manufacturers (�OEM�), and retail packaged products. CALs provide access rights to certain server products, including SQL Server and Windows Server, and revenue is reported along with the associated server product. - -Nuance and GitHub include both cloud and on-premises offerings. Nuance provides healthcare and enterprise AI solutions. GitHub provides a collaboration platform and code hosting service for developers. - -Enterprise Services - -Enterprise Services, including Enterprise Support Services, Microsoft Consulting Services, and Nuance Professional Services, assist customers in developing, deploying, and managing Microsoft server solutions, Microsoft desktop solutions, and Nuance conversational AI and ambient intelligent solutions, along with providing training and certification to developers and IT professionals on various Microsoft products. - -Competition - -Azure faces diverse competition from companies such as Amazon, Google, IBM, Oracle, VMware, and open source offerings. Our Enterprise Mobility + Security offerings also compete with products from a range of competitors including identity vendors, security solution vendors, and numerous other security point solution vendors. Azure�s competitive advantage includes enabling a hybrid cloud, allowing deployment of existing datacenters with our public cloud into a single, cohesive infrastructure, and the ability to run at a scale that meets the needs of businesses of all sizes and complexities. We believe our cloud�s global scale, coupled with our broad portfolio of identity and security solutions, allows us to effectively solve complex cybersecurity challenges for our customers and differentiates us from the competition. - -Our server products face competition from a wide variety of server operating systems and applications offered by companies with a range of market approaches. Vertically integrated computer manufacturers such as Hewlett-Packard, IBM, and Oracle offer their own versions of the Unix operating system preinstalled on server hardware. Nearly all computer manufacturers offer server hardware for the Linux operating system and many contribute to Linux operating system development. The competitive position of Linux has also benefited from the large number of compatible applications now produced by many commercial and non-commercial software developers. A number of companies, such as Red Hat, supply versions of Linux. - -13 - - -PART I -Item 1 - -We compete to provide enterprise-wide computing solutions and point solutions with numerous commercial software vendors that offer solutions and middleware technology platforms, software applications for connectivity (both Internet and intranet), security, hosting, database, and e-business servers. IBM and Oracle lead a group of companies focused on the Java Platform Enterprise Edition that competes with our enterprise-wide computing solutions. Commercial competitors for our server applications for PC-based distributed client-server environments include CA Technologies, IBM, and Oracle. Our web application platform software competes with open source software such as Apache, Linux, MySQL, and PHP. In middleware, we compete against Java vendors. - -Our database, business intelligence, and data warehousing solutions offerings compete with products from IBM, Oracle, SAP, Snowflake, and other companies. Our system management solutions compete with server management and server virtualization platform providers, such as BMC, CA Technologies, Hewlett-Packard, IBM, and VMware. Our products for software developers compete against offerings from Adobe, IBM, Oracle, and other companies, and also against open-source projects, including Eclipse (sponsored by CA Technologies, IBM, Oracle, and SAP), PHP, and Ruby on Rails. - -We believe our server products provide customers with advantages in performance, total costs of ownership, and productivity by delivering superior applications, development tools, compatibility with a broad base of hardware and software applications, security, and manageability. - -Our Enterprise Services business competes with a wide range of companies that provide strategy and business planning, application development, and infrastructure services, including multinational consulting firms and small niche businesses focused on specific technologies. - -More Personal Computing - -Our More Personal Computing segment consists of products and services that put customers at the center of the experience with our technology. This segment primarily comprises: - -� Windows, including Windows OEM licensing (�Windows OEM�) and other non-volume licensing of the Windows operating system; Windows Commercial, comprising volume licensing of the Windows operating system, Windows cloud services, and other Windows commercial offerings; patent licensing; and Windows Internet of Things. - -� Devices, including Surface and PC accessories. - -� Gaming, including Xbox hardware and Xbox content and services, comprising first- and third-party content (including games and in-game content), Xbox Game Pass and other subscriptions, Xbox Cloud Gaming, third-party disc royalties, advertising, and other cloud services. - -� Search and news advertising. - -Windows - -The Windows operating system is designed to deliver a more personal computing experience for users by enabling consistency of experience, applications, and information across their devices. Windows OEM revenue is impacted significantly by the number of Windows operating system licenses purchased by OEMs, which they pre-install on the devices they sell. In addition to computing device market volume, Windows OEM revenue is impacted by: - -� The mix of computing devices based on form factor and screen size. - -� Differences in device market demand between developed markets and growth markets. - -� Attachment of Windows to devices shipped. - -� Customer mix between consumer, small and medium businesses, and large enterprises. - -� Changes in inventory levels in the OEM channel. - -� Pricing changes and promotions, pricing variation that occurs when the mix of devices manufactured shifts from local and regional system builders to large multinational OEMs, and different pricing of Windows versions licensed. - -� Constraints in the supply chain of device components. - -� Piracy. - -14 - - -PART I -Item 1 - - -Windows Commercial revenue, which includes volume licensing of the Windows operating system and Windows cloud services such as Microsoft Defender for Endpoint, is affected mainly by the demand from commercial customers for volume licensing and Software Assurance (�SA�), as well as advanced security offerings. Windows Commercial revenue often reflects the number of information workers in a licensed enterprise and is relatively independent of the number of PCs sold in a given year. - -Patent licensing includes our programs to license patents we own for use across a broad array of technology areas, including mobile devices and cloud offerings. - -Windows IoT extends the power of Windows and the cloud to intelligent systems by delivering specialized operating systems, tools, and services for use in embedded devices. - -Devices - -We design and sell devices, including Surface and PC accessories. Our devices are designed to enable people and organizations to connect to the people and content that matter most using Windows and integrated Microsoft products and services. Surface is designed to help organizations, students, and consumers be more productive. Growth in Devices is dependent on total PC shipments, the ability to attract new customers, our product roadmap, and expanding into new categories. - -Gaming - -Our gaming platform is designed to provide a variety of entertainment through a unique combination of content, community, and cloud. Our exclusive game content is created through Xbox Game Studios, a collection of first-party studios creating iconic and differentiated gaming experiences. We continue to invest in new gaming studios and content to expand our IP roadmap and leverage new content creators. These unique gaming experiences are the cornerstone of Xbox Game Pass, a subscription service and gaming community with access to a curated library of over 100 first- and third-party console and PC titles. - -The gamer remains at the heart of the Xbox ecosystem. We continue to open new opportunities for gamers to engage both on- and off-console with both the launch of Xbox Cloud Gaming, our game streaming service, and continued investment in gaming hardware. Xbox Cloud Gaming utilizes Microsoft�s Azure cloud technology to allow direct and on-demand streaming of games to PCs, consoles, and mobile devices, enabling gamers to take their favorite games with them and play on the device most convenient to them. - -Xbox enables people to connect and share online gaming experiences that are accessible on Xbox consoles, Windows-enabled devices, and other devices. Xbox is designed to benefit users by providing access to a network of certified applications and services and to benefit our developer and partner ecosystems by providing access to a large customer base. Xbox revenue is mainly affected by subscriptions and sales of first- and third-party content, as well as advertising. Growth of our Gaming business is determined by the overall active user base through Xbox enabled content, availability of games, providing exclusive game content that gamers seek, the computational power and reliability of the devices used to access our content and services, and the ability to create new experiences through first-party content creators. - -Search and News Advertising - -Our Search and news advertising business is designed to deliver relevant search, native, and display advertising to a global audience. We have several partnerships with other companies, including Yahoo, through which we provide and monetize search queries. Growth depends on our ability to attract new users, understand intent, and match intent with relevant content and advertiser offerings. - -On June 6, 2022, we acquired Xandr, Inc., a technology platform with tools to accelerate the delivery of our digital advertising solutions. - -Competition - -Windows faces competition from various software products and from alternative platforms and devices, mainly from Apple and Google. We believe Windows competes effectively by giving customers choice, value, flexibility, security, an easy-to-use interface, and compatibility with a broad range of hardware and software applications, including those that enable productivity. - -15 - - -PART I -Item 1 - -Devices face competition from various computer, tablet, and hardware manufacturers who offer a unique combination of high-quality industrial design and innovative technologies across various price points. These manufacturers, many of which are also current or potential partners and customers, include Apple and our Windows OEMs. - -Xbox and our cloud gaming services face competition from various online gaming ecosystems and game streaming services, including those operated by Amazon, Apple, Meta, Google, and Tencent. We also compete with other providers of entertainment services such as video streaming platforms. Our gaming platform competes with console platforms from Nintendo and Sony, both of which have a large, established base of customers. We believe our gaming platform is effectively positioned against, and uniquely differentiated from, competitive products and services based on significant innovation in hardware architecture, user interface, developer tools, online gaming and entertainment services, and continued strong exclusive content from our own first-party game franchises as well as other digital content offerings. - -Our Search and news advertising business competes with Google and a wide array of websites, social platforms like Meta, and portals that provide content and online offerings to end users. - -OPERATIONS - -We have operations centers that support operations in their regions, including customer contract and order processing, credit and collections, information processing, and vendor management and logistics. The regional center in Ireland supports the European, Middle Eastern, and African region; the center in Singapore supports the Japan, India, Greater China, and Asia-Pacific region; and the centers in Fargo, North Dakota, Fort Lauderdale, Florida, Puerto Rico, Redmond, Washington, and Reno, Nevada support Latin America and North America. In addition to the operations centers, we also operate datacenters throughout the Americas, Europe, Australia, and Asia, as well as in the Middle East and Africa. - -To serve the needs of customers around the world and to improve the quality and usability of products in international markets, we localize many of our products to reflect local languages and conventions. Localizing a product may require modifying the user interface, altering dialog boxes, and translating text. - -Our devices are primarily manufactured by third-party contract manufacturers. For the majority of our products, we have the ability to use other manufacturers if a current vendor becomes unavailable or unable to meet our requirements. However, some of our products contain certain components for which there are very few qualified suppliers. For these components, we have limited near-term flexibility to use other manufacturers if a current vendor becomes unavailable or is unable to meet our requirements. Extended disruptions at these suppliers and/or manufacturers could lead to a similar disruption in our ability to manufacture devices on time to meet consumer demand. - -RESEARCH AND DEVELOPMENT - -Product and Service Development, and Intellectual Property - -We develop most of our products and services internally through the following engineering groups. - -� Cloud and AI, focuses on making IT professionals, developers, and their systems more productive and efficient through development of cloud infrastructure, server, database, CRM, ERP, software development tools and services (including GitHub), AI cognitive services, and other business process applications and services for enterprises. - -� Experiences and Devices, focuses on instilling a unifying product ethos across our end-user experiences and devices, including Office, Windows, Teams, consumer web experiences (including search and news advertising), and the Surface line of devices. - -� Security, Compliance, Identity, and Management, focuses on cloud platform and application security, identity and network access, enterprise mobility, information protection, and managed services. - -� Technology and Research, focuses on our AI innovations and other forward-looking research and development efforts spanning infrastructure, services, and applications. - -� LinkedIn, focuses on our services that transform the way customers hire, market, sell, and learn. - -� Gaming, focuses on developing hardware, content, and services across a large range of platforms to help grow our user base through game experiences and social interaction. - -16 - - -PART I -Item 1 - -Internal development allows us to maintain competitive advantages that come from product differentiation and closer technical control over our products and services. It also gives us the freedom to decide which modifications and enhancements are most important and when they should be implemented. We strive to obtain information as early as possible about changing usage patterns and hardware advances that may affect software and hardware design. Before releasing new software platforms, and as we make significant modifications to existing platforms, we provide application vendors with a range of resources and guidelines for development, training, and testing. Generally, we also create product documentation internally. - -We protect our intellectual property investments in a variety of ways. We work actively in the U.S. and internationally to ensure the enforcement of copyright, trademark, trade secret, and other protections that apply to our software and hardware products, services, business plans, and branding. We are a leader among technology companies in pursuing patents and currently have a portfolio of over 69,000 U.S. and international patents issued and over 19,000 pending worldwide. While we employ much of our internally-developed intellectual property exclusively in our products and services, we also engage in outbound licensing of specific patented technologies that are incorporated into licensees� products. From time to time, we enter into broader cross-license agreements with other technology companies covering entire groups of patents. We may also purchase or license technology that we incorporate into our products and services. At times, we make select intellectual property broadly available at no or low cost to achieve a strategic objective, such as promoting industry standards, advancing interoperability, supporting societal and/or environmental efforts, or attracting and enabling our external development community. Our increasing engagement with open source software will also cause us to license our intellectual property rights broadly in certain situations. - -While it may be necessary in the future to seek or renew licenses relating to various aspects of our products, services, and business methods, we believe, based upon past experience and industry practice, such licenses generally can be obtained on commercially reasonable terms. We believe our continuing research and product development are not materially dependent on any single license or other agreement with a third party relating to the development of our products. - -Investing in the Future - -Our success is based on our ability to create new and compelling products, services, and experiences for our users, to initiate and embrace disruptive technology trends, to enter new geographic and product markets, and to drive broad adoption of our products and services. We invest in a range of emerging technology trends and breakthroughs that we believe offer significant opportunities to deliver value to our customers and growth for the Company. Based on our assessment of key technology trends, we maintain our long-term commitment to research and development across a wide spectrum of technologies, tools, and platforms spanning digital work and life experiences, cloud computing, AI, devices, and operating systems. - -While our main product research and development facilities are located in Redmond, Washington, we also operate research and development facilities in other parts of the U.S. and around the world. This global approach helps us remain competitive in local markets and enables us to continue to attract top talent from across the world. - -We plan to continue to make significant investments in a broad range of product research and development activities, and as appropriate we will coordinate our research and development across operating segments and leverage the results across the Company. - -In addition to our main research and development operations, we also operate Microsoft Research. Microsoft Research is one of the world�s largest corporate research organizations and works in close collaboration with top universities around the world to advance the state-of-the-art in computer science and a broad range of other disciplines, providing us a unique perspective on future trends and contributing to our innovation. - -DISTRIBUTION, SALES, AND MARKETING - -We market and distribute our products and services through the following channels: OEMs, direct, and distributors and resellers. Our sales force performs a variety of functions, including working directly with commercial enterprises and public-sector organizations worldwide to identify and meet their technology and digital transformation requirements; managing OEM relationships; and supporting system integrators, independent software vendors, and other partners who engage directly with our customers to perform sales, consulting, and fulfillment functions for our products and services. - -17 - - -PART I -Item 1 - - -OEMs - -We distribute our products and services through OEMs that pre-install our software on new devices and servers they sell. The largest component of the OEM business is the Windows operating system pre-installed on devices. OEMs also sell devices pre-installed with other Microsoft products and services, including applications such as Office and the capability to subscribe to Office 365. - -There are two broad categories of OEMs. The largest category of OEMs are direct OEMs as our relationship with them is managed through a direct agreement between Microsoft and the OEM. We have distribution agreements covering one or more of our products with virtually all the multinational OEMs, including Dell, Hewlett-Packard, Lenovo, and with many regional and local OEMs. The second broad category of OEMs are system builders consisting of lower-volume PC manufacturers, which source Microsoft software for pre-installation and local redistribution primarily through the Microsoft distributor channel rather than through a direct agreement or relationship with Microsoft. - -Direct - -Many organizations that license our products and services transact directly with us through Enterprise Agreements and Enterprise Services contracts, with sales support from system integrators, independent software vendors, web agencies, and partners that advise organizations on licensing our products and services (�Enterprise Agreement Software Advisors� or �ESA�). Microsoft offers direct sales programs targeted to reach small, medium, and corporate customers, in addition to those offered through the reseller channel. A large network of partner advisors support many of these sales. - -We also sell commercial and consumer products and services directly to customers, such as cloud services, search, and gaming, through our digital marketplaces and online stores. In fiscal year 2021, we closed our Microsoft Store physical locations and opened our Microsoft Experience Centers. Microsoft Experience Centers are designed to facilitate deeper engagement with our partners and customers across industries. - -Distributors and Resellers - -Organizations also license our products and services indirectly, primarily through licensing solution partners (�LSP�), distributors, value-added resellers (�VAR�), and retailers. Although each type of reselling partner may reach organizations of all sizes, LSPs are primarily engaged with large organizations, distributors resell primarily to VARs, and VARs typically reach small and medium organizations. ESAs are also typically authorized as LSPs and operate as resellers for our other volume licensing programs. Microsoft Cloud Solution Provider is our main partner program for reselling cloud services. - -We distribute our retail packaged products primarily through independent non-exclusive distributors, authorized replicators, resellers, and retail outlets. Individual consumers obtain these products primarily through retail outlets. We distribute our devices through third-party retailers. We have a network of field sales representatives and field support personnel that solicit orders from distributors and resellers and provide product training and sales support. - -Our Dynamics business solutions are also licensed to enterprises through a global network of channel partners providing vertical solutions and specialized services. - -LICENSING OPTIONS - -We offer options for organizations that want to purchase our cloud services, on-premises software, and SA. We license software to organizations under volume licensing agreements to allow the customer to acquire multiple licenses of products and services instead of having to acquire separate licenses through retail channels. We use different programs designed to provide flexibility for organizations of various sizes. While these programs may differ in various parts of the world, generally they include those discussed below. - -SA conveys rights to new software and upgrades for perpetual licenses released over the contract period. It also provides support, tools, training, and other licensing benefits to help customers deploy and use software efficiently. SA is included with certain volume licensing agreements and is an optional purchase with others. - -18 - - -PART I -Item 1 - - -Volume Licensing Programs - -Enterprise Agreement - -Enterprise Agreements offer large organizations a manageable volume licensing program that gives them the flexibility to buy cloud services and software licenses under one agreement. Enterprise Agreements are designed for medium or large organizations that want to license cloud services and on-premises software organization-wide over a three-year period. Organizations can elect to purchase perpetual licenses or subscribe to licenses. SA is included. - -Microsoft Customer Agreement - -A Microsoft Customer Agreement is a simplified purchase agreement presented, accepted, and stored through a digital experience. A Microsoft Customer Agreement is a non-expiring agreement that is designed to support all customers over time, whether purchasing through a partner or directly from Microsoft. - -Microsoft Online Subscription Agreement - -A Microsoft Online Subscription Agreement is designed for small and medium organizations that want to subscribe to, activate, provision, and maintain cloud services seamlessly and directly via the web. The agreement allows customers to acquire monthly or annual subscriptions for cloud-based services. - -Microsoft Products and Services Agreement - -Microsoft Products and Services Agreements are designed for medium and large organizations that want to license cloud services and on-premises software as needed, with no organization-wide commitment, under a single, non-expiring agreement. Organizations purchase perpetual licenses or subscribe to licenses. SA is optional for customers that purchase perpetual licenses. - -Open Value - -Open Value agreements are a simple, cost-effective way to acquire the latest Microsoft technology. These agreements are designed for small and medium organizations that want to license cloud services and on-premises software over a three-year period. Under Open Value agreements, organizations can elect to purchase perpetual licenses or subscribe to licenses and SA is included. - -Select Plus - -A Select Plus agreement is designed for government and academic organizations to acquire on-premises licenses at any affiliate or department level, while realizing advantages as one organization. Organizations purchase perpetual licenses and SA is optional. - -Partner Programs - -The Microsoft Cloud Solution Provider program offers customers an easy way to license the cloud services they need in combination with the value-added services offered by their systems integrator, managed services provider, or cloud reseller partner. Partners in this program can easily package their own products and services to directly provision, manage, and support their customer subscriptions. - -The Microsoft Services Provider License Agreement allows hosting service providers and independent software vendors who want to license eligible Microsoft software products to provide software services and hosted applications to their end customers. Partners license software over a three-year period and are billed monthly based on consumption. - -The Independent Software Vendor Royalty program enables partners to integrate Microsoft products into other applications and then license the unified business solution to their end users. - -19 - - -PART I -Item 1 - - -CUSTOMERS - -Our customers include individual consumers, small and medium organizations, large global enterprises, public-sector institutions, Internet service providers, application developers, and OEMs. Our practice is to ship our products promptly upon receipt of purchase orders from customers; consequently, backlog is not significant. - -20 - - -PART I -Item 1 - - INFORMATION ABOUT OUR EXECUTIVE OFFICERS Our executive officers as of July 28, 2022 were as follows: - -Name -Age -Position with the Company - - - -Satya Nadella -54 -Chairman of the Board and Chief Executive Officer -Judson Althoff -49 -Executive Vice President and Chief Commercial Officer -Christopher C. Capossela -52 -Executive Vice President, Marketing and Consumer Business, and Chief Marketing Officer -Kathleen T. Hogan -56 -Executive Vice President, Human Resources -Amy E. Hood -50 -Executive Vice President, Chief Financial Officer -Bradford L. Smith -63 -President and Vice Chair -Christopher D. Young -50 -Executive Vice President, Business Development, Strategy, and Ventures - - - - -Mr. Nadella was appointed Chairman of the Board in June 2021 and Chief Executive Officer in February 2014. He served as Executive Vice President, Cloud and Enterprise from July 2013 until that time. From 2011 to 2013, Mr. Nadella served as President, Server and Tools. From 2009 to 2011, he was Senior Vice President, Online Services Division. From 2008 to 2009, he was Senior Vice President, Search, Portal, and Advertising. Since joining Microsoft in 1992, Mr. Nadella�s roles also included Vice President of the Business Division. Mr. Nadella also serves on the Board of Directors of Starbucks Corporation. - -Mr. Althoff was appointed Executive Vice President and Chief Commercial Officer in July 2021. He served as Executive Vice President, Worldwide Commercial Business from July 2017 until that time. Prior to that, Mr. Althoff served as the President of Microsoft North America. Mr. Althoff joined Microsoft in March 2013 as President of Microsoft North America. - -Mr. Capossela was appointed Executive Vice President, Marketing and Consumer Business, and Chief Marketing Officer in July 2016. He had served as Executive Vice President, Chief Marketing Officer since March 2014. Previously, he served as the worldwide leader of the Consumer Channels Group, responsible for sales and marketing activities with OEMs, operators, and retail partners. In his more than 25 years at Microsoft, Mr. Capossela has held a variety of marketing leadership roles in the Microsoft Office Division. He was responsible for marketing productivity solutions including Microsoft Office, Office 365, SharePoint, Exchange, Skype for Business, Project, and Visio. - -Ms. Hogan was appointed Executive Vice President, Human Resources in November 2014. Prior to that Ms. Hogan was Corporate Vice President of Microsoft Services. She also served as Corporate Vice President of Customer Service and Support. Ms. Hogan joined Microsoft in 2003. Ms. Hogan also serves on the Board of Directors of Alaska Air Group, Inc. - -Ms. Hood was appointed Executive Vice President and Chief Financial Officer in July 2013, subsequent to her appointment as Chief Financial Officer in May 2013. From 2010 to 2013, Ms. Hood was Chief Financial Officer of the Microsoft Business Division. From 2006 through 2009, Ms. Hood was General Manager, Microsoft Business Division Strategy. Since joining Microsoft in 2002, Ms. Hood has also held finance-related positions in the Server and Tools Business and the corporate finance organization. Ms. Hood also serves on the Board of Directors of 3M Corporation. - -Mr. Smith was appointed President and Vice Chair in September 2021. Prior to that, he served as President and Chief Legal Officer since September 2015. He served as Executive Vice President, General Counsel, and Secretary from 2011 to 2015, and served as Senior Vice President, General Counsel, and Secretary from 2001 to 2011. Mr. Smith was also named Chief Compliance Officer in 2002. Since joining Microsoft in 1993, he was Deputy General Counsel for Worldwide Sales and previously was responsible for managing the European Law and Corporate Affairs Group, based in Paris. Mr. Smith also serves on the Board of Directors of Netflix, Inc. - -Mr. Young has served as Executive Vice President, Business Development, Strategy, and Ventures since joining Microsoft in November 2020. Prior to Microsoft, he served as the Chief Executive Officer of McAfee, LLC from 2017 to 2020, and served as a Senior Vice President and General Manager of Intel Security Group from 2014 until 2017, when he led the initiative to spin out McAfee into a standalone company. Mr. Young also serves on the Board of Directors of American Express Company. - -21 - - -PART I -Item 1 - - -AVAILABLE INFORMATION - -Our Internet address is www.microsoft.com. At our Investor Relations website, www.microsoft.com/investor, we make available free of charge a variety of information for investors. Our goal is to maintain the Investor Relations website as a portal through which investors can easily find or navigate to pertinent information about us, including: - -� Our annual report on Form 10-K, quarterly reports on Form 10-Q, current reports on Form 8-K, and any amendments to those reports, as soon as reasonably practicable after we electronically file that material with or furnish it to the Securities and Exchange Commission (�SEC�) at www.sec.gov. - -� Information on our business strategies, financial results, and metrics for investors. - -� Announcements of investor conferences, speeches, and events at which our executives talk about our product, service, and competitive strategies. Archives of these events are also available. - -� Press releases on quarterly earnings, product and service announcements, legal developments, and international news. - -� Corporate governance information including our articles of incorporation, bylaws, governance guidelines, committee charters, codes of conduct and ethics, global corporate social responsibility initiatives, and other governance-related policies. - -� Other news and announcements that we may post from time to time that investors might find useful or interesting. - -� Opportunities to sign up for email alerts to have information pushed in real time. - -We publish a variety of reports and resources related to our Corporate Social Responsibility programs and progress on our Reports Hub website, www.microsoft.com/corporate-responsibility/reports-hub, including reports on sustainability, responsible sourcing, accessibility, digital trust, and public policy engagement. - -The information found on these websites is not part of, or incorporated by reference into, this or any other report we file with, or furnish to, the SEC. In addition to these channels, we use social media to communicate to the public. It is possible that the information we post on social media could be deemed to be material to investors. We encourage investors, the media, and others interested in Microsoft to review the information we post on the social media channels listed on our Investor Relations website. - - - - -22 - - -PART I -Item 1A - -ITEM 1A. RISK FACTORS - -Our operations and financial results are subject to various risks and uncertainties, including those described below, that could adversely affect our business, financial condition, results of operations, cash flows, and the trading price of our common stock. - -STRATEGIC AND COMPETITIVE RISKS - -We face intense competition across all markets for our products and services, which may lead to lower revenue or operating margins. - -Competition in the technology sector - -Our competitors range in size from diversified global companies with significant research and development resources to small, specialized firms whose narrower product lines may let them be more effective in deploying technical, marketing, and financial resources. Barriers to entry in many of our businesses are low and many of the areas in which we compete evolve rapidly with changing and disruptive technologies, shifting user needs, and frequent introductions of new products and services. Our ability to remain competitive depends on our success in making innovative products, devices, and services that appeal to businesses and consumers. - -Competition among platform-based ecosystems - -An important element of our business model has been to create platform-based ecosystems on which many participants can build diverse solutions. A well-established ecosystem creates beneficial network effects among users, application developers, and the platform provider that can accelerate growth. Establishing significant scale in the marketplace is necessary to achieve and maintain attractive margins. We face significant competition from firms that provide competing platforms. - -� A competing vertically-integrated model, in which a single firm controls the software and hardware elements of a product and related services, has succeeded with some consumer products such as personal computers, tablets, phones, gaming consoles, wearables, and other endpoint devices. Competitors pursuing this model also earn revenue from services integrated with the hardware and software platform, including applications and content sold through their integrated marketplaces. They may also be able to claim security and performance benefits from their vertically integrated offer. We also offer some vertically-integrated hardware and software products and services. To the extent we shift a portion of our business to a vertically integrated model we increase our cost of revenue and reduce our operating margins. - -� We derive substantial revenue from licenses of Windows operating systems on PCs. We face significant competition from competing platforms developed for new devices and form factors such as smartphones and tablet computers. These devices compete on multiple bases including price and the perceived utility of the device and its platform. Users are increasingly turning to these devices to perform functions that in the past were performed by personal computers. Even if many users view these devices as complementary to a personal computer, the prevalence of these devices may make it more difficult to attract application developers to our PC operating system platforms. Competing with operating systems licensed at low or no cost may decrease our PC operating system margins. Popular products or services offered on competing platforms could increase their competitive strength. In addition, some of our devices compete with products made by our original equipment manufacturer (�OEM�) partners, which may affect their commitment to our platform. - -� Competing platforms have content and application marketplaces with scale and significant installed bases. The variety and utility of content and applications available on a platform are important to device purchasing decisions. Users may incur costs to move data and buy new content and applications when switching platforms. To compete, we must successfully enlist developers to write applications for our platform and ensure that these applications have high quality, security, customer appeal, and value. Efforts to compete with competitors� content and application marketplaces may increase our cost of revenue and lower our operating margins. Competitors� rules governing their content and applications marketplaces may restrict our ability to distribute products and services through them in accordance with our technical and business model objectives. - -23 - - -PART I -Item 1A - - -Business model competition - -Companies compete with us based on a growing variety of business models. - -� Even as we transition more of our business to infrastructure-, platform-, and software-as-a-service business model, the license-based proprietary software model generates a substantial portion of our software revenue. We bear the costs of converting original ideas into software products through investments in research and development, offsetting these costs with the revenue received from licensing our products. Many of our competitors also develop and sell software to businesses and consumers under this model. - -� Other competitors develop and offer free applications, online services and content, and make money by selling third-party advertising. Advertising revenue funds development of products and services these competitors provide to users at no or little cost, competing directly with our revenue-generating products. - -� Some companies compete with us by modifying and then distributing open source software at little or no cost to end users, and earning revenue on advertising or integrated products and services. These firms do not bear the full costs of research and development for the open source software. Some open source software mimics the features and functionality of our products. - -The competitive pressures described above may cause decreased sales volumes, price reductions, and/or increased operating costs, such as for research and development, marketing, and sales incentives. This may lead to lower revenue, gross margins, and operating income. - -Our increasing focus on cloud-based services presents execution and competitive risks. A growing part of our business involves cloud-based services available across the spectrum of computing devices. Our strategic vision is to compete and grow by building best-in-class platforms and productivity services for an intelligent cloud and an intelligent edge infused with artificial intelligence (�AI�). At the same time, our competitors are rapidly developing and deploying cloud-based services for consumers and business customers. Pricing and delivery models are evolving. Devices and form factors influence how users access services in the cloud and sometimes the user�s choice of which cloud-based services to use. We are devoting significant resources to develop and deploy our cloud-based strategies. The Windows ecosystem must continue to evolve with this changing environment. We embrace cultural and organizational changes to drive accountability and eliminate obstacles to innovation. Our intelligent cloud and intelligent edge worldview is connected with the growth of the Internet of Things (�IoT�). Our success in the IoT will depend on the level of adoption of our offerings such as Azure, Azure Stack, Azure IoT Edge, and Azure Sphere. We may not establish market share sufficient to achieve scale necessary to meet our business objectives. - -Besides software development costs, we are incurring costs to build and maintain infrastructure to support cloud computing services. These costs will reduce the operating margins we have previously achieved. Whether we succeed in cloud-based services depends on our execution in several areas, including: - -� Continuing to bring to market compelling cloud-based experiences that generate increasing traffic and market share. - -� Maintaining the utility, compatibility, and performance of our cloud-based services on the growing array of computing devices, including PCs, smartphones, tablets, gaming consoles, and other devices, as well as sensors and other IoT endpoints. - -� Continuing to enhance the attractiveness of our cloud platforms to third-party developers. - -� Ensuring our cloud-based services meet the reliability expectations of our customers and maintain the security of their data as well as help them meet their own compliance needs. - -� Making our suite of cloud-based services platform-agnostic, available on a wide range of devices and ecosystems, including those of our competitors. - -It is uncertain whether our strategies will attract the users or generate the revenue required to succeed. If we are not effective in executing organizational and technical changes to increase efficiency and accelerate innovation, or if we fail to generate sufficient usage of our new products and services, we may not grow revenue in line with the infrastructure and development investments described above. This may negatively impact gross margins and operating income. - -24 - - -PART I -Item 1A - - -RISKS RELATING TO THE EVOLUTION OF OUR BUSINESS - -We make significant investments in products and services that may not achieve expected returns. We will continue to make significant investments in research, development, and marketing for existing products, services, and technologies, including the Windows operating system, Microsoft 365, Office, Bing, SQL Server, Windows Server, Azure, Office 365, Xbox, LinkedIn, and other products and services. We also invest in the development and acquisition of a variety of hardware for productivity, communication, and entertainment including PCs, tablets, gaming devices, and HoloLens. Investments in new technology are speculative. Commercial success depends on many factors, including innovativeness, developer support, and effective distribution and marketing. If customers do not perceive our latest offerings as providing significant new functionality or other value, they may reduce their purchases of new software and hardware products or upgrades, unfavorably affecting revenue. We may not achieve significant revenue from new product, service, and distribution channel investments for several years, if at all. New products and services may not be profitable, and even if they are profitable, operating margins for some new products and businesses will not be as high as the margins we have experienced historically. We may not get engagement in certain features, like Edge and Bing, that drive post-sale monetization opportunities. Our data handling practices across our products and services will continue to be under scrutiny and perceptions of mismanagement, driven by regulatory activity or negative public reaction to our practices or product experiences, could negatively impact product and feature adoption, product design, and product quality. - -Developing new technologies is complex. It can require long development and testing periods. Significant delays in new releases or significant problems in creating new products or services could adversely affect our revenue. - -Acquisitions, joint ventures, and strategic alliances may have an adverse effect on our business. We expect to continue making acquisitions and entering into joint ventures and strategic alliances as part of our long-term business strategy. For example, in March 2021 we completed our acquisition of ZeniMax Media Inc. for $ 8.1 billion, and in March 2022 we completed our acquisition of Nuance Communications, Inc. for $18.8 billion. In January 2022 we announced a definitive agreement to acquire Activision Blizzard, Inc. for $68.7 billion. These acquisitions and other transactions and arrangements involve significant challenges and risks, including that they do not advance our business strategy, that we get an unsatisfactory return on our investment, that they raise new compliance-related obligations and challenges, that we have difficulty integrating and retaining new employees, business systems, and technology, that they distract management from our other businesses, or that announced transactions may not be completed. If an arrangement fails to adequately anticipate changing circumstances and interests of a party, it may result in early termination or renegotiation of the arrangement. The success of these transactions and arrangements will depend in part on our ability to leverage them to enhance our existing products and services or develop compelling new ones, as well as acquired companies� ability to meet our policies and processes in areas such as data governance, privacy, and cybersecurity. It may take longer than expected to realize the full benefits from these transactions and arrangements such as increased revenue or enhanced efficiencies, or the benefits may ultimately be smaller than we expected. These events could adversely affect our consolidated financial statements. - -If our goodwill or amortizable intangible assets become impaired, we may be required to record a significant charge to earnings. We acquire other companies and intangible assets and may not realize all the economic benefit from those acquisitions, which could cause an impairment of goodwill or intangibles. We review our amortizable intangible assets for impairment when events or changes in circumstances indicate the carrying value may not be recoverable. We test goodwill for impairment at least annually. Factors that may be a change in circumstances, indicating that the carrying value of our goodwill or amortizable intangible assets may not be recoverable, include a decline in our stock price and market capitalization, reduced future cash flow estimates, and slower growth rates in industry segments in which we participate. We have in the past recorded, and may in the future be required to record, a significant charge in our consolidated financial statements during the period in which any impairment of our goodwill or amortizable intangible assets is determined, negatively affecting our results of operations. - -25 - - -PART I -Item 1A - - -CYBERSECURITY, DATA PRIVACY, AND PLATFORM ABUSE RISKS - -Cyberattacks and security vulnerabilities could lead to reduced revenue, increased costs, liability claims, or harm to our reputation or competitive position. - -Security of our information technology - -Threats to IT security can take a variety of forms. Individual and groups of hackers and sophisticated organizations, including state-sponsored organizations or nation-states, continuously undertake attacks that pose threats to our customers and our IT. These actors may use a wide variety of methods, which may include developing and deploying malicious software or exploiting vulnerabilities in hardware, software, or other infrastructure in order to attack our products and services or gain access to our networks and datacenters, using social engineering techniques to induce our employees, users, partners, or customers to disclose passwords or other sensitive information or take other actions to gain access to our data or our users� or customers� data, or acting in a coordinated manner to launch distributed denial of service or other coordinated attacks. Nation-state and state-sponsored actors can deploy significant resources to plan and carry out exploits. Nation-state attacks against us or our customers may intensify during periods of intense diplomatic or armed conflict, such as the ongoing conflict in Ukraine. Inadequate account security practices may also result in unauthorized access to confidential data. For example, system administrators may fail to timely remove employee account access when no longer appropriate. Employees or third parties may intentionally compromise our or our users� security or systems or reveal confidential information. Malicious actors may employ the IT supply chain to introduce malware through software updates or compromised supplier accounts or hardware. - -Cyberthreats are constantly evolving and becoming increasingly sophisticated and complex, increasing the difficulty of detecting and successfully defending against them. We may have no current capability to detect certain vulnerabilities, which may allow them to persist in the environment over long periods of time. Cyberthreats can have cascading impacts that unfold with increasing speed across our internal networks and systems and those of our partners and customers. Breaches of our facilities, network, or data security could disrupt the security of our systems and business applications, impair our ability to provide services to our customers and protect the privacy of their data, result in product development delays, compromise confidential or technical business information harming our reputation or competitive position, result in theft or misuse of our intellectual property or other assets, subject us to ransomware attacks, require us to allocate more resources to improve technologies or remediate the impacts of attacks, or otherwise adversely affect our business. - -The cyberattacks uncovered in late 2020 known as �Solorigate� or �Nobelium� are an example of a supply chain attack where malware was introduced to a software provider�s customers, including us, through software updates. The attackers were later able to create false credentials that appeared legitimate to certain customers� systems. We may be targets of further attacks similar to Solorigate/Nobelium as both a supplier and consumer of IT. - -In addition, our internal IT environment continues to evolve. Often, we are early adopters of new devices and technologies. We embrace new ways of sharing data and communicating internally and with partners and customers using methods such as social networking and other consumer-oriented technologies. Our business policies and internal security controls may not keep pace with these changes as new threats emerge, or emerging cybersecurity regulations in jurisdictions worldwide. - -26 - - -PART I -Item 1A - - -Security of our products, services, devices, and customers� data - -The security of our products and services is important in our customers� decisions to purchase or use our products or services across cloud and on-premises environments. Security threats are a significant challenge to companies like us whose business is providing technology products and services to others. Threats to our own IT infrastructure can also affect our customers. Customers using our cloud-based services rely on the security of our infrastructure, including hardware and other elements provided by third parties, to ensure the reliability of our services and the protection of their data. Adversaries tend to focus their efforts on the most popular operating systems, programs, and services, including many of ours, and we expect that to continue. In addition, adversaries can attack our customers� on-premises or cloud environments, sometimes exploiting previously unknown (�zero day�) vulnerabilities, such as occurred in early calendar year 2021 with several of our Exchange Server on-premises products. Vulnerabilities in these or any product can persist even after we have issued security patches if customers have not installed the most recent updates, or if the attackers exploited the vulnerabilities before patching to install additional malware to further compromise customers� systems. Adversaries will continue to attack customers using our cloud services as customers embrace digital transformation. Adversaries that acquire user account information can use that information to compromise our users� accounts, including where accounts share the same attributes such as passwords. Inadequate account security practices may also result in unauthorized access, and user activity may result in ransomware or other malicious software impacting a customer�s use of our products or services. We are increasingly incorporating open source software into our products. There may be vulnerabilities in open source software that may make our products susceptible to cyberattacks. - -Our customers operate complex IT systems with third-party hardware and software from multiple vendors that may include systems acquired over many years. They expect our products and services to support all these systems and products, including those that no longer incorporate the strongest current security advances or standards. As a result, we may not be able to discontinue support in our services for a product, service, standard, or feature solely because a more secure alternative is available. Failure to utilize the most current security advances and standards can increase our customers� vulnerability to attack. Further, customers of widely varied size and technical sophistication use our technology, and consequently may have limited capabilities and resources to help them adopt and implement state of the art cybersecurity practices and technologies. In addition, we must account for this wide variation of technical sophistication when defining default settings for our products and services, including security default settings, as these settings may limit or otherwise impact other aspects of IT operations and some customers may have limited capability to review and reset these defaults. - -Cyberattacks such as Solorigate/Nobelium may adversely impact our customers even if our production services are not directly compromised. We are committed to notifying our customers whose systems have been impacted as we become aware and have available information and actions for customers to help protect themselves. We are also committed to providing guidance and support on detection, tracking, and remediation. We may not be able to detect the existence or extent of these attacks for all of our customers or have information on how to detect or track an attack, especially where an attack involves on-premises software such as Exchange Server where we may have no or limited visibility into our customers� computing environments. - -Development and deployment of defensive measures - -To defend against security threats to our internal IT systems, our cloud-based services, and our customers� systems, we must continuously engineer more secure products and services, enhance security and reliability features, improve the deployment of software updates to address security vulnerabilities in our own products as well as those provided by others, develop mitigation technologies that help to secure customers from attacks even when software updates are not deployed, maintain the digital security infrastructure that protects the integrity of our network, products, and services, and provide security tools such as firewalls, anti-virus software, and advanced security and information about the need to deploy security measures and the impact of doing so. Customers in certain industries such as financial services, health care, and government may have enhanced or specialized requirements to which we must engineer our products and services. - -27 - - -PART I -Item 1A - -The cost of measures to protect products and customer-facing services could reduce our operating margins. If we fail to do these things well, actual or perceived security vulnerabilities in our products and services, data corruption issues, or reduced performance could harm our reputation and lead customers to reduce or delay future purchases of products or subscriptions to services, or to use competing products or services. Customers may also spend more on protecting their existing computer systems from attack, which could delay adoption of additional products or services. Customers, and third parties granted access to their systems, may fail to update their systems, continue to run software or operating systems we no longer support, or may fail timely to install or enable security patches, or may otherwise fail to adopt adequate security practices. Any of these could adversely affect our reputation and revenue. Actual or perceived vulnerabilities may lead to claims against us. Our license agreements typically contain provisions that eliminate or limit our exposure to liability, but there is no assurance these provisions will withstand legal challenges. At times, to achieve commercial objectives, we may enter into agreements with larger liability exposure to customers. - -Our products operate in conjunction with and are dependent on products and components across a broad ecosystem of third parties. If there is a security vulnerability in one of these components, and if there is a security exploit targeting it, we could face increased costs, liability claims, reduced revenue, or harm to our reputation or competitive position. - -Disclosure and misuse of personal data could result in liability and harm our reputation. As we continue to grow the number, breadth, and scale of our cloud-based offerings, we store and process increasingly large amounts of personal data of our customers and users. The continued occurrence of high-profile data breaches provides evidence of an external environment increasingly hostile to information security. Despite our efforts to improve the security controls across our business groups and geographies, it is possible our security controls over personal data, our training of employees and third parties on data security, and other practices we follow may not prevent the improper disclosure or misuse of customer or user data we or our vendors store and manage. In addition, third parties who have limited access to our customer or user data may use this data in unauthorized ways. Improper disclosure or misuse could harm our reputation, lead to legal exposure to customers or users, or subject us to liability under laws that protect personal data, resulting in increased costs or loss of revenue. Our software products and services also enable our customers and users to store and process personal data on-premises or, increasingly, in a cloud-based environment we host. Government authorities can sometimes require us to produce customer or user data in response to valid legal orders. In the U.S. and elsewhere, we advocate for transparency concerning these requests and appropriate limitations on government authority to compel disclosure. Despite our efforts to protect customer and user data, perceptions that the collection, use, and retention of personal information is not satisfactorily protected could inhibit sales of our products or services and could limit adoption of our cloud-based solutions by consumers, businesses, and government entities. Additional security measures we may take to address customer or user concerns, or constraints on our flexibility to determine where and how to operate datacenters in response to customer or user expectations or governmental rules or actions, may cause higher operating expenses or hinder growth of our products and services. - -We may not be able to protect information in our products and services from use by others. LinkedIn and other Microsoft products and services contain valuable information and content protected by contractual restrictions or technical measures. In certain cases, we have made commitments to our members and users to limit access to or use of this information. Changes in the law or interpretations of the law may weaken our ability to prevent third parties from scraping or gathering information or content through use of bots or other measures and using it for their own benefit, thus diminishing the value of our products and services. - -Abuse of our platforms may harm our reputation or user engagement. - -Advertising, professional, marketplace, and gaming platform abuses - -For platform products and services that provide content or host ads that come from or can be influenced by third parties, including GitHub, LinkedIn, Microsoft Advertising, Microsoft News, Microsoft Store, Bing, and Xbox, our reputation or user engagement may be negatively affected by activity that is hostile or inappropriate. This activity may come from users impersonating other people or organizations, dissemination of information that may be viewed as misleading or intended to manipulate the opinions of our users, or the use of our products or services that violates our terms of service or otherwise for objectionable or illegal ends. Preventing or responding to these actions may require us to make substantial investments in people and technology and these investments may not be successful, adversely affecting our business and consolidated financial statements. - -28 - - -PART I -Item 1A - - -Other digital safety abuses - -Our hosted consumer services as well as our enterprise services may be used to disseminate harmful or illegal content in violation of our terms or applicable law. We may not proactively discover such content due to scale, the limitations of existing technologies, and conflicting legal frameworks. When discovered by users, such content may negatively affect our reputation, our brands, and user engagement. Regulations and other initiatives to make platforms responsible for preventing or eliminating harmful content online have been enacted, and we expect this to continue. We may be subject to enhanced regulatory oversight, civil or criminal liability, or reputational damage if we fail to comply with content moderation regulations, adversely affecting our business and consolidated financial statements. - -The development of the IoT presents security, privacy, and execution risks. To support the growth of the intelligent cloud and the intelligent edge, we are developing products, services, and technologies to power the IoT, a network of distributed and interconnected devices employing sensors, data, and computing capabilities including AI. The IoT�s great potential also carries substantial risks. IoT products and services may contain defects in design, manufacture, or operation that make them insecure or ineffective for their intended purposes. An IoT solution has multiple layers of hardware, sensors, processors, software, and firmware, several of which we may not develop or control. Each layer, including the weakest layer, can impact the security of the whole system. Many IoT devices have limited interfaces and ability to be updated or patched. IoT solutions may collect large amounts of data, and our handling of IoT data may not satisfy customers or regulatory requirements. IoT scenarios may increasingly affect personal health and safety. If IoT solutions that include our technologies do not work as intended, violate the law, or harm individuals or businesses, we may be subject to legal claims or enforcement actions. These risks, if realized, may increase our costs, damage our reputation or brands, or negatively impact our revenues or margins. - -Issues in the development and use of AI may result in reputational harm or liability. We are building AI into many of our offerings, including our productivity services, and we are also making first- and third-party AI available for our customers to use in solutions that they build. We expect these elements of our business to grow. We envision a future in which AI operating in our devices, applications, and the cloud helps our customers be more productive in their work and personal lives. As with many innovations, AI presents risks and challenges that could affect its adoption, and therefore our business. AI algorithms may be flawed. Datasets may be insufficient or contain biased information. Ineffective or inadequate AI development or deployment practices by Microsoft or others could result in incidents that impair the acceptance of AI solutions or cause harm to individuals or society. These deficiencies and other failures of AI systems could subject us to competitive harm, regulatory action, legal liability, including under new proposed legislation regulating AI in jurisdictions such as the European Union (�EU�), and brand or reputational harm. Some AI scenarios present ethical issues. If we enable or offer AI solutions that are controversial because of their impact on human rights, privacy, employment, or other social, economic, or political issues, we may experience brand or reputational harm. - -OPERATIONAL RISKS - -We may have excessive outages, data losses, and disruptions of our online services if we fail to maintain an adequate operations infrastructure. Our increasing user traffic, growth in services, and the complexity of our products and services demand more computing power. We spend substantial amounts to build, purchase, or lease datacenters and equipment and to upgrade our technology and network infrastructure to handle more traffic on our websites and in our datacenters. Our datacenters depend on predictable energy and networking supplies, the cost or availability of which could be adversely affected by a variety of factors, including the transition to a clean energy economy and geopolitical disruptions. These demands continue to increase as we introduce new products and services and support the growth of existing services such as Bing, Azure, Microsoft Account services, Microsoft 365, Microsoft Teams, Dynamics 365, OneDrive, SharePoint Online, Skype, Xbox, and Outlook.com. We are rapidly growing our business of providing a platform and back-end hosting for services provided by third parties to their end users. Maintaining, securing, and expanding this infrastructure is expensive and complex, and requires development of principles for datacenter builds in geographies with higher safety risks. It requires that we maintain an Internet connectivity infrastructure and storage and compute capacity that is robust and reliable within competitive and regulatory constraints that continue to evolve. Inefficiencies or operational failures, including temporary or permanent loss of customer data, insufficient Internet connectivity, or inadequate storage and compute capacity, could diminish the quality of our products, services, and user experience resulting in contractual liability, claims by customers and other third parties, regulatory actions, damage to our reputation, and loss of current and potential users, subscribers, and advertisers, each of which may adversely impact our consolidated financial statements. - -29 - - -PART I -Item 1A - -We may experience quality or supply problems. Our hardware products such as Xbox consoles, Surface devices, and other devices we design and market are highly complex and can have defects in design, manufacture, or associated software. We could incur significant expenses, lost revenue, and reputational harm as a result of recalls, safety alerts, or product liability claims if we fail to prevent, detect, or address such issues through design, testing, or warranty repairs. - -Our software products and services also may experience quality or reliability problems. The highly sophisticated software we develop may contain bugs and other defects that interfere with their intended operation. Our customers increasingly rely on us for critical business functions and multiple workloads. Many of our products and services are interdependent with one another. Each of these circumstances potentially magnifies the impact of quality or reliability issues. Any defects we do not detect and fix in pre-release testing could cause reduced sales and revenue, damage to our reputation, repair or remediation costs, delays in the release of new products or versions, or legal liability. Although our license agreements typically contain provisions that eliminate or limit our exposure to liability, there is no assurance these provisions will withstand legal challenge. - -There are limited suppliers for certain device and datacenter components. Our competitors use some of the same suppliers and their demand for hardware components can affect the capacity available to us. If components are delayed or become unavailable, whether because of supplier capacity constraint, industry shortages, legal or regulatory changes that restrict supply sources, or other reasons, we may not obtain timely replacement supplies, resulting in reduced sales or inadequate datacenter capacity. Component shortages, excess or obsolete inventory, or price reductions resulting in inventory adjustments may increase our cost of revenue. Xbox consoles, Surface devices, datacenter servers, and other hardware are assembled in Asia and other geographies that may be subject to disruptions in the supply chain, resulting in shortages that would affect our revenue and operating margins. - -LEGAL, REGULATORY, AND LITIGATION RISKS - -Government litigation and regulatory activity relating to competition rules may limit how we design and market our products. As a leading global software and device maker, government agencies closely scrutinize us under U.S. and foreign competition laws. Governments are actively enforcing competition laws and regulations, and this includes scrutiny in potentially large markets such as the EU, the U.S., and China. Some jurisdictions also allow competitors or consumers to assert claims of anti-competitive conduct. U.S. federal and state antitrust authorities have previously brought enforcement actions and continue to scrutinize our business. - -The European Commission (�the Commission�) closely scrutinizes the design of high-volume Microsoft products and the terms on which we make certain technologies used in these products, such as file formats, programming interfaces, and protocols, available to other companies. Flagship product releases such as Windows can receive significant scrutiny under competition laws. For example, in 2004, the Commission ordered us to create new versions of our Windows operating system that do not include certain multimedia technologies and to provide our competitors with specifications for how to implement certain proprietary Windows communications protocols in their own products. In 2009, the Commission accepted a set of commitments we offered to address the Commission�s concerns relating to competition in web browsing software, including an undertaking to address Commission concerns relating to interoperability. The web browsing commitments expired in 2014. The remaining obligations may limit our ability to innovate in Windows or other products in the future, diminish the developer appeal of the Windows platform, and increase our product development costs. The availability of licenses related to protocols and file formats may enable competitors to develop software products that better mimic the functionality of our products, which could hamper sales of our products. - -Our portfolio of first-party devices continues to grow; at the same time our OEM partners offer a large variety of devices for our platforms. As a result, increasingly we both cooperate and compete with our OEM partners, creating a risk that we fail to do so in compliance with competition rules. Regulatory scrutiny in this area may increase. Certain foreign governments, particularly in China and other countries in Asia, have advanced arguments under their competition laws that exert downward pressure on royalties for our intellectual property. - -30 - - -PART I -Item 1A - -Government regulatory actions and court decisions such as these may result in fines or hinder our ability to provide the benefits of our software to consumers and businesses, reducing the attractiveness of our products and the revenue that comes from them. New competition law actions could be initiated, potentially using previous actions as precedent. The outcome of such actions, or steps taken to avoid them, could adversely affect us in a variety of ways, including: - -� We may have to choose between withdrawing products from certain geographies to avoid fines or designing and developing alternative versions of those products to comply with government rulings, which may entail a delay in a product release and removing functionality that customers want or on which developers rely. - -� We may be required to make available licenses to our proprietary technologies on terms that do not reflect their fair market value or do not protect our associated intellectual property. - -� We are subject to a variety of ongoing commitments because of court or administrative orders, consent decrees, or other voluntary actions we have taken. If we fail to comply with these commitments, we may incur litigation costs and be subject to substantial fines or other remedial actions. - -� Our ability to realize anticipated Windows post-sale monetization opportunities may be limited. - -� Regulatory scrutiny may inhibit our ability to consummate acquisitions or impose conditions that reduce the ultimate value of such transactions. - -Our global operations subject us to potential consequences under anti-corruption, trade, and other laws and regulations. The Foreign Corrupt Practices Act (�FCPA�) and other anti-corruption laws and regulations (�Anti-Corruption Laws�) prohibit corrupt payments by our employees, vendors, or agents, and the accounting provisions of the FCPA require us to maintain accurate books and records and adequate internal controls. From time to time, we receive inquiries from authorities in the U.S. and elsewhere which may be based on reports from employees and others about our business activities outside the U.S. and our compliance with Anti-Corruption Laws. Periodically, we receive such reports directly and investigate them, and also cooperate with investigations by U.S. and foreign law enforcement authorities. An example of increasing international regulatory complexity is the EU Whistleblower Directive, initiated in 2021, which may present compliance challenges to the extent it is implemented in different forms by EU member states. Most countries in which we operate also have competition laws that prohibit competitors from colluding or otherwise attempting to reduce competition between themselves. While we devote substantial resources to our U.S. and international compliance programs and have implemented policies, training, and internal controls designed to reduce the risk of corrupt payments and collusive activity, our employees, vendors, or agents may violate our policies. Our failure to comply with Anti-Corruption Laws or competition laws could result in significant fines and penalties, criminal sanctions against us, our officers, or our employees, prohibitions on the conduct of our business, and damage to our reputation. - -Increasing trade laws, policies, sanctions, and other regulatory requirements also affect our operations in and outside the U.S. relating to trade and investment. Economic sanctions in the U.S., the EU, and other countries prohibit most business with restricted entities or countries such as Crimea, Cuba, Iran, North Korea, and Syria. U.S. export controls restrict Microsoft from offering many of its products and services to, or making investments in, certain entities in specified countries. U.S. import controls restrict us from integrating certain information and communication technologies into our supply chain and allow for government review of transactions involving information and communications technology from countries determined to be foreign adversaries. Periods of intense diplomatic or armed conflict, such as the ongoing conflict in Ukraine, may result in (1) new and rapidly evolving sanctions and trade restrictions, which may impair trade with sanctioned individuals and countries, and (2) negative impacts to regional trade ecosystems among our customers, partners, and us. Non-compliance with sanctions as well as general ecosystem disruptions could result in reputational harm, operational delays, monetary fines, loss of revenues, increased costs, loss of export privileges, or criminal sanctions. - -31 - - -PART I -Item 1A - -Other regulatory areas that may apply to our products and online services offerings include requirements related to user privacy, telecommunications, data storage and protection, advertising, and online content. For example, some regulators are taking the position that our offerings such as Microsoft Teams and Skype are covered by existing laws regulating telecommunications services, and some new laws, including EU Member State laws under the European Electronic Communications Code, are defining more of our services as regulated telecommunications services. This trend may continue and will result in these offerings being subjected to additional data protection, security, and law enforcement surveillance obligations. Regulators may assert that our collection, use, and management of customer data and other information is inconsistent with their laws and regulations, including laws that apply to the tracking of users via technology such as cookies. Legislative or regulatory action relating to cybersecurity requirements may increase the costs to develop, implement, or secure our products and services. Legislative and regulatory action is emerging in the areas of AI and content moderation, which could increase costs or restrict opportunity. Applying these laws and regulations to our business is often unclear, subject to change over time, and sometimes may conflict from jurisdiction to jurisdiction. Additionally, these laws and governments� approach to their enforcement, and our products and services, are continuing to evolve. Compliance with these types of regulation may involve significant costs or require changes in products or business practices that result in reduced revenue. Noncompliance could result in the imposition of penalties or orders we stop the alleged noncompliant activity. - -We strive to empower all people and organizations to achieve more, and accessibility of our products is an important aspect of this goal. There is increasing pressure from advocacy groups, regulators, competitors, customers, and other stakeholders to make technology more accessible. If our products do not meet customer expectations or global accessibility requirements, we could lose sales opportunities or face regulatory or legal actions. - -Laws and regulations relating to the handling of personal data may impede the adoption of our services or result in increased costs, legal claims, fines against us, or reputational damage. The growth of our Internet- and cloud-based services internationally relies increasingly on the movement of data across national boundaries. Legal requirements relating to the collection, storage, handling, and transfer of personal data continue to evolve. For example, in July 2020 the Court of Justice of the EU invalidated a framework called Privacy Shield for companies to transfer data from EU member states to the United States. This ruling continues to generate uncertainty about the legal requirements for data transfers from the EU under other legal mechanisms and has resulted in some EU data protection authorities blocking the use of U.S.-based services that involve the transfer of data to the U.S. The U.S. and the EU in March 2022 agreed in principle on a replacement framework for the Privacy Shield, called the Trans-Atlantic Data Privacy Framework. A failure of the U.S. and EU to finalize the Trans-Atlantic Data Privacy Framework could compound that uncertainty and result in additional blockages of data transfers. Potential new rules and restrictions on the flow of data across borders could increase the cost and complexity of delivering our products and services in some markets. For example, the EU General Data Protection Regulation (�GDPR�) applies to all of our activities conducted from an establishment in the EU or related to products and services offered in the EU, imposes a range of compliance obligations regarding the handling of personal data. More recently, the EU has been developing new requirements related to the use of data, including in the Digital Markets Act, the Digital Services Act, and the Data Act, that will add additional rules and restriction on the use of data in our products and services. Engineering efforts to build and maintain capabilities to facilitate compliance with these laws involve substantial expense and the diversion of engineering resources from other projects. We might experience reduced demand for our offerings if we are unable to engineer products that meet our legal duties or help our customers meet their obligations under the GDPR and other data regulations, or if our implementation to comply with the GDPR makes our offerings less attractive. Compliance with these obligations depends in part on how particular regulators interpret and apply them. If we fail to comply, or if regulators assert we have failed to comply (including in response to complaints made by customers), it may lead to regulatory enforcement actions, which can result in monetary penalties (of up to 4% of worldwide revenue in the case of GDPR), private lawsuits, reputational damage, blockage of international data transfers, and loss of customers. The highest fines assessed under GDPR have recently been increasing, especially against large technology companies. Jurisdictions around the world, such as China, India, and states in the U.S. have adopted, or are considering adopting or expanding, laws and regulations imposing obligations regarding the handling or transfer of personal data. - -32 - - -PART I -Item 1A - -The Company�s investment in gaining insights from data is becoming central to the value of the services we deliver to customers, to our operational efficiency and key opportunities in monetization, customer perceptions of quality, and operational efficiency. Our ability to use data in this way may be constrained by regulatory developments that impede realizing the expected return from this investment. Ongoing legal analyses, reviews, and inquiries by regulators of Microsoft practices, or relevant practices of other organizations, may result in burdensome or inconsistent requirements, including data sovereignty and localization requirements, affecting the location, movement, collection, and use of our customer and internal employee data as well as the management of that data. Compliance with applicable laws and regulations regarding personal data may require changes in services, business practices, or internal systems that result in increased costs, lower revenue, reduced efficiency, or greater difficulty in competing with foreign-based firms. Compliance with data regulations might limit our ability to innovate or offer certain features and functionality in some jurisdictions where we operate. Failure to comply with existing or new rules may result in significant penalties or orders to stop the alleged noncompliant activity, as well as negative publicity and diversion of management time and effort. - -We have claims and lawsuits against us that may result in adverse outcomes. We are subject to a variety of claims and lawsuits. These claims may arise from a wide variety of business practices and initiatives, including major new product releases such as Windows, significant business transactions, warranty or product claims, and employment practices. Adverse outcomes in some or all of these claims may result in significant monetary damages or injunctive relief that could adversely affect our ability to conduct our business. The litigation and other claims are subject to inherent uncertainties and management�s view of these matters may change in the future. A material adverse impact in our consolidated financial statements could occur for the period in which the effect of an unfavorable outcome becomes probable and reasonably estimable. - -Our business with government customers may present additional uncertainties. We derive substantial revenue from government contracts. Government contracts generally can present risks and challenges not present in private commercial agreements. For instance, we may be subject to government audits and investigations relating to these contracts, we could be suspended or debarred as a governmental contractor, we could incur civil and criminal fines and penalties, and under certain circumstances contracts may be rescinded. Some agreements may allow a government to terminate without cause and provide for higher liability limits for certain losses. Some contracts may be subject to periodic funding approval, reductions, or delays which could adversely impact public-sector demand for our products and services. These events could negatively impact our results of operations, financial condition, and reputation. - -We may have additional tax liabilities. We are subject to income taxes in the U.S. and many foreign jurisdictions. Significant judgment is required in determining our worldwide provision for income taxes. In the course of our business, there are many transactions and calculations where the ultimate tax determination is uncertain. For example, compliance with the 2017 United States Tax Cuts and Jobs Act (�TCJA�) and possible future legislative changes may require the collection of information not regularly produced within the Company, the use of estimates in our consolidated financial statements, and the exercise of significant judgment in accounting for its provisions. As regulations and guidance evolve with respect to the TCJA or possible future legislative changes, and as we gather more information and perform more analysis, our results may differ from previous estimates and may materially affect our consolidated financial statements. - -We regularly are under audit by tax authorities in different jurisdictions. Although we believe that our provision for income taxes and our tax estimates are reasonable, tax authorities may disagree with certain positions we have taken. In addition, economic and political pressures to increase tax revenue in various jurisdictions may make resolving tax disputes favorably more difficult. We are currently under Internal Revenue Service audit for prior tax years, with the primary unresolved issues relating to transfer pricing. The final resolution of those audits, and other audits or litigation, may differ from the amounts recorded in our consolidated financial statements and may materially affect our consolidated financial statements in the period or periods in which that determination is made. - -We earn a significant amount of our operating income outside the U.S. A change in the mix of earnings and losses in countries with differing statutory tax rates, changes in our business or structure, or the expiration of or disputes about certain tax agreements in a particular country may result in higher effective tax rates for the Company. In addition, changes in U.S. federal and state or international tax laws applicable to corporate multinationals, other fundamental law changes currently being considered by many countries, including in the U.S., and changes in taxing jurisdictions� administrative interpretations, decisions, policies, and positions may materially adversely impact our consolidated financial statements. - -33 - - -PART I -Item 1A - - -INTELLECTUAL PROPERTY RISKS - -We may not be able to protect our source code from copying if there is an unauthorized disclosure. Source code, the detailed program commands for our operating systems and other software programs, is critical to our business. Although we license portions of our application and operating system source code to several licensees, we take significant measures to protect the secrecy of large portions of our source code. If our source code leaks, we might lose future trade secret protection for that code. It may then become easier for third parties to compete with our products by copying functionality, which could adversely affect our revenue and operating margins. Unauthorized disclosure of source code also could increase the security risks described elsewhere in these risk factors. - -Legal changes, our evolving business model, piracy, and other factors may decrease the value of our intellectual property. Protecting our intellectual property rights and combating unlicensed copying and use of our software and other intellectual property on a global basis is difficult. While piracy adversely affects U.S. revenue, the impact on revenue from outside the U.S. is more significant, particularly countries in which the legal system provides less protection for intellectual property rights. Our revenue in these markets may grow more slowly than the underlying device market. Similarly, the absence of harmonized patent laws makes it more difficult to ensure consistent respect for patent rights. Throughout the world, we educate users about the benefits of licensing genuine products and obtaining indemnification benefits for intellectual property risks, and we educate lawmakers about the advantages of a business climate where intellectual property rights are protected. Reductions in the legal protection for software intellectual property rights could adversely affect revenue. - -We expend significant resources to patent the intellectual property we create with the expectation that we will generate revenues by incorporating that intellectual property in our products or services or, in some instances, by licensing or cross-licensing our patents to others in return for a royalty and/or increased freedom to operate. Changes in the law may continue to weaken our ability to prevent the use of patented technology or collect revenue for licensing our patents. These include legislative changes and regulatory actions that make it more difficult to obtain injunctions, and the increasing use of legal process to challenge issued patents. Similarly, licensees of our patents may fail to satisfy their obligations to pay us royalties or may contest the scope and extent of their obligations. The royalties we can obtain to monetize our intellectual property may decline because of the evolution of technology, price changes in products using licensed patents, greater value from cross-licensing, or the difficulty of discovering infringements. Finally, our increasing engagement with open source software will also cause us to license our intellectual property rights broadly in certain situations and may negatively impact revenue. - -Third parties may claim we infringe their intellectual property rights. From time to time, others claim we infringe their intellectual property rights. The number of these claims may grow because of constant technological change in the markets in which we compete, the extensive patent coverage of existing technologies, the rapid rate of issuance of new patents, and our offering of first-party devices, such as Surface. To resolve these claims, we may enter into royalty and licensing agreements on terms that are less favorable than currently available, stop selling or redesign affected products or services, or pay damages to satisfy indemnification commitments with our customers. These outcomes may cause operating margins to decline. Besides money damages, in some jurisdictions plaintiffs can seek injunctive relief that may limit or prevent importing, marketing, and selling our products or services that have infringing technologies. In some countries, such as Germany, an injunction can be issued before the parties have fully litigated the validity of the underlying patents. We have paid significant amounts to settle claims related to the use of technology and intellectual property rights and to procure intellectual property rights as part of our strategy to manage this risk, and may continue to do so. - -GENERAL RISKS - -If our reputation or our brands are damaged, our business and operating results may be harmed. Our reputation and brands are globally recognized and are important to our business. Our reputation and brands affect our ability to attract and retain consumer, business, and public-sector customers. There are numerous ways our reputation or brands could be damaged. These include product safety or quality issues, our environmental impact and sustainability, supply chain practices, or human rights record. We may experience backlash from customers, government entities, advocacy groups, employees, and other stakeholders that disagree with our product offering decisions or public policy positions. Damage to our reputation or our brands may occur from, among other things: - -� The introduction of new features, products, services, or terms of service that customers, users, or partners do not like. - -34 - - -PART I -Item 1A - -� Public scrutiny of our decisions regarding user privacy, data practices, or content. - -� Data security breaches, compliance failures, or actions of partners or individual employees. - -The proliferation of social media may increase the likelihood, speed, and magnitude of negative brand events. If our brands or reputation are damaged, it could negatively impact our revenues or margins, or ability to attract the most highly qualified employees. - -Adverse economic or market conditions may harm our business. Worsening economic conditions, including inflation, recession, pandemic, or other changes in economic conditions, may cause lower IT spending and adversely affect our revenue. If demand for PCs, servers, and other computing devices declines, or consumer or business spending for those products declines, our revenue will be adversely affected. - -Our product distribution system relies on an extensive partner and retail network. OEMs building devices that run our software have also been a significant means of distribution. The impact of economic conditions on our partners, such as the bankruptcy of a major distributor, OEM, or retailer, could cause sales channel disruption. - -Challenging economic conditions also may impair the ability of our customers to pay for products and services they have purchased. As a result, allowances for doubtful accounts and write-offs of accounts receivable may increase. - -We maintain an investment portfolio of various holdings, types, and maturities. These investments are subject to general credit, liquidity, market, and interest rate risks, which may be exacerbated by market downturns or events that affect global financial markets. A significant part of our investment portfolio comprises U.S. government securities. If global financial markets decline for long periods, or if there is a downgrade of the U.S. government credit rating due to an actual or threatened default on government debt, our investment portfolio may be adversely affected and we could determine that more of our investments have experienced a decline in fair value, requiring impairment charges that could adversely affect our consolidated financial statements. - -Catastrophic events or geopolitical conditions may disrupt our business. A disruption or failure of our systems or operations because of a major earthquake, weather event, cyberattack, terrorist attack, pandemic, or other catastrophic event could cause delays in completing sales, providing services, or performing other critical functions. Our corporate headquarters, a significant portion of our research and development activities, and certain other essential business operations are in the Seattle, Washington area, and we have other business operations in the Silicon Valley area of California, both of which are seismically active regions. A catastrophic event that results in the destruction or disruption of any of our critical business or IT systems, or the infrastructure or systems they rely on, such as power grids, could harm our ability to conduct normal business operations. Providing our customers with more services and solutions in the cloud puts a premium on the resilience of our systems and strength of our business continuity management plans and magnifies the potential impact of prolonged service outages in our consolidated financial statements. - -Abrupt political change, terrorist activity, and armed conflict, such as the ongoing conflict in Ukraine, pose a risk of general economic disruption in affected countries, which may increase our operating costs and negatively impact our ability to sell to and collect from customers in affected markets. These conditions also may add uncertainty to the timing and budget for technology investment decisions by our customers and may cause supply chain disruptions for hardware manufacturers. Geopolitical change may result in changing regulatory systems and requirements and market interventions that could impact our operating strategies, access to national, regional, and global markets, hiring, and profitability. Geopolitical instability may lead to sanctions and impact our ability to do business in some markets or with some public-sector customers. Any of these changes may negatively impact our revenues. - -The occurrence of regional epidemics or a global pandemic such as COVID-19 may adversely affect our operations, financial condition, and results of operations. The COVID-19 pandemic has had widespread, rapidly evolving, and unpredictable impacts on global society, economies, financial markets, and business practices. The extent to which global pandemics impact our business going forward will depend on factors such as the duration and scope of the pandemic; governmental, business, and individuals' actions in response to the pandemic; and the impact on economic activity including the possibility of recession or financial market instability. - -35 - - -PART I -Item 1A - -Measures to contain a global pandemic may intensify other risks described in these Risk Factors. Any of these measures may adversely impact our ability to: - -� Maintain our operations infrastructure, including the reliability and adequate capacity of cloud services. - -� Satisfy our contractual and regulatory compliance obligations as we adapt to changing usage patterns, such as through datacenter load balancing. - -� Ensure a high-quality and consistent supply chain and manufacturing operations for our hardware devices and datacenter operations. - -� Effectively manage our international operations through changes in trade practices and policies. - -� Hire and deploy people where we most need them. - -� Sustain the effectiveness and productivity of our operations including our sales, marketing, engineering, and distribution functions. - -We may incur increased costs to effectively manage these aspects of our business. If we are unsuccessful it may adversely impact our revenues, cash flows, market share growth, and reputation. - -The long-term effects of climate change on the global economy and the IT industry in particular are unclear. Environmental regulations or changes in the supply, demand or available sources of energy or other resources may affect the availability or cost of goods and services, including natural resources, necessary to run our business. Changes in climate where we operate may increase the costs of powering and cooling computer hardware we use to develop software and provide cloud-based services. - -Our global business exposes us to operational and economic risks. Our customers are located throughout the world and a significant part of our revenue comes from international sales. The global nature of our business creates operational, economic, and geopolitical risks. Our results of operations may be affected by global, regional, and local economic developments, monetary policy, inflation, and recession, as well as political and military disputes. In addition, our international growth strategy includes certain markets, the developing nature of which presents several risks, including deterioration of social, political, labor, or economic conditions in a country or region, and difficulties in staffing and managing foreign operations. Emerging nationalist and protectionist trends and concerns about human rights and political expression in specific countries may significantly alter the trade and commercial environments. Changes to trade policy or agreements as a result of populism, protectionism, or economic nationalism may result in higher tariffs, local sourcing initiatives, and non-local sourcing restrictions, export controls, investment restrictions, or other developments that make it more difficult to sell our products in foreign countries. Disruptions of these kinds in developed or emerging markets could negatively impact demand for our products and services or increase operating costs. Although we hedge a portion of our international currency exposure, significant fluctuations in foreign exchange rates between the U.S. dollar and foreign currencies may adversely affect our results of operations. - -Our business depends on our ability to attract and retain talented employees. Our business is based on successfully attracting and retaining talented employees representing diverse backgrounds, experiences, and skill sets. The market for highly skilled workers and leaders in our industry is extremely competitive. Maintaining our brand and reputation, as well as a diverse and inclusive work environment that enables all our employees to thrive, are important to our ability to recruit and retain employees. We are also limited in our ability to recruit internationally by restrictive domestic immigration laws. Changes to U.S. immigration policies that restrain the flow of technical and professional talent may inhibit our ability to adequately staff our research and development efforts. If we are less successful in our recruiting efforts, or if we cannot retain highly skilled workers and key leaders, our ability to develop and deliver successful products and services may be adversely affected. Effective succession planning is also important to our long-term success. Failure to ensure effective transfer of knowledge and smooth transitions involving key employees could hinder our strategic planning and execution. How employment-related laws are interpreted and applied to our workforce practices may result in increased operating costs and less flexibility in how we meet our workforce needs. Our global workforce is primarily non-unionized, but we have several unions and works councils outside of the United States. In the U.S., there has been a general increase in workers exercising their right to form or join a union. While Microsoft has not received such petitions in the U.S., the unionization of significant employee populations could result in higher costs and other operational changes necessary to respond to changing conditions and to establish new relationships with worker representatives. - - - -36 - - -PART I -Item 1B, 2, 3, 4 - -ITEM 1B. UNRESOLVED STAFF COMMENTS - -We have received no written comments regarding our periodic or current reports from the staff of the Securities and Exchange Commission that were issued 180 days or more preceding the end of our fiscal year 2022 that remain unresolved. - -ITEM 2. PROPERTIES - -Our corporate headquarters are located in Redmond, Washington. We have approximately 15 million square feet of space located in King County, Washington that is used for engineering, sales, marketing, and operations, among other general and administrative purposes. These facilities include approximately 10 million square feet of owned space situated on approximately 520 acres of land we own at our corporate headquarters, and approximately 5 million square feet of space we lease. In addition, we own and lease space domestically that includes office and datacenter space. - -We also own and lease facilities internationally for datacenters, research and development, and other operations. The largest owned properties include space in the following locations: China, India, Ireland, the Netherlands, and Singapore. The largest leased properties include space in the following locations: Australia, Canada, China, France, Germany, India, Ireland, Israel, Japan, the Netherlands, and the United Kingdom. - -In addition to the above locations, we have various product development facilities, both domestically and internationally, as described under Research and Development (Part I, Item 1 of this Form 10-K). - -The table below shows a summary of the square footage of our office, datacenter, and other facilities owned and leased domestically and internationally as of June 30, 2022: - -(Square feet in millions) - - -Location -Owned -Leased -Total - - - - - - - -U.S. -25 -19 -44 -International -8 -21 -29 - - - - - - - -Total -33 -40 -73 - - - - - - - - -ITEM 3. LEGAL PROCEEDINGS - -Refer to Note 15 � Contingencies of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for information regarding legal proceedings in which we are involved. - -ITEM 4. MINE SAFETY DISCLOSURES - -Not applicable. - - - -37 - - -PART II -Item 5 - - -PART II - -ITEM 5. MARKET FOR REGISTRANT�S COMMON EQUITY, RELATED STOCKHOLDER MATTERS, AND ISSUER PURCHASES OF EQUITY SECURITIES - -MARKET AND STOCKHOLDERS - -Our common stock is traded on the NASDAQ Stock Market under the symbol MSFT. On July 25, 2022, there were 86,465 registered holders of record of our common stock. - -SHARE REPURCHASES AND DIVIDENDS - -Following are our monthly share repurchases for the fourth quarter of fiscal year 2022: - - - - - - -Total Number of - - - - - - - - -Shares Purchased as - -Approximate Dollar Value of - -Total Number - -Average -Part of Publicly - -Shares That May Yet Be - -of Shares - -Price Paid -Announced Plans - -Purchased Under the Plans -Period -Purchased - -Per Share -or Programs - -or Programs - - - - - - - - - - - - - - - - - - -(In millions) - - - - - - - - - - -April 1, 2022 � April 30, 2022 -9,124,963 -$ -289.34 -9,124,963 -$ -45,869 -May 1, 2022 � May 31, 2022 -9,809,727 - -265.95 -9,809,727 - -43,260 -June 1, 2022 � June 30, 2022 -9,832,841 - -259.42 -9,832,841 - -40,709 - - - - - - - - - - - -28,767,531 - - -28,767,531 - - - - - - - - - - - - - - - -All share repurchases were made using cash resources. Our share repurchases may occur through open market purchases or pursuant to a Rule 10b5-1 trading plan. The above table excludes shares repurchased to settle employee tax withholding related to the vesting of stock awards. - -Our Board of Directors declared the following dividends during the fourth quarter of fiscal year 2022: - - - - - -Dividend - - - -Declaration Date -Record Date -Payment Date - -Per Share - -Amount - - - - - - - - - - - - - - -(In millions) - - - - - - - - -June 14, 2022 -August 18, 2022 -September 8, 2022 -$ -0.62 -$ -4,627 - - - - - - - - - - -We returned $12.4 billion to shareholders in the form of share repurchases and dividends in the fourth quarter of fiscal year 2022. Refer to Note 16 � Stockholders� Equity of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for further discussion regarding share repurchases and dividends. - - - -38 - - -PART II -Item 6 - -ITEM 6. [RESERVED] - - - - -39 - - -PART II -Item 7 - -ITEM 7. MANAGEMENT�S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS - -The following Management�s Discussion and Analysis of Financial Condition and Results of Operations (�MD&A�) is intended to help the reader understand the results of operations and financial condition of Microsoft Corporation. MD&A is provided as a supplement to, and should be read in conjunction with, our consolidated financial statements and the accompanying Notes to Financial Statements (Part II, Item 8 of this Form 10-K). This section generally discusses the results of our operations for the year ended June 30, 2022 compared to the year ended June 30, 2021. For a discussion of the year ended June 30, 2021 compared to the year ended June 30, 2020, please refer to Part II, Item 7, �Management�s Discussion and Analysis of Financial Condition and Results of Operations� in our Annual Report on Form 10-K for the year ended June 30, 2021. - -OVERVIEW - -Microsoft is a technology company whose mission is to empower every person and every organization on the planet to achieve more. We strive to create local opportunity, growth, and impact in every country around the world. Our platforms and tools help drive small business productivity, large business competitiveness, and public-sector efficiency. They also support new startups, improve educational and health outcomes, and empower human ingenuity. - -We generate revenue by offering a wide range of cloud-based and other services to people and businesses; licensing and supporting an array of software products; designing, manufacturing, and selling devices; and delivering relevant online advertising to a global audience. Our most significant expenses are related to compensating employees; designing, manufacturing, marketing, and selling our products and services; datacenter costs in support of our cloud-based services; and income taxes. - -Highlights from fiscal year 2022 compared with fiscal year 2021 included: - -� Microsoft Cloud (formerly commercial cloud) revenue increased 32% to $91.2 billion. - -� Office Commercial products and cloud services revenue increased 13% driven by Office 365 Commercial growth of 18%. - -� Office Consumer products and cloud services revenue increased 11% and Microsoft 365 Consumer subscribers grew to 59.7 million. - -� LinkedIn revenue increased 34%. - -� Dynamics products and cloud services revenue increased 25% driven by Dynamics 365 growth of 39%. - -� Server products and cloud services revenue increased 28% driven by Azure and other cloud services growth of 45%. - -� Windows original equipment manufacturer licensing (�Windows OEM�) revenue increased 11%. - -� Windows Commercial products and cloud services revenue increased 11%. - -� Xbox content and services revenue increased 3%. - -� Search and news advertising revenue excluding traffic acquisition costs increased 27%. - -� Surface revenue increased 3%. - -On March 4, 2022, we completed our acquisition of Nuance Communications, Inc. (�Nuance�) for a total purchase price of $ 18.8 billion, consisting primarily of cash. Nuance is a cloud and artificial intelligence (�AI�) software provider with healthcare and enterprise AI experience, and the acquisition will build on our industry-specific cloud offerings. The financial results of Nuance have been included in our consolidated financial statements since the date of the acquisition. Nuance is reported as part of our Intelligent Cloud segment. Refer to Note 8 � Business Combinations of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for further discussion. - -Industry Trends - -Our industry is dynamic and highly competitive, with frequent changes in both technologies and business models. Each industry shift is an opportunity to conceive new products, new technologies, or new ideas that can further transform the industry and our business. At Microsoft, we push the boundaries of what is possible through a broad range of research and development activities that seek to identify and address the changing demands of customers and users, industry trends, and competitive forces. - -40 - - -PART II -Item 7 - - -Economic Conditions, Challenges, and Risks - -The markets for software, devices, and cloud-based services are dynamic and highly competitive. Our competitors are developing new software and devices, while also deploying competing cloud-based services for consumers and businesses. The devices and form factors customers prefer evolve rapidly, and influence how users access services in the cloud, and in some cases, the user�s choice of which suite of cloud-based services to use. We must continue to evolve and adapt over an extended time in pace with this changing environment. The investments we are making in infrastructure and devices will continue to increase our operating costs and may decrease our operating margins. - -Our success is highly dependent on our ability to attract and retain qualified employees. We hire a mix of university and industry talent worldwide. We compete for talented individuals globally by offering an exceptional working environment, broad customer reach, scale in resources, the ability to grow one�s career across many different products and businesses, and competitive compensation and benefits. Aggregate demand for our software, services, and devices is correlated to global macroeconomic and geopolitical factors, which remain dynamic. - -Our devices are primarily manufactured by third-party contract manufacturers, some of which contain certain components for which there are very few qualified suppliers. For these components, we have limited near-term flexibility to use other manufacturers if a current vendor becomes unavailable or is unable to meet our requirements. Extended disruptions at these suppliers and/or manufacturers could lead to a similar disruption in our ability to manufacture devices on time to meet consumer demand. - -Our international operations provide a significant portion of our total revenue and expenses. Many of these revenue and expenses are denominated in currencies other than the U.S. dollar. As a result, changes in foreign exchange rates may significantly affect revenue and expenses. Fluctuations in the U.S. dollar relative to certain foreign currencies did not have a material impact on reported revenue or expenses from our international operations in fiscal year 2022. - -Refer to Risk Factors (Part I, Item 1A of this Form 10-K) for a discussion of these factors and other risks. - -Seasonality - -Our revenue fluctuates quarterly and is generally higher in the second and fourth quarters of our fiscal year. Second quarter revenue is driven by corporate year-end spending trends in our major markets and holiday season spending by consumers, and fourth quarter revenue is driven by the volume of multi-year on-premises contracts executed during the period. - -Reportable Segments - -We report our financial performance based on the following segments: Productivity and Business Processes, Intelligent Cloud, and More Personal Computing. The segment amounts included in MD&A are presented on a basis consistent with our internal management reporting. Additional information on our reportable segments is contained in Note 19 � Segment Information and Geographic Data of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K). - -Metrics - -We use metrics in assessing the performance of our business and to make informed decisions regarding the allocation of resources. We disclose metrics to enable investors to evaluate progress against our ambitions, provide transparency into performance trends, and reflect the continued evolution of our products and services. Our commercial and other business metrics are fundamentally connected based on how customers use our products and services. The metrics are disclosed in the MD&A or the Notes to Financial Statements (Part II, Item 8 of this Form 10-K). Financial metrics are calculated based on financial results prepared in accordance with accounting principles generally accepted in the United States of America (�GAAP�), and growth comparisons relate to the corresponding period of last fiscal year. - -In the first quarter of fiscal year 2022, we made updates to the presentation and method of calculation for certain metrics, most notably changes to incorporate all current and anticipated revenue streams within our Office Consumer and Server products and cloud services metrics and changes to align with how we manage our Windows OEM and Search and news advertising businesses. None of these changes had a material impact on previously reported amounts in our MD&A. - -41 - - -PART II -Item 7 - -In the third quarter of fiscal year 2022, we completed our acquisition of Nuance. Nuance is included in all commercial metrics and our Server products and cloud services revenue growth metric. Azure and other cloud services revenue includes Nuance cloud services, and Server products revenue includes Nuance on-premises offerings. - -Commercial - -Our commercial business primarily consists of Server products and cloud services, Office Commercial, Windows Commercial, the commercial portion of LinkedIn, Enterprise Services, and Dynamics. Our commercial metrics allow management and investors to assess the overall health of our commercial business and include leading indicators of future performance. - - -Commercial remaining performance obligation - - - -Commercial portion of revenue allocated to remaining performance obligations, which includes unearned revenue and amounts that will be invoiced and recognized as revenue in future periods - - - -Microsoft Cloud revenue - - -Revenue from Azure -commercial portion of - - -and other LinkedIn, - - -cloud services, Dynamics 365, - - -Office 365 and other - - - Commercial, the commercial cloud - - -properties - - -Microsoft Cloud gross margin percentage - - - -Gross margin percentage for our Microsoft Cloud business - - - -Productivity and Business Processes and Intelligent Cloud - -Metrics related to our Productivity and Business Processes and Intelligent Cloud segments assess the health of our core businesses within these segments. The metrics reflect our cloud and on-premises product strategies and trends. - -Office Commercial products and cloud services revenue growth Revenue from Office Commercial products and cloud services (Office 365 subscriptions, the Office 365 portion of Microsoft 365 Commercial subscriptions, and Office licensed on-premises), comprising Office, Exchange, SharePoint, Microsoft Teams, Office 365 Security and Compliance, and Microsoft Viva - -Office Consumer products and cloud services revenue growth -Revenue from Office Consumer products and cloud services, including - -Microsoft 365 Consumer subscriptions, Office licensed on-premises, and - -other Office services -Office 365 Commercial seat growth -The number of Office 365 Commercial seats at end of period where seats - -are paid users covered by an Office 365 Commercial subscription -Microsoft 365 Consumer subscribers -The number of Microsoft 365 Consumer subscribers at end of period -Dynamics products and cloud services revenue growth -Revenue from Dynamics products and cloud services, including Dynamics - -365, comprising a set of intelligent, cloud-based applications across ERP, - -CRM, Customer Insights, Power Apps, and Power Automate; and on- - -premises ERP and CRM applications -LinkedIn revenue growth -Revenue from LinkedIn, including Talent Solutions, Marketing Solutions, - -Premium Subscriptions, and Sales Solutions -Server products and cloud services revenue growth -Revenue from Server products and cloud services, including Azure and - -other cloud services; SQL Server, Windows Server, Visual Studio, System - -Center, and related Client Access Licenses (�CALs�); and Nuance and - -GitHub - -42 - - - - -PART II -Item 7 - - -More Personal Computing - -Metrics related to our More Personal Computing segment assess the performance of key lines of business within this segment. These metrics provide strategic product insights which allow us to assess the performance across our commercial and consumer businesses. As we have diversity of target audiences and sales motions within the Windows business, we monitor metrics that are reflective of those varying motions. - -Windows OEM revenue growth -Revenue from sales of Windows Pro and non-Pro licenses sold - - -through the OEM channel - - - - -Windows Commercial products and cloud services revenue growth -Revenue from Windows Commercial products and cloud services, - - -comprising volume licensing of the Windows operating system, - - -Windows cloud services, and other Windows commercial offerings -Surface revenue growth -Revenue from Surface devices and accessories - - -Xbox content and services revenue growth -Revenue from Xbox content and services, comprising first- and third- - - -party content (including games and in-game content), Xbox Game - - -Pass and other subscriptions, Xbox Cloud Gaming, third-party disc - - -royalties, advertising, and other cloud services - - -Search and news advertising revenue, excluding TAC, growth -Revenue from search and news advertising excluding traffic acquisition - - -costs (�TAC�) paid to Bing Ads network publishers and news partners - -SUMMARY RESULTS OF OPERATIONS - - - - - - - - - - - -Percentage -(In millions, except percentages and per share amounts) - -2022 - -2021 -Change - - - - - - - - - - - - - - - - - - -Revenue -$ -198,270 -$ -168,088 -18% -Gross margin - -135,620 - -115,856 -17% -Operating income - -83,383 - -69,916 -19% -Net income - -72,738 - -61,271 -19% -Diluted earnings per share - -9.65 - -8.05 -20% -Adjusted net income (non-GAAP) - -69,447 - -60,651 -15% -Adjusted diluted earnings per share (non-GAAP) - -9.21 - -7.97 -16% - - - - - - - - - - -Adjusted net income and adjusted diluted earnings per share (�EPS�) are non-GAAP financial measures which exclude the net income tax benefit related to transfer of intangible properties in the first quarter of fiscal year 2022 and the net income tax benefit related to an India Supreme Court decision on withholding taxes in the third quarter of fiscal year 2021. Refer to the Non-GAAP Financial Measures section below for a reconciliation of our financial results reported in accordance with GAAP to non-GAAP financial results. See Note 12 � Income Taxes of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for further discussion. - -Fiscal Year 2022 Compared with Fiscal Year 2021 - -Revenue increased $30.2 billion or 18% driven by growth across each of our segments. Intelligent Cloud revenue increased driven by Azure and other cloud services. Productivity and Business Processes revenue increased driven by Office 365 Commercial and LinkedIn. More Personal Computing revenue increased driven by Search and news advertising and Windows. - -Cost of revenue increased $10.4 billion or 20% driven by growth in Microsoft Cloud. - -Gross margin increased $19.8 billion or 17% driven by growth across each of our segments. - -� Gross margin percentage decreased slightly. Excluding the impact of the fiscal year 2021 change in accounting estimate for the useful lives of our server and network equipment, gross margin percentage increased 1 point driven by improvement in Productivity and Business Processes. - -� Microsoft Cloud gross margin percentage decreased slightly to 70%. Excluding the impact of the change in accounting estimate, Microsoft Cloud gross margin percentage increased 3 points driven by improvement across our cloud services, offset in part by sales mix shift to Azure and other cloud services. - -43 - - -PART II -Item 7 - -Operating expenses increased $6.3 billion or 14% driven by investments in cloud engineering, LinkedIn, Gaming, and commercial sales. - -Key changes in operating expenses were: - -� Research and development expenses increased $3.8 billion or 18% driven by investments in cloud engineering, Gaming, and LinkedIn. - -� Sales and marketing expenses increased $1.7 billion or 8% driven by investments in commercial sales and LinkedIn. Sales and marketing included a favorable foreign currency impact of 2%. - -� General and administrative expenses increased $793 million or 16% driven by investments in corporate functions. - -Operating income increased $13.5 billion or 19% driven by growth across each of our segments. - -Current year net income and diluted EPS were positively impacted by the net tax benefit related to the transfer of intangible properties, which resulted in an increase to net income and diluted EPS of $3.3 billion and $0.44, respectively. Prior year net income and diluted EPS were positively impacted by the net tax benefit related to the India Supreme Court decision on withholding taxes, which resulted in an increase to net income and diluted EPS of $620 million and $0.08, respectively. - -Gross margin and operating income both included an unfavorable foreign currency impact of 2%. - -SEGMENT RESULTS OF OPERATIONS - - - - - - - - -Percentage - -(In millions, except percentages) - - -2022 - -2021 -Change - - - - - - - - - - - -Revenue - - - - - - - - - - - - - - - - - - - -Productivity and Business Processes -$ -63,364 -$ -53,915 -18% - -Intelligent Cloud - - -75,251 - -60,080 -25% - -More Personal Computing - - -59,655 - -54,093 -10% - - - - - - - - - - -Total -$ -198,270 -$ -168,088 -18% - -Operating Income - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Productivity and Business Processes -$ -29,687 -$ -24,351 -22% - -Intelligent Cloud - - -32,721 - -26,126 -25% - -More Personal Computing - - -20,975 - -19,439 -8% - - - - - - - - - - -Total -$ -83,383 -$ -69,916 -19% - - - - - - - - - - - - -Reportable Segments - -Fiscal Year 2022 Compared with Fiscal Year 2021 - -Productivity and Business Processes - -Revenue increased $9.4 billion or 18%. - -� Office Commercial products and cloud services revenue increased $4.4 billion or 13%. Office 365 Commercial revenue grew 18% driven by seat growth of 14%, with continued momentum in small and medium business and frontline worker offerings, as well as growth in revenue per user. Office Commercial products revenue declined 22% driven by continued customer shift to cloud offerings. - -� Office Consumer products and cloud services revenue increased $641 million or 11% driven by Microsoft 365 Consumer subscription revenue. Microsoft 365 Consumer subscribers grew 15% to 59.7 million. - -� LinkedIn revenue increased $3.5 billion or 34% driven by a strong job market in our Talent Solutions business and advertising demand in our Marketing Solutions business. - -� Dynamics products and cloud services revenue increased 25% driven by Dynamics 365 growth of 39%. - -44 - - -PART II -Item 7 - - -Operating income increased $5.3 billion or 22%. - -� Gross margin increased $7.3 billion or 17% driven by growth in Office 365 Commercial and LinkedIn. Gross margin percentage was relatively unchanged. Excluding the impact of the change in accounting estimate, gross margin percentage increased 2 points driven by improvement across all cloud services. - -� Operating expenses increased $2.0 billion or 11% driven by investments in LinkedIn and cloud engineering. - -Gross margin and operating income both included an unfavorable foreign currency impact of 2%. - -Intelligent Cloud - -Revenue increased $15.2 billion or 25%. - -� Server products and cloud services revenue increased $14.7 billion or 28% driven by Azure and other cloud services. Azure and other cloud services revenue grew 45% driven by growth in our consumption-based services. Server products revenue increased 5% driven by hybrid solutions, including Windows Server and SQL Server running in multi-cloud environments. - -� Enterprise Services revenue increased $464 million or 7% driven by growth in Enterprise Support Services. - -Operating income increased $6.6 billion or 25%. - -� Gross margin increased $9.4 billion or 22% driven by growth in Azure and other cloud services. Gross margin percentage decreased. Excluding the impact of the change in accounting estimate, gross margin percentage was relatively unchanged driven by improvement in Azure and other cloud services, offset in part by sales mix shift to Azure and other cloud services. - -� Operating expenses increased $2.8 billion or 16% driven by investments in Azure and other cloud services. - -Revenue and operating income included an unfavorable foreign currency impact of 2% and 3%, respectively. - -More Personal Computing - -Revenue increased $5.6 billion or 10%. - -� Windows revenue increased $2.3 billion or 10% driven by growth in Windows OEM and Windows Commercial. Windows OEM revenue increased 11% driven by continued strength in the commercial PC market, which has higher revenue per license. Windows Commercial products and cloud services revenue increased 11% driven by demand for Microsoft 365. - -� Search and news advertising revenue increased $2.3 billion or 25%. Search and news advertising revenue excluding traffic acquisition costs increased 27% driven by higher revenue per search and search volume. - -� Gaming revenue increased $860 million or 6% on a strong prior year comparable that benefited from Xbox Series X|S launches and stay-at-home scenarios, driven by growth in Xbox hardware and Xbox content and services. Xbox hardware revenue increased 16% due to continued demand for Xbox Series X|S. Xbox content and services revenue increased 3% driven by growth in Xbox Game Pass subscriptions and first-party content, offset in part by a decline in third-party content. - -� Surface revenue increased $226 million or 3%. - -Operating income increased $1.5 billion or 8%. - -� Gross margin increased $3.1 billion or 10% driven by growth in Windows and Search and news advertising. Gross margin percentage was relatively unchanged. - -� Operating expenses increased $1.5 billion or 14% driven by investments in Gaming, Search and news advertising, and Windows marketing. - -45 - - -PART II -Item 7 - - -OPERATING EXPENSES - -Research and Development - - - - - - -Percentage -(In millions, except percentages) - -2022 - -2021 -Change - - - - - - - - - - - - - - -Research and development -$ -24,512 -$ -20,716 -18% - -As a percent of revenue - -12% - -12% -0ppt - - -Research and development expenses include payroll, employee benefits, stock-based compensation expense, and other headcount-related expenses associated with product development. Research and development expenses also include third-party development and programming costs, localization costs incurred to translate software for international markets, and the amortization of purchased software code and services content. - -Research and development expenses increased $3.8 billion or 18% driven by investments in cloud engineering, Gaming, and LinkedIn. - -Sales and Marketing - - - - - - -Percentage -(In millions, except percentages) - -2022 - -2021 -Change - - - - - - - - - - - - - - -Sales and marketing -$ -21,825 -$ -20,117 -8% - -As a percent of revenue - -11% - -12% -(1)ppt - - -Sales and marketing expenses include payroll, employee benefits, stock-based compensation expense, and other headcount-related expenses associated with sales and marketing personnel, and the costs of advertising, promotions, trade shows, seminars, and other programs. - -Sales and marketing expenses increased $1.7 billion or 8% driven by investments in commercial sales and LinkedIn. Sales and marketing included a favorable foreign currency impact of 2%. - -General and Administrative - - - - - - -Percentage -(In millions, except percentages) - -2022 - -2021 -Change - - - - - - - - - - - - - - -General and administrative -$ -5,900 -$ -5,107 -16% - -As a percent of revenue - -3% - -3% -0ppt - - -General and administrative expenses include payroll, employee benefits, stock-based compensation expense, and other headcount-related expenses associated with finance, legal, facilities, certain human resources and other administrative personnel, certain taxes, and legal and other administrative fees. - -General and administrative expenses increased $793 million or 16% driven by investments in corporate functions. - -46 - - -PART II -Item 7 - - -OTHER INCOME (EXPENSE), NET - -The components of other income (expense), net were as follows: - -(In millions) - - -Year Ended June 30, - - -2022 - - -2021 - - - - - - - - - -Interest and dividends income -$ -2,094 -$ -2,131 -Interest expense - - -(2,063) - - -(2,346) -Net recognized gains on investments - - -461 - - -1,232 -Net gains (losses) on derivatives - - -(52) - - -17 -Net gains (losses) on foreign currency remeasurements - - -(75) - - -54 -Other, net - - -(32) - - -98 - - - - - - - - - -Total -$ -333 -$ -1,186 - - - - - - - - - - -We use derivative instruments to manage risks related to foreign currencies, equity prices, interest rates, and credit; enhance investment returns; and facilitate portfolio diversification. Gains and losses from changes in fair values of derivatives that are not designated as hedging instruments are primarily recognized in other income (expense), net. - -Interest and dividends income decreased due to lower portfolio balances. Interest expense decreased due to a decrease in outstanding long-term debt due to debt maturities. Net recognized gains on investments decreased primarily due to lower gains on equity securities. - -INCOME TAXES - -Effective Tax Rate - -Our effective tax rate for fiscal years 2022 and 2021 was 13% and 14%, respectively. The decrease in our effective tax rate was primarily due to a $3.3 billion net income tax benefit in the first quarter of fiscal year 2022 related to the transfer of intangible properties, offset in part by changes in the mix of our income before income taxes between the U.S. and foreign countries, as well as tax benefits in the prior year from the India Supreme Court decision on withholding taxes in the case of Engineering Analysis Centre of Excellent Private Limited vs The Commissioner of Income Tax, an agreement between the U.S. and India tax authorities related to transfer pricing, and final Tax Cuts and Jobs Act (�TCJA�) regulations. - -In the first quarter of fiscal year 2022, we transferred certain intangible properties from our Puerto Rico subsidiary to the U.S. The transfer of intangible properties resulted in a $3.3 billion net income tax benefit in the first quarter of fiscal year 2022, as the value of future U.S. tax deductions exceeds the current tax liability from the U.S. global intangible low-taxed income tax. - -We have historically paid India withholding taxes on software sales through distributor withholding and tax audit assessments in India. In March 2021, the India Supreme Court ruled favorably for companies in 86 separate appeals, some dating back to 2012, holding that software sales are not subject to India withholding taxes. Although we were not a party to the appeals, our software sales in India were determined to be not subject to withholding taxes. Therefore, we recorded a net income tax benefit of $ 620 million in the third quarter of fiscal year 2021 to reflect the results of the India Supreme Court decision impacting fiscal year 1996 through fiscal year 2016. - -Our effective tax rate was lower than the U.S. federal statutory rate, primarily due to the net income tax benefit related to the transfer of intangible properties, earnings taxed at lower rates in foreign jurisdictions resulting from producing and distributing our products and services through our foreign regional operations center in Ireland, and tax benefits relating to stock-based compensation. - -The mix of income before income taxes between the U.S. and foreign countries impacted our effective tax rate as a result of the geographic distribution of, and customer demand for, our products and services. In fiscal year 2022, our U.S. income before income taxes was $47.8 billion and our foreign income before income taxes was $35.9 billion. In fiscal year 2021, our U.S. income before income taxes was $35.0 billion and our foreign income before income taxes was $36.1 billion. - -47 - - -PART II -Item 7 - - -Uncertain Tax Positions - -We settled a portion of the Internal Revenue Service (�IRS�) audit for tax years 2004 to 2006 in fiscal year 2011. In February 2012, the IRS withdrew its 2011 Revenue Agents Report related to unresolved issues for tax years 2004 to 2006 and reopened the audit phase of the examination. We also settled a portion of the IRS audit for tax years 2007 to 2009 in fiscal year 2016, and a portion of the IRS audit for tax years 2010 to 2013 in fiscal year 2018. In the second quarter of fiscal year 2021, we settled an additional portion of the IRS audits for tax years 2004 to 2013 and made a payment of $1.7 billion, including tax and interest. We remain under audit for tax years 2004 to 2017. - -As of June 30, 2022, the primary unresolved issues for the IRS audits relate to transfer pricing, which could have a material impact in our consolidated financial statements when the matters are resolved. We believe our allowances for income tax contingencies are adequate. We have not received a proposed assessment for the unresolved key transfer pricing issues and do not expect a final resolution of these issues in the next 12 months. Based on the information currently available, we do not anticipate a significant increase or decrease to our tax contingencies for these issues within the next 12 months. - -We are subject to income tax in many jurisdictions outside the U.S. Our operations in certain jurisdictions remain subject to examination for tax years 1996 to 2021, some of which are currently under audit by local tax authorities. The resolution of each of these audits is not expected to be material to our consolidated financial statements. - -NON-GAAP FINANCIAL MEASURES - -Adjusted net income and adjusted diluted EPS are non-GAAP financial measures which exclude the net tax benefit related to the transfer of intangible properties in the first quarter of fiscal year 2022 and the net income tax benefit related to an India Supreme Court decision on withholding taxes in the third quarter of fiscal year 2021. We believe these non-GAAP measures aid investors by providing additional insight into our operational performance and help clarify trends affecting our business. For comparability of reporting, management considers non-GAAP measures in conjunction with GAAP financial results in evaluating business performance. These non-GAAP financial measures presented should not be considered a substitute for, or superior to, the measures of financial performance prepared in accordance with GAAP. - -The following table reconciles our financial results reported in accordance with GAAP to non-GAAP financial results: - - - - - - - - - - -Percentage - -(In millions, except percentages and per share amounts) - -2022 - - -2021 -Change - - - - - - - - - - - - - - - - - - - - - - - - -Net income -$ -72,738 -$ -61,271 -19% - -Net income tax benefit related to transfer of intangible properties - -(3,291) - - -0 -* - -Net income tax benefit related to India Supreme Court decision on withholding - -0 - - - - - - - - -taxes - - - - -(620) -* - - - - - - - - - - - - - - -Adjusted net income (non-GAAP) -$ -69,447 -$ -60,651 -15% - - - - - - - - - - - - - - - - - - - - - - - - -Diluted earnings per share -$ -9.65 -$ -8.05 -20% - -Net income tax benefit related to transfer of intangible properties - -(0.44) - - -0 -* - -Net income tax benefit related to India Supreme Court decision on withholding - -0 - - - - - - - - -taxes - - - - -(0.08) -* - - - - - - - - - - - - - - -Adjusted diluted earnings per share (non-GAAP) -$ -9.21 -$ -7.97 -16% - -* -Not meaningful. - - - - - - - - - - - - - - - - - - - - - - - -48 - - -PART II -Item 7 - - - -LIQUIDITY AND CAPITAL RESOURCES - -We expect existing cash, cash equivalents, short-term investments, cash flows from operations, and access to capital markets to continue to be sufficient to fund our operating activities and cash commitments for investing and financing activities, such as dividends, share repurchases, debt maturities, material capital expenditures, and the transition tax related to the TCJA, for at least the next 12 months and thereafter for the foreseeable future. - -Cash, Cash Equivalents, and Investments - -Cash, cash equivalents, and short-term investments totaled $104.8 billion and $130.3 billion as of June 30, 2022 and 2021, respectively. Equity investments were $6.9 billion and $6.0 billion as of June 30, 2022 and 2021, respectively. Our short-term investments are primarily intended to facilitate liquidity and capital preservation. They consist predominantly of highly liquid investment-grade fixed-income securities, diversified among industries and individual issuers. The investments are predominantly U.S. dollar-denominated securities, but also include foreign currency-denominated securities to diversify risk. Our fixed-income investments are exposed to interest rate risk and credit risk. The credit risk and average maturity of our fixed-income portfolio are managed to achieve economic returns that correlate to certain fixed-income indices. The settlement risk related to these investments is insignificant given that the short-term investments held are primarily highly liquid investment-grade fixed-income securities. - -Valuation - -In general, and where applicable, we use quoted prices in active markets for identical assets or liabilities to determine the fair value of our financial instruments. This pricing methodology applies to our Level 1 investments, such as U.S. government securities, common and preferred stock, and mutual funds. If quoted prices in active markets for identical assets or liabilities are not available to determine fair value, then we use quoted prices for similar assets and liabilities or inputs other than the quoted prices that are observable either directly or indirectly. This pricing methodology applies to our Level 2 investments, such as commercial paper, certificates of deposit, U.S. agency securities, foreign government bonds, mortgage- and asset-backed securities, corporate notes and bonds, and municipal securities. Level 3 investments are valued using internally-developed models with unobservable inputs. Assets and liabilities measured at fair value on a recurring basis using unobservable inputs are an immaterial portion of our portfolio. - -A majority of our investments are priced by pricing vendors and are generally Level 1 or Level 2 investments as these vendors either provide a quoted market price in an active market or use observable inputs for their pricing without applying significant adjustments. Broker pricing is used mainly when a quoted price is not available, the investment is not priced by our pricing vendors, or when a broker price is more reflective of fair values in the market in which the investment trades. Our broker-priced investments are generally classified as Level 2 investments because the broker prices these investments based on similar assets without applying significant adjustments. In addition, all our broker-priced investments have a sufficient level of trading volume to demonstrate that the fair values used are appropriate for these investments. Our fair value processes include controls that are designed to ensure appropriate fair values are recorded. These controls include model validation, review of key model inputs, analysis of period-over-period fluctuations, and independent recalculation of prices where appropriate. - -Cash Flows - -Cash from operations increased $12.3 billion to $89.0 billion for fiscal year 2022, mainly due to an increase in cash received from customers, offset in part by an increase in cash paid to suppliers and employees. Cash used in financing increased $10.4 billion to $58.9 billion for fiscal year 2022, mainly due to a $5.3 billion increase in common stock repurchases and a $5.3 billion increase in repayments of debt. Cash used in investing increased $2.7 billion to $30.3 billion for fiscal year 2022, mainly due to a $13.1 billion increase in cash used for acquisitions of companies, net of cash acquired, and purchases of intangible and other assets, and a $3.3 billion increase in additions to property and equipment, offset in part by a $15.6 billion increase in cash from net investment purchases, sales, and maturities. - -49 - - -PART II -Item 7 - - -Debt Proceeds - -We issue debt to take advantage of favorable pricing and liquidity in the debt markets, reflecting our credit rating and the low interest rate environment. The proceeds of these issuances were or will be used for general corporate purposes, which may include, among other things, funding for working capital, capital expenditures, repurchases of capital stock, acquisitions, and repayment of existing debt. In March 2021 and June 2020, we exchanged a portion of our existing debt at a premium for cash and new debt with longer maturities to take advantage of favorable financing rates in the debt markets, reflecting our credit rating and the low interest rate environment. Refer to Note 11 � Debt of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for further discussion. - -Unearned Revenue - -Unearned revenue comprises mainly unearned revenue related to volume licensing programs, which may include Software Assurance (�SA�) and cloud services. Unearned revenue is generally invoiced annually at the beginning of each contract period for multi-year agreements and recognized ratably over the coverage period. Unearned revenue also includes payments for other offerings for which we have been paid in advance and earn the revenue when we transfer control of the product or service. Refer to Note 1 � Accounting Policies of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for further discussion. - -The following table outlines the expected future recognition of unearned revenue as of June 30, 2022: - -(In millions) - - -Three Months Ending - - - - - - - -September 30, 2022 -$ -17,691 - -December 31, 2022 - -13,923 - -March 31, 2023 - -9,491 - -June 30, 2023 - -4,433 - -Thereafter - -2,870 - - - - - -Total -$ -48,408 - - - - - - -If our customers choose to license cloud-based versions of our products and services rather than licensing transaction-based products and services, the associated revenue will shift from being recognized at the time of the transaction to being recognized over the subscription period or upon consumption, as applicable. - -50 - - -PART II -Item 7 - - -Material Cash Requirements and Other Obligations - -Contractual Obligations - -The following table summarizes the payments due by fiscal year for our outstanding contractual obligations as of June 30, 2022: - -(In millions) - -2023 - - -Thereafter - -Total - - - - - - - - - - - - - - - - - - - - -Long-term debt: (a) - - - - - - - - - -Principal payments -$ -2,750 -$ -52,761 -$ -55,511 -Interest payments - -1,468 - -21,139 - -22,607 -Construction commitments (b) - -7,942 - -576 - - -8,518 -Operating and finance leases, including imputed interest (c) - -4,609 - -44,045 - -48,654 -Purchase commitments (d) - -42,669 - -2,985 - -45,654 - - - - - - - - - - -Total -$ -59,438 -$ -121,506 -$ -180,944 - - - - - - - - - - - -(a) Refer to Note 11 � Debt of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K). - -(b) Refer to Note 7 � Property and Equipment of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K). -(c) Refer to Note 14 � Leases of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K). -(d) Purchase commitments primarily relate to datacenters and include open purchase orders and take-or-pay contracts that are not presented as construction commitments above. - -Income Taxes - -As a result of the TCJA, we are required to pay a one-time transition tax on deferred foreign income not previously subject to U.S. income tax. Under the TCJA, the transition tax is payable in interest-free installments over eight years, with 8% due in each of the first five years, 15% in year six, 20% in year seven, and 25% in year eight. We have paid transition tax of $6.2 billion, which included $1.5 billion for fiscal year 2022. The remaining transition tax of $12.0 billion is payable over the next four years, with $1.3 billion payable within 12 months. - -Provisions enacted in the TCJA related to the capitalization for tax purposes of research and experimental expenditures became effective on July 1, 2022. These provisions require us to capitalize research and experimental expenditures and amortize them on the U.S. tax return over five or fifteen years, depending on where research is conducted. The final foreign tax credit regulations, also effective on July 1, 2022, introduced significant changes to foreign tax credit calculations in the U.S. tax return. While these provisions are not expected to have a material impact on our fiscal year 2023 effective tax rate on a net basis, our cash paid for taxes would increase unless these provisions are postponed or modified through legislative processes. - -Share Repurchases - -During fiscal years 2022 and 2021, we repurchased 95 million shares and 101 million shares of our common stock for $28.0 billion and $23.0 billion, respectively, through our share repurchase programs. All repurchases were made using cash resources. As of June 30, 2022, $40.7 billion remained of our $60 billion share repurchase program. Refer to Note 16 � Stockholders� Equity of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for further discussion. - -Dividends - -During fiscal year 2022, our Board of Directors declared quarterly dividends of $0.62 per share. We intend to continue returning capital to shareholders in the form of dividends, subject to declaration by our Board of Directors. Refer to Note 16 � Stockholders� Equity of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for further discussion. - -51 - - -PART II -Item 7 - - -Other Planned Uses of Capital - -On January 18, 2022, we entered into a definitive agreement to acquire Activision Blizzard, Inc. (�Activision Blizzard�) for $95.00 per share in an all-cash transaction valued at $68.7 billion, inclusive of Activision Blizzard�s net cash. The acquisition has been approved by Activision Blizzard�s shareholders, and we expect it to close in fiscal year 2023, subject to the satisfaction of certain regulatory approvals and other customary closing conditions. - -We will continue to invest in sales, marketing, product support infrastructure, and existing and advanced areas of technology, as well as continue making acquisitions that align with our business strategy. Additions to property and equipment will continue, including new facilities, datacenters, and computer systems for research and development, sales and marketing, support, and administrative staff. We expect capital expenditures to increase in coming years to support growth in our cloud offerings. We have operating and finance leases for datacenters, corporate offices, research and development facilities, Microsoft Experience Centers, and certain equipment. We have not engaged in any related party transactions or arrangements with unconsolidated entities or other persons that are reasonably likely to materially affect liquidity or the availability of capital resources. - -RECENT ACCOUNTING GUIDANCE - -Refer to Note 1 � Accounting Policies of the Notes to Financial Statements (Part II, Item 8 of this Form 10-K) for further discussion. - -CRITICAL ACCOUNTING ESTIMATES - -Our consolidated financial statements and accompanying notes are prepared in accordance with GAAP. Preparing consolidated financial statements requires management to make estimates and assumptions that affect the reported amounts of assets, liabilities, revenue, and expenses. Critical accounting estimates are those estimates that involve a significant level of estimation uncertainty and could have a material impact on our financial condition or results of operations. We have critical accounting estimates in the areas of revenue recognition, impairment of investment securities, goodwill, research and development costs, legal and other contingencies, income taxes, and inventories. - -Revenue Recognition - -Our contracts with customers often include promises to transfer multiple products and services to a customer. Determining whether products and services are considered distinct performance obligations that should be accounted for separately versus together may require significant judgment. When a cloud-based service includes both on-premises software licenses and cloud services, judgment is required to determine whether the software license is considered distinct and accounted for separately, or not distinct and accounted for together with the cloud service and recognized over time. Certain cloud services, primarily Office 365, depend on a significant level of integration, interdependency, and interrelation between the desktop applications and cloud services, and are accounted for together as one performance obligation. Revenue from Office 365 is recognized ratably over the period in which the cloud services are provided. - -Judgment is required to determine the stand-alone selling price (�SSP") for each distinct performance obligation. We use a single amount to estimate SSP for items that are not sold separately, including on-premises licenses sold with SA or software updates provided at no additional charge. We use a range of amounts to estimate SSP when we sell each of the products and services separately and need to determine whether there is a discount to be allocated based on the relative SSP of the various products and services. - -In instances where SSP is not directly observable, such as when we do not sell the product or service separately, we determine the SSP using information that may include market conditions and other observable inputs. We typically have more than one SSP for individual products and services due to the stratification of those products and services by customers and circumstances. In these instances, we may use information such as the size of the customer and geographic region in determining the SSP. - -Due to the various benefits from and the nature of our SA program, judgment is required to assess the pattern of delivery, including the exercise pattern of certain benefits across our portfolio of customers. - -52 - - -PART II -Item 7 - -Our products are generally sold with a right of return, we may provide other credits or incentives, and in certain instances we estimate customer usage of our products and services, which are accounted for as variable consideration when determining the amount of revenue to recognize. Returns and credits are estimated at contract inception and updated at the end of each reporting period if additional information becomes available. Changes to our estimated variable consideration were not material for the periods presented. - -Impairment of Investment Securities - -We review debt investments quarterly for credit losses and impairment. If the cost of an investment exceeds its fair value, we evaluate, among other factors, general market conditions, credit quality of debt instrument issuers, and the extent to which the fair value is less than cost. This determination requires significant judgment. In making this judgment, we employ a systematic methodology that considers available quantitative and qualitative evidence in evaluating potential impairment of our investments. In addition, we consider specific adverse conditions related to the financial health of, and business outlook for, the investee. If we have plans to sell the security or it is more likely than not that we will be required to sell the security before recovery, then a decline in fair value below cost is recorded as an impairment charge in other income (expense), net and a new cost basis in the investment is established. If market, industry, and/or investee conditions deteriorate, we may incur future impairments. - -Equity investments without readily determinable fair values are written down to fair value if a qualitative assessment indicates that the investment is impaired and the fair value of the investment is less than carrying value. We perform a qualitative assessment on a periodic basis. We are required to estimate the fair value of the investment to determine the amount of the impairment loss. Once an investment is determined to be impaired, an impairment charge is recorded in other income (expense), net. - -Goodwill - -We allocate goodwill to reporting units based on the reporting unit expected to benefit from the business combination. We evaluate our reporting units on an annual basis and, if necessary, reassign goodwill using a relative fair value allocation approach. Goodwill is tested for impairment at the reporting unit level (operating segment or one level below an operating segment) on an annual basis (May 1 for us) and between annual tests if an event occurs or circumstances change that would more likely than not reduce the fair value of a reporting unit below its carrying value. These events or circumstances could include a significant change in the business climate, legal factors, operating performance indicators, competition, or sale or disposition of a significant portion of a reporting unit. - -Application of the goodwill impairment test requires judgment, including the identification of reporting units, assignment of assets and liabilities to reporting units, assignment of goodwill to reporting units, and determination of the fair value of each reporting unit. The fair value of each reporting unit is estimated primarily through the use of a discounted cash flow methodology. This analysis requires significant judgments, including estimation of future cash flows, which is dependent on internal forecasts, estimation of the long-term rate of growth for our business, estimation of the useful life over which cash flows will occur, and determination of our weighted average cost of capital. - -The estimates used to calculate the fair value of a reporting unit change from year to year based on operating results, market conditions, and other factors. Changes in these estimates and assumptions could materially affect the determination of fair value and goodwill impairment for each reporting unit. - -Research and Development Costs - -Costs incurred internally in researching and developing a computer software product are charged to expense until technological feasibility has been established for the product. Once technological feasibility is established, software costs are capitalized until the product is available for general release to customers. Judgment is required in determining when technological feasibility of a product is established. We have determined that technological feasibility for our software products is reached after all high-risk development issues have been resolved through coding and testing. Generally, this occurs shortly before the products are released to production. The amortization of these costs is included in cost of revenue over the estimated life of the products. - -53 - - -PART II -Item 7 - - -Legal and Other Contingencies - -The outcomes of legal proceedings and claims brought against us are subject to significant uncertainty. An estimated loss from a loss contingency such as a legal proceeding or claim is accrued by a charge to income if it is probable that an asset has been impaired or a liability has been incurred and the amount of the loss can be reasonably estimated. In determining whether a loss should be accrued we evaluate, among other factors, the degree of probability of an unfavorable outcome and the ability to make a reasonable estimate of the amount of loss. Changes in these factors could materially impact our consolidated financial statements. - -Income Taxes - -The objectives of accounting for income taxes are to recognize the amount of taxes payable or refundable for the current year, and deferred tax liabilities and assets for the future tax consequences of events that have been recognized in an entity�s financial statements or tax returns. We recognize the tax benefit from an uncertain tax position only if it is more likely than not that the tax position will be sustained on examination by the taxing authorities, based on the technical merits of the position. The tax benefits recognized in the financial statements from such a position are measured based on the largest benefit that has a greater than 50% likelihood of being realized upon ultimate settlement. Accounting literature also provides guidance on derecognition of income tax assets and liabilities, classification of deferred income tax assets and liabilities, accounting for interest and penalties associated with tax positions, and income tax disclosures. Judgment is required in assessing the future tax consequences of events that have been recognized in our consolidated financial statements or tax returns. Variations in the actual outcome of these future tax consequences could materially impact our consolidated financial statements. - -Inventories - -Inventories are stated at average cost, subject to the lower of cost or net realizable value. Cost includes materials, labor, and manufacturing overhead related to the purchase and production of inventories. Net realizable value is the estimated selling price less estimated costs of completion, disposal, and transportation. We regularly review inventory quantities on hand, future purchase commitments with our suppliers, and the estimated utility of our inventory. These reviews include analysis of demand forecasts, product life cycle status, product development plans, current sales levels, pricing strategy, and component cost trends. If our review indicates a reduction in utility below carrying value, we reduce our inventory to a new cost basis through a charge to cost of revenue. - -CHANGE IN ACCOUNTING ESTIMATE - -In July 2022, we completed an assessment of the useful lives of our server and network equipment. Due to investments in software that increased efficiencies in how we operate our server and network equipment, as well as advances in technology, we determined we should increase the estimated useful lives of both server and network equipment from four years to six years. This change in accounting estimate will be effective beginning fiscal year 2023. Based on the carrying amount of server and network equipment included in property and equipment, net as of June 30, 2022, it is estimated this change will increase our fiscal year 2023 operating income by $3.7 billion. We had previously increased the estimated useful lives of both server and network equipment in July 2020. - - -54 - - -PART II -Item 7 - - -STATEMENT OF MANAGEMENT�S RESPONSIBILITY FOR FINANCIAL STATEMENTS - -Management is responsible for the preparation of the consolidated financial statements and related information that are presented in this report. The consolidated financial statements, which include amounts based on management�s estimates and judgments, have been prepared in conformity with accounting principles generally accepted in the United States of America. - -The Company designs and maintains accounting and internal control systems to provide reasonable assurance at reasonable cost that assets are safeguarded against loss from unauthorized use or disposition, and that the financial records are reliable for preparing consolidated financial statements and maintaining accountability for assets. These systems are augmented by written policies, an organizational structure providing division of responsibilities, careful selection and training of qualified personnel, and a program of internal audits. - -The Company engaged Deloitte & Touche LLP, an independent registered public accounting firm, to audit and render an opinion on the consolidated financial statements and internal control over financial reporting in accordance with the standards of the Public Company Accounting Oversight Board (United States). - -The Board of Directors, through its Audit Committee, consisting solely of independent directors of the Company, meets periodically with management, internal auditors, and our independent registered public accounting firm to ensure that each is meeting its responsibilities and to discuss matters concerning internal controls and financial reporting. Deloitte & Touche LLP and the internal auditors each have full and free access to the Audit Committee. - -Satya Nadella -Chief Executive Officer - -Amy E. Hood -Executive Vice President and Chief Financial Officer - -Alice L. Jolla -Corporate Vice President and Chief Accounting Officer - -55 - - -PART II -Item 7A - -ITEM 7A. QUANTITATIVE AND QUALITATIVE DISCLOSURES ABOUT MARKET RISK - -RISKS - -We are exposed to economic risk from foreign exchange rates, interest rates, credit risk, and equity prices. We use derivatives instruments to manage these risks, however, they may still impact our consolidated financial statements. - -Foreign Currencies - -Certain forecasted transactions, assets, and liabilities are exposed to foreign currency risk. We monitor our foreign currency exposures daily to maximize the economic effectiveness of our foreign currency positions, including hedges. Principal currency exposures include the Euro, Japanese yen, British pound, Canadian dollar, and Australian dollar. - -Interest Rate - -Securities held in our fixed-income portfolio are subject to different interest rate risks based on their maturities. We manage the average maturity of the fixed-income portfolio to achieve economic returns that correlate to certain global fixed-income indices. - -Credit - -Our fixed-income portfolio is diversified and consists primarily of investment-grade securities. We manage credit exposures relative to broad-based indices and to facilitate portfolio diversification. - -Equity - -Securities held in our equity investments portfolio are subject to price risk. - -SENSITIVITY ANALYSIS - -The following table sets forth the potential loss in future earnings or fair values, including associated derivatives, resulting from hypothetical changes in relevant market rates or prices: - -(In millions) - - - - - -June 30, - - -Risk Categories -Hypothetical Change - -2022 -Impact - - - - - - -Foreign currency � Revenue -10% decrease in foreign exchange rates -$ -(6,822) -Earnings -Foreign currency � Investments -10% decrease in foreign exchange rates - -(94) -Fair Value -Interest rate -100 basis point increase in U.S. treasury interest rates - -(2,536) -Fair Value -Credit -100 basis point increase in credit spreads - -(350) -Fair Value -Equity -10% decrease in equity market prices - -(637) -Earnings - - - - - - - - - - -56 - - -PART II -Item 8 - -ITEM 8. FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA - -INCOME STATEMENTS - -(In millions, except per share amounts) - - -Year Ended June 30, - - -2022 - -2021 - - - -2020 - - - - - - - - - - - - - - -Revenue: - - - - - - - - - - - - -Product -$ -72,732 -$ -71,074 -$ -68,041 - -Service and other - - -125,538 - -97,014 - - -74,974 - - - - - - - - - - - - - - -Total revenue - - -198,270 - -168,088 - - - -143,015 - - - - - - - - - - - - - - - - - - - - - - - - - - - -Cost of revenue: - - - - - - - - - - - - -Product - - -19,064 - -18,219 - - -16,017 - -Service and other - - -43,586 - -34,013 - - -30,061 - - - - - - - - - - - - - - -Total cost of revenue - - -62,650 - -52,232 - - -46,078 - - - - - - - - - - - - - - - - - - - - - - - - - - - -Gross margin - - -135,620 - -115,856 - - - -96,937 - -Research and development - - -24,512 - -20,716 - - -19,269 - -Sales and marketing - - -21,825 - -20,117 - - -19,598 - -General and administrative - - -5,900 - -5,107 - - -5,111 - - - - - - - - - - - - - - -Operating income - - -83,383 - -69,916 - - -52,959 - -Other income, net - - -333 - - -1,186 - - -77 - - - - - - - - - - - - - - -Income before income taxes - - -83,716 - -71,102 - - -53,036 - -Provision for income taxes - - -10,978 - -9,831 - - -8,755 - - - - - - - - - - - - - - -Net income -$ -72,738 -$ -61,271 -$ -44,281 - -Earnings per share: - - - - - - - - - - - - - - - - - - - - - - - - - -Basic -$ -9.70 - -$ -8.12 - -$ -5.82 - -Diluted -$ -9.65 - -$ -8.05 - -$ -5.76 - -Weighted average shares outstanding: - - - - - - - - - - - - -Basic - - -7,496 - -7,547 - - -7,610 - -Diluted - - -7,540 - -7,608 - - -7,683 - - - - - - - - - - - - - - - -Refer to accompanying notes. - - -57 - - -PART II -Item 8 - - -COMPREHENSIVE INCOME STATEMENTS - -(In millions) - - -Year Ended June 30, - - -2022 - -2021 - - -2020 - - - - - - - - - - - - - - -Net income -$ -72,738 -$ -61,271 -$ -44,281 - -Other comprehensive income (loss), net of tax: - - - - - - - - - - - - - - - - - - - - - - - - - -Net change related to derivatives - - -6 - - -19 - - -(38) - -Net change related to investments - - -(5,360) - -(2,266) - - -3,990 - -Translation adjustments and other - - -(1,146) - -873 - - -(426) - - - - - - - - - - - - - - -Other comprehensive income (loss) - - -(6,500) - -(1,374) - - -3,526 - - - - - - - - - - - - - - - - - - - - - - - - - - - -Comprehensive income -$ -66,238 -$ -59,897 -$ -47,807 - - - - - - - - - - - - - - - -Refer to accompanying notes. - -58 - - -PART II -Item 8 - - -BALANCE SHEETS - -(In millions) - - -June 30, - -2022 - - - -2021 - - - - - - - - -Assets - - - - - - - -Current assets: - - - - - - - -Cash and cash equivalents -$ -13,931 -$ -14,224 -Short-term investments - -90,826 - - -116,110 - - - - - - - - -Total cash, cash equivalents, and short-term investments - -104,757 - - - -130,334 -Accounts receivable, net of allowance for doubtful accounts of $633 and $751 - -44,261 - - -38,043 -Inventories - -3,742 - - -2,636 -Other current assets - -16,924 - - -13,393 - - - - - - - - -Total current assets - -169,684 - - - -184,406 -Property and equipment, net of accumulated depreciation of $59,660 and $51,351 - -74,398 - - -59,715 -Operating lease right-of-use assets - -13,148 - - -11,088 -Equity investments - -6,891 - - -5,984 -Goodwill - -67,524 - - -49,711 -Intangible assets, net - -11,298 - - -7,800 -Other long-term assets - -21,897 - - -15,075 - - - - - - - - -Total assets -$ -364,840 - -$ -333,779 - - - - - - - - - - - - - - - - -Liabilities and stockholders� equity - - - - - - - -Current liabilities: - - - - - - - -Accounts payable -$ -19,000 -$ -15,163 -Current portion of long-term debt - -2,749 - - -8,072 -Accrued compensation - -10,661 - - -10,057 -Short-term income taxes - -4,067 - - -2,174 -Short-term unearned revenue - -45,538 - - -41,525 -Other current liabilities - -13,067 - - -11,666 - - - - - - - - -Total current liabilities - -95,082 - - -88,657 -Long-term debt - -47,032 - - -50,074 -Long-term income taxes - -26,069 - - -27,190 -Long-term unearned revenue - -2,870 - - -2,616 -Deferred income taxes - -230 - - - -198 -Operating lease liabilities - -11,489 - - -9,629 -Other long-term liabilities - -15,526 - - -13,427 - - - - - - - - -Total liabilities - -198,298 - - - -191,791 - - - - - - - - - - - - - - - - -Commitments and contingencies - - - - - - - -Stockholders� equity: - - - - - - - -Common stock and paid-in capital � shares authorized 24,000; outstanding 7,464 and 7,519 - -86,939 - - -83,111 -Retained earnings - -84,281 - - -57,055 -Accumulated other comprehensive income (loss) - -(4,678) - - -1,822 - - - - - - - - -Total stockholders� equity - -166,542 - - - -141,988 - - - - - - - - - - - - - - - - -Total liabilities and stockholders� equity -$ -364,840 - -$ -333,779 - - - - - - - - - -Refer to accompanying notes. - -59 - - -PART II -Item 8 - - -CASH FLOWS STATEMENTS - -(In millions) - - -Year Ended June 30, - -2022 - - -2021 - -2020 - - - - - - - - - - - - - - - -Operations - - - - - - - - - - - - -Net income -$ -72,738 -$ -61,271 -$ -44,281 - - -Adjustments to reconcile net income to net cash from operations: - - - - - - - - - - - - -Depreciation, amortization, and other - -14,460 - - -11,686 - -12,796 - - -Stock-based compensation expense - -7,502 - - -6,118 - -5,289 - - -Net recognized gains on investments and derivatives - -(409) - - -(1,249) - -(219) - -Deferred income taxes - -(5,702) - - -(150) - -11 - - -Changes in operating assets and liabilities: - - - - - - - - - - - - -Accounts receivable - -(6,834) - - -(6,481) - -(2,577) - -Inventories - -(1,123) - - -(737) - -168 - - -Other current assets - -(709) - - -(932) - -(2,330) - -Other long-term assets - -(2,805) - - -(3,459) - -(1,037) - -Accounts payable - -2,943 - - -2,798 - -3,018 - - -Unearned revenue - -5,109 - - -4,633 - -2,212 - - -Income taxes - -696 - - -(2,309) - -(3,631) - -Other current liabilities - -2,344 - - -4,149 - -1,346 - - -Other long-term liabilities - -825 - - -1,402 - -1,348 - - - - - - - - - - - - - - - -Net cash from operations - -89,035 - - -76,740 - -60,675 - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Financing - - - - - - - - - - - - -Cash premium on debt exchange - -0 - - -(1,754) - -(3,417) - -Repayments of debt - -(9,023) - - -(3,750) - -(5,518) - -Common stock issued - -1,841 - - -1,693 - -1,343 - - -Common stock repurchased - -(32,696) - - -(27,385) - -(22,968) - -Common stock cash dividends paid - -(18,135) - - -(16,521) - -(15,137) - -Other, net - -(863) - - -(769) - -(334) - - - - - - - - - - - - - -Net cash used in financing - -(58,876) - - -(48,486) - -(46,031) - - - - - - - - - - - - - - - - - - - - - - - - - - - -Investing - - - - - - - - - - - - -Additions to property and equipment - -(23,886) - - -(20,622) - -(15,441) - -Acquisition of companies, net of cash acquired, and purchases of intangible and other - -(22,038) - - - - - - - - - -assets - - - - -(8,909) - -(2,521) - -Purchases of investments - -(26,456) - - -(62,924) - -(77,190) - -Maturities of investments - -16,451 - - -51,792 - -66,449 - - -Sales of investments - -28,443 - - -14,008 - -17,721 - - -Other, net - -(2,825) - - -(922) - -(1,241) - - - - - - - - - - - - - -Net cash used in investing - -(30,311) - - -(27,577) - -(12,223) - - - - - - - - - - - - - - - - - - - - - - - - - - - -Effect of foreign exchange rates on cash and cash equivalents - -(141) - - -(29) - -(201) - - - - - - - - - - - - - - -Net change in cash and cash equivalents - -(293) - - -648 - -2,220 - - -Cash and cash equivalents, beginning of period - -14,224 - - -13,576 - -11,356 - - - - - - - - - - - - - - - -Cash and cash equivalents, end of period -$ -13,931 -$ -14,224 -$ -13,576 - - - - - - - - - - - - - - - - -Refer to accompanying notes. - -60 - - -PART II -Item 8 - -STOCKHOLDERS� EQUITY STATEMENTS - -(In millions, except per share amounts) - - -Year Ended June 30, - - -2022 - - -2021 - - -2020 - - - - - - - - - - - - - -Common stock and paid-in capital - - - - - - - - - - - - -Balance, beginning of period -$ -83,111 -$ -80,552 -$ -78,520 -Common stock issued - - -1,841 - - -1,963 - - -1,343 -Common stock repurchased - - -(5,688) - - -(5,539) - - -(4,599) -Stock-based compensation expense - - -7,502 - - -6,118 - - -5,289 -Other, net - - -173 - - - -17 - - -(1) - - - - - - - - - - - - - -Balance, end of period - - -86,939 - - -83,111 - - -80,552 - - - - - - - - - - - - - -Retained earnings - - - - - - - - - - - - - - - - - - - - - - - - - -Balance, beginning of period - - -57,055 - - -34,566 - - -24,150 -Net income - - -72,738 - - -61,271 - - -44,281 -Common stock cash dividends - - -(18,552) - - -(16,871) - - -(15,483) -Common stock repurchased - - -(26,960) - - -(21,879) - - -(18,382) -Cumulative effect of accounting changes - - -0 - - -(32) - - -0 - - - - - - - - - - - - - -Balance, end of period - - -84,281 - - -57,055 - - -34,566 - - - - - - - - - - - - - - - - - - - - - - - - - - -Accumulated other comprehensive income (loss) - - - - - - - - - - - - -Balance, beginning of period - - -1,822 - - -3,186 - - -(340) -Other comprehensive income (loss) - - -(6,500) - - -(1,374) - - -3,526 -Cumulative effect of accounting changes - - -0 - - -10 - - -0 - - - - - - - - - - - - - -Balance, end of period - - -(4,678) - - -1,822 - - -3,186 - - - - - - - - - - - - -Total stockholders� equity -$ -166,542 - -$ -141,988 -$ -118,304 - - - - - - - - - - - - - - - - - - - - - - - - - - -Cash dividends declared per common share -$ -2.48 - -$ -2.24 -$ -2.04 - - - - - - - - - - - - - -Refer to accompanying notes. - - - - - - - - - - - - - -61 - - - - - - - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -NOTES TO FINANCIAL STATEMENTS - -NOTE 1 � ACCOUNTING POLICIES - -Accounting Principles - -Our consolidated financial statements and accompanying notes are prepared in accordance with accounting principles generally accepted in the United States of America (�GAAP�). - -We have recast certain prior period amounts to conform to the current period presentation. The recast of these prior period amounts had no impact on our consolidated balance sheets, consolidated income statements, or consolidated cash flows statements. - -Principles of Consolidation - -The consolidated financial statements include the accounts of Microsoft Corporation and its subsidiaries. Intercompany transactions and balances have been eliminated. - -Estimates and Assumptions - -Preparing financial statements requires management to make estimates and assumptions that affect the reported amounts of assets, liabilities, revenue, and expenses. Examples of estimates and assumptions include: for revenue recognition, determining the nature and timing of satisfaction of performance obligations, and determining the standalone selling price (�SSP�) of performance obligations, variable consideration, and other obligations such as product returns and refunds; loss contingencies; product warranties; the fair value of and/or potential impairment of goodwill and intangible assets for our reporting units; product life cycles; useful lives of our tangible and intangible assets; allowances for doubtful accounts; the market value of, and demand for, our inventory; stock-based compensation forfeiture rates; when technological feasibility is achieved for our products; the potential outcome of uncertain tax positions that have been recognized in our consolidated financial statements or tax returns; and determining the timing and amount of impairments for investments. Actual results and outcomes may differ from management�s estimates and assumptions due to risks and uncertainties. - -In July 2022, we completed an assessment of the useful lives of our server and network equipment. Due to investments in software that increased efficiencies in how we operate our server and network equipment, as well as advances in technology, we determined we should increase the estimated useful lives of both server and network equipment from four years to six years. This change in accounting estimate will be effective beginning fiscal year 2023. We had previously increased the estimated useful lives of both server and network equipment in July 2020. - -Foreign Currencies - -Assets and liabilities recorded in foreign currencies are translated at the exchange rate on the balance sheet date. Revenue and expenses are translated at average rates of exchange prevailing during the year. Translation adjustments resulting from this process are recorded to other comprehensive income. - -Revenue - -Product Revenue and Service and Other Revenue - -Product revenue includes sales from operating systems, cross-device productivity applications, server applications, business solution applications, desktop and server management tools, software development tools, video games, and hardware such as PCs, tablets, gaming and entertainment consoles, other intelligent devices, and related accessories. - -Service and other revenue includes sales from cloud-based solutions that provide customers with software, services, platforms, and content such as Office 365, Azure, Dynamics 365, and Xbox; solution support; and consulting services. Service and other revenue also includes sales from online advertising and LinkedIn. - -62 - - -PART II -Item 8 - - -Revenue Recognition - -Revenue is recognized upon transfer of control of promised products or services to customers in an amount that reflects the consideration we expect to receive in exchange for those products or services. We enter into contracts that can include various combinations of products and services, which are generally capable of being distinct and accounted for as separate performance obligations. Revenue is recognized net of allowances for returns and any taxes collected from customers, which are subsequently remitted to governmental authorities. - -Nature of Products and Services - -Licenses for on-premises software provide the customer with a right to use the software as it exists when made available to the customer. Customers may purchase perpetual licenses or subscribe to licenses, which provide customers with the same functionality and differ mainly in the duration over which the customer benefits from the software. Revenue from distinct on-premises licenses is recognized upfront at the point in time when the software is made available to the customer. In cases where we allocate revenue to software updates, primarily because the updates are provided at no additional charge, revenue is recognized as the updates are provided, which is generally ratably over the estimated life of the related device or license. - -Certain volume licensing programs, including Enterprise Agreements, include on-premises licenses combined with Software Assurance (�SA�). SA conveys rights to new software and upgrades released over the contract period and provides support, tools, and training to help customers deploy and use products more efficiently. On-premises licenses are considered distinct performance obligations when sold with SA. Revenue allocated to SA is generally recognized ratably over the contract period as customers simultaneously consume and receive benefits, given that SA comprises distinct performance obligations that are satisfied over time. - -Cloud services, which allow customers to use hosted software over the contract period without taking possession of the software, are provided on either a subscription or consumption basis. Revenue related to cloud services provided on a subscription basis is recognized ratably over the contract period. Revenue related to cloud services provided on a consumption basis, such as the amount of storage used in a period, is recognized based on the customer utilization of such resources. When cloud services require a significant level of integration and interdependency with software and the individual components are not considered distinct, all revenue is recognized over the period in which the cloud services are provided. - -Revenue from search advertising is recognized when the advertisement appears in the search results or when the action necessary to earn the revenue has been completed. Revenue from consulting services is recognized as services are provided. - -Our hardware is generally highly dependent on, and interrelated with, the underlying operating system and cannot function without the operating system. In these cases, the hardware and software license are accounted for as a single performance obligation and revenue is recognized at the point in time when ownership is transferred to resellers or directly to end customers through retail stores and online marketplaces. - -Refer to Note 19 � Segment Information and Geographic Data for further information, including revenue by significant product and service offering. - -Significant Judgments - -Our contracts with customers often include promises to transfer multiple products and services to a customer. Determining whether products and services are considered distinct performance obligations that should be accounted for separately versus together may require significant judgment. When a cloud-based service includes both on-premises software licenses and cloud services, judgment is required to determine whether the software license is considered distinct and accounted for separately, or not distinct and accounted for together with the cloud service and recognized over time. Certain cloud services, primarily Office 365, depend on a significant level of integration, interdependency, and interrelation between the desktop applications and cloud services, and are accounted for together as one performance obligation. Revenue from Office 365 is recognized ratably over the period in which the cloud services are provided. - -63 - - -PART II -Item 8 - -Judgment is required to determine the SSP for each distinct performance obligation. We use a single amount to estimate SSP for items that are not sold separately, including on-premises licenses sold with SA or software updates provided at no additional charge. We use a range of amounts to estimate SSP when we sell each of the products and services separately and need to determine whether there is a discount to be allocated based on the relative SSP of the various products and services. - -In instances where SSP is not directly observable, such as when we do not sell the product or service separately, we determine the SSP using information that may include market conditions and other observable inputs. We typically have more than one SSP for individual products and services due to the stratification of those products and services by customers and circumstances. In these instances, we may use information such as the size of the customer and geographic region in determining the SSP. - -Due to the various benefits from and the nature of our SA program, judgment is required to assess the pattern of delivery, including the exercise pattern of certain benefits across our portfolio of customers. - -Our products are generally sold with a right of return, we may provide other credits or incentives, and in certain instances we estimate customer usage of our products and services, which are accounted for as variable consideration when determining the amount of revenue to recognize. Returns and credits are estimated at contract inception and updated at the end of each reporting period if additional information becomes available. Changes to our estimated variable consideration were not material for the periods presented. - -Contract Balances and Other Receivables - -Timing of revenue recognition may differ from the timing of invoicing to customers. We record a receivable when revenue is recognized prior to invoicing, or unearned revenue when revenue is recognized subsequent to invoicing. For multi-year agreements, we generally invoice customers annually at the beginning of each annual coverage period. We record a receivable related to revenue recognized for multi-year on-premises licenses as we have an unconditional right to invoice and receive payment in the future related to those licenses. - -Unearned revenue comprises mainly unearned revenue related to volume licensing programs, which may include SA and cloud services. Unearned revenue is generally invoiced annually at the beginning of each contract period for multi-year agreements and recognized ratably over the coverage period. Unearned revenue also includes payments for consulting services to be performed in the future, LinkedIn subscriptions, Office 365 subscriptions, Xbox subscriptions, Windows post-delivery support, Dynamics business solutions, and other offerings for which we have been paid in advance and earn the revenue when we transfer control of the product or service. - -Refer to Note 13 � Unearned Revenue for further information, including unearned revenue by segment and changes in unearned revenue during the period. - -Payment terms and conditions vary by contract type, although terms generally include a requirement of payment within 30 to 60 days. In instances where the timing of revenue recognition differs from the timing of invoicing, we have determined our contracts generally do not include a significant financing component. The primary purpose of our invoicing terms is to provide customers with simplified and predictable ways of purchasing our products and services, not to receive financing from our customers or to provide customers with financing. Examples include invoicing at the beginning of a subscription term with revenue recognized ratably over the contract period, and multi-year on-premises licenses that are invoiced annually with revenue recognized upfront. - -As of June 30, 2022 and 2021, other receivables due from suppliers were $1.0 billion and $965 million, respectively, and are included in accounts receivable, net in our consolidated balance sheets. - -As of June 30, 2022 and 2021, long-term accounts receivable, net of allowance for doubtful accounts, was $3.8 billion and $3.4 billion, respectively, and is included in other long-term assets in our consolidated balance sheets. - -The allowance for doubtful accounts reflects our best estimate of probable losses inherent in the accounts receivable balance. We determine the allowance based on known troubled accounts, historical experience, and other currently available evidence. - -64 - - -PART II -Item 8 - -Activity in the allowance for doubtful accounts was as follows: - -(In millions) - - -Year Ended June 30, - - - -2022 - -2021 - - - -2020 - - - - - - - - - - - - - - - - - - -Balance, beginning of period -$ -798 -$ -816 -$ -434 - -Charged to costs and other - - - -157 - -234 - - - -560 - -Write-offs - - - -(245) - -(252) - - - -(178) - - - - - - - - - - - - - - - - - -Balance, end of period -$ -710 -$ -798 -$ -816 - -Allowance for doubtful accounts included in our consolidated balance sheets: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -(In millions) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -June 30, - - - -2022 - - -2021 - - - - -2020 - - - - - - - - - - - - - - - - - -Accounts receivable, net of allowance for doubtful accounts -$ -633 - - -$ -751 - -$ -788 - - -Other long-term assets - - - -77 - - -47 - - - - -28 - - - - - - - - - - - - - - - - - - - -Total -$ -710 - - -$ -798 - -$ -816 - - - - - - - - - - - - - - - - - - - - -We record financing receivables when we offer certain of our customers the option to acquire our software products and services offerings through a financing program in a limited number of countries. As of June 30, 2022 and 2021, our financing receivables, net were $4.1 billion and $4.4 billion, respectively, for short-term and long-term financing receivables, which are included in other current assets and other long-term assets in our consolidated balance sheets. We record an allowance to cover expected losses based on troubled accounts, historical experience, and other currently available evidence. - -Assets Recognized from Costs to Obtain a Contract with a Customer - -We recognize an asset for the incremental costs of obtaining a contract with a customer if we expect the benefit of those costs to be longer than one year. We have determined that certain sales incentive programs meet the requirements to be capitalized. Total capitalized costs to obtain a contract were immaterial during the periods presented and are included in other current and long-term assets in our consolidated balance sheets. - -We apply a practical expedient to expense costs as incurred for costs to obtain a contract with a customer when the amortization period would have been one year or less. These costs include our internal sales force compensation program and certain partner sales incentive programs as we have determined annual compensation is commensurate with annual sales activities. - -Cost of Revenue - -Cost of revenue includes: manufacturing and distribution costs for products sold and programs licensed; operating costs related to product support service centers and product distribution centers; costs incurred to include software on PCs sold by original equipment manufacturers (�OEM�), to drive traffic to our websites, and to acquire online advertising space; costs incurred to support and maintain online products and services, including datacenter costs and royalties; warranty costs; inventory valuation adjustments; costs associated with the delivery of consulting services; and the amortization of capitalized software development costs. Capitalized software development costs are amortized over the estimated lives of the products. - -Product Warranty - -We provide for the estimated costs of fulfilling our obligations under hardware and software warranties at the time the related revenue is recognized. For hardware warranties, we estimate the costs based on historical and projected product failure rates, historical and projected repair costs, and knowledge of specific product failures (if any). The specific hardware warranty terms and conditions vary depending upon the product sold and the country in which we do business, but generally include parts and labor over a period generally ranging from 90 days to three years. For software warranties, we estimate the costs to provide bug fixes, such as security patches, over the estimated life of the software. We regularly reevaluate our estimates to assess the adequacy of the recorded warranty liabilities and adjust the amounts as necessary. - -65 - - -PART II -Item 8 - - -Research and Development - -Research and development expenses include payroll, employee benefits, stock-based compensation expense, and other headcount-related expenses associated with product development. Research and development expenses also include third-party development and programming costs, localization costs incurred to translate software for international markets, and the amortization of purchased software code and services content. Such costs related to software development are included in research and development expense until the point that technological feasibility is reached, which for our software products, is generally shortly before the products are released to production. Once technological feasibility is reached, such costs are capitalized and amortized to cost of revenue over the estimated lives of the products. - -Sales and Marketing - -Sales and marketing expenses include payroll, employee benefits, stock-based compensation expense, and other headcount-related expenses associated with sales and marketing personnel, and the costs of advertising, promotions, trade shows, seminars, and other programs. Advertising costs are expensed as incurred. Advertising expense was $1.5 billion, $1.5 billion, and $1.6 billion in fiscal years 2022, 2021, and 2020, respectively. - -Stock-Based Compensation - -Compensation cost for stock awards, which include restricted stock units (�RSUs�) and performance stock units (�PSUs�), is measured at the fair value on the grant date and recognized as expense, net of estimated forfeitures, over the related service or performance period. The fair value of stock awards is based on the quoted price of our common stock on the grant date less the present value of expected dividends not received during the vesting period. We measure the fair value of PSUs using a Monte Carlo valuation model. Compensation cost for RSUs is recognized using the straight-line method and for PSUs is recognized using the accelerated method. - -Compensation expense for the employee stock purchase plan (�ESPP�) is measured as the discount the employee is entitled to upon purchase and is recognized in the period of purchase. - -Income Taxes - -Income tax expense includes U.S. and international income taxes, and interest and penalties on uncertain tax positions. Certain income and expenses are not reported in tax returns and financial statements in the same year. The tax effect of such temporary differences is reported as deferred income taxes. Deferred tax assets are reported net of a valuation allowance when it is more likely than not that a tax benefit will not be realized. All deferred income taxes are classified as long-term in our consolidated balance sheets. - -Financial Instruments - -Investments - -We consider all highly liquid interest-earning investments with a maturity of three months or less at the date of purchase to be cash equivalents. The fair values of these investments approximate their carrying values. In general, investments with original maturities of greater than three months and remaining maturities of less than one year are classified as short-term investments. Investments with maturities beyond one year may be classified as short-term based on their highly liquid nature and because such marketable securities represent the investment of cash that is available for current operations. - -66 - - -PART II -Item 8 - -Debt investments are classified as available-for-sale and realized gains and losses are recorded using the specific identification method. Changes in fair value, excluding credit losses and impairments, are recorded in other comprehensive income. Fair value is calculated based on publicly available market information or other estimates determined by management. If the cost of an investment exceeds its fair value, we evaluate, among other factors, general market conditions, credit quality of debt instrument issuers, and the extent to which the fair value is less than cost. To determine credit losses, we employ a systematic methodology that considers available quantitative and qualitative evidence. In addition, we consider specific adverse conditions related to the financial health of, and business outlook for, the investee. If we have plans to sell the security or it is more likely than not that we will be required to sell the security before recovery, then a decline in fair value below cost is recorded as an impairment charge in other income (expense), net and a new cost basis in the investment is established. If market, industry, and/or investee conditions deteriorate, we may incur future impairments. - -Equity investments with readily determinable fair values are measured at fair value. Equity investments without readily determinable fair values are measured using the equity method or measured at cost with adjustments for observable changes in price or impairments (referred to as the measurement alternative). We perform a qualitative assessment on a periodic basis and recognize an impairment if there are sufficient indicators that the fair value of the investment is less than carrying value. Changes in value are recorded in other income (expense), net. - -Derivatives - -Derivative instruments are recognized as either assets or liabilities and measured at fair value. The accounting for changes in the fair value of a derivative depends on the intended use of the derivative and the resulting designation. - -For derivative instruments designated as fair value hedges, gains and losses are recognized in other income (expense), net with offsetting gains and losses on the hedged items. Gains and losses representing hedge components excluded from the assessment of effectiveness are recognized in other income (expense), net. - -For derivative instruments designated as cash flow hedges, gains and losses are initially reported as a component of other comprehensive income and subsequently recognized in other income (expense), net with the corresponding hedged item. Gains and losses representing hedge components excluded from the assessment of effectiveness are recognized in other income (expense), net. - -For derivative instruments that are not designated as hedges, gains and losses from changes in fair values are primarily recognized in other income (expense), net. - -Fair Value Measurements - -We account for certain assets and liabilities at fair value. The hierarchy below lists three levels of fair value based on the extent to which inputs used in measuring fair value are observable in the market. We categorize each of our fair value measurements in one of these three levels based on the lowest level input that is significant to the fair value measurement in its entirety. These levels are: - -� Level 1 � inputs are based upon unadjusted quoted prices for identical instruments in active markets. Our Level 1 investments include U.S. government securities, common and preferred stock, and mutual funds. Our Level 1 derivative assets and liabilities include those actively traded on exchanges. - -� Level 2 � inputs are based upon quoted prices for similar instruments in active markets, quoted prices for identical or similar instruments in markets that are not active, and model-based valuation techniques (e.g. the Black-Scholes model) for which all significant inputs are observable in the market or can be corroborated by observable market data for substantially the full term of the assets or liabilities. Where applicable, these models project future cash flows and discount the future amounts to a present value using market-based observable inputs including interest rate curves, credit spreads, foreign exchange rates, and forward and spot prices for currencies. Our Level 2 investments include commercial paper, certificates of deposit, U.S. agency securities, foreign government bonds, mortgage- and asset-backed securities, corporate notes and bonds, and municipal securities. Our Level 2 derivative assets and liabilities include certain over-the-counter forward, option, and swap contracts. - -67 - - -PART II -Item 8 - -� Level 3 � inputs are generally unobservable and typically reflect management�s estimates of assumptions that market participants would use in pricing the asset or liability. The fair values are therefore determined using model-based techniques, including option pricing models and discounted cash flow models. Our Level 3 assets and liabilities include investments in corporate notes and bonds, municipal securities, and goodwill and intangible assets, when they are recorded at fair value due to an impairment charge. Unobservable inputs used in the models are significant to the fair values of the assets and liabilities. - -We measure equity investments without readily determinable fair values on a nonrecurring basis. The fair values of these investments are determined based on valuation techniques using the best information available, and may include quoted market prices, market comparables, and discounted cash flow projections. - -Our other current financial assets and current financial liabilities have fair values that approximate their carrying values. - -Inventories - -Inventories are stated at average cost, subject to the lower of cost or net realizable value. Cost includes materials, labor, and manufacturing overhead related to the purchase and production of inventories. Net realizable value is the estimated selling price less estimated costs of completion, disposal, and transportation. We regularly review inventory quantities on hand, future purchase commitments with our suppliers, and the estimated utility of our inventory. If our review indicates a reduction in utility below carrying value, we reduce our inventory to a new cost basis through a charge to cost of revenue. - -Property and Equipment - -Property and equipment is stated at cost less accumulated depreciation, and depreciated using the straight-line method over the shorter of the estimated useful life of the asset or the lease term. The estimated useful lives of our property and equipment are generally as follows: computer software developed or acquired for internal use, three to seven years; computer equipment, two to four years; buildings and improvements, five to 15 years; leasehold improvements, three to 20 years; and furniture and equipment, one to 10 years. Land is not depreciated. - -Leases - -We determine if an arrangement is a lease at inception. Operating leases are included in operating lease right-of-use (�ROU�) assets, other current liabilities, and operating lease liabilities in our consolidated balance sheets. Finance leases are included in property and equipment, other current liabilities, and other long-term liabilities in our consolidated balance sheets. - -ROU assets represent our right to use an underlying asset for the lease term and lease liabilities represent our obligation to make lease payments arising from the lease. Operating lease ROU assets and liabilities are recognized at commencement date based on the present value of lease payments over the lease term. As most of our leases do not provide an implicit rate, we generally use our incremental borrowing rate based on the estimated rate of interest for collateralized borrowing over a similar term of the lease payments at commencement date. The operating lease ROU asset also includes any lease payments made and excludes lease incentives. Our lease terms may include options to extend or terminate the lease when it is reasonably certain that we will exercise that option. Lease expense for lease payments is recognized on a straight-line basis over the lease term. - -We have lease agreements with lease and non-lease components, which are generally accounted for separately. For certain equipment leases, such as vehicles, we account for the lease and non-lease components as a single lease component. Additionally, for certain equipment leases, we apply a portfolio approach to effectively account for the operating lease ROU assets and liabilities. - -Goodwill - -Goodwill is tested for impairment at the reporting unit level (operating segment or one level below an operating segment) on an annual basis (May 1 for us) and between annual tests if an event occurs or circumstances change that would more likely than not reduce the fair value of a reporting unit below its carrying value. - -68 - - -PART II -Item 8 - - -Intangible Assets - -Our intangible assets are subject to amortization and are amortized using the straight-line method over their estimated period of benefit, ranging from one to 20 years. We evaluate the recoverability of intangible assets periodically by taking into account events or circumstances that may warrant revised estimates of useful lives or that indicate the asset may be impaired. - -Recent Accounting Guidance - -Accounting for Income Taxes - -In December 2019, the Financial Accounting Standards Board issued a new standard to simplify the accounting for income taxes. The guidance eliminates certain exceptions related to the approach for intraperiod tax allocation, the methodology for calculating income taxes in an interim period, and the recognition of deferred tax liabilities for outside basis differences related to changes in ownership of equity method investments and foreign subsidiaries. The guidance also simplifies aspects of accounting for franchise taxes and enacted changes in tax laws or rates and clarifies the accounting for transactions that result in a step-up in the tax basis of goodwill. We adopted the standard effective July 1, 2021. Adoption of the standard did not have a material impact on our consolidated financial statements. - -NOTE 2 � EARNINGS PER SHARE - -Basic earnings per share (�EPS�) is computed based on the weighted average number of shares of common stock outstanding during the period. Diluted EPS is computed based on the weighted average number of shares of common stock plus the effect of dilutive potential common shares outstanding during the period using the treasury stock method. Dilutive potential common shares include outstanding stock options and stock awards. - -The components of basic and diluted EPS were as follows: - -(In millions, except earnings per share) - - -Year Ended June 30, - -2022 - - -2021 - - -2020 - - - - - - - - - - - - - - - -Net income available for common shareholders (A) -$ -72,738 -$ -61,271 -$ -44,281 - - -Weighted average outstanding shares of common stock (B) - -7,496 - - -7,547 - - -7,610 - - -Dilutive effect of stock-based awards - -44 - - -61 - - -73 - - - - - - - - - - - - - - - -Common stock and common stock equivalents (C) - -7,540 - - -7,608 - - -7,683 - - -Earnings Per Share - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Basic (A/B) -$ -9.70 -$ -8.12 -$ -5.82 - - -Diluted (A/C) -$ -9.65 -$ -8.05 -$ -5.76 - - - -Anti-dilutive stock-based awards excluded from the calculations of diluted EPS were immaterial during the periods presented. - - NOTE 3 � OTHER INCOME (EXPENSE), NET The components of other income (expense), net were as follows: - -(In millions) - - -Year Ended June 30, - - -2022 - - -2021 - - -2020 - - - - - - - - - - - - - - - -Interest and dividends income -$ -2,094 -$ -2,131 -$ -2,680 - -Interest expense - - -(2,063) - - -(2,346) - - -(2,591) - -Net recognized gains on investments - - -461 - - -1,232 - - -32 - -Net gains (losses) on derivatives - - -(52) - - -17 - - -187 - -Net gains (losses) on foreign currency remeasurements - - -(75) - - -54 - - -(191) - -Other, net - - -(32) - - -98 - - -(40) - - - - - - - - - - - - - - - -Total -$ -333 -$ -1,186 -$ -77 - - -69 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - - -Net Recognized Gains (Losses) on Investments - -Net recognized gains (losses) on debt investments were as follows: - -(In millions) - - -Year Ended June 30, - -2022 - - -2021 - -2020 - - - - - - - - - - - -Realized gains from sales of available-for-sale securities -$ -162 -$ -105 -$ -50 -Realized losses from sales of available-for-sale securities - -(138) - - -(40) - -(37) -Impairments and allowance for credit losses - -(81) - - -(2) - -(17) - - - - - - - - - - - -Total -$ -(57) -$ -63 -$ -(4) - - - - - - - - - - - - -Net recognized gains (losses) on equity investments were as follows: - -(In millions) - - -Year Ended June 30, - - -2022 - - -2021 - -2020 - - - - - - - - - - - - - - -Net realized gains on investments sold -$ -29 -$ -123 -$ -83 - -Net unrealized gains on investments still held - - -509 - - -1,057 - -69 - -Impairments of investments - - -(20) - - -(11) - -(116) - - - - - - - - - - - - - -Total -$ -518 -$ -1,169 -$ -36 - - -70 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -NOTE 4 � INVESTMENTS - -Investment Components - -The components of investments were as follows: - - - - - - - - - - - - - - - - - - - -Cash - - - - - - - - - -Fair Value - - -Adjusted - - -Unrealized - -Unrealized - -Recorded - - -and Cash - -Short-term - - -Equity - -(In millions) -Level - - -Cost Basis - - -Gains - -Losses - -Basis - -Equivalents -Investments - -Investments - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -June 30, 2022 - - - - - - - - - - - - - - - - - - - - - - - - - - - -Changes in Fair Value Recorded in Other - - - - - - - - - - - - - - - - - - - - - - - - - - - -Comprehensive Income - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Commercial paper -Level 2 -$ -2,500 -$ -0 - -$ -0 - -$ -2,500 -$ -2,498 -$ -2 -$ -0 - -Certificates of deposit -Level 2 - - -2,071 - - -0 - - -0 - - -2,071 - - -2,032 - -39 - - -0 - -U.S. government securities -Level 1 - - -79,696 - - - -29 - - -(2,178) - -77,547 - - - -9 - - -77,538 - - - -0 - -U.S. agency securities -Level 2 - - -419 - - - -0 - - -(9) - -410 - - -0 - - -410 - - -0 - -Foreign government bonds -Level 2 - - -506 - - - -0 - - -(24) - -482 - - -0 - - -482 - - -0 - -Mortgage- and asset-backed - - - -727 - - - -1 - - -(30) - -698 - - -0 - - -698 - - -0 - -securities -Level 2 - - - - - - - - - - - - - - - - - - - - - - -Corporate notes and bonds -Level 2 - - -11,661 - - - -4 - - -(554) - -11,111 - - - -0 - - -11,111 - - - -0 - -Corporate notes and bonds -Level 3 - - -67 - - - -0 - - -0 - - -67 - - -0 - - -67 - - -0 - -Municipal securities -Level 2 - - -368 - - - -19 - - -(13) - -374 - - -0 - - -374 - - -0 - -Municipal securities -Level 3 - - -103 - - - -0 - - -(6) - -97 - - -0 - - -97 - - -0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Total debt investments - -$ -98,118 - -$ -53 - -$ -(2,814) -$ -95,357 - -$ -4,539 -$ -90,818 - -$ -0 - -Changes in Fair Value Recorded in Net - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Income - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Equity investments -Level 1 - - - - - - - - - - - -$ -1,590 -$ -1,134 -$ -0 -$ -456 - -Equity investments -Other - - - - - - - - - - - - -6,435 - - -0 - - -0 - - -6,435 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Total equity investments - - - - - - - - - - - - -$ -8,025 -$ -1,134 -$ -0 -$ -6,891 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Cash - - - - - - - - - - - - -$ -8,258 -$ -8,258 -$ -0 -$ -0 - -Derivatives, net (a) - - - - - - - - - - - - - -8 - - -0 - - -8 - - -0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Total - - - - - - - - - - - - -$ -111,648 - -$ -13,931 - -$ -90,826 - -$ -6,891 - - - - - - - - - -71 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - - - - - - - - - - - - - - - - - - - - -Cash - - - - - - - - - -Fair Value - -Adjusted - - -Unrealized - - -Unrealized - -Recorded - - -and Cash - - -Short-term - -Equity - -(In millions) -Level - -Cost Basis - - -Gains - - -Losses - -Basis - - -Equivalents - -Investments - -Investments - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -June 30, 2021 - - - - - - - - - - - - - - - - - - - - - - - - - - - -Changes in Fair - - - - - - - - - - - - - - - - - - - - - - - - - - - -Value Recorded - - - - - - - - - - - - - - - - - - - - - - - - - - - -in Other - - - - - - - - - - - - - - - - - - - - - - - - - - - -Comprehensive - - - - - - - - - - - - - - - - - - - - - - - - - - - -Income - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Commercial - - - - - - - - - - - - - - - - - - - - - - - - - - - -paper -Level 2 -$ -4,316 -$ -0 -$ -0 -$ -4,316 -$ -1,331 -$ -2,985 -$ -0 - -Certificates of - - - - - - - - - - - - - - - - - - - - - - - - - - - -deposit -Level 2 - -3,615 - - -0 - - -0 - -3,615 - - -2,920 - - -695 - -0 - -U.S. government - - - - - - - - - - - - - - - - - - - - - - - - - - - -securities -Level 1 - -90,664 - - - -3,832 - - -(111) - -94,385 - - - -1,500 - - -92,885 - - -0 - -U.S. agency - - - - - - - - - - - - - - - - - - - - - - - - - - - -securities -Level 2 - -807 - - - -2 - - -0 - -809 - - - -0 - - - -809 - -0 - -Foreign - - - - - - - - - - - - - - - - - - - - - - - - - - - -government - - - - - - - - - - - - - - - - - - - - - - - - - - - -bonds -Level 2 - -6,213 - - -9 - - -(2) - -6,220 - - -225 - - - -5,995 - -0 - -Mortgage- and - - - - - - - - - - - - - - - - - - - - - - - - - - - -asset-backed - - - - - - - - - - - - - - - - - - - - - - - - - - - -securities -Level 2 - -3,442 - - -22 - - -(6) - -3,458 - - -0 - - - -3,458 - -0 - -Corporate notes - - - - - - - - - - - - - - - - - - - - - - - - - - - -and bonds -Level 2 - -8,443 - - -249 - - -(9) - -8,683 - - -0 - - - -8,683 - -0 - -Corporate notes - - - - - - - - - - - - - - - - - - - - - - - - - - - -and bonds -Level 3 - -63 - - - -0 - - -0 - -63 - - - -0 - - - -63 - -0 - -Municipal - - - - - - - - - - - - - - - - - - - - - - - - - - - -securities -Level 2 - -308 - - - -63 - - -0 - -371 - - - -0 - - - -371 - -0 - -Municipal - - - - - - - - - - - - - - - - - - - - - - - - - - - -securities -Level 3 - -95 - - - -0 - - -(7) - -88 - - - -0 - - - -88 - -0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Total debt - - - - - - - - - - - - - - - - - - - - - - - - - - - -investments - -$ -117,966 - -$ -4,177 -$ -(135) -$ -122,008 - -$ -5,976 - -$ -116,032 - -$ -0 - -Changes in Fair - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Value Recorded - - - - - - - - - - - - - - - - - - - - - - - - - - - -in Net Income - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Equity - - - - - - - - - - - - - - - - - - - - - - - - - - - -investments -Level 1 - - - - - - - - - - - -$ -1,582 -$ -976 - -$ -0 -$ -606 - -Equity - - - - - - - - - - - - - - - - - - - - - - - - - - - -investments -Other - - - - - - - - - - - - -5,378 - - -0 - - - -0 - -5,378 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Total equity - - - - - - - - - - - - - - - - - - - - - - - - - - - -investments - - - - - - - - - - - - -$ -6,960 -$ -976 - -$ -0 -$ -5,984 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Cash - - - - - - - - - - - - -$ -7,272 -$ -7,272 -$ -0 -$ -0 - -Derivatives, net - - - - - - - - - - - - - -78 - - - -0 - - - -78 - -0 - -(a) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Total - - - - - - - - - - - - -$ -136,318 - -$ -14,224 - -$ -116,110 - -$ -5,984 - - - -(a) Refer to Note 5 � Derivatives for further information on the fair value of our derivative instruments. - -Equity investments presented as �Other� in the tables above include investments without readily determinable fair values measured using the equity method or measured at cost with adjustments for observable changes in price or impairments, and investments measured at fair value using net asset value as a practical expedient which are not categorized in the fair value hierarchy. As of June 30, 2022 and 2021, equity investments without readily determinable fair values measured at cost with adjustments for observable changes in price or impairments were $3.8 billion and $3.3 billion, respectively. - -Unrealized Losses on Debt Investments - -Debt investments with continuous unrealized losses for less than 12 months and 12 months or greater and their related fair values were as follows: - - - - - -Less than 12 Months - - -12 Months or Greater - - - - - -Total - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Unrealized - - - - - - -Unrealized - - -Total - -Unrealized - -(In millions) - - -Fair Value - - -Losses - - -Fair Value - - -Losses - - -Fair Value - -Losses - - - - - - - - - - - - - - - - - - - - - - - - - -June 30, 2022 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -U.S. government and agency securities -$ -59,092 -$ -(1,835) -$ -2,210 -$ -(352) -$ -61,302 -$ -(2,187) - -Foreign government bonds - - -418 - - -(18) - - -27 - - -(6) - - -445 - -(24) - -Mortgage- and asset-backed securities - - -510 - - -(26) - - -41 - - -(4) - - -551 - -(30) - -Corporate notes and bonds - - -9,443 - - -(477) - - -786 - - - -(77) - - -10,229 - -(554) - -Municipal securities - - -178 - - -(12) - - -74 - - -(7) - - -252 - -(19) - - - - - - - - - - - - - - - - - - - - - - - - -Total -$ -69,641 -$ -(2,368) -$ -3,138 -$ -(446) -$ -72,779 -$ -(2,814) - - - - - - - - - - - - - - - - - - - - - - - - - - - - -72 - - -PART II -Item 8 - - - - - - - -Less than 12 Months - -12 Months or Greater - - - - - -Total - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Unrealized - - - - - -Unrealized - - -Total - -Unrealized - -(In millions) - - -Fair Value - - -Losses - -Fair Value - - -Losses - - -Fair Value - -Losses - - - - - - - - - - - - - - - - - - - - - - - - -June 30, 2021 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -U.S. government and agency securities -$ -5,294 -$ -(111) -$ -0 -$ -0 -$ -5,294 -$ -(111) - -Foreign government bonds - - -3,148 - - -(1) - -5 - - -(1) - - -3,153 - -(2) - -Mortgage- and asset-backed securities - - -1,211 - - -(5) - -87 - - -(1) - - -1,298 - -(6) - -Corporate notes and bonds - - -1,678 - - -(8) - -34 - - -(1) - - -1,712 - -(9) - -Municipal securities - - -58 - - -(7) - -1 - - -0 - - -59 - -(7) - - - - - - - - - - - - - - - - - - - - - - - -Total -$ -11,389 -$ -(132) -$ -127 -$ -(3) -$ -11,516 -$ -(135) - - - - - - - - - - - - - - - - - - - - - - - - - -Unrealized losses from fixed-income securities are primarily attributable to changes in interest rates. Management does not believe any remaining unrealized losses represent impairments based on our evaluation of available evidence. - -Debt Investment Maturities - - - - -Adjusted - - -Estimated -(In millions) - - -Cost Basis - - -Fair Value - - - - - - - - - -June 30, 2022 - - - - - - - - - - - - - - - - - -Due in one year or less -$ -26,480 -$ -26,470 -Due after one year through five years - - -52,006 - - -50,748 -Due after five years through 10 years - - -18,274 - - -16,880 -Due after 10 years - - -1,358 - - -1,259 - - - - - - - - - -Total -$ -98,118 -$ -95,357 - - - - - - - - - - -NOTE 5 � DERIVATIVES - -We use derivative instruments to manage risks related to foreign currencies, interest rates, equity prices, and credit; to enhance investment returns; and to facilitate portfolio diversification. Our objectives for holding derivatives include reducing, eliminating, and efficiently managing the economic impact of these exposures as effectively as possible. Our derivative programs include strategies that both qualify and do not qualify for hedge accounting treatment. - -Foreign Currencies - -Certain forecasted transactions, assets, and liabilities are exposed to foreign currency risk. We monitor our foreign currency exposures daily to maximize the economic effectiveness of our foreign currency hedge positions. - -Foreign currency risks related to certain non-U.S. dollar-denominated investments are hedged using foreign exchange forward contracts that are designated as fair value hedging instruments. Foreign currency risks related to certain Euro-denominated debt are hedged using foreign exchange forward contracts that are designated as cash flow hedging instruments. - -Certain options and forwards not designated as hedging instruments are also used to manage the variability in foreign exchange rates on certain balance sheet amounts and to manage other foreign currency exposures. - -Interest Rate - -Interest rate risks related to certain fixed-rate debt are hedged using interest rate swaps that are designated as fair value hedging instruments to effectively convert the fixed interest rates to floating interest rates. - -73 - - -PART II -Item 8 - -Securities held in our fixed-income portfolio are subject to different interest rate risks based on their maturities. We manage the average maturity of our fixed-income portfolio to achieve economic returns that correlate to certain broad-based fixed-income indices using exchange-traded option and futures contracts and over-the-counter swap and option contracts. These contracts are not designated as hedging instruments and are included in �Other contracts� in the tables below. - -Equity - -Securities held in our equity investments portfolio are subject to market price risk. At times, we may hold options, futures, and swap contracts. - -These contracts are not designated as hedging instruments and are included in �Other contracts� in the tables below. - -Credit - -Our fixed-income portfolio is diversified and consists primarily of investment-grade securities. We use credit default swap contracts to manage credit exposures relative to broad-based indices and to facilitate portfolio diversification. These contracts are not designated as hedging instruments and are included in �Other contracts� in the tables below. - -Credit-Risk-Related Contingent Features - -Certain of our counterparty agreements for derivative instruments contain provisions that require our issued and outstanding long-term unsecured debt to maintain an investment grade credit rating and require us to maintain minimum liquidity of $1.0 billion. To the extent we fail to meet these requirements, we will be required to post collateral, similar to the standard convention related to over-the-counter derivatives. As of June 30, 2022, our long-term unsecured debt rating was AAA, and cash investments were in excess of $1.0 billion. As a result, no collateral was required to be posted. - -The following table presents the notional amounts of our outstanding derivative instruments measured in U.S. dollar equivalents: - - - -June 30, - -June 30, -(In millions) - -2022 - -2021 - - - - - - -Designated as Hedging Instruments - - - - - - - - - - - -Foreign exchange contracts purchased -$ -635 -$ -635 -Foreign exchange contracts sold - -0 - -6,081 -Interest rate contracts purchased - -1,139 - -1,247 -Not Designated as Hedging Instruments - - - - - - - - - - - -Foreign exchange contracts purchased - -10,322 - -14,223 -Foreign exchange contracts sold - -21,606 - -23,391 -Other contracts purchased - -2,773 - -2,456 -Other contracts sold - -544 - -763 - -74 - - - - - - - - - - - - -PART II -Item 8 - - -Fair Values of Derivative Instruments - -The following table presents our derivative instruments: - - - -Derivative - -Derivative - - -Derivative - -Derivative - -(In millions) - -Assets - -Liabilities - - -Assets - -Liabilities - - - - - - - - - - - - - - - - - - - - -June 30, - - - - - -June 30, - - - - - - -2022 - - - - - -2021 - - -Designated as Hedging Instruments - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Foreign exchange contracts -$ -0 -$ -(77) -$ -76 -$ -(8) - -Interest rate contracts - -3 - -0 - - -40 - -0 - - -Not Designated as Hedging Instruments - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Foreign exchange contracts - -333 - -(362) - - -227 - -(291) - -Other contracts - -20 - -(112) - - -56 - -(36) - - - - - - - - - - - - - - - - -Gross amounts of derivatives - -356 - -(551) - - -399 - -(335) - -Gross amounts of derivatives offset in the balance sheet - -(130) - -133 - - -(141) - -142 - - -Cash collateral received - -0 - -(75) - - -0 - -(42) - - - - - - - - - - - - - - - - -Net amounts of derivatives -$ -226 -$ -(493) -$ -258 -$ -(235) - -Reported as - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Short-term investments -$ -8 -$ -0 -$ -78 -$ -0 - - -Other current assets - -218 - -0 - - -137 - -0 - - -Other long-term assets - -0 - -0 - - -43 - -0 - - -Other current liabilities - -0 - -(298) - - -0 - -(182) - -Other long-term liabilities - -0 - -(195) - - -0 - -(53) - - - - - - - - - - - - - - - - -Total -$ -226 -$ -(493) -$ -258 -$ -(235) - - - - - - - - - - - - - - - - - -Gross derivative assets and liabilities subject to legally enforceable master netting agreements for which we have elected to offset were $343 million and $550 million, respectively, as of June 30, 2022, and $395 million and $335 million, respectively, as of June 30, 2021. - -The following table presents the fair value of our derivatives instruments on a gross basis: - -(In millions) - -Level 1 - -Level 2 - -Level 3 - -Total - - - - - - - - - - -June 30, 2022 - - - - - - - - - - - - - - - - - - - -Derivative assets -$ -1 -$ -349 -$ -6 -$ -356 -Derivative liabilities - -0 - -(551) - -0 - -(551) -June 30, 2021 - - - - - - - - - - - - - - - - - - - -Derivative assets - -0 - -396 - -3 - -399 -Derivative liabilities - -0 - -(335) - -0 - -(335) - - - - - - - - - - - -75 - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -Gains (losses) on derivative instruments recognized in other income (expense), net were as follows: - -(In millions) - - -Year Ended June 30, - -2022 - -2021 - -2020 - - -Designated as Fair Value Hedging Instruments - - - - - - - - - - - - - - - - - -Foreign exchange contracts - - - - - - - - -Derivatives -$ -49 -$ -193 -$ -1 - - -Hedged items - -(50) - -(188) - -3 - - -Excluded from effectiveness assessment - -4 - -30 - -139 - - -Interest rate contracts - - - - - - - - -Derivatives - -(92) - -(37) - -93 - - -Hedged items - -108 - -53 - -(93) - -Designated as Cash Flow Hedging Instruments - - - - - - - - - - - - - - - - - -Foreign exchange contracts - - - - - - - - -Amount reclassified from accumulated other comprehensive - -(79) - - - - - - -income - - - -17 - -0 - - -Not Designated as Hedging Instruments - - - - - - - - - - - - - - - - - -Foreign exchange contracts - -383 - -27 - -(123) - -Other contracts - -(72) - -9 - -50 - - - -Gains (losses), net of tax, on derivative instruments recognized in our consolidated comprehensive income statements were as follows: - -(In millions) - - -Year Ended June 30, - - - -2022 - - -2021 - - - - -2020 - -Designated as Cash Flow Hedging Instruments - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Foreign exchange contracts - - - - - - - - - - - - - - -Included in effectiveness assessment -$ - -(57) -$ - -34 -$ - - - -(38) - - -NOTE 6 � INVENTORIES - - - - - - - - - - - -The components of inventories were as follows: - - - - - - - - - - - - - - -(In millions) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -June 30, - - - - - - - -2022 - - - -2021 - - - - - - - - - - - - - - - -Raw materials - - - - -$ -1,144 -$ -1,190 - -Work in process - - - - - - - -82 - - -79 - -Finished goods - - - - - - -2,516 - - -1,367 - - - - - - - - - - - - - - -Total - - - - -$ -3,742 -$ -2,636 - - -76 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -NOTE 7 � PROPERTY AND EQUIPMENT - -The components of property and equipment were as follows: - -(In millions) - - -June 30, - - -2022 - -2021 - - - - - - - - - -Land -$ -4,734 -$ -3,660 - -Buildings and improvements - - -55,014 - -43,928 -Leasehold improvements - - -7,819 - -6,884 - -Computer equipment and software - - -60,631 - -51,250 -Furniture and equipment - - -5,860 - -5,344 - - - - - - - - - -Total, at cost - - -134,058 - -111,066 - -Accumulated depreciation - - -(59,660) - -(51,351) - - - - - - - -Total, net -$ -74,398 -$ -59,715 - - - - - - - - - - -During fiscal years 2022, 2021, and 2020, depreciation expense was $12.6 billion, $9.3 billion, and $10.7 billion, respectively. We have committed $8.5 billion, primarily related to datacenters, for the construction of new buildings, building improvements, and leasehold improvements as of June 30, 2022. - -NOTE 8 � BUSINESS COMBINATIONS - -Nuance Communications, Inc. - -On March 4, 2022, we completed our acquisition of Nuance Communications, Inc. (�Nuance�) for a total purchase price of $ 18.8 billion, consisting primarily of cash. Nuance is a cloud and artificial intelligence (�AI�) software provider with healthcare and enterprise AI experience, and the acquisition will build on our industry-specific cloud offerings. The financial results of Nuance have been included in our consolidated financial statements since the date of the acquisition. Nuance is reported as part of our Intelligent Cloud segment. - -The purchase price allocation as of the date of acquisition was based on a preliminary valuation and is subject to revision as more detailed analyses are completed and additional information about the fair value of assets acquired and liabilities assumed becomes available. - -The major classes of assets and liabilities to which we have preliminarily allocated the purchase price were as follows: - -(In millions) - - -Goodwill (a) -$ -16,308 -Intangible assets - -4,365 -Other assets - -59 -Other liabilities (b) - -(1,971) - - - - - - - - -Total -$ -18,761 - - - - - -(a) Goodwill was assigned to our Intelligent Cloud segment and was primarily attributed to increased synergies that are expected to be achieved from the integration of Nuance. None of the goodwill is expected to be deductible for income tax purposes. - -(b) Includes $986 million of convertible senior notes issued by Nuance in 2015 and 2017, of which $985 million was redeemed prior to June 30, 2022. The remaining $1 million of notes are redeemable through their respective maturity dates and are included in other current liabilities on our consolidated balance sheets as of June 30, 2022. - -77 - - -PART II -Item 8 - -Following are the details of the purchase price allocated to the intangible assets acquired: - - - - -Weighted -(In millions, except average life) - -Amount -Average Life - - - - - - - - - - -Customer-related -$ -2,610 -9 years -Technology-based - -1,540 -5 years -Marketing-related - -215 -4 years - - - - - -Total -$ -4,365 -7 years - - - - - - -ZeniMax Media Inc. - -On March 9, 2021, we completed our acquisition of ZeniMax Media Inc. (�ZeniMax�), the parent company of Bethesda Softworks LLC (�Bethesda�), for a total purchase price of $8.1 billion, consisting primarily of cash. The purchase price included $766 million of cash and cash equivalents acquired. Bethesda is one of the largest, privately held game developers and publishers in the world, and brings a broad portfolio of games, technology, and talent to Xbox. The financial results of ZeniMax have been included in our consolidated financial statements since the date of the acquisition. ZeniMax is reported as part of our More Personal Computing segment. - -The allocation of the purchase price to goodwill was completed as of December 31, 2021. The major classes of assets and liabilities to which we have allocated the purchase price were as follows: - -(In millions) - - -Cash and cash equivalents -$ -766 - -Goodwill - -5,510 - -Intangible assets - -1,968 - -Other assets - -121 - -Other liabilities - -(244) - - - - -Total -$ -8,121 - - - - - - -Goodwill was assigned to our More Personal Computing segment. The goodwill was primarily attributed to increased synergies that are expected to be achieved from the integration of ZeniMax. None of the goodwill is expected to be deductible for income tax purposes. - -Following are details of the purchase price allocated to the intangible assets acquired: - - - - - - -Weighted -(In millions, except average life) - - -Amount -Average Life - - - - - - - - - - - - - - -Technology-based -$ -1,341 -4 years -Marketing-related - - -627 -11 years - - - - - - - - - - - - - - -Total -$ -1,968 -6 years - - - - - - - - -Activision Blizzard, Inc. - -On January 18, 2022, we entered into a definitive agreement to acquire Activision Blizzard, Inc. (�Activision Blizzard�) for $95.00 per share in an all-cash transaction valued at $68.7 billion, inclusive of Activision Blizzard�s net cash. Activision Blizzard is a leader in game development and an interactive entertainment content publisher. The acquisition will accelerate the growth in our gaming business across mobile, PC, console, and cloud and will provide building blocks for the metaverse. The acquisition has been approved by Activision Blizzard�s shareholders, and we expect it to close in fiscal year 2023, subject to the satisfaction of certain regulatory approvals and other customary closing conditions. - -78 - - -PART II -Item 8 - - -NOTE 9 � GOODWILL - -Changes in the carrying amount of goodwill were as follows: - - -(In millions) - - - -June 30, - -2020 - - - -Acquisitions - - - -Other - - - - -June 30, - -2021 - - - -Acquisitions - - - -Other - - - -June 30, - -2022 - - -Productivity and Business - - - - - - - - - - - -$ -24,317 -$ -599 -$ -(105) -$ -24,811 - -Processes -$ -24,190 -$ -0 -$ -127 - - - - - - - - - -Intelligent Cloud - -12,697 - - -505 - - -54 - - -13,256 - - -16,879 - - -47 - - -30,182 - - - - - - - - - - - - - - - - - - - - -(b) - - -(b) - - - -More Personal Computing - -6,464 - - -5,556(a) - - -118(a) - - -12,138 - - -648 - - -(255) - - -12,531 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Total -$ -43,351 -$ -6,061 -$ -299 - -$ -49,711 -$ -18,126 -$ -(313) -$ -67,524 - - - -(a) Includes goodwill of $5.5 billion related to ZeniMax. See Note 8 � Business Combinations for further information. - -(b) Includes goodwill of $16.3 billion related to Nuance. See Note 8 � Business Combinations for further information. - -The measurement periods for the valuation of assets acquired and liabilities assumed end as soon as information on the facts and circumstances that existed as of the acquisition dates becomes available, but do not exceed 12 months. Adjustments in purchase price allocations may require a change in the amounts allocated to goodwill during the periods in which the adjustments are determined. - -Any change in the goodwill amounts resulting from foreign currency translations and purchase accounting adjustments are presented as �Other� in the table above. Also included in �Other� are business dispositions and transfers between segments due to reorganizations, as applicable. - -Goodwill Impairment - -We test goodwill for impairment annually on May 1 at the reporting unit level, primarily using a discounted cash flow methodology with a peer-based, risk-adjusted weighted average cost of capital. We believe use of a discounted cash flow approach is the most reliable indicator of the fair values of the businesses. - -No instances of impairment were identified in our May 1, 2022, May 1, 2021, or May 1, 2020 tests. As of June 30, 2022 and 2021, accumulated goodwill impairment was $11.3 billion. - -NOTE 10 � INTANGIBLE ASSETS - -The components of intangible assets, all of which are finite-lived, were as follows: - - - -Gross - - - - - - - - -Gross - - - - - - - - -Carrying - - -Accumulated - -Net Carrying - -Carrying - -Accumulated - -Net Carrying -(In millions) - -Amount - - -Amortization - -Amount - -Amount - -Amortization - -Amount - - - - - - - - - - - - - - - - - - - -June 30, - - - - - - - - -2022 - - - - - - - -2021 - - - - - - - - - - - - - - - - - - - - - -Technology-based -$ -11,277 -$ -(6,958) -$ -4,319 -$ -9,779 -$ -(7,007) -$ -2,772 - -Customer-related - -7,342 - - -(3,171) - -4,171 - -4,958 - -(2,859) - -2,099 - -Marketing-related - -4,942 - - -(2,143) - -2,799 - -4,792 - -(1,878) - -2,914 - -Contract-based - -16 - - -(7) - -9 - -446 - -(431) - -15 - - - - - - - - - - - - - - - - - - - - -Total -$ -23,577(a) $ -(12,279) -$ -11,298 -$ -19,975(b) -$ -(12,175) -$ -7,800 - - - - - - - - - - - - - - - - - - - - - - -(a) Includes intangible assets of $4.4 billion related to Nuance. See Note 8 � Business Combinations for further information. - -(b) Includes intangible assets of $2.0 billion related to ZeniMax. See Note 8 � Business Combinations for further information. - -No material impairments of intangible assets were identified during fiscal years 2022, 2021, or 2020. We estimate that we have no significant residual value related to our intangible assets. - -79 - - -PART II -Item 8 - -The components of intangible assets acquired during the periods presented were as follows: - - - - - - -Weighted - - - -Weighted -(In millions) - - -Amount -Average Life - -Amount -Average Life - - - - - - - - - -Year Ended June 30, - - -2022 - - -2021 - - - - - - - - - - - - - -Technology-based -$ -2,611 -4 years -$ -1,628 -4 years -Customer-related - - -2,837 -9 years - -96 -4 years -Marketing-related - - -233 -4 years - -625 -6 years -Contract-based - - -0 -0 years - -10 -3 years - - - - - - - - - - - -Total -$ -5,681 -7 years -$ -2,359 -5 years - - - - - - - - - - - - -Intangible assets amortization expense was $2.0 billion, $1.6 billion, and $1.6 billion for fiscal years 2022, 2021, and 2020, respectively. - -The following table outlines the estimated future amortization expense related to intangible assets held as of June 30, 2022: - -(In millions) - - -Year Ending June 30, - - - - - - - - - - - -2023 -$ -2,654 - -2024 - - -2,385 - -2025 - - -1,631 - -2026 - - -1,227 - -2027 - - -809 - -Thereafter - - -2,592 - - - - - - - -Total -$ -11,298 - - -80 - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -NOTE 11 � DEBT - -The components of debt were as follows: - - -Maturities -Stated Interest -Effective Interest - - -June 30, - -June 30, -(In millions, issuance by calendar year) -(calendar year) -Rate -Rate - - -2022 - -2021 - - - - - - - - - - - - - - - - - - - - - - - -2009 issuance of $3.8 billion (a) -2039 -5.20% -5.24% -$ -520 -$ -520 - -2010 issuance of $4.8 billion (a) -2040 -4.50% -4.57% - - -486 - -486 - -2011 issuance of $2.3 billion (a) -2041 -5.30% -5.36% - - -718 - -718 - -2012 issuance of $2.3 billion (a) -2022 � 2042 -2.13% � 3.50% -2.24% � 3.57% - - -1,204 - -1,204 - -2013 issuance of $5.2 billion (a) -2023 � 2043 -2.38% � 4.88% -2.47% � 4.92% - - -2,814 - -2,814 - -2013 issuance of �4.1 billion -2028 � 2033 -2.63% � 3.13% -2.69% � 3.22% - - -2,404 - -4,803 - -2015 issuance of $23.8 billion (a) -2022 � 2055 -2.65% � 4.75% -2.72% � 4.78% - - -10,805 - -12,305 - -2016 issuance of $19.8 billion (a) -2023 � 2056 -2.00% � 3.95% -2.10% � 4.03% - - -9,430 - -12,180 - -2017 issuance of $17.0 billion (a) -2024 � 2057 -2.88% � 4.50% -3.04% � 4.53% - - -8,945 - -10,695 - -2020 issuance of $10.0 billion (a) -2050 � 2060 -2.53% � 2.68% -2.53% � 2.68% - - -10,000 - -10,000 - -2021 issuance of $8.2 billion (a) -2052 � 2062 -2.92% � 3.04% -2.92% � 3.04% - - -8,185 - -8,185 - - - - - - - - - - - - -Total face value - - - - - -55,511 - -63,910 - -Unamortized discount and issuance costs - - - - - -(471) - -(511) -Hedge fair value adjustments (b) - - - - - -(68) - -40 - -Premium on debt exchange (a) - - - - - -(5,191) - -(5,293) - - - - - - - - - - - -Total debt - - - - - -49,781 - -58,146 - -Current portion of long-term debt - - - - - -(2,749) - -(8,072) - - - - - - - - - - -Long-term debt - - - -$ -47,032 -$ -50,074 - - - - - - - - - - - - - -(a) In March 2021 and June 2020, we exchanged a portion of our existing debt at a premium for cash and new debt with longer maturities. The premiums are amortized over the terms of the new debt. - -(b) Refer to Note 5 � Derivatives for further information on the interest rate swaps related to fixed-rate debt. - -As of June 30, 2022 and 2021, the estimated fair value of long-term debt, including the current portion, was $50.9 billion and $70.0 billion, respectively. The estimated fair values are based on Level 2 inputs. - -Debt in the table above is comprised of senior unsecured obligations and ranks equally with our other outstanding obligations. Interest is paid semi-annually, except for the Euro-denominated debt, which is paid annually. Cash paid for interest on our debt for fiscal years 2022, 2021, and 2020 was $1.9 billion, $2.0 billion, and $2.4 billion, respectively. - -The following table outlines maturities of our long-term debt, including the current portion, as of June 30, 2022: - -(In millions) - - -Year Ending June 30, - - - - - - - - - - - -2023 -$ -2,750 - -2024 - - -5,250 - -2025 - - -2,250 - -2026 - - -3,000 - -2027 - - -8,000 - -Thereafter - - -34,261 - - - - - - - -Total -$ -55,511 - - -81 - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -NOTE 12 � INCOME TAXES - -Provision for Income Taxes - -The components of the provision for income taxes were as follows: - -(In millions) - - -Year Ended June 30, - -2022 - -2021 - - -2020 - -Current Taxes - - - - - - - - - - - - - - - - - - - - - - - -U.S. federal -$ -8,329 -$ -3,285 -$ -3,537 - -U.S. state and local - -1,679 - -1,229 - - -763 - -Foreign - -6,672 - -5,467 - - -4,444 - - - - - - - - - - - - - -Current taxes -$ -16,680 -$ -9,981 -$ -8,744 - -Deferred Taxes - - - - - - - - - - - - - - - - - - - - - - - -U.S. federal -$ -(4,815) -$ -25 -$ -58 - -U.S. state and local - -(1,062) - -(204) - - -(6) - -Foreign - -175 - -29 - - -(41) - - - - - - - - - - - - - -Deferred taxes -$ -(5,702) -$ -(150) -$ -11 - - - - - - - - - - - - - - - - - - - - - - - - - -Provision for income taxes -$ -10,978 -$ -9,831 -$ -8,755 - -U.S. and foreign components of income before income taxes were as follows: - - - - - - - - - - - - - - - - - - - - - - - -(In millions) - - - - - - - - - - - - - - - - - - - - - -Year Ended June 30, - -2022 - -2021 - - -2020 - - - - - - - - - -U.S. -$ -47,837 -$ -34,972 -$ -24,116 - -Foreign - -35,879 - -36,130 - - -28,920 - - - - - - - - - - - - - -Income before income taxes -$ -83,716 -$ -71,102 -$ -53,036 - - - - - - - - - - - - - - -Effective Tax Rate - -The items accounting for the difference between income taxes computed at the U.S. federal statutory rate and our effective rate were as follows: - - -Year Ended June 30, -2022 -2021 -2020 - - - - - - - -Federal statutory rate -21.0% -21.0% -21.0% -Effect of: - - - - - - -Foreign earnings taxed at lower rates -(1.3)% -(2.7)% -(3.7)% -Impact of intangible property transfers -(3.9)% -0% -0% -Foreign-derived intangible income deduction -(1.1)% -(1.3)% -(1.1)% -State income taxes, net of federal benefit -1.4% -1.4% -1.3% -Research and development credit -(0.9)% -(0.9)% -(1.1)% -Excess tax benefits relating to stock-based compensation -(1.9)% -(2.4)% -(2.2)% -Interest, net -0.5% -0.5% -1.0% -Other reconciling items, net -(0.7)% -(1.8)% -1.3% - - - - - - - -Effective rate -13.1% -13.8% -16.5% - - - - - - - - -In the first quarter of fiscal year 2022, we transferred certain intangible properties from our Puerto Rico subsidiary to the U.S. The transfer of intangible properties resulted in a $3.3 billion net income tax benefit in the first quarter of fiscal year 2022, as the value of future U.S. tax deductions exceeds the current tax liability from the U.S. global intangible low-taxed income (�GILTI�) tax. - - -82 - - -PART II -Item 8 - - -We have historically paid India withholding taxes on software sales through distributor withholding and tax audit assessments in India. In March 2021, the India Supreme Court ruled favorably in the case of Engineering Analysis Centre of Excellence Private Limited vs The Commissioner of Income Tax for companies in 86 separate appeals, some dating back to 2012, holding that software sales are not subject to India withholding taxes. Although we were not a party to the appeals, our software sales in India were determined to be not subject to withholding taxes. Therefore, we recorded a net income tax benefit of $ 620 million in the third quarter of fiscal year 2021 to reflect the results of the India Supreme Court decision impacting fiscal year 1996 through fiscal year 2016. - -The decrease from the federal statutory rate in fiscal year 2022 is primarily due to the net income tax benefit related to the transfer of intangible properties, earnings taxed at lower rates in foreign jurisdictions resulting from producing and distributing our products and services through our foreign regional operations center in Ireland, and tax benefits relating to stock-based compensation. The decrease from the federal statutory rate in fiscal year 2021 is primarily due to earnings taxed at lower rates in foreign jurisdictions resulting from producing and distributing our products and services through our foreign regional operations centers in Ireland and Puerto Rico, tax benefits relating to stock-based compensation, and tax benefits from the India Supreme Court decision on withholding taxes. The decrease from the federal statutory rate in fiscal year 2020 is primarily due to earnings taxed at lower rates in foreign jurisdictions resulting from producing and distributing our products and services through our foreign regional operations centers in Ireland and Puerto Rico, and tax benefits relating to stock-based compensation. In fiscal years 2022, 2021, and 2020, our foreign regional operating centers in Ireland and Puerto Rico, which are taxed at rates lower than the U.S. rate, generated 71%, 82%, and 86% of our foreign income before tax. Other reconciling items, net consists primarily of tax credits and GILTI tax, and in fiscal year 2021, includes tax benefits from the India Supreme Court decision on withholding taxes. In fiscal years 2022, 2021, and 2020, there were no individually significant other reconciling items. - -The decrease in our effective tax rate for fiscal year 2022 compared to fiscal year 2021 was primarily due to a $3.3 billion net income tax benefit in the first quarter of fiscal year 2022 related to the transfer of intangible properties, offset in part by changes in the mix of our income before income taxes between the U.S. and foreign countries, as well as tax benefits in the prior year from the India Supreme Court decision on withholding taxes, an agreement between the U.S. and India tax authorities related to transfer pricing, and final Tax Cuts and Jobs Act (�TCJA�) regulations. The decrease in our effective tax rate for fiscal year 2021 compared to fiscal year 2020 was primarily due to tax benefits from the India Supreme Court decision on withholding taxes, an agreement between the U.S. and India tax authorities related to transfer pricing, final TCJA regulations, and an increase in tax benefits relating to stock-based compensation. - -83 - - -PART II -Item 8 - -The components of the deferred income tax assets and liabilities were as follows: - -(In millions) - - -June 30, - -2022 - -2021 - - -Deferred Income Tax Assets - - - - - - - - - - - - - - - - - -Stock-based compensation expense -$ -601 -$ -502 - - -Accruals, reserves, and other expenses - -2,874 - -2,960 - - -Loss and credit carryforwards - -1,546 - -1,090 - - -Amortization - -10,656 - -6,346 - - -Leasing liabilities - -4,557 - -4,060 - - -Unearned revenue - -2,876 - -2,659 - - -Other - -461 - -319 - - - - - - - - - - -Deferred income tax assets - -23,571 - -17,936 - -Less valuation allowance - -(1,012) - -(769) - - - - - - - - - -Deferred income tax assets, net of valuation allowance -$ -22,559 -$ -17,167 - - - - - - - - - - -Deferred Income Tax Liabilities - - - - - - - - - - - - - - - - - -Book/tax basis differences in investments and debt -$ -(174) -$ -(2,381) - -Leasing assets - -(4,291) - -(3,834) - -Depreciation - -(1,602) - -(1,010) - -Deferred tax on foreign earnings - -(3,104) - -(2,815) - -Other - -(103) - -(144) - - - - - - - - - -Deferred income tax liabilities -$ -(9,274) -$ -(10,184) - - - - - - - - - - - - - - - - - - - -Net deferred income tax assets -$ -13,285 -$ -6,983 - - -Reported As - - - - - - - - - - - - - - - - - -Other long-term assets -$ -13,515 -$ -7,181 - - -Long-term deferred income tax liabilities - -(230) - -(198) - - - - - - - - - - -Net deferred income tax assets -$ -13,285 -$ -6,983 - - - - - - - - - - - - - -Deferred income tax balances reflect the effects of temporary differences between the carrying amounts of assets and liabilities and their tax bases and are stated at enacted tax rates expected to be in effect when the taxes are paid or recovered. - -As of June 30, 2022, we had federal, state, and foreign net operating loss carryforwards of $318 million, $ 1.3 billion, and $2.1 billion, respectively. The federal and state net operating loss carryforwards will expire in various years from fiscal 2023 through 2042, if not utilized. The majority of our foreign net operating loss carryforwards do not expire. Certain acquired net operating loss carryforwards are subject to an annual limitation but are expected to be realized with the exception of those which have a valuation allowance. As of June 30, 2022, we had $1.3 billion federal capital loss carryforwards for U.S. tax purposes from our acquisition of Nuance. The federal capital loss carryforwards are subject to an annual limitation and will expire in various years from fiscal 2023 through 2025. - -The valuation allowance disclosed in the table above relates to the foreign net operating loss carryforwards, federal capital loss carryforwards, and other net deferred tax assets that may not be realized. - -Income taxes paid, net of refunds, were $16.0 billion, $13.4 billion, and $12.5 billion in fiscal years 2022, 2021, and 2020, respectively. - -Uncertain Tax Positions - -Gross unrecognized tax benefits related to uncertain tax positions as of June 30, 2022, 2021, and 2020, were $15.6 billion, $14.6 billion, and $13.8 billion, respectively, which were primarily included in long-term income taxes in our consolidated balance sheets. If recognized, the resulting tax benefit would affect our effective tax rates for fiscal years 2022, 2021, and 2020 by $13.3 billion, $12.5 billion, and $12.1 billion, respectively. - -84 - - -PART II -Item 8 - -As of June 30, 2022, 2021, and 2020, we had accrued interest expense related to uncertain tax positions of $4.3 billion, $4.3 billion, and $4.0 billion, respectively, net of income tax benefits. The provision for income taxes for fiscal years 2022, 2021, and 2020 included interest expense related to uncertain tax positions of $36 million, $274 million, and $579 million, respectively, net of income tax benefits. - -The aggregate changes in the gross unrecognized tax benefits related to uncertain tax positions were as follows: - -(In millions) - - -Year Ended June 30, - -2022 - -2021 - - -2020 - - - - - - - - - - - -Beginning unrecognized tax benefits -$ -14,550 -$ -13,792 -$ -13,146 -Decreases related to settlements - -(317) - -(195) - - -(31) -Increases for tax positions related to the current year - -1,145 - -790 - - -647 -Increases for tax positions related to prior years - -461 - -461 - - -366 -Decreases for tax positions related to prior years - -(246) - -(297) - - -(331) -Decreases due to lapsed statutes of limitations - -0 - -(1) - - -(5) - - - - - - - - - - - -Ending unrecognized tax benefits -$ -15,593 -$ -14,550 -$ -13,792 - - - - - - - - - - - - -We settled a portion of the Internal Revenue Service (�IRS�) audit for tax years 2004 to 2006 in fiscal year 2011. In February 2012, the IRS withdrew its 2011 Revenue Agents Report related to unresolved issues for tax years 2004 to 2006 and reopened the audit phase of the examination. We also settled a portion of the IRS audit for tax years 2007 to 2009 in fiscal year 2016, and a portion of the IRS audit for tax years 2010 to 2013 in fiscal year 2018. In the second quarter of fiscal year 2021, we settled an additional portion of the IRS audits for tax years 2004 to 2013 and made a payment of $1.7 billion, including tax and interest. We remain under audit for tax years 2004 to 2017. - -As of June 30, 2022, the primary unresolved issues for the IRS audits relate to transfer pricing, which could have a material impact in our consolidated financial statements when the matters are resolved. We believe our allowances for income tax contingencies are adequate. We have not received a proposed assessment for the unresolved key transfer pricing issues and do not expect a final resolution of these issues in the next 12 months. Based on the information currently available, we do not anticipate a significant increase or decrease to our tax contingencies for these issues within the next 12 months. - -We are subject to income tax in many jurisdictions outside the U.S. Our operations in certain jurisdictions remain subject to examination for tax years 1996 to 2021, some of which are currently under audit by local tax authorities. The resolution of each of these audits is not expected to be material to our consolidated financial statements. - -NOTE 13 � UNEARNED REVENUE - -Unearned revenue by segment was as follows: - -(In millions) - - -June 30, - - -2022 - - - - -2021 - - - - - - - - - - - - - -Productivity and Business Processes -$ -24,558 -$ -22,120 - -Intelligent Cloud - - -19,371 - - - -17,710 - -More Personal Computing - - -4,479 - - - -4,311 - - - - - - - - - - - -Total -$ -48,408 -$ -44,141 - -Changes in unearned revenue were as follows: - - - - - - - - - - - - - - - - - - - - - -(In millions) - - - - - - - - - - - - - - - - - - - - - -Year Ended June 30, 2022 - - - - - - - - - - - - - - - - - - - -Balance, beginning of period - - - - -$ -44,141 - -Deferral of revenue - - - - - - - -110,455 - -Recognition of unearned revenue - - - - - - - -(106,188) - - - - - - - - - - -Balance, end of period - - - - -$ -48,408 - - -85 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -Revenue allocated to remaining performance obligations, which includes unearned revenue and amounts that will be invoiced and recognized as revenue in future periods, was $ 193 billion as of June 30, 2022, of which $189 billion is related to the commercial portion of revenue. We expect to recognize approximately 45% of this revenue over the next 12 months and the remainder thereafter. - -NOTE 14 � LEASES - -We have operating and finance leases for datacenters, corporate offices, research and development facilities, Microsoft Experience Centers, and certain equipment. Our leases have remaining lease terms of 1 year to 19 years, some of which include options to extend the leases for up to 5 years, and some of which include options to terminate the leases within 1 year. - -The components of lease expense were as follows: - -(In millions) - - -Year Ended June 30, - -2022 - -2021 - -2020 - - - - - - - - - - - - -Operating lease cost -$ -2,461 -$ -2,127 -$ -2,043 - -Finance lease cost: - - - - - - - - - - - - - - - - - - - - - -Amortization of right-of-use assets -$ -980 -$ -921 - -$ -611 - -Interest on lease liabilities - -429 - -386 - - -336 - - - - - - - - - - - - -Total finance lease cost -$ -1,409 -$ -1,307 -$ -947 - -Supplemental cash flow information related to leases was as follows: - - - - - - - - - - - - - - - - - - - - - -(In millions) - - - - - - - - - - - - - - - - - - - -Year Ended June 30, - -2022 - -2021 - -2020 - - - - - - - - - - - - -Cash paid for amounts included in the measurement of lease liabilities: - - - - - - - - - - -Operating cash flows from operating leases -$ -2,368 -$ -2,052 -$ -1,829 - -Operating cash flows from finance leases - -429 - -386 - - -336 - -Financing cash flows from finance leases - -896 - -648 - - -409 - -Right-of-use assets obtained in exchange for lease obligations: - - - - - - - - - - -Operating leases - -5,268 - -4,380 - -3,677 - -Finance leases - -4,234 - -3,290 - -3,467 - - - - - - - - - - - - -86 - - - - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -Supplemental balance sheet information related to leases was as follows: - -(In millions, except lease term and discount rate) - - -June 30, - - -2022 - - - - - -2021 - - - -Operating Leases - - - - - - - - - - - - - - - - - - - - - - - - - -Operating lease right-of-use assets -$ -13,148 - -$ -11,088 - - - - - - - - - - - - - - - -Other current liabilities -$ -2,228 - -$ -1,962 - - -Operating lease liabilities - - -11,489 - - - - -9,629 - - - - - - - - - - - - - - - -Total operating lease liabilities -$ -13,717 - -$ -11,591 - - -Finance Leases - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Property and equipment, at cost -$ -17,388 - -$ -14,107 - - -Accumulated depreciation - - -(3,285) - - - -(2,306) - - - - - - - - - - - - - - -Property and equipment, net -$ -14,103 - -$ -11,801 - - - - - - - - - - - - - - - -Other current liabilities -$ -1,060 - -$ -791 - - -Other long-term liabilities - - -13,842 - - - - -11,750 - - - - - - - - - - - - - - - -Total finance lease liabilities -$ -14,902 - -$ -12,541 - - -Weighted Average Remaining Lease Term - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Operating leases - - -8 years - - - -8 years - -Finance leases - - -12 years - - - -12 years - -Weighted Average Discount Rate - - - - - - - - - - - - - - - - - - - - - - - - - -Operating leases - - -2.1% - - - - -2.2% - - -Finance leases - - -3.1% - - - - -3.4% - - -The following table outlines maturities of our lease liabilities as of June 30, 2022: - - - - - - - - - - - - -(In millions) - - - - - - - - - - - - - - - - - - - - - - - - - -Operating - - - -Finance - -Year Ending June 30, - - -Leases - - - -Leases - - - - - - - - - - - -2023 -$ -2,456 -$ -1,477 - -2024 - - -2,278 - - - -1,487 - -2025 - - -1,985 - - - -1,801 - -2026 - - -1,625 - - - -1,483 - -2027 - - -1,328 - - - -1,489 - -Thereafter - - -5,332 - - - -9,931 - - - - - - - - - - - - - - -Total lease payments - - -15,004 - - - -17,668 - -Less imputed interest - - -(1,287) - - - -(2,766) - - - - - - - - - - - - - - -Total -$ -13,717 -$ -14,902 - - - - - - - - - - - - - - - -As of June 30, 2022, we have additional operating and finance leases, primarily for datacenters, that have not yet commenced of $7.2 billion and $ 8.8 billion, respectively. These operating and finance leases will commence between fiscal year 2023 and fiscal year 2028 with lease terms of 1 year to 18 years. - - -87 - - -PART II -Item 8 - - -NOTE 15 � CONTINGENCIES - -Antitrust Litigation and Claims - -China State Administration for Market Regulation Investigation - -In 2014, Microsoft was informed that China�s State Agency for Market Regulation (�SAMR�) (formerly State Administration for Industry and Commerce) had begun a formal investigation relating to China�s Anti-Monopoly Law, and the SAMR conducted onsite inspections of Microsoft offices in Beijing, Shanghai, Guangzhou, and Chengdu. In 2019, the SAMR presented preliminary views as to certain possible violations of China�s Anti-Monopoly Law. - -Product-Related Litigation - -U.S. Cell Phone Litigation - -Microsoft Mobile Oy, a subsidiary of Microsoft, along with other handset manufacturers and network operators, is a defendant in 46 lawsuits, including 45 lawsuits filed in the Superior Court for the District of Columbia by individual plaintiffs who allege that radio emissions from cellular handsets caused their brain tumors and other adverse health effects. We assumed responsibility for these claims in our agreement to acquire Nokia�s Devices and Services business and have been substituted for the Nokia defendants. Nine of these cases were filed in 2002 and are consolidated for certain pre-trial proceedings; the remaining cases are stayed. In a separate 2009 decision, the Court of Appeals for the District of Columbia held that adverse health effect claims arising from the use of cellular handsets that operate within the U.S. Federal Communications Commission radio frequency emission guidelines (�FCC Guidelines�) are preempted by federal law. The plaintiffs allege that their handsets either operated outside the FCC Guidelines or were manufactured before the FCC Guidelines went into effect. The lawsuits also allege an industry-wide conspiracy to manipulate the science and testing around emission guidelines. - -In 2013, the defendants in the consolidated cases moved to exclude the plaintiffs� expert evidence of general causation on the basis of flawed scientific methodologies. In 2014, the trial court granted in part and denied in part the defendants� motion to exclude the plaintiffs� general causation experts. The defendants filed an interlocutory appeal to the District of Columbia Court of Appeals challenging the standard for evaluating expert scientific evidence. In October 2016, the Court of Appeals issued its decision adopting the standard advocated by the defendants and remanding the cases to the trial court for further proceedings under that standard. The plaintiffs have filed supplemental expert evidence, portions of which the defendants have moved to strike. In August 2018, the trial court issued an order striking portions of the plaintiffs� expert reports. A hearing on general causation is scheduled for September of 2022. - -Other Contingencies - -We also are subject to a variety of other claims and suits that arise from time to time in the ordinary course of our business. Although management currently believes that resolving claims against us, individually or in aggregate, will not have a material adverse impact in our consolidated financial statements, these matters are subject to inherent uncertainties and management�s view of these matters may change in the future. - -As of June 30, 2022, we accrued aggregate legal liabilities of $364 million. While we intend to defend these matters vigorously, adverse outcomes that we estimate could reach approximately $600 million in aggregate beyond recorded amounts are reasonably possible. Were unfavorable final outcomes to occur, there exists the possibility of a material adverse impact in our consolidated financial statements for the period in which the effects become reasonably estimable. - -88 - - -PART II -Item 8 - - -NOTE 16 � STOCKHOLDERS� EQUITY - -Shares Outstanding - -Shares of common stock outstanding were as follows: - -(In millions) - - -Year Ended June 30, -2022 - -2021 -2020 - - - - - - - - -Balance, beginning of year -7,519 -7,571 -7,643 - -Issued -40 -49 -54 - -Repurchased -(95) -(101) -(126) - - - - - - - -Balance, end of year -7,464 -7,519 -7,571 - - - - - - - - - -Share Repurchases - -On September 20, 2016, our Board of Directors approved a share repurchase program authorizing up to $40.0 billion in share repurchases. -This share repurchase program commenced in December 2016 and was completed in February 2020. - -On September 18, 2019, our Board of Directors approved a share repurchase program authorizing up to $40.0 billion in share repurchases. -This share repurchase program commenced in February 2020 and was completed in November 2021. - -On September 14, 2021, our Board of Directors approved a share repurchase program authorizing up to $60.0 billion in share repurchases. This share repurchase program commenced in November 2021, following completion of the program approved on September 18, 2019, has no expiration date, and may be terminated at any time. As of June 30, 2022, $40.7 billion remained of this $60.0 billion share repurchase program. - -We repurchased the following shares of common stock under the share repurchase programs: - -(In millions) -Shares - - -Amount -Shares - - -Amount -Shares - -Amount - - - - - - - - - - - - - - - - -Year Ended June 30, - - - - -2022 - - - - -2021 - - - -2020 - - - - - - - - - - - - - - - - - - - -First Quarter -21 -$ -6,200 -25 -$ -5,270 -29 -$ -4,000 - -Second Quarter -20 - - -6,233 -27 - - -5,750 -32 - -4,600 - -Third Quarter -26 - - -7,800 -25 - - -5,750 -37 - -6,000 - -Fourth Quarter -28 - - -7,800 -24 - - -6,200 -28 - -5,088 - - - - - - - - - - - - - - - - - - - -Total -95 -$ -28,033 -101 -$ -22,970 -126 - -$ -19,688 - - - - - - - - - - - - - - - - - - - -All repurchases were made using cash resources. Shares repurchased during the fourth and third quarters of fiscal year 2022 were under the share repurchase program approved on September 14, 2021. Shares repurchased during the second quarter of fiscal year 2022 were under the share repurchase programs approved on both September 14, 2021 and September 18, 2019. Shares repurchased during the first quarter of fiscal year 2022, fiscal year 2021, and the fourth quarter of fiscal year 2020 were under the share repurchase program approved on September 18, 2019. Shares repurchased during the third quarter of fiscal year 2020 were under the share repurchase programs approved on both September 20, 2016 and September 18, 2019. All other shares repurchased were under the share repurchase program approved on September 20, 2016. The above table excludes shares repurchased to settle employee tax withholding related to the vesting of stock awards of $4.7 billion, $4.4 billion, and $3.3 billion for fiscal years 2022, 2021, and 2020, respectively. - -89 - - -PART II -Item 8 - - -Dividends - -Our Board of Directors declared the following dividends: - - - - - -Dividend - - - - - -Declaration Date -Record Date -Payment Date - -Per Share - - -Amount - - - - - - - - - - - - -Fiscal Year 2022 - - - - - - - -(In millions) - - - - - - - - - - - - -September 14, 2021 -November 18, 2021 -December 9, 2021 -$ -0.62 -$ -4,652 - -December 7, 2021 -February 17, 2022 -March 10, 2022 - -0.62 - - -4,645 - -March 14, 2022 -May 19, 2022 -June 9, 2022 - -0.62 - - -4,632 - -June 14, 2022 -August 18, 2022 -September 8, 2022 - -0.62 - - -4,627 - - - - - - - - - - - - -Total - - -$ -2.48 -$ -18,556 - -Fiscal Year 2021 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -September 15, 2020 -November 19, 2020 -December 10, 2020 -$ -0.56 -$ -4,230 - -December 2, 2020 -February 18, 2021 -March 11, 2021 - -0.56 - - -4,221 - -March 16, 2021 -May 20, 2021 -June 10, 2021 - -0.56 - - -4,214 - -June 16, 2021 -August 19, 2021 -September 9, 2021 - -0.56 - - -4,206 - - - - - - - - - - - - - - - - - - - - - - - -Total - - -$ -2.24 -$ -16,871 - - - - - - - - - - - - - -The dividend declared on June 14, 2022 was included in other current liabilities as of June 30, 2022. - -90 - - -PART II -Item 8 - - - -NOTE 17 � ACCUMULATED OTHER COMPREHENSIVE INCOME (LOSS) - -The following table summarizes the changes in accumulated other comprehensive income (loss) by component: - -(In millions) - - -Year Ended June 30, - - -2022 - - -2021 - -2020 - -Derivatives - - - - - - - - - - - - - - - - - - - - - - - - - -Balance, beginning of period -$ -(19) -$ -(38) -$ -0 - -Unrealized gains (losses), net of tax of $(15), $ 9, and $(10) - - -(57) - - -34 - -(38) -Reclassification adjustments for (gains) losses included in other income (expense), net - - -79 - - -(17) - -0 - -Tax expense (benefit) included in provision for income taxes - - -(16) - - -2 - -0 - - - - - - - - - - - - - - -Amounts reclassified from accumulated other comprehensive income (loss) - - -63 - - -(15) - -0 - - - - - - - - - - - - - -Net change related to derivatives, net of tax of $1, $7, and $(10) - - -6 - - -19 - -(38) - - - - - - - - - - - - - - - - - - - - - - - - - - -Balance, end of period -$ -(13) -$ -(19) -$ -(38) - - - - - - - - - - - - - -Investments - - - - - - - - - - - - - - - - - - - - - - - - - -Balance, beginning of period -$ -3,222 -$ -5,478 -$ -1,488 - -Unrealized gains (losses), net of tax of $(1,440), $(589), and $1,057 - - -(5,405) - - -(2,216) - -3,987 - -Reclassification adjustments for (gains) losses included in other income (expense), net - - -57 - - -(63) - -4 - -Tax expense (benefit) included in provision for income taxes - - -(12) - - -13 - -(1) - - - - - - - - - - - - - -Amounts reclassified from accumulated other comprehensive income (loss) - - -45 - - -(50) - -3 - - - - - - - - - - - - - - -Net change related to investments, net of tax of $(1,428), $(602), and $1,058 - - -(5,360) - - -(2,266) - -3,990 - -Cumulative effect of accounting changes - - -0 - - -10 - -0 - - - - - - - - - - - - - -Balance, end of period -$ -(2,138) -$ -3,222 -$ -5,478 - - - - - - - - - - - - - - -Translation Adjustments and Other - - - - - - - - - - - - - - - - - - - - - - - - - -Balance, beginning of period -$ -(1,381) -$ -(2,254) -$ -(1,828) -Translation adjustments and other, net of tax of $0, $(9), and $1 - - -(1,146) - - -873 - -(426) - - - - - - - - - - - - - -Balance, end of period -$ -(2,527) -$ -(1,381) -$ -(2,254) - - - - - - - - - - - - -Accumulated other comprehensive income (loss), end of period -$ -(4,678) -$ -1,822 -$ -3,186 - - - - - - - - - - - - - - - -NOTE 18 � EMPLOYEE STOCK AND SAVINGS PLANS - -We grant stock-based compensation to employees and directors. Awards that expire or are canceled without delivery of shares generally become available for issuance under the plans. We issue new shares of Microsoft common stock to satisfy vesting of awards granted under our stock plans. We also have an ESPP for all eligible employees. - -Stock-based compensation expense and related income tax benefits were as follows: - -(In millions) - - -Year Ended June 30, - -2022 - -2021 - -2020 - - - - - - - - - -Stock-based compensation expense -$ -7,502 -$ -6,118 -$ -5,289 - -Income tax benefits related to stock-based compensation - -1,293 - -1,065 - -938 - - -Stock Plans - -Stock awards entitle the holder to receive shares of Microsoft common stock as the award vests. Stock awards generally vest over a service period of four years or five years. - -91 - - -PART II -Item 8 - - -Executive Incentive Plan - -Under the Executive Incentive Plan, the Compensation Committee approves stock awards to executive officers and certain senior executives. RSUs generally vest ratably over a service period of four years. PSUs generally vest over a performance period of three years. The number of shares the PSU holder receives is based on the extent to which the corresponding performance goals have been achieved. - -Activity for All Stock Plans - -The fair value of stock awards was estimated on the date of grant using the following assumptions: - - -Year ended June 30, - -2022 -2021 - - - -2020 - - - - - - - - - - - - -Dividends per share (quarterly amounts) -$ -0.56�0.62 $ -0.51�0.56 -$ -0.46�0.51 -Interest rates - -0.03%�3.6% -0.01%�1.5% - - -0.1%�2.2% -During fiscal year 2022, the following activity occurred under our stock plans: - - - - - - - - - - - - - - - -Weighted Average - - - - - - - -Grant-Date Fair - - - -Shares - - -Value - - - - - - - - - - - - -(In millions) - - - - - -Stock Awards - - - - - - - - - - - - - - - - - - - -Nonvested balance, beginning of year - - -100 - -$ -152.51 - -Granted (a) - - -50 - - -291.22 - -Vested - - -(47) - - -143.10 - -Forfeited - - -(10) - - -189.88 - - - - - - - - - - - -Nonvested balance, end of year - - -93 -$ -227.59 - - - - - - - - - - - - -(a) Includes 1 million, 2 million, and 2 million of PSUs granted at target and performance adjustments above target levels for fiscal years 2022, 2021, and 2020, respectively. - -As of June 30, 2022, there was approximately $16.7 billion of total unrecognized compensation costs related to stock awards. These costs are expected to be recognized over a weighted average period of three years. The weighted average grant-date fair value of stock awards granted was $291.22, $221.13, and $ 140.49 for fiscal years 2022, 2021, and 2020, respectively. The fair value of stock awards vested was $14.1 billion, $13.4 billion, and $10.1 billion, for fiscal years 2022, 2021, and 2020, respectively. As of June 30, 2022, an aggregate of 211 million shares were authorized for future grant under our stock plans. - -Employee Stock Purchase Plan - -We have an ESPP for all eligible employees. Shares of our common stock may be purchased by employees at three-month intervals at 90% of the fair market value on the last trading day of each three-month period. Employees may purchase shares having a value not exceeding 15% of their gross compensation during an offering period. Under the terms of the ESPP that were approved in 2012, the plan was set to terminate on December 31, 2022. At our 2021 Annual Shareholders Meeting, our shareholders approved a successor ESPP with a January 1, 2022 effective date and ten-year expiration of December 31, 2031. No additional shares were requested at this meeting. - -Employees purchased the following shares during the periods presented: - -(Shares in millions) - - -Year Ended June 30, -2022 - -2021 - -2020 - - - - - - - - -Shares purchased -7 - -8 - -9 - -Average price per share -$ 259.55 -$ -207.88 -$ -142.22 - - -As of June 30, 2022, 81 million shares of our common stock were reserved for future issuance through the ESPP. 92 - - -PART II -Item 8 - - -Savings Plans - -We have savings plans in the U.S. that qualify under Section 401(k) of the Internal Revenue Code, and a number of savings plans in international locations. Eligible U.S. employees may contribute a portion of their salary into the savings plans, subject to certain limitations. We match a portion of each dollar a participant contributes into the plans. Employer-funded retirement benefits for all plans were $1.4 billion, $1.2 billion, and $1.0 billion in fiscal years 2022, 2021, and 2020, respectively, and were expensed as contributed. - -NOTE 19 � SEGMENT INFORMATION AND GEOGRAPHIC DATA - -In its operation of the business, management, including our chief operating decision maker, who is also our Chief Executive Officer, reviews certain financial information, including segmented internal profit and loss statements prepared on a basis not consistent with GAAP. During the periods presented, we reported our financial performance based on the following segments: Productivity and Business Processes, Intelligent Cloud, and More Personal Computing. - -Our reportable segments are described below. - -Productivity and Business Processes - -Our Productivity and Business Processes segment consists of products and services in our portfolio of productivity, communication, and information services, spanning a variety of devices and platforms. This segment primarily comprises: - -� Office Commercial (Office 365 subscriptions, the Office 365 portion of Microsoft 365 Commercial subscriptions, and Office licensed on-premises), comprising Office, Exchange, SharePoint, Microsoft Teams, Office 365 Security and Compliance, and Microsoft Viva. - -� Office Consumer, including Microsoft 365 Consumer subscriptions, Office licensed on-premises, and other Office services. - -� LinkedIn, including Talent Solutions, Marketing Solutions, Premium Subscriptions, and Sales Solutions. - -� Dynamics business solutions, including Dynamics 365, comprising a set of intelligent, cloud-based applications across ERP, CRM, Customer Insights, Power Apps, and Power Automate; and on-premises ERP and CRM applications. - -Intelligent Cloud - -Our Intelligent Cloud segment consists of our public, private, and hybrid server products and cloud services that can power modern business and developers. This segment primarily comprises: - -� Server products and cloud services, including Azure and other cloud services; SQL Server, Windows Server, Visual Studio, System Center, and related Client Access Licenses (�CALs�); and Nuance and GitHub. - -� Enterprise Services, including Enterprise Support Services, Microsoft Consulting Services, and Nuance professional services. - -More Personal Computing - -Our More Personal Computing segment consists of products and services that put customers at the center of the experience with our technology. This segment primarily comprises: - -� Windows, including Windows OEM licensing and other non-volume licensing of the Windows operating system; Windows Commercial, comprising volume licensing of the Windows operating system, Windows cloud services, and other Windows commercial offerings; patent licensing; and Windows Internet of Things. - -� Devices, including Surface and PC accessories. - -� Gaming, including Xbox hardware and Xbox content and services, comprising first- and third-party content (including games and in-game content), Xbox Game Pass and other subscriptions, Xbox Cloud Gaming, third-party disc royalties, advertising, and other cloud services. - -� Search and news advertising. - -93 - - -PART II -Item 8 - - -Revenue and costs are generally directly attributed to our segments. However, due to the integrated structure of our business, certain revenue recognized and costs incurred by one segment may benefit other segments. Revenue from certain contracts is allocated among the segments based on the relative value of the underlying products and services, which can include allocation based on actual prices charged, prices when sold separately, or estimated costs plus a profit margin. Cost of revenue is allocated in certain cases based on a relative revenue methodology. Operating expenses that are allocated primarily include those relating to marketing of products and services from which multiple segments benefit and are generally allocated based on relative gross margin. - -In addition, certain costs incurred at a corporate level that are identifiable and that benefit our segments are allocated to them. These allocated costs include legal, including settlements and fines, information technology, human resources, finance, excise taxes, field selling, shared facilities services, and customer service and support. Each allocation is measured differently based on the specific facts and circumstances of the costs being allocated. - -Segment revenue and operating income were as follows during the periods presented: - -(In millions) - - -Year Ended June 30, - -2022 - - -2021 - -2020 - - -Revenue - - - - - - - - - - - - - - - - - - - - - - - -Productivity and Business Processes -$ -63,364 -$ -53,915 -$ -46,398 - - -Intelligent Cloud - -75,251 - - -60,080 - -48,366 - - -More Personal Computing - -59,655 - - -54,093 - -48,251 - - - - - - - - - - - - - - -Total -$ -198,270 -$ -168,088 -$ -143,015 - - -Operating Income - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Productivity and Business Processes -$ -29,687 -$ -24,351 -$ -18,724 - - -Intelligent Cloud - -32,721 - - -26,126 - -18,324 - - -More Personal Computing - -20,975 - - -19,439 - -15,911 - - - - - - - - - - - - - - -Total -$ -83,383 -$ -69,916 -$ -52,959 - - - - - - - - - - - - - - - -No sales to an individual customer or country other than the United States accounted for more than 10% of revenue for fiscal years 2022, 2021, or 2020. Revenue, classified by the major geographic areas in which our customers were located, was as follows: - -(In millions) - - -Year Ended June 30, - -2022 - - - -2021 - - - -2020 - - - - - - - - - - - - - -United States (a) -$ -100,218 -$ -83,953 -$ -73,160 -Other countries - -98,052 - - -84,135 - - -69,855 - - - - - - - - - - - - -Total -$ -198,270 -$ -168,088 -$ -143,015 - - - - - - - - - - - - - -(a) Includes billings to OEMs and certain multinational organizations because of the nature of these businesses and the impracticability of determining the geographic source of the revenue. - -94 - - -PART II -Item 8 - -Revenue, classified by significant product and service offerings, was as follows: - -(In millions) - - -Year Ended June 30, - -2022 - - - -2021 - - -2020 - - - - - - - - - - - - -Server products and cloud services -$ -67,321 -$ -52,589 -$ -41,379 -Office products and cloud services - -44,862 - - -39,872 - -35,316 -Windows - -24,761 - - -22,488 - -21,510 -Gaming - -16,230 - - -15,370 - -11,575 -LinkedIn - -13,816 - - -10,289 - -8,077 - -Search and news advertising - -11,591 - - -9,267 - -8,524 - -Enterprise Services - -7,407 - - -6,943 - -6,409 - -Devices - -6,991 - - -6,791 - -6,457 - -Other - -5,291 - - -4,479 - -3,768 - - - - - - - - - - - - -Total -$ -198,270 -$ -168,088 -$ -143,015 - - - - - - - - - - - - - -We have recast certain previously reported amounts in the table above to conform to the way we internally manage and monitor our business. - -Our Microsoft Cloud (formerly commercial cloud) revenue, which includes Azure and other cloud services, Office 365 Commercial, the commercial portion of LinkedIn, Dynamics 365, and other commercial cloud properties, was $ 91.2 billion, $ 69.1 billion and $51.7 billion in fiscal years 2022, 2021, and 2020, respectively. These amounts are primarily included in Server products and cloud services, Office products and cloud services, and LinkedIn in the table above. - -Assets are not allocated to segments for internal reporting presentations. A portion of amortization and depreciation is included with various other costs in an overhead allocation to each segment. It is impracticable for us to separately identify the amount of amortization and depreciation by segment that is included in the measure of segment profit or loss. - -Long-lived assets, excluding financial instruments and tax assets, classified by the location of the controlling statutory company and with countries over 10% of the total shown separately, were as follows: - -(In millions) - - -June 30, - - -2022 - - - -2021 - - - -2020 - - - - - - - - - - - - - - - - -United States -$ -106,430 -$ -76,153 -$ -60,789 - -Ireland - - -15,505 - - -13,303 - - -12,734 - -Other countries - - -44,433 - - -38,858 - - -29,770 - - - - - - - - - - - - - - -Total -$ -166,368 -$ -128,314 -$ -103,293 - - -95 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -PART II -Item 8 - - -REPORT OF INDEPENDENT REGISTERED PUBLIC ACCOUNTING FIRM To the Stockholders and the Board of Directors of Microsoft Corporation Opinion on the Financial Statements -We have audited the accompanying consolidated balance sheets of Microsoft Corporation and subsidiaries (the "Company") as of June 30, 2022 and 2021, the related consolidated statements of income, comprehensive income, cash flows, and stockholders' equity, for each of the three years in the period ended June 30, 2022, and the related notes (collectively referred to as the "financial statements"). In our opinion, the financial statements present fairly, in all material respects, the financial position of the Company as of June 30, 2022 and 2021, and the results of its operations and its cash flows for each of the three years in the period ended June 30, 2022, in conformity with accounting principles generally accepted in the United States of America. - -We have also audited, in accordance with the standards of the Public Company Accounting Oversight Board (United States) (PCAOB), the Company's internal control over financial reporting as of June 30, 2022, based on criteria established in Internal Control � Integrated Framework (2013) issued by the Committee of Sponsoring Organizations of the Treadway Commission and our report dated July 28, 2022, expressed an unqualified opinion on the Company's internal control over financial reporting. - -Basis for Opinion - -These financial statements are the responsibility of the Company's management. Our responsibility is to express an opinion on the Company's financial statements based on our audits. We are a public accounting firm registered with the PCAOB and are required to be independent with respect to the Company in accordance with the U.S. federal securities laws and the applicable rules and regulations of the Securities and Exchange Commission and the PCAOB. - -We conducted our audits in accordance with the standards of the PCAOB. Those standards require that we plan and perform the audit to obtain reasonable assurance about whether the financial statements are free of material misstatement, whether due to error or fraud. Our audits included performing procedures to assess the risks of material misstatement of the financial statements, whether due to error or fraud, and performing procedures that respond to those risks. Such procedures included examining, on a test basis, evidence regarding the amounts and disclosures in the financial statements. Our audits also included evaluating the accounting principles used and significant estimates made by management, as well as evaluating the overall presentation of the financial statements. We believe that our audits provide a reasonable basis for our opinion. - -Critical Audit Matters - -The critical audit matters communicated below are matters arising from the current-period audit of the financial statements that were communicated or required to be communicated to the audit committee and that (1) relate to accounts or disclosures that are material to the financial statements and (2) involved our especially challenging, subjective, or complex judgments. The communication of critical audit matters does not alter in any way our opinion on the financial statements, taken as a whole, and we are not, by communicating the critical audit matters below, providing separate opinions on the critical audit matters or on the accounts or disclosures to which they relate. - - -96 - - -PART II -Item 8 - - -Revenue Recognition � Refer to Note 1 to the financial statements Critical Audit Matter Description - -The Company recognizes revenue upon transfer of control of promised products or services to customers in an amount that reflects the consideration the Company expects to receive in exchange for those products or services. The Company offers customers the ability to acquire multiple licenses of software products and services, including cloud-based services, in its customer agreements through its volume licensing programs. - -Significant judgment is exercised by the Company in determining revenue recognition for these customer agreements, and includes the following: - -� Determination of whether products and services are considered distinct performance obligations that should be accounted for separately versus together, such as software licenses and related services that are sold with cloud-based services. - -� The pattern of delivery (i.e., timing of when revenue is recognized) for each distinct performance obligation. - -� Identification and treatment of contract terms that may impact the timing and amount of revenue recognized (e.g., variable consideration, optional purchases, and free services). - -� Determination of stand-alone selling prices for each distinct performance obligation and for products and services that are not sold separately. - -Given these factors and due to the volume of transactions, the related audit effort in evaluating management's judgments in determining revenue recognition for these customer agreements was extensive and required a high degree of auditor judgment. - -How the Critical Audit Matter Was Addressed in the Audit - -Our principal audit procedures related to the Company's revenue recognition for these customer agreements included the following: - -� We tested the effectiveness of controls related to the identification of distinct performance obligations, the determination of the timing of revenue recognition, and the estimation of variable consideration. - -� We evaluated management's significant accounting policies related to these customer agreements for reasonableness. - -� We selected a sample of customer agreements and performed the following procedures: - -- Obtained and read contract source documents for each selection, including master agreements, and other documents that were part of the agreement. - -- Tested management's identification and treatment of contract terms. - -- Assessed the terms in the customer agreement and evaluated the appropriateness of management's application of their accounting policies, along with their use of estimates, in the determination of revenue recognition conclusions. - -� We evaluated the reasonableness of management's estimate of stand-alone selling prices for products and services that are not sold separately. - -� We tested the mathematical accuracy of management's calculations of revenue and the associated timing of revenue recognized in the financial statements. - - -97 - - -PART II -Item 8 - - -Income Taxes � Uncertain Tax Positions � Refer to Note 12 to the financial statements Critical Audit Matter Description - -The Company's long-term income taxes liability includes uncertain tax positions related to transfer pricing issues that remain unresolved with the Internal Revenue Service ("IRS"). The Company remains under IRS audit, or subject to IRS audit, for tax years subsequent to 2003. While the Company has settled a portion of the IRS audits, resolution of the remaining matters could have a material impact on the Company's financial statements. - -Conclusions on recognizing and measuring uncertain tax positions involve significant estimates and management judgment and include complex considerations of the Internal Revenue Code, related regulations, tax case laws, and prior-year audit settlements. Given the complexity and the subjective nature of the transfer pricing issues that remain unresolved with the IRS, evaluating management's estimates relating to their determination of uncertain tax positions required extensive audit effort and a high degree of auditor judgment, including involvement of our tax specialists. - -How the Critical Audit Matter Was Addressed in the Audit - -Our principal audit procedures to evaluate management's estimates of uncertain tax positions related to unresolved transfer pricing issues included the following: - -� We evaluated the appropriateness and consistency of management's methods and assumptions used in the identification, recognition, measurement, and disclosure of uncertain tax positions, which included testing the effectiveness of the related internal controls. - -� We read and evaluated management's documentation, including relevant accounting policies and information obtained by management from outside tax specialists, that detailed the basis of the uncertain tax positions. - -� We tested the reasonableness of management's judgments regarding the future resolution of the uncertain tax positions, including an evaluation of the technical merits of the uncertain tax positions. - -� For those uncertain tax positions that had not been effectively settled, we evaluated whether management had appropriately considered new information that could significantly change the recognition, measurement or disclosure of the uncertain tax positions. - -� We evaluated the reasonableness of management's estimates by considering how tax law, including statutes, regulations and case law, impacted management's judgments. - -/s/ DELOITTE & TOUCHE LLP - -Seattle, Washington -July 28, 2022 - -We have served as the Company's auditor since 1983. - - - -98 - - -PART II -Item 9, 9A - -ITEM 9. CHANGES IN AND DISAGREEMENTS WITH ACCOUNTANTS ON ACCOUNTING AND FINANCIAL DISCLOSURE - -Not applicable. - -ITEM 9A. CONTROLS AND PROCEDURES - -Under the supervision and with the participation of our management, including the Chief Executive Officer and Chief Financial Officer, we have evaluated the effectiveness of our disclosure controls and procedures as required by Exchange Act Rule 13a-15(b) as of the end of the period covered by this report. Based on that evaluation, the Chief Executive Officer and Chief Financial Officer have concluded that these disclosure controls and procedures are effective. - -REPORT OF MANAGEMENT ON INTERNAL CONTROL OVER FINANCIAL REPORTING - -Our management is responsible for establishing and maintaining adequate internal control over financial reporting for the Company. Internal control over financial reporting is a process to provide reasonable assurance regarding the reliability of our financial reporting for external purposes in accordance with accounting principles generally accepted in the United States of America. Internal control over financial reporting includes maintaining records that in reasonable detail accurately and fairly reflect our transactions; providing reasonable assurance that transactions are recorded as necessary for preparation of our consolidated financial statements; providing reasonable assurance that receipts and expenditures of company assets are made in accordance with management authorization; and providing reasonable assurance that unauthorized acquisition, use, or disposition of company assets that could have a material effect on our consolidated financial statements would be prevented or detected on a timely basis. Because of its inherent limitations, internal control over financial reporting is not intended to provide absolute assurance that a misstatement of our consolidated financial statements would be prevented or detected. - -Management conducted an evaluation of the effectiveness of our internal control over financial reporting based on the framework in Internal Control � Integrated Framework (2013) issued by the Committee of Sponsoring Organizations of the Treadway Commission. Based on this evaluation, management concluded that the Company�s internal control over financial reporting was effective as of June 30, 2022. There were no changes in our internal control over financial reporting during the quarter ended June 30, 2022 that have materially affected, or are reasonably likely to materially affect, our internal control over financial reporting. Deloitte & Touche LLP has audited our internal control over financial reporting as of June 30, 2022; their report is included in Item 9A. - - - -99 - - -PART II -Item 9A - - -REPORT OF INDEPENDENT REGISTERED PUBLIC ACCOUNTING FIRM To the Stockholders and the Board of Directors of Microsoft Corporation Opinion on Internal Control over Financial Reporting -We have audited the internal control over financial reporting of Microsoft Corporation and subsidiaries (the "Company") as of June 30, 2022, based on criteria established in Internal Control � Integrated Framework (2013) issued by the Committee of Sponsoring Organizations of the Treadway Commission (COSO). In our opinion, the Company maintained, in all material respects, effective internal control over financial reporting as of June 30, 2022, based on criteria established in Internal Control � Integrated Framework (2013) issued by COSO. - -We have also audited, in accordance with the standards of the Public Company Accounting Oversight Board (United States) (PCAOB), the consolidated financial statements as of and for the year ended June 30, 2022, of the Company and our report dated July 28, 2022, expressed an unqualified opinion on those financial statements. - -Basis for Opinion - -The Company's management is responsible for maintaining effective internal control over financial reporting and for its assessment of the effectiveness of internal control over financial reporting, included in the accompanying Report of Management on Internal Control over Financial Reporting. Our responsibility is to express an opinion on the Company's internal control over financial reporting based on our audit. We are a public accounting firm registered with the PCAOB and are required to be independent with respect to the Company in accordance with the U.S. federal securities laws and the applicable rules and regulations of the Securities and Exchange Commission and the PCAOB. - -We conducted our audit in accordance with the standards of the PCAOB. Those standards require that we plan and perform the audit to obtain reasonable assurance about whether effective internal control over financial reporting was maintained in all material respects. Our audit included obtaining an understanding of internal control over financial reporting, assessing the risk that a material weakness exists, testing and evaluating the design and operating effectiveness of internal control based on the assessed risk, and performing such other procedures as we considered necessary in the circumstances. We believe that our audit provides a reasonable basis for our opinion. - -Definition and Limitations of Internal Control over Financial Reporting - -A company's internal control over financial reporting is a process designed to provide reasonable assurance regarding the reliability of financial reporting and the preparation of financial statements for external purposes in accordance with generally accepted accounting principles. A company's internal control over financial reporting includes those policies and procedures that (1) pertain to the maintenance of records that, in reasonable detail, accurately and fairly reflect the transactions and dispositions of the assets of the company; (2) provide reasonable assurance that transactions are recorded as necessary to permit preparation of financial statements in accordance with generally accepted accounting principles, and that receipts and expenditures of the company are being made only in accordance with authorizations of management and directors of the company; and (3) provide reasonable assurance regarding prevention or timely detection of unauthorized acquisition, use, or disposition of the company's assets that could have a material effect on the financial statements. - -Because of its inherent limitations, internal control over financial reporting may not prevent or detect misstatements. Also, projections of any evaluation of effectiveness to future periods are subject to the risk that controls may become inadequate because of changes in conditions, or that the degree of compliance with the policies or procedures may deteriorate. - -/s/ DELOITTE & TOUCHE LLP - -Seattle, Washington -July 28, 2022 - - - - - -100 - - -PART II, III -Item 9B, 9C, 10, 11, 12, 13, 14 - -ITEM 9B. OTHER INFORMATION - -Not applicable. - -ITEM 9C. DISCLOSURE REGARDING FOREIGN JURISDICTIONS THAT PREVENT INSPECTIONS - -Not applicable. - -PART III - -ITEM 10. DIRECTORS, EXECUTIVE OFFICERS AND CORPORATE GOVERNANCE - -A list of our executive officers and biographical information appears in Part I, Item 1 of this Form 10-K. Information about our directors may be found under the caption �Our Director Nominees� in our Proxy Statement for the Annual Meeting of Shareholders to be held December 13, 2022 (the �Proxy Statement�). Information about our Audit Committee may be found under the caption �Board Committees� in the Proxy Statement. That information is incorporated herein by reference. - -We have adopted the Microsoft Finance Code of Professional Conduct (the �finance code of ethics�), a code of ethics that applies to our Chief Executive Officer, Chief Financial Officer, Chief Accounting Officer, and other finance organization employees. The finance code of ethics is publicly available on our website at https://aka.ms/FinanceCodeProfessionalConduct. If we make any substantive amendments to the finance code of ethics or grant any waiver, including any implicit waiver, from a provision of the code to our Chief Executive Officer, Chief Financial Officer, or Chief Accounting Officer, we will disclose the nature of the amendment or waiver on that website or in a report on Form 8-K. - -ITEM 11. EXECUTIVE COMPENSATION - -The information in the Proxy Statement set forth under the captions �Director Compensation,� �Named Executive Officer Compensation,� �Compensation Committee Report,� and, if required, �Compensation Committee Interlocks and Insider Participation,� is incorporated herein by reference. - -ITEM 12. SECURITY OWNERSHIP OF CERTAIN BENEFICIAL OWNERS AND MANAGEMENT AND RELATED STOCKHOLDER MATTERS - -The information in the Proxy Statement set forth under the captions �Stock Ownership Information,� �Principal Shareholders� and �Equity Compensation Plan Information� is incorporated herein by reference. - -ITEM 13. CERTAIN RELATIONSHIPS AND RELATED TRANSACTIONS, AND DIRECTOR INDEPENDENCE - -The information set forth in the Proxy Statement under the captions �Director Independence Guidelines� and �Certain Relationships and Related Transactions� is incorporated herein by reference. - -ITEM 14. PRINCIPAL ACCOUNTANT FEES AND SERVICES - -Information concerning fees and services provided by our principal accountant, Deloitte & Touche LLP (PCAOB ID No. 34), appears in the Proxy Statement under the headings �Fees Billed by Deloitte & Touche� and �Policy on Audit Committee Pre-Approval of Audit and Permissible Non-Audit Services of Independent Auditor� and is incorporated herein by reference. - -101 - - -PART IV -Item 15 - - -PART IV - -ITEM 15. EXHIBIT AND FINANCIAL STATEMENT SCHEDULES - -(a) Financial Statements and Schedules - -The financial statements are set forth under Part II, Item 8 of this Form 10-K, as indexed below. Financial statement schedules have been omitted since they either are not required, not applicable, or the information is otherwise included. - -Index to Financial Statements -Page -Income Statements - -57 -Comprehensive Income Statements -58 -Balance Sheets - -59 - - - - - - -Cash Flows Statements -60 - - - - - -Stockholders� Equity Statements - -61 -Notes to Financial Statements -62 - - - -Report of Independent Registered Public Accounting Firm -96 - -(b) Exhibit Listing - -Exhibit - - - - - - - - - - - - - - - -Filed - -Incorporated by Reference - - - - - - - - - - - - - - - - - - - - -Period - - - -Number -Exhibit Description -Herewith -Form -Ending -Exhibit -Filing Date - -3.1 -Amended and Restated Articles of Incorporation of - - -8-K - -3.1 -12/1/16 - - -Microsoft Corporation - - - - - - - - - - - - - - - - - - - - - - - - - - - - -3.2 -Bylaws of Microsoft Corporation - -8-K - -3.2 -6/14/17 - - - - - - - - - - - - - - - - - - - - - - - -4.1 -Indenture, dated as of May 18, 2009, between - - -S-3ASR - -4.1 -10/29/15 - - -Microsoft Corporation and The Bank of New York - - - - - - - -Mellon Trust Company, N.A., as Trustee (�Base - - - - - - - - -Indenture�) - - - - - - - - -4.2 -Form of First Supplemental Indenture for 2.95% - -8-K - -4.2 -5/15/09 - - -Notes due 2014, 4.20% Notes due 2019, and - - - - - - - - - - - - - - - - - - - - - - - - -5.20% Notes due 2039, dated as of May 18, 2009, - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee, - - - - - - - -to the Base Indenture - - - - - - - - - - - - - - - - - - - - -4.5 -Form of Second Supplemental Indenture for - -8-K - -4.2 -9/27/10 - - - - - - - - - - - - - - -0.875% Notes due 2013, 1.625% Notes due 2015, - - - - - - - - -3.00% Notes due 2020, and 4.50% Notes due - - - - - - - - - - - - - - - - - -2040, dated as of September 27, 2010, between - - - - - - - - -Microsoft Corporation and The Bank of New York -Mellon Trust Company, N.A., as Trustee, to the -Indenture, dated as of May 18, 2009, between -Microsoft Corporation and The Bank of New York - -Mellon Trust Company, N.A., as Trustee - -102 - - -PART IV -Item 15 - -Exhibit - - - - - - - - - - - - - - - - - - -Filed - -Incorporated by Reference - - - - - - - - - - - - - - - - - - - - - - - -Period - - - -Number -Exhibit Description -Herewith -Form -Ending -Exhibit -Filing Date - -4.6 -Third Supplemental Indenture for 2.500% Notes - - -8-K - -4.2 -2/8/11 - - -due 2016, 4.000% Notes due 2021, and 5.300% - - - - - - - - -Notes due 2041, dated as of February 8, 2011, - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee, - - - - - - - -to the Indenture, dated as of May 18, 2009, - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee - - - - - - - -4.7 -Fourth Supplemental Indenture for 0.875% Notes - -8-K - -4.1 -11/7/12 - - -due 2017, 2.125% Notes due 2022, and 3.500% - - - - - - - - - -Notes due 2042, dated as of November 7, 2012, - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee, - - - - - - - -to the Indenture, dated as of May 18, 2009, - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee - - - - - - - -4.8 -Fifth Supplemental Indenture for 2.625% Notes - - -8-K - -4.1 -5/1/13 - - -due 2033, dated as of May 2, 2013, between - - - - - - - - - - - - - - - - - - - - - - - - - - -Microsoft Corporation and The Bank of New York - - - - - - - -Mellon Trust Company, N.A., as Trustee, to the - - - - - - - - - -Indenture, dated as of May 18, 2009, between - - - - - - - - -Microsoft Corporation and The Bank of New York - - - - - - - -Mellon Trust Company, N.A., as Trustee - - - - - - - - - - - - - - - - - - - - - - - -4.9 -Sixth Supplemental Indenture for 1.000% Notes - -8-K - -4.2 -5/1/13 - - - - - - - - - - - - - - - - - -due 2018, 2.375% Notes due 2023, and 3.750% - - - - - - - - -Notes due 2043, dated as of May 2, 2013, - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee, - - - - - - - -to the Indenture, dated as of May 18, 2009, - - - - - - - - - - - - - - - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee - - - - - - - -4.10 -Seventh Supplemental Indenture for 2.125% Notes - -8-K - -4.1 -12/6/13 - - -due 2021 and 3.125% Notes due 2028, dated as - - - - - - - - - -of December 6, 2013, between Microsoft - - - - - - - - - - - - - - - - - -Corporation and The Bank of New York Mellon - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Trust Company, N.A., as Trustee, to the Indenture, -dated as of May 18, 2009, between Microsoft -Corporation and The Bank of New York Mellon -Trust Company, N.A., as Trustee - -103 - - -PART IV -Item 15 - -Exhibit - - - - - - - - - - - - - - - - - - - -Filed - -Incorporated by Reference - - - - - - - - - - - - - - - - - - - - - - - - -Period - - - -Number -Exhibit Description -Herewith -Form -Ending -Exhibit -Filing Date - -4.11 -Eighth Supplemental Indenture for 1.625% Notes - - -8-K - -4.2 -12/6/13 - - -due 2018, 3.625% Notes due 2023, and 4.875% - - - - - - - - -Notes due 2043, dated as of December 6, 2013, - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee, - - - - - - - -to the Indenture, dated as of May 18, 2009, - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee - - - - - - - -4.12 -Ninth Supplemental Indenture for 1.850% Notes - -8-K - -4.1 -2/12/15 - - - - - - - - - - - - - - - - - - - - - - - - -due 2020, 2.375% Notes due 2022, 2.700% Notes - - - - - - - - -due 2025, 3.500% Notes due 2035, 3.750% Notes - - - - - - - - -due 2045, and 4.000% Notes due 2055, dated as - - - - - - - -of February 12, 2015, between Microsoft - - - - - - - - - -Corporation and U.S. Bank National Association, - - - - - - - - - - - - - - - - - - - - - - - - - - - -as Trustee, to the Indenture, dated as of May 18, - - - - - - - -2009, between Microsoft Corporation and The - - - - - - - - - - - - - - - - - - - - - - - - - - - -Bank of New York Mellon Trust Company, N.A., as - - - - - - - -trustee - - - - - - - - - - - - - - - - - - - - - - - - - -4.13 -Tenth Supplemental Indenture for 1.300% Notes - - -8-K - -4.1 -11/3/15 - - -due 2018, 2.000% Notes due 2020, 2.650% Notes - - - - - - - - -due 2022, 3.125% Notes due 2025, 4.200% Notes - - - - - - - - -due 2035, 4.450% Notes due 2045, and 4.750% - - - - - - - - -Notes due 2055, dated as of November 3, 2015, - - - - - - - -between Microsoft Corporation and U.S. Bank - - - - - - - - - - - - - - - - - - - - - -National Association, as Trustee, to the Indenture, - - - - - - - - -dated as of May 18, 2009, between Microsoft - - - - - - - - -Corporation and The Bank of New York Mellon - - - - - - - - -Trust Company, N.A., as trustee - - - - - - - - - - - - - - - - -4.14 -Eleventh Supplemental Indenture for 1.100% Notes - - -8-K - -4.1 -8/5/16 - - -due 2019, 1.550% Notes due 2021, 2.000% Notes - - - - - - - - -due 2023, 2.400% Notes due 2026, 3.450% Notes - - - - - - - - -due 2036, 3.700% Notes due 2046, and -3.950% Notes due 2056, dated as of August 8, -2016, between Microsoft Corporation and U.S. -Bank, National Association, as Trustee, to the -Indenture, dated as of May 18, 2009, between -Microsoft Corporation and The Bank of New York -Mellon Trust Company, N.A., as trustee - -104 - - -PART IV -Item 15 - -Exhibit - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Filed - -Incorporated by Reference - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Period - - - -Number -Exhibit Description -Herewith -Form -Ending -Exhibit -Filing Date - -4.15 -Twelfth Supplemental Indenture for 1.850% Notes - - -8-K - -4.1 -2/3/17 - - -due 2020, 2.400% Notes due 2022, 2.875% Notes - - - - - - - - -due 2024, 3.300% Notes due 2027, 4.100% Notes - - - - - - - - -due 2037, 4.250% Notes due 2047, and 4.500% - - - - - - - - -Notes due 2057, dated as of February 6, 2017, - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - -New York Mellon Trust Company, N.A., as Trustee, - - - - - - - -to the Indenture, dated as of May 18, 2009, - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - -New York Mellon Trust Company, N.A., as trustee - - - - - - -4.16 -Thirteenth Supplemental Indenture for 2.525% - - - -8-K - -4.1 -6/1/20 - - -Notes due 2050 and 2.675% Notes due 2060, - - - - - - - -dated as of June 1, 2020, between Microsoft - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Corporation and U.S. Bank National Association, as - - - - - - - - -Trustee, to the Indenture, dated as of May 18, 2009, - - - - - - - - -between Microsoft Corporation and The Bank of - - - - - - - - -New York Mellon Trust Company, N.A., as trustee - - - - - - -4.17 -Fourteenth Supplemental Indenture for 2.921% - - - -8-K - -4.1 -3/17/21 - - -Notes due 2052 and 3.041% Notes due 2062, - - - - - - - - -dated as of March 17, 2021, between Microsoft - - - - - - - - -Corporation and The Bank of New York Mellon - - - - - - - - -Trust Company, N.A., as Trustee, to the Indenture, - - - - - - - - -dated as of May 18, 2009, between Microsoft - - - - - - - - -Corporation and The Bank of New York Mellon - - - - - - - - -Trust Company, N.A., as trustee - - - - - - - -4.18 -Description of Securities - -10-K -6/30/19 -4.16 -8/1/19 - - - - - - - - - - - - - - - - - - - - - -10.1* -Microsoft Corporation 2001 Stock Plan - - -10-Q -9/30/16 -10.1 -10/20/16 - -10.4* -Microsoft Corporation Employee Stock Purchase - -10-K -6/30/12 -10.4 -7/26/12 - - -Plan - - - - - - - - -10.5* -Microsoft Corporation Deferred Compensation - -10-K -6/30/18 -10.5 -8/3/18 - - -Plan - - - - - - - - -10.6* -Microsoft Corporation 2017 Stock Plan - -DEF14A - -Annex C -10/16/17 - - - - - - - - - - - - - - - - -10.7* -Form of Stock Award Agreement Under the Microsoft - - -10-Q -3/31/2018 -10.26 -4/26/18 - - -Corporation 2017 Stock Plan - - - - - - - -10.8* -Form of Performance Stock Award Agreement - - -10-Q -3/31/2018 -10.27 -4/26/18 - - -Under the Microsoft Corporation 2017 Stock Plan - - - - - - -10.9 -Amended and Restated Officers� Indemnification - - - -10-Q -9/30/16 -10.12 -10/20/16 - - -Trust Agreement between Microsoft Corporation - - - - - - - - -and The Bank of New York Mellon Trust Company, -N.A., as trustee - -105 - - -PART IV -Item 15 - -Exhibit - - - - - - - - - - - - - - - - - - - - - - -Filed - -Incorporated by Reference - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Period - - - - -Number -Exhibit Description -Herewith -Form -Ending -Exhibit -Filing Date - -10.10 -Assumption of Beneficiaries� Representative - - -10-K -6/30/2020 -10.25 -7/30/2020 - - -Obligations Under Amended and Restated - - - - - - - - -Officers� Indemnification Trust Agreement - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10.11 -Form of Indemnification Agreement and Amended - - -10-K -6/30/19 -10.13 -8/1/19 - - -and Restated Directors� Indemnification Trust - - - - - - - - - -Agreement between Microsoft Corporation and - - - - - - - - - -The Bank of New York Mellon Trust Company, - - - - - - - - - -N.A., as trustee - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10.12 -Assumption of Beneficiaries� Representative - - -10-K -6/30/2020 -10.26 -7/30/2020 - - -Obligations Under Amended and Restated - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Directors� Indemnification Trust Agreement - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10.14* -Microsoft Corporation Deferred Compensation - -10-Q -12/31/17 -10.14 -1/31/18 - - -Plan for Non-Employee Directors - - - - - - - - - -10.15* -Microsoft Corporation Executive Incentive Plan - - -8-K - -10.1 -9/19/18 - -10.19* -Microsoft Corporation Executive Incentive Plan - - -10-Q -9/30/16 -10.17 -10/20/16 - -10.20* -Form of Executive Incentive Plan (Executive - -10-Q -9/30/16 -10.18 -10/20/16 - - - - - - - - - - - - - - - - - - - - - - - - - - - -Officer SAs) Stock Award Agreement under the - - - - - - - - - -Microsoft Corporation 2001 Stock Plan - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10.21* -Form of Executive Incentive Plan Performance - - -10-Q -9/30/16 -10.25 -10/20/16 - - -Stock Award Agreement under the Microsoft - - - - - - - - - -Corporation 2001 Stock Plan - - - - - - - - - - - - - - - - - - - - - - - - - - - -10.22* -Senior Executive Severance Benefit Plan - - -10-Q -9/30/16 -10.22 -10/20/16 - -10.23* -Offer Letter, dated February 3, 2014, between - -8-K - -10.1 -2/4/14 - - -Microsoft Corporation and Satya Nadella - - - - - - - - - - - - - - - - - - - - - - - - - -10.24* -Long-Term Performance Stock Award Agreement - -10-Q -12/31/14 -10.24 -1/26/15 - - - - - - - - - - - - - - - - - - - -between Microsoft Corporation and Satya Nadella - - - - - - - - -21 -Subsidiaries of Registrant - -X - - - - - - -23.1 -Consent of Independent Registered Public -X - - - - - - - -Accounting Firm - - - - - - - - - - - - - - - - - - - - - -31.1 -Certification of Chief Executive Officer Pursuant to - -X - - - - - - - -Section 302 of the Sarbanes-Oxley Act of 2002 - - - - - - - - -31.2 -Certification of Chief Financial Officer Pursuant to - -X - - - - - - - -Section 302 of the Sarbanes-Oxley Act of 2002 - - - - - - - - -32.1** -Certification of Chief Executive Officer Pursuant to - -X - - - - - - - -Section 906 of the Sarbanes-Oxley Act of 2002 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -106 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -PART IV -Item 15 - -Exhibit - - - -Filed - -Incorporated by Reference - - - - - - - - -Period - - - -Number -Exhibit Description -Herewith -Form -Ending -Exhibit -Filing Date - -32.2** -Certification of Chief Financial Officer Pursuant to - -X - - - - - - -Section 906 of the Sarbanes-Oxley Act of 2002 - - - - - - - -101.INS -Inline XBRL Instance Document�the instance -X - - - - - - -document does not appear in the Interactive Data - - - - - - - -File as its XBRL tags are embedded within the - - - - - - - -Inline XBRL document - - - - - - -101.SCH -Inline XBRL Taxonomy Extension Schema -X - - - - - -101.CAL -Inline XBRL Taxonomy Extension Calculation -X - - - - - - -Linkbase - - - - - - -101.DEF -Inline XBRL Taxonomy Extension Definition -X - - - - - - -Linkbase - - - - - - -101.LAB -Inline XBRL Taxonomy Extension Label Linkbase -X - - - - - -101.PRE -Inline XBRL Taxonomy Extension Presentation -X - - - - - - -Linkbase - - - - - - -104 -Cover page formatted as Inline XBRL and -X - - - - - - -contained in Exhibit 101 - - - - - - - -* Indicates a management contract or compensatory plan or arrangement. - -**Furnished, not filed. - - - -107 - - -PART IV -Item 16 - -ITEM 16. FORM 10-K SUMMARY - -None. - - - -108 - - -SIGNATURES - -Pursuant to the requirements of Section 13 or 15(d) of the Securities Exchange Act of 1934, the Registrant has duly caused this report to be signed on its behalf by the undersigned; thereunto duly authorized, in the City of Redmond, State of Washington, on July 28, 2022. - -MICROSOFT CORPORATION - -/s/ ALICE L. JOLLA - -Alice L. Jolla -Corporate Vice President and Chief Accounting Officer (Principal -Accounting Officer) - - -109 - - -Pursuant to the requirements of the Securities Exchange Act of 1934, this report has been signed below by the following persons on behalf of Registrant and in the capacities indicated on July 28, 2022. - -Signature - -Title - - - -/s/ SATYA NADELLA - -Chairman and Chief Executive Officer (Principal Executive Officer) -Satya Nadella - - -/s/ REID HOFFMAN - -Director -Reid Hoffman - - -/s/ HUGH F. JOHNSTON - -Director -Hugh F. Johnston - - -/s/ TERI L. LIST - -Director -Teri L. List - - -/s/ SANDRA E. PETERSON - -Director -Sandra E. Peterson - - -/s/ PENNY S. PRITZKER - -Director -Penny S. Pritzker - - -/s/ CARLOS A. RODRIGUEZ - -Director -Carlos A. Rodriguez - - -/s/ CHARLES W. SCHARF - -Director -Charles W. Scharf - - -/s/ JOHN W. STANTON - -Director -John W. Stanton - - -/s/ JOHN W. THOMPSON - -Lead Independent Director -John W. Thompson - - -/s/ EMMA N. WALMSLEY - -Director -Emma N. Walmsley - - -/s/ PADMASREE WARRIOR - -Director -Padmasree Warrior - - -/s/ AMY E. HOOD - -Executive Vice President and Chief Financial Officer -Amy E. Hood - -(Principal Financial Officer) -/s/ ALICE L. JOLLA - -Corporate Vice President and Chief Accounting Officer (Principal -Alice L. Jolla - -Accounting Officer) - -110 - -Exhibit 21 - -SUBSIDIARIES OF REGISTRANT - -The following is a list of subsidiaries of Microsoft Corporation as of June 30, 2022, omitting subsidiaries which, considered in the aggregate, would not constitute a significant subsidiary. - -Name -Where Incorporated -Microsoft Ireland Research -Ireland -Microsoft Global Finance -Ireland -Microsoft Ireland Operations Limited -Ireland -Microsoft Online, Inc. -United States -LinkedIn Corporation -United States -LinkedIn Ireland Unlimited Company -Ireland -Nuance Communications, Inc. -United States - -Exhibit 23.1 - -CONSENT OF INDEPENDENT REGISTERED PUBLIC ACCOUNTING FIRM - -We consent to the incorporation by reference in Registration Statement Nos. 333-109185, 333-118764, 333-52852, 333-132100, 333-161516, 333-75243, 333-185757, and 333-221833 on Form S-8 and Registration Statement Nos. 333-240227 and 333-261590 on Form S-3 of our reports dated July 28, 2022, relating to the financial statements of Microsoft Corporation, and the effectiveness of Microsoft Corporation�s internal control over financial reporting appearing in this Annual Report on Form 10-K of Microsoft Corporation for the year ended June 30, 2022. - -/s/ DELOITTE & TOUCHE LLP - -Seattle, Washington -July 28, 2022 - -Exhibit 31.1 - -CERTIFICATION - -I, Satya Nadella, certify that: - -1. I have reviewed this annual report on Form 10-K of Microsoft Corporation; - -2. Based on my knowledge, this report does not contain any untrue statement of a material fact or omit to state a material fact necessary to make the statements made, in light of the circumstances under which such statements were made, not misleading with respect to the period covered by this report; - -3. Based on my knowledge, the financial statements, and other financial information included in this report, fairly present in all material respects the financial condition, results of operations and cash flows of the registrant as of, and for, the periods presented in this report; - -4. The registrant�s other certifying officer and I are responsible for establishing and maintaining disclosure controls and procedures (as defined in Exchange Act Rules 13a-15(e) and 15d-15(e)) and internal control over financial reporting (as defined in Exchange Act Rules 13a-15(f) and 15d-15(f)) for the registrant and have: - -a) Designed such disclosure controls and procedures, or caused such disclosure controls and procedures to be designed under our supervision, to ensure that material information relating to the registrant, including its consolidated subsidiaries, is made known to us by others within those entities, particularly during the period in which this report is being prepared; - -b) Designed such internal control over financial reporting, or caused such internal control over financial reporting to be designed under our supervision, to provide reasonable assurance regarding the reliability of financial reporting and the preparation of financial statements for external purposes in accordance with generally accepted accounting principles; - -c) Evaluated the effectiveness of the registrant�s disclosure controls and procedures and presented in this report our conclusions about the effectiveness of the disclosure controls and procedures, as of the end of the period covered by this report based on such evaluation; and - -d) Disclosed in this report any change in the registrant�s internal control over financial reporting that occurred during the registrant�s most recent fiscal quarter (the registrant�s fourth fiscal quarter in the case of an annual report) that has materially affected, or is reasonably likely to materially affect, the registrant�s internal control over financial reporting; and - -5. The registrant�s other certifying officer and I have disclosed, based on our most recent evaluation of internal control over financial reporting, to the registrant�s auditors and the audit committee of registrant�s Board of Directors (or persons performing the equivalent functions): - -a) All significant deficiencies and material weaknesses in the design or operation of internal control over financial reporting which are reasonably likely to adversely affect the registrant�s ability to record, process, summarize and report financial information; and - -b) Any fraud, whether or not material, that involves management or other employees who have a significant role in the registrant�s internal control over financial reporting. - -/s/ SATYA NADELLA - -Satya Nadella -Chief Executive Officer - -July 28, 2022 - -Exhibit 31.2 - -CERTIFICATION - -I, Amy E. Hood, certify that: - -1. I have reviewed this annual report on Form 10-K of Microsoft Corporation; - -2. Based on my knowledge, this report does not contain any untrue statement of a material fact or omit to state a material fact necessary to make the statements made, in light of the circumstances under which such statements were made, not misleading with respect to the period covered by this report; - -3. Based on my knowledge, the financial statements, and other financial information included in this report, fairly present in all material respects the financial condition, results of operations and cash flows of the registrant as of, and for, the periods presented in this report; - -4. The registrant�s other certifying officer and I are responsible for establishing and maintaining disclosure controls and procedures (as defined in Exchange Act Rules 13a-15(e) and 15d-15(e)) and internal control over financial reporting (as defined in Exchange Act Rules 13a-15(f) and 15d-15(f)) for the registrant and have: - -a) Designed such disclosure controls and procedures, or caused such disclosure controls and procedures to be designed under our supervision, to ensure that material information relating to the registrant, including its consolidated subsidiaries, is made known to us by others within those entities, particularly during the period in which this report is being prepared; - -b) Designed such internal control over financial reporting, or caused such internal control over financial reporting to be designed under our supervision, to provide reasonable assurance regarding the reliability of financial reporting and the preparation of financial statements for external purposes in accordance with generally accepted accounting principles; - -c) Evaluated the effectiveness of the registrant�s disclosure controls and procedures and presented in this report our conclusions about the effectiveness of the disclosure controls and procedures, as of the end of the period covered by this report based on such evaluation; and - -d) Disclosed in this report any change in the registrant�s internal control over financial reporting that occurred during the registrant�s most recent fiscal quarter (the registrant�s fourth fiscal quarter in the case of an annual report) that has materially affected, or is reasonably likely to materially affect, the registrant�s internal control over financial reporting; and - -5. The registrant�s other certifying officer and I have disclosed, based on our most recent evaluation of internal control over financial reporting, to the registrant�s auditors and the audit committee of registrant�s Board of Directors (or persons performing the equivalent functions): - -a) All significant deficiencies and material weaknesses in the design or operation of internal control over financial reporting which are reasonably likely to adversely affect the registrant�s ability to record, process, summarize and report financial information; and - -b) Any fraud, whether or not material, that involves management or other employees who have a significant role in the registrant�s internal control over financial reporting. - - -/s/ AMY E. HOOD - -Amy E. Hood -Executive Vice President and -Chief Financial Officer - -July 28, 2022 - -Exhibit 32.1 - -CERTIFICATION PURSUANT TO - -SECTION 906 OF THE SARBANES-OXLEY ACT OF 2002 -(18 U.S.C. SECTION 1350) - -In connection with the Annual Report of Microsoft Corporation, a Washington corporation (the �Company�), on Form 10-K for the year ended June 30, 2022, as filed with the Securities and Exchange Commission (the �Report�), Satya Nadella, Chief Executive Officer of the Company, does hereby certify, pursuant to � 906 of the Sarbanes-Oxley Act of 2002 (18 U.S.C. � 1350), that to his knowledge: - -(1) The Report fully complies with the requirements of section 13(a) or 15(d) of the Securities Exchange Act of 1934; and - -(2) The information contained in the Report fairly presents, in all material respects, the financial condition and results of operations of the Company. - -/s/ SATYA NADELLA - -Satya Nadella -Chief Executive Officer - -July 28, 2022 - - -Exhibit 32.2 - -CERTIFICATION PURSUANT TO - -SECTION 906 OF THE SARBANES-OXLEY ACT OF 2002 -(18 U.S.C. SECTION 1350) - -In connection with the Annual Report of Microsoft Corporation, a Washington corporation (the �Company�), on Form 10-K for the year ended June 30, 2022, as filed with the Securities and Exchange Commission (the �Report�), Amy E. Hood, Chief Financial Officer of the Company, does hereby certify, pursuant to � 906 of the Sarbanes-Oxley Act of 2002 (18 U.S.C. � 1350), that to her knowledge: - -(1) The Report fully complies with the requirements of section 13(a) or 15(d) of the Securities Exchange Act of 1934; and - -(2) The information contained in the Report fairly presents, in all material respects, the financial condition and results of operations of the Company. - -/s/ AMY E. HOOD - -Amy E. Hood -Executive Vice President and -Chief Financial Officer - -July 28, 2022 diff --git a/samples/apps/copilot-chat-app/scripts/Configure.ps1 b/samples/apps/copilot-chat-app/scripts/Configure.ps1 deleted file mode 100644 index 622bb1927bda..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Configure.ps1 +++ /dev/null @@ -1,136 +0,0 @@ -<# -.SYNOPSIS -Configure user secrets, appsettings.Development.json, and .env for Copilot Chat. - -.PARAMETER OpenAI -Switch to configure for OpenAI. - -.PARAMETER AzureOpenAI -Switch to configure for Azure OpenAI. - -.PARAMETER Endpoint -Set when using Azure OpenAI. - -.PARAMETER ApiKey -The API key for the AI service. - -.PARAMETER CompletionModel -The chat completion model to use (e.g., gpt-3.5-turbo or gpt-4). - -.PARAMETER EmbeddingModel -The embedding model to use (e.g., text-embedding-ada-002). - -.PARAMETER PlannerModel -The chat completion model to use for planning (e.g., gpt-3.5-turbo or gpt-4). - -.PARAMETER ClientID -The client (application) ID associated with your AAD app registration. - -.PARAMETER Tenant -The tenant (directory) associated with your AAD app registration. -Defaults to 'common'. -See https://learn.microsoft.com/en-us/azure/active-directory/develop/msal-client-application-configuration#authority. -#> - -param( - [Parameter(ParameterSetName='OpenAI',Mandatory=$false)] - [switch]$OpenAI, - - [Parameter(ParameterSetName='AzureOpenAI',Mandatory=$false)] - [switch]$AzureOpenAI, - - [Parameter(ParameterSetName='AzureOpenAI',Mandatory=$true)] - [string]$Endpoint, - - [Parameter(Mandatory=$true)] - [string]$ApiKey, - - [Parameter(Mandatory=$false)] - [string]$CompletionModel = "gpt-3.5-turbo", - - [Parameter(Mandatory=$false)] - [string]$EmbeddingModel = "text-embedding-ada-002", - - [Parameter(Mandatory=$false)] - [string]$PlannerModel = "gpt-3.5-turbo", - - [Parameter(Mandatory = $true)] - [string] $ClientId, - - [Parameter(Mandatory = $false)] - [string] $Tenant = 'common' -) - -Write-Host "#########################" -Write-Host "# Backend configuration #" -Write-Host "#########################" - -# Install dev certificate -if ($IsLinux) -{ - dotnet dev-certs https - if ($LASTEXITCODE -ne 0) { exit(1) } -} -else # Windows/MacOS -{ - dotnet dev-certs https --trust - if ($LASTEXITCODE -ne 0) { exit(1) } -} - -if ($OpenAI) -{ - $aiServiceType = "OpenAI" - $Endpoint = "" -} -elseif ($AzureOpenAI) -{ - $aiServiceType = "AzureOpenAI" - - # Azure OpenAI has a different model name for gpt-3.5-turbo (no decimal). - $CompletionModel = $CompletionModel.Replace("3.5", "35") - $EmbeddingModel = $EmbeddingModel.Replace("3.5", "35") - $PlannerModel = $PlannerModel.Replace("3.5", "35") -} -else { - Write-Error "Please specify either -OpenAI or -AzureOpenAI" - exit(1) -} - -$appsettingsOverrides = @{ AIService = @{ Type = $aiServiceType; Endpoint = $Endpoint; Models = @{ Completion = $CompletionModel; Embedding = $EmbeddingModel; Planner = $PlannerModel } } } - -$webapiProjectPath = Join-Path "$PSScriptRoot" '../webapi' -$appsettingsOverridesFilePath = Join-Path $webapiProjectPath 'appsettings.Development.json' - -Write-Host "Setting 'AIService:Key' user secret for $aiServiceType..." -dotnet user-secrets set --project $webapiProjectPath AIService:Key $ApiKey -if ($LASTEXITCODE -ne 0) { exit(1) } - -Write-Host "Setting up 'appsettings.Development.json' for $aiServiceType..." -ConvertTo-Json $appsettingsOverrides | Out-File -Encoding utf8 $appsettingsOverridesFilePath - -Write-Host "($appsettingsOverridesFilePath)" -Write-Host "========" -Get-Content $appsettingsOverridesFilePath | Write-Host -Write-Host "========" - -Write-Host "" -Write-Host "##########################" -Write-Host "# Frontend configuration #" -Write-Host "##########################" - -$envFilePath = Join-Path "$PSScriptRoot" '../webapp/.env' - -Write-Host "Setting up '.env'..." -Set-Content -Path $envFilePath -Value "REACT_APP_BACKEND_URI=https://localhost:40443/" -Add-Content -Path $envFilePath -Value "REACT_APP_AAD_AUTHORITY=https://login.microsoftonline.com/$Tenant" -Add-Content -Path $envFilePath -Value "REACT_APP_AAD_CLIENT_ID=$ClientId" -Add-Content -Path $envFilePath -Value "" -Add-Content -Path $envFilePath -Value "# Web Service API key (not required when running locally)" -Add-Content -Path $envFilePath -Value "REACT_APP_SK_API_KEY=" - -Write-Host "($envFilePath)" -Write-Host "========" -Get-Content $envFilePath | Write-Host -Write-Host "========" - -Write-Host "Done!" diff --git a/samples/apps/copilot-chat-app/scripts/Configure.sh b/samples/apps/copilot-chat-app/scripts/Configure.sh deleted file mode 100644 index 06fa0e6f9197..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Configure.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -# Configure user secrets, appsettings.Development.json, and .env for Copilot Chat. - -set -e - -# Defaults -COMPLETION_MODEL="gpt-3.5-turbo" -EMBEDDING_MODEL="text-embedding-ada-002" -PLANNER_MODEL="gpt-3.5-turbo" -TENANT_ID="common" - -# Argument parsing -POSITIONAL_ARGS=() - -while [[ $# -gt 0 ]]; do - case $1 in - --openai) - OPENAI=YES - shift # past argument - ;; - --azureopenai) - AZURE_OPENAI=YES - shift - ;; - -e|--endpoint) - ENDPOINT="$2" - shift # past argument - shift # past value - ;; - -a|--apikey) - API_KEY="$2" - shift - shift - ;; - --completion) - COMPLETION_MODEL="$2" - shift - shift - ;; - --embedding) - EMBEDDING_MODEL="$2" - shift - shift - ;; - --planner) - PLANNER_MODEL="$2" - shift - shift - ;; - -c|--clientid) - CLIENT_ID="$2" - shift - shift - ;; - -t|--tenantid) - TENANT_ID="$2" - shift - shift - ;; - -*|--*) - echo "Unknown option $1" - exit 1 - ;; - *) - POSITIONAL_ARGS+=("$1") # save positional arg - shift # past argument - ;; - esac -done - -set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters - -SCRIPT_DIRECTORY="$(dirname $0)" - -# Validate arguments -if [ -z "$API_KEY" ]; then - echo "Please specify an API key with -a or --apikey."; exit 1; -fi -if [ -z "$CLIENT_ID" ]; then - echo "Please specify a client (application) ID with -c or --clientid."; exit 1; -fi -if [ "$AZURE_OPENAI" = "YES" ] && [ -z "$ENDPOINT" ]; then - echo "When using --azureopenti, please specify an endpoint with -e or --endpoint."; exit 1; -fi - -echo "#########################" -echo "# Backend configuration #" -echo "#########################" - -# Install dev certificate -case "$OSTYPE" in - darwin*) - dotnet dev-certs https --trust - if [ $? -ne 0 ]; then `exit` 1; fi ;; - msys*) - dotnet dev-certs https --trust - if [ $? -ne 0 ]; then exit 1; fi ;; - cygwin*) - dotnet dev-certs https --trust - if [ $? -ne 0 ]; then exit 1; fi ;; - linux*) - dotnet dev-certs https - if [ $? -ne 0 ]; then exit 1; fi ;; -esac - -if [ "$OPENAI" = "YES" ]; then - AI_SERVICE_TYPE="OpenAI" -elif [ "$AZURE_OPENAI" = "YES" ]; then - # Azure OpenAI has a different model name for gpt-3.5-turbo (no decimal). - AI_SERVICE_TYPE="AzureOpenAI" - COMPLETION_MODEL="${COMPLETION_MODEL/3.5/"35"}" - EMBEDDING_MODEL="${EMBEDDING_MODEL/3.5/"35"}" - PLANNER_MODEL="${PLANNER_MODEL/3.5/"35"}" -else - echo "Please specify either --openai or --azureopenai." - exit 1 -fi - -APPSETTINGS_JSON="{ \"AIService\": { \"Type\": \"${AI_SERVICE_TYPE}\", \"Endpoint\": \"${ENDPOINT}\", \"Models\": { \"Completion\": \"${COMPLETION_MODEL}\", \"Embedding\": \"${EMBEDDING_MODEL}\", \"Planner\": \"${PLANNER_MODEL}\" } } }" -WEBAPI_PROJECT_PATH="${SCRIPT_DIRECTORY}/../webapi" -APPSETTINGS_OVERRIDES_FILEPATH="${WEBAPI_PROJECT_PATH}/appsettings.Development.json" - -echo "Setting 'AIService:Key' user secret for $AI_SERVICE_TYPE..." -dotnet user-secrets set --project $WEBAPI_PROJECT_PATH AIService:Key $API_KEY -if [ $? -ne 0 ]; then exit 1; fi - -echo "Setting up 'appsettings.Development.json' for $AI_SERVICE_TYPE..." -echo $APPSETTINGS_JSON > $APPSETTINGS_OVERRIDES_FILEPATH - -echo "($APPSETTINGS_OVERRIDES_FILEPATH)" -echo "========" -cat $APPSETTINGS_OVERRIDES_FILEPATH -echo "========" - -echo "" -echo "##########################" -echo "# Frontend configuration #" -echo "##########################" - -ENV_FILEPATH="${SCRIPT_DIRECTORY}/../webapp/.env" - -echo "Setting up '.env'..." -echo "REACT_APP_BACKEND_URI=https://localhost:40443/" > $ENV_FILEPATH -echo "REACT_APP_AAD_AUTHORITY=https://login.microsoftonline.com/$TENANT_ID" >> $ENV_FILEPATH -echo "REACT_APP_AAD_CLIENT_ID=$CLIENT_ID" >> $ENV_FILEPATH -echo "# Web Service API key (not required when running locally)" >> $ENV_FILEPATH -echo "REACT_APP_SK_API_KEY=" >> $ENV_FILEPATH - -echo "($ENV_FILEPATH)" -echo "========" -cat $ENV_FILEPATH -echo "========" - -echo "Done!" \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/scripts/Install-Requirements-UbuntuDebian.sh b/samples/apps/copilot-chat-app/scripts/Install-Requirements-UbuntuDebian.sh deleted file mode 100644 index 5a012d0292b6..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Install-Requirements-UbuntuDebian.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Installs the requirements for running Copilot Chat. - -set -e - -# Add Yarn's package repository to the system -curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - -echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list - -# Add .NET's package repository to the system -wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb -sudo dpkg -i packages-microsoft-prod.deb -rm packages-microsoft-prod.deb - -# Add NodeJS's package repository to the system -curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - - -# Install the requirements -sudo apt update; -sudo apt install yarn -y; -sudo apt install dotnet-sdk-6.0 -y; -sudo apt install nodejs -y; - -echo "" -echo "YARN $(yarn --version) installed." -echo "NODEJS $(node --version) installed." -echo "DOTNET $(dotnet --version) installed." \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/scripts/Install-Requirements.ps1 b/samples/apps/copilot-chat-app/scripts/Install-Requirements.ps1 deleted file mode 100644 index c7020ea5c423..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Install-Requirements.ps1 +++ /dev/null @@ -1,33 +0,0 @@ -<# -.SYNOPSIS -Installs the requirements for running Copilot Chat. -#> - -if ($IsLinux) -{ - Write-Host "ERROR: This script is not supported for your operating system." - exit 1; -} - -Set-ExecutionPolicy Bypass -Scope Process -Force -[System.Net.ServicePointManager]::SecurityProtocol = 3072 - -# Install chocolatey if not already installed -$ChocoInstalled = $false -if (Get-Command choco.exe -ErrorAction SilentlyContinue) { - $ChocoInstalled = $true -} -if (!$ChocoInstalled) -{ - Write-Host "Installing Chocolatey..." - Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1') - $env:PATH += ";%ALLUSERSPROFILE%\chocolatey\bin" - refreshenv -} - -# Ensure required packages are installed -$Packages = 'dotnet-6.0-sdk', 'nodejs', 'yarn' -foreach ($PackageName in $Packages) -{ - choco install $PackageName -y -} diff --git a/samples/apps/copilot-chat-app/scripts/README.md b/samples/apps/copilot-chat-app/scripts/README.md deleted file mode 100644 index de51f5bf0e97..000000000000 --- a/samples/apps/copilot-chat-app/scripts/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Copilot Chat Setup Scripts (local deployment) - -## Before You Begin -To run Copilot Chat, you will need the following: -- *Application ID* - - This is the Client ID (i.e., Application ID) associated with your Azure Active Directory (AAD) application registration, which you can find in the Azure portal. -- *Azure OpenAI or OpenAI API Key* - - This is your API key for Azure OpenAI or OpenAI - -## 1. Configure your environment -### Windows -Open a PowerShell terminal as an administrator, navigate to this directory, and run the following command: -```powershell -./Install-Requirements.ps1 -``` -> This script uses the Chocolatey package manager install .NET 6.0 SDK, latest Node.js, and Yarn package manager. - -### Ubuntu/Debian Linux -Open a bash terminal as an administrator, navigate to this directory, and run the following command: -```bash -./Install-Requirements-UbuntuDebian.ps1 -``` - -### Other Linux/MacOS -For all other operating systems, ensure NET 6.0 SDK (or newer), Node.js 14 (or newer), and Yarn classic ([v1.22.19](https://classic.yarnpkg.com/)) package manager are installed before proceeding. - -## 2. Configure CopilotChat -Configure the projects with your AI service and application registration information from above. - -**Powershell** -```powershell -./Configure.ps1 -AzureOpenAI -Endpoint {AZURE_OPENAI_ENDPOINT} -ApiKey {AZURE_OPENAI_API_KEY} -ClientId {CLIENT_ID} -``` -> For OpenAI, replace `-AzureOpenAI` with `-OpenAI` and omit `-Endpoint`. - -**Bash** -```bash -./Configure.sh --azureopenai --endpoint {AZURE_OPENAI_ENDPOINT} --apikey {AZURE_OPENAI_API_KEY} --clientid {CLIENT_ID} -``` -> For OpenAI, replace `--azureopenai` with `--openai` and omit `--endpoint`. - -> **Note:** `Configure.ps1`/`Configure.sh` scripts also have parameters for setting additional options, such as AI models and Azure Active Directory tenant IDs. - -## 3. Run Copilot Chat -The `Start` script initializes and runs the WebApp (frontend) and WebApi (backend) for Copilot Chat on your local machine. - -### PowerShell -Open a PowerShell window, navigate to this directory, and run the following command: -```powershell -./Start.ps1 -``` - -### Bash -Open a Bash window, navigate to this directory, and run the following commands: -```bash -# Ensure ./Start.sh is executable -chmod +x Start.sh -# Start CopilotChat -./Start.sh -``` -> **Note:** The first time you run this may take a few minutes for Yarn packages to install. -> **Note:** This script starts `CopilotChatWebApi.exe` as a background process. Be sure to terminate it when you are finished. - -# Troubleshooting -## 1. "A fatal error occurred. The folder [/usr/share/dotnet/host/fxr] does not exist" when running dotnet commands on Linux. -> From https://stackoverflow.com/questions/73753672/a-fatal-error-occurred-the-folder-usr-share-dotnet-host-fxr-does-not-exist - -When .NET (Core) was first released for Linux, it was not yet available in the official Ubuntu repo. So instead, many of us added the Microsoft APT repo in order to install it. Now, the packages are part of the Ubuntu repo, and they are conflicting with the Microsoft packages. This error is a result of mixed packages. -```bash -# Remove all existing packages to get to a clean state: -sudo apt remove --assume-yes dotnet*; -sudo apt remove --assume-yes aspnetcore*; -sudo apt remove --assume-yes netstandard*; -# Set the Microsoft package provider priority -echo -e "Package: *\nPin: origin \"packages.microsoft.com\"\nPin-Priority: 1001" | sudo tee /etc/apt/preferences.d/99microsoft-dotnet.pref; -# Update and install dotnet -sudo apt update; -sudo apt install --assume-yes dotnet-sdk-6.0; -``` \ No newline at end of file diff --git a/samples/apps/copilot-chat-app/scripts/Start-Backend.ps1 b/samples/apps/copilot-chat-app/scripts/Start-Backend.ps1 deleted file mode 100644 index 2a459d6085df..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Start-Backend.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -<# -.SYNOPSIS -Builds and runs the Copilot Chat backend. -#> - -Join-Path "$PSScriptRoot" '../webapi' | Set-Location -dotnet build -dotnet run diff --git a/samples/apps/copilot-chat-app/scripts/Start-Backend.sh b/samples/apps/copilot-chat-app/scripts/Start-Backend.sh deleted file mode 100644 index dce319620d29..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Start-Backend.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Builds and runs the Copilot Chat backend. - -set -e - -ScriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$ScriptDir/../WebApi" - -# Build and run the backend API server -dotnet build && dotnet run diff --git a/samples/apps/copilot-chat-app/scripts/Start-Frontend.ps1 b/samples/apps/copilot-chat-app/scripts/Start-Frontend.ps1 deleted file mode 100644 index 13e870c91f2c..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Start-Frontend.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -<# -.SYNOPSIS -Builds and runs the Copilot Chat frontend. -#> - -Join-Path "$PSScriptRoot" '../webapp' | Set-Location -yarn install -yarn start diff --git a/samples/apps/copilot-chat-app/scripts/Start-Frontend.sh b/samples/apps/copilot-chat-app/scripts/Start-Frontend.sh deleted file mode 100644 index 152b544bfafd..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Start-Frontend.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Initializes and runs the Copilot Chat frontend. - -set -e - -ScriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$ScriptDir/../webapp" - -# Build and run the frontend application -yarn install && yarn start diff --git a/samples/apps/copilot-chat-app/scripts/Start.ps1 b/samples/apps/copilot-chat-app/scripts/Start.ps1 deleted file mode 100644 index e5f046f29d39..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Start.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -<# -.SYNOPSIS -Builds and runs both the backend and frontend for Copilot Chat. -#> - -$BackendScript = Join-Path "$PSScriptRoot" 'Start-Backend.ps1' -$FrontendScript = Join-Path "$PSScriptRoot" 'Start-Frontend.ps1' - -# Start backend (in new PS process) -Start-Process pwsh -ArgumentList "-noexit", "-command $BackendScript" - -# Start frontend (in current PS process) -& $FrontendScript diff --git a/samples/apps/copilot-chat-app/scripts/Start.sh b/samples/apps/copilot-chat-app/scripts/Start.sh deleted file mode 100644 index edf88aff4330..000000000000 --- a/samples/apps/copilot-chat-app/scripts/Start.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Initializes and runs both the backend and frontend for Copilot Chat. - -set -e - -ScriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$ScriptDir" - -# Start backend (in background) -./Start-Backend.sh & - -# Start frontend -./Start-Frontend.sh diff --git a/samples/apps/copilot-chat-app/webapi/Auth/ApiKeyAuthenticationHandler.cs b/samples/apps/copilot-chat-app/webapi/Auth/ApiKeyAuthenticationHandler.cs deleted file mode 100644 index 550f862b1b0e..000000000000 --- a/samples/apps/copilot-chat-app/webapi/Auth/ApiKeyAuthenticationHandler.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Security.Claims; -using System.Text.Encodings.Web; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authentication; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Primitives; - -namespace SemanticKernel.Service.Auth; - -/// -/// Class implementing API key authentication. -/// -public class ApiKeyAuthenticationHandler : AuthenticationHandler -{ - public const string AuthenticationScheme = "ApiKey"; - public const string ApiKeyHeaderName = "x-sk-api-key"; - - /// - /// Constructor - /// - public ApiKeyAuthenticationHandler( - IOptionsMonitor options, - ILoggerFactory loggerFactory, - UrlEncoder encoder, - ISystemClock clock) : base(options, loggerFactory, encoder, clock) - { - } - - protected override Task HandleAuthenticateAsync() - { - this.Logger.LogInformation("Checking API key"); - - if (string.IsNullOrWhiteSpace(this.Options.ApiKey)) - { - const string ErrorMessage = "API key not configured on server"; - - this.Logger.LogError(ErrorMessage); - - return Task.FromResult(AuthenticateResult.Fail(ErrorMessage)); - } - - if (!this.Request.Headers.TryGetValue(ApiKeyHeaderName, out StringValues apiKeyFromHeader)) - { - const string WarningMessage = "No API key provided"; - - this.Logger.LogWarning(WarningMessage); - - return Task.FromResult(AuthenticateResult.Fail(WarningMessage)); - } - - if (!string.Equals(apiKeyFromHeader, this.Options.ApiKey, StringComparison.Ordinal)) - { - const string WarningMessage = "Incorrect API key"; - - this.Logger.LogWarning(WarningMessage); - - return Task.FromResult(AuthenticateResult.Fail(WarningMessage)); - } - - var principal = new ClaimsPrincipal(new ClaimsIdentity(AuthenticationScheme)); - var ticket = new AuthenticationTicket(principal, this.Scheme.Name); - - this.Logger.LogInformation("Request authorized by API key"); - - return Task.FromResult(AuthenticateResult.Success(ticket)); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/Auth/ApiKeyAuthenticationSchemeOptions.cs b/samples/apps/copilot-chat-app/webapi/Auth/ApiKeyAuthenticationSchemeOptions.cs deleted file mode 100644 index 33e8943f2a0a..000000000000 --- a/samples/apps/copilot-chat-app/webapi/Auth/ApiKeyAuthenticationSchemeOptions.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.AspNetCore.Authentication; - -namespace SemanticKernel.Service.Auth; - -/// -/// Options for API key authentication. -/// -public class ApiKeyAuthenticationSchemeOptions : AuthenticationSchemeOptions -{ - /// - /// The API key against which to authenticate. - /// - public string? ApiKey { get; set; } -} diff --git a/samples/apps/copilot-chat-app/webapi/Auth/PassThroughAuthenticationHandler.cs b/samples/apps/copilot-chat-app/webapi/Auth/PassThroughAuthenticationHandler.cs deleted file mode 100644 index e12320b898ec..000000000000 --- a/samples/apps/copilot-chat-app/webapi/Auth/PassThroughAuthenticationHandler.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Security.Claims; -using System.Text.Encodings.Web; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authentication; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace SemanticKernel.Service.Auth; - -/// -/// Class implementing "authentication" that lets all requests pass through. -/// -public class PassThroughAuthenticationHandler : AuthenticationHandler -{ - public const string AuthenticationScheme = "PassThrough"; - - /// - /// Constructor - /// - public PassThroughAuthenticationHandler( - IOptionsMonitor options, - ILoggerFactory loggerFactory, - UrlEncoder encoder, - ISystemClock clock) : base(options, loggerFactory, encoder, clock) - { - } - - protected override Task HandleAuthenticateAsync() - { - this.Logger.LogInformation("Allowing request to pass through"); - - var principal = new ClaimsPrincipal(new ClaimsIdentity(AuthenticationScheme)); - var ticket = new AuthenticationTicket(principal, this.Scheme.Name); - - return Task.FromResult(AuthenticateResult.Success(ticket)); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/ConfigurationExtensions.cs b/samples/apps/copilot-chat-app/webapi/ConfigurationExtensions.cs deleted file mode 100644 index 81e7bf3697a4..000000000000 --- a/samples/apps/copilot-chat-app/webapi/ConfigurationExtensions.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Reflection; -using Azure.Identity; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Hosting; - -namespace SemanticKernel.Service; - -internal static class ConfigExtensions -{ - /// - /// Build the configuration for the service. - /// - public static IHostBuilder AddConfiguration(this IHostBuilder host) - { - string? environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); - - host.ConfigureAppConfiguration((builderContext, configBuilder) => - { - configBuilder.AddJsonFile( - path: "appsettings.json", - optional: false, - reloadOnChange: true); - - configBuilder.AddJsonFile( - path: $"appsettings.{environment}.json", - optional: true, - reloadOnChange: true); - - configBuilder.AddEnvironmentVariables(); - - configBuilder.AddUserSecrets( - assembly: Assembly.GetExecutingAssembly(), - optional: true, - reloadOnChange: true); - - // For settings from Key Vault, see https://learn.microsoft.com/en-us/aspnet/core/security/key-vault-configuration?view=aspnetcore-8.0 - string? keyVaultUri = builderContext.Configuration["Service:KeyVault"]; - if (!string.IsNullOrWhiteSpace(keyVaultUri)) - { - configBuilder.AddAzureKeyVault( - new Uri(keyVaultUri), - new DefaultAzureCredential()); - - // for more information on how to use DefaultAzureCredential, see https://learn.microsoft.com/en-us/dotnet/api/azure.identity.defaultazurecredential?view=azure-dotnet - } - }); - - return host; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/BotController.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/BotController.cs deleted file mode 100644 index a9ab3ac76397..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/BotController.cs +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Memory; -using SemanticKernel.Service.CopilotChat.Extensions; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Options; -using SemanticKernel.Service.CopilotChat.Storage; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Controllers; - -[ApiController] -public class BotController : ControllerBase -{ - private readonly ILogger _logger; - private readonly IMemoryStore? _memoryStore; - private readonly ISemanticTextMemory _semanticMemory; - private readonly ChatSessionRepository _chatRepository; - private readonly ChatMessageRepository _chatMessageRepository; - private readonly ChatParticipantRepository _chatParticipantRepository; - - private readonly BotSchemaOptions _botSchemaOptions; - private readonly AIServiceOptions _embeddingOptions; - private readonly DocumentMemoryOptions _documentMemoryOptions; - - /// - /// The constructor of BotController. - /// - /// Optional memory store. - /// High level semantic memory implementations, such as Azure Cognitive Search, do not allow for providing embeddings when storing memories. - /// We wrap the memory store in an optional memory store to allow controllers to pass dependency injection validation and potentially optimize - /// for a lower-level memory implementation (e.g. Qdrant). Lower level memory implementations (i.e., IMemoryStore) allow for reusing embeddings, - /// whereas high level memory implementation (i.e., ISemanticTextMemory) assume embeddings get recalculated on every write. - /// - /// The chat session repository. - /// The chat message repository. - /// The chat participant repository. - /// The AI service options where we need the embedding settings from. - /// The bot schema options. - /// The document memory options. - /// The logger. - public BotController( - OptionalIMemoryStore optionalIMemoryStore, - ISemanticTextMemory semanticMemory, - ChatSessionRepository chatRepository, - ChatMessageRepository chatMessageRepository, - ChatParticipantRepository chatParticipantRepository, - IOptions aiServiceOptions, - IOptions botSchemaOptions, - IOptions documentMemoryOptions, - ILogger logger) - { - this._memoryStore = optionalIMemoryStore.MemoryStore; - this._logger = logger; - this._semanticMemory = semanticMemory; - this._chatRepository = chatRepository; - this._chatMessageRepository = chatMessageRepository; - this._chatParticipantRepository = chatParticipantRepository; - this._botSchemaOptions = botSchemaOptions.Value; - this._embeddingOptions = aiServiceOptions.Value; - this._documentMemoryOptions = documentMemoryOptions.Value; - } - - /// - /// Upload a bot. - /// - /// The Semantic Kernel instance. - /// The user id. - /// The bot object from the message body - /// The cancellation token. - /// The HTTP action result with new chat session object. - [Authorize] - [HttpPost] - [Route("bot/upload")] - [ProducesResponseType(StatusCodes.Status201Created)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task> UploadAsync( - [FromServices] IKernel kernel, - [FromQuery] string userId, - [FromBody] Bot bot, - CancellationToken cancellationToken) - { - // TODO: We should get userId from server context instead of from request for privacy/security reasons when support multiple users. - this._logger.LogDebug("Received call to upload a bot"); - - if (!IsBotCompatible( - externalBotSchema: bot.Schema, - externalBotEmbeddingConfig: bot.EmbeddingConfigurations, - embeddingOptions: this._embeddingOptions, - botSchemaOptions: this._botSchemaOptions)) - { - return this.BadRequest("Incompatible schema. " + - $"The supported bot schema is {this._botSchemaOptions.Name}/{this._botSchemaOptions.Version} " + - $"for the {this._embeddingOptions.Models.Embedding} model from {this._embeddingOptions.Type}. " + - $"But the uploaded file is with schema {bot.Schema.Name}/{bot.Schema.Version} " + - $"for the {bot.EmbeddingConfigurations.DeploymentOrModelId} model from {bot.EmbeddingConfigurations.AIService}."); - } - - string chatTitle = $"{bot.ChatTitle} - Clone"; - string chatId = string.Empty; - ChatSession newChat; - - // Upload chat history into chat repository and embeddings into memory. - - // 1. Create a new chat and get the chat id. - newChat = new ChatSession(chatTitle); - await this._chatRepository.CreateAsync(newChat); - await this._chatParticipantRepository.CreateAsync(new ChatParticipant(userId, newChat.Id)); - chatId = newChat.Id; - - string oldChatId = bot.ChatHistory.First().ChatId; - - // 2. Update the app's chat storage. - foreach (var message in bot.ChatHistory) - { - var chatMessage = new ChatMessage( - message.UserId, - message.UserName, - chatId, - message.Content, - message.Prompt, - ChatMessage.AuthorRoles.Participant) - { - Timestamp = message.Timestamp - }; - await this._chatMessageRepository.CreateAsync(chatMessage); - } - - // 3. Update the memory. - await this.BulkUpsertMemoryRecordsAsync(oldChatId, chatId, bot.Embeddings, cancellationToken); - - // TODO: Revert changes if any of the actions failed - - return this.CreatedAtAction( - nameof(ChatHistoryController.GetChatSessionByIdAsync), - nameof(ChatHistoryController).Replace("Controller", "", StringComparison.OrdinalIgnoreCase), - new { chatId }, - newChat); - } - - /// - /// Download a bot. - /// - /// The Semantic Kernel instance. - /// The chat id to be downloaded. - /// The serialized Bot object of the chat id. - [Authorize] - [HttpGet] - [ActionName("DownloadAsync")] - [Route("bot/download/{chatId:guid}")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task> DownloadAsync( - [FromServices] IKernel kernel, - Guid chatId) - { - this._logger.LogDebug("Received call to download a bot"); - var memory = await this.CreateBotAsync(kernel: kernel, chatId: chatId); - - return JsonSerializer.Serialize(memory); - } - - /// - /// Check if an external bot file is compatible with the application. - /// - /// - /// If the embeddings are not generated from the same model, the bot file is not compatible. - /// - /// The external bot schema. - /// The external bot embedding configuration. - /// The embedding options. - /// The bot schema options. - /// True if the bot file is compatible with the app; otherwise false. - private static bool IsBotCompatible( - BotSchemaOptions externalBotSchema, - BotEmbeddingConfig externalBotEmbeddingConfig, - AIServiceOptions embeddingOptions, - BotSchemaOptions botSchemaOptions) - { - // The app can define what schema/version it supports before the community comes out with an open schema. - return externalBotSchema.Name.Equals(botSchemaOptions.Name, StringComparison.OrdinalIgnoreCase) - && externalBotSchema.Version == botSchemaOptions.Version - && externalBotEmbeddingConfig.AIService == embeddingOptions.Type - && externalBotEmbeddingConfig.DeploymentOrModelId.Equals(embeddingOptions.Models.Embedding, StringComparison.OrdinalIgnoreCase); - } - - /// - /// Get memory from memory store and append the memory records to a given list. - /// It will update the memory collection name in the new list if the newCollectionName is provided. - /// - /// The Semantic Kernel instance. - /// The current collection name. Used to query the memory storage. - /// The embeddings list where we will append the fetched memory records. - /// - /// The new collection name when appends to the embeddings list. Will use the old collection name if not provided. - /// - private static async Task GetMemoryRecordsAndAppendToEmbeddingsAsync( - IKernel kernel, - string collectionName, - List>> embeddings, - string newCollectionName = "") - { - List collectionMemoryRecords = await kernel.Memory.SearchAsync( - collectionName, - "abc", // dummy query since we don't care about relevance. An empty string will cause exception. - limit: 999999999, // temp solution to get as much as record as a workaround. - minRelevanceScore: -1, // no relevance required since the collection only has one entry - withEmbeddings: true, - cancellationToken: default - ).ToListAsync(); - - embeddings.Add(new KeyValuePair>( - string.IsNullOrEmpty(newCollectionName) ? collectionName : newCollectionName, - collectionMemoryRecords)); - } - - /// - /// Prepare the bot information of a given chat. - /// - /// The semantic kernel object. - /// The chat id of the bot - /// A Bot object that represents the chat session. - private async Task CreateBotAsync(IKernel kernel, Guid chatId) - { - var chatIdString = chatId.ToString(); - var bot = new Bot - { - // get the bot schema version - Schema = this._botSchemaOptions, - - // get the embedding configuration - EmbeddingConfigurations = new BotEmbeddingConfig - { - AIService = this._embeddingOptions.Type, - DeploymentOrModelId = this._embeddingOptions.Models.Embedding - } - }; - - // get the chat title - ChatSession chat = await this._chatRepository.FindByIdAsync(chatIdString); - bot.ChatTitle = chat.Title; - - // get the chat history - bot.ChatHistory = await this.GetAllChatMessagesAsync(chatIdString); - - // get the memory collections associated with this chat - // TODO: filtering memory collections by name might be fragile. - var chatCollections = (await kernel.Memory.GetCollectionsAsync()) - .Where(collection => collection.StartsWith(chatIdString, StringComparison.OrdinalIgnoreCase)); - - foreach (var collection in chatCollections) - { - await GetMemoryRecordsAndAppendToEmbeddingsAsync(kernel: kernel, collectionName: collection, embeddings: bot.Embeddings); - } - - // get the document memory collection names (global scope) - await GetMemoryRecordsAndAppendToEmbeddingsAsync( - kernel: kernel, - collectionName: this._documentMemoryOptions.GlobalDocumentCollectionName, - embeddings: bot.DocumentEmbeddings); - - // get the document memory collection names (user scope) - await GetMemoryRecordsAndAppendToEmbeddingsAsync( - kernel: kernel, - collectionName: this._documentMemoryOptions.ChatDocumentCollectionNamePrefix + chatIdString, - embeddings: bot.DocumentEmbeddings); - - return bot; - } - - /// - /// Get chat messages of a given chat id. - /// - /// The chat id - /// The list of chat messages in descending order of the timestamp - private async Task> GetAllChatMessagesAsync(string chatId) - { - // TODO: We might want to set limitation on the number of messages that are pulled from the storage. - return (await this._chatMessageRepository.FindByChatIdAsync(chatId)) - .OrderByDescending(m => m.Timestamp).ToList(); - } - - /// - /// Bulk upsert memory records into memory store. - /// - /// The original chat id of the memory records. - /// The new chat id that will replace the original chat id. - /// The list of embeddings of the chat id. - /// The function doesn't return anything. - private async Task BulkUpsertMemoryRecordsAsync(string oldChatId, string chatId, List>> embeddings, CancellationToken cancellationToken = default) - { - foreach (var collection in embeddings) - { - foreach (var record in collection.Value) - { - if (record != null && record.Embedding != null) - { - var newCollectionName = collection.Key.Replace(oldChatId, chatId, StringComparison.OrdinalIgnoreCase); - - if (this._memoryStore == null) - { - await this._semanticMemory.SaveInformationAsync( - collection: newCollectionName, - text: record.Metadata.Text, - id: record.Metadata.Id, - cancellationToken: cancellationToken); - } - else - { - MemoryRecord data = MemoryRecord.LocalRecord( - id: record.Metadata.Id, - text: record.Metadata.Text, - embedding: record.Embedding.Value, - description: null, - additionalMetadata: null); - - if (!(await this._memoryStore.DoesCollectionExistAsync(newCollectionName, default))) - { - await this._memoryStore.CreateCollectionAsync(newCollectionName, default); - } - - await this._memoryStore.UpsertAsync(newCollectionName, data, default); - } - } - } - } - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs deleted file mode 100644 index ae4d795f2e21..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatController.cs +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; -using Microsoft.Graph; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Skills.MsGraph; -using Microsoft.SemanticKernel.Skills.MsGraph.Connectors; -using Microsoft.SemanticKernel.Skills.MsGraph.Connectors.Client; -using Microsoft.SemanticKernel.Skills.OpenAPI.Authentication; -using Microsoft.SemanticKernel.Skills.OpenAPI.Extensions; -using SemanticKernel.Service.CopilotChat.Hubs; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Skills.ChatSkills; -using SemanticKernel.Service.Diagnostics; -using SemanticKernel.Service.Models; - -namespace SemanticKernel.Service.CopilotChat.Controllers; - -/// -/// Controller responsible for handling chat messages and responses. -/// -[ApiController] -public class ChatController : ControllerBase, IDisposable -{ - private readonly ILogger _logger; - private readonly List _disposables; - private readonly ITelemetryService _telemetryService; - private const string ChatSkillName = "ChatSkill"; - private const string ChatFunctionName = "Chat"; - private const string ReceiveResponseClientCall = "ReceiveResponse"; - private const string GeneratingResponseClientCall = "ReceiveBotTypingState"; - - public ChatController(ILogger logger, ITelemetryService telemetryService) - { - this._logger = logger; - this._telemetryService = telemetryService; - this._disposables = new List(); - } - - /// - /// Invokes the chat skill to get a response from the bot. - /// - /// Semantic kernel obtained through dependency injection. - /// Message Hub that performs the real time relay service. - /// Planner to use to create function sequences. - /// Prompt along with its parameters. - /// Authentication headers to connect to OpenAPI Skills. - /// Results containing the response from the model. - [Authorize] - [Route("chat")] - [HttpPost] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task ChatAsync( - [FromServices] IKernel kernel, - [FromServices] IHubContext messageRelayHubContext, - [FromServices] CopilotChatPlanner planner, - [FromBody] Ask ask, - [FromHeader] OpenApiSkillsAuthHeaders openApiSkillsAuthHeaders) - { - this._logger.LogDebug("Chat request received."); - - // Put ask's variables in the context we will use. - var contextVariables = new ContextVariables(ask.Input); - foreach (var input in ask.Variables) - { - contextVariables.Set(input.Key, input.Value); - } - - // Register plugins that have been enabled - await this.RegisterPlannerSkillsAsync(planner, openApiSkillsAuthHeaders, contextVariables); - - // Get the function to invoke - ISKFunction? function = null; - try - { - function = kernel.Skills.GetFunction(ChatSkillName, ChatFunctionName); - } - catch (KernelException ke) - { - this._logger.LogError("Failed to find {0}/{1} on server: {2}", ChatSkillName, ChatFunctionName, ke); - - return this.NotFound($"Failed to find {ChatSkillName}/{ChatFunctionName} on server"); - } - - // Broadcast bot typing state to all users - if (ask.Variables.Where(v => v.Key == "chatId").Any()) - { - var chatId = ask.Variables.Where(v => v.Key == "chatId").First().Value; - await messageRelayHubContext.Clients.Group(chatId).SendAsync(GeneratingResponseClientCall, chatId, true); - } - - // Run the function. - SKContext? result = null; - try - { - result = await kernel.RunAsync(contextVariables, function!); - } - finally - { - this._telemetryService.TrackSkillFunction(ChatSkillName, ChatFunctionName, (!result?.ErrorOccurred) ?? false); - } - - if (result.ErrorOccurred) - { - if (result.LastException is AIException aiException && aiException.Detail is not null) - { - return this.BadRequest(string.Concat(aiException.Message, " - Detail: " + aiException.Detail)); - } - - return this.BadRequest(result.LastErrorDescription); - } - - AskResult chatSkillAskResult = new() - { - Value = result.Result, - Variables = result.Variables.Select( - v => new KeyValuePair(v.Key, v.Value)) - }; - - // Broadcast AskResult to all users - if (ask.Variables.Where(v => v.Key == "chatId").Any()) - { - var chatId = ask.Variables.Where(v => v.Key == "chatId").First().Value; - await messageRelayHubContext.Clients.Group(chatId).SendAsync(ReceiveResponseClientCall, chatSkillAskResult, chatId); - await messageRelayHubContext.Clients.Group(chatId).SendAsync(GeneratingResponseClientCall, chatId, false); - } - - return this.Ok(chatSkillAskResult); - } - - /// - /// Register skills with the planner's kernel. - /// - private async Task RegisterPlannerSkillsAsync(CopilotChatPlanner planner, OpenApiSkillsAuthHeaders openApiSkillsAuthHeaders, ContextVariables variables) - { - // Register authenticated skills with the planner's kernel only if the request includes an auth header for the skill. - - // Klarna Shopping - if (openApiSkillsAuthHeaders.KlarnaAuthentication != null) - { - // Register the Klarna shopping ChatGPT plugin with the planner's kernel. There is no authentication required for this plugin. - await planner.Kernel.ImportChatGptPluginSkillFromUrlAsync("KlarnaShoppingSkill", new Uri("https://www.klarna.com/.well-known/ai-plugin.json"), new OpenApiSkillExecutionParameters()); - } - - // GitHub - if (!string.IsNullOrWhiteSpace(openApiSkillsAuthHeaders.GithubAuthentication)) - { - this._logger.LogInformation("Enabling GitHub skill."); - BearerAuthenticationProvider authenticationProvider = new(() => Task.FromResult(openApiSkillsAuthHeaders.GithubAuthentication)); - await planner.Kernel.ImportOpenApiSkillFromFileAsync( - skillName: "GitHubSkill", - filePath: Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "CopilotChat", "Skills", "OpenApiSkills/GitHubSkill/openapi.json"), - new OpenApiSkillExecutionParameters - { - AuthCallback = authenticationProvider.AuthenticateRequestAsync, - }); - } - - // Jira - if (!string.IsNullOrWhiteSpace(openApiSkillsAuthHeaders.JiraAuthentication)) - { - this._logger.LogInformation("Registering Jira Skill"); - var authenticationProvider = new BasicAuthenticationProvider(() => { return Task.FromResult(openApiSkillsAuthHeaders.JiraAuthentication); }); - var hasServerUrlOverride = variables.TryGetValue("jira-server-url", out string? serverUrlOverride); - - await planner.Kernel.ImportOpenApiSkillFromFileAsync( - skillName: "JiraSkill", - filePath: Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "CopilotChat", "Skills", "OpenApiSkills/JiraSkill/openapi.json"), - new OpenApiSkillExecutionParameters - { - AuthCallback = authenticationProvider.AuthenticateRequestAsync, - ServerUrlOverride = hasServerUrlOverride ? new Uri(serverUrlOverride!) : null, - }); - } - - // Microsoft Graph - if (!string.IsNullOrWhiteSpace(openApiSkillsAuthHeaders.GraphAuthentication)) - { - this._logger.LogInformation("Enabling Microsoft Graph skill(s)."); - BearerAuthenticationProvider authenticationProvider = new(() => Task.FromResult(openApiSkillsAuthHeaders.GraphAuthentication)); - GraphServiceClient graphServiceClient = this.CreateGraphServiceClient(authenticationProvider.AuthenticateRequestAsync); - - planner.Kernel.ImportSkill(new TaskListSkill(new MicrosoftToDoConnector(graphServiceClient)), "todo"); - planner.Kernel.ImportSkill(new CalendarSkill(new OutlookCalendarConnector(graphServiceClient)), "calendar"); - planner.Kernel.ImportSkill(new EmailSkill(new OutlookMailConnector(graphServiceClient)), "email"); - } - } - - /// - /// Create a Microsoft Graph service client. - /// - /// The delegate to authenticate the request. - private GraphServiceClient CreateGraphServiceClient(AuthenticateRequestAsyncDelegate authenticateRequestAsyncDelegate) - { - MsGraphClientLoggingHandler graphLoggingHandler = new(this._logger); - this._disposables.Add(graphLoggingHandler); - - IList graphMiddlewareHandlers = - GraphClientFactory.CreateDefaultHandlers(new DelegateAuthenticationProvider(authenticateRequestAsyncDelegate)); - graphMiddlewareHandlers.Add(graphLoggingHandler); - - HttpClient graphHttpClient = GraphClientFactory.Create(graphMiddlewareHandlers); - this._disposables.Add(graphHttpClient); - - GraphServiceClient graphServiceClient = new(graphHttpClient); - return graphServiceClient; - } - - /// - /// Dispose of the object. - /// - protected virtual void Dispose(bool disposing) - { - if (disposing) - { - foreach (IDisposable disposable in this._disposables) - { - disposable.Dispose(); - } - } - } - - /// - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - this.Dispose(disposing: true); - GC.SuppressFinalize(this); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatHistoryController.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatHistoryController.cs deleted file mode 100644 index 8f911fec4b3e..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatHistoryController.cs +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.SemanticKernel; -using SemanticKernel.Service.CopilotChat.Hubs; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Options; -using SemanticKernel.Service.CopilotChat.Storage; - -namespace SemanticKernel.Service.CopilotChat.Controllers; - -/// -/// Controller for chat history. -/// This controller is responsible for creating new chat sessions, retrieving chat sessions, -/// retrieving chat messages, and editing chat sessions. -/// -[ApiController] -[Authorize] -public class ChatHistoryController : ControllerBase -{ - private readonly ILogger _logger; - private readonly ChatSessionRepository _sessionRepository; - private readonly ChatMessageRepository _messageRepository; - private readonly ChatParticipantRepository _participantRepository; - private readonly ChatMemorySourceRepository _sourceRepository; - private readonly PromptsOptions _promptOptions; - private const string ChatEditedClientCall = "ChatEdited"; - - /// - /// Initializes a new instance of the class. - /// - /// The logger. - /// The chat session repository. - /// The chat message repository. - /// The chat participant repository. - /// The chat memory resource repository. - /// The prompts options. - public ChatHistoryController( - ILogger logger, - ChatSessionRepository sessionRepository, - ChatMessageRepository messageRepository, - ChatParticipantRepository participantRepository, - ChatMemorySourceRepository sourceRepository, - IOptions promptsOptions) - { - this._logger = logger; - this._sessionRepository = sessionRepository; - this._messageRepository = messageRepository; - this._participantRepository = participantRepository; - this._sourceRepository = sourceRepository; - this._promptOptions = promptsOptions.Value; - } - - /// - /// Create a new chat session and populate the session with the initial bot message. - /// - /// Contains the title of the chat. - /// The HTTP action result. - [HttpPost] - [Route("chatSession/create")] - [ProducesResponseType(StatusCodes.Status201Created)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - public async Task CreateChatSessionAsync([FromBody] CreateChatParameters chatParameter) - { - if (chatParameter.UserId == null || chatParameter.Title == null) - { - return this.BadRequest("Chat session parameters cannot be null."); - } - - // Create a new chat session - var newChat = new ChatSession(chatParameter.Title); - await this._sessionRepository.CreateAsync(newChat); - - var initialBotMessage = this._promptOptions.InitialBotMessage; - // The initial bot message doesn't need a prompt. - var chatMessage = ChatMessage.CreateBotResponseMessage( - newChat.Id, - initialBotMessage, - string.Empty); - await this._messageRepository.CreateAsync(chatMessage); - - // Add the user to the chat session - await this._participantRepository.CreateAsync(new ChatParticipant(chatParameter.UserId, newChat.Id)); - - this._logger.LogDebug("Created chat session with id {0}.", newChat.Id); - return this.CreatedAtAction(nameof(this.GetChatSessionByIdAsync), new { chatId = newChat.Id }, newChat); - } - - /// - /// Get a chat session by id. - /// - /// The chat id. - [HttpGet] - [ActionName("GetChatSessionByIdAsync")] - [Route("chatSession/getChat/{chatId:guid}")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task GetChatSessionByIdAsync(Guid chatId) - { - ChatSession? chat = null; - if (await this._sessionRepository.TryFindByIdAsync(chatId.ToString(), v => chat = v)) - { - return this.Ok(chat); - } - - return this.NotFound($"No chat session found for chat id '{chatId}'."); - } - - /// - /// Get all chat sessions associated with a user. Return an empty list if no chats are found. - /// The regex pattern that is used to match the user id will match the following format: - /// - 2 period separated groups of one or more hyphen-delimited alphanumeric strings. - /// The pattern matches two GUIDs in canonical textual representation separated by a period. - /// - /// The user id. - /// A list of chat sessions. An empty list if the user is not in any chat session. - [HttpGet] - [Route("chatSession/getAllChats/{userId:regex(([[a-z0-9]]+-)+[[a-z0-9]]+\\.([[a-z0-9]]+-)+[[a-z0-9]]+)}")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task GetAllChatSessionsAsync(string userId) - { - // Get all participants that belong to the user. - // Then get all the chats from the list of participants. - var chatParticipants = await this._participantRepository.FindByUserIdAsync(userId); - - var chats = new List(); - foreach (var chatParticipant in chatParticipants) - { - ChatSession? chat = null; - if (await this._sessionRepository.TryFindByIdAsync(chatParticipant.ChatId, v => chat = v)) - { - chats.Add(chat!); - } - else - { - this._logger.LogDebug( - "Failed to find chat session with id {0} for participant {1}", chatParticipant.ChatId, chatParticipant.Id); - return this.NotFound( - $"Failed to find chat session with id {chatParticipant.ChatId} for participant {chatParticipant.Id}"); - } - } - - return this.Ok(chats); - } - - /// - /// Get all chat messages for a chat session. - /// The list will be ordered with the first entry being the most recent message. - /// - /// The chat id. - /// The start index at which the first message will be returned. - /// The number of messages to return. -1 will return all messages starting from startIdx. - /// [Authorize] - [HttpGet] - [Route("chatSession/getChatMessages/{chatId:guid}")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task GetChatMessagesAsync( - Guid chatId, - [FromQuery] int startIdx = 0, - [FromQuery] int count = -1) - { - // TODO: the code mixes strings and Guid without being explicit about the serialization format - var chatMessages = await this._messageRepository.FindByChatIdAsync(chatId.ToString()); - if (!chatMessages.Any()) - { - return this.NotFound($"No messages found for chat id '{chatId}'."); - } - - chatMessages = chatMessages.OrderByDescending(m => m.Timestamp).Skip(startIdx); - if (count >= 0) { chatMessages = chatMessages.Take(count); } - - return this.Ok(chatMessages); - } - - /// - /// Edit a chat session. - /// - /// Object that contains the parameters to edit the chat. - [HttpPost] - [Route("chatSession/edit")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task EditChatSessionAsync( - [FromServices] IHubContext messageRelayHubContext, - [FromBody] ChatSession chatParameters) - { - string chatId = chatParameters.Id; - - ChatSession? chat = null; - if (await this._sessionRepository.TryFindByIdAsync(chatId, v => chat = v)) - { - chat!.Title = chatParameters.Title; - await this._sessionRepository.UpsertAsync(chat); - await messageRelayHubContext.Clients.Group(chatId).SendAsync(ChatEditedClientCall, chat); - return this.Ok(chat); - } - - return this.NotFound($"No chat session found for chat id '{chatId}'."); - } - - /// - /// Service API to get a list of imported sources. - /// - [Authorize] - [Route("chatSession/{chatId:guid}/sources")] - [HttpGet] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task>> GetSourcesAsync( - [FromServices] IKernel kernel, - Guid chatId) - { - this._logger.LogInformation("Get imported sources of chat session {0}", chatId); - - if (await this._sessionRepository.TryFindByIdAsync(chatId.ToString(), v => _ = v)) - { - var sources = await this._sourceRepository.FindByChatIdAsync(chatId.ToString()); - return this.Ok(sources); - } - - return this.NotFound($"No chat session found for chat id '{chatId}'."); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatParticipantController.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatParticipantController.cs deleted file mode 100644 index b08ff8c1f2d6..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/ChatParticipantController.cs +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; -using SemanticKernel.Service.CopilotChat.Hubs; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Storage; - -namespace SemanticKernel.Service.CopilotChat.Controllers; - -/// -/// Controller for managing invitations and participants in a chat session. -/// This controller is responsible for: -/// 1. Creating invitation links. -/// 2. Accepting/rejecting invitation links. -/// 3. Managing participants in a chat session. -/// -[ApiController] -[Authorize] -public class ChatParticipantController : ControllerBase -{ - private const string UserJoinedClientCall = "UserJoined"; - private readonly ILogger _logger; - private readonly ChatParticipantRepository _chatParticipantRepository; - private readonly ChatSessionRepository _chatSessionRepository; - - /// - /// Initializes a new instance of the class. - /// - /// The logger. - /// The chat participant repository. - /// The chat session repository. - public ChatParticipantController( - ILogger logger, - ChatParticipantRepository chatParticipantRepository, - ChatSessionRepository chatSessionRepository) - { - this._logger = logger; - this._chatParticipantRepository = chatParticipantRepository; - this._chatSessionRepository = chatSessionRepository; - } - - /// - /// Join a use to a chat session given a chat id and a user id. - /// - /// Message Hub that performs the real time relay service. - /// Contains the user id and chat id. - [HttpPost] - [Route("chatParticipant/join")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - public async Task JoinChatAsync( - [FromServices] IHubContext messageRelayHubContext, - [FromBody] ChatParticipant chatParticipantParam) - { - string userId = chatParticipantParam.UserId; - string chatId = chatParticipantParam.ChatId; - - // Make sure the chat session exists. - if (!await this._chatSessionRepository.TryFindByIdAsync(chatId, v => _ = v)) - { - return this.BadRequest("Chat session does not exist."); - } - - // Make sure the user is not already in the chat session. - if (await this._chatParticipantRepository.IsUserInChatAsync(userId, chatId)) - { - return this.BadRequest("User is already in the chat session."); - } - - var chatParticipant = new ChatParticipant(userId, chatId); - await this._chatParticipantRepository.CreateAsync(chatParticipant); - - // Broadcast the user joined event to all the connected clients. - // Note that the client who initiated the request may not have joined the group. - await messageRelayHubContext.Clients.Group(chatId).SendAsync(UserJoinedClientCall, chatId, userId); - - return this.Ok(chatParticipant); - } - - /// - /// Get a list of chat participants that have the same chat id. - /// - /// The Id of the chat to get all the participants from. - [HttpGet] - [Route("chatParticipant/getAllParticipants/{chatId:guid}")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status404NotFound)] - public async Task GetAllParticipantsAsync(Guid chatId) - { - // Make sure the chat session exists. - if (!await this._chatSessionRepository.TryFindByIdAsync(chatId.ToString(), v => _ = v)) - { - return this.NotFound("Chat session does not exist."); - } - - var chatParticipants = await this._chatParticipantRepository.FindByChatIdAsync(chatId.ToString()); - return this.Ok(chatParticipants); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/DocumentImportController.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/DocumentImportController.cs deleted file mode 100644 index d53020553534..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/DocumentImportController.cs +++ /dev/null @@ -1,562 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Text; -using SemanticKernel.Service.CopilotChat.Hubs; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Options; -using SemanticKernel.Service.CopilotChat.Storage; -using SemanticKernel.Service.Services; -using Tesseract; -using UglyToad.PdfPig; -using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor; -using static SemanticKernel.Service.CopilotChat.Models.MemorySource; - -namespace SemanticKernel.Service.CopilotChat.Controllers; - -/// -/// Controller for importing documents. -/// -[ApiController] -public class DocumentImportController : ControllerBase -{ - /// - /// Supported file types for import. - /// - private enum SupportedFileType - { - /// - /// .txt - /// - Txt, - - /// - /// .pdf - /// - Pdf, - - /// - /// .jpg - /// - Jpg, - - /// - /// .png - /// - Png, - - /// - /// .tif or .tiff - /// - Tiff - }; - - private readonly ILogger _logger; - private readonly DocumentMemoryOptions _options; - private readonly ChatSessionRepository _sessionRepository; - private readonly ChatMemorySourceRepository _sourceRepository; - private readonly ChatMessageRepository _messageRepository; - private readonly ChatParticipantRepository _participantRepository; - private const string GlobalDocumentUploadedClientCall = "GlobalDocumentUploaded"; - private const string ChatDocumentUploadedClientCall = "ChatDocumentUploaded"; - private readonly ITesseractEngine _tesseractEngine; - - /// - /// Initializes a new instance of the class. - /// - public DocumentImportController( - ILogger logger, - IOptions documentMemoryOptions, - ChatSessionRepository sessionRepository, - ChatMemorySourceRepository sourceRepository, - ChatMessageRepository messageRepository, - ChatParticipantRepository participantRepository, - ITesseractEngine tesseractEngine) - { - this._logger = logger; - this._options = documentMemoryOptions.Value; - this._sessionRepository = sessionRepository; - this._sourceRepository = sourceRepository; - this._messageRepository = messageRepository; - this._participantRepository = participantRepository; - this._tesseractEngine = tesseractEngine; - } - - /// - /// Service API for importing a document. - /// - [Authorize] - [Route("importDocuments")] - [HttpPost] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - public async Task ImportDocumentsAsync( - [FromServices] IKernel kernel, - [FromServices] IHubContext messageRelayHubContext, - [FromForm] DocumentImportForm documentImportForm) - { - try - { - await this.ValidateDocumentImportFormAsync(documentImportForm); - } - catch (ArgumentException ex) - { - return this.BadRequest(ex.Message); - } - - this._logger.LogInformation("Importing {0} document(s)...", documentImportForm.FormFiles.Count()); - - // TODO: Perform the import in parallel. - DocumentMessageContent documentMessageContent = new(); - IEnumerable importResults = new List(); - foreach (var formFile in documentImportForm.FormFiles) - { - var importResult = await this.ImportDocumentHelperAsync(kernel, formFile, documentImportForm); - documentMessageContent.AddDocument( - formFile.FileName, - this.GetReadableByteString(formFile.Length), - importResult.IsSuccessful); - importResults = importResults.Append(importResult); - } - - // Broadcast the document uploaded event to other users. - if (documentImportForm.DocumentScope == DocumentImportForm.DocumentScopes.Chat) - { - var chatMessage = await this.TryCreateDocumentUploadMessage( - documentMessageContent, - documentImportForm); - if (chatMessage == null) - { - foreach (var importResult in importResults) - { - await this.RemoveMemoriesAsync(kernel, importResult); - } - return this.BadRequest("Failed to create chat message. All documents are removed."); - } - - var chatId = documentImportForm.ChatId.ToString(); - await messageRelayHubContext.Clients.Group(chatId) - .SendAsync(ChatDocumentUploadedClientCall, chatMessage, chatId); - - return this.Ok(chatMessage); - } - - await messageRelayHubContext.Clients.All.SendAsync( - GlobalDocumentUploadedClientCall, - documentMessageContent.ToFormattedStringNamesOnly(), - documentImportForm.UserName - ); - - return this.Ok("Documents imported successfully to global scope."); - } - - #region Private - - /// - /// A class to store a document import results. - /// - private sealed class ImportResult - { - /// - /// A boolean indicating whether the import is successful. - /// - public bool IsSuccessful => this.Keys.Any(); - - /// - /// The name of the collection that the document is inserted to. - /// - public string CollectionName { get; set; } - - /// - /// The keys of the inserted document chunks. - /// - public IEnumerable Keys { get; set; } = new List(); - - /// - /// Create a new instance of the class. - /// - /// The name of the collection that the document is inserted to. - public ImportResult(string collectionName) - { - this.CollectionName = collectionName; - } - - /// - /// Create a new instance of the class representing a failed import. - /// - public static ImportResult Fail() => new(string.Empty); - - /// - /// Add a key to the list of keys. - /// - /// The key to be added. - public void AddKey(string key) - { - this.Keys = this.Keys.Append(key); - } - } - - /// - /// Validates the document import form. - /// - /// The document import form. - /// - /// Throws ArgumentException if validation fails. - private async Task ValidateDocumentImportFormAsync(DocumentImportForm documentImportForm) - { - // Make sure the user has access to the chat session if the document is uploaded to a chat session. - if (documentImportForm.DocumentScope == DocumentImportForm.DocumentScopes.Chat - && !(await this.UserHasAccessToChatAsync(documentImportForm.UserId, documentImportForm.ChatId))) - { - throw new ArgumentException("User does not have access to the chat session."); - } - - var formFiles = documentImportForm.FormFiles; - - if (!formFiles.Any()) - { - throw new ArgumentException("No files were uploaded."); - } - else if (formFiles.Count() > this._options.FileCountLimit) - { - throw new ArgumentException($"Too many files uploaded. Max file count is {this._options.FileCountLimit}."); - } - - // Loop through the uploaded files and validate them before importing. - foreach (var formFile in formFiles) - { - if (formFile.Length == 0) - { - throw new ArgumentException($"File {formFile.FileName} is empty."); - } - - if (formFile.Length > this._options.FileSizeLimit) - { - throw new ArgumentException($"File {formFile.FileName} size exceeds the limit."); - } - - // Make sure the file type is supported. - var fileType = this.GetFileType(Path.GetFileName(formFile.FileName)); - switch (fileType) - { - case SupportedFileType.Txt: - case SupportedFileType.Pdf: - break; - default: - throw new ArgumentException($"Unsupported file type: {fileType}"); - } - } - } - - /// - /// Import a single document. - /// - /// The kernel. - /// The form file. - /// The document import form. - /// Import result. - private async Task ImportDocumentHelperAsync(IKernel kernel, IFormFile formFile, DocumentImportForm documentImportForm) - { - var fileType = this.GetFileType(Path.GetFileName(formFile.FileName)); - var documentContent = string.Empty; - switch (fileType) - { - case SupportedFileType.Txt: - documentContent = await this.ReadTxtFileAsync(formFile); - break; - case SupportedFileType.Pdf: - documentContent = this.ReadPdfFile(formFile); - break; - case SupportedFileType.Jpg: - case SupportedFileType.Png: - case SupportedFileType.Tiff: - { - documentContent = await this.ReadTextFromImageFileAsync(formFile); - break; - } - - default: - // This should never happen. Validation should have already caught this. - return ImportResult.Fail(); - } - - this._logger.LogInformation("Importing document {0}", formFile.FileName); - - // Create memory source - var memorySource = await this.TryCreateAndUpsertMemorySourceAsync(formFile, documentImportForm); - if (memorySource == null) - { - return ImportResult.Fail(); - } - - // Parse document content to memory - ImportResult importResult = ImportResult.Fail(); - try - { - importResult = await this.ParseDocumentContentToMemoryAsync( - kernel, - formFile.FileName, - documentContent, - documentImportForm, - memorySource.Id - ); - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - await this._sourceRepository.DeleteAsync(memorySource); - await this.RemoveMemoriesAsync(kernel, importResult); - return ImportResult.Fail(); - } - - return importResult; - } - - /// - /// Try to create and upsert a memory source. - /// - /// The file to be uploaded - /// The document upload form that contains additional necessary info - /// A MemorySource object if successful, null otherwise - private async Task TryCreateAndUpsertMemorySourceAsync( - IFormFile formFile, - DocumentImportForm documentImportForm) - { - var chatId = documentImportForm.ChatId.ToString(); - var userId = documentImportForm.UserId; - var memorySource = new MemorySource( - chatId, - formFile.FileName, - userId, - MemorySourceType.File, - formFile.Length, - null); - - try - { - await this._sourceRepository.UpsertAsync(memorySource); - return memorySource; - } - catch (Exception ex) when (ex is ArgumentOutOfRangeException) - { - return null; - } - } - - /// - /// Try to create a chat message that represents document upload. - /// - /// The chat id - /// The user id - /// The document message content - /// The document upload form that contains additional necessary info - /// A ChatMessage object if successful, null otherwise - private async Task TryCreateDocumentUploadMessage( - DocumentMessageContent documentMessageContent, - DocumentImportForm documentImportForm) - { - var chatId = documentImportForm.ChatId.ToString(); - var userId = documentImportForm.UserId; - var userName = documentImportForm.UserName; - - var chatMessage = ChatMessage.CreateDocumentMessage( - userId, - userName, - chatId, - documentMessageContent - ); - - try - { - await this._messageRepository.CreateAsync(chatMessage); - return chatMessage; - } - catch (Exception ex) when (ex is ArgumentOutOfRangeException) - { - return null; - } - } - - /// - /// Converts a `long` byte count to a human-readable string. - /// - /// Byte count - /// Human-readable string of bytes - private string GetReadableByteString(long bytes) - { - string[] sizes = { "B", "KB", "MB", "GB", "TB" }; - int i; - double dblsBytes = bytes; - for (i = 0; i < sizes.Length && bytes >= 1024; i++, bytes /= 1024) - { - dblsBytes = bytes / 1024.0; - } - - return string.Format(CultureInfo.InvariantCulture, "{0:0.#}{1}", dblsBytes, sizes[i]); - } - - /// - /// Get the file type from the file extension. - /// - /// Name of the file. - /// A SupportedFileType. - /// - private SupportedFileType GetFileType(string fileName) - { - string extension = Path.GetExtension(fileName); - return extension switch - { - ".txt" => SupportedFileType.Txt, - ".pdf" => SupportedFileType.Pdf, - ".jpg" => SupportedFileType.Jpg, - ".jpeg" => SupportedFileType.Jpg, - ".png" => SupportedFileType.Png, - ".tif" => SupportedFileType.Tiff, - ".tiff" => SupportedFileType.Tiff, - _ => throw new ArgumentOutOfRangeException($"Unsupported file type: {extension}"), - }; - } - - /// - /// Reads the text content from an image file. - /// - /// An IFormFile object. - /// A string of the content of the file. - private async Task ReadTextFromImageFileAsync(IFormFile file) - { - await using (var ms = new MemoryStream()) - { - await file.CopyToAsync(ms); - var fileBytes = ms.ToArray(); - await using var imgStream = new MemoryStream(fileBytes); - - using var img = Pix.LoadFromMemory(imgStream.ToArray()); - - using var page = this._tesseractEngine.Process(img); - return page.GetText(); - } - } - - /// - /// Read the content of a text file. - /// - /// An IFormFile object. - /// A string of the content of the file. - private async Task ReadTxtFileAsync(IFormFile file) - { - using var streamReader = new StreamReader(file.OpenReadStream()); - return await streamReader.ReadToEndAsync(); - } - - /// - /// Read the content of a PDF file, ignoring images. - /// - /// An IFormFile object. - /// A string of the content of the file. - private string ReadPdfFile(IFormFile file) - { - var fileContent = string.Empty; - - using var pdfDocument = PdfDocument.Open(file.OpenReadStream()); - foreach (var page in pdfDocument.GetPages()) - { - var text = ContentOrderTextExtractor.GetText(page); - fileContent += text; - } - - return fileContent; - } - - /// - /// Parse the content of the document to memory. - /// - /// The kernel instance from the service - /// The name of the uploaded document - /// The file content read from the uploaded document - /// The document upload form that contains additional necessary info - /// The ID of the MemorySource that the document content is linked to - private async Task ParseDocumentContentToMemoryAsync( - IKernel kernel, - string documentName, - string content, - DocumentImportForm documentImportForm, - string memorySourceId) - { - var targetCollectionName = documentImportForm.DocumentScope == DocumentImportForm.DocumentScopes.Global - ? this._options.GlobalDocumentCollectionName - : this._options.ChatDocumentCollectionNamePrefix + documentImportForm.ChatId; - var importResult = new ImportResult(targetCollectionName); - - // Split the document into lines of text and then combine them into paragraphs. - // Note that this is only one of many strategies to chunk documents. Feel free to experiment with other strategies. - var lines = TextChunker.SplitPlainTextLines(content, this._options.DocumentLineSplitMaxTokens); - var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, this._options.DocumentParagraphSplitMaxLines); - - // TODO: Perform the save in parallel. - for (var i = 0; i < paragraphs.Count; i++) - { - var paragraph = paragraphs[i]; - var key = $"{memorySourceId}-{i}"; - await kernel.Memory.SaveInformationAsync( - collection: targetCollectionName, - text: paragraph, - id: key, - description: $"Document: {documentName}"); - importResult.AddKey(key); - } - - this._logger.LogInformation( - "Parsed {0} paragraphs from local file {1}", - paragraphs.Count, - documentName - ); - - return importResult; - } - - /// - /// Check if the user has access to the chat session. - /// - /// The user ID. - /// The chat session ID. - /// A boolean indicating whether the user has access to the chat session. - private async Task UserHasAccessToChatAsync(string userId, Guid chatId) - { - return await this._participantRepository.IsUserInChatAsync(userId, chatId.ToString()); - } - - /// - /// Remove the memories that were created during the import process if subsequent steps fail. - /// - /// The kernel instance from the service - /// The import result that contains the keys of the memories to be removed - /// - private async Task RemoveMemoriesAsync(IKernel kernel, ImportResult importResult) - { - foreach (var key in importResult.Keys) - { - try - { - await kernel.Memory.RemoveAsync(importResult.CollectionName, key); - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - this._logger.LogError(ex, "Failed to remove memory {0} from collection {1}. Skipped.", key, importResult.CollectionName); - continue; - } - } - } - - #endregion -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/SpeechTokenController.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/SpeechTokenController.cs deleted file mode 100644 index 8dea7b3b90f4..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/SpeechTokenController.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Options; - -namespace SemanticKernel.Service.CopilotChat.Controllers; - -[Authorize] -[ApiController] -public class SpeechTokenController : ControllerBase -{ - private sealed class TokenResult - { - public string? Token { get; set; } - public HttpStatusCode? ResponseCode { get; set; } - } - - private readonly ILogger _logger; - private readonly AzureSpeechOptions _options; - - public SpeechTokenController(IOptions options, ILogger logger) - { - this._logger = logger; - this._options = options.Value; - } - - /// - /// Get an authorization token and region - /// - [Route("speechToken")] - [HttpGet] - [ProducesResponseType(StatusCodes.Status200OK)] - public async Task> GetAsync() - { - // Azure Speech token support is optional. If the configuration is missing or incomplete, return an unsuccessful token response. - if (string.IsNullOrWhiteSpace(this._options.Region) || - string.IsNullOrWhiteSpace(this._options.Key)) - { - return new SpeechTokenResponse { IsSuccess = false }; - } - - string fetchTokenUri = "https://" + this._options.Region + ".api.cognitive.microsoft.com/sts/v1.0/issueToken"; - - TokenResult tokenResult = await this.FetchTokenAsync(fetchTokenUri, this._options.Key); - var isSuccess = tokenResult.ResponseCode != HttpStatusCode.NotFound; - return new SpeechTokenResponse { Token = tokenResult.Token, Region = this._options.Region, IsSuccess = isSuccess }; - } - - private async Task FetchTokenAsync(string fetchUri, string subscriptionKey) - { - // TODO: get the HttpClient from the DI container - using var client = new HttpClient(); - client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey); - UriBuilder uriBuilder = new(fetchUri); - - var result = await client.PostAsync(uriBuilder.Uri, null); - if (result.IsSuccessStatusCode) - { - var response = result.EnsureSuccessStatusCode(); - this._logger.LogDebug("Token Uri: {0}", uriBuilder.Uri.AbsoluteUri); - string token = await result.Content.ReadAsStringAsync(); - return new TokenResult { Token = token, ResponseCode = response.StatusCode }; - } - - return new TokenResult { Token = "", ResponseCode = HttpStatusCode.NotFound }; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/IAsyncEnumerableExtensions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/IAsyncEnumerableExtensions.cs deleted file mode 100644 index 033364acc2e2..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/IAsyncEnumerableExtensions.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace SemanticKernel.Service.CopilotChat.Extensions; - -/// -/// Extension methods for enabling async LINQ operations on IAsyncEnumerable sequence. -/// -public static class IAsyncEnumerableExtensions -{ - /// - /// Creates a List from an IAsyncEnumerable by enumerating it asynchronously. - /// - internal static async Task> ToListAsync(this IAsyncEnumerable source) - { - var result = new List(); - await foreach (var item in source.ConfigureAwait(false)) - { - result.Add(item); - } - - return result; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/SemanticKernelExtensions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/SemanticKernelExtensions.cs deleted file mode 100644 index ac9a47a9cce3..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/SemanticKernelExtensions.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.SemanticKernel; -using SemanticKernel.Service.CopilotChat.Options; -using SemanticKernel.Service.CopilotChat.Skills.ChatSkills; -using SemanticKernel.Service.CopilotChat.Storage; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Extensions; - -/// -/// Extension methods for registering Copilot Chat components to Semantic Kernel. -/// -public static class CopilotChatSemanticKernelExtensions -{ - /// - /// Add Planner services - /// - public static IServiceCollection AddCopilotChatPlannerServices(this IServiceCollection services) - { - IOptions? plannerOptions = services.BuildServiceProvider().GetService>(); - services.AddScoped(sp => - { - IKernel plannerKernel = Kernel.Builder - .WithLogger(sp.GetRequiredService>()) - // TODO verify planner has AI service configured - .WithPlannerBackend(sp.GetRequiredService>().Value) - .Build(); - return new CopilotChatPlanner(plannerKernel, plannerOptions?.Value); - }); - - // Register Planner skills (AI plugins) here. - // TODO: Move planner skill registration from ChatController to here. - - return services; - } - - /// - /// Register the Copilot chat skills with the kernel. - /// - public static IKernel RegisterCopilotChatSkills(this IKernel kernel, IServiceProvider sp) - { - // Chat skill - kernel.ImportSkill(new ChatSkill( - kernel: kernel, - chatMessageRepository: sp.GetRequiredService(), - chatSessionRepository: sp.GetRequiredService(), - promptOptions: sp.GetRequiredService>(), - documentImportOptions: sp.GetRequiredService>(), - planner: sp.GetRequiredService(), - logger: sp.GetRequiredService>()), - nameof(ChatSkill)); - - return kernel; - } - - /// - /// Add the completion backend to the kernel config for the planner. - /// - private static KernelBuilder WithPlannerBackend(this KernelBuilder kernelBuilder, AIServiceOptions options) - { - return options.Type switch - { - AIServiceOptions.AIServiceType.AzureOpenAI => kernelBuilder.WithAzureChatCompletionService(options.Models.Planner, options.Endpoint, options.Key), - AIServiceOptions.AIServiceType.OpenAI => kernelBuilder.WithOpenAIChatCompletionService(options.Models.Planner, options.Key), - _ => throw new ArgumentException($"Invalid {nameof(options.Type)} value in '{AIServiceOptions.PropertyName}' settings."), - }; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/ServiceExtensions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/ServiceExtensions.cs deleted file mode 100644 index eff7b3854faf..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/ServiceExtensions.cs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Options; -using SemanticKernel.Service.CopilotChat.Storage; -using SemanticKernel.Service.Options; -using SemanticKernel.Service.Services; -using Tesseract; - -namespace SemanticKernel.Service.CopilotChat.Extensions; - -/// -/// Extension methods for . -/// Add options and services for Copilot Chat. -/// -public static class CopilotChatServiceExtensions -{ - /// - /// Parse configuration into options. - /// - public static IServiceCollection AddCopilotChatOptions(this IServiceCollection services, ConfigurationManager configuration) - { - // AI service configurations for Copilot Chat. - // They are using the same configuration section as Semantic Kernel. - services.AddOptions(AIServiceOptions.PropertyName) - .Bind(configuration.GetSection(AIServiceOptions.PropertyName)) - .ValidateOnStart() - .PostConfigure(TrimStringProperties); - - // Chat log storage configuration - services.AddOptions() - .Bind(configuration.GetSection(ChatStoreOptions.PropertyName)) - .ValidateOnStart() - .PostConfigure(TrimStringProperties); - - // Azure speech token configuration - services.AddOptions() - .Bind(configuration.GetSection(AzureSpeechOptions.PropertyName)) - .ValidateOnStart() - .PostConfigure(TrimStringProperties); - - // Bot schema configuration - services.AddOptions() - .Bind(configuration.GetSection(BotSchemaOptions.PropertyName)) - .ValidateOnStart() - .PostConfigure(TrimStringProperties); - - // Document memory options - services.AddOptions() - .Bind(configuration.GetSection(DocumentMemoryOptions.PropertyName)) - .ValidateOnStart() - .PostConfigure(TrimStringProperties); - - // Chat prompt options - services.AddOptions() - .Bind(configuration.GetSection(PromptsOptions.PropertyName)) - .ValidateOnStart() - .PostConfigure(TrimStringProperties); - - // Planner options - services.AddOptions() - .Bind(configuration.GetSection(PlannerOptions.PropertyName)) - .ValidateOnStart() - .PostConfigure(TrimStringProperties); - - // OCR support options - services.AddOptions() - .Bind(configuration.GetSection(OcrSupportOptions.PropertyName)) - .ValidateOnStart() - .PostConfigure(TrimStringProperties); - - return services; - } - - /// - /// Adds persistent OCR support service. - /// - /// - public static IServiceCollection AddPersistentOcrSupport(this IServiceCollection services) - { - OcrSupportOptions ocrSupportConfig = services.BuildServiceProvider().GetRequiredService>().Value; - - switch (ocrSupportConfig.Type) - { - case OcrSupportOptions.OcrSupportType.Tesseract: - { - services.AddSingleton(sp => new TesseractEngineWrapper(new TesseractEngine(ocrSupportConfig.Tesseract!.FilePath, ocrSupportConfig.Tesseract!.Language, EngineMode.Default))); - break; - } - - case OcrSupportOptions.OcrSupportType.None: - { - services.AddSingleton(sp => new NullTesseractEngine()); - break; - } - - default: - { - throw new InvalidOperationException($"Unsupported OcrSupport:Type '{ocrSupportConfig.Type}'"); - } - } - - return services; - } - - /// - /// Add persistent chat store services. - /// - public static IServiceCollection AddPersistentChatStore(this IServiceCollection services) - { - IStorageContext chatSessionStorageContext; - IStorageContext chatMessageStorageContext; - IStorageContext chatMemorySourceStorageContext; - IStorageContext chatParticipantStorageContext; - - ChatStoreOptions chatStoreConfig = services.BuildServiceProvider().GetRequiredService>().Value; - - switch (chatStoreConfig.Type) - { - case ChatStoreOptions.ChatStoreType.Volatile: - { - chatSessionStorageContext = new VolatileContext(); - chatMessageStorageContext = new VolatileContext(); - chatMemorySourceStorageContext = new VolatileContext(); - chatParticipantStorageContext = new VolatileContext(); - break; - } - - case ChatStoreOptions.ChatStoreType.Filesystem: - { - if (chatStoreConfig.Filesystem == null) - { - throw new InvalidOperationException("ChatStore:Filesystem is required when ChatStore:Type is 'Filesystem'"); - } - - string fullPath = Path.GetFullPath(chatStoreConfig.Filesystem.FilePath); - string directory = Path.GetDirectoryName(fullPath) ?? string.Empty; - chatSessionStorageContext = new FileSystemContext( - new FileInfo(Path.Combine(directory, $"{Path.GetFileNameWithoutExtension(fullPath)}_sessions{Path.GetExtension(fullPath)}"))); - chatMessageStorageContext = new FileSystemContext( - new FileInfo(Path.Combine(directory, $"{Path.GetFileNameWithoutExtension(fullPath)}_messages{Path.GetExtension(fullPath)}"))); - chatMemorySourceStorageContext = new FileSystemContext( - new FileInfo(Path.Combine(directory, $"{Path.GetFileNameWithoutExtension(fullPath)}_memorysources{Path.GetExtension(fullPath)}"))); - chatParticipantStorageContext = new FileSystemContext( - new FileInfo(Path.Combine(directory, $"{Path.GetFileNameWithoutExtension(fullPath)}_participants{Path.GetExtension(fullPath)}"))); - break; - } - - case ChatStoreOptions.ChatStoreType.Cosmos: - { - if (chatStoreConfig.Cosmos == null) - { - throw new InvalidOperationException("ChatStore:Cosmos is required when ChatStore:Type is 'Cosmos'"); - } -#pragma warning disable CA2000 // Dispose objects before losing scope - objects are singletons for the duration of the process and disposed when the process exits. - chatSessionStorageContext = new CosmosDbContext( - chatStoreConfig.Cosmos.ConnectionString, chatStoreConfig.Cosmos.Database, chatStoreConfig.Cosmos.ChatSessionsContainer); - chatMessageStorageContext = new CosmosDbContext( - chatStoreConfig.Cosmos.ConnectionString, chatStoreConfig.Cosmos.Database, chatStoreConfig.Cosmos.ChatMessagesContainer); - chatMemorySourceStorageContext = new CosmosDbContext( - chatStoreConfig.Cosmos.ConnectionString, chatStoreConfig.Cosmos.Database, chatStoreConfig.Cosmos.ChatMemorySourcesContainer); - chatParticipantStorageContext = new CosmosDbContext( - chatStoreConfig.Cosmos.ConnectionString, chatStoreConfig.Cosmos.Database, chatStoreConfig.Cosmos.ChatParticipantsContainer); -#pragma warning restore CA2000 // Dispose objects before losing scope - break; - } - - default: - { - throw new InvalidOperationException( - "Invalid 'ChatStore' setting 'chatStoreConfig.Type'."); - } - } - - services.AddSingleton(new ChatSessionRepository(chatSessionStorageContext)); - services.AddSingleton(new ChatMessageRepository(chatMessageStorageContext)); - services.AddSingleton(new ChatMemorySourceRepository(chatMemorySourceStorageContext)); - services.AddSingleton(new ChatParticipantRepository(chatParticipantStorageContext)); - - return services; - } - - /// - /// Trim all string properties, recursively. - /// - private static void TrimStringProperties(T options) where T : class - { - Queue targets = new(); - targets.Enqueue(options); - - while (targets.Count > 0) - { - object target = targets.Dequeue(); - Type targetType = target.GetType(); - foreach (PropertyInfo property in targetType.GetProperties()) - { - // Skip enumerations - if (property.PropertyType.IsEnum) - { - continue; - } - - // Property is a built-in type, readable, and writable. - if (property.PropertyType.Namespace == "System" && - property.CanRead && - property.CanWrite) - { - // Property is a non-null string. - if (property.PropertyType == typeof(string) && - property.GetValue(target) != null) - { - property.SetValue(target, property.GetValue(target)!.ToString()!.Trim()); - } - } - else - { - // Property is a non-built-in and non-enum type - queue it for processing. - if (property.GetValue(target) != null) - { - targets.Enqueue(property.GetValue(target)!); - } - } - } - } - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Hubs/MessageRelayHub.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Hubs/MessageRelayHub.cs deleted file mode 100644 index ec624907108f..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Hubs/MessageRelayHub.cs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Threading.Tasks; -using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; - -namespace SemanticKernel.Service.CopilotChat.Hubs; - -/// -/// Represents a chat hub for real-time communication. -/// -public class MessageRelayHub : Hub -{ - private const string ReceiveMessageClientCall = "ReceiveMessage"; - private const string ReceiveUserTypingStateClientCall = "ReceiveUserTypingState"; - private readonly ILogger _logger; - - /// - /// Initializes a new instance of the class. - /// - /// The logger. - public MessageRelayHub(ILogger logger) - { - this._logger = logger; - } - - /// - /// Adds the user to the groups that they are a member of. - /// Groups are identified by the chat ID. - /// TODO: Retrieve the user ID from the claims and call this method - /// from the OnConnectedAsync method instead of the frontend. - /// - /// The ChatID used as group id for SignalR. - public async Task AddClientToGroupAsync(string chatId) - { - await this.Groups.AddToGroupAsync(this.Context.ConnectionId, chatId); - } - - /// - /// Sends a message to all users except the sender. - /// - /// The ChatID used as group id for SignalR. - /// The message to send. - public async Task SendMessageAsync(string chatId, object message) - { - await this.Clients.OthersInGroup(chatId).SendAsync(ReceiveMessageClientCall, message, chatId); - } - - /// - /// Sends the typing state to all users except the sender. - /// - /// The ChatID used as group id for SignalR. - /// The user ID of the user who is typing. - /// Whether the user is typing. - /// A task that represents the asynchronous operation. - public async Task SendUserTypingStateAsync(string chatId, string userId, bool isTyping) - { - await this.Clients.OthersInGroup(chatId).SendAsync(ReceiveUserTypingStateClientCall, chatId, userId, isTyping); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/Bot.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/Bot.cs deleted file mode 100644 index 302ec6cfad63..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/Bot.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using Microsoft.SemanticKernel.Memory; -using SemanticKernel.Service.CopilotChat.Options; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// The data model of a bot for portability. -/// -public class Bot -{ - /// - /// The schema information of the bot data model. - /// - public BotSchemaOptions Schema { get; set; } = new BotSchemaOptions(); - - /// - /// The embedding configurations. - /// - public BotEmbeddingConfig EmbeddingConfigurations { get; set; } = new BotEmbeddingConfig(); - - /// - /// The title of the chat with the bot. - /// - public string ChatTitle { get; set; } = string.Empty; - - /// - /// The chat history. It contains all the messages in the conversation with the bot. - /// - public List ChatHistory { get; set; } = new List(); - - // TODO: Change from MemoryQueryResult to MemoryRecord - /// - /// The embeddings of the bot. - /// - public List>> Embeddings { get; set; } = new List>>(); - - // TODO: Change from MemoryQueryResult to MemoryRecord - /// - /// The embeddings of uploaded documents in Copilot Chat. It represents the document memory which is accessible to all chat sessions of a given user. - /// - public List>> DocumentEmbeddings { get; set; } = new List>>(); -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/BotEmbeddingConfig.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/BotEmbeddingConfig.cs deleted file mode 100644 index e8e4768a1d70..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/BotEmbeddingConfig.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel.DataAnnotations; -using System.Text.Json.Serialization; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// The embedding configuration of a bot. Used in the Bot object for portability. -/// -public class BotEmbeddingConfig -{ - /// - /// The AI service. - /// - [Required] - [JsonConverter(typeof(JsonStringEnumConverter))] - public AIServiceOptions.AIServiceType AIService { get; set; } = AIServiceOptions.AIServiceType.AzureOpenAI; - - /// - /// The deployment or the model id. - /// - public string DeploymentOrModelId { get; set; } = string.Empty; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatMessage.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatMessage.cs deleted file mode 100644 index 16d3c5d2a496..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatMessage.cs +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Globalization; -using System.Text.Json; -using System.Text.Json.Serialization; -using SemanticKernel.Service.CopilotChat.Storage; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// Information about a single chat message. -/// -public class ChatMessage : IStorageEntity -{ - /// - /// Role of the author of a chat message. - /// - public enum AuthorRoles - { - /// - /// The current user of the chat. - /// - User = 0, - - /// - /// The bot. - /// - Bot, - - /// - /// The participant who is not the current user nor the bot of the chat. - /// - Participant - } - - /// - /// Type of the chat message. - /// - public enum ChatMessageType - { - /// - /// A standard message - /// - Message, - - /// - /// A message for a Plan - /// - Plan, - - /// - /// An uploaded document notification - /// - Document, - } - - /// - /// Timestamp of the message. - /// - [JsonPropertyName("timestamp")] - public DateTimeOffset Timestamp { get; set; } - - /// - /// Id of the user who sent this message. - /// - [JsonPropertyName("userId")] - public string UserId { get; set; } - - /// - /// Name of the user who sent this message. - /// - [JsonPropertyName("userName")] - public string UserName { get; set; } - - /// - /// Id of the chat this message belongs to. - /// - [JsonPropertyName("chatId")] - public string ChatId { get; set; } - - /// - /// Content of the message. - /// - [JsonPropertyName("content")] - public string Content { get; set; } - - /// - /// Id of the message. - /// - [JsonPropertyName("id")] - public string Id { get; set; } - - /// - /// Role of the author of the message. - /// - [JsonPropertyName("authorRole")] - public AuthorRoles AuthorRole { get; set; } - - /// - /// Prompt used to generate the message. - /// Will be empty if the message is not generated by a prompt. - /// - [JsonPropertyName("prompt")] - public string Prompt { get; set; } = string.Empty; - - /// - /// Type of the message. - /// - [JsonPropertyName("type")] - public ChatMessageType Type { get; set; } - - /// - /// Create a new chat message. Timestamp is automatically generated. - /// - /// Id of the user who sent this message - /// Name of the user who sent this message - /// The chat ID that this message belongs to - /// The message - /// The prompt used to generate the message - /// Role of the author - /// Type of the message - public ChatMessage( - string userId, - string userName, - string chatId, - string content, - string prompt = "", - AuthorRoles authorRole = AuthorRoles.User, - ChatMessageType type = ChatMessageType.Message) - { - this.Timestamp = DateTimeOffset.Now; - this.UserId = userId; - this.UserName = userName; - this.ChatId = chatId; - this.Content = content; - this.Id = Guid.NewGuid().ToString(); - this.Prompt = prompt; - this.AuthorRole = authorRole; - this.Type = type; - } - - /// - /// Create a new chat message for the bot response. - /// - /// The chat ID that this message belongs to - /// The message - /// The prompt used to generate the message - public static ChatMessage CreateBotResponseMessage(string chatId, string content, string prompt) - { - return new ChatMessage("bot", "bot", chatId, content, prompt, AuthorRoles.Bot, IsPlan(content) ? ChatMessageType.Plan : ChatMessageType.Message); - } - - /// - /// Create a new chat message for a document upload. - /// - /// The user ID that uploaded the document - /// The user name that uploaded the document - /// The chat ID that this message belongs to - /// The document message content - public static ChatMessage CreateDocumentMessage(string userId, string userName, string chatId, DocumentMessageContent documentMessageContent) - { - return new ChatMessage(userId, userName, chatId, documentMessageContent.ToString(), string.Empty, AuthorRoles.User, ChatMessageType.Document); - } - - /// - /// Serialize the object to a formatted string. - /// - /// A formatted string - public string ToFormattedString() - { - var content = this.Content; - if (this.Type == ChatMessageType.Document) - { - var documentMessageContent = DocumentMessageContent.FromString(content); - content = (documentMessageContent != null) ? documentMessageContent.ToFormattedString() : "Uploaded documents"; - } - - return $"[{this.Timestamp.ToString("G", CultureInfo.CurrentCulture)}] {this.UserName}: {content}"; - } - - /// - /// Serialize the object to a JSON string. - /// - /// A serialized json string - public override string ToString() - { - return JsonSerializer.Serialize(this); - } - - /// - /// Deserialize a JSON string to a ChatMessage object. - /// - /// A json string - /// A ChatMessage object - public static ChatMessage? FromString(string json) - { - return JsonSerializer.Deserialize(json); - } - - /// - /// Check if the response is a Plan. - /// This is a copy of the `isPlan` function on the frontend. - /// - /// The response from the bot. - /// True if the response represents Plan, false otherwise. - private static bool IsPlan(string response) - { - var planPrefix = "proposedPlan\":"; - return response.IndexOf(planPrefix, StringComparison.Ordinal) != -1; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatParticipant.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatParticipant.cs deleted file mode 100644 index 3c28912b5094..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatParticipant.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Text.Json.Serialization; -using SemanticKernel.Service.CopilotChat.Storage; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// A chat participant is a user that is part of a chat. -/// A user can be part of multiple chats, thus a user can have multiple chat participants. -/// -public class ChatParticipant : IStorageEntity -{ - /// - /// Participant ID that is persistent and unique. - /// - [JsonPropertyName("id")] - public string Id { get; set; } - - /// - /// User ID that is persistent and unique. - /// - [JsonPropertyName("userId")] - public string UserId { get; set; } - - /// - /// Chat ID that this participant belongs to. - /// - [JsonPropertyName("chatId")] - public string ChatId { get; set; } - - public ChatParticipant(string userId, string chatId) - { - this.Id = Guid.NewGuid().ToString(); - this.UserId = userId; - this.ChatId = chatId; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatSession.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatSession.cs deleted file mode 100644 index 9cb58ca86ff3..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ChatSession.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Text.Json.Serialization; -using SemanticKernel.Service.CopilotChat.Storage; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// A chat session -/// -public class ChatSession : IStorageEntity -{ - /// - /// Chat ID that is persistent and unique. - /// - [JsonPropertyName("id")] - public string Id { get; set; } - - /// - /// Title of the chat. - /// - [JsonPropertyName("title")] - public string Title { get; set; } - - /// - /// Timestamp of the chat creation. - /// - [JsonPropertyName("createdOn")] - public DateTimeOffset CreatedOn { get; set; } - - public ChatSession(string title) - { - this.Id = Guid.NewGuid().ToString(); - this.Title = title; - this.CreatedOn = DateTimeOffset.Now; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/CreateChatParameters.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/CreateChatParameters.cs deleted file mode 100644 index 5c25fcb13016..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/CreateChatParameters.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// Json body for creating a new chat session. -/// -public class CreateChatParameters -{ - /// - /// Id of the user who sent this message. - /// - [JsonPropertyName("userId")] - public string? UserId { get; set; } - - /// - /// Title of the chat. - /// - [JsonPropertyName("title")] - public string? Title { get; set; } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentData.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentData.cs deleted file mode 100644 index 04f937df1cb1..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentData.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; - -namespace SemanticKernel.Service.CopilotChat.Models; - -public sealed class DocumentData -{ - /// - /// Name of the uploaded document. - /// - [JsonPropertyName("name")] - public string Name { get; set; } = string.Empty; - - /// - /// Size of the uploaded document in bytes. - /// - [JsonPropertyName("size")] - public string Size { get; set; } = string.Empty; - - /// - /// Status of the uploaded document. - /// If true, the document is successfully uploaded. False otherwise. - /// - [JsonPropertyName("isUploaded")] - public bool IsUploaded { get; set; } = false; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentImportForm.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentImportForm.cs deleted file mode 100644 index e259734b7fb3..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentImportForm.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.AspNetCore.Http; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// Form for importing a document from a POST Http request. -/// -public class DocumentImportForm -{ - /// - /// Scope of the document. This determines the collection name in the document memory. - /// - public enum DocumentScopes - { - Global, - Chat, - } - - /// - /// The file to import. - /// - public IEnumerable FormFiles { get; set; } = Enumerable.Empty(); - - /// - /// Scope of the document. This determines the collection name in the document memory. - /// - public DocumentScopes DocumentScope { get; set; } = DocumentScopes.Chat; - - /// - /// The ID of the chat that owns the document. - /// This is used to create a unique collection name for the chat. - /// If the chat ID is not specified or empty, the documents will be stored in a global collection. - /// If the document scope is set to global, this value is ignored. - /// - public Guid ChatId { get; set; } = Guid.Empty; - - /// - /// The ID of the user who is importing the document to a chat session. - /// Will be use to validate if the user has access to the chat session. - /// - public string UserId { get; set; } = string.Empty; - - /// - /// Name of the user who sent this message. - /// Will be used to create the chat message representing the document upload. - /// - public string UserName { get; set; } = string.Empty; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentMessageContent.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentMessageContent.cs deleted file mode 100644 index ae590148969f..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/DocumentMessageContent.cs +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// Value of `Content` for a `ChatMessage` of type `ChatMessageType.Document`. -/// -public class DocumentMessageContent -{ - /// - /// List of documents contained in the message. - /// - [JsonPropertyName("documents")] - public IEnumerable Documents { get; private set; } = Enumerable.Empty(); - - /// - /// Add a document to the list of documents. - /// - /// Name of the uploaded document - /// Size of the uploaded document in bytes - /// Status of the uploaded document - public void AddDocument(string name, string size, bool isUploaded) - { - this.Documents = this.Documents.Append(new DocumentData - { - Name = name, - Size = size, - IsUploaded = isUploaded, - }); - } - - /// - /// Serialize the object to a JSON string. - /// - /// A serialized JSON string - public override string ToString() - { - return JsonSerializer.Serialize(this); - } - - /// - /// Serialize the object to a formatted string. - /// Only successful uploads will be included in the formatted string. - /// - /// A formatted string - public string ToFormattedString() - { - if (!this.Documents.Any()) - { - return string.Empty; - } - - var formattedStrings = this.Documents - .Where(document => document.IsUploaded) - .Select(document => $"[Name: {document.Name}, Size: {document.Size}]").ToList(); - - if (formattedStrings.Count == 1) - { - return $"Uploaded a document {formattedStrings.First()}."; - } - - return $"Uploaded documents: {string.Join(", ", formattedStrings)}."; - } - - /// - /// Serialize the object to a formatted string that only - /// contains document names separated by comma. - /// - /// A formatted string - public string ToFormattedStringNamesOnly() - { - if (!this.Documents.Any()) - { - return string.Empty; - } - - var formattedStrings = this.Documents - .Where(document => document.IsUploaded) - .Select(document => document.Name).ToList(); - - if (formattedStrings.Count == 1) - { - return formattedStrings.First(); - } - - return string.Join(", ", formattedStrings); - } - - /// - /// Deserialize a JSON string to a DocumentMessageContent object. - /// - /// A JSON string - /// A DocumentMessageContent object - public static DocumentMessageContent? FromString(string json) - { - return JsonSerializer.Deserialize(json); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/MemorySource.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/MemorySource.cs deleted file mode 100644 index f531d542287f..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/MemorySource.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Text.Json.Serialization; -using SemanticKernel.Service.CopilotChat.Storage; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// The external memory source. -/// -public class MemorySource : IStorageEntity -{ - /// - /// Type of the memory source. - /// - public enum MemorySourceType - { - // A file source. - File, - } - - /// - /// Source ID that is persistent and unique. - /// - [JsonPropertyName("id")] - public string Id { get; set; } = string.Empty; - - /// - /// The Chat ID. - /// - [JsonPropertyName("chatId")] - public string ChatId { get; set; } = string.Empty; - - /// - /// The type of the source. - /// - [JsonConverter(typeof(JsonStringEnumConverter))] - [JsonPropertyName("sourceType")] - public MemorySourceType SourceType { get; set; } = MemorySourceType.File; - - /// - /// The name of the source. - /// - [JsonPropertyName("name")] - public string Name { get; set; } = string.Empty; - - /// - /// The external link to the source. - /// - [JsonPropertyName("hyperlink")] - public Uri? HyperLink { get; set; } = null; - - /// - /// The user ID of who shared the source. - /// - [JsonPropertyName("sharedBy")] - public string SharedBy { get; set; } = string.Empty; - - /// - /// When the source is created in the bot. - /// - [JsonPropertyName("createdOn")] - public DateTimeOffset CreatedOn { get; set; } - - /// - /// The size of the source in bytes. - /// - [JsonPropertyName("size")] - public long Size { get; set; } - - /// - /// Empty constructor for serialization. - /// - public MemorySource() - { - } - - public MemorySource(string chatId, string name, string sharedBy, MemorySourceType type, long size, Uri? hyperlink) - { - this.Id = Guid.NewGuid().ToString(); - this.ChatId = chatId; - this.Name = name; - this.SourceType = type; - this.HyperLink = hyperlink; - this.SharedBy = sharedBy; - this.CreatedOn = DateTimeOffset.Now; - this.Size = size; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/OpenApiSkillsAuthHeaders.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/OpenApiSkillsAuthHeaders.cs deleted file mode 100644 index bac14f65a7d8..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/OpenApiSkillsAuthHeaders.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.AspNetCore.Mvc; - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// /// -/// Represents the authentication headers for imported OpenAPI Plugin Skills. -/// -public class OpenApiSkillsAuthHeaders -{ - /// - /// Gets or sets the MS Graph authentication header value. - /// - [FromHeader(Name = "x-sk-copilot-graph-auth")] - public string? GraphAuthentication { get; set; } - - /// - /// Gets or sets the Jira authentication header value. - /// - [FromHeader(Name = "x-sk-copilot-jira-auth")] - public string? JiraAuthentication { get; set; } - - /// - /// Gets or sets the GitHub authentication header value. - /// - [FromHeader(Name = "x-sk-copilot-github-auth")] - public string? GithubAuthentication { get; set; } - - /// - /// Gets or sets the Klarna header value. - /// - [FromHeader(Name = "x-sk-copilot-klarna-auth")] - public string? KlarnaAuthentication { get; set; } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ProposedPlan.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ProposedPlan.cs deleted file mode 100644 index f84990b88ce7..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/ProposedPlan.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; -using Microsoft.SemanticKernel.Planning; - -namespace SemanticKernel.Service.CopilotChat.Models; - -// Type of Plan -public enum PlanType -{ - Action, // single-step - Sequential, // multi-step -} - -// State of Plan -public enum PlanState -{ - NoOp, // Plan has not received any user input - Approved, - Rejected, -} - -/// -/// Information about a single proposed plan. -/// -public class ProposedPlan -{ - /// - /// Plan object to be approved or invoked. - /// - [JsonPropertyName("proposedPlan")] - public Plan Plan { get; set; } - - /// - /// Indicates whether plan is Action (single-step) or Sequential (multi-step). - /// - [JsonPropertyName("type")] - public PlanType Type { get; set; } - - /// - /// State of plan - /// - [JsonPropertyName("state")] - public PlanState State { get; set; } - - /// - /// Create a new proposed plan. - /// - /// Proposed plan object - public ProposedPlan(Plan plan, PlanType type, PlanState state) - { - this.Plan = plan; - this.Type = type; - this.State = state; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/SpeechTokenResponse.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/SpeechTokenResponse.cs deleted file mode 100644 index 977fe0de05eb..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Models/SpeechTokenResponse.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace SemanticKernel.Service.CopilotChat.Models; - -/// -/// Token Response is a simple wrapper around the token and region -/// -public class SpeechTokenResponse -{ - public string? Token { get; set; } - public string? Region { get; set; } - public bool? IsSuccess { get; set; } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/AzureSpeechOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/AzureSpeechOptions.cs deleted file mode 100644 index 94a442d230d0..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/AzureSpeechOptions.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Configuration options for Azure speech recognition. -/// -public sealed class AzureSpeechOptions -{ - public const string PropertyName = "AzureSpeech"; - - /// - /// Location of the Azure speech service to use (e.g. "South Central US") - /// - public string? Region { get; set; } = string.Empty; - - /// - /// Key to access the Azure speech service. - /// - public string? Key { get; set; } = string.Empty; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/BotSchemaOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/BotSchemaOptions.cs deleted file mode 100644 index 193b3493535f..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/BotSchemaOptions.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel.DataAnnotations; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Configuration options for the bot file schema that is supported by this application. -/// -public class BotSchemaOptions -{ - public const string PropertyName = "BotSchema"; - - /// - /// The name of the schema. - /// - [Required, NotEmptyOrWhitespace] - public string Name { get; set; } = string.Empty; - - /// - /// The version of the schema. - /// - [Range(0, int.MaxValue)] - public int Version { get; set; } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/ChatStoreOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/ChatStoreOptions.cs deleted file mode 100644 index d0a9860f5193..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/ChatStoreOptions.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Configuration settings for the chat store. -/// -public class ChatStoreOptions -{ - public const string PropertyName = "ChatStore"; - - /// - /// The type of chat store to use. - /// - public enum ChatStoreType - { - /// - /// Non-persistent chat store - /// - Volatile, - - /// - /// File-system based persistent chat store. - /// - Filesystem, - - /// - /// Azure CosmosDB based persistent chat store. - /// - Cosmos - } - - /// - /// Gets or sets the type of chat store to use. - /// - public ChatStoreType Type { get; set; } = ChatStoreType.Volatile; - - /// - /// Gets or sets the configuration for the file system chat store. - /// - [RequiredOnPropertyValue(nameof(Type), ChatStoreType.Filesystem)] - public FileSystemOptions? Filesystem { get; set; } - - /// - /// Gets or sets the configuration for the Azure CosmosDB chat store. - /// - [RequiredOnPropertyValue(nameof(Type), ChatStoreType.Cosmos)] - public CosmosOptions? Cosmos { get; set; } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/CosmosOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/CosmosOptions.cs deleted file mode 100644 index a639dbf49b0c..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/CosmosOptions.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel.DataAnnotations; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Configuration settings for connecting to Azure CosmosDB. -/// -public class CosmosOptions -{ - /// - /// Gets or sets the Cosmos database name. - /// - [Required, NotEmptyOrWhitespace] - public string Database { get; set; } = string.Empty; - - /// - /// Gets or sets the Cosmos connection string. - /// - [Required, NotEmptyOrWhitespace] - public string ConnectionString { get; set; } = string.Empty; - - /// - /// Gets or sets the Cosmos container for chat sessions. - /// - [Required, NotEmptyOrWhitespace] - public string ChatSessionsContainer { get; set; } = string.Empty; - - /// - /// Gets or sets the Cosmos container for chat messages. - /// - [Required, NotEmptyOrWhitespace] - public string ChatMessagesContainer { get; set; } = string.Empty; - - /// - /// Gets or sets the Cosmos container for chat memory sources. - /// - [Required, NotEmptyOrWhitespace] - public string ChatMemorySourcesContainer { get; set; } = string.Empty; - - /// - /// Gets or sets the Cosmos container for chat participants. - /// - [Required, NotEmptyOrWhitespace] - public string ChatParticipantsContainer { get; set; } = string.Empty; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/DocumentMemoryOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/DocumentMemoryOptions.cs deleted file mode 100644 index 3df81b1cc611..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/DocumentMemoryOptions.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel.DataAnnotations; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Configuration options for handling memorized documents. -/// -public class DocumentMemoryOptions -{ - public const string PropertyName = "DocumentMemory"; - - /// - /// Gets or sets the name of the global document collection. - /// - [Required, NotEmptyOrWhitespace] - public string GlobalDocumentCollectionName { get; set; } = "global-documents"; - - /// - /// Gets or sets the prefix for the chat document collection name. - /// - [Required, NotEmptyOrWhitespace] - public string ChatDocumentCollectionNamePrefix { get; set; } = "chat-documents-"; - - /// - /// Gets or sets the maximum number of tokens to use when splitting a document into lines. - /// Default token limits are suggested by OpenAI: - /// https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them - /// - [Range(0, int.MaxValue)] - public int DocumentLineSplitMaxTokens { get; set; } = 30; - - /// - /// Gets or sets the maximum number of lines to use when combining lines into paragraphs. - /// Default token limits are suggested by OpenAI: - /// https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them - /// - [Range(0, int.MaxValue)] - public int DocumentParagraphSplitMaxLines { get; set; } = 100; - - /// - /// Maximum size in bytes of a document to be allowed for importing. - /// Prevent large uploads by setting a file size limit (in bytes) as suggested here: - /// https://learn.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads?view=aspnetcore-6.0 - /// - [Range(0, int.MaxValue)] - public int FileSizeLimit { get; set; } = 1000000; - - /// - /// Maximum number of files to be allowed for importing in a single request. - /// - [Range(0, int.MaxValue)] - public int FileCountLimit { get; set; } = 10; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/FileSystemOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/FileSystemOptions.cs deleted file mode 100644 index e4bb92b6b1a0..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/FileSystemOptions.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel.DataAnnotations; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// File system storage configuration. -/// -public class FileSystemOptions -{ - /// - /// Gets or sets the file path for persistent file system storage. - /// - [Required, NotEmptyOrWhitespace] - public string FilePath { get; set; } = string.Empty; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/OcrSupportOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/OcrSupportOptions.cs deleted file mode 100644 index a1744b47b8c6..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/OcrSupportOptions.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Ocr Support Configuration Options -/// -public class OcrSupportOptions -{ - public const string PropertyName = "OcrSupport"; - - public enum OcrSupportType - { - /// - /// No OCR Support - /// - None, - - /// - /// Tesseract OCR Support - /// - Tesseract - } - - /// - /// Gets or sets the type of OCR support to use. - /// - public OcrSupportType Type { get; set; } = OcrSupportType.None; - - /// - /// Gets or sets the configuration for the Tesseract OCR support. - /// - [RequiredOnPropertyValue(nameof(Type), OcrSupportType.Tesseract)] - public TesseractOptions? Tesseract { get; set; } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/PlannerOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/PlannerOptions.cs deleted file mode 100644 index c2827035add9..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/PlannerOptions.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -using System.ComponentModel.DataAnnotations; -using SemanticKernel.Service.CopilotChat.Models; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Configuration options for the planner. -/// -public class PlannerOptions -{ - public const string PropertyName = "Planner"; - - /// - /// Define if the planner must be Sequential or not. - /// - [Required] - public PlanType Type { get; set; } = PlanType.Action; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/PromptsOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/PromptsOptions.cs deleted file mode 100644 index 3b83b30adfff..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/PromptsOptions.cs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Configuration options for the chat -/// -public class PromptsOptions -{ - public const string PropertyName = "Prompts"; - - /// - /// Token limit of the chat model. - /// - /// https://platform.openai.com/docs/models/overview for token limits. - [Required, Range(0, int.MaxValue)] public int CompletionTokenLimit { get; set; } - - /// - /// The token count left for the model to generate text after the prompt. - /// - [Required, Range(0, int.MaxValue)] public int ResponseTokenLimit { get; set; } - - /// - /// Weight of memories in the contextual part of the final prompt. - /// Contextual prompt excludes all the system commands and user intent. - /// - internal double MemoriesResponseContextWeight { get; } = 0.3; - - /// - /// Weight of documents in the contextual part of the final prompt. - /// Contextual prompt excludes all the system commands and user intent. - /// - internal double DocumentContextWeight { get; } = 0.3; - - /// - /// Weight of information returned from planner (i.e., responses from OpenAPI skills). - /// Contextual prompt excludes all the system commands and user intent. - /// - internal double ExternalInformationContextWeight { get; } = 0.3; - - /// - /// Minimum relevance of a semantic memory to be included in the final prompt. - /// The higher the value, the answer will be more relevant to the user intent. - /// - internal double SemanticMemoryMinRelevance { get; } = 0.8; - - /// - /// Minimum relevance of a document memory to be included in the final prompt. - /// The higher the value, the answer will be more relevant to the user intent. - /// - internal double DocumentMemoryMinRelevance { get; } = 0.8; - - // System - [Required, NotEmptyOrWhitespace] public string KnowledgeCutoffDate { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string InitialBotMessage { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string SystemDescription { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string SystemResponse { get; set; } = string.Empty; - - internal string[] SystemAudiencePromptComponents => new string[] - { - this.SystemAudience, - "{{ChatSkill.ExtractChatHistory}}", - this.SystemAudienceContinuation - }; - - internal string SystemAudienceExtraction => string.Join("\n", this.SystemAudiencePromptComponents); - - internal string[] SystemIntentPromptComponents => new string[] - { - this.SystemDescription, - this.SystemIntent, - "{{ChatSkill.ExtractChatHistory}}", - this.SystemIntentContinuation - }; - - internal string SystemIntentExtraction => string.Join("\n", this.SystemIntentPromptComponents); - - // Intent extraction - [Required, NotEmptyOrWhitespace] public string SystemIntent { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string SystemIntentContinuation { get; set; } = string.Empty; - - // Audience extraction - [Required, NotEmptyOrWhitespace] public string SystemAudience { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string SystemAudienceContinuation { get; set; } = string.Empty; - - // Memory extraction - [Required, NotEmptyOrWhitespace] public string SystemCognitive { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string MemoryFormat { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string MemoryAntiHallucination { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string MemoryContinuation { get; set; } = string.Empty; - - // Long-term memory - [Required, NotEmptyOrWhitespace] public string LongTermMemoryName { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string LongTermMemoryExtraction { get; set; } = string.Empty; - - internal string[] LongTermMemoryPromptComponents => new string[] - { - this.SystemCognitive, - $"{this.LongTermMemoryName} Description:\n{this.LongTermMemoryExtraction}", - this.MemoryAntiHallucination, - $"Chat Description:\n{this.SystemDescription}", - "{{ChatSkill.ExtractChatHistory}}", - this.MemoryContinuation - }; - - internal string LongTermMemory => string.Join("\n", this.LongTermMemoryPromptComponents); - - // Working memory - [Required, NotEmptyOrWhitespace] public string WorkingMemoryName { get; set; } = string.Empty; - [Required, NotEmptyOrWhitespace] public string WorkingMemoryExtraction { get; set; } = string.Empty; - - internal string[] WorkingMemoryPromptComponents => new string[] - { - this.SystemCognitive, - $"{this.WorkingMemoryName} Description:\n{this.WorkingMemoryExtraction}", - this.MemoryAntiHallucination, - $"Chat Description:\n{this.SystemDescription}", - "{{ChatSkill.ExtractChatHistory}}", - this.MemoryContinuation - }; - - internal string WorkingMemory => string.Join("\n", this.WorkingMemoryPromptComponents); - - // Memory map - internal IDictionary MemoryMap => new Dictionary() - { - { this.LongTermMemoryName, this.LongTermMemory }, - { this.WorkingMemoryName, this.WorkingMemory } - }; - - // Chat commands - internal string SystemChatContinuation = "SINGLE RESPONSE FROM BOT TO USER:\n[{{TimeSkill.Now}} {{timeSkill.Second}}] bot:"; - - internal string[] SystemChatPromptComponents => new string[] - { - this.SystemDescription, - this.SystemResponse, - "{{$audience}}", - "{{$userIntent}}", - "{{$chatContext}}", - this.SystemChatContinuation - }; - - internal string SystemChatPrompt => string.Join("\n\n", this.SystemChatPromptComponents); - - internal double ResponseTemperature { get; } = 0.7; - internal double ResponseTopP { get; } = 1; - internal double ResponsePresencePenalty { get; } = 0.5; - internal double ResponseFrequencyPenalty { get; } = 0.5; - - internal double IntentTemperature { get; } = 0.7; - internal double IntentTopP { get; } = 1; - internal double IntentPresencePenalty { get; } = 0.5; - internal double IntentFrequencyPenalty { get; } = 0.5; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/TesseractOptions.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/TesseractOptions.cs deleted file mode 100644 index 0fe50f104667..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Options/TesseractOptions.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel.DataAnnotations; -using SemanticKernel.Service.Options; - -namespace SemanticKernel.Service.CopilotChat.Options; - -/// -/// Configuration options for Tesseract OCR support. -/// -public sealed class TesseractOptions -{ - public const string PropertyName = "Tesseract"; - - /// - /// The file path where the Tesseract language file is stored (e.g. "./data") - /// - [Required, NotEmptyOrWhitespace] - public string? FilePath { get; set; } = string.Empty; - - /// - /// The language file prefix name (e.g. "eng") - /// - [Required, NotEmptyOrWhitespace] - public string? Language { get; set; } = string.Empty; -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/ChatSkill.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/ChatSkill.cs deleted file mode 100644 index 2cf2fd0c16ba..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/ChatSkill.cs +++ /dev/null @@ -1,632 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Globalization; -using System.Linq; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.TemplateEngine; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Options; -using SemanticKernel.Service.CopilotChat.Storage; - -namespace SemanticKernel.Service.CopilotChat.Skills.ChatSkills; - -/// -/// ChatSkill offers a more coherent chat experience by using memories -/// to extract conversation history and user intentions. -/// -public class ChatSkill -{ - /// - /// A kernel instance to create a completion function since each invocation - /// of the function will generate a new prompt dynamically. - /// - private readonly IKernel _kernel; - - /// - /// A repository to save and retrieve chat messages. - /// - private readonly ChatMessageRepository _chatMessageRepository; - - /// - /// A repository to save and retrieve chat sessions. - /// - private readonly ChatSessionRepository _chatSessionRepository; - - /// - /// Settings containing prompt texts. - /// - private readonly PromptsOptions _promptOptions; - - /// - /// A semantic chat memory skill instance to query semantic memories. - /// - private readonly SemanticChatMemorySkill _semanticChatMemorySkill; - - /// - /// A document memory skill instance to query document memories. - /// - private readonly DocumentMemorySkill _documentMemorySkill; - - /// - /// A skill instance to acquire external information. - /// - private readonly ExternalInformationSkill _externalInformationSkill; - - /// - /// Create a new instance of . - /// - public ChatSkill( - IKernel kernel, - ChatMessageRepository chatMessageRepository, - ChatSessionRepository chatSessionRepository, - IOptions promptOptions, - IOptions documentImportOptions, - CopilotChatPlanner planner, - ILogger logger) - { - this._kernel = kernel; - this._chatMessageRepository = chatMessageRepository; - this._chatSessionRepository = chatSessionRepository; - this._promptOptions = promptOptions.Value; - - this._semanticChatMemorySkill = new SemanticChatMemorySkill( - promptOptions); - this._documentMemorySkill = new DocumentMemorySkill( - promptOptions, - documentImportOptions); - this._externalInformationSkill = new ExternalInformationSkill( - promptOptions, - planner); - } - - /// - /// Extract user intent from the conversation history. - /// - /// The SKContext. - [SKFunction, Description("Extract user intent")] - [SKParameter("chatId", "Chat ID to extract history from")] - [SKParameter("audience", "The audience the chat bot is interacting with.")] - public async Task ExtractUserIntentAsync(SKContext context) - { - var tokenLimit = this._promptOptions.CompletionTokenLimit; - var historyTokenBudget = - tokenLimit - - this._promptOptions.ResponseTokenLimit - - Utilities.TokenCount(string.Join("\n", new string[] - { - this._promptOptions.SystemDescription, - this._promptOptions.SystemIntent, - this._promptOptions.SystemIntentContinuation - }) - ); - - // Clone the context to avoid modifying the original context variables. - var intentExtractionContext = Utilities.CopyContextWithVariablesClone(context); - intentExtractionContext.Variables.Set("tokenLimit", historyTokenBudget.ToString(new NumberFormatInfo())); - intentExtractionContext.Variables.Set("knowledgeCutoff", this._promptOptions.KnowledgeCutoffDate); - - var completionFunction = this._kernel.CreateSemanticFunction( - this._promptOptions.SystemIntentExtraction, - skillName: nameof(ChatSkill), - description: "Complete the prompt."); - - var result = await completionFunction.InvokeAsync( - intentExtractionContext, - settings: this.CreateIntentCompletionSettings() - ); - - if (result.ErrorOccurred) - { - context.Log.LogError("{0}: {1}", result.LastErrorDescription, result.LastException); - context.Fail(result.LastErrorDescription); - return string.Empty; - } - - return $"User intent: {result}"; - } - - /// - /// Extract the list of participants from the conversation history. - /// Note that only those who have spoken will be included. - /// - [SKFunction, Description("Extract audience list")] - [SKParameter("chatId", "Chat ID to extract history from")] - public async Task ExtractAudienceAsync(SKContext context) - { - var tokenLimit = this._promptOptions.CompletionTokenLimit; - var historyTokenBudget = - tokenLimit - - this._promptOptions.ResponseTokenLimit - - Utilities.TokenCount(string.Join("\n", new string[] - { - this._promptOptions.SystemAudience, - this._promptOptions.SystemAudienceContinuation, - }) - ); - - // Clone the context to avoid modifying the original context variables. - var audienceExtractionContext = Utilities.CopyContextWithVariablesClone(context); - audienceExtractionContext.Variables.Set("tokenLimit", historyTokenBudget.ToString(new NumberFormatInfo())); - - var completionFunction = this._kernel.CreateSemanticFunction( - this._promptOptions.SystemAudienceExtraction, - skillName: nameof(ChatSkill), - description: "Complete the prompt."); - - var result = await completionFunction.InvokeAsync( - audienceExtractionContext, - settings: this.CreateIntentCompletionSettings() - ); - - if (result.ErrorOccurred) - { - context.Log.LogError("{0}: {1}", result.LastErrorDescription, result.LastException); - context.Fail(result.LastErrorDescription); - return string.Empty; - } - - return $"List of participants: {result}"; - } - - /// - /// Extract chat history. - /// - /// Contains the 'tokenLimit' controlling the length of the prompt. - [SKFunction, Description("Extract chat history")] - public async Task ExtractChatHistoryAsync( - [Description("Chat ID to extract history from")] string chatId, - [Description("Maximum number of tokens")] int tokenLimit) - { - var messages = await this._chatMessageRepository.FindByChatIdAsync(chatId); - var sortedMessages = messages.OrderByDescending(m => m.Timestamp); - - var remainingToken = tokenLimit; - - string historyText = ""; - foreach (var chatMessage in sortedMessages) - { - var formattedMessage = chatMessage.ToFormattedString(); - - // Plan object is not meaningful content in generating bot response, so shorten to intent only to save on tokens - if (formattedMessage.Contains("proposedPlan\":", StringComparison.InvariantCultureIgnoreCase)) - { - string pattern = @"(\[.*?\]).*User Intent:User intent: (.*)(?=""}})"; - Match match = Regex.Match(formattedMessage, pattern); - if (match.Success) - { - string timestamp = match.Groups[1].Value.Trim(); - string userIntent = match.Groups[2].Value.Trim(); - - formattedMessage = $"{timestamp} Bot proposed plan to fulfill user intent: {userIntent}"; - } - else - { - formattedMessage = "Bot proposed plan"; - } - } - - var tokenCount = Utilities.TokenCount(formattedMessage); - - if (remainingToken - tokenCount >= 0) - { - historyText = $"{formattedMessage}\n{historyText}"; - remainingToken -= tokenCount; - } - else - { - break; - } - } - - return $"Chat history:\n{historyText.Trim()}"; - } - - /// - /// This is the entry point for getting a chat response. It manages the token limit, saves - /// messages to memory, and fill in the necessary context variables for completing the - /// prompt that will be rendered by the template engine. - /// - [SKFunction, Description("Get chat response")] - public async Task ChatAsync( - [Description("The new message")] string message, - [Description("Unique and persistent identifier for the user")] string userId, - [Description("Name of the user")] string userName, - [Description("Unique and persistent identifier for the chat")] string chatId, - [Description("Type of the message")] string messageType, - [Description("Previously proposed plan that is approved"), DefaultValue(null), SKName("proposedPlan")] string? planJson, - [Description("ID of the response message for planner"), DefaultValue(null), SKName("responseMessageId")] string? messageId, - SKContext context) - { - // Save this new message to memory such that subsequent chat responses can use it - await this.SaveNewMessageAsync(message, userId, userName, chatId, messageType); - - // Clone the context to avoid modifying the original context variables. - var chatContext = Utilities.CopyContextWithVariablesClone(context); - chatContext.Variables.Set("knowledgeCutoff", this._promptOptions.KnowledgeCutoffDate); - - // Check if plan exists in ask's context variables. - // If plan was returned at this point, that means it was approved or cancelled. - // Update the response previously saved in chat history with state - if (!string.IsNullOrWhiteSpace(planJson) && - !string.IsNullOrEmpty(messageId)) - { - await this.UpdateResponseAsync(planJson, messageId); - } - - var response = chatContext.Variables.ContainsKey("userCancelledPlan") - ? "I am sorry the plan did not meet your goals." - : await this.GetChatResponseAsync(chatId, chatContext); - - if (chatContext.ErrorOccurred) - { - context.Fail(chatContext.LastErrorDescription); - return context; - } - - // Retrieve the prompt used to generate the response - // and return it to the caller via the context variables. - chatContext.Variables.TryGetValue("prompt", out string? prompt); - prompt ??= string.Empty; - context.Variables.Set("prompt", prompt); - - // Save this response to memory such that subsequent chat responses can use it - ChatMessage botMessage = await this.SaveNewResponseAsync(response, prompt, chatId); - context.Variables.Set("messageId", botMessage.Id); - context.Variables.Set("messageType", ((int)botMessage.Type).ToString(CultureInfo.InvariantCulture)); - - // Extract semantic chat memory - await SemanticChatMemoryExtractor.ExtractSemanticChatMemoryAsync( - chatId, - this._kernel, - chatContext, - this._promptOptions); - - context.Variables.Update(response); - return context; - } - - #region Private - - /// - /// Generate the necessary chat context to create a prompt then invoke the model to get a response. - /// - /// The SKContext. - /// A response from the model. - private async Task GetChatResponseAsync(string chatId, SKContext chatContext) - { - // 0. Get the audience - var audience = await this.GetAudienceAsync(chatContext); - if (chatContext.ErrorOccurred) - { - return string.Empty; - } - - // 1. Extract user intent from the conversation history. - var userIntent = await this.GetUserIntentAsync(chatContext); - if (chatContext.ErrorOccurred) - { - return string.Empty; - } - - // 2. Calculate the remaining token budget. - var remainingToken = this.GetChatContextTokenLimit(userIntent); - - // 3. Acquire external information from planner - var externalInformationTokenLimit = (int)(remainingToken * this._promptOptions.ExternalInformationContextWeight); - var planResult = await this.AcquireExternalInformationAsync(chatContext, userIntent, externalInformationTokenLimit); - if (chatContext.ErrorOccurred) - { - return string.Empty; - } - - // If plan is suggested, send back to user for approval before running - if (this._externalInformationSkill.ProposedPlan != null) - { - return JsonSerializer.Serialize(this._externalInformationSkill.ProposedPlan); - } - - // 4. Query relevant semantic memories - var chatMemoriesTokenLimit = (int)(remainingToken * this._promptOptions.MemoriesResponseContextWeight); - var chatMemories = await this._semanticChatMemorySkill.QueryMemoriesAsync(userIntent, chatId, chatMemoriesTokenLimit, chatContext.Memory); - if (chatContext.ErrorOccurred) - { - return string.Empty; - } - - // 5. Query relevant document memories - var documentContextTokenLimit = (int)(remainingToken * this._promptOptions.DocumentContextWeight); - var documentMemories = await this._documentMemorySkill.QueryDocumentsAsync(userIntent, chatId, documentContextTokenLimit, chatContext.Memory); - if (chatContext.ErrorOccurred) - { - return string.Empty; - } - - // 6. Fill in the chat history if there is any token budget left - var chatContextComponents = new List() { chatMemories, documentMemories, planResult }; - var chatContextText = string.Join("\n\n", chatContextComponents.Where(c => !string.IsNullOrEmpty(c))); - var chatContextTextTokenCount = remainingToken - Utilities.TokenCount(chatContextText); - if (chatContextTextTokenCount > 0) - { - var chatHistory = await this.ExtractChatHistoryAsync(chatId, chatContextTextTokenCount); - if (chatContext.ErrorOccurred) - { - return string.Empty; - } - chatContextText = $"{chatContextText}\n{chatHistory}"; - } - - // Invoke the model - chatContext.Variables.Set("audience", audience); - chatContext.Variables.Set("UserIntent", userIntent); - chatContext.Variables.Set("ChatContext", chatContextText); - - var promptRenderer = new PromptTemplateEngine(); - var renderedPrompt = await promptRenderer.RenderAsync( - this._promptOptions.SystemChatPrompt, - chatContext); - - var completionFunction = this._kernel.CreateSemanticFunction( - renderedPrompt, - skillName: nameof(ChatSkill), - description: "Complete the prompt."); - - chatContext = await completionFunction.InvokeAsync( - context: chatContext, - settings: this.CreateChatResponseCompletionSettings() - ); - - // Allow the caller to view the prompt used to generate the response - chatContext.Variables.Set("prompt", renderedPrompt); - - if (chatContext.ErrorOccurred) - { - return string.Empty; - } - - return chatContext.Result; - } - - /// - /// Helper function create the correct context variables to - /// extract audience from the conversation history. - /// - private async Task GetAudienceAsync(SKContext context) - { - var contextVariables = new ContextVariables(); - contextVariables.Set("chatId", context["chatId"]); - - var audienceContext = new SKContext( - contextVariables, - context.Memory, - context.Skills, - context.Log, - context.CancellationToken - ); - - var audience = await this.ExtractAudienceAsync(audienceContext); - - // Propagate the error - if (audienceContext.ErrorOccurred) - { - context.Fail(audienceContext.LastErrorDescription); - } - - return audience; - } - - /// - /// Helper function create the correct context variables to - /// extract user intent from the conversation history. - /// - private async Task GetUserIntentAsync(SKContext context) - { - // TODO: Regenerate user intent if plan was modified - if (!context.Variables.TryGetValue("planUserIntent", out string? userIntent)) - { - var contextVariables = new ContextVariables(); - contextVariables.Set("chatId", context["chatId"]); - contextVariables.Set("audience", context["userName"]); - - var intentContext = new SKContext( - contextVariables, - context.Memory, - context.Skills, - context.Log, - context.CancellationToken - ); - - userIntent = await this.ExtractUserIntentAsync(intentContext); - // Propagate the error - if (intentContext.ErrorOccurred) - { - context.Fail(intentContext.LastErrorDescription); - } - } - - return userIntent; - } - - /// - /// Helper function create the correct context variables to - /// query chat memories from the chat memory store. - /// - private Task QueryChatMemoriesAsync(SKContext context, string userIntent, int tokenLimit) - { - return this._semanticChatMemorySkill.QueryMemoriesAsync(userIntent, context["chatId"], tokenLimit, context.Memory); - } - - /// - /// Helper function create the correct context variables to - /// query document memories from the document memory store. - /// - private Task QueryDocumentsAsync(SKContext context, string userIntent, int tokenLimit) - { - return this._documentMemorySkill.QueryDocumentsAsync(userIntent, context["chatId"], tokenLimit, context.Memory); - } - - /// - /// Helper function create the correct context variables to acquire external information. - /// - private async Task AcquireExternalInformationAsync(SKContext context, string userIntent, int tokenLimit) - { - var contextVariables = context.Variables.Clone(); - contextVariables.Set("tokenLimit", tokenLimit.ToString(new NumberFormatInfo())); - - var planContext = new SKContext( - contextVariables, - context.Memory, - context.Skills, - context.Log, - context.CancellationToken - ); - - var plan = await this._externalInformationSkill.AcquireExternalInformationAsync(userIntent, planContext); - - // Propagate the error - if (planContext.ErrorOccurred) - { - context.Fail(planContext.LastErrorDescription); - } - - return plan; - } - - /// - /// Save a new message to the chat history. - /// - /// The message - /// The user ID - /// - /// The chat ID - /// Type of the message - private async Task SaveNewMessageAsync(string message, string userId, string userName, string chatId, string type) - { - // Make sure the chat exists. - if (!await this._chatSessionRepository.TryFindByIdAsync(chatId, v => _ = v)) - { - throw new ArgumentException("Chat session does not exist."); - } - - var chatMessage = new ChatMessage( - userId, - userName, - chatId, - message, - "", - ChatMessage.AuthorRoles.User, - // Default to a standard message if the `type` is not recognized - Enum.TryParse(type, out ChatMessage.ChatMessageType typeAsEnum) && Enum.IsDefined(typeof(ChatMessage.ChatMessageType), typeAsEnum) - ? typeAsEnum - : ChatMessage.ChatMessageType.Message); - - await this._chatMessageRepository.CreateAsync(chatMessage); - return chatMessage; - } - - /// - /// Save a new response to the chat history. - /// - /// Response from the chat. - /// Prompt used to generate the response. - /// The chat ID - /// The created chat message. - private async Task SaveNewResponseAsync(string response, string prompt, string chatId) - { - // Make sure the chat exists. - if (!await this._chatSessionRepository.TryFindByIdAsync(chatId, v => _ = v)) - { - throw new ArgumentException("Chat session does not exist."); - } - - var chatMessage = ChatMessage.CreateBotResponseMessage(chatId, response, prompt); - await this._chatMessageRepository.CreateAsync(chatMessage); - - return chatMessage; - } - - /// - /// Updates previously saved response in the chat history. - /// - /// Updated response from the chat. - /// The chat message ID - private async Task UpdateResponseAsync(string updatedResponse, string messageId) - { - // Make sure the chat exists. - var chatMessage = await this._chatMessageRepository.FindByIdAsync(messageId); - chatMessage.Content = updatedResponse; - - await this._chatMessageRepository.UpsertAsync(chatMessage); - } - - /// - /// Create a completion settings object for chat response. Parameters are read from the PromptSettings class. - /// - private CompleteRequestSettings CreateChatResponseCompletionSettings() - { - var completionSettings = new CompleteRequestSettings - { - MaxTokens = this._promptOptions.ResponseTokenLimit, - Temperature = this._promptOptions.ResponseTemperature, - TopP = this._promptOptions.ResponseTopP, - FrequencyPenalty = this._promptOptions.ResponseFrequencyPenalty, - PresencePenalty = this._promptOptions.ResponsePresencePenalty - }; - - return completionSettings; - } - - /// - /// Create a completion settings object for intent response. Parameters are read from the PromptSettings class. - /// - private CompleteRequestSettings CreateIntentCompletionSettings() - { - var completionSettings = new CompleteRequestSettings - { - MaxTokens = this._promptOptions.ResponseTokenLimit, - Temperature = this._promptOptions.IntentTemperature, - TopP = this._promptOptions.IntentTopP, - FrequencyPenalty = this._promptOptions.IntentFrequencyPenalty, - PresencePenalty = this._promptOptions.IntentPresencePenalty, - StopSequences = new string[] { "] bot:" } - }; - - return completionSettings; - } - - /// - /// Calculate the remaining token budget for the chat response prompt. - /// This is the token limit minus the token count of the user intent and the system commands. - /// - /// The user intent returned by the model. - /// The remaining token limit. - private int GetChatContextTokenLimit(string userIntent) - { - var tokenLimit = this._promptOptions.CompletionTokenLimit; - var remainingToken = - tokenLimit - - Utilities.TokenCount(userIntent) - - this._promptOptions.ResponseTokenLimit - - Utilities.TokenCount(string.Join("\n", new string[] - { - this._promptOptions.SystemDescription, - this._promptOptions.SystemResponse, - this._promptOptions.SystemChatContinuation - }) - ); - - return remainingToken; - } - - # endregion -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/CopilotChatPlanner.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/CopilotChatPlanner.cs deleted file mode 100644 index f41106acee89..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/CopilotChatPlanner.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SkillDefinition; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Options; - -namespace SemanticKernel.Service.CopilotChat.Skills.ChatSkills; - -/// -/// A lightweight wrapper around a planner to allow for curating which skills are available to it. -/// -public class CopilotChatPlanner -{ - /// - /// The planner's kernel. - /// - public IKernel Kernel { get; } - - /// - /// Options for the planner. - /// - private readonly PlannerOptions? _plannerOptions; - - /// - /// Gets the pptions for the planner. - /// - public PlannerOptions? PlannerOptions => this._plannerOptions; - - /// - /// Initializes a new instance of the class. - /// - /// The planner's kernel. - public CopilotChatPlanner(IKernel plannerKernel, PlannerOptions? plannerOptions) - { - this.Kernel = plannerKernel; - this._plannerOptions = plannerOptions; - } - - /// - /// Create a plan for a goal. - /// - /// The goal to create a plan for. - /// The plan. - public Task CreatePlanAsync(string goal) - { - FunctionsView plannerFunctionsView = this.Kernel.Skills.GetFunctionsView(true, true); - if (plannerFunctionsView.NativeFunctions.IsEmpty && plannerFunctionsView.SemanticFunctions.IsEmpty) - { - // No functions are available - return an empty plan. - return Task.FromResult(new Plan(goal)); - } - - if (this._plannerOptions?.Type == PlanType.Sequential) - { - return new SequentialPlanner(this.Kernel).CreatePlanAsync(goal); - } - - return new ActionPlanner(this.Kernel).CreatePlanAsync(goal); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/DocumentMemorySkill.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/DocumentMemorySkill.cs deleted file mode 100644 index ac9f8793808a..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/DocumentMemorySkill.cs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.Options; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.SkillDefinition; -using SemanticKernel.Service.CopilotChat.Options; - -namespace SemanticKernel.Service.CopilotChat.Skills.ChatSkills; - -/// -/// This skill provides the functions to query the document memory. -/// -public class DocumentMemorySkill -{ - /// - /// Prompt settings. - /// - private readonly PromptsOptions _promptOptions; - - /// - /// Configuration settings for importing documents to memory. - /// - private readonly DocumentMemoryOptions _documentImportOptions; - - /// - /// Create a new instance of DocumentMemorySkill. - /// - public DocumentMemorySkill( - IOptions promptOptions, - IOptions documentImportOptions) - { - this._promptOptions = promptOptions.Value; - this._documentImportOptions = documentImportOptions.Value; - } - - /// - /// Query the document memory collection for documents that match the query. - /// - /// Query to match. - /// The SkContext. - [SKFunction, Description("Query documents in the memory given a user message")] - public async Task QueryDocumentsAsync( - [Description("Query to match.")] string query, - [Description("ID of the chat that owns the documents")] string chatId, - [Description("Maximum number of tokens")] int tokenLimit, - ISemanticTextMemory textMemory) - { - var remainingToken = tokenLimit; - - // Search for relevant document snippets. - string[] documentCollections = new string[] - { - this._documentImportOptions.ChatDocumentCollectionNamePrefix + chatId, - this._documentImportOptions.GlobalDocumentCollectionName - }; - - List relevantMemories = new(); - foreach (var documentCollection in documentCollections) - { - var results = textMemory.SearchAsync( - documentCollection, - query, - limit: 100, - minRelevanceScore: this._promptOptions.DocumentMemoryMinRelevance); - await foreach (var memory in results) - { - relevantMemories.Add(memory); - } - } - - relevantMemories = relevantMemories.OrderByDescending(m => m.Relevance).ToList(); - - // Concatenate the relevant document snippets. - string documentsText = string.Empty; - foreach (var memory in relevantMemories) - { - var tokenCount = Utilities.TokenCount(memory.Metadata.Text); - if (remainingToken - tokenCount > 0) - { - documentsText += $"\n\nSnippet from {memory.Metadata.Description}: {memory.Metadata.Text}"; - remainingToken -= tokenCount; - } - else - { - break; - } - } - - if (string.IsNullOrEmpty(documentsText)) - { - // No relevant documents found - return string.Empty; - } - - return $"User has also shared some document snippets:\n{documentsText}"; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/ExternalInformationSkill.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/ExternalInformationSkill.cs deleted file mode 100644 index 9fea94f6220a..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/ExternalInformationSkill.cs +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Globalization; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SkillDefinition; -using SemanticKernel.Service.CopilotChat.Models; -using SemanticKernel.Service.CopilotChat.Options; -using SemanticKernel.Service.CopilotChat.Skills.OpenApiSkills.GitHubSkill.Model; -using SemanticKernel.Service.CopilotChat.Skills.OpenApiSkills.JiraSkill.Model; - -namespace SemanticKernel.Service.CopilotChat.Skills.ChatSkills; - -/// -/// This skill provides the functions to acquire external information. -/// -public class ExternalInformationSkill -{ - /// - /// Prompt settings. - /// - private readonly PromptsOptions _promptOptions; - - /// - /// CopilotChat's planner to gather additional information for the chat context. - /// - private readonly CopilotChatPlanner _planner; - - /// - /// Proposed plan to return for approval. - /// - public ProposedPlan? ProposedPlan { get; private set; } - - /// - /// Preamble to add to the related information text. - /// - private const string PromptPreamble = "[RELATED START]"; - - /// - /// Postamble to add to the related information text. - /// - private const string PromptPostamble = "[RELATED END]"; - - /// - /// Create a new instance of ExternalInformationSkill. - /// - public ExternalInformationSkill( - IOptions promptOptions, - CopilotChatPlanner planner) - { - this._promptOptions = promptOptions.Value; - this._planner = planner; - } - - /// - /// Extract relevant additional knowledge using a planner. - /// - [SKFunction, Description("Acquire external information")] - [SKParameter("tokenLimit", "Maximum number of tokens")] - [SKParameter("proposedPlan", "Previously proposed plan that is approved")] - public async Task AcquireExternalInformationAsync( - [Description("The intent to whether external information is needed")] string userIntent, - SKContext context) - { - FunctionsView functions = this._planner.Kernel.Skills.GetFunctionsView(true, true); - if (functions.NativeFunctions.IsEmpty && functions.SemanticFunctions.IsEmpty) - { - return string.Empty; - } - - // Check if plan exists in ask's context variables. - var planExists = context.Variables.TryGetValue("proposedPlan", out string? proposedPlanJson); - var deserializedPlan = planExists && !string.IsNullOrWhiteSpace(proposedPlanJson) ? JsonSerializer.Deserialize(proposedPlanJson) : null; - - // Run plan if it was approved - if (deserializedPlan != null && deserializedPlan.State == PlanState.Approved) - { - string planJson = JsonSerializer.Serialize(deserializedPlan.Plan); - // Reload the plan with the planner's kernel so - // it has full context to be executed - var newPlanContext = new SKContext( - null, - this._planner.Kernel.Memory, - this._planner.Kernel.Skills, - this._planner.Kernel.Log - ); - var plan = Plan.FromJson(planJson, newPlanContext); - - // Invoke plan - newPlanContext = await plan.InvokeAsync(newPlanContext); - int tokenLimit = - int.Parse(context["tokenLimit"], new NumberFormatInfo()) - - Utilities.TokenCount(PromptPreamble) - - Utilities.TokenCount(PromptPostamble); - - // The result of the plan may be from an OpenAPI skill. Attempt to extract JSON from the response. - bool extractJsonFromOpenApi = - this.TryExtractJsonFromOpenApiPlanResult(newPlanContext, newPlanContext.Result, out string planResult); - if (extractJsonFromOpenApi) - { - planResult = this.OptimizeOpenApiSkillJson(planResult, tokenLimit, plan); - } - else - { - // If not, use result of the plan execution result directly. - planResult = newPlanContext.Variables.Input; - } - - return $"{PromptPreamble}\n{planResult.Trim()}\n{PromptPostamble}\n"; - } - else - { - // Create a plan and set it in context for approval. - var contextString = string.Join("\n", context.Variables.Where(v => v.Key != "userIntent").Select(v => $"{v.Key}: {v.Value}")); - Plan plan = await this._planner.CreatePlanAsync($"Given the following context, accomplish the user intent.\nContext:{contextString}\nUser Intent:{userIntent}"); - - if (plan.Steps.Count > 0) - { - // Parameters stored in plan's top level - this.MergeContextIntoPlan(context.Variables, plan.Parameters); - - // TODO: Improve Kernel to give developers option to skip this override - // (i.e., keep functions regardless of whether they're available in the planner's context or not) - Plan sanitizedPlan = this.SanitizePlan(plan, context); - sanitizedPlan.Parameters.Update(plan.Parameters); - - this.ProposedPlan = new ProposedPlan(sanitizedPlan, this._planner.PlannerOptions!.Type, PlanState.NoOp); - } - } - - return string.Empty; - } - - #region Private - - /// - /// Scrubs plan of functions not available in Planner's kernel. - /// - private Plan SanitizePlan(Plan plan, SKContext context) - { - List sanitizedSteps = new(); - var availableFunctions = this._planner.Kernel.Skills.GetFunctionsView(true); - - foreach (var step in plan.Steps) - { - if (this._planner.Kernel.Skills.TryGetFunction(step.SkillName, step.Name, out var function)) - { - this.MergeContextIntoPlan(context.Variables, step.Parameters); - sanitizedSteps.Add(step); - } - } - - return new Plan(plan.Description, sanitizedSteps.ToArray()); - } - - /// - /// Merge any variables from the context into plan parameters as these will be used on plan execution. - /// These context variables come from user input, so they are prioritized. - /// - private void MergeContextIntoPlan(ContextVariables variables, ContextVariables planParams) - { - foreach (var param in planParams) - { - if (param.Key.Equals("INPUT", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (variables.TryGetValue(param.Key, out string? value)) - { - planParams.Set(param.Key, value); - } - } - } - - /// - /// Try to extract json from the planner response as if it were from an OpenAPI skill. - /// - private bool TryExtractJsonFromOpenApiPlanResult(SKContext context, string openApiSkillResponse, out string json) - { - try - { - JsonNode? jsonNode = JsonNode.Parse(openApiSkillResponse); - string contentType = jsonNode?["contentType"]?.ToString() ?? string.Empty; - if (contentType.StartsWith("application/json", StringComparison.InvariantCultureIgnoreCase)) - { - var content = jsonNode?["content"]?.ToString() ?? string.Empty; - if (!string.IsNullOrWhiteSpace(content)) - { - json = content; - return true; - } - } - } - catch (JsonException) - { - context.Log.LogDebug("Unable to extract JSON from planner response, it is likely not from an OpenAPI skill."); - } - catch (InvalidOperationException) - { - context.Log.LogDebug("Unable to extract JSON from planner response, it may already be proper JSON."); - } - - json = string.Empty; - return false; - } - - /// - /// Try to optimize json from the planner response - /// based on token limit - /// - private string OptimizeOpenApiSkillJson(string jsonContent, int tokenLimit, Plan plan) - { - // Remove all new line characters + leading and trailing white space - jsonContent = Regex.Replace(jsonContent.Trim(), @"[\n\r]", string.Empty); - var document = JsonDocument.Parse(jsonContent); - string lastSkillInvoked = plan.Steps[^1].SkillName; - string lastSkillFunctionInvoked = plan.Steps[^1].Name; - bool trimSkillResponse = false; - - // The json will be deserialized based on the response type of the particular operation that was last invoked by the planner - // The response type can be a custom trimmed down json structure, which is useful in staying within the token limit - Type skillResponseType = this.GetOpenApiSkillResponseType(ref document, ref lastSkillInvoked, ref lastSkillFunctionInvoked, ref trimSkillResponse); - - if (trimSkillResponse) - { - // Deserializing limits the json content to only the fields defined in the respective OpenApiSkill's Model classes - var skillResponse = JsonSerializer.Deserialize(jsonContent, skillResponseType); - jsonContent = skillResponse != null ? JsonSerializer.Serialize(skillResponse) : string.Empty; - document = JsonDocument.Parse(jsonContent); - } - - int jsonContentTokenCount = Utilities.TokenCount(jsonContent); - - // Return the JSON content if it does not exceed the token limit - if (jsonContentTokenCount < tokenLimit) - { - return jsonContent; - } - - List itemList = new(); - - // Some APIs will return a JSON response with one property key representing an embedded answer. - // Extract this value for further processing - string resultsDescriptor = ""; - - if (document.RootElement.ValueKind == JsonValueKind.Object) - { - int propertyCount = 0; - foreach (JsonProperty property in document.RootElement.EnumerateObject()) - { - propertyCount++; - } - - if (propertyCount == 1) - { - // Save property name for result interpolation - JsonProperty firstProperty = document.RootElement.EnumerateObject().First(); - tokenLimit -= Utilities.TokenCount(firstProperty.Name); - resultsDescriptor = string.Format(CultureInfo.InvariantCulture, "{0}: ", firstProperty.Name); - - // Extract object to be truncated - JsonElement value = firstProperty.Value; - document = JsonDocument.Parse(value.GetRawText()); - } - } - - // Detail Object - // To stay within token limits, attempt to truncate the list of properties - if (document.RootElement.ValueKind == JsonValueKind.Object) - { - foreach (JsonProperty property in document.RootElement.EnumerateObject()) - { - int propertyTokenCount = Utilities.TokenCount(property.ToString()); - - if (tokenLimit - propertyTokenCount > 0) - { - itemList.Add(property); - tokenLimit -= propertyTokenCount; - } - else - { - break; - } - } - } - - // Summary (List) Object - // To stay within token limits, attempt to truncate the list of results - if (document.RootElement.ValueKind == JsonValueKind.Array) - { - foreach (JsonElement item in document.RootElement.EnumerateArray()) - { - int itemTokenCount = Utilities.TokenCount(item.ToString()); - - if (tokenLimit - itemTokenCount > 0) - { - itemList.Add(item); - tokenLimit -= itemTokenCount; - } - else - { - break; - } - } - } - - return itemList.Count > 0 - ? string.Format(CultureInfo.InvariantCulture, "{0}{1}", resultsDescriptor, JsonSerializer.Serialize(itemList)) - : string.Format(CultureInfo.InvariantCulture, "JSON response for {0} is too large to be consumed at this time.", lastSkillInvoked); - } - - private Type GetOpenApiSkillResponseType(ref JsonDocument document, ref string lastSkillInvoked, ref string lastSkillFunctionInvoked, ref bool trimSkillResponse) - { - Type skillResponseType = typeof(object); // Use a reasonable default response type - - // Different operations under the skill will return responses as json structures; - // Prune each operation response according to the most important/contextual fields only to avoid going over the token limit - // Check what the last skill invoked was and deserialize the JSON content accordingly - if (string.Equals(lastSkillInvoked, "GitHubSkill", StringComparison.Ordinal)) - { - trimSkillResponse = true; - skillResponseType = this.GetGithubSkillResponseType(ref document); - } - else if (string.Equals(lastSkillInvoked, "JiraSkill", StringComparison.Ordinal)) - { - trimSkillResponse = true; - skillResponseType = this.GetJiraSkillResponseType(ref document, ref lastSkillFunctionInvoked); - } - - return skillResponseType; - } - - private Type GetGithubSkillResponseType(ref JsonDocument document) - { - return document.RootElement.ValueKind == JsonValueKind.Array ? typeof(PullRequest[]) : typeof(PullRequest); - } - - private Type GetJiraSkillResponseType(ref JsonDocument document, ref string lastSkillFunctionInvoked) - { - if (lastSkillFunctionInvoked == "GetIssue") - { - return document.RootElement.ValueKind == JsonValueKind.Array ? typeof(IssueResponse[]) : typeof(IssueResponse); - } - - return typeof(IssueResponse); - } - - #endregion -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemory.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemory.cs deleted file mode 100644 index 889af6f2cd12..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemory.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace SemanticKernel.Service.CopilotChat.Skills.ChatSkills; - -/// -/// A collection of semantic chat memory. -/// -public class SemanticChatMemory -{ - /// - /// The chat memory items. - /// - [JsonPropertyName("items")] - public List Items { get; set; } = new List(); - - /// - /// Create and add a chat memory item. - /// - /// Label for the chat memory item. - /// Details for the chat memory item. - public void AddItem(string label, string details) - { - this.Items.Add(new SemanticChatMemoryItem(label, details)); - } - - /// - /// Serialize the chat memory to a Json string. - /// - /// A Json string representing the chat memory. - public override string ToString() - { - return JsonSerializer.Serialize(this); - } - - /// - /// Create a semantic chat memory from a Json string. - /// - /// Json string to deserialize. - /// A semantic chat memory. - public static SemanticChatMemory FromJson(string json) - { - var result = JsonSerializer.Deserialize(json); - return result ?? throw new ArgumentException("Failed to deserialize chat memory to json."); - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemoryExtractor.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemoryExtractor.cs deleted file mode 100644 index e08bd64f0aa2..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemoryExtractor.cs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Globalization; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using SemanticKernel.Service.CopilotChat.Extensions; -using SemanticKernel.Service.CopilotChat.Options; - -namespace SemanticKernel.Service.CopilotChat.Skills.ChatSkills; - -/// -/// Helper class to extract and create semantic memory from chat history. -/// -internal static class SemanticChatMemoryExtractor -{ - /// - /// Returns the name of the semantic text memory collection that stores chat semantic memory. - /// - /// Chat ID that is persistent and unique for the chat session. - /// Name of the memory category - internal static string MemoryCollectionName(string chatId, string memoryName) => $"{chatId}-{memoryName}"; - - /// - /// Extract and save semantic memory. - /// - /// The Chat ID. - /// The semantic kernel. - /// The context containing the memory. - /// The prompts options. - internal static async Task ExtractSemanticChatMemoryAsync( - string chatId, - IKernel kernel, - SKContext context, - PromptsOptions options) - { - foreach (var memoryName in options.MemoryMap.Keys) - { - try - { - var semanticMemory = await ExtractCognitiveMemoryAsync( - memoryName, - kernel, - context, - options - ); - foreach (var item in semanticMemory.Items) - { - await CreateMemoryAsync(item, chatId, context, memoryName, options); - } - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - // Skip semantic memory extraction for this item if it fails. - // We cannot rely on the model to response with perfect Json each time. - context.Log.LogInformation("Unable to extract semantic memory for {0}: {1}. Continuing...", memoryName, ex.Message); - continue; - } - } - } - - /// - /// Extracts the semantic chat memory from the chat session. - /// - /// Name of the memory category - /// The semantic kernel. - /// The SKContext - /// The prompts options. - /// A SemanticChatMemory object. - internal static async Task ExtractCognitiveMemoryAsync( - string memoryName, - IKernel kernel, - SKContext context, - PromptsOptions options) - { - if (!options.MemoryMap.TryGetValue(memoryName, out var memoryPrompt)) - { - throw new ArgumentException($"Memory name {memoryName} is not supported."); - } - - // Token limit for chat history - var tokenLimit = options.CompletionTokenLimit; - var remainingToken = - tokenLimit - - options.ResponseTokenLimit - - Utilities.TokenCount(memoryPrompt); ; - - var memoryExtractionContext = Utilities.CopyContextWithVariablesClone(context); - memoryExtractionContext.Variables.Set("tokenLimit", remainingToken.ToString(new NumberFormatInfo())); - memoryExtractionContext.Variables.Set("memoryName", memoryName); - memoryExtractionContext.Variables.Set("format", options.MemoryFormat); - memoryExtractionContext.Variables.Set("knowledgeCutoff", options.KnowledgeCutoffDate); - - var completionFunction = kernel.CreateSemanticFunction(memoryPrompt); - var result = await completionFunction.InvokeAsync( - context: memoryExtractionContext, - settings: CreateMemoryExtractionSettings(options) - ); - - SemanticChatMemory memory = SemanticChatMemory.FromJson(result.ToString()); - return memory; - } - - /// - /// Create a memory item in the memory collection. - /// If there is already a memory item that has a high similarity score with the new item, it will be skipped. - /// - /// A SemanticChatMemoryItem instance - /// The ID of the chat the memories belong to - /// The context that contains the memory - /// Name of the memory - /// The prompts options. - internal static async Task CreateMemoryAsync( - SemanticChatMemoryItem item, - string chatId, - SKContext context, - string memoryName, - PromptsOptions options) - { - var memoryCollectionName = SemanticChatMemoryExtractor.MemoryCollectionName(chatId, memoryName); - - var memories = await context.Memory.SearchAsync( - collection: memoryCollectionName, - query: item.ToFormattedString(), - limit: 1, - minRelevanceScore: options.SemanticMemoryMinRelevance, - cancellationToken: context.CancellationToken - ) - .ToListAsync() - .ConfigureAwait(false); - - if (memories.Count == 0) - { - await context.Memory.SaveInformationAsync( - collection: memoryCollectionName, - text: item.ToFormattedString(), - id: Guid.NewGuid().ToString(), - description: memoryName, - cancellationToken: context.CancellationToken - ); - } - } - - /// - /// Create a completion settings object for chat response. Parameters are read from the PromptSettings class. - /// - private static CompleteRequestSettings CreateMemoryExtractionSettings(PromptsOptions options) - { - var completionSettings = new CompleteRequestSettings - { - MaxTokens = options.ResponseTokenLimit, - Temperature = options.ResponseTemperature, - TopP = options.ResponseTopP, - FrequencyPenalty = options.ResponseFrequencyPenalty, - PresencePenalty = options.ResponsePresencePenalty - }; - - return completionSettings; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemoryItem.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemoryItem.cs deleted file mode 100644 index 1eef7ab91c54..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemoryItem.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; - -namespace SemanticKernel.Service.CopilotChat.Skills.ChatSkills; - -/// -/// A single entry in the chat memory. -/// -public class SemanticChatMemoryItem -{ - /// - /// Label for the chat memory item. - /// - [JsonPropertyName("label")] - public string Label { get; set; } - - /// - /// Details for the chat memory item. - /// - [JsonPropertyName("details")] - public string Details { get; set; } - - /// - /// Create a new chat memory item. - /// - /// Label of the item. - /// Details of the item. - public SemanticChatMemoryItem(string label, string details) - { - this.Label = label; - this.Details = details; - } - - /// - /// Format the chat memory item as a string. - /// - /// A formatted string representing the item. - public string ToFormattedString() - { - return $"{this.Label}: {this.Details}"; - } -} diff --git a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemorySkill.cs b/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemorySkill.cs deleted file mode 100644 index b9d7efaeb158..000000000000 --- a/samples/apps/copilot-chat-app/webapi/CopilotChat/Skills/ChatSkills/SemanticChatMemorySkill.cs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.Options; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.SkillDefinition; -using SemanticKernel.Service.CopilotChat.Options; - -namespace SemanticKernel.Service.CopilotChat.Skills.ChatSkills; - -/// -/// This skill provides the functions to query the semantic chat memory. -/// -public class SemanticChatMemorySkill -{ - /// - /// Prompt settings. - /// - private readonly PromptsOptions _promptOptions; - - /// - /// Create a new instance of SemanticChatMemorySkill. - /// - public SemanticChatMemorySkill( - IOptions promptOptions) - { - this._promptOptions = promptOptions.Value; - } - - /// - /// Query relevant memories based on the query. - /// - /// Query to match. - /// The SKContext - /// A string containing the relevant memories. - [SKFunction, Description("Query chat memories")] - public async Task QueryMemoriesAsync( - [Description("Query to match.")] string query, - [Description("Chat ID to query history from")] string chatId, - [Description("Maximum number of tokens")] int tokenLimit, - ISemanticTextMemory textMemory) - { - var remainingToken = tokenLimit; - - // Search for relevant memories. - List relevantMemories = new(); - foreach (var memoryName in this._promptOptions.MemoryMap.Keys) - { - var results = textMemory.SearchAsync( - SemanticChatMemoryExtractor.MemoryCollectionName(chatId, memoryName), - query, - limit: 100, - minRelevanceScore: this._promptOptions.SemanticMemoryMinRelevance); - await foreach (var memory in results) - { - relevantMemories.Add(memory); - } - } - - relevantMemories = relevantMemories.OrderByDescending(m => m.Relevance).ToList(); - - string memoryText = ""; - foreach (var memory in relevantMemories) - { - var tokenCount = Utilities.TokenCount(memory.Metadata.Text); - if (remainingToken - tokenCount > 0) - { - memoryText += $"\n[{memory.Metadata.Description}] {memory.Metadata.Text}"; - remainingToken -= tokenCount; - } - else - { - break; - } - } - - if (string.IsNullOrEmpty(memoryText)) - { - // No relevant memories found - return string.Empty; - } - - return $"Past memories (format: [memory type]