Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SOLR-17068: Resolve mish mash of bin/post and bin/solr post references in favour of bin/solr post. #2227

Merged
merged 12 commits into from
Feb 2, 2024
Merged
22 changes: 19 additions & 3 deletions solr/core/src/java/org/apache/solr/cli/PostTool.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,16 @@ public List<Option> getOptions() {
Option.builder("url")
.argName("url")
.hasArg()
.required(true)
.required(false)
.desc("<base Solr update URL>")
.build(),
Option.builder("c")
.longOpt("name")
.argName("NAME")
.hasArg()
.required(false)
.desc("Name of the collection.")
.build(),
Option.builder("commit").required(false).desc("Issue a commit at end of post").build(),
Option.builder("optimize").required(false).desc("Issue an optimize at end of post").build(),
Option.builder("mode")
Expand Down Expand Up @@ -103,8 +110,17 @@ public List<Option> getOptions() {
public void runImpl(CommandLine cli) throws Exception {
SolrCLI.raiseLogLevelUnlessVerbose(cli);

String url = cli.getOptionValue("url");
URL solrUrl = new URL(url);
URL solrUrl = null;
if (cli.hasOption("url")) {
String url = cli.getOptionValue("url");
solrUrl = new URL(url);
} else if (cli.hasOption("c")) {
String url = SolrCLI.getDefaultSolrUrl() + "/solr/" + cli.getOptionValue("c") + "/update";
solrUrl = new URL(url);
} else {
throw new IllegalArgumentException(
"Must specify either -url or -c parameter to post documents.");
}

String mode = SimplePostTool.DEFAULT_DATA_MODE;
if (cli.hasOption("mode")) {
Expand Down
3 changes: 0 additions & 3 deletions solr/core/src/java/org/apache/solr/cli/SimplePostTool.java
Original file line number Diff line number Diff line change
Expand Up @@ -1089,9 +1089,6 @@ private void basicAuth(HttpURLConnection urlc) throws Exception {
if (!credentials.contains(":")) {
throw new Exception("credentials '" + credentials + "' must be of format user:pass");
}
// urlc.setRequestProperty(
// "Authorization",
// "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(UTF_8)));
urlc.setRequestProperty(
"Authorization",
"Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(UTF_8)));
Expand Down
2 changes: 1 addition & 1 deletion solr/core/src/java/org/apache/solr/cli/SolrCLI.java
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ private static void printHelp() {

/**
* Strips off the end of solrUrl any /solr when a legacy solrUrl like http://localhost:8983/solr
* is used, and warns those users. In the future we'll have url's with /api as well.
* is used, and warns those users. In the future we'll have urls ending with /api as well.
*
* @param solrUrl The user supplied url to Solr.
* @return the solrUrl in the format that Solr expects to see internally.
Expand Down
32 changes: 27 additions & 5 deletions solr/core/src/test/org/apache/solr/cli/PostToolTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.security.BasicAuthPlugin;
import org.apache.solr.security.RuleBasedAuthorizationPlugin;
Expand Down Expand Up @@ -68,7 +69,7 @@ public static void setupClusterWithSecurityEnabled() throws Exception {
singletonMap(USER, getSaltedHashedValue(PASS)))));

configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.addConfig("conf1", configset("cloud-minimal"))
.withSecurityJson(SECURITY_JSON)
.configure();
}
Expand All @@ -80,27 +81,48 @@ private <T extends SolrRequest<? extends SolrResponse>> T withBasicAuth(T req) {

@Test
public void testBasicRun() throws Exception {
final String collection = "aliasedCollection";
final String collection = "testBasicRun";

withBasicAuth(CollectionAdminRequest.createCollection(collection, "config", 1, 1, 0, 0))
withBasicAuth(CollectionAdminRequest.createCollection(collection, "conf1", 1, 1, 0, 0))
.processAndWait(cluster.getSolrClient(), 10);

File jsonDoc = File.createTempFile("temp", "json");
File jsonDoc = File.createTempFile("temp", ".json");

FileWriter fw = new FileWriter(jsonDoc, StandardCharsets.UTF_8);
Utils.writeJson(Utils.toJSONString(Map.of("id", "1", "title", "mytitle")), fw, true);

String[] args = {
"post",
"-url",
cluster.getJettySolrRunner(0).getBaseUrl() + "/" + collection,
cluster.getJettySolrRunner(0).getBaseUrl() + "/" + collection + "/update",
"-credentials",
USER + ":" + PASS,
jsonDoc.getAbsolutePath()
};
assertEquals(0, runTool(args));
}

@Test
public void testRunWithCollectionParam() throws Exception {
final String collection = "testRunWithCollectionParam";

// Provide the port as an environment variable for the PostTool to look up.
EnvUtils.setEnv("SOLR_PORT", cluster.getJettySolrRunner(0).getLocalPort() + "");

withBasicAuth(CollectionAdminRequest.createCollection(collection, "conf1", 1, 1, 0, 0))
.processAndWait(cluster.getSolrClient(), 10);

File jsonDoc = File.createTempFile("temp", "json");

FileWriter fw = new FileWriter(jsonDoc, StandardCharsets.UTF_8);
Utils.writeJson(Utils.toJSONString(Map.of("id", "1", "title", "mytitle")), fw, true);

String[] args = {
"post", "-c", collection, "-credentials", USER + ":" + PASS, jsonDoc.getAbsolutePath()
};
assertEquals(0, runTool(args));
}

private int runTool(String[] args) throws Exception {
Tool tool = findTool(args);
assertTrue(tool instanceof PostTool);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,10 @@
import org.slf4j.LoggerFactory;

/**
* Emulates bin/solr -e cloud -noprompt; bin/post -c gettingstarted example/exampledocs/*.xml; this
* test is useful for catching regressions in indexing the example docs in collections that use data
* driven functionality and managed schema features of the default configset (configsets/_default).
* Emulates bin/solr start -e cloud -noprompt; bin/solr post -c gettingstarted
* example/exampledocs/*.xml; this test is useful for catching regressions in indexing the example
* docs in collections that use data driven functionality and managed schema features of the default
* configset (configsets/_default).
*/
public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase {

Expand Down Expand Up @@ -114,7 +115,7 @@ public void testLoadDocsIntoGettingStartedCollection() throws Exception {
invalidToolExitStatus,
tool.runTool(cli));

// now index docs like bin/post would, but we can't use SimplePostTool because it uses
// now index docs like bin/solr post would, but we can't use SimplePostTool because it uses
// System.exit when it encounters an error, which JUnit doesn't like ...
log.info("Created collection, now posting example docs!");
Path exampleDocsDir = Path.of(ExternalPaths.SOURCE_HOME, "example", "exampledocs");
Expand Down
10 changes: 3 additions & 7 deletions solr/docker/scripts/solr-demo
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,9 @@ else
/opt/solr/bin/solr create -c "$CORE"
echo "Created $CORE"
echo "Loading example data"
post_args=()
if [[ -n "${SOLR_PORT:-}" ]]; then
post_args+=(-p "$SOLR_PORT")
fi
/opt/solr/bin/post "${post_args[@]}" -c $CORE -commit no example/exampledocs/*.xml
/opt/solr/bin/post "${post_args[@]}" -c $CORE -commit no example/exampledocs/books.json
/opt/solr/bin/post "${post_args[@]}" -c $CORE -commit yes example/exampledocs/books.csv
/opt/solr/bin/solr post -c $CORE -commit no example/exampledocs/*.xml
/opt/solr/bin/solr post -c $CORE -commit no example/exampledocs/books.json
/opt/solr/bin/solr post -c $CORE -commit yes example/exampledocs/books.csv
echo "Loaded example data"
stop-local-solr

Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/create_core/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ docker run --name "$container_name" -d "$tag" solr-create -c gettingstarted
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c gettingstarted example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c gettingstarted -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/gettingstarted/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/create_core_exec/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ wait_for_container_and_solr "$container_name"
echo "Creating core"
docker exec --user=solr "$container_name" bin/solr create -c gettingstarted
echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c gettingstarted example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c gettingstarted -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/gettingstarted/select?q=id%3Adell')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ docker run --user 7777:0 --name "$container_name" -d "$tag" solr-create -c getti
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c gettingstarted example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c gettingstarted -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/gettingstarted/select?q=id%3Adell')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c getting-started example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c getting-started -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/getting-started/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/empty-varsolr-dir-solr/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c getting-started example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c getting-started -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/getting-started/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/empty-varsolr-dir-user/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c getting-started example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c getting-started -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/getting-started/select?q=id%3Adell')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c getting-started example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c getting-started -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/getting-started/select?q=id%3Adell')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c getting-started example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c getting-started -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/getting-started/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/empty-varsolr-vol-solr/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c getting-started example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c getting-started -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/getting-started/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/empty-varsolr-vol-user/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c getting-started example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c getting-started -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/getting-started/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/gosu/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ docker run --user 0:0 --name "$container_name" -d -e VERBOSE=yes \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c gettingstarted example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c gettingstarted -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/gettingstarted/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/precreate_core/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ docker run --name "$container_name" -d -e VERBOSE=yes "$tag" solr-precreate gett
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c gettingstarted example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c gettingstarted -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/gettingstarted/select?q=id%3Adell')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c gettingstarted example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c gettingstarted -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/gettingstarted/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/test_log4j/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ docker run --name "$container_name" -d -e VERBOSE=yes \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c gettingstarted example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c gettingstarted -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/gettingstarted/select?q=id%3Adell')
Expand Down
2 changes: 1 addition & 1 deletion solr/docker/tests/cases/user_volume/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ docker run \
wait_for_container_and_solr "$container_name"

echo "Loading data"
docker exec --user=solr "$container_name" bin/post -c mycore example/exampledocs/manufacturers.xml
docker exec --user=solr "$container_name" bin/solr post -c mycore -commit example/exampledocs/manufacturers.xml
sleep 1
echo "Checking data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/mycore/select?q=id%3Adell')
Expand Down
4 changes: 2 additions & 2 deletions solr/example/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ After starting a Solr example, direct your Web browser to:
http://localhost:8983/solr/
```

To add documents to the index, use bin/post, for example:
To add documents to the index, use bin/solr post, for example:

```
bin/post -c techproducts example/exampledocs/*.xml
bin/solr post -c techproducts example/exampledocs/*.xml
```

(where "techproducts" is the Solr core name)
Expand Down
3 changes: 1 addition & 2 deletions solr/example/exampledocs/utf8-example.xml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
-->

<!--
After posting this to Solr with bin/post, searching for "êâîôû" from
After posting this to Solr with bin/solr post, searching for "êâîôû" from
the solr/admin/ search page must return this document.
-->

Expand All @@ -39,4 +39,3 @@
<field name="inStock">true</field>
</doc>
</add>

13 changes: 12 additions & 1 deletion solr/packaging/test/test_post.bats
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ teardown() {
@test "Check help command" {

run solr post
assert_output --partial 'Failed to parse command-line arguments due to: Missing required option: url'
assert_output --partial 'Must specify either -url or -c parameter'

run solr post -h
assert_output --partial 'usage: post'
Expand All @@ -63,6 +63,17 @@ teardown() {
refute_output --partial 'ERROR'
}

@test "basic post with c parameter instead of url parameter" {

run solr create -c monitors_c_param -d _default
assert_output --partial "Created collection 'monitors_c_param'"

run solr post -type application/xml -c monitors_c_param ${SOLR_TIP}/example/exampledocs/monitor.xml

assert_output --partial '1 files indexed.'
refute_output --partial 'ERROR'
}

@test "basic post WITHOUT a type specified" {

solr create -c monitors_no_type -d _default
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
-->

<!--
After posting this to Solr with bin/post, searching for "êâîôû" from
After posting this to Solr with bin/solr post, searching for "êâîôû" from
the solr/admin/ search page must return this document.
-->

Expand All @@ -39,4 +39,3 @@
<field name="inStock">true</field>
</doc>
</add>

Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ To load data, and see it was split over shards:

[source,bash,subs="attributes"]
----
docker exec -it --user=solr zksolr1 bin/post -c my_collection1 example/exampledocs/manufacturers.xml
docker exec -it --user=solr zksolr1 bin/solr post -c my_collection1 example/exampledocs/manufacturers.xml
# /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -classpath /opt/solr/server/lib/ext/*:/opt/solr/server/solr-webapp/webapp/WEB-INF/lib/* -Dauto=yes -Dc=my_collection1 -Ddata=files org.apache.solr.cli.SimplePostTool example/exampledocs/manufacturers.xml
# SimplePostTool version {solr-full-version}
# Posting files to [base] url http://localhost:8983/solr/my_collection1/update...
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,8 @@ If you ran through the xref:getting-started:solr-tutorial.adoc[], you have alrea
Here are the examples included with Solr:

exampledocs::
This is a small set of simple CSV, XML, and JSON files that can be used with `bin/post` when first getting started with Solr.
For more information about using `bin/post` with these files, see xref:indexing-guide:post-tool.adoc[].
This is a small set of simple CSV, XML, and JSON files that can be used with `bin/solr post` when first getting started with Solr.
For more information about using `bin/solr post` with these files, see xref:indexing-guide:post-tool.adoc[].

files::
The `files` directory provides a basic search UI for documents such as Word or PDF that you may have stored locally.
Expand Down
Loading
Loading