Solr automatic core update as deployment step
For my projects I use the awesome tool Capistrano for deployment. In a project that uses Solr, I want to update the Solr core as a deployment step if the Solr configuration of that core is changed. After the reload of the configuration I want to do a full import with the Solr Data Import Handler.
If you have a large import then this can take a while. But it is possible to do this without any downtime! The trick is to work with two cores. A “live” core and a “on deck” core. The PHP script below copies the new core configuration from the project to the on deck core, for example “/opt/solr/my-core-ondeck”. After that it reloads the on deck core and starts the full import.
The PHP script will poll every 15 seconds if the Data Import Handler is finished. When finished, the “live” and “on deck” core instance directories are swapped in an eye blink so there is minimal downtime! Of course if you have a Delta Import running from time to time, it will still have to catch up for the time the full import was running, but that won’t be as much of a problem.
I have this script located at “/opt/solr/update-core”. Example usage:
./update-core example /var/www/example.com/data/solr/example/conf
<?php
// USAGE ./update-core <core> <projectCoreConfDir>
/**
* Updates the specified core
*
* @param string $core
* @param array $options
*/
function updateCore($core, $options = array())
{
echo 'Updating core ' . $core . ":\n";
if (!file_exists('/opt/solr/' . $core . '-ondeck')) {
mkdir('/opt/solr/' . $core . '-ondeck');
mkdir('/opt/solr/' . $core . '-ondeck/data');
system('chgrp -Rf jetty /opt/solr/' . $core . '-ondeck');
$createOnDeckCore = true;
} else {
$createOnDeckCore = false;
}
system('rm -rf /opt/solr/' . $core . '-ondeck/conf');
system('cp -a ' . $options['project_core_conf_dir'] . ' /opt/solr/' . $core . '-ondeck/conf');
system('chgrp -Rf jetty /opt/solr/' . $core . '-ondeck/conf');
if ($createOnDeckCore) {
echo "Creating on deck core...\n";
$url = 'http://localhost:8085/solr/admin/cores?wt=json&action=CREATE&name=' . $core . '-ondeck';
$content = file_get_contents($url);
echo 'GET ' . $url . "\n" . $content;
// Wait 10 seconds for creation
sleep(10);
echo "done\n";
}
if (!file_exists('/opt/solr/' . $core)) {
echo "Creating core...\n";
mkdir('/opt/solr/' . $core);
mkdir('/opt/solr/' . $core . '/data');
system('chgrp -Rf jetty /opt/solr/' . $core);
system('cp -a ' . $options['project_core_conf_dir'] . ' /opt/solr/' . $core . '/conf');
system('chgrp -Rf jetty /opt/solr/' . $core .'/conf');
$url = 'http://localhost:8085/solr/admin/cores?wt=json&action=CREATE&name=' . $core;
$content = file_get_contents($url);
echo 'GET ' . $url . "\n" . $content;
// Wait 10 seconds for creation
sleep(10);
echo "done\n";
}
// Reload on deck core
echo "Reloading on deck core...\n";
$url = 'http://localhost:8085/solr/admin/cores?wt=json&action=RELOAD&core=' . $core . '-ondeck';
$content = file_get_contents($url);
echo 'GET ' . $url . "\n" . $content;
// Wait 10 seconds for reload
sleep(10);
echo "done\n";
// Run full import for on deck core
echo "Running full import for on deck core...\n";
$url = 'http://localhost:8085/solr/' . $core . '-ondeck/dataimport?command=full-import&verbose=false&clean=true&commit=true&optimize=true';
$content = file_get_contents($url);
echo 'GET ' . $url . "\n" . $content;
$startTime = microtime(true);
while (true) {
if ($startTime + (3600 * 10) < time()) {
echo "Stopped importing because of timeout after 10 hours";
exit(1);
}
$url = 'http://localhost:8085/solr/' . $core . '-ondeck/dataimport?command=status&wt=json';
$content = file_get_contents($url);
echo 'GET ' . $url . "\n" . $content;
if (stripos($content, 'failed') !== false) {
echo "Full import failed\n";
exit(1);
} elseif (stripos($content, 'idle') !== false) {
break;
}
sleep(15);
}
echo "done\n";
// Swap live core and on deck core
echo "Swapping live core and on deck core...\n";
rename('/opt/solr/' . $core, '/opt/solr/' . $core . '.tmp');
echo 'Moved: /opt/solr/' . $core . ' to /opt/solr/' . $core . '.tmp' . "\n";
rename('/opt/solr/' . $core . '-ondeck', '/opt/solr/' . $core);
echo 'Moved: /opt/solr/' . $core . '-ondeck to /opt/solr/' . $core . "\n";
rename('/opt/solr/' . $core . '.tmp', '/opt/solr/' . $core . '-ondeck');
echo 'Moved: /opt/solr/' . $core . '.tmp to /opt/solr/' . $core . '-ondeck' . "\n";
// Reload live core
echo "Reloading live core...\n";
$url = 'http://localhost:8085/solr/admin/cores?wt=json&action=RELOAD&core=' . $core;
$content = file_get_contents($url);
echo 'GET ' . $url . "\n" . $content;
// Reload on deck core
echo "Reloading on deck core...\n";
$url = 'http://localhost:8085/solr/admin/cores?wt=json&action=RELOAD&core=' . $core . '-ondeck';
$content = file_get_contents($url);
echo 'GET ' . $url . "\n" . $content;
echo "done\n";
echo "Finished updating core " . $core . "\n";
}
if (isset($argv[1]) && '' != $argv[1]) {
$core = strtolower(trim($argv[1]));
} else {
echo "Please enter the core to update:\n";
$handle = fopen('php://stdin','r');
$core = strtolower(trim(fgets($handle)));
if ('' == $core) {
echo "No core specified\n";
exit(1);
}
}
if (isset($argv[2]) && '' != $argv[2]) {
$projectCoreConfDir = trim($argv[2]);
} else {
echo "Please enter the project core conf directory:\n";
$handle = fopen('php://stdin','r');
$projectCoreConfDir = trim(fgets($handle));
if ('' == $projectCoreConfDir) {
echo "No project core conf directory specified\n";
exit(1);
}
}
$options = array(
'project_core_conf_dir' => $projectCoreConfDir,
);
updateCore($core, $options);