Current File : //opt/cpanel/ea-ruby27/src/passenger-release-6.0.23/test/cxx/Core/ApplicationPool/PoolTest.cpp
#include <TestSupport.h>
#include <jsoncpp/json.h>
#include <Core/ApplicationPool/Pool.h>
#include <LoggingKit/Context.h>
#include <FileTools/FileManip.h>
#include <StrIntTools/StrIntUtils.h>
#include <IOTools/MessageSerialization.h>
#include <map>
#include <vector>
#include <cerrno>
#include <signal.h>
using namespace std;
using namespace Passenger;
using namespace Passenger::ApplicationPool2;
namespace tut {
struct Core_ApplicationPool_PoolTest: public TestBase {
WrapperRegistry::Registry wrapperRegistry;
SpawningKit::Context::Schema skContextSchema;
SpawningKit::Context::DebugSupport skDebugSupport;
SpawningKit::Context skContext;
SpawningKit::FactoryPtr skFactory;
Context context;
PoolPtr pool;
Pool::DebugSupportPtr debug;
Ticket ticket;
GetCallback callback;
SessionPtr currentSession;
ExceptionPtr currentException;
AtomicInt number;
boost::mutex syncher;
list<SessionPtr> sessions;
bool retainSessions;
Core_ApplicationPool_PoolTest()
: skContext(skContextSchema)
{
retainSessions = false;
wrapperRegistry.finalize();
skContext.resourceLocator = resourceLocator;
skContext.wrapperRegistry = &wrapperRegistry;
skContext.integrationMode = "standalone";
skContext.debugSupport = &skDebugSupport;
skContext.spawnDir = getSystemTempDir();
skContext.finalize();
context.spawningKitFactory = boost::make_shared<SpawningKit::Factory>(&skContext);
context.finalize();
pool = boost::make_shared<Pool>(&context);
pool->initialize();
callback.func = _callback;
callback.userData = this;
Json::Value config;
vector<ConfigKit::Error> errors;
LoggingKit::ConfigChangeRequest req;
config["level"] = "warn";
config["app_output_log_level"] = "debug";
if (LoggingKit::context->prepareConfigChange(config, errors, req)) {
LoggingKit::context->commitConfigChange(req);
} else {
P_BUG("Error configuring LoggingKit: " << ConfigKit::toString(errors));
}
}
~Core_ApplicationPool_PoolTest() {
// Explicitly destroy these here because they can run
// additional code that depend on other fields in this
// class.
TRACE_POINT();
clearAllSessions();
UPDATE_TRACE_POINT();
pool->destroy();
UPDATE_TRACE_POINT();
pool.reset();
Json::Value config;
vector<ConfigKit::Error> errors;
LoggingKit::ConfigChangeRequest req;
config["level"] = DEFAULT_LOG_LEVEL_NAME;
config["app_output_log_level"] = DEFAULT_APP_OUTPUT_LOG_LEVEL_NAME;
if (LoggingKit::context->prepareConfigChange(config, errors, req)) {
LoggingKit::context->commitConfigChange(req);
} else {
P_BUG("Error configuring LoggingKit: " << ConfigKit::toString(errors));
}
SystemTime::releaseAll();
}
void initPoolDebugging() {
pool->initDebugging();
debug = pool->debugSupport;
}
void clearAllSessions() {
SessionPtr myCurrentSession;
list<SessionPtr> mySessions;
{
LockGuard l(syncher);
myCurrentSession = currentSession;
mySessions = sessions;
currentSession.reset();
sessions.clear();
}
myCurrentSession.reset();
mySessions.clear();
}
Options createOptions() {
Options options;
options.spawnMethod = "dummy";
options.appRoot = "stub/rack";
options.appType = "ruby";
options.appStartCommand = "ruby start.rb";
options.startupFile = "start.rb";
options.loadShellEnvvars = false;
options.user = testConfig["normal_user_1"].asCString();
options.defaultUser = testConfig["default_user"].asCString();
options.defaultGroup = testConfig["default_group"].asCString();
return options;
}
static void _callback(const AbstractSessionPtr &_session, const ExceptionPtr &e,
void *userData)
{
Core_ApplicationPool_PoolTest *self = (Core_ApplicationPool_PoolTest *) userData;
SessionPtr session = static_pointer_cast<Session>(_session);
SessionPtr oldSession;
{
LockGuard l(self->syncher);
oldSession = self->currentSession;
self->currentSession = session;
self->currentException = e;
self->number++;
if (self->retainSessions && session != NULL) {
self->sessions.push_back(session);
}
}
// destroy old session object outside the lock.
}
void sendHeaders(int connection, ...) {
va_list ap;
const char *arg;
vector<StaticString> args;
va_start(ap, connection);
while ((arg = va_arg(ap, const char *)) != NULL) {
args.push_back(StaticString(arg, strlen(arg) + 1));
}
va_end(ap);
shared_array<StaticString> args_array(new StaticString[args.size() + 1]);
unsigned int totalSize = 0;
for (unsigned int i = 0; i < args.size(); i++) {
args_array[i + 1] = args[i];
totalSize += args[i].size();
}
char sizeHeader[sizeof(uint32_t)];
Uint32Message::generate(sizeHeader, totalSize);
args_array[0] = StaticString(sizeHeader, sizeof(uint32_t));
gatheredWrite(connection, args_array.get(), args.size() + 1, NULL);
}
string stripHeaders(const string &str) {
string::size_type pos = str.find("\r\n\r\n");
if (pos == string::npos) {
return str;
} else {
string result = str;
result.erase(0, pos + 4);
return result;
}
}
string sendRequest(const Options &options, const char *path) {
int oldNumber = number;
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == oldNumber + 1;
);
if (currentException != NULL) {
P_ERROR("get() exception: " << currentException->what());
abort();
}
currentSession->initiate();
sendHeaders(currentSession->fd(),
"PATH_INFO", path,
"REQUEST_METHOD", "GET",
NULL);
shutdown(currentSession->fd(), SHUT_WR);
string body = stripHeaders(readAll(currentSession->fd(),
1024 * 1024).first);
ProcessPtr process = currentSession->getProcess()->shared_from_this();
currentSession.reset();
EVENTUALLY(5,
result = process->busyness() == 0;
);
return body;
}
// Ensure that n processes exist.
Options ensureMinProcesses(unsigned int n) {
Options options = createOptions();
options.minProcesses = n;
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
EVENTUALLY(5,
result = pool->getProcessCount() == n;
);
currentSession.reset();
return options;
}
void disableProcess(ProcessPtr process, AtomicInt *result) {
*result = (int) pool->disableProcess(process->getGupid());
}
};
DEFINE_TEST_GROUP_WITH_LIMIT(Core_ApplicationPool_PoolTest, 100);
TEST_METHOD(1) {
// Test initial state.
ensure(!pool->atFullCapacity());
}
/*********** Test asyncGet() behavior on a single Group ***********/
TEST_METHOD(2) {
// asyncGet() actions on empty pools cannot be immediately satisfied.
// Instead a new process will be spawned. In the mean time get()
// actions are put on a wait list which will be processed as soon
// as the new process is done spawning.
Options options = createOptions();
ScopedLock l(pool->syncher);
pool->asyncGet(options, callback, false);
ensure_equals("(1)", number, 0);
ensure("(2)", pool->getWaitlist.empty());
ensure("(3)", !pool->groups.empty());
l.unlock();
EVENTUALLY(5,
result = pool->getProcessCount() == 1;
);
EVENTUALLY(5,
result = number == 1;
);
ensure("(4)", currentSession != NULL);
ensure("(5)", currentException == NULL);
}
TEST_METHOD(3) {
// If one matching process already exists and it's not at full
// capacity then asyncGet() will immediately use it.
Options options = createOptions();
// Spawn a process and opens a session with it.
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
// Close the session so that the process is now idle.
ProcessPtr process = currentSession->getProcess()->shared_from_this();
currentSession.reset();
ensure_equals(process->busyness(), 0);
ensure(!process->isTotallyBusy());
// Verify test assertion.
ScopedLock l(pool->syncher);
pool->asyncGet(options, callback, false);
ensure_equals("callback is immediately called", number, 2);
}
TEST_METHOD(4) {
// If one matching process already exists but it's at full capacity,
// and the limits prevent spawning of a new process,
// then asyncGet() will put the get action on the group's wait
// queue. When the process is no longer at full capacity it will
// process the request.
// Spawn a process and verify that it's at full capacity.
// Keep its session open.
Options options = createOptions();
options.appGroupName = "test";
pool->setMax(1);
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
SessionPtr session1 = currentSession;
ProcessPtr process = session1->getProcess()->shared_from_this();
currentSession.reset();
ensure_equals(process->sessions, 1);
ensure(process->isTotallyBusy());
// Now call asyncGet() again.
pool->asyncGet(options, callback);
ensure_equals("callback is not yet called", number, 1);
ensure_equals("the get action has been put on the wait list",
pool->groups.lookupCopy("test")->getWaitlist.size(), 1u);
session1.reset();
ensure_equals("callback is called after the process becomes idle",
number, 2);
ensure_equals("the get wait list has been processed",
pool->groups.lookupCopy("test")->getWaitlist.size(), 0u);
ensure_equals(process->sessions, 1);
}
TEST_METHOD(5) {
// If one matching process already exists but it's at full utilization,
// and the limits and pool capacity allow spawning of a new process,
// then get() will put the get action on the group's wait
// queue while spawning a process in the background.
// Either the existing process or the newly spawned process
// will process the action, whichever becomes first available.
// Here we test the case in which the existing process becomes
// available first.
initPoolDebugging();
// Spawn a regular process and keep its session open.
Options options = createOptions();
debug->messages->send("Proceed with spawn loop iteration 1");
SessionPtr session1 = pool->get(options, &ticket);
ProcessPtr process1 = session1->getProcess()->shared_from_this();
// Now spawn a process that never finishes.
pool->asyncGet(options, callback);
// Release the session on the first process.
session1.reset();
EVENTUALLY(1,
result = number == 1;
);
ensure_equals("The first process handled the second asyncGet() request",
currentSession->getProcess(), process1.get());
debug->messages->send("Proceed with spawn loop iteration 2");
EVENTUALLY(5,
result = number == 1;
);
}
TEST_METHOD(6) {
// Here we test the case in which the new process becomes
// available first.
// Spawn a regular process.
Options options = createOptions();
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
SessionPtr session1 = currentSession;
ProcessPtr process1 = currentSession->getProcess()->shared_from_this();
currentSession.reset();
// As long as we don't release process1 the following get
// action will be processed by the newly spawned process.
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = pool->getProcessCount() == 2;
);
ensure_equals(number, 2);
ensure(currentSession->getProcess() != process1.get());
}
TEST_METHOD(7) {
// If multiple matching processes exist, and one of them is idle,
// then asyncGet() will use that.
// Spawn 3 processes and keep a session open with 1 of them.
Options options = createOptions();
options.minProcesses = 3;
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
EVENTUALLY(5,
result = pool->getProcessCount() == 3;
);
SessionPtr session1 = currentSession;
ProcessPtr process1 = currentSession->getProcess()->shared_from_this();
currentSession.reset();
// Now open another session. It should complete immediately
// and should not use the first process.
ScopedLock l(pool->syncher);
pool->asyncGet(options, callback, false);
ensure_equals("asyncGet() completed immediately", number, 2);
SessionPtr session2 = currentSession;
ProcessPtr process2 = currentSession->getProcess()->shared_from_this();
l.unlock();
currentSession.reset();
ensure(process2 != process1);
// Now open yet another session. It should also complete immediately
// and should not use the first or the second process.
l.lock();
pool->asyncGet(options, callback, false);
ensure_equals("asyncGet() completed immediately", number, 3);
SessionPtr session3 = currentSession;
ProcessPtr process3 = currentSession->getProcess()->shared_from_this();
l.unlock();
currentSession.reset();
ensure(process3 != process1);
ensure(process3 != process2);
}
TEST_METHOD(8) {
// If multiple matching processes exist, then asyncGet() will use
// the one with the smallest utilization number.
// Spawn 2 processes, each with a concurrency of 2.
skDebugSupport.dummyConcurrency = 2;
Options options = createOptions();
options.minProcesses = 2;
pool->setMax(2);
GroupPtr group = pool->findOrCreateGroup(options);
{
LockGuard l(pool->syncher);
group->spawn();
}
EVENTUALLY(5,
result = pool->getProcessCount() == 2;
);
// asyncGet() selects some process.
pool->asyncGet(options, callback);
ensure_equals("(1)", number, 1);
SessionPtr session1 = currentSession;
ProcessPtr process1 = currentSession->getProcess()->shared_from_this();
currentSession.reset();
// The first process now has 1 session, so next asyncGet() should
// select the other process.
pool->asyncGet(options, callback);
ensure_equals("(2)", number, 2);
SessionPtr session2 = currentSession;
ProcessPtr process2 = currentSession->getProcess()->shared_from_this();
currentSession.reset();
ensure("(3)", process1 != process2);
// Both processes now have an equal number of sessions. Next asyncGet()
// can select either.
pool->asyncGet(options, callback);
ensure_equals("(4)", number, 3);
SessionPtr session3 = currentSession;
ProcessPtr process3 = currentSession->getProcess()->shared_from_this();
currentSession.reset();
// One process now has the lowest number of sessions. Next
// asyncGet() should select that one.
pool->asyncGet(options, callback);
ensure_equals("(5)", number, 4);
SessionPtr session4 = currentSession;
ProcessPtr process4 = currentSession->getProcess()->shared_from_this();
currentSession.reset();
ensure("(6)", process3 != process4);
}
TEST_METHOD(9) {
// If multiple matching processes exist, and all of them are at full capacity,
// and no more processes may be spawned,
// then asyncGet() will put the action on the group's wait queue.
// The process that first becomes not at full capacity will process the action.
// Spawn 2 processes and open 4 sessions.
Options options = createOptions();
options.appGroupName = "test";
options.minProcesses = 2;
pool->setMax(2);
skDebugSupport.dummyConcurrency = 2;
vector<SessionPtr> sessions;
int expectedNumber = 1;
for (int i = 0; i < 4; i++) {
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == expectedNumber;
);
expectedNumber++;
sessions.push_back(currentSession);
currentSession.reset();
}
EVENTUALLY(5,
result = pool->getProcessCount() == 2;
);
GroupPtr group = pool->groups.lookupCopy("test");
ensure_equals(group->getWaitlist.size(), 0u);
ensure(pool->atFullCapacity());
// Now try to open another session.
pool->asyncGet(options, callback);
ensure_equals("The get request has been put on the wait list",
pool->groups.lookupCopy("test")->getWaitlist.size(), 1u);
// Close an existing session so that one process is no
// longer at full utilization.
sessions[0].reset();
ensure_equals("The get request has been removed from the wait list",
pool->groups.lookupCopy("test")->getWaitlist.size(), 0u);
ensure(pool->atFullCapacity());
}
TEST_METHOD(10) {
// If multiple matching processes exist, and all of them are at full utilization,
// and a new process may be spawned,
// then asyncGet() will put the action on the group's wait queue and spawn the
// new process.
// The process that first becomes not at full utilization
// or the newly spawned process
// will process the action, whichever is earlier.
// Here we test the case where an existing process is earlier.
// Spawn 2 processes and open 4 sessions.
skDebugSupport.dummyConcurrency = 2;
Options options = createOptions();
options.minProcesses = 2;
pool->setMax(3);
GroupPtr group = pool->findOrCreateGroup(options);
vector<SessionPtr> sessions;
int expectedNumber = 1;
for (int i = 0; i < 4; i++) {
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == expectedNumber;
);
expectedNumber++;
sessions.push_back(currentSession);
currentSession.reset();
}
EVENTUALLY(5,
result = pool->getProcessCount() == 2;
);
// The next asyncGet() should spawn a new process and the action should be queued.
ScopedLock l(pool->syncher);
skDebugSupport.dummySpawnDelay = 5000000;
pool->asyncGet(options, callback, false);
ensure(group->spawning());
ensure_equals(group->getWaitlist.size(), 1u);
l.unlock();
// Close one of the sessions. Now it will process the action.
ProcessPtr process = sessions[0]->getProcess()->shared_from_this();
sessions[0].reset();
ensure_equals(number, 5);
ensure_equals(currentSession->getProcess(), process.get());
ensure_equals(group->getWaitlist.size(), 0u);
ensure_equals(pool->getProcessCount(), 2u);
}
TEST_METHOD(11) {
// Here we test the case where the newly spawned process is earlier.
// Spawn 2 processes and open 4 sessions.
Options options = createOptions();
options.minProcesses = 2;
pool->setMax(3);
GroupPtr group = pool->findOrCreateGroup(options);
skDebugSupport.dummyConcurrency = 2;
vector<SessionPtr> sessions;
int expectedNumber = 1;
for (int i = 0; i < 4; i++) {
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == expectedNumber;
);
expectedNumber++;
sessions.push_back(currentSession);
currentSession.reset();
}
EVENTUALLY(5,
result = pool->getProcessCount() == 2;
);
// The next asyncGet() should spawn a new process. After it's done
// spawning it will process the action.
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = pool->getProcessCount() == 3;
);
EVENTUALLY(5,
result = number == 5;
);
ensure_equals(currentSession->getProcess()->getPid(), 3);
ensure_equals(group->getWaitlist.size(), 0u);
}
TEST_METHOD(12) {
// Test shutting down.
ensureMinProcesses(2);
ensure(pool->detachGroupByName("stub/rack"));
ensure_equals(pool->getGroupCount(), 0u);
}
TEST_METHOD(13) {
// Test shutting down while Group is restarting.
initPoolDebugging();
debug->messages->send("Proceed with spawn loop iteration 1");
ensureMinProcesses(1);
ensure(pool->restartGroupByName("stub/rack"));
debug->debugger->recv("About to end restarting");
ensure(pool->detachGroupByName("stub/rack"));
ensure_equals(pool->getGroupCount(), 0u);
}
TEST_METHOD(14) {
// Test shutting down while Group is spawning.
initPoolDebugging();
Options options = createOptions();
pool->asyncGet(options, callback);
debug->debugger->recv("Begin spawn loop iteration 1");
ensure(pool->detachGroupByName("stub/rack"));
ensure_equals(pool->getGroupCount(), 0u);
}
TEST_METHOD(17) {
// Test that restartGroupByName() spawns more processes to ensure
// that minProcesses and other constraints are met.
ensureMinProcesses(1);
ensure(pool->restartGroupByName("stub/rack"));
EVENTUALLY(5,
result = pool->getProcessCount() == 1;
);
}
TEST_METHOD(18) {
// Test getting from an app for which minProcesses is set to 0,
// and restart.txt already existed.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
options.minProcesses = 0;
initPoolDebugging();
debug->spawning = false;
SystemTime::forceAll(1000000);
pool->get(options, &ticket);
SystemTime::forceAll(20000000);
touchFile("tmp.wsgi/tmp/restart.txt", 1);
pool->asyncGet(options, callback);
debug->debugger->recv("About to end restarting");
debug->messages->send("Finish restarting");
EVENTUALLY(5,
result = number == 1;
);
ensure_equals(pool->getProcessCount(), 1u);
}
/*********** Test asyncGet() behavior on multiple Groups ***********/
TEST_METHOD(20) {
// If the pool is full, and one tries to asyncGet() from a nonexistant group,
// then it will kill the oldest idle process and spawn a new process.
Options options = createOptions();
pool->setMax(2);
// Get from /foo and close its session immediately.
options.appRoot = "/foo";
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
ProcessPtr process1 = currentSession->getProcess()->shared_from_this();
GroupPtr group1 = process1->getGroup()->shared_from_this();
currentSession.reset();
// Get from /bar and keep its session open.
options.appRoot = "/bar";
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 2;
);
SessionPtr session2 = currentSession;
currentSession.reset();
// Get from /baz. The process for /foo should be killed now.
options.appRoot = "/baz";
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 3;
);
ensure_equals(pool->getProcessCount(), 2u);
ensure_equals(group1->getProcessCount(), 0u);
}
TEST_METHOD(21) {
// If the pool is full, and one tries to asyncGet() from a nonexistant group,
// and all existing processes are non-idle, then it will
// kill the oldest process and spawn a new process.
Options options = createOptions();
pool->setMax(2);
// Get from /foo and close its session immediately.
options.appRoot = "/foo";
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
ProcessPtr process1 = currentSession->getProcess()->shared_from_this();
GroupPtr group1 = process1->getGroup()->shared_from_this();
// Get from /bar and keep its session open.
options.appRoot = "/bar";
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 2;
);
SessionPtr session2 = currentSession;
currentSession.reset();
// Get from /baz. The process for /foo should be killed now.
options.appRoot = "/baz";
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 3;
);
ensure_equals(pool->getProcessCount(), 2u);
ensure_equals(group1->getProcessCount(), 0u);
}
TEST_METHOD(22) {
// Suppose the pool is at full capacity, and one tries to asyncGet() from an
// existant group that does not have any processes. It should kill a process
// from another group, and the request should succeed.
Options options = createOptions();
SessionPtr session;
pid_t pid1, pid2;
pool->setMax(1);
// Create a group /foo.
options.appRoot = "/foo";
SystemTime::force(1);
session = pool->get(options, &ticket);
pid1 = session->getPid();
session.reset();
// Create a group /bar.
options.appRoot = "/bar";
SystemTime::force(2);
session = pool->get(options, &ticket);
pid2 = session->getPid();
session.reset();
// Sleep for a short while to give Pool a chance to shutdown
// the first process.
usleep(300000);
ensure_equals("(1)", pool->getProcessCount(), 1u);
// Get from /foo.
options.appRoot = "/foo";
SystemTime::force(3);
session = pool->get(options, &ticket);
ensure("(2)", session->getPid() != pid1);
ensure("(3)", session->getPid() != pid2);
ensure_equals("(4)", pool->getProcessCount(), 1u);
}
TEST_METHOD(23) {
// Suppose the pool is at full capacity, and one tries to asyncGet() from an
// existant group that does not have any processes, and that happens to need
// restarting. It should kill a process from another group and the request
// should succeed.
Options options1 = createOptions();
Options options2 = createOptions();
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
SessionPtr session;
pid_t pid1, pid2;
pool->setMax(1);
// Create a group tmp.wsgi.
options1.appRoot = "tmp.wsgi";
options1.appType = "wsgi";
options1.startupFile = "passenger_wsgi.py";
options1.spawnMethod = "direct";
SystemTime::force(1);
session = pool->get(options1, &ticket);
pid1 = session->getPid();
session.reset();
// Create a group bar.
options2.appRoot = "bar";
SystemTime::force(2);
session = pool->get(options2, &ticket);
pid2 = session->getPid();
session.reset();
// Sleep for a short while to give Pool a chance to shutdown
// the first process.
usleep(300000);
ensure_equals("(1)", pool->getProcessCount(), 1u);
// Get from tmp.wsgi.
SystemTime::force(3);
touchFile("tmp.wsgi/tmp/restart.txt", 4);
session = pool->get(options1, &ticket);
ensure("(2)", session->getPid() != pid1);
ensure("(3)", session->getPid() != pid2);
ensure_equals("(4)", pool->getProcessCount(), 1u);
}
TEST_METHOD(24) {
// Suppose the pool is at full capacity, with two groups:
// - one that is spawning a process.
// - one with no processes.
// When one tries to asyncGet() from the second group, there should
// be no process to kill, but when the first group is done spawning
// it should throw away that process immediately to allow the second
// group to spawn.
Options options1 = createOptions();
Options options2 = createOptions();
initPoolDebugging();
debug->restarting = false;
pool->setMax(1);
// Create a group foo.
options1.appRoot = "foo";
options1.noop = true;
SystemTime::force(1);
pool->get(options1, &ticket);
// Create a group bar, but don't let it finish spawning.
options2.appRoot = "bar";
options2.noop = true;
SystemTime::force(2);
GroupPtr barGroup = pool->get(options2, &ticket)->getGroup()->shared_from_this();
{
LockGuard l(pool->syncher);
ensure_equals("(1)", barGroup->spawn(), SR_OK);
}
debug->debugger->recv("Begin spawn loop iteration 1");
// Now get from foo again and let the request be queued.
options1.noop = false;
SystemTime::force(3);
pool->asyncGet(options1, callback);
// Nothing should happen while bar is spawning.
SHOULD_NEVER_HAPPEN(100,
result = number > 0;
);
ensure_equals("(2)", pool->getProcessCount(), 0u);
// Now let bar finish spawning. Eventually there should
// only be one process: the one for foo.
debug->messages->send("Proceed with spawn loop iteration 1");
debug->debugger->recv("Spawn loop done");
debug->messages->send("Proceed with spawn loop iteration 2");
debug->debugger->recv("Spawn loop done");
EVENTUALLY(5,
LockGuard l(pool->syncher);
vector<ProcessPtr> processes = pool->getProcesses(false);
if (processes.size() == 1) {
GroupPtr group = processes[0]->getGroup()->shared_from_this();
result = group->getName() == "foo";
} else {
result = false;
}
);
}
TEST_METHOD(25) {
// Suppose the pool is at full capacity, with two groups:
// - one that is spawning a process, and has a queued request.
// - one with no processes.
// When one tries to asyncGet() from the second group, there should
// be no process to kill, but when the first group is done spawning
// it should throw away that process immediately to allow the second
// group to spawn.
Options options1 = createOptions();
Options options2 = createOptions();
initPoolDebugging();
debug->restarting = false;
pool->setMax(1);
// Create a group foo.
options1.appRoot = "foo";
options1.noop = true;
SystemTime::force(1);
pool->get(options1, &ticket);
// Create a group bar with a queued request, but don't let it finish spawning.
options2.appRoot = "bar";
SystemTime::force(2);
pool->asyncGet(options2, callback);
debug->debugger->recv("Begin spawn loop iteration 1");
// Now get from foo again and let the request be queued.
options1.noop = false;
SystemTime::force(3);
pool->asyncGet(options1, callback);
// Nothing should happen while bar is spawning.
SHOULD_NEVER_HAPPEN(100,
result = number > 0;
);
ensure_equals("(1)", pool->getProcessCount(), 0u);
// Now let bar finish spawning. The request for bar should be served.
debug->messages->send("Proceed with spawn loop iteration 1");
debug->debugger->recv("Spawn loop done");
EVENTUALLY(5,
result = number == 1;
);
ensure_equals(currentSession->getGroup()->getName(), "bar");
// When that request is done, the process for bar should be killed,
// and a process for foo should be spawned.
currentSession.reset();
debug->messages->send("Proceed with spawn loop iteration 2");
debug->debugger->recv("Spawn loop done");
EVENTUALLY(5,
LockGuard l(pool->syncher);
vector<ProcessPtr> processes = pool->getProcesses(false);
if (processes.size() == 1) {
GroupPtr group = processes[0]->getGroup()->shared_from_this();
result = group->getName() == "foo";
} else {
result = false;
}
);
EVENTUALLY(5,
result = number == 2;
);
}
/*********** Test detachProcess() ***********/
TEST_METHOD(30) {
// detachProcess() detaches the process from the group. The pool
// will restore the minimum number of processes afterwards.
Options options = createOptions();
options.appGroupName = "test";
options.minProcesses = 2;
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = pool->getProcessCount() == 2;
);
EVENTUALLY(5,
result = number == 1;
);
ProcessPtr process = currentSession->getProcess()->shared_from_this();
pool->detachProcess(process);
{
LockGuard l(pool->syncher);
ensure(process->enabled == Process::DETACHED);
}
EVENTUALLY(5,
result = pool->getProcessCount() == 2;
);
currentSession.reset();
EVENTUALLY(5,
result = process->isDead();
);
}
TEST_METHOD(31) {
// If the containing group had waiters on it, and detachProcess()
// detaches the only process in the group, then a new process
// is automatically spawned to handle the waiters.
Options options = createOptions();
options.appGroupName = "test";
pool->setMax(1);
skDebugSupport.dummySpawnDelay = 1000000;
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
SessionPtr session1 = currentSession;
currentSession.reset();
pool->asyncGet(options, callback);
{
LockGuard l(pool->syncher);
ensure_equals(pool->groups.lookupCopy("test")->getWaitlist.size(), 1u);
}
pool->detachProcess(session1->getProcess()->shared_from_this());
{
LockGuard l(pool->syncher);
ensure(pool->groups.lookupCopy("test")->spawning());
ensure_equals(pool->groups.lookupCopy("test")->enabledCount, 0);
ensure_equals(pool->groups.lookupCopy("test")->getWaitlist.size(), 1u);
}
EVENTUALLY(5,
result = number == 2;
);
}
TEST_METHOD(32) {
// If the pool had waiters on it then detachProcess() will
// automatically create the Groups that were requested
// by the waiters.
Options options = createOptions();
options.appGroupName = "test";
options.minProcesses = 0;
pool->setMax(1);
skDebugSupport.dummySpawnDelay = 30000;
// Begin spawning a process.
pool->asyncGet(options, callback);
ensure(pool->atFullCapacity());
// asyncGet() on another group should now put it on the waiting list.
Options options2 = createOptions();
options2.appGroupName = "test2";
options2.minProcesses = 0;
skDebugSupport.dummySpawnDelay = 90000;
pool->asyncGet(options2, callback);
{
LockGuard l(pool->syncher);
ensure_equals(pool->getWaitlist.size(), 1u);
}
// Eventually the dummy process for "test" is now done spawning.
// We then detach it.
EVENTUALLY(5,
result = number == 1;
);
SessionPtr session1 = currentSession;
currentSession.reset();
pool->detachProcess(session1->getProcess()->shared_from_this());
{
LockGuard l(pool->syncher);
ensure(pool->groups.lookupCopy("test2") != NULL);
ensure_equals(pool->getWaitlist.size(), 0u);
}
EVENTUALLY(5,
result = number == 2;
);
}
TEST_METHOD(33) {
// A Group does not become garbage collectable
// after detaching all its processes.
Options options = createOptions();
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
ProcessPtr process = currentSession->getProcess()->shared_from_this();
currentSession.reset();
GroupPtr group = process->getGroup()->shared_from_this();
pool->detachProcess(process);
LockGuard l(pool->syncher);
ensure_equals(pool->groups.size(), 1u);
ensure(group->isAlive());
ensure(!group->garbageCollectable());
}
TEST_METHOD(34) {
// When detaching a process, it waits until all sessions have
// finished before telling the process to shut down.
Options options = createOptions();
options.spawnMethod = "direct";
options.minProcesses = 0;
SessionPtr session = pool->get(options, &ticket);
ProcessPtr process = session->getProcess()->shared_from_this();
ensure(pool->detachProcess(process));
{
LockGuard l(pool->syncher);
ensure_equals(process->enabled, Process::DETACHED);
}
SHOULD_NEVER_HAPPEN(100,
LockGuard l(pool->syncher);
result = !process->isAlive()
|| !process->osProcessExists();
);
session.reset();
EVENTUALLY(1,
LockGuard l(pool->syncher);
result = process->enabled == Process::DETACHED
&& !process->osProcessExists()
&& process->isDead();
);
}
TEST_METHOD(35) {
// When detaching a process, it waits until the OS processes
// have exited before cleaning up the in-memory data structures.
Options options = createOptions();
options.spawnMethod = "direct";
options.minProcesses = 0;
ProcessPtr process = pool->get(options, &ticket)->getProcess()->shared_from_this();
ScopeGuard g(boost::bind(::kill, process->getPid(), SIGCONT));
kill(process->getPid(), SIGSTOP);
ensure(pool->detachProcess(process));
{
LockGuard l(pool->syncher);
ensure_equals(process->enabled, Process::DETACHED);
}
EVENTUALLY(1,
result = process->getLifeStatus() == Process::SHUTDOWN_TRIGGERED;
);
SHOULD_NEVER_HAPPEN(100,
LockGuard l(pool->syncher);
result = process->isDead()
|| !process->osProcessExists();
);
kill(process->getPid(), SIGCONT);
g.clear();
EVENTUALLY(1,
LockGuard l(pool->syncher);
result = process->enabled == Process::DETACHED
&& !process->osProcessExists()
&& process->isDead();
);
}
TEST_METHOD(36) {
// Detaching a process that is already being detached, works.
Options options = createOptions();
options.appGroupName = "test";
options.minProcesses = 0;
initPoolDebugging();
debug->restarting = false;
debug->spawning = false;
debug->detachedProcessesChecker = true;
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = pool->getProcessCount() == 1;
);
EVENTUALLY(5,
result = number == 1;
);
ProcessPtr process = currentSession->getProcess()->shared_from_this();
pool->detachProcess(process);
debug->debugger->recv("About to start detached processes checker");
{
LockGuard l(pool->syncher);
ensure(process->enabled == Process::DETACHED);
}
// detachProcess() will spawn a new process. Prevent it from being
// spawned too soon.
debug->spawning = true;
pool->detachProcess(process);
debug->messages->send("Proceed with starting detached processes checker");
debug->messages->send("Proceed with starting detached processes checker");
debug->messages->send("Proceed with spawn loop iteration 2");
EVENTUALLY(5,
result = pool->getProcessCount() == 0;
);
currentSession.reset();
EVENTUALLY(5,
result = process->isDead();
);
}
/*********** Test disabling and enabling processes ***********/
TEST_METHOD(40) {
// Disabling a process under idle conditions should succeed immediately.
ensureMinProcesses(2);
vector<ProcessPtr> processes = pool->getProcesses();
ensure_equals("Disabling succeeds",
pool->disableProcess(processes[0]->getGupid()), DR_SUCCESS);
LockGuard l(pool->syncher);
ensure(processes[0]->isAlive());
ensure_equals("Process is disabled",
processes[0]->enabled,
Process::DISABLED);
ensure("Other processes are not affected",
processes[1]->isAlive());
ensure_equals("Other processes are not affected",
processes[1]->enabled, Process::ENABLED);
}
TEST_METHOD(41) {
// Disabling the sole process in a group, in case the pool settings allow
// spawning another process, should trigger a new process spawn.
ensureMinProcesses(1);
Options options = createOptions();
SessionPtr session = pool->get(options, &ticket);
ensure_equals(pool->getProcessCount(), 1u);
ensure(!pool->isSpawning());
skDebugSupport.dummySpawnDelay = 60000;
AtomicInt code = -1;
TempThread thr(boost::bind(&Core_ApplicationPool_PoolTest::disableProcess,
this, session->getProcess()->shared_from_this(), &code));
EVENTUALLY2(100, 1,
result = pool->isSpawning();
);
EVENTUALLY(1,
result = pool->getProcessCount() == 2u;
);
ensure_equals((int) code, -1);
session.reset();
EVENTUALLY(1,
result = code == (int) DR_SUCCESS;
);
}
TEST_METHOD(42) {
// Disabling the sole process in a group, in case pool settings don't allow
// spawning another process, should fail.
pool->setMax(1);
ensureMinProcesses(1);
vector<ProcessPtr> processes = pool->getProcesses();
ensure_equals("(1)", processes.size(), 1u);
if (defaultLogLevel == (LoggingKit::Level) DEFAULT_LOG_LEVEL) {
// If the user did not customize the test's log level,
// then we'll want to tone down the noise.
LoggingKit::setLevel(LoggingKit::ERROR);
}
DisableResult result = pool->disableProcess(processes[0]->getGupid());
ensure_equals("(2)", result, DR_ERROR);
ensure_equals("(3)", pool->getProcessCount(), 1u);
}
TEST_METHOD(43) {
// If there are no enabled processes in the group, then disabling should
// succeed after the new process has been spawned.
initPoolDebugging();
debug->messages->send("Proceed with spawn loop iteration 1");
debug->messages->send("Proceed with spawn loop iteration 2");
Options options = createOptions();
SessionPtr session1 = pool->get(options, &ticket);
SessionPtr session2 = pool->get(options, &ticket);
ensure_equals(pool->getProcessCount(), 2u);
GroupPtr group = session1->getGroup()->shared_from_this();
ProcessPtr process1 = session1->getProcess()->shared_from_this();
ProcessPtr process2 = session2->getProcess()->shared_from_this();
AtomicInt code1 = -1, code2 = -2;
TempThread thr(boost::bind(&Core_ApplicationPool_PoolTest::disableProcess,
this, process1, &code1));
TempThread thr2(boost::bind(&Core_ApplicationPool_PoolTest::disableProcess,
this, process2, &code2));
EVENTUALLY(5,
LockGuard l(pool->syncher);
result = group->enabledCount == 0
&& group->disablingCount == 2
&& group->disabledCount == 0;
);
session1.reset();
session2.reset();
SHOULD_NEVER_HAPPEN(20,
result = code1 != -1 || code2 != -2;
);
debug->messages->send("Proceed with spawn loop iteration 3");
EVENTUALLY(5,
result = code1 == DR_SUCCESS;
);
EVENTUALLY(5,
result = code2 == DR_SUCCESS;
);
{
LockGuard l(pool->syncher);
ensure_equals(group->enabledCount, 1);
ensure_equals(group->disablingCount, 0);
ensure_equals(group->disabledCount, 2);
}
}
TEST_METHOD(44) {
// Suppose that a previous disable command triggered a new process spawn,
// and the spawn fails. Then any disabling processes should become enabled
// again, and the callbacks for the previous disable commands should be called.
initPoolDebugging();
debug->messages->send("Proceed with spawn loop iteration 1");
debug->messages->send("Proceed with spawn loop iteration 2");
Options options = createOptions();
options.minProcesses = 2;
SessionPtr session1 = pool->get(options, &ticket);
SessionPtr session2 = pool->get(options, &ticket);
ensure_equals(pool->getProcessCount(), 2u);
AtomicInt code1 = -1, code2 = -1;
TempThread thr(boost::bind(&Core_ApplicationPool_PoolTest::disableProcess,
this, session1->getProcess()->shared_from_this(), &code1));
TempThread thr2(boost::bind(&Core_ApplicationPool_PoolTest::disableProcess,
this, session2->getProcess()->shared_from_this(), &code2));
EVENTUALLY(2,
GroupPtr group = session1->getGroup()->shared_from_this();
LockGuard l(pool->syncher);
result = group->enabledCount == 0
&& group->disablingCount == 2
&& group->disabledCount == 0;
);
SHOULD_NEVER_HAPPEN(20,
result = code1 != -1 || code2 != -1;
);
if (defaultLogLevel == (LoggingKit::Level) DEFAULT_LOG_LEVEL) {
// If the user did not customize the test's log level,
// then we'll want to tone down the noise.
LoggingKit::setLevel(LoggingKit::CRIT);
}
debug->messages->send("Fail spawn loop iteration 3");
EVENTUALLY(5,
result = code1 == DR_ERROR;
);
EVENTUALLY(5,
result = code2 == DR_ERROR;
);
{
GroupPtr group = session1->getGroup()->shared_from_this();
LockGuard l(pool->syncher);
ensure_equals(group->enabledCount, 2);
ensure_equals(group->disablingCount, 0);
ensure_equals(group->disabledCount, 0);
}
}
// TODO: asyncGet() should not select a disabling process if there are enabled processes.
// TODO: asyncGet() should not select a disabling process when non-rolling restarting.
// TODO: asyncGet() should select a disabling process if there are no enabled processes
// in the group. If this happens then asyncGet() will also spawn a new process.
// TODO: asyncGet() should not select a disabled process.
// TODO: If there are no enabled processes and all disabling processes are at full
// utilization, and the process that was being spawned becomes available
// earlier than any of the disabling processes, then the newly spawned process
// should handle the request.
// TODO: A disabling process becomes disabled as soon as it's done with
// all its request.
TEST_METHOD(50) {
// Disabling a process that's already being disabled should result in the
// callback being called after disabling is done.
ensureMinProcesses(2);
Options options = createOptions();
SessionPtr session = pool->get(options, &ticket);
AtomicInt code = -1;
TempThread thr(boost::bind(&Core_ApplicationPool_PoolTest::disableProcess,
this, session->getProcess()->shared_from_this(), &code));
SHOULD_NEVER_HAPPEN(100,
result = code != -1;
);
session.reset();
EVENTUALLY(5,
result = code != -1;
);
ensure_equals(code, (int) DR_SUCCESS);
}
// TODO: Enabling a process that's disabled succeeds immediately.
// TODO: Enabling a process that's disabling succeeds immediately. The disable
// callbacks will be called with DR_CANCELED.
TEST_METHOD(51) {
// If the number of processes is already at maximum, then disabling
// a process will cause that process to be disabled, without spawning
// a new process.
pool->setMax(2);
ensureMinProcesses(2);
vector<ProcessPtr> processes = pool->getProcesses();
ensure_equals(processes.size(), 2u);
DisableResult result = pool->disableProcess(
processes[0]->getGupid());
ensure_equals(result, DR_SUCCESS);
{
ScopedLock l(pool->syncher);
GroupPtr group = processes[0]->getGroup()->shared_from_this();
ensure_equals(group->enabledCount, 1);
ensure_equals(group->disablingCount, 0);
ensure_equals(group->disabledCount, 1);
}
}
/*********** Other tests ***********/
TEST_METHOD(60) {
// The pool is considered to be at full capacity if and only
// if all Groups are at full capacity.
Options options = createOptions();
Options options2 = createOptions();
options2.appGroupName = "test";
pool->setMax(2);
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
pool->asyncGet(options2, callback);
EVENTUALLY(5,
result = number == 2;
);
ensure_equals(pool->getProcessCount(), 2u);
ensure(pool->atFullCapacity());
clearAllSessions();
pool->detachGroupByName("test");
ensure(!pool->atFullCapacity());
}
TEST_METHOD(61) {
// If the pool is at full capacity, then increasing 'max' will cause
// new processes to be spawned. Any queued get requests are processed
// as those new processes become available or as existing processes
// become available.
Options options = createOptions();
retainSessions = true;
pool->setMax(1);
pool->asyncGet(options, callback);
pool->asyncGet(options, callback);
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
pool->setMax(4);
EVENTUALLY(5,
result = number == 3;
);
ensure_equals(pool->getProcessCount(), 3u);
}
TEST_METHOD(62) {
// Each spawned process has a GUPID, which can be looked up
// through findProcessByGupid().
Options options = createOptions();
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
string gupid = currentSession->getProcess()->getGupid().toString();
ensure(!gupid.empty());
ensure_equals(currentSession->getProcess(),
pool->findProcessByGupid(gupid).get());
}
TEST_METHOD(63) {
// findProcessByGupid() returns a NULL pointer if there is
// no matching process.
ensure(pool->findProcessByGupid("none") == NULL);
}
TEST_METHOD(64) {
// Test process idle cleaning.
Options options = createOptions();
pool->setMaxIdleTime(50000);
SessionPtr session1 = pool->get(options, &ticket);
SessionPtr session2 = pool->get(options, &ticket);
ensure_equals(pool->getProcessCount(), 2u);
session2.reset();
// One of the processes still has a session open and should
// not be idle cleaned.
EVENTUALLY(2,
result = pool->getProcessCount() == 1;
);
SHOULD_NEVER_HAPPEN(150,
result = pool->getProcessCount() == 0;
);
// It shouldn't clean more processes than minInstances allows.
sessions.clear();
SHOULD_NEVER_HAPPEN(150,
result = pool->getProcessCount() == 0;
);
}
TEST_METHOD(65) {
// Test spawner idle cleaning.
Options options = createOptions();
options.appGroupName = "test1";
Options options2 = createOptions();
options2.appGroupName = "test2";
retainSessions = true;
pool->setMaxIdleTime(50000);
pool->asyncGet(options, callback);
pool->asyncGet(options2, callback);
EVENTUALLY(2,
result = number == 2;
);
ensure_equals(pool->getProcessCount(), 2u);
EVENTUALLY(2,
SpawningKit::SpawnerPtr spawner = pool->getGroup("test1")->spawner;
result = static_pointer_cast<SpawningKit::DummySpawner>(spawner)->cleanCount >= 1;
);
EVENTUALLY(2,
SpawningKit::SpawnerPtr spawner = pool->getGroup("test2")->spawner;
result = static_pointer_cast<SpawningKit::DummySpawner>(spawner)->cleanCount >= 1;
);
}
TEST_METHOD(66) {
// It should restart the app if restart.txt is created or updated.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
options.statThrottleRate = 0;
pool->setMax(1);
// Send normal request.
ensure_equals(sendRequest(options, "/"), "front page");
// Modify application; it shouldn't have effect yet.
writeFile("tmp.wsgi/passenger_wsgi.py",
"def application(env, start_response):\n"
" start_response('200 OK', [('Content-Type', 'text/html')])\n"
" return ['restarted']\n");
ensure_equals(sendRequest(options, "/"), "front page");
// Create restart.txt and send request again. The change should now be activated.
touchFile("tmp.wsgi/tmp/restart.txt", 1);
ensure_equals(sendRequest(options, "/"), "restarted");
// Modify application again; it shouldn't have effect yet.
writeFile("tmp.wsgi/passenger_wsgi.py",
"def application(env, start_response):\n"
" start_response('200 OK', [('Content-Type', 'text/html')])\n"
" return ['restarted 2']\n");
ensure_equals(sendRequest(options, "/"), "restarted");
// Touch restart.txt and send request again. The change should now be activated.
touchFile("tmp.wsgi/tmp/restart.txt", 2);
ensure_equals(sendRequest(options, "/"), "restarted 2");
}
TEST_METHOD(67) {
// Test spawn exceptions.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
writeFile("tmp.wsgi/passenger_wsgi.py",
"import sys\n"
"sys.stderr.write('Something went wrong!')\n"
"exit(1)\n");
if (defaultLogLevel == (LoggingKit::Level) DEFAULT_LOG_LEVEL) {
// If the user did not customize the test's log level,
// then we'll want to tone down the noise.
LoggingKit::setLevel(LoggingKit::CRIT);
}
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
ensure(currentException != NULL);
boost::shared_ptr<SpawningKit::SpawnException> e = dynamic_pointer_cast<SpawningKit::SpawnException>(currentException);
ensure(e->getProblemDescriptionHTML().find("Something went wrong!") != string::npos);
}
TEST_METHOD(68) {
// If a process fails to spawn, then it stops trying to spawn minProcesses processes.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
options.minProcesses = 4;
writeFile("tmp.wsgi/counter", "0");
chmod("tmp.wsgi/counter", 0666);
// Our application starts successfully the first two times,
// and fails all the other times.
writeFile("tmp.wsgi/passenger_wsgi.py",
"import sys\n"
"def application(env, start_response):\n"
" pass\n"
"counter = int(open('counter', 'r').read())\n"
"f = open('counter', 'w')\n"
"f.write(str(counter + 1))\n"
"f.close()\n"
"if counter >= 2:\n"
" sys.stderr.write('Something went wrong!')\n"
" exit(1)\n");
if (defaultLogLevel == (LoggingKit::Level) DEFAULT_LOG_LEVEL) {
// If the user did not customize the test's log level,
// then we'll want to tone down the noise.
LoggingKit::setLevel(LoggingKit::CRIT);
}
pool->asyncGet(options, callback);
EVENTUALLY(10,
result = number == 1;
);
EVENTUALLY(10,
result = pool->getProcessCount() == 2;
);
EVENTUALLY(10,
result = !pool->isSpawning();
);
SHOULD_NEVER_HAPPEN(500,
result = pool->getProcessCount() > 2;
);
}
TEST_METHOD(69) {
// It removes the process from the pool if session->initiate() fails.
Options options = createOptions();
options.appRoot = "stub/wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
options.minProcesses = 0;
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = number == 1;
);
pid_t pid = currentSession->getPid();
kill(pid, SIGTERM);
// Wait until process is gone.
EVENTUALLY(5,
result = kill(pid, 0) == -1 && (errno == ESRCH || errno == EPERM || errno == ECHILD);
);
try {
currentSession->initiate();
fail("Initiate is supposed to fail");
} catch (const SystemException &e) {
ensure_equals(e.code(), ECONNREFUSED);
}
ensure_equals(pool->getProcessCount(), 0u);
}
TEST_METHOD(70) {
// When a process has become idle, and there are waiters on the pool,
// consider detaching it in order to satisfy a waiter.
Options options1 = createOptions();
Options options2 = createOptions();
options2.appRoot = "stub/wsgi";
retainSessions = true;
pool->setMax(2);
pool->asyncGet(options1, callback);
pool->asyncGet(options1, callback);
EVENTUALLY(3,
result = pool->getProcessCount() == 2;
);
pool->asyncGet(options2, callback);
ensure_equals(pool->getWaitlist.size(), 1u);
ensure_equals(number, 2);
currentSession.reset();
sessions.pop_front();
EVENTUALLY(3,
result = number == 3;
);
ensure_equals(pool->getProcessCount(), 2u);
GroupPtr group1 = pool->groups.lookupCopy("stub/rack");
GroupPtr group2 = pool->groups.lookupCopy("stub/rack");
ensure_equals(group1->enabledCount, 1);
ensure_equals(group2->enabledCount, 1);
}
TEST_METHOD(71) {
// A process is detached after processing maxRequests sessions.
Options options = createOptions();
options.minProcesses = 0;
options.maxRequests = 5;
pool->setMax(1);
SessionPtr session = pool->get(options, &ticket);
ensure_equals(pool->getProcessCount(), 1u);
pid_t origPid = session->getPid();
session.reset();
for (int i = 0; i < 3; i++) {
pool->get(options, &ticket).reset();
ensure_equals(pool->getProcessCount(), 1u);
ensure_equals(pool->getProcesses()[0]->getPid(), origPid);
}
pool->get(options, &ticket).reset();
EVENTUALLY(2,
result = pool->getProcessCount() == 0;
);
}
TEST_METHOD(72) {
// If we restart while spawning is in progress, and the restart
// finishes before the process is done spawning, then that
// process will not be attached and the original spawn loop will
// abort. A new spawn loop will start to ensure that resource
// constraints are met.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
initPoolDebugging();
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.minProcesses = 3;
options.statThrottleRate = 0;
// Trigger spawn loop and freeze it at the point where it's spawning
// the second process.
pool->asyncGet(options, callback);
debug->debugger->recv("Begin spawn loop iteration 1");
debug->messages->send("Proceed with spawn loop iteration 1");
debug->debugger->recv("Begin spawn loop iteration 2");
ensure_equals("(1)", pool->getProcessCount(), 1u);
// Trigger restart, wait until it's finished.
touchFile("tmp.wsgi/tmp/restart.txt", 1);
pool->asyncGet(options, callback);
debug->messages->send("Finish restarting");
debug->debugger->recv("Restarting done");
ensure_equals("(2)", pool->getProcessCount(), 0u);
// The restarter should have created a new spawn loop and
// instructed the old one to stop.
debug->debugger->recv("Begin spawn loop iteration 3");
// We let the old spawn loop continue, which should drop
// the second process and abort.
debug->messages->send("Proceed with spawn loop iteration 2");
debug->debugger->recv("Spawn loop done");
ensure_equals("(3)", pool->getProcessCount(), 0u);
// We let the new spawn loop continue.
debug->messages->send("Proceed with spawn loop iteration 3");
debug->messages->send("Proceed with spawn loop iteration 4");
debug->messages->send("Proceed with spawn loop iteration 5");
debug->debugger->recv("Spawn loop done");
ensure_equals("(4)", pool->getProcessCount(), 3u);
}
TEST_METHOD(73) {
// If a get() request comes in while the restart is in progress, then
// that get() request will be put into the get waiters list, which will
// be processed after spawning is done.
// Spawn 2 processes.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.minProcesses = 2;
options.statThrottleRate = 0;
pool->asyncGet(options, callback);
EVENTUALLY(2,
result = pool->getProcessCount() == 2;
);
// Trigger a restart. The creation of the new spawner should take a while.
skDebugSupport.spawnerCreationSleepTime = 20000;
touchFile("tmp.wsgi/tmp/restart.txt");
pool->asyncGet(options, callback);
GroupPtr group = pool->findOrCreateGroup(options);
ensure_equals("(1)", pool->getProcessCount(), 0u);
ensure_equals("(2)", group->getWaitlist.size(), 1u);
// Now that the restart is in progress, perform a get().
pool->asyncGet(options, callback);
ensure_equals("(3)", group->getWaitlist.size(), 2u);
EVENTUALLY(2,
result = number == 3;
);
ensure_equals("(4) The restart function respects minProcesses",
pool->getProcessCount(), 2u);
}
TEST_METHOD(74) {
// If a process fails to spawn, it sends a SpawnException result to all get waiters.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
chmod("tmp.wsgi", 0777);
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
pool->setMax(1);
writeFile("tmp.wsgi/passenger_wsgi.py",
"import os, time, sys\n"
"\n"
"def file_exists(filename):\n"
" try:\n"
" os.stat(filename)\n"
" return True\n"
" except OSError:\n"
" return False\n"
"\n"
"f = open('spawned.txt', 'w')\n"
"f.write(str(os.getpid()))\n"
"f.close()\n"
"while not file_exists('continue.txt'):\n"
" time.sleep(0.05)\n"
"sys.stderr.write('Something went wrong!')\n"
"exit(1)\n");
retainSessions = true;
if (defaultLogLevel == (LoggingKit::Level) DEFAULT_LOG_LEVEL) {
// If the user did not customize the test's log level,
// then we'll want to tone down the noise.
LoggingKit::setLevel(LoggingKit::CRIT);
}
pool->asyncGet(options, callback);
pool->asyncGet(options, callback);
pool->asyncGet(options, callback);
pool->asyncGet(options, callback);
EVENTUALLY(5,
result = fileExists("tmp.wsgi/spawned.txt");
);
usleep(20000);
writeFile("tmp.wsgi/passenger_wsgi.py", unsafeReadFile("stub/wsgi/passenger_wsgi.py"));
pid_t pid = (pid_t) stringToLL(unsafeReadFile("tmp.wsgi/spawned.txt"));
kill(pid, SIGTERM);
EVENTUALLY(5,
result = number == 4;
);
ensure_equals(pool->getProcessCount(), 0u);
ensure(sessions.empty());
}
TEST_METHOD(75) {
// If a process fails to spawn, the existing processes
// are kept alive.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
options.minProcesses = 2;
// Spawn 2 processes.
retainSessions = true;
pool->asyncGet(options, callback);
pool->asyncGet(options, callback);
EVENTUALLY(10,
result = number == 2;
);
ensure_equals(pool->getProcessCount(), 2u);
// Mess up the application and spawn a new one.
writeFile("tmp.wsgi/passenger_wsgi.py",
"import sys\n"
"sys.stderr.write('Something went wrong!')\n"
"exit(1)\n");
if (defaultLogLevel == (LoggingKit::Level) DEFAULT_LOG_LEVEL) {
// If the user did not customize the test's log level,
// then we'll want to tone down the noise.
LoggingKit::setLevel(LoggingKit::CRIT);
}
try {
currentSession = pool->get(options, &ticket);
fail("SpawnException expected");
} catch (const SpawningKit::SpawnException &) {
ensure_equals(pool->getProcessCount(), 2u);
}
}
TEST_METHOD(76) {
// No more than maxOutOfBandWorkInstances process will be performing
// out-of-band work at the same time.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
options.maxOutOfBandWorkInstances = 2;
initPoolDebugging();
debug->restarting = false;
debug->spawning = false;
debug->oobw = true;
// Spawn 3 processes and initiate 2 OOBW requests.
SessionPtr session1 = pool->get(options, &ticket);
SessionPtr session2 = pool->get(options, &ticket);
SessionPtr session3 = pool->get(options, &ticket);
session1->requestOOBW();
session1.reset();
session2->requestOOBW();
session2.reset();
// 2 OOBW requests eventually start.
debug->debugger->recv("OOBW request about to start");
debug->debugger->recv("OOBW request about to start");
// Request another OOBW, but this one is not initiated.
session3->requestOOBW();
session3.reset();
SHOULD_NEVER_HAPPEN(100,
result = debug->debugger->peek("OOBW request about to start") != NULL;
);
// Let one OOBW request finish. The third one should eventually
// start.
debug->messages->send("Proceed with OOBW request");
debug->debugger->recv("OOBW request about to start");
debug->messages->send("Proceed with OOBW request");
debug->messages->send("Proceed with OOBW request");
debug->debugger->recv("OOBW request finished");
debug->debugger->recv("OOBW request finished");
debug->debugger->recv("OOBW request finished");
}
TEST_METHOD(77) {
// If the getWaitlist already has maxRequestQueueSize items,
// then an exception is returned.
Options options = createOptions();
options.appGroupName = "test1";
options.maxRequestQueueSize = 3;
GroupPtr group = pool->findOrCreateGroup(options);
skDebugSupport.dummyConcurrency = 3;
initPoolDebugging();
pool->setMax(1);
for (int i = 0; i < 3; i++) {
pool->asyncGet(options, callback);
}
ensure_equals(number, 0);
{
LockGuard l(pool->syncher);
ensure_equals(group->getWaitlist.size(),
3u);
}
try {
pool->get(options, &ticket);
fail("Expected RequestQueueFullException");
} catch (const RequestQueueFullException &e) {
// OK
}
debug->messages->send("Proceed with spawn loop iteration 1");
debug->messages->send("Spawn loop done");
EVENTUALLY(5,
result = number == 3;
);
}
TEST_METHOD(78) {
// Test restarting while a previous restart was already being finalized.
// The previous finalization should abort.
Options options = createOptions();
initPoolDebugging();
debug->spawning = false;
pool->get(options, &ticket);
ensure_equals(pool->restartGroupsByAppRoot(options.appRoot), 1u);
debug->debugger->recv("About to end restarting");
ensure_equals(pool->restartGroupsByAppRoot(options.appRoot), 1u);
debug->debugger->recv("About to end restarting");
debug->messages->send("Finish restarting");
debug->messages->send("Finish restarting");
debug->debugger->recv("Restarting done");
debug->debugger->recv("Restarting aborted");
}
TEST_METHOD(79) {
// Test sticky sessions.
// Spawn 2 processes and get their sticky session IDs and PIDs.
ensureMinProcesses(2);
Options options = createOptions();
SessionPtr session1 = pool->get(options, &ticket);
SessionPtr session2 = pool->get(options, &ticket);
int id1 = session1->getStickySessionId();
int id2 = session2->getStickySessionId();
pid_t pid1 = session1->getPid();
pid_t pid2 = session2->getPid();
session1.reset();
session2.reset();
// Make two requests with id1 as sticky session ID. They should
// both go to process pid1.
options.stickySessionId = id1;
session1 = pool->get(options, &ticket);
ensure_equals("Request 1.1 goes to process 1", session1->getPid(), pid1);
// The second request should be queued, and should not finish until
// the first request is finished.
ensure_equals(number, 1);
pool->asyncGet(options, callback);
SHOULD_NEVER_HAPPEN(100,
result = number > 1;
);
session1.reset();
EVENTUALLY(1,
result = number == 2;
);
ensure_equals("Request 1.2 goes to process 1", currentSession->getPid(), pid1);
currentSession.reset();
// Make two requests with id2 as sticky session ID. They should
// both go to process pid2.
options.stickySessionId = id2;
session1 = pool->get(options, &ticket);
ensure_equals("Request 2.1 goes to process 2", session1->getPid(), pid2);
// The second request should be queued, and should not finish until
// the first request is finished.
pool->asyncGet(options, callback);
SHOULD_NEVER_HAPPEN(100,
result = number > 2;
);
session1.reset();
EVENTUALLY(1,
result = number == 3;
);
ensure_equals("Request 2.2 goes to process 2", currentSession->getPid(), pid2);
currentSession.reset();
}
// TODO: Persistent connections.
// TODO: If one closes the session before it has reached EOF, and process's maximum concurrency
// has already been reached, then the pool should ping the process so that it can detect
// when the session's connection has been released by the app.
/*********** Test previously discovered bugs ***********/
TEST_METHOD(85) {
// Test detaching, then restarting. This should not violate any invariants.
TempDirCopy dir("stub/wsgi", "tmp.wsgi");
Options options = createOptions();
options.appRoot = "tmp.wsgi";
options.appType = "wsgi";
options.startupFile = "passenger_wsgi.py";
options.spawnMethod = "direct";
options.statThrottleRate = 0;
SessionPtr session = pool->get(options, &ticket);
string gupid = session->getProcess()->getGupid().toString();
session.reset();
pool->detachProcess(gupid);
touchFile("tmp.wsgi/tmp/restart.txt", 1);
pool->get(options, &ticket).reset();
}
/*****************************/
}
Mini Shell