00001 #include "Archiver.h"
00002 #include "ArchivingWorkerGroup.h"
00003 #include "Sct/Archive/IOManagerArchiveFile.h"
00004 #include "Sct/SctNames.h"
00005 #include "Sct/IpcObjectException.h"
00006 #include "Sct/UnsupportedOperationError.h"
00007 #include "Sct/StdExceptionWrapper.h"
00008 #include "Sct/ISStreamerWrapper.h"
00009 #include "is/isinfo.h"
00010 #include <pmg/pmg_initSync.h>
00011
00012 #include "unistd.h"
00013 #include <memory>
00014
00015 #include "TransferCommand.h"
00016 #include "IsGetCommand.h"
00017 #include "IsPutCommand.h"
00018 #include "ArchiveGetCommand.h"
00019 #include "ArchivePutCommand.h"
00020 #include "Sct/Archive/IOParamsArchive.h"
00021 #include "Sct/OmniMarshalling.h"
00022 #include "Sct/ApplicationStartupDebugTools.h"
00023
00024 #include "ipc/core.h"
00025
00026 using namespace std;
00027
00028 using namespace Sct;
00029 using namespace Sct::Archive;
00030 using namespace Sct::IS;
00031
00032 using boost::recursive_mutex;
00033
00034
00035
00036
00037
00038
00039
00040 int main(int argc, char** argv) {
00041 using namespace SctArchiving;
00042 setExceptionHandlers(argv[0]);
00043
00044 IPCCore::init(argc,argv);
00045 Sct::ApplicationStartupDebugTools::announceStartOfMain(argc, argv, __FILE__, __LINE__);
00046
00047
00048 nice(10);
00049
00050 ArchiverArguments args(argc, argv);
00051 args.print(std::cout);
00052
00053 try {
00054 Archiver& s=Archiver::initialize(args);
00055 if (!s.publish()){
00056 throw IpcObjectException("ArchivingService failed to publish", __FILE__, __LINE__);
00057 }
00058 s.go(1);
00059 pmg_initSync();
00060 s.getArchiverServer().run();
00061 } catch (Throwable& e) {
00062 e.sendToMrs(MRS_FATAL);
00063 terminate();
00064 }
00065 }
00066
00067 namespace SctArchiving {
00068
00069 Archiver* Archiver::archiver = 0;
00070
00071 Archiver& Archiver::instance() {
00072 if(!archiver) throw IllegalStateError("Attempt to use uninitialised Archiver", __FILE__, __LINE__);
00073 return *archiver;
00074 }
00075
00076 Archiver& Archiver::initialize(ArchiverArguments args){
00077 archiver = new Archiver(args);
00078 return *archiver;
00079 }
00080
00081 Archiver::Archiver(ArchiverArguments args) :
00082 IPCNamedObject<POA_ArchivingServiceI::ArchivingServiceInterface>(SctNames::getPartition(), args.instanceName()),
00083 nArchived(), nRetrieved(0),
00084 nValidated(0), isTimeTaken(0.),
00085 fileTimeTaken(0.), m_args(args) {
00086
00087 auto_ptr<ISInfoReceiver> ir(new ISInfoReceiver(SctNames::getPartition()));
00088 if (!ir.get()) throw ConfigurationException("Archiver::initialize can't make infoReceiver ", __FILE__, __LINE__) ;
00089 infoReceiver = ir;
00090 workergroup = new ArchivingWorkerGroup();
00091 workergroup->reportTo(args.getISStatusName());
00092 ostringstream oss;
00093 oss << "Started with persistent directory=" << m_args.getPersistentDirectory()
00094 << " and retrieval server = " << m_args.getOutputISServer();
00095
00096 SctNames::Mrs() << "ARCHIVING_SERVICE" << MRS_TEXT(oss.str())
00097 << MRS_INFORMATION << ENDM;
00098 m_archive_manager = &IOManagerArchiveFile::instance();
00099 m_retrieval_is_server=Sct::SctNames::getRetrievedDataName();
00100 m_suspend_callback=false;
00101 m_archive_manager->setCompressionLevel(m_args.getCompressionLevel());
00102 }
00103
00104 Archiver::~Archiver() throw() {
00105 while (workergroup->busy() || workergroup->queueSize() ) {
00106 sleep(1);
00107 }
00108 workergroup->join();
00109 }
00110
00111 void Archiver::addCommand(boost::shared_ptr<ArchivingCommand> command)const {
00112 workergroup->push(command);
00113 }
00114
00115 IPCServer& Archiver::getArchiverServer() throw() {
00116 static IPCServer archiverServer;
00117 return archiverServer;
00118 }
00119
00120 inline bool isControl(std::string name){
00121 return ( name.find("TestData")!=string::npos || name.find("ControlData")!=string::npos);
00122 };
00123
00124 void Archiver::archive_callback(ISCallbackInfo *isc){
00125 try{
00126 bool over_write=isControl(isc->name());
00127 if (isc->reason() == ISInfoUpdated && !over_write ) {
00128 throw IoException(string("That is odd ... ") + isc->name() + " has been modified! I'm not going to overwrite the data."
00129 , __FILE__, __LINE__);
00130 }
00131 if (isc->reason() != ISInfoCreated && isc->reason() != ISInfoUpdated) return;
00132 if (Archiver::instance().m_suspend_callback){
00133 return;
00134 }
00135
00136 shared_ptr<IONameIS> name ( new IONameIS(isc->name()));
00137 shared_ptr<GetCommand> get (new IsGetCommand(name));
00138 shared_ptr<PutCommand> put (new ArchivePutCommand());
00139
00140 if (over_write){
00141 shared_ptr<Sct::Archive::IOParamsArchive> params;
00142 params = shared_ptr<Sct::Archive::IOParamsArchive>(new IOParamsArchive());
00143 params->setOverWrite();
00144 put->setParams(params);
00145 }
00146
00147 shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00148 Archiver::instance().workergroup->push( command );
00149
00150 } catch (Sct::Throwable& e){
00151 e.sendToMrs();
00152 }catch(std::exception& e){
00153 StdExceptionWrapper(e).sendToMrs();
00154 }
00155 }
00156
00157 IOManagerArchive& Archiver::getIOManagerArchive() const {
00158 return *m_archive_manager;
00159 }
00160
00161 string Archiver::internal_getRetrieveIsServer() const throw(){
00162 return m_retrieval_is_server;
00163 }
00164
00165
00166 char * Archiver::getRetrieveIsServer() throw() {
00167 return copyStringToCorba(internal_getRetrieveIsServer());
00168 };
00169
00170 void Archiver::setRetrieveIsServer (const char * serverName){
00171 m_retrieval_is_server=serverName;
00172 }
00173
00174 void Archiver::setPersistentDirectory(const char* newDirectory) {
00175 SctNames::setPersistentDir(newDirectory);
00176 }
00177
00178 void Archiver::suspendCallbacks (CORBA::Boolean doSuspend){
00179 if (doSuspend) SctNames::Mrs() << MRS_TEXT("ArchivingService being suspended from callbacks")
00180 << "ARCHIVE_SUSPEND" << MRS_INFORMATION << ENDM;
00181 m_suspend_callback=doSuspend;
00182 }
00183
00184
00185 void Archiver::subscribe(const string& servername, const string& regexp, ISCallbackInfo::Callback callback){
00186 if (servername==m_args.getOutputISServer()){
00187 ostringstream oss;
00188 oss << "ArchivingService is not allowed to subscribe to [" << servername
00189 << "] since it is publishing there and this could set up a loop";
00190 IoException(oss.str(), __FILE__, __LINE__).sendToMrs();
00191 }
00192 ISInfo::Status s=infoReceiver->subscribe(servername.c_str(), regexp.c_str(), callback);
00193 if (s!=ISInfo::Success) {
00194 ostringstream os;
00195 os <<"Archiving service could not subscribe to [" << servername
00196 << "] to retrieve [" << regexp << "]";
00197 IsException(s, os.str(), __FILE__, __LINE__).sendToMrs();
00198 } else {
00199 ostringstream oss;
00200 oss << "Archiving service subscribed to IS server [" << servername
00201 << "] to retrieve [" << regexp << "]";
00202 SctNames::Mrs() << "ARCHIVE_SUBSCRIBE" << MRS_TEXT(oss.str())
00203 << MRS_INFORMATION << ENDM;
00204 }
00205 }
00206
00207 void Archiver::go(unsigned nWorker) throw(IsException) {
00208 const list<SctService::Arguments::Subscription> theList = m_args.getInputISServers();
00209 for (list<SctService::Arguments::Subscription>::const_iterator i=theList.begin();
00210 i!= theList.end(); ++i){
00211 subscribe((*i).server, (*i).regexp, archive_callback);
00212 }
00213 workergroup->go(m_args.getNWorkers());
00214 if (m_args.recoveryMode()) recover();
00215 }
00216
00217 void Archiver::incrimentNArchived() throw(){
00218 recursive_mutex::scoped_lock scoped_lock(counterMutex);
00219 ++nArchived;
00220 }
00221
00222 void Archiver::incrimentNRetrieved() throw(){
00223 recursive_mutex::scoped_lock scoped_lock(counterMutex);
00224 ++nRetrieved;
00225 }
00226
00227 void Archiver::incrimentNValidated() throw(){
00228 recursive_mutex::scoped_lock scoped_lock(counterMutex);
00229 ++nValidated;
00230 }
00231
00232 void Archiver::addISTime(double time) throw(){
00233 recursive_mutex::scoped_lock scoped_lock(counterMutex);
00234 isTimeTaken += time;
00235 }
00236
00237 void Archiver::addFileTime(double time) throw(){
00238 recursive_mutex::scoped_lock scoped_lock(counterMutex);
00239 fileTimeTaken += time;
00240 }
00241
00242 const char* Archiver::getStatus() const throw(){
00243 recursive_mutex::scoped_lock scoped_lock(counterMutex);
00244 ostringstream os;
00245 os << "\t=== Archiving Service ===" << endl
00246 << "nWorkers=" << workergroup->nWorkers() << endl
00247 << "\t(Busy=" << workergroup->busy() << ")" << endl
00248 << "Queue = " << workergroup->queueSize() << endl
00249 << "# Archived = " << nArchived << endl
00250 << "# Retrieved = " << nRetrieved <<endl
00251 << "# Validated = " << nValidated << endl
00252 << " Approx average time / command / thread :" << endl
00253 << "\tArchive = " << getAverageTime(fileTimeTaken) << endl
00254 << "\tIS/ISProxy = " << getAverageTime(isTimeTaken) << endl;
00255 os << "Compression level = " << getIOManagerArchive().getCompressionLevel() << endl;
00256 os << "IOManagerArchive Information: \n"
00257 << getIOManagerArchive().status() << endl;
00258 return os.str().c_str();
00259 }
00260
00261 double Archiver::getAverageTime(double time) const throw(){
00262 recursive_mutex::scoped_lock scoped_lock(counterMutex);
00263 long ncommands = nArchived + nRetrieved + nValidated;
00264 return (ncommands>0) ? time/ncommands : 0.;
00265 }
00266
00267 char* Archiver::status(){
00268 recursive_mutex::scoped_lock scoped_lock(counterMutex);
00269 std::string status=getStatus();
00270 char *statusMessage = new char[status.size()+1];
00271 strcpy(statusMessage, status.c_str());
00272 return statusMessage;
00273 }
00274
00275 void Archiver::archiveISName(const char* ioNameIS){
00276 try {
00277 shared_ptr<IONameIS> name( new IONameIS(ioNameIS));
00278 shared_ptr<GetCommand> get (new IsGetCommand(name));
00279 shared_ptr<PutCommand> put (new ArchivePutCommand());
00280
00281 if (isControl(ioNameIS)){
00282 shared_ptr<Sct::Archive::IOParamsArchive> params;
00283 params = shared_ptr<Sct::Archive::IOParamsArchive>(new IOParamsArchive());
00284 params->setOverWrite();
00285 put->setParams(params);
00286 }
00287
00288 shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00289 workergroup->push( command );
00290 }catch(Throwable& e){
00291 e.sendToMrs();
00292 }
00293 }
00294
00295 void Archiver::retrieveISName (const char* ioNameIS){
00296 try{
00297 IONameIS nameis(ioNameIS);
00298 IONameArchiveFile archfile( nameis.getUniqueID(), nameis.getClassName());
00300 char* blah = const_cast<char*> (archfile.getIOName().c_str());
00301 retrieveArchName (blah);
00302 }catch(Throwable& e){
00303 e.sendToMrs();
00304 }
00305 }
00306
00307 void Archiver::retrieve(const char* runNumber, const char* scanNumber, const char* className, const char* specifier){
00308 try{
00309 std::ostringstream oss;
00310 oss << m_args.getPersistentDirectory() << "/" << className
00311 << "." << runNumber << "." << scanNumber
00312 << "." << specifier << ".gz";
00313 IONameArchiveFile arch(oss.str());
00314 char* blah = const_cast<char*>(arch.getIOName().c_str());
00315 retrieveArchName(blah);
00316 }catch(Throwable& e){
00317 e.sendToMrs();
00318 }
00319 }
00320
00321 void Archiver::retrieveArchName (const char* archivingName){
00322 try{
00323 shared_ptr<IONameArchiveFile> name(new IONameArchiveFile(archivingName));
00324 shared_ptr<GetCommand> get (new ArchiveGetCommand(name));
00325 shared_ptr<IsPutCommand> put (new IsPutCommand());
00326 put->setServer(internal_getRetrieveIsServer());
00327 shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00328 workergroup->push( command );
00329 }catch(Throwable& e){
00330 e.sendToMrs();
00331 }
00332 }
00333
00334
00335 CORBA::Short Archiver::busy(){
00336 return workergroup->busy();
00337 }
00338
00339 CORBA::Short Archiver::queueLength(){
00340 return workergroup->queueSize();
00341 }
00342
00343 CORBA::Short Archiver::getCompressionLevel () {
00344 return getIOManagerArchive().getCompressionLevel();
00345 }
00346
00347 void Archiver::setCompressionLevel (CORBA::Short level) {
00348 getIOManagerArchive().setCompressionLevel(level);
00349 }
00350 void Archiver::recover(){
00351 SctNames::Mrs() << MRS_TEXT("Sorry - recovery not yet implimented")
00352 << "ANALYSIS_RECOVER" << MRS_DIAGNOSTIC << ENDM;
00353 }
00354 }