Main Page | Modules | Namespace List | Class Hierarchy | Alphabetical List | Data Structures | File List | Namespace Members | Data Fields | Related Pages

Archiver.cpp

00001 #include "Archiver.h"
00002 #include "ArchivingWorkerGroup.h"
00003 #include "Sct/Archive/IOManagerArchiveFile.h" 
00004 #include "Sct/SctNames.h"
00005 #include "Sct/IpcObjectException.h"
00006 #include "Sct/UnsupportedOperationError.h"
00007 #include "Sct/StdExceptionWrapper.h"
00008 #include "Sct/ISStreamerWrapper.h"
00009 #include "is/isinfo.h"
00010 #include <pmg/pmg_initSync.h>
00011 
00012 #include "unistd.h"
00013 #include <memory>
00014 
00015 #include "TransferCommand.h"
00016 #include "IsGetCommand.h"
00017 #include "IsPutCommand.h"
00018 #include "ArchiveGetCommand.h"
00019 #include "ArchivePutCommand.h"
00020 #include "Sct/Archive/IOParamsArchive.h"
00021 #include "Sct/OmniMarshalling.h"
00022 #include "Sct/ApplicationStartupDebugTools.h"
00023 
00024 #include "ipc/core.h"
00025 
00026 using namespace std;
00027 
00028 using namespace Sct;
00029 using namespace Sct::Archive;
00030 using namespace Sct::IS;
00031 
00032 using boost::recursive_mutex;
00033 
00034 /* Obsolete:
00035 void pmgSynch(void *) {
00036     pmg_initSync();
00037 }
00038 */
00039 
00040 int main(int argc, char** argv) {
00041     using namespace SctArchiving;
00042     setExceptionHandlers(argv[0]);
00043 
00044     IPCCore::init(argc,argv);
00045     Sct::ApplicationStartupDebugTools::announceStartOfMain(argc, argv, __FILE__, __LINE__);
00046  
00047     // renice this job!
00048     nice(10);
00049     
00050     ArchiverArguments args(argc, argv);
00051     args.print(std::cout);
00052 
00053     try {
00054       Archiver& s=Archiver::initialize(args);
00055       if (!s.publish()){
00056     throw IpcObjectException("ArchivingService failed to publish", __FILE__, __LINE__);
00057       }
00058       s.go(1);
00059       pmg_initSync();
00060       s.getArchiverServer().run();
00061     } catch (Throwable& e) {
00062       e.sendToMrs(MRS_FATAL);
00063       terminate();
00064     }
00065 }
00066 
00067 namespace SctArchiving {
00068 
00069 Archiver* Archiver::archiver = 0;
00070 
00071 Archiver& Archiver::instance() {
00072   if(!archiver) throw IllegalStateError("Attempt to use uninitialised Archiver", __FILE__, __LINE__);
00073   return *archiver;
00074 }
00075 
00076 Archiver& Archiver::initialize(ArchiverArguments args){
00077   archiver = new Archiver(args);
00078   return *archiver;
00079 }
00080 
00081 Archiver::Archiver(ArchiverArguments args) : 
00082     IPCNamedObject<POA_ArchivingServiceI::ArchivingServiceInterface>(SctNames::getPartition(), args.instanceName()),
00083     nArchived(),     nRetrieved(0), 
00084     nValidated(0),   isTimeTaken(0.), 
00085     fileTimeTaken(0.), m_args(args) {
00086   
00087   auto_ptr<ISInfoReceiver> ir(new ISInfoReceiver(SctNames::getPartition()));
00088   if (!ir.get()) throw ConfigurationException("Archiver::initialize can't make infoReceiver ", __FILE__, __LINE__) ;
00089   infoReceiver = ir;
00090   workergroup = new ArchivingWorkerGroup();
00091   workergroup->reportTo(args.getISStatusName());
00092   ostringstream oss;
00093   oss << "Started with persistent directory=" << m_args.getPersistentDirectory()
00094       << " and retrieval server = " << m_args.getOutputISServer();
00095 
00096   SctNames::Mrs() << "ARCHIVING_SERVICE" << MRS_TEXT(oss.str()) 
00097           << MRS_INFORMATION << ENDM;
00098   m_archive_manager = &IOManagerArchiveFile::instance();
00099   m_retrieval_is_server=Sct::SctNames::getRetrievedDataName();
00100   m_suspend_callback=false;
00101   m_archive_manager->setCompressionLevel(m_args.getCompressionLevel());
00102 }
00103 
00104 Archiver::~Archiver() throw() {
00105   while (workergroup->busy() || workergroup->queueSize() ) {
00106     sleep(1);
00107   }
00108   workergroup->join();
00109 }
00110 
00111 void Archiver::addCommand(boost::shared_ptr<ArchivingCommand> command)const {
00112   workergroup->push(command);
00113 }
00114 
00115 IPCServer& Archiver::getArchiverServer() throw() {
00116   static IPCServer archiverServer;
00117   return archiverServer;
00118 }
00119 
00120 inline bool isControl(std::string name){
00121   return ( name.find("TestData")!=string::npos || name.find("ControlData")!=string::npos);
00122 };
00123 
00124 void Archiver::archive_callback(ISCallbackInfo *isc){
00125   try{
00126     bool over_write=isControl(isc->name());
00127     if (isc->reason() == ISInfoUpdated && !over_write ) {
00128       throw IoException(string("That is odd ... ") + isc->name() +  " has been modified! I'm not going to overwrite the data."
00129             , __FILE__, __LINE__);
00130     }
00131     if (isc->reason() != ISInfoCreated && isc->reason() != ISInfoUpdated) return;
00132     if (Archiver::instance().m_suspend_callback){
00133         return;
00134     }
00135 
00136     shared_ptr<IONameIS> name ( new IONameIS(isc->name()));
00137     shared_ptr<GetCommand> get (new IsGetCommand(name));
00138     shared_ptr<PutCommand> put (new ArchivePutCommand());
00139 
00140     if (over_write){
00141       shared_ptr<Sct::Archive::IOParamsArchive> params;
00142       params = shared_ptr<Sct::Archive::IOParamsArchive>(new IOParamsArchive());
00143       params->setOverWrite();
00144       put->setParams(params);
00145     }
00146 
00147     shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00148     Archiver::instance().workergroup->push( command );
00149     
00150   } catch (Sct::Throwable& e){
00151     e.sendToMrs();
00152   }catch(std::exception& e){
00153     StdExceptionWrapper(e).sendToMrs();
00154   }
00155 }
00156 
00157 IOManagerArchive& Archiver::getIOManagerArchive() const {
00158   return *m_archive_manager;
00159 }
00160 
00161 string Archiver::internal_getRetrieveIsServer() const throw(){
00162   return  m_retrieval_is_server;
00163 }
00164 
00165   
00166 char * Archiver::getRetrieveIsServer() throw() {
00167   return copyStringToCorba(internal_getRetrieveIsServer());
00168 };
00169 
00170 void  Archiver::setRetrieveIsServer (const char * serverName){
00171   m_retrieval_is_server=serverName;
00172 }
00173 
00174 void Archiver::setPersistentDirectory(const char* newDirectory) {
00175   SctNames::setPersistentDir(newDirectory);
00176 }
00177 
00178 void Archiver::suspendCallbacks (CORBA::Boolean doSuspend){
00179    if (doSuspend) SctNames::Mrs() << MRS_TEXT("ArchivingService being suspended from callbacks") 
00180                   << "ARCHIVE_SUSPEND" << MRS_INFORMATION << ENDM;
00181    m_suspend_callback=doSuspend;
00182 }
00183 
00184 
00185 void Archiver::subscribe(const string& servername, const string& regexp, ISCallbackInfo::Callback callback){
00186   if (servername==m_args.getOutputISServer()){
00187     ostringstream oss;
00188     oss << "ArchivingService is not allowed to subscribe to [" << servername 
00189     << "] since it is publishing there and this could set up a loop";
00190     IoException(oss.str(), __FILE__, __LINE__).sendToMrs();
00191   }
00192   ISInfo::Status s=infoReceiver->subscribe(servername.c_str(), regexp.c_str(), callback);
00193   if (s!=ISInfo::Success) {
00194     ostringstream os;
00195     os <<"Archiving service could not subscribe to [" << servername
00196        << "] to retrieve [" << regexp << "]";
00197     IsException(s, os.str(), __FILE__, __LINE__).sendToMrs();
00198   } else {
00199     ostringstream oss;
00200     oss << "Archiving service subscribed to IS server [" << servername
00201     << "] to retrieve [" << regexp << "]";
00202     SctNames::Mrs() << "ARCHIVE_SUBSCRIBE" << MRS_TEXT(oss.str())
00203             << MRS_INFORMATION << ENDM;
00204   }
00205 }
00206   
00207   void Archiver::go(unsigned nWorker) throw(IsException) {
00208     const list<SctService::Arguments::Subscription> theList = m_args.getInputISServers();
00209     for (list<SctService::Arguments::Subscription>::const_iterator i=theList.begin(); 
00210      i!= theList.end(); ++i){
00211       subscribe((*i).server, (*i).regexp, archive_callback);
00212     }
00213     workergroup->go(m_args.getNWorkers());
00214     if (m_args.recoveryMode()) recover();
00215   }
00216 
00217   void Archiver::incrimentNArchived() throw(){
00218     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00219     ++nArchived;
00220   }
00221   
00222   void Archiver::incrimentNRetrieved() throw(){
00223     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00224     ++nRetrieved;
00225   }
00226   
00227   void Archiver::incrimentNValidated() throw(){
00228     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00229     ++nValidated;
00230   }
00231   
00232   void Archiver::addISTime(double time) throw(){
00233     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00234     isTimeTaken += time;
00235   }
00236 
00237   void Archiver::addFileTime(double time) throw(){
00238     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00239     fileTimeTaken += time;
00240   }
00241 
00242 const char* Archiver::getStatus() const throw(){
00243   recursive_mutex::scoped_lock scoped_lock(counterMutex);
00244   ostringstream os;
00245   os << "\t=== Archiving Service ===" << endl
00246      << "nWorkers=" << workergroup->nWorkers() << endl
00247      << "\t(Busy=" << workergroup->busy() << ")" << endl
00248      << "Queue = " << workergroup->queueSize();
00249   if (workergroup->isFifo()) {
00250     os << " [FIFO]\n";
00251   }else{
00252     os << " [FILO]\n";
00253   }
00254   os<< "# Archived = " << nArchived << endl
00255      << "# Retrieved = " << nRetrieved  <<endl
00256      << "# Validated = " << nValidated << endl
00257      << " Approx average time / command / thread :" << endl 
00258      << "\tArchive = " << getAverageTime(fileTimeTaken) << endl
00259      << "\tIS/ISProxy = " << getAverageTime(isTimeTaken) << endl;
00260   os << "Compression level = " << getIOManagerArchive().getCompressionLevel() << endl;
00261   os << "IOManagerArchive Information: \n"
00262      << getIOManagerArchive().status() << endl;
00263   return os.str().c_str();
00264 }
00265 
00266   double Archiver::getAverageTime(double time) const throw(){
00267     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00268     long ncommands =  nArchived + nRetrieved + nValidated;
00269     return (ncommands>0) ? time/ncommands : 0.;
00270   }
00271 
00272   char* Archiver::status(){
00273     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00274     std::string status=getStatus();
00275     char *statusMessage = new char[status.size()+1];
00276     strcpy(statusMessage, status.c_str());
00277     return statusMessage;
00278   }
00279   
00280   void Archiver::archiveISName(const char* ioNameIS){
00281     try {
00282       shared_ptr<IONameIS> name( new IONameIS(ioNameIS));
00283       shared_ptr<GetCommand> get (new IsGetCommand(name));
00284       shared_ptr<PutCommand> put (new ArchivePutCommand());
00285 
00286       if (isControl(ioNameIS)){
00287     shared_ptr<Sct::Archive::IOParamsArchive> params;
00288     params = shared_ptr<Sct::Archive::IOParamsArchive>(new IOParamsArchive());
00289     params->setOverWrite();
00290     put->setParams(params);
00291       }
00292 
00293       shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00294       workergroup->push( command );
00295     }catch(Throwable& e){
00296       e.sendToMrs();
00297     }
00298   }
00299 
00300   void Archiver::retrieveISName (const char* ioNameIS){
00301     try{ 
00302       IONameIS nameis(ioNameIS);
00303       IONameArchiveFile archfile( nameis.getUniqueID(), nameis.getClassName());
00305       char* blah = const_cast<char*> (archfile.getIOName().c_str());
00306       retrieveArchName (blah);
00307     }catch(Throwable& e){
00308       e.sendToMrs();
00309     }
00310   }
00311 
00312   void Archiver::retrieve(const char* runNumber, const char* scanNumber, const char* className, const char* specifier){
00313     try{ 
00314       std::ostringstream oss;
00315       oss << m_args.getPersistentDirectory() << "/" << className 
00316       << "." << runNumber << "." << scanNumber 
00317       << "." << specifier << ".gz";
00318       IONameArchiveFile arch(oss.str());
00319       char* blah = const_cast<char*>(arch.getIOName().c_str());
00320       retrieveArchName(blah);
00321     }catch(Throwable& e){
00322       e.sendToMrs();
00323     }
00324   }
00325 
00326   void Archiver::retrieveArchName (const char* archivingName){
00327     try{
00328       shared_ptr<IONameArchiveFile> name(new IONameArchiveFile(archivingName));
00329       shared_ptr<GetCommand> get (new ArchiveGetCommand(name));
00330       shared_ptr<IsPutCommand> put (new IsPutCommand());
00331       put->setServer(internal_getRetrieveIsServer());
00332       shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00333       workergroup->push( command );
00334     }catch(Throwable& e){
00335       e.sendToMrs();
00336     }
00337   }
00338 
00339   
00340   CORBA::Short Archiver::busy(){
00341     return workergroup->busy();
00342   }
00343 
00344   CORBA::Short Archiver::queueLength(){
00345     return workergroup->queueSize();
00346   }
00347 
00348   CORBA::Short Archiver::getCompressionLevel () {
00349     return getIOManagerArchive().getCompressionLevel();
00350   }
00351   
00352   bool Archiver::isFifo(){
00353     return workergroup->isFifo();
00354   }
00355 
00356   void Archiver::setFifo(bool val){
00357     workergroup->setFifo(val);
00358   }
00359 
00360   void Archiver::setCompressionLevel (CORBA::Short level) {
00361     getIOManagerArchive().setCompressionLevel(level);
00362   }
00363   void Archiver::recover(){
00364     SctNames::Mrs() << MRS_TEXT("Sorry - recovery not yet implimented") 
00365             << "ANALYSIS_RECOVER" << MRS_DIAGNOSTIC << ENDM;
00366   }
00367 } // end of namespace SctArchiving

Generated on Fri Sep 16 18:01:48 2005 for SCT DAQ/DCS Software - C++ by doxygen 1.3.5