#include "TProofDataSetManager.h"
#include "Riostream.h"
#include "TEnv.h"
#include "TFileCollection.h"
#include "TFileInfo.h"
#include "TMD5.h"
#include "THashList.h"
#include "TObjArray.h"
#include "TObjString.h"
#include "TParameter.h"
#include "TPRegexp.h"
#include "TRegexp.h"
#include "TSystem.h"
#include "TVirtualMonitoring.h"
#define DSM_ONE_GB (1073741824)
TString TProofDataSetManager::fgCommonDataSetTag = "COMMON";
ClassImp(TProofDataSetManager)
TProofDataSetManager::TProofDataSetManager(const char *group, const char *user,
const char *options)
: fGroup(group),
fUser(user), fCommonUser(), fCommonGroup(),
fGroupQuota(), fGroupUsed(),
fUserUsed(), fNTouchedFiles(0), fNOpenedFiles(0),
fNDisappearedFiles(0), fMTimeGroupConfig(-1)
{
if (fGroup.IsNull())
fGroup = "default";
if (fUser.IsNull()) {
fUser = "--nouser--";
UserGroup_t *pw = gSystem->GetUserInfo();
if (pw) {
fUser = pw->fUser;
delete pw;
}
}
fGroupQuota.SetOwner();
fGroupUsed.SetOwner();
fUserUsed.SetOwner();
fCommonUser = "COMMON";
fCommonGroup = "COMMON";
fNTouchedFiles = -1;
fNOpenedFiles = -1;
fNDisappearedFiles = -1;
fMTimeGroupConfig = -1;
fAvgFileSize = 50000000;
ParseInitOpts(options);
if (!fUser.IsNull() && !fGroup.IsNull()) {
if (!TestBit(TProofDataSetManager::kIsSandbox))
fBase.SetUri(TString(Form("/%s/%s/", fGroup.Data(), fUser.Data())));
}
ReadGroupConfig(gEnv->GetValue("Proof.GroupFile", ""));
}
TProofDataSetManager::~TProofDataSetManager()
{
fGroupQuota.DeleteAll();
fGroupUsed.DeleteAll();
fUserUsed.DeleteAll();
}
void TProofDataSetManager::ParseInitOpts(const char *opts)
{
ResetBit(TProofDataSetManager::kCheckQuota);
SetBit(TProofDataSetManager::kAllowRegister);
SetBit(TProofDataSetManager::kAllowVerify);
ResetBit(TProofDataSetManager::kAllowStaging);
ResetBit(TProofDataSetManager::kIsSandbox);
if (opts && strlen(opts) > 0) {
TString opt(opts);
Int_t ip = opt.Index("opt:");
if (ip != kNPOS) opt.Remove(0, ip + 4);
ip = opt.Index(" ");
if (ip != kNPOS) opt.Remove(ip);
if (opt.Contains("Cq:") && !opt.Contains("-Cq:"))
SetBit(TProofDataSetManager::kCheckQuota);
if (opt.Contains("-Ar:"))
ResetBit(TProofDataSetManager::kAllowRegister);
if (opt.Contains("-Av:"))
ResetBit(TProofDataSetManager::kAllowVerify);
if (opt.Contains("As:") && !opt.Contains("-As:"))
SetBit(TProofDataSetManager::kAllowStaging);
if (opt.Contains("Sb:") && !opt.Contains("-Sb:"))
SetBit(TProofDataSetManager::kIsSandbox);
}
if (TestBit(TProofDataSetManager::kAllowStaging)) {
SetBit(TProofDataSetManager::kAllowVerify);
}
if (TestBit(TProofDataSetManager::kAllowVerify)) {
SetBit(TProofDataSetManager::kAllowRegister);
}
}
Bool_t TProofDataSetManager::ReadGroupConfig(const char *cf)
{
FileStat_t st;
if (!cf || (strlen(cf) <= 0) || !strcmp(cf, fGroupConfigFile.Data())) {
if (fGroupConfigFile.IsNull()) {
if (gDebug > 0)
Info("ReadGroupConfig", "path to config file undefined - nothing to do");
return kFALSE;
}
if (gSystem->GetPathInfo(fGroupConfigFile, st)) {
Error("ReadGroupConfig", "could not stat %s", fGroupConfigFile.Data());
return kFALSE;
}
if (st.fMtime <= fMTimeGroupConfig) {
if (gDebug > 0)
Info("ReadGroupConfig","file has not changed - do nothing");
return kTRUE;
}
}
if (cf && (strlen(cf) > 0)) {
if (gSystem->GetPathInfo(cf, st)) {
Error("ReadGroupConfig", "could not stat %s", cf);
return kFALSE;
}
if (gSystem->AccessPathName(cf, kReadPermission)) {
Error("ReadGroupConfig", "cannot read %s", cf);
return kFALSE;
}
fGroupConfigFile = cf;
fMTimeGroupConfig = st.fMtime;
}
if (gDebug > 0)
Info("ReadGroupConfig","reading group config from %s", cf);
ifstream in;
in.open(cf);
if (!in.is_open()) {
Error("ReadGroupConfig", "could not open config file %s", cf);
return kFALSE;
}
TString tmpCommonUser;
TString line;
while (in.good()) {
line.ReadLine(in);
Ssiz_t from = 0;
TString key;
if (!line.Tokenize(key, from, " "))
continue;
if (key == "property") {
TString grp;
if (!line.Tokenize(grp, from, " ")) {
if (gDebug > 0)
Info("ReadGroupConfig","incomplete line: '%s'", line.Data());
continue;
}
TString type;
if (!line.Tokenize(type, from, " "))
continue;
if (type == "diskquota") {
TString sdq;
if (!line.Tokenize(sdq, from, " "))
continue;
if (sdq.IsDigit()) sdq += "G";
Long64_t quota = ToBytes(sdq);
if (quota > -1) {
fGroupQuota.Add(new TObjString(grp),
new TParameter<Long64_t> ("group quota", quota));
} else {
Warning("ReadGroupConfig",
"problems parsing string: wrong or unsupported suffix? %s",
sdq.Data());
}
} else if (type == "commonuser") {
TString comusr;
if (!line.Tokenize(comusr, from, " "))
continue;
}
} else if (key == "dataset") {
TString type;
if (!line.Tokenize(type, from, " ")) {
if (gDebug > 0)
Info("ReadGroupConfig","incomplete line: '%s'", line.Data());
continue;
}
if (type == "commonuser") {
TString comusr;
if (!line.Tokenize(comusr, from, " "))
continue;
fCommonUser = comusr;
} else if (type == "commongroup") {
TString comgrp;
if (!line.Tokenize(comgrp, from, " "))
continue;
fCommonGroup = comgrp;
} else if (type == "diskquota") {
TString on;
if (!line.Tokenize(on, from, " "))
continue;
if (on == "on") {
SetBit(TProofDataSetManager::kCheckQuota);
} else if (on == "off") {
ResetBit(TProofDataSetManager::kCheckQuota);
}
}
} else if (key == "averagefilesize") {
TString avgsize;
if (!line.Tokenize(avgsize, from, " ")) {
if (gDebug > 0)
Info("ReadGroupConfig","incomplete line: '%s'", line.Data());
continue;
}
Long64_t avgsz = ToBytes(avgsize);
if (avgsz > -1) {
fAvgFileSize = avgsz;
} else {
Warning("ReadGroupConfig",
"problems parsing string: wrong or unsupported suffix? %s",
avgsize.Data());
}
}
}
in.close();
return kTRUE;
}
Long64_t TProofDataSetManager::ToBytes(const char *size)
{
Long64_t lsize = -1;
if (!size || strlen(size) <= 0) return lsize;
TString s(size);
Long64_t fact = 1;
if (!s.IsDigit()) {
const char *unit[5] = { "k", "M", "G", "T", "P"};
fact = 1024;
Int_t jj = 0;
while (jj <= 4) {
if (s.EndsWith(unit[jj], TString::kIgnoreCase)) {
s.Remove(s.Length()-1);
break;
}
fact *= 1024;
jj++;
}
}
if (s.IsDigit())
lsize = s.Atoi() * fact;
return lsize;
}
TFileCollection *TProofDataSetManager::GetDataSet(const char *)
{
AbstractMethod("GetDataSet");
return (TFileCollection *)0;
}
Bool_t TProofDataSetManager::RemoveDataSet(const char *)
{
AbstractMethod("RemoveDataSet");
return kFALSE;
}
Bool_t TProofDataSetManager::ExistsDataSet(const char *)
{
AbstractMethod("ExistsDataSet");
return kFALSE;
}
TMap *TProofDataSetManager::GetDataSets(const char *, UInt_t)
{
AbstractMethod("GetDataSets");
return (TMap *)0;
}
Int_t TProofDataSetManager::ScanDataSet(const char *, UInt_t)
{
AbstractMethod("ScanDataSet");
return -1;
}
void TProofDataSetManager::GetQuota(const char *group, const char *user,
const char *dsName, TFileCollection *dataset)
{
if (gDebug > 0)
Info("GetQuota", "processing dataset %s %s %s", group, user, dsName);
if (dataset->GetTotalSize() > 0) {
TParameter<Long64_t> *size =
dynamic_cast<TParameter<Long64_t>*> (fGroupUsed.GetValue(group));
if (!size) {
size = new TParameter<Long64_t> ("group used", 0);
fGroupUsed.Add(new TObjString(group), size);
}
size->SetVal(size->GetVal() + dataset->GetTotalSize());
TMap *userMap = dynamic_cast<TMap*> (fUserUsed.GetValue(group));
if (!userMap) {
userMap = new TMap;
fUserUsed.Add(new TObjString(group), userMap);
}
size = dynamic_cast<TParameter<Long64_t>*> (userMap->GetValue(user));
if (!size) {
size = new TParameter<Long64_t> ("user used", 0);
userMap->Add(new TObjString(user), size);
}
size->SetVal(size->GetVal() + dataset->GetTotalSize());
}
}
void TProofDataSetManager::ShowQuota(const char *opt)
{
UpdateUsedSpace();
TMap *groupQuotaMap = GetGroupQuotaMap();
TMap *userUsedMap = GetUserUsedMap();
if (!groupQuotaMap || !userUsedMap)
return;
Bool_t noInfo = kTRUE;
TIter iter(groupQuotaMap);
TObjString *group = 0;
while ((group = dynamic_cast<TObjString*> (iter.Next()))) {
noInfo = kFALSE;
Long64_t groupQuota = GetGroupQuota(group->String());
Long64_t groupUsed = GetGroupUsed(group->String());
Printf(" +++ Group %s uses %.1f GB out of %.1f GB", group->String().Data(),
(Float_t) groupUsed / DSM_ONE_GB,
(Float_t) groupQuota / DSM_ONE_GB);
if (opt && !TString(opt).Contains("U", TString::kIgnoreCase))
continue;
TMap *userMap = dynamic_cast<TMap*> (userUsedMap->GetValue(group->String()));
if (!userMap)
continue;
TIter iter2(userMap);
TObjString *user = 0;
while ((user = dynamic_cast<TObjString*> (iter2.Next()))) {
TParameter<Long64_t> *size2 =
dynamic_cast<TParameter<Long64_t>*> (userMap->GetValue(user->String().Data()));
if (!size2)
continue;
Printf(" +++ User %s uses %.1f GB", user->String().Data(),
(Float_t) size2->GetVal() / DSM_ONE_GB);
}
Printf("------------------------------------------------------");
}
if (noInfo) {
Printf(" +++ Quota check enabled but no quota info available +++ ");
}
}
void TProofDataSetManager::PrintUsedSpace()
{
Info("PrintUsedSpace", "listing used space");
TIter iter(&fUserUsed);
TObjString *group = 0;
while ((group = dynamic_cast<TObjString*> (iter.Next()))) {
TMap *userMap = dynamic_cast<TMap*> (fUserUsed.GetValue(group->String()));
TParameter<Long64_t> *size =
dynamic_cast<TParameter<Long64_t>*> (fGroupUsed.GetValue(group->String()));
if (userMap && size) {
Printf("Group %s: %lld B = %.2f GB", group->String().Data(), size->GetVal(),
(Float_t) size->GetVal() / DSM_ONE_GB);
TIter iter2(userMap);
TObjString *user = 0;
while ((user = dynamic_cast<TObjString*> (iter2.Next()))) {
TParameter<Long64_t> *size2 =
dynamic_cast<TParameter<Long64_t>*> (userMap->GetValue(user->String().Data()));
if (size2)
Printf(" User %s: %lld B = %.2f GB", user->String().Data(), size2->GetVal(),
(Float_t) size2->GetVal() / DSM_ONE_GB);
}
Printf("------------------------------------------------------");
}
}
}
void TProofDataSetManager::MonitorUsedSpace(TVirtualMonitoringWriter *monitoring)
{
Info("MonitorUsedSpace", "sending used space to monitoring server");
TIter iter(&fUserUsed);
TObjString *group = 0;
while ((group = dynamic_cast<TObjString*> (iter.Next()))) {
TMap *userMap = dynamic_cast<TMap*> (fUserUsed.GetValue(group->String()));
TParameter<Long64_t> *size =
dynamic_cast<TParameter<Long64_t>*> (fGroupUsed.GetValue(group->String()));
if (!userMap || !size)
continue;
TList *list = new TList;
list->SetOwner();
list->Add(new TParameter<Long64_t>("_TOTAL_", size->GetVal()));
Long64_t groupQuota = GetGroupQuota(group->String());
if (groupQuota != -1)
list->Add(new TParameter<Long64_t>("_QUOTA_", groupQuota));
TIter iter2(userMap);
TObjString *user = 0;
while ((user = dynamic_cast<TObjString*> (iter2.Next()))) {
TParameter<Long64_t> *size2 =
dynamic_cast<TParameter<Long64_t>*> (userMap->GetValue(user->String().Data()));
if (!size2)
continue;
list->Add(new TParameter<Long64_t>(user->String().Data(), size2->GetVal()));
}
monitoring->SendParameters(list, group->String());
delete list;
}
}
Long64_t TProofDataSetManager::GetGroupUsed(const char *group)
{
if (fgCommonDataSetTag == group)
group = fCommonGroup;
TParameter<Long64_t> *size =
dynamic_cast<TParameter<Long64_t>*> (fGroupUsed.GetValue(group));
if (!size) {
if (gDebug > 0)
Info("GetGroupUsed", "group %s not found", group);
return 0;
}
return size->GetVal();
}
Long64_t TProofDataSetManager::GetGroupQuota(const char *group)
{
if (fgCommonDataSetTag == group)
group = fCommonGroup;
TParameter<Long64_t> *value =
dynamic_cast<TParameter<Long64_t>*> (fGroupQuota.GetValue(group));
if (!value) {
if (gDebug > 0)
Info("GetGroupQuota", "group %s not found", group);
return 0;
}
return value->GetVal();
}
void TProofDataSetManager::UpdateUsedSpace()
{
AbstractMethod("UpdateUsedSpace");
}
Int_t TProofDataSetManager::RegisterDataSet(const char *,
TFileCollection *,
const char *)
{
AbstractMethod("RegisterDataSet");
return -1;
}
TString TProofDataSetManager::CreateUri(const char *dsGroup, const char *dsUser,
const char *dsName, const char *dsObjPath)
{
TString uri;
if (dsGroup && strlen(dsGroup) > 0) {
if (dsUser && strlen(dsUser) > 0) {
uri += Form("/%s/%s/", dsGroup, dsUser);
} else {
uri += Form("/%s/*/", dsGroup);
}
} else if (dsUser && strlen(dsUser) > 0) {
uri += Form("%s/", dsUser);
}
if (dsName && strlen(dsName) > 0)
uri += dsName;
if (dsObjPath && strlen(dsObjPath) > 0)
uri += Form("#%s", dsObjPath);
return uri;
}
Bool_t TProofDataSetManager::ParseUri(const char *uri,
TString *dsGroup, TString *dsUser,
TString *dsName, TString *dsTree,
Bool_t onlyCurrent, Bool_t wildcards)
{
TString uristr(uri);
if (wildcards && uristr.Length() > 0 && !uristr.EndsWith("/"))
uristr += '/';
TUri resolved = TUri::Transform(uristr, fBase);
if (resolved.HasQuery())
Info ("ParseUri", "URI query part <%s> ignored", resolved.GetQuery().Data());
TString path(resolved.GetPath());
Int_t pc = path.CountChar('/');
if (pc != 3) {
if (!TestBit(TProofDataSetManager::kIsSandbox)) {
Error ("ParseUri", "illegal dataset path: %s", uri);
return kFALSE;
} else if (pc >= 0 && pc < 3) {
TString sls("/");
if (pc == 2) {
sls = "/";
} else if (pc == 1) {
sls = Form("/%s/", fGroup.Data());
} else if (pc == 0) {
sls = Form("/%s/%s/", fGroup.Data(), fUser.Data());
}
path.Insert(0, sls);
}
}
if (gDebug > 1)
Info("ParseUri", "path: '%s'", path.Data());
Int_t from = 1;
TString group, user, name;
path.Tokenize(group, from, "/");
path.Tokenize(user, from, "/");
path.Tokenize(name, from, "/");
TString tree = resolved.GetFragment();
if (tree.EndsWith("/"))
tree.Remove(tree.Length()-1);
if (gDebug > 1)
Info("ParseUri", "group: '%s', user: '%s', dsname:'%s', seg: '%s'",
group.Data(), user.Data(), name.Data(), tree.Data());
if ((user == "*" || group == "*") && !wildcards) {
Error ("ParseUri", "no wildcards allowed for user/group in this context");
return kFALSE;
}
if (name.IsNull() && !wildcards) {
Error ("ParseUri", "DataSet name is empty");
return kFALSE;
}
TPRegexp wcExp (wildcards ? "^(?:[A-Za-z0-9-]*|[*])$" : "^[A-Za-z0-9-]*$");
if (!wcExp.Match(group)) {
Error("ParseUri", "illegal characters in group");
return kFALSE;
}
if (!wcExp.Match(user)) {
Error("ParseUri", "illegal characters in user");
return kFALSE;
}
if (name.Contains(TRegexp("[^A-Za-z0-9-._]"))) {
Error("ParseUri", "illegal characters in dataset name");
return kFALSE;
}
if (tree.Contains(TRegexp("[^A-Za-z0-9-/_]"))) {
Error("ParseUri", "Illegal characters in subdir/object name");
return kFALSE;
}
if (onlyCurrent && (group.CompareTo(fGroup) || user.CompareTo(fUser))) {
Error("ParseUri", "only datasets from your group/user allowed");
return kFALSE;
}
if (dsGroup)
*dsGroup = group;
if (dsUser)
*dsUser = user;
if (dsName)
*dsName = name;
if (dsTree)
*dsTree = tree;
return kTRUE;
}
Last change: Wed Jun 25 08:51:06 2008
Last generated: 2008-06-25 08:51
This page has been automatically generated. If you have any comments or suggestions about the page layout send a mail to ROOT support, or contact the developers with any questions or problems regarding ROOT.