#include "Validation.h" #include "DataFile.h" #include "DBConnPool.h" #include "Database.h" #include "Util.h" #include "Logger.h" #include "ReportLog.h" #include "ServiceConfig.h" #include #include #define FLAG_SYNC "SYNC" #define FLAG_DEL "DEL" #define FLAG_NSYNC "NSYNC" #define FLAG_COPY "COPY" #define FLAG_MOVE "MOVE" // uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn class DIFFFILE_FIELD { public: enum FIELD { uri = 0, filename_hash, resource_type, get_content_length, file_lastmodified, deleted_yn, MAX_FILED }; }; // meta table filed class CURRENT_FIELD { public: enum FIELD { uri = 0, filename_hash, host_name, resource_type, depth, creation_date, display_name, get_content_length, get_content_type, get_lastmodified, source, deleted_yn, is_cache, file_lastmodified, MAX_FILED }; }; // flag|uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn|host_name|creation_date|get_content_type|src_uri|slave_tran_id class SYNCFILE_FIELD { public: // SYNCFILE_FIELD::flag // - SYCN : µ¿±âÈ­ ÇÊ¿ä // - DEL : »èÁ¦ ÇÊ¿ä // - COPY : º¹»çµÈ ÆÄÀÏÀÓ // - MOVE : À̵¿µÈ ÆÄÀÏÀÓ enum FIELD { flag = 0, uri, filename_hash, resource_type, get_content_length, file_lastmodified, deleted_yn, host_name, creation_date, get_content_type, src_uri, slave_tran_id, uri_ignore_case, // copy ¼Ó¼ºÀ» ã±â À§Çؼ­ Ãß°¡(sync file °æ¿ì filename_hash Çʵ忡 ÇØ´ç °ª »ç¿ë) src_filename_hash, MAX_FILED }; }; static int StringCount(string str, string delim) { int nCount = 0; string::size_type cutAt; while ((cutAt = str.find_first_of(delim)) != str.npos) { if (cutAt > 0) { nCount++; } str = str.substr(cutAt + 1); } if (str.length() > 0) nCount++; return nCount; } static int64_t convert(string &s) { if (s.size() == 0) return 0; int64_t v = 0; size_t i = 0; char sign = (s[0] == '-' || s[0] == '+') ? (++i, s[0]) : '+'; for (; i < s.size(); ++i) { if (s[i] < '0' || s[i] > '9') return 0; v = v * 10 + s[i] - '0'; } return sign == '-' ? -v : v; } CValidation::CValidation(CSyncInfo& info, string data) : m_syncinfo(info), m_diffdata(data) { } CValidation::~CValidation() { } bool CValidation::CheckValidation(bool is_master /*= true*/) { CMasterDBPool* pool = CMasterDBPool::getInstance(); if (pool == NULL) { LOG(LERR, "DB Coon Pool is NULL"); return false; } // uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn vector vectdiff; StringSplit(m_diffdata, "|", vectdiff, true); if (vectdiff.size() != DIFFFILE_FIELD::MAX_FILED) { LOG(LERR, "validation sync file string split caution :[%s]", m_diffdata.c_str()); return false; } DataBase *db = pool->GetConnFromPool(); if (!db) { LOG(LERR, "Failed GetDB."); return false; } string tran_id = (is_master ? m_syncinfo.sync_master : m_syncinfo.sync_slave); string tblname = pool->GetMetaTableName(tran_id.c_str()); bool useDisyplay = pool->UseDisplayname(); // check 1 : diffdata¿¡ ´ëÇÑ RCDB ÃֽŠ»óÅ ºñ±³ /// Get Current Meta if (GetCurrentMeta(db, vectdiff, tblname, tran_id, useDisyplay) == false) { pool->ReleaseConnToPool(db, false); return false; } /// Compare input & current vectorvectOut = CompareData(vectdiff, tran_id); if (vectOut.empty()) { pool->ReleaseConnToPool(db); LOG(LERR, "Failed CompareData.[%s]", m_diffdata.c_str()) return false; } // write sync file formate if (WriteSyncFile(vectOut) == false) { pool->ReleaseConnToPool(db); LOG(LERR, "Failed Write Sync File.[%s]", m_diffdata.c_str()) return false; } pool->ReleaseConnToPool(db); return true; } string CValidation::GetFullFilenameHash(vector & vectdiff, string &tran_id) { string r; std::vector vecMount = CServiceConfig::GetInstance()->GetFHSMount(); for (vector::size_type i = 0; i < vecMount.size(); ++i) { string hash = "/" + tran_id + vectdiff[DIFFFILE_FIELD::filename_hash]; if (i > 0) r += ","; r += vecMount[i] + hash; } // 2014.06.07 dadamin // ANY ÇÔ¼ö »ç¿ëÀ» {} Æ÷ÇÔµÈ °æ¿ì array ·Î ÀεǴ °æ¿ì ¹æÁö r = stringreplace(r, "{", "\\{"); r = stringreplace(r, "}", "\\}"); r = "{" + r + "}"; return r; } bool CValidation::GetCurrentMeta(DataBase* d, vector & vectdiff, string &tblname, string &tran_id, bool useDisyplay) { ostringstream sql; ostringstream t; const char *paramValues[5]; int index = 0; // check 1 : diffdata¿¡ ´ëÇÑ RCDB ÃֽŠ»óÅ sql << "SELECT "; // ¸ÞŸ º¯°æ¿¡ ´Ù¸¥ ÂüÁ¶ÇÒ uri Ä÷³ °ªÀÌ ´Ù¸§ // ±âÁ¸ uri µ¿ÀÏÇÑ °ªÀº display_name ÀÓ if (useDisyplay) sql << "display_name,"; else sql << "uri,"; sql << "filename_hash, host_name, resource_type, "; sql << "depth, creation_date, display_name, "; sql << "get_content_length, get_content_type, "; sql << "get_lastmodified, source, deleted_yn, "; sql << "is_cache, file_lastmodified "; sql << "FROM " << tblname << " "; sql << "WHERE "; string strdepth, strfulluri, strfullhash; if (vectdiff[DIFFFILE_FIELD::resource_type] == "0") { // file sql << "filename_hash = ANY ($1) "; sql << "AND resource_type = $2 "; sql << "AND get_content_length = $3 "; strfullhash = GetFullFilenameHash(vectdiff, tran_id); // filename_hash paramValues[0] = strfullhash.c_str(); // resource_type paramValues[1] = vectdiff[DIFFFILE_FIELD::resource_type].c_str(); // get_content_length paramValues[2] = vectdiff[DIFFFILE_FIELD::get_content_length].c_str(); index = 3; LOG(LDBG, "Value [Get Current Info(file)] => %s,%s,%s", paramValues[0], paramValues[1], paramValues[2]); } else { sql << "depth = $1 "; if (useDisyplay && m_syncinfo.uri_ignore_case == "Y") { sql << "AND uri = lower($2) "; } else { sql << "AND uri = $2 "; } // dir sql << "AND resource_type = $3 "; sql << "AND get_content_length = $4 "; // depth °ª ±¸Çϱâ. int nDEPTH = StringCount(vectdiff[DIFFFILE_FIELD::uri], "/"); t << nDEPTH; strdepth = t.str(); t.str(""); // Full URI strfulluri = "/" + tran_id + vectdiff[DIFFFILE_FIELD::uri]; // depth paramValues[0] = strdepth.c_str(); // uri paramValues[1] = strfulluri.c_str(); // resource_type paramValues[2] = vectdiff[DIFFFILE_FIELD::resource_type].c_str(); // get_content_length paramValues[3] = vectdiff[DIFFFILE_FIELD::get_content_length].c_str(); index = 4; LOG(LDBG, "Value [Get Current Info(dir)] => %s,%s,%s,%s", paramValues[0], paramValues[1], paramValues[2], paramValues[3]); } sql << "AND deleted_yn = 'N' AND is_cache = 'N';"; LOG(LDBG, "SQL[Get Current Info] => %s", sql.str().c_str()); d->PgDoExecParams((char*)sql.str().c_str(), index, paramValues); // Query °á°ú¸¦ °¡Á®¿Â´Ù. if (d->PgResult(DataBase::NOT_CLEAR) < 0) { LOG(LERR, "RCDB Meta failed.[%s][%s]", vectdiff[DIFFFILE_FIELD::uri].c_str(), d->GetErrorMessage().c_str()); return false; } for (int i = 0; i < d->GetNoTuples(); ++i) { vector vectcurr(CURRENT_FIELD::MAX_FILED); string struri = d->GetValue(i, CURRENT_FIELD::uri); size_t found = struri.find_first_of("/", 1); if (found != string::npos) { vectcurr[CURRENT_FIELD::uri] = struri.substr(found); } else { LOG(LNOT, "Use the uri as in RCDB.[%s]", struri.c_str()); vectcurr[CURRENT_FIELD::uri] = struri; } if (vectdiff[DIFFFILE_FIELD::resource_type] == "0") vectcurr[CURRENT_FIELD::filename_hash] = d->GetValue(i, CURRENT_FIELD::filename_hash); vectcurr[CURRENT_FIELD::host_name] = d->GetValue(i, CURRENT_FIELD::host_name); vectcurr[CURRENT_FIELD::resource_type] = d->GetValue(i, CURRENT_FIELD::resource_type); vectcurr[CURRENT_FIELD::depth] = d->GetValue(i, CURRENT_FIELD::depth); vectcurr[CURRENT_FIELD::creation_date] = d->GetValue(i, CURRENT_FIELD::creation_date); vectcurr[CURRENT_FIELD::display_name] = d->GetValue(i, CURRENT_FIELD::display_name); vectcurr[CURRENT_FIELD::get_content_length] = d->GetValue(i, CURRENT_FIELD::get_content_length); vectcurr[CURRENT_FIELD::get_content_type] = d->GetValue(i, CURRENT_FIELD::get_content_type); vectcurr[CURRENT_FIELD::get_lastmodified] = d->GetValue(i, CURRENT_FIELD::get_lastmodified); vectcurr[CURRENT_FIELD::source] = d->GetValue(i, CURRENT_FIELD::source); vectcurr[CURRENT_FIELD::deleted_yn] = d->GetValue(i, CURRENT_FIELD::deleted_yn); vectcurr[CURRENT_FIELD::is_cache] = d->GetValue(i, CURRENT_FIELD::is_cache); vectcurr[CURRENT_FIELD::file_lastmodified] = d->GetValue(i, CURRENT_FIELD::file_lastmodified); m_current.insert(CCurretMap::value_type(struri, vectcurr)); } d->PgClear(); return true; } vector CValidation::CompareData(vector & vectdiff, string &tran_id) { // flag|uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn|host_name|src_uri|slave tran_id vector r(SYNCFILE_FIELD::MAX_FILED); r[SYNCFILE_FIELD::uri] = vectdiff[DIFFFILE_FIELD::uri]; r[SYNCFILE_FIELD::filename_hash] = vectdiff[DIFFFILE_FIELD::filename_hash]; r[SYNCFILE_FIELD::resource_type] = vectdiff[DIFFFILE_FIELD::resource_type]; r[SYNCFILE_FIELD::get_content_length] = vectdiff[DIFFFILE_FIELD::get_content_length]; r[SYNCFILE_FIELD::file_lastmodified] = vectdiff[DIFFFILE_FIELD::file_lastmodified]; r[SYNCFILE_FIELD::deleted_yn] = vectdiff[DIFFFILE_FIELD::deleted_yn]; r[SYNCFILE_FIELD::slave_tran_id] = m_syncinfo.sync_slave; r[SYNCFILE_FIELD::uri_ignore_case] = m_syncinfo.uri_ignore_case; do { if (m_current.empty()) { // m_diffdata¿¡ Æ÷ÇÔµÈ uir, filename_hash ÇØ´ç ÇÏ´Â °ªÀÌ Á¸Àç ÇÏÁö ¾ÊÀ¸¹Ç·Î »èÁ¦ ó¸® r[SYNCFILE_FIELD::flag] = FLAG_DEL; break; } string struri = "/" + tran_id + vectdiff[DIFFFILE_FIELD::uri]; int eqcnt = m_current.count(struri); if (eqcnt > 1) { // µð·ºÅ丮 °æ¿ì »ý¼ºÇÔ if (vectdiff[DIFFFILE_FIELD::resource_type] == "1") { CCurretMap::iterator curr = m_current.find(struri); r[SYNCFILE_FIELD::creation_date] = (*curr).second[CURRENT_FIELD::creation_date]; r[SYNCFILE_FIELD::file_lastmodified] = (*curr).second[CURRENT_FIELD::file_lastmodified]; r[SYNCFILE_FIELD::get_content_type] = (*curr).second[CURRENT_FIELD::get_content_type]; r[SYNCFILE_FIELD::flag] = FLAG_NSYNC; break; } // ÇØ´ç ÆÄÀÏ ½ÌÅ© ÇÏÁö ¾ÊÀ½(report log ±â·Ï) CReportLog rlog; if (rlog.Init(CServiceConfig::GetInstance()->GetLogPath())) { rlog.Write("WAR", "[%s] [The duplicate files.[uri :%s]]", PROG_NAME, struri.c_str()); } break; } // diffµÈ uri == RCDB : 1°³ if (eqcnt == 1 ) { if (m_current.size() == 1) { CCurretMap::iterator curr = m_current.find(struri); r[SYNCFILE_FIELD::creation_date] = (*curr).second[CURRENT_FIELD::creation_date]; r[SYNCFILE_FIELD::get_content_type] = (*curr).second[CURRENT_FIELD::get_content_type]; r[SYNCFILE_FIELD::file_lastmodified] = (*curr).second[CURRENT_FIELD::file_lastmodified]; if ((*curr).second[CURRENT_FIELD::resource_type] == "1" || (*curr).second[CURRENT_FIELD::file_lastmodified] != vectdiff[DIFFFILE_FIELD::file_lastmodified]) { r[SYNCFILE_FIELD::flag] = FLAG_NSYNC; break; } r[SYNCFILE_FIELD::flag] = FLAG_SYNC; r[SYNCFILE_FIELD::host_name] = (*curr).second[CURRENT_FIELD::host_name]; //r[SYNCFILE_FIELD::src_uri] = current[CURR_FIELD::uri]; r[SYNCFILE_FIELD::src_filename_hash] = (*curr).second[CURRENT_FIELD::filename_hash]; } else { // copy vector src, self; int64_t o = 0, n = 0; CCurretMap::iterator i = m_current.begin(); while (i != m_current.end()) // Loop until end is reached. { //cout << (*i).first << " -> " << (*i).second << "\n"; if ((*i).first != struri) { // get_lastmodified ÀÛÀº °ªÀ» ¿øº» ¼Ò½º Ãë±Þ n = convert((*i).second[CURRENT_FIELD::get_lastmodified]); if (o > n || o == 0) { o = n; src = (*i).second; } } else { self = (*i).second;; } ++i; } if (src.empty()) { CReportLog rlog; if (rlog.Init(CServiceConfig::GetInstance()->GetLogPath())) { rlog.Write("WAR", "[%s] [Not Found Source File.[uri :?? => %s]]", PROG_NAME, struri.c_str()); } r.clear(); break; } if (n < convert(self[CURRENT_FIELD::get_lastmodified])) { r[SYNCFILE_FIELD::flag] = FLAG_COPY; r[SYNCFILE_FIELD::src_uri] = src[CURRENT_FIELD::uri]; r[SYNCFILE_FIELD::host_name] = src[CURRENT_FIELD::host_name]; r[SYNCFILE_FIELD::src_filename_hash] = src[CURRENT_FIELD::filename_hash]; r[SYNCFILE_FIELD::creation_date] = self[CURRENT_FIELD::creation_date]; r[SYNCFILE_FIELD::get_content_type] = self[CURRENT_FIELD::get_content_type]; } else { //r.clear(); r[SYNCFILE_FIELD::flag] = FLAG_COPY; r[SYNCFILE_FIELD::host_name] = src[CURRENT_FIELD::host_name]; r[SYNCFILE_FIELD::src_filename_hash] = src[CURRENT_FIELD::filename_hash]; r[SYNCFILE_FIELD::creation_date] = self[CURRENT_FIELD::creation_date]; r[SYNCFILE_FIELD::get_content_type] = self[CURRENT_FIELD::get_content_type]; } } break; } if (eqcnt == 0) { // move vector dest; CCurretMap::iterator i = m_current.begin(); if (m_current.size() == 1) { r[SYNCFILE_FIELD::flag] = FLAG_MOVE; dest = (*i).second; r[SYNCFILE_FIELD::src_uri] = r[SYNCFILE_FIELD::uri]; r[SYNCFILE_FIELD::uri] = dest[CURRENT_FIELD::uri]; r[SYNCFILE_FIELD::host_name] = dest[CURRENT_FIELD::host_name]; r[SYNCFILE_FIELD::file_lastmodified] = dest[CURRENT_FIELD::file_lastmodified]; r[SYNCFILE_FIELD::src_filename_hash] = dest[CURRENT_FIELD::filename_hash]; r[SYNCFILE_FIELD::creation_date] = dest[CURRENT_FIELD::creation_date]; r[SYNCFILE_FIELD::get_content_type] = dest[CURRENT_FIELD::get_content_type]; } else { // 2°³ ÀÌ»ó Á¸Àç ¾î¶² °ÍÀ¸·Î À̵¿µÈÁö ¾Ë¼ö ¾øÀ¸¹Ç·Î ÇØ´ç ÆÄÀÏ »èÁ¦ ó¸® r[SYNCFILE_FIELD::flag] = FLAG_DEL; } break; } } while (false); return r; } bool CValidation::WriteSyncFile(vector& outdata) { // flag|uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn|host_name|creation_date|get_content_type|src_uri|slave_tran_id|uri_ignore_case CDataFile syncfile; syncfile.Init(m_syncinfo.file_sync_name.c_str(), false); vector::size_type n = SYNCFILE_FIELD::MAX_FILED - (SYNCFILE_FIELD::MAX_FILED - SYNCFILE_FIELD::src_filename_hash); string w; for (vector::size_type i = 0; i < n; ++i) { if (i> 0) w += "|"; if( i == SYNCFILE_FIELD::filename_hash) { if (outdata[SYNCFILE_FIELD::src_filename_hash].size()) { w += outdata[SYNCFILE_FIELD::src_filename_hash]; continue; } } w += outdata[i]; } bool r = syncfile.Write("%s", w.c_str()); return r; }