This commit is contained in:
biosvos
2026-08-07 17:38:18 +09:00
commit 873193a243
9613 changed files with 2755992 additions and 0 deletions
+548
View File
@@ -0,0 +1,548 @@
#include "Validation.h"
#include "DataFile.h"
#include "DBConnPool.h"
#include "Database.h"
#include "Util.h"
#include "Logger.h"
#include "ReportLog.h"
#include "ServiceConfig.h"
#include <sys/types.h>
#include <string.h>
#define FLAG_SYNC "SYNC"
#define FLAG_DEL "DEL"
#define FLAG_NSYNC "NSYNC"
#define FLAG_COPY "COPY"
#define FLAG_MOVE "MOVE"
// uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn
class DIFFFILE_FIELD
{
public:
enum FIELD
{
uri = 0, filename_hash, resource_type, get_content_length,
file_lastmodified, deleted_yn,
MAX_FILED
};
};
// meta table filed
class CURRENT_FIELD
{
public:
enum FIELD
{
uri = 0, filename_hash, host_name, resource_type,
depth, creation_date, display_name,
get_content_length, get_content_type,
get_lastmodified, source, deleted_yn,
is_cache, file_lastmodified,
MAX_FILED
};
};
// flag|uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn|host_name|creation_date|get_content_type|src_uri|slave_tran_id
class SYNCFILE_FIELD
{
public:
// SYNCFILE_FIELD::flag
// - SYCN : 동기화 필요
// - DEL : 삭제 필요
// - COPY : 복사된 파일임
// - MOVE : 이동된 파일임
enum FIELD
{
flag = 0, uri, filename_hash, resource_type, get_content_length,
file_lastmodified, deleted_yn, host_name, creation_date, get_content_type,
src_uri, slave_tran_id, uri_ignore_case,
// copy 속성을 찾기 위해서 추가(sync file 경우 filename_hash 필드에 해당 값 사용)
src_filename_hash,
MAX_FILED
};
};
static int StringCount(string str, string delim)
{
int nCount = 0;
string::size_type cutAt;
while ((cutAt = str.find_first_of(delim)) != str.npos)
{
if (cutAt > 0)
{
nCount++;
}
str = str.substr(cutAt + 1);
}
if (str.length() > 0)
nCount++;
return nCount;
}
static int64_t convert(string &s)
{
if (s.size() == 0)
return 0;
int64_t v = 0;
size_t i = 0;
char sign = (s[0] == '-' || s[0] == '+') ? (++i, s[0]) : '+';
for (; i < s.size(); ++i)
{
if (s[i] < '0' || s[i] > '9')
return 0;
v = v * 10 + s[i] - '0';
}
return sign == '-' ? -v : v;
}
CValidation::CValidation(CSyncInfo& info, string data)
: m_syncinfo(info), m_diffdata(data)
{
}
CValidation::~CValidation()
{
}
bool CValidation::CheckValidation(bool is_master /*= true*/)
{
CMasterDBPool* pool = CMasterDBPool::getInstance();
if (pool == NULL)
{
LOG(LERR, "DB Coon Pool is NULL");
return false;
}
// uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn
vector<string> vectdiff;
StringSplit(m_diffdata, "|", vectdiff, true);
if (vectdiff.size() != DIFFFILE_FIELD::MAX_FILED)
{
LOG(LERR, "validation sync file string split caution :[%s]", m_diffdata.c_str());
return false;
}
DataBase *db = pool->GetConnFromPool();
if (!db)
{
LOG(LERR, "Failed GetDB.");
return false;
}
string tran_id = (is_master ? m_syncinfo.sync_master : m_syncinfo.sync_slave);
string tblname = pool->GetMetaTableName(tran_id.c_str());
bool useDisyplay = pool->UseDisplayname();
// check 1 : diffdata에 대한 RCDB 최신 상태 비교
/// Get Current Meta
if (GetCurrentMeta(db, vectdiff, tblname, tran_id, useDisyplay) == false)
{
pool->ReleaseConnToPool(db, false);
return false;
}
/// Compare input & current
vector<string>vectOut = CompareData(vectdiff, tran_id);
if (vectOut.empty())
{
pool->ReleaseConnToPool(db);
LOG(LERR, "Failed CompareData.[%s]", m_diffdata.c_str())
return false;
}
// write sync file formate
if (WriteSyncFile(vectOut) == false)
{
pool->ReleaseConnToPool(db);
LOG(LERR, "Failed Write Sync File.[%s]", m_diffdata.c_str())
return false;
}
pool->ReleaseConnToPool(db);
return true;
}
string CValidation::GetFullFilenameHash(vector<string> & vectdiff, string &tran_id)
{
string r;
std::vector<std::string> vecMount = CServiceConfig::GetInstance()->GetFHSMount();
for (vector<string>::size_type i = 0; i < vecMount.size(); ++i)
{
string hash = "/" + tran_id + vectdiff[DIFFFILE_FIELD::filename_hash];
if (i > 0)
r += ",";
r += vecMount[i] + hash;
}
// 2014.06.07 dadamin
// ANY 함수 사용을 {} 포함된 경우 array 로 인되는 경우 방지
r = stringreplace(r, "{", "\\{");
r = stringreplace(r, "}", "\\}");
r = "{" + r + "}";
return r;
}
bool CValidation::GetCurrentMeta(DataBase* d, vector<string> & vectdiff, string &tblname, string &tran_id, bool useDisyplay)
{
ostringstream sql;
ostringstream t;
const char *paramValues[5];
int index = 0;
// check 1 : diffdata에 대한 RCDB 최신 상태
sql << "SELECT ";
// 메타 변경에 다른 참조할 uri 컬럼 값이 다름
// 기존 uri 동일한 값은 display_name 임
if (useDisyplay)
sql << "display_name,";
else
sql << "uri,";
sql << "filename_hash, host_name, resource_type, ";
sql << "depth, creation_date, display_name, ";
sql << "get_content_length, get_content_type, ";
sql << "get_lastmodified, source, deleted_yn, ";
sql << "is_cache, file_lastmodified ";
sql << "FROM " << tblname << " ";
sql << "WHERE ";
string strdepth, strfulluri, strfullhash;
if (vectdiff[DIFFFILE_FIELD::resource_type] == "0")
{
// file
sql << "filename_hash = ANY ($1) ";
sql << "AND resource_type = $2 ";
sql << "AND get_content_length = $3 ";
strfullhash = GetFullFilenameHash(vectdiff, tran_id);
// filename_hash
paramValues[0] = strfullhash.c_str();
// resource_type
paramValues[1] = vectdiff[DIFFFILE_FIELD::resource_type].c_str();
// get_content_length
paramValues[2] = vectdiff[DIFFFILE_FIELD::get_content_length].c_str();
index = 3;
LOG(LDBG, "Value [Get Current Info(file)] => %s,%s,%s",
paramValues[0], paramValues[1], paramValues[2]);
}
else
{
sql << "depth = $1 ";
if (useDisyplay && m_syncinfo.uri_ignore_case == "Y")
{
sql << "AND uri = lower($2) ";
}
else
{
sql << "AND uri = $2 ";
}
// dir
sql << "AND resource_type = $3 ";
sql << "AND get_content_length = $4 ";
// depth 값 구하기.
int nDEPTH = StringCount(vectdiff[DIFFFILE_FIELD::uri], "/");
t << nDEPTH;
strdepth = t.str(); t.str("");
// Full URI
strfulluri = "/" + tran_id + vectdiff[DIFFFILE_FIELD::uri];
// depth
paramValues[0] = strdepth.c_str();
// uri
paramValues[1] = strfulluri.c_str();
// resource_type
paramValues[2] = vectdiff[DIFFFILE_FIELD::resource_type].c_str();
// get_content_length
paramValues[3] = vectdiff[DIFFFILE_FIELD::get_content_length].c_str();
index = 4;
LOG(LDBG, "Value [Get Current Info(dir)] => %s,%s,%s,%s",
paramValues[0], paramValues[1], paramValues[2], paramValues[3]);
}
sql << "AND deleted_yn = 'N' AND is_cache = 'N';";
LOG(LDBG, "SQL[Get Current Info] => %s", sql.str().c_str());
d->PgDoExecParams((char*)sql.str().c_str(), index, paramValues);
// Query 결과를 가져온다.
if (d->PgResult(DataBase::NOT_CLEAR) < 0)
{
LOG(LERR, "RCDB Meta failed.[%s][%s]",
vectdiff[DIFFFILE_FIELD::uri].c_str(), d->GetErrorMessage().c_str());
return false;
}
for (int i = 0; i < d->GetNoTuples(); ++i)
{
vector<string> vectcurr(CURRENT_FIELD::MAX_FILED);
string struri = d->GetValue(i, CURRENT_FIELD::uri);
size_t found = struri.find_first_of("/", 1);
if (found != string::npos)
{
vectcurr[CURRENT_FIELD::uri] = struri.substr(found);
}
else
{
LOG(LNOT, "Use the uri as in RCDB.[%s]", struri.c_str());
vectcurr[CURRENT_FIELD::uri] = struri;
}
if (vectdiff[DIFFFILE_FIELD::resource_type] == "0")
vectcurr[CURRENT_FIELD::filename_hash] = d->GetValue(i, CURRENT_FIELD::filename_hash);
vectcurr[CURRENT_FIELD::host_name] = d->GetValue(i, CURRENT_FIELD::host_name);
vectcurr[CURRENT_FIELD::resource_type] = d->GetValue(i, CURRENT_FIELD::resource_type);
vectcurr[CURRENT_FIELD::depth] = d->GetValue(i, CURRENT_FIELD::depth);
vectcurr[CURRENT_FIELD::creation_date] = d->GetValue(i, CURRENT_FIELD::creation_date);
vectcurr[CURRENT_FIELD::display_name] = d->GetValue(i, CURRENT_FIELD::display_name);
vectcurr[CURRENT_FIELD::get_content_length] = d->GetValue(i, CURRENT_FIELD::get_content_length);
vectcurr[CURRENT_FIELD::get_content_type] = d->GetValue(i, CURRENT_FIELD::get_content_type);
vectcurr[CURRENT_FIELD::get_lastmodified] = d->GetValue(i, CURRENT_FIELD::get_lastmodified);
vectcurr[CURRENT_FIELD::source] = d->GetValue(i, CURRENT_FIELD::source);
vectcurr[CURRENT_FIELD::deleted_yn] = d->GetValue(i, CURRENT_FIELD::deleted_yn);
vectcurr[CURRENT_FIELD::is_cache] = d->GetValue(i, CURRENT_FIELD::is_cache);
vectcurr[CURRENT_FIELD::file_lastmodified] = d->GetValue(i, CURRENT_FIELD::file_lastmodified);
m_current.insert(CCurretMap::value_type(struri, vectcurr));
}
d->PgClear();
return true;
}
vector<string> CValidation::CompareData(vector<string> & vectdiff, string &tran_id)
{
// flag|uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn|host_name|src_uri|slave tran_id
vector<string> r(SYNCFILE_FIELD::MAX_FILED);
r[SYNCFILE_FIELD::uri] = vectdiff[DIFFFILE_FIELD::uri];
r[SYNCFILE_FIELD::filename_hash] = vectdiff[DIFFFILE_FIELD::filename_hash];
r[SYNCFILE_FIELD::resource_type] = vectdiff[DIFFFILE_FIELD::resource_type];
r[SYNCFILE_FIELD::get_content_length] = vectdiff[DIFFFILE_FIELD::get_content_length];
r[SYNCFILE_FIELD::file_lastmodified] = vectdiff[DIFFFILE_FIELD::file_lastmodified];
r[SYNCFILE_FIELD::deleted_yn] = vectdiff[DIFFFILE_FIELD::deleted_yn];
r[SYNCFILE_FIELD::slave_tran_id] = m_syncinfo.sync_slave;
r[SYNCFILE_FIELD::uri_ignore_case] = m_syncinfo.uri_ignore_case;
do
{
if (m_current.empty())
{
// m_diffdata에 포함된 uir, filename_hash 해당 하는 값이 존재 하지 않으므로 삭제 처리
r[SYNCFILE_FIELD::flag] = FLAG_DEL;
break;
}
string struri = "/" + tran_id + vectdiff[DIFFFILE_FIELD::uri];
int eqcnt = m_current.count(struri);
if (eqcnt > 1)
{
// 디렉토리 경우 생성함
if (vectdiff[DIFFFILE_FIELD::resource_type] == "1")
{
CCurretMap::iterator curr = m_current.find(struri);
r[SYNCFILE_FIELD::creation_date] = (*curr).second[CURRENT_FIELD::creation_date];
r[SYNCFILE_FIELD::file_lastmodified] = (*curr).second[CURRENT_FIELD::file_lastmodified];
r[SYNCFILE_FIELD::get_content_type] = (*curr).second[CURRENT_FIELD::get_content_type];
r[SYNCFILE_FIELD::flag] = FLAG_NSYNC;
break;
}
// 해당 파일 싱크 하지 않음(report log 기록)
CReportLog rlog;
if (rlog.Init(CServiceConfig::GetInstance()->GetLogPath()))
{
rlog.Write("WAR", "[%s] [The duplicate files.[uri :%s]]", PROG_NAME, struri.c_str());
}
break;
}
// diff된 uri == RCDB : 1개
if (eqcnt == 1 )
{
if (m_current.size() == 1)
{
CCurretMap::iterator curr = m_current.find(struri);
r[SYNCFILE_FIELD::creation_date] = (*curr).second[CURRENT_FIELD::creation_date];
r[SYNCFILE_FIELD::get_content_type] = (*curr).second[CURRENT_FIELD::get_content_type];
r[SYNCFILE_FIELD::file_lastmodified] = (*curr).second[CURRENT_FIELD::file_lastmodified];
if ((*curr).second[CURRENT_FIELD::resource_type] == "1" ||
(*curr).second[CURRENT_FIELD::file_lastmodified] != vectdiff[DIFFFILE_FIELD::file_lastmodified])
{
r[SYNCFILE_FIELD::flag] = FLAG_NSYNC;
break;
}
r[SYNCFILE_FIELD::flag] = FLAG_SYNC;
r[SYNCFILE_FIELD::host_name] = (*curr).second[CURRENT_FIELD::host_name];
//r[SYNCFILE_FIELD::src_uri] = current[CURR_FIELD::uri];
r[SYNCFILE_FIELD::src_filename_hash] = (*curr).second[CURRENT_FIELD::filename_hash];
}
else
{
// copy
vector<string> src, self;
int64_t o = 0, n = 0;
CCurretMap::iterator i = m_current.begin();
while (i != m_current.end()) // Loop until end is reached.
{
//cout << (*i).first << " -> " << (*i).second << "\n";
if ((*i).first != struri)
{
// get_lastmodified 작은 값을 원본 소스 취급
n = convert((*i).second[CURRENT_FIELD::get_lastmodified]);
if (o > n || o == 0)
{
o = n;
src = (*i).second;
}
}
else
{
self = (*i).second;;
}
++i;
}
if (src.empty())
{
CReportLog rlog;
if (rlog.Init(CServiceConfig::GetInstance()->GetLogPath()))
{
rlog.Write("WAR", "[%s] [Not Found Source File.[uri :?? => %s]]", PROG_NAME, struri.c_str());
}
r.clear();
break;
}
if (n < convert(self[CURRENT_FIELD::get_lastmodified]))
{
r[SYNCFILE_FIELD::flag] = FLAG_COPY;
r[SYNCFILE_FIELD::src_uri] = src[CURRENT_FIELD::uri];
r[SYNCFILE_FIELD::host_name] = src[CURRENT_FIELD::host_name];
r[SYNCFILE_FIELD::src_filename_hash] = src[CURRENT_FIELD::filename_hash];
r[SYNCFILE_FIELD::creation_date] = self[CURRENT_FIELD::creation_date];
r[SYNCFILE_FIELD::get_content_type] = self[CURRENT_FIELD::get_content_type];
}
else
{
//r.clear();
r[SYNCFILE_FIELD::flag] = FLAG_COPY;
r[SYNCFILE_FIELD::host_name] = src[CURRENT_FIELD::host_name];
r[SYNCFILE_FIELD::src_filename_hash] = src[CURRENT_FIELD::filename_hash];
r[SYNCFILE_FIELD::creation_date] = self[CURRENT_FIELD::creation_date];
r[SYNCFILE_FIELD::get_content_type] = self[CURRENT_FIELD::get_content_type];
}
}
break;
}
if (eqcnt == 0)
{
// move
vector<string> dest;
CCurretMap::iterator i = m_current.begin();
if (m_current.size() == 1)
{
r[SYNCFILE_FIELD::flag] = FLAG_MOVE;
dest = (*i).second;
r[SYNCFILE_FIELD::src_uri] = r[SYNCFILE_FIELD::uri];
r[SYNCFILE_FIELD::uri] = dest[CURRENT_FIELD::uri];
r[SYNCFILE_FIELD::host_name] = dest[CURRENT_FIELD::host_name];
r[SYNCFILE_FIELD::file_lastmodified] = dest[CURRENT_FIELD::file_lastmodified];
r[SYNCFILE_FIELD::src_filename_hash] = dest[CURRENT_FIELD::filename_hash];
r[SYNCFILE_FIELD::creation_date] = dest[CURRENT_FIELD::creation_date];
r[SYNCFILE_FIELD::get_content_type] = dest[CURRENT_FIELD::get_content_type];
}
else
{
// 2개 이상 존재 어떤 것으로 이동된지 알수 없으므로 해당 파일 삭제 처리
r[SYNCFILE_FIELD::flag] = FLAG_DEL;
}
break;
}
} while (false);
return r;
}
bool CValidation::WriteSyncFile(vector<string>& outdata)
{
// flag|uri|filename_hash|resource_type|get_content_length|file_lastmodified|deleted_yn|host_name|creation_date|get_content_type|src_uri|slave_tran_id|uri_ignore_case
CDataFile syncfile;
syncfile.Init(m_syncinfo.file_sync_name.c_str(), false);
vector<string>::size_type n = SYNCFILE_FIELD::MAX_FILED - (SYNCFILE_FIELD::MAX_FILED - SYNCFILE_FIELD::src_filename_hash);
string w;
for (vector<string>::size_type i = 0; i < n; ++i)
{
if (i> 0)
w += "|";
if( i == SYNCFILE_FIELD::filename_hash)
{
if (outdata[SYNCFILE_FIELD::src_filename_hash].size())
{
w += outdata[SYNCFILE_FIELD::src_filename_hash];
continue;
}
}
w += outdata[i];
}
bool r = syncfile.Write("%s", w.c_str());
return r;
}