Skip to content

Commit

Permalink
Merge pull request #224 from LLNL/feature/kausik1/plugins
Browse files Browse the repository at this point in the history
Use RAJA plugin capability
  • Loading branch information
adayton1 authored Jul 14, 2023
2 parents bbb9bb8 + 52504cd commit 223caaa
Show file tree
Hide file tree
Showing 14 changed files with 433 additions and 406 deletions.
19 changes: 10 additions & 9 deletions src/care/Accessor.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
#define CHAI_Accessor__HPP

#include "care/config.h"
#include "care/RAJAPlugin.h"
#include "care/PluginData.h"

#include <algorithm>
#include <stdio.h>
#include <cstdlib>
#include <functional>
#include <set>
Expand Down Expand Up @@ -93,17 +94,17 @@ class RaceConditionAccessor : public NoOpAccessor<T> {

CARE_HOST_DEVICE RaceConditionAccessor<T>(RaceConditionAccessor<T> const & other ) : m_shallow_copy_of_cpu_data(other.m_shallow_copy_of_cpu_data), m_deep_copy_of_previous_state_of_cpu_data(other.m_deep_copy_of_previous_state_of_cpu_data), m_accesses(other.m_accesses), m_size_in_bytes(other.m_size_in_bytes), m_name(other.m_name) {
#ifndef CARE_GPUCC
if (RAJAPlugin::isParallelContext()) {
if (PluginData::isParallelContext()) {
auto data = m_shallow_copy_of_cpu_data;
if (!RAJAPlugin::post_parallel_forall_action_registered((void *)data)) {
if (!PluginData::post_parallel_forall_action_registered((void *)data)) {
auto len = m_size_in_bytes / sizeof(T);
m_deep_copy_of_previous_state_of_cpu_data = new typename std::remove_const<T>::type[len];
std::copy_n(data, len, m_deep_copy_of_previous_state_of_cpu_data);
auto prev_data = m_deep_copy_of_previous_state_of_cpu_data;
m_accesses = new std::unordered_map<int, std::set<int>> {};
auto accesses = m_accesses;
const char * name = m_name;
RAJAPlugin::register_post_parallel_forall_action((void *)data, std::bind(detectRaceCondition<T>, data, prev_data, accesses, len, name, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));
PluginData::register_post_parallel_forall_action((void *)data, std::bind(detectRaceCondition<T>, data, prev_data, accesses, len, name, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));
}
}
#endif
Expand All @@ -115,8 +116,8 @@ class RaceConditionAccessor : public NoOpAccessor<T> {
const
{
#ifndef CARE_GPUCC
if (m_accesses && RAJAPlugin::isParallelContext()) {
(*m_accesses)[i].insert(RAJAPlugin::s_threadID);
if (m_accesses && PluginData::isParallelContext()) {
(*m_accesses)[i].insert(PluginData::s_threadID);
}
#endif
}
Expand Down Expand Up @@ -153,12 +154,12 @@ class RaceConditionAccessorWithCallback : public RaceConditionAccessor<T> {
const
{
m_callback(i);
if (RaceConditionAccessor<T>::m_accesses && RAJAPlugin::isParallelContext()) {
printf("inserting %i into (%p) accesses[%i] for pointer %p\n", RAJAPlugin::s_threadID, (void *) RaceConditionAccessor<T>::m_accesses, i, RaceConditionAccessor<T>::m_shallow_copy_of_cpu_data);
if (RaceConditionAccessor<T>::m_accesses && PluginData::isParallelContext()) {
printf("inserting %i into (%p) accesses[%i] for pointer %p\n", PluginData::s_threadID, (void *) RaceConditionAccessor<T>::m_accesses, i, RaceConditionAccessor<T>::m_shallow_copy_of_cpu_data);
printf("size of accesses before insert %lu\n", RaceConditionAccessor<T>::m_accesses->size());
}
RaceConditionAccessor<T>::operator[](i);
if (RaceConditionAccessor<T>::m_accesses && RAJAPlugin::isParallelContext()) {
if (RaceConditionAccessor<T>::m_accesses && PluginData::isParallelContext()) {
printf("size of accesses after insert %lu\n", RaceConditionAccessor<T>::m_accesses->size());
printf("size of accesses[%i] after insert %lu\n", i, (*RaceConditionAccessor<T>::m_accesses)[i].size());
}
Expand Down
24 changes: 12 additions & 12 deletions src/care/CHAICallback.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// CARE headers
#include "care/CHAICallback.h"
#ifndef CARE_DISABLE_RAJAPLUGIN
#include "care/RAJAPlugin.h"
#include "care/PluginData.h"
#endif

// Other library headers
Expand Down Expand Up @@ -301,7 +301,7 @@ namespace care {
void CHAICallback::deregisterRecord(const chai::PointerRecord* record) {
#ifndef CARE_DISABLE_RAJAPLUGIN
if (s_log_data > 0) {
RAJAPlugin::removeActivePointer(record);
PluginData::removeActivePointer(record);
}
#endif
NameMap& s_names = getNameMap();
Expand Down Expand Up @@ -433,16 +433,16 @@ namespace care {
fprintf(s_log_file,
"[CARE] [CHAI] %s: Moved %lu bytes to space %i at %s:%i\n",
name.c_str(), size, (int) space,
RAJAPlugin::getCurrentLoopFileName().c_str(),
RAJAPlugin::getCurrentLoopLineNumber());
PluginData::getFileName(),
PluginData::getLineNumber());
}
else {
fprintf(s_log_file,
"[CARE] [CHAI] %s (%p): Moved %lu bytes to space %i (%p) at %s:%i\n",
name.c_str(), m_record, size, (int) space,
m_record->m_pointers[space],
RAJAPlugin::getCurrentLoopFileName().c_str(),
RAJAPlugin::getCurrentLoopLineNumber());
PluginData::getFileName(),
PluginData::getLineNumber());
}
#endif
}
Expand All @@ -456,20 +456,20 @@ namespace care {
fprintf(s_log_file,
"[CARE] [CHAI] %s: Captured %lu bytes to space %i at %s:%i\n",
name.c_str(), size, (int) space,
RAJAPlugin::getCurrentLoopFileName().c_str(),
RAJAPlugin::getCurrentLoopLineNumber());
PluginData::getFileName(),
PluginData::getLineNumber());
}
else {
fprintf(s_log_file,
"[CARE] [CHAI] %s (%p): Captured %lu bytes to space %i (%p) at %s:%i\n",
name.c_str(), m_record, size, (int) space,
m_record->m_pointers[space],
RAJAPlugin::getCurrentLoopFileName().c_str(),
RAJAPlugin::getCurrentLoopLineNumber());
PluginData::getFileName(),
PluginData::getLineNumber());
}
}
if (s_log_data > 0) {
RAJAPlugin::addActivePointer(m_record);
PluginData::addActivePointer(m_record);
}
#endif
break;
Expand Down Expand Up @@ -522,7 +522,7 @@ namespace care {
case chai::ACTION_CAPTURED:
#ifndef CARE_DISABLE_RAJAPLUGIN
if (s_log_data > 0) {
RAJAPlugin::addActivePointer(m_record);
PluginData::addActivePointer(m_record);
}
#endif
break;
Expand Down
10 changes: 7 additions & 3 deletions src/care/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,10 @@ set(care_headers
numeric.h
PointerTypes.h
policies.h
care.h
RAJAPlugin.h
care.h
PluginData.h
DebugPlugin.h
ProfilePlugin.h
scan.h
scan_impl.h
Setup.h
Expand All @@ -65,7 +67,9 @@ set(care_sources
CHAICallback.cpp
ExecutionSpace.cpp
LoopFuser.cpp
RAJAPlugin.cpp
PluginData.cpp
DebugPlugin.cpp
ProfilePlugin.cpp
scan.cpp
)

Expand Down
139 changes: 139 additions & 0 deletions src/care/DebugPlugin.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
#include "care/DebugPlugin.h"
#include "chai/ExecutionSpaces.hpp"
#include "care/CHAICallback.h"
#include "care/PluginData.h"

// Std library headers
#if defined(CARE_DEBUG) && !defined(_WIN32)
#include <execinfo.h>
#endif // defined(CARE_DEBUG) && !defined(_WIN32)

#include <unordered_set>

namespace care{
void DebugPlugin::registerPlugin() {
static RAJA::util::PluginRegistry::add<care::DebugPlugin> L ("Debug plugin", "CARE plugin for debugging");
}

void DebugPlugin::preLaunch(const RAJA::util::PluginContext& /* p */) {
#if !defined(CHAI_DISABLE_RM)
// Prepare to record CHAI data
if (CHAICallback::isActive()) {
PluginData::clearActivePointers();

#if defined(CARE_GPUCC) && defined(CARE_DEBUG)
GPUWatchpoint::setOrCheckWatchpoint<int>();
#endif // defined(CARE_GPUCC) && defined(CARE_DEBUG)
}
#endif // !defined(CHAI_DISABLE_RM)
}


void DebugPlugin::postLaunch(const RAJA::util::PluginContext& p) {
#if !defined(CHAI_DISABLE_RM)
chai::ExecutionSpace space;

switch (p.platform) {
case RAJA::Platform::host:
space = chai::CPU; break;
#if defined(CHAI_ENABLE_CUDA)
case RAJA::Platform::cuda:
space = chai::GPU; break;
#endif
#if defined(CHAI_ENABLE_HIP)
case RAJA::Platform::hip:
space = chai::GPU; break;
#endif
default:
space = chai::NONE;
}

if (CHAICallback::isActive()) {
writeLoopData(space, PluginData::getFileName(), PluginData::getLineNumber());

#if defined(CARE_GPUCC) && defined(CARE_DEBUG)
GPUWatchpoint::setOrCheckWatchpoint<int>();
#endif // defined(CARE_GPUCC) && defined(CARE_DEBUG)
}

if (PluginData::isParallelContext()) {
for (auto const & it : PluginData::get_post_parallel_forall_actions()) {
it.second(space, PluginData::getFileName(), PluginData::getLineNumber());
}
PluginData::clear_post_parallel_forall_actions();
PluginData::s_threadID = -1;
}
#endif // !defined(CHAI_DISABLE_RM)
}

void DebugPlugin::writeLoopData(chai::ExecutionSpace space, const char * fileName, int lineNumber) {
if (CHAICallback::loggingIsEnabled()) {
const int s_log_data = CHAICallback::getLogData();

if (s_log_data == (int) space ||
s_log_data == (int) chai::ExecutionSpace::NUM_EXECUTION_SPACES) {
// Get the output file
FILE* callback_output_file = CHAICallback::getLogFile();

// Write the loop header
int numArrays = 0;
std::unordered_set<const chai::PointerRecord*> usedRecords;

for (const chai::PointerRecord* record : PluginData::getActivePointers()) {
if (usedRecords.count(record) > 0) {
continue;
}
else {
usedRecords.emplace(record);
++numArrays;
}
}

fprintf(callback_output_file, "[CARE] AFTER LOOP EXECUTION %s:%i (%i arrays)\n", fileName, lineNumber, numArrays);

#if defined(CARE_DEBUG) && !defined(_WIN32)
// Write the stack trace
const int stackDepth = 16;
void *stackArray[stackDepth];
size_t stackSize = backtrace(stackArray, stackDepth);
char **stackStrings = backtrace_symbols(stackArray, stackSize);

// Skip the first two contexts
for (size_t i = 2 ; i < stackSize ; ++i) {
fprintf(callback_output_file, "[CARE] [STACK] %lu: %s\n", i, stackStrings[i]);
}

if (CHAICallback::isDiffFriendly()) {
// For diff-friendly output, keep the number of lines constant
for (size_t i = stackSize ; i < stackDepth ; ++i) {
fprintf(callback_output_file, "[CARE] [STACK] %lu: <empty>\n", i);
}
}

free(stackStrings);
#endif // defined(CARE_DEBUG) && !defined(_WIN32)

// Flush to the output file
fflush(callback_output_file);

#ifdef CARE_DEBUG
// Write the arrays captured in the loop
usedRecords.clear();

for (const chai::PointerRecord* record : PluginData::getActivePointers()) {
if (record && usedRecords.find(record) == usedRecords.end()) {
usedRecords.emplace(record);
CHAICallback::writeArray(record, space);
fflush(callback_output_file);
}
}
#else // CARE_DEBUG
// Write message that -logchaidata is not supported
fprintf(callback_output_file, "[CARE] [CHAI] -logchaidata is not supported in this build\n");
fflush(callback_output_file);
#endif // CARE_DEBUG
}
}
}
}

33 changes: 33 additions & 0 deletions src/care/DebugPlugin.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#ifndef _CARE_DebugPlugin_H_
#define _CARE_DebugPlugin_H_

#include "RAJA/util/PluginStrategy.hpp"
#include "chai/ExecutionSpaces.hpp"

namespace care{

class DebugPlugin : public RAJA::util::PluginStrategy
{
public:
DebugPlugin() = default;

static void registerPlugin();

void preLaunch(const RAJA::util::PluginContext& p) override;

void postLaunch(const RAJA::util::PluginContext& p) override;

/////////////////////////////////////////////////////////////////////////////////
///
/// @brief Writes out debugging information after a loop is executed.
///
/// @arg[in] space The execution space
/// @arg[in] fileName The file where the loop macro was called
/// @arg[in] lineNumber The line number where the loop macro was called
///
/////////////////////////////////////////////////////////////////////////////////
static void writeLoopData(chai::ExecutionSpace space, const char * fileName, int lineNumber);
};
}

#endif
2 changes: 1 addition & 1 deletion src/care/DefaultMacros.h
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,7 @@
////////////////////////////////////////////////////////////////////////////////

#ifdef CARE_ENABLE_RACE_DETECTION
#define CARE_SET_THREAD_ID(INDEX) care::RAJAPlugin::s_threadID = INDEX ;
#define CARE_SET_THREAD_ID(INDEX) care::DebugPlugin::s_threadID = INDEX ;
#else
#define CARE_SET_THREAD_ID(INDEX)
#endif
Expand Down
58 changes: 58 additions & 0 deletions src/care/PluginData.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#include "PluginData.h"

namespace care{
const char * PluginData::s_file_name = "N/A";
int PluginData::s_line_number = -1;
bool PluginData::s_parallel_context = false;
ActionMap PluginData::s_post_parallel_forall_actions = ActionMap{};
std::vector<const chai::PointerRecord*> PluginData::s_active_pointers_in_loop = std::vector<const chai::PointerRecord*>{};
int PluginData::s_threadID = -1;

void PluginData::setFileName(const char * name) {PluginData::s_file_name = name;}

void PluginData::setLineNumber(int num) {PluginData::s_line_number = num;}

const char * PluginData::getFileName() {return s_file_name;}

int PluginData::getLineNumber() {return s_line_number;}

void PluginData::setParallelContext(bool isParallel) {
s_parallel_context = isParallel;
}

bool PluginData::isParallelContext(){
return s_parallel_context;
}

ActionMap PluginData::get_post_parallel_forall_actions() {
return s_post_parallel_forall_actions;
}

void PluginData::register_post_parallel_forall_action(void * key, std::function<void(chai::ExecutionSpace, const char *, int)> action) {
s_post_parallel_forall_actions[key] = action;
}

bool PluginData::post_parallel_forall_action_registered(void * key) {
bool registered = s_post_parallel_forall_actions.count(key) > 0;
return registered;
}

void PluginData::clear_post_parallel_forall_actions(){s_post_parallel_forall_actions.clear();}

std::vector<const chai::PointerRecord*> PluginData::getActivePointers() {return s_active_pointers_in_loop;}

void PluginData::addActivePointer(const chai::PointerRecord* record) {
s_active_pointers_in_loop.emplace_back(record);
}

void PluginData::removeActivePointer(const chai::PointerRecord* record) {
for (size_t i = 0; i < s_active_pointers_in_loop.size(); ++i) {
if (s_active_pointers_in_loop[i] == record) {
s_active_pointers_in_loop[i] = nullptr;
}
}
}

void PluginData::clearActivePointers() {s_active_pointers_in_loop.clear();}

}
Loading

0 comments on commit 223caaa

Please sign in to comment.