Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add missing atomic operators, refactor atomic operators, move atomic operators to detail namespace. #14962

Merged
merged 13 commits into from
Mar 12, 2024
Merged

Add missing atomic operators, refactor atomic operators, move atomic operators to detail namespace. #14962

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
5cf4c16
Refactor atomicAdd to cudf::detail::atomic_add.
bdice Feb 3, 2024
69dc774
Refactor other atomic operators.
bdice Feb 3, 2024
25a1125
Refactor docs and clean up device_atomic.cuh.
bdice Feb 3, 2024
5ecf52f
Revert atomic_add in most places, update tests.
bdice Feb 4, 2024
b69ed50
Merge remote-tracking branch 'upstream/branch-24.04' into refactor-at…
bdice Feb 28, 2024
c10ce78
Merge remote-tracking branch 'upstream/branch-24.04' into refactor-at…
bdice Mar 4, 2024
b1b8a5e
Merge remote-tracking branch 'upstream/branch-24.04' into refactor-at…
bdice Mar 4, 2024
d8fe49d
Merge branch 'branch-24.04' into refactor-atomics
bdice Mar 4, 2024
86e325d
Merge branch 'branch-24.04' into refactor-atomics
bdice Mar 8, 2024
0d46d99
Fix copyright.
bdice Mar 8, 2024
282a341
Add missing headers that were implicitly included before this PR.
bdice Mar 9, 2024
2ccc3fc
Merge remote-tracking branch 'upstream/branch-24.04' into refactor-at…
bdice Mar 9, 2024
c42b5f3
Merge branch 'branch-24.04' into refactor-atomics
bdice Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion cpp/benchmarks/join/generate_input_tables.cuh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@

#pragma once

#include <cudf/detail/utilities/device_atomics.cuh>
#include <cudf/types.hpp>
#include <cudf/utilities/default_stream.hpp>
#include <cudf/utilities/error.hpp>

#include <rmm/device_uvector.hpp>
#include <rmm/exec_policy.hpp>

#include <thrust/distance.h>
Expand Down
46 changes: 24 additions & 22 deletions cpp/include/cudf/detail/aggregation/aggregation.cuh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2023, NVIDIA CORPORATION.
* Copyright (c) 2019-2024, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -144,8 +144,8 @@ struct update_target_element<
if (source_has_nulls and source.is_null(source_index)) { return; }

using Target = target_type_t<Source, aggregation::MIN>;
atomicMin(&target.element<Target>(target_index),
static_cast<Target>(source.element<Source>(source_index)));
cudf::detail::atomic_min(&target.element<Target>(target_index),
static_cast<Target>(source.element<Source>(source_index)));

if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
}
Expand All @@ -170,8 +170,8 @@ struct update_target_element<
using DeviceTarget = device_storage_type_t<Target>;
using DeviceSource = device_storage_type_t<Source>;

atomicMin(&target.element<DeviceTarget>(target_index),
static_cast<DeviceTarget>(source.element<DeviceSource>(source_index)));
cudf::detail::atomic_min(&target.element<DeviceTarget>(target_index),
static_cast<DeviceTarget>(source.element<DeviceSource>(source_index)));

if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
}
Expand All @@ -193,8 +193,8 @@ struct update_target_element<
if (source_has_nulls and source.is_null(source_index)) { return; }

using Target = target_type_t<Source, aggregation::MAX>;
atomicMax(&target.element<Target>(target_index),
static_cast<Target>(source.element<Source>(source_index)));
cudf::detail::atomic_max(&target.element<Target>(target_index),
static_cast<Target>(source.element<Source>(source_index)));

if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
}
Expand All @@ -219,8 +219,8 @@ struct update_target_element<
using DeviceTarget = device_storage_type_t<Target>;
using DeviceSource = device_storage_type_t<Source>;

atomicMax(&target.element<DeviceTarget>(target_index),
static_cast<DeviceTarget>(source.element<DeviceSource>(source_index)));
cudf::detail::atomic_max(&target.element<DeviceTarget>(target_index),
static_cast<DeviceTarget>(source.element<DeviceSource>(source_index)));

if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
}
Expand All @@ -242,8 +242,8 @@ struct update_target_element<
if (source_has_nulls and source.is_null(source_index)) { return; }

using Target = target_type_t<Source, aggregation::SUM>;
atomicAdd(&target.element<Target>(target_index),
static_cast<Target>(source.element<Source>(source_index)));
cudf::detail::atomic_add(&target.element<Target>(target_index),
static_cast<Target>(source.element<Source>(source_index)));

if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
}
Expand All @@ -268,8 +268,8 @@ struct update_target_element<
using DeviceTarget = device_storage_type_t<Target>;
using DeviceSource = device_storage_type_t<Source>;

atomicAdd(&target.element<DeviceTarget>(target_index),
static_cast<DeviceTarget>(source.element<DeviceSource>(source_index)));
cudf::detail::atomic_add(&target.element<DeviceTarget>(target_index),
static_cast<DeviceTarget>(source.element<DeviceSource>(source_index)));

if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
}
Expand Down Expand Up @@ -368,7 +368,7 @@ struct update_target_element<Source,

using Target = target_type_t<Source, aggregation::SUM_OF_SQUARES>;
auto value = static_cast<Target>(source.element<Source>(source_index));
atomicAdd(&target.element<Target>(target_index), value * value);
cudf::detail::atomic_add(&target.element<Target>(target_index), value * value);
if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
}
};
Expand All @@ -387,8 +387,8 @@ struct update_target_element<Source,
if (source_has_nulls and source.is_null(source_index)) { return; }

using Target = target_type_t<Source, aggregation::PRODUCT>;
atomicMul(&target.element<Target>(target_index),
static_cast<Target>(source.element<Source>(source_index)));
cudf::detail::atomic_mul(&target.element<Target>(target_index),
static_cast<Target>(source.element<Source>(source_index)));
if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); }
}
};
Expand All @@ -408,7 +408,7 @@ struct update_target_element<
if (source_has_nulls and source.is_null(source_index)) { return; }

using Target = target_type_t<Source, aggregation::COUNT_VALID>;
atomicAdd(&target.element<Target>(target_index), Target{1});
cudf::detail::atomic_add(&target.element<Target>(target_index), Target{1});

// It is assumed the output for COUNT_VALID is initialized to be all valid
}
Expand All @@ -427,7 +427,7 @@ struct update_target_element<
size_type source_index) const noexcept
{
using Target = target_type_t<Source, aggregation::COUNT_ALL>;
atomicAdd(&target.element<Target>(target_index), Target{1});
cudf::detail::atomic_add(&target.element<Target>(target_index), Target{1});

// It is assumed the output for COUNT_ALL is initialized to be all valid
}
Expand All @@ -449,10 +449,11 @@ struct update_target_element<
if (source_has_nulls and source.is_null(source_index)) { return; }

using Target = target_type_t<Source, aggregation::ARGMAX>;
auto old = atomicCAS(&target.element<Target>(target_index), ARGMAX_SENTINEL, source_index);
auto old = cudf::detail::atomic_cas(
&target.element<Target>(target_index), ARGMAX_SENTINEL, source_index);
if (old != ARGMAX_SENTINEL) {
while (source.element<Source>(source_index) > source.element<Source>(old)) {
old = atomicCAS(&target.element<Target>(target_index), old, source_index);
old = cudf::detail::atomic_cas(&target.element<Target>(target_index), old, source_index);
}
}

Expand All @@ -476,10 +477,11 @@ struct update_target_element<
if (source_has_nulls and source.is_null(source_index)) { return; }

using Target = target_type_t<Source, aggregation::ARGMIN>;
auto old = atomicCAS(&target.element<Target>(target_index), ARGMIN_SENTINEL, source_index);
auto old = cudf::detail::atomic_cas(
&target.element<Target>(target_index), ARGMIN_SENTINEL, source_index);
if (old != ARGMIN_SENTINEL) {
while (source.element<Source>(source_index) < source.element<Source>(old)) {
old = atomicCAS(&target.element<Target>(target_index), old, source_index);
old = cudf::detail::atomic_cas(&target.element<Target>(target_index), old, source_index);
}
}

Expand Down