Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implementation of a new softmax version with PLANE_WISE mode support #3022

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open

Implementation of a new softmax version with PLANE_WISE mode support #3022

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
4d698fc
Fix Stride Indexing Bugs in `reorg` and `reorg_gradient` Functions (C…
Cydral Sep 16, 2024
1d73b6c
'add_to' parameter missing in cuda call reorg_gradient.launch_kernel()
Cydral Sep 20, 2024
c343779
Cleanup: remove using namespace std; (#3016)
arrufat Sep 23, 2024
724ec09
Merge branch 'refs/heads/master' into Cydral-master
davisking Sep 23, 2024
4dca9b2
fix build error
davisking Sep 23, 2024
2f68a11
Adjust comment formatting to be like other dlib comments
davisking Sep 23, 2024
64e3471
Merge branch 'davisking:master' into master
Cydral Sep 23, 2024
640c02f
Add positional encodings layer to Dlib
Cydral Sep 24, 2024
6750d2e
Implementation of a new softmax version with plane-wise mode support
Cydral Sep 27, 2024
684c56c
Updates
Cydral Sep 27, 2024
1a18dc2
Updates
Cydral Sep 27, 2024
6fb34f9
Updates
Cydral Sep 27, 2024
847ab9d
Updates
Cydral Sep 27, 2024
ee07b5f
Updates
Cydral Sep 27, 2024
0ca05c7
Merge branch 'master' into softmaxm-layer
Cydral Oct 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .vscode/settings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"githubPullRequests.ignoredPullRequestBranches": [
"master"
]
}
214 changes: 135 additions & 79 deletions dlib/cuda/cpu_dlib.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1620,122 +1620,178 @@ namespace dlib

namespace ttimpl
{
void softmax (
const long num_locations,
const long num_channels,
tensor& dest,
const tensor& src
)
{
DLIB_ASSERT(num_channels*num_locations == src.nr()*src.nc()*src.k());
DLIB_CASSERT(have_same_dimensions(dest,src));
const auto d = dest.host();
const auto s = src.host();
void softmax(
const long num_locations,
const long num_channels,
tensor& dest,
const tensor& src,
size_t mode = 0
)
{
DLIB_ASSERT(num_channels * num_locations == src.nr() * src.nc() * src.k());
DLIB_CASSERT(have_same_dimensions(dest, src));
const auto d = dest.host();
const auto s = src.host();

// Note that we subtract out the max values in each channel before applying
// exp() to avoid numeric overflow in the subsequent computations. Doing this
// doesn't change the resulting output, it just makes it more numerically
// stable.
for (long n = 0; n < src.num_samples(); ++n)
{
auto ss = s + num_locations*num_channels*n;
auto dd = d + num_locations*num_channels*n;
for (long i = 0; i < num_locations; ++i)
for (long n = 0; n < src.num_samples(); ++n)
{
float max_val = -std::numeric_limits<float>::infinity();
for (long k = 0; k < num_channels; ++k)
max_val = std::max(max_val, ss[k*num_locations]);
auto ss = s + num_locations * num_channels * n;
auto dd = d + num_locations * num_channels * n;

for (long k = 0; k < num_channels; ++k)
dd[k*num_locations] = std::exp(ss[k*num_locations]-max_val);
if (mode == 0) // softmax_mode::CHANNEL_WISE
{
for (long i = 0; i < num_locations; ++i)
{
float max_val = -std::numeric_limits<float>::infinity();
for (long k = 0; k < num_channels; ++k)
max_val = std::max(max_val, ss[k * num_locations]);

++ss;
++dd;
}
}
float sum = 0.0f;
for (long k = 0; k < num_channels; ++k)
{
dd[k * num_locations] = std::exp(ss[k * num_locations] - max_val);
sum += dd[k * num_locations];
}
sum += std::numeric_limits<float>::epsilon();
for (long k = 0; k < num_channels; ++k)
dd[k * num_locations] /= sum;

// Now normalize each channel so they sum to 1.
for (long n = 0; n < src.num_samples(); ++n)
{
const auto dd = d + num_locations*num_channels*n;
for (long i = 0; i < num_locations; ++i)
{
const auto ddd = dd+i;
++ss;
++dd;
}
}
else if (mode == 1) // softmax_mode::PLANE_WISE
{
for (long k = 0; k < num_channels; ++k)
{
auto s_channel = ss + k * num_locations;
auto d_channel = dd + k * num_locations;
for (long r = 0; r < src.nr(); ++r)
{
float max_val = -std::numeric_limits<float>::infinity();
for (long c = 0, idx = r * src.nc(); c < src.nc(); ++c, ++idx)
max_val = std::max(max_val, s_channel[idx]);

float temp = 0;
for (long k = 0; k < num_channels; ++k)
temp += ddd[k*num_locations];
for (long k = 0; k < num_channels; ++k)
ddd[k*num_locations] /= temp;
if (max_val == -std::numeric_limits<float>::infinity())
{
for (long c = 0, idx = r * src.nc(); c < src.nc(); ++c, ++idx)
d_channel[idx] = 0.0f;
}
else
{
float sum = 0.0f;
for (long c = 0, idx = r * src.nc(); c < src.nc(); ++c, ++idx)
{
d_channel[idx] = std::exp(s_channel[idx] - max_val);
sum += d_channel[idx];
}
sum += std::numeric_limits<float>::epsilon();
for (long c = 0, idx = r * src.nc(); c < src.nc(); ++c, ++idx)
d_channel[idx] /= sum;
}
}
}
}
}
}
}

void softmax_gradient (
const long num_locations,
const long num_channels,
tensor& grad,
const tensor& dest,
const tensor& gradient_input
)
{
DLIB_ASSERT(num_channels*num_locations == grad.nr()*grad.nc()*grad.k());
DLIB_CASSERT(have_same_dimensions(grad,dest));
DLIB_CASSERT(have_same_dimensions(grad,gradient_input));
const auto d = dest.host();
const auto g = grad.host();
const auto in = gradient_input.host();


for (long n = 0; n < grad.num_samples(); ++n)
void softmax_gradient(
const long num_locations,
const long num_channels,
tensor& grad,
const tensor& dest,
const tensor& gradient_input,
size_t mode = 0
)
{
const auto d2 = d + num_locations*num_channels*n;
const auto g2 = g + num_locations*num_channels*n;
const auto in2 = in + num_locations*num_channels*n;
for (long i = 0; i < num_locations; ++i)
DLIB_ASSERT(num_channels * num_locations == grad.nr() * grad.nc() * grad.k());
DLIB_CASSERT(have_same_dimensions(grad, dest));
DLIB_CASSERT(have_same_dimensions(grad, gradient_input));

const auto d = dest.host();
const auto g = grad.host();
const auto in = gradient_input.host();
for (long n = 0; n < grad.num_samples(); ++n)
{
const auto d3 = d2+i;
const auto g3 = g2+i;
const auto in3 = in2+i;
const auto d2 = d + num_locations * num_channels * n;
const auto g2 = g + num_locations * num_channels * n;
const auto in2 = in + num_locations * num_channels * n;

float temp = 0;
for (long k = 0; k < num_channels; ++k)
temp += -d3[k*num_locations]*in3[k*num_locations];
if (is_same_object(gradient_input, grad))
if (mode == 0) // softmax_mode::CHANNEL_WISE
{
for (long k = 0; k < num_channels; ++k)
g3[k*num_locations] = d3[k*num_locations]*(temp+in3[k*num_locations]);
for (long i = 0; i < num_locations; ++i)
{
const auto d3 = d2 + i;
const auto g3 = g2 + i;
const auto in3 = in2 + i;
float sum = 0.0f;
for (long k = 0; k < num_channels; ++k)
sum += -d3[k * num_locations] * in3[k * num_locations];
if (is_same_object(gradient_input, grad))
{
for (long k = 0; k < num_channels; ++k)
g3[k * num_locations] = d3[k * num_locations] * (sum + in3[k * num_locations]);
}
else
{
for (long k = 0; k < num_channels; ++k)
g3[k * num_locations] += d3[k * num_locations] * (sum + in3[k * num_locations]);
}
}
}
else
else if (mode == 1) // softmax_mode::PLANE_WISE
{
for (long k = 0; k < num_channels; ++k)
g3[k*num_locations] += d3[k*num_locations]*(temp+in3[k*num_locations]);
{
const auto d_channel = d2 + k * num_locations;
const auto g_channel = g2 + k * num_locations;
const auto in_channel = in2 + k * num_locations;
for (long r = 0; r < grad.nr(); ++r)
{
float sum = 0.0f;
for (long c = 0, idx = r * grad.nc(); c < grad.nc(); ++c, ++idx)
sum += -d_channel[idx] * in_channel[idx];
if (is_same_object(gradient_input, grad))
{
for (long c = 0, idx = r * grad.nc(); c < grad.nc(); ++c, ++idx)
g_channel[idx] = d_channel[idx] * (sum + in_channel[idx]);
}
else
{
for (long c = 0, idx = r * grad.nc(); c < grad.nc(); ++c, ++idx)
g_channel[idx] += d_channel[idx] * (sum + in_channel[idx]);
}
}
}
}
}
}
}
}

// ----------------------------------------------------------------------------------------

void softmax (
tensor& dest,
const tensor& src
const tensor& src,
size_t mode
)
{
DLIB_CASSERT(have_same_dimensions(dest,src));
ttimpl::softmax(src.nr()*src.nc(), src.k(), dest, src);
DLIB_CASSERT(mode == 0 /*CHANNEL_WISE*/ || mode == 1 /*PLANE_WISE*/, "Invalid softmax mode");
ttimpl::softmax(src.nr()*src.nc(), src.k(), dest, src, mode);
}

void softmax_gradient (
tensor& grad,
const tensor& dest,
const tensor& gradient_input
const tensor& gradient_input,
size_t mode
)
{
DLIB_CASSERT(have_same_dimensions(grad,dest));
DLIB_CASSERT(have_same_dimensions(grad,gradient_input));
ttimpl::softmax_gradient(grad.nr()*grad.nc(), grad.k(), grad, dest, gradient_input);
DLIB_CASSERT(mode == 0 /*CHANNEL_WISE*/ || mode == 1 /*PLANE_WISE*/, "Invalid softmax mode");
ttimpl::softmax_gradient(grad.nr()*grad.nc(), grad.k(), grad, dest, gradient_input, mode);
}

// ------------------------------------------------------------------------------------
Expand Down
6 changes: 4 additions & 2 deletions dlib/cuda/cpu_dlib.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,13 +293,15 @@ namespace dlib

void softmax (
tensor& dest,
const tensor& src
const tensor& src,
size_t mode = 0
);

void softmax_gradient (
tensor& grad,
const tensor& dest,
const tensor& gradient_input
const tensor& gradient_input,
size_t mode = 0
);

// ------------------------------------------------------------------------------------
Expand Down
Loading
Loading