how to implement hazelcast in-memory distributed cache using dotnet core 6

Question

I am using Hazelcast c# client for creating an in-memory distributed cache. I am not sure if my implementation is correct or missing something. I am testing it against an MSSQL server that is faster than Hazelcast. [Testing with same data, memory and gpus]

appsettings.json

"Hazelcast": {
  "Networking": {
    "Addresses": [ "localhost:5701" ],
    "Authentication": {
      "Token": "my-token"
    }
  }
}

program.cs

var hazelcastOptions = builder.Configuration.GetSection("hazelcast").Get<HazelcastOptions>();
builder.Services.AddCacheService(hazelcastOptions);

Dependency Injection

public static void AddCacheService(this IServiceCollection services, HazelcastOptions hazelcastOptions)
{
    services.AddSingleton<IHazelcastStudentHelper, HazelcastStudentHelper>(_ => new HazelcastStudentHelper(hazelcastOptions));
}

HazelcastStudentHelper.cs

private readonly HazelcastOptions _options;
private const string _mapStudent = "students";
private const string _mapSubject = "subjects";
private const string _mapExam = "exams";
private const string _mapMark = "marks";

public HazelcastStudentHelper(HazelcastOptions options)
{
    _options = options;
}

Map creating:

public async Task<int> CacheStudentListAsync(IEnumerable<Student> studentList, CancellationToken token = default)
        {
            try
            {
                await using var client = await HazelcastClientFactory.StartNewClientAsync(_options, cancellationToken: token).ConfigureAwait(false);
                await using var map = await client.GetMapAsync<long, HazelcastJsonValue>(_mapStudent).ConfigureAwait(false);

                var studentsDictionary = new Dictionary<long, HazelcastJsonValue>();

                foreach (var student in studentList)
                {
                    string jsonString = JsonSerializer.Serialize(student);
                    var jsonObj = new HazelcastJsonValue(jsonString);
                    studentsDictionary.Add(student.Id, jsonObj);
                }

                // Create a sorted index on the __key attribute
                await map.AddIndexAsync(IndexType.Sorted, "__key").ConfigureAwait(false);

                // Create a hash index on the Name attribute
                await map.AddIndexAsync(IndexType.Hashed, "Name").ConfigureAwait(false);

                #region bitmap indexes are not supported by Hazelcast SQL
                // Create a bitmap index on the RollNumber attribute
                // await map.AddIndexAsync(IndexType.Bitmap, "RollNumber").ConfigureAwait(false);
                #endregion

                await client.Sql.ExecuteCommandAsync($@"
CREATE OR REPLACE MAPPING 
{map.Name} (
__key BIGINT,
Id BIGINT,
Name VARCHAR,
RollNumber VARCHAR,
CreatedAt TIMESTAMP WITH TIME ZONE,
ModifiedAt TIMESTAMP WITH TIME ZONE)
TYPE IMap OPTIONS ('keyFormat'='bigint', 'valueFormat'='json-flat')", cancellationToken: token).ConfigureAwait(false);

                await map.SetAllAsync(studentsDictionary).ConfigureAwait(false);
                int count = await map.GetSizeAsync().ConfigureAwait(false);

                await map.DisposeAsync().ConfigureAwait(false);
                await client.DisposeAsync().ConfigureAwait(false);

                return count;
            }
            catch (Exception)
            {

                throw;
            }
        }

public async Task<int> CacheSubjectListAsync(IEnumerable<Subject> subjectList, CancellationToken token = default)
        {
            try
            {
                await using var client = await HazelcastClientFactory.StartNewClientAsync(_options, cancellationToken: token).ConfigureAwait(false);
                await using var map = await client.GetMapAsync<long, HazelcastJsonValue>(_mapSubject).ConfigureAwait(false);

                var subjectsDictionary = new Dictionary<long, HazelcastJsonValue>();

                foreach (var subject in subjectList)
                {
                    string jsonString = JsonSerializer.Serialize(subject);
                    var jsonObj = new HazelcastJsonValue(jsonString);
                    subjectsDictionary.Add(subject.Id, jsonObj);
                }

                // Create a sorted index on the __key attribute
                await map.AddIndexAsync(IndexType.Sorted, "__key").ConfigureAwait(false);

                // Create a hash index on the Name attribute
                await map.AddIndexAsync(IndexType.Hashed, "Name").ConfigureAwait(false);

                await client.Sql.ExecuteCommandAsync($@"
CREATE OR REPLACE MAPPING 
{map.Name} (
__key BIGINT,
Id BIGINT,
Name VARCHAR,
Description VARCHAR,
CreatedAt TIMESTAMP WITH TIME ZONE,
ModifiedAt TIMESTAMP WITH TIME ZONE)
TYPE IMap OPTIONS ('keyFormat'='bigint', 'valueFormat'='json-flat')", cancellationToken: token).ConfigureAwait(false);

                await map.SetAllAsync(subjectsDictionary).ConfigureAwait(false);
                int count = await map.GetSizeAsync().ConfigureAwait(false);

                await map.DisposeAsync().ConfigureAwait(false);
                await client.DisposeAsync().ConfigureAwait(false);

                return count;
            }
            catch (Exception)
            {

                throw;

            }
        }

public async Task<int> CacheExamListAsync(IEnumerable<Exam> examList, CancellationToken token = default)
        {
            try
            {
                await using var client = await HazelcastClientFactory.StartNewClientAsync(_options, cancellationToken: token).ConfigureAwait(false);
                await using var map = await client.GetMapAsync<long, HazelcastJsonValue>(_mapExam).ConfigureAwait(false);

                var examsDictionary = new Dictionary<long, HazelcastJsonValue>();

                foreach (var exam in examList)
                {
                    string jsonString = JsonSerializer.Serialize(exam);
                    var jsonObj = new HazelcastJsonValue(jsonString);
                    examsDictionary.Add(exam.Id, jsonObj);
                }

                // Create a sorted index on the __key attribute
                await map.AddIndexAsync(IndexType.Sorted, "__key").ConfigureAwait(false);

                // Create a sorted index on the ExamDate attribute
                await map.AddIndexAsync(IndexType.Sorted, "ExamDate").ConfigureAwait(false);

                await client.Sql.ExecuteCommandAsync($@"
CREATE OR REPLACE MAPPING 
{map.Name} (
__key BIGINT,
Id BIGINT,
Name VARCHAR,
ExamDate TIMESTAMP WITH TIME ZONE,
CreatedAt TIMESTAMP WITH TIME ZONE,
ModifiedAt TIMESTAMP WITH TIME ZONE)
TYPE IMap OPTIONS ('keyFormat'='bigint', 'valueFormat'='json-flat')", cancellationToken: token).ConfigureAwait(false);

                await map.SetAllAsync(examsDictionary).ConfigureAwait(false);
                int count = await map.GetSizeAsync().ConfigureAwait(false);

                await map.DisposeAsync().ConfigureAwait(false);
                await client.DisposeAsync().ConfigureAwait(false);

                return count;
            }
            catch (Exception)
            {

                throw;
            }
        }

public async Task<int> CacheMarkListAsync(IEnumerable<Mark> markList, CancellationToken token = default)
        {
            try
            {
                await using var client = await HazelcastClientFactory.StartNewClientAsync(_options, cancellationToken: token).ConfigureAwait(false);
                await using var map = await client.GetMapAsync<long, HazelcastJsonValue>(_mapMark).ConfigureAwait(false);

                var marksDictionary = new Dictionary<long, HazelcastJsonValue>();

                foreach (var mark in markList)
                {
                    string jsonString = JsonSerializer.Serialize(
                        new MarkDto
                        {
                            Id = mark.Id,
                            MarkValue = mark.MarkValue,
                            StudentId = mark.Student.Id,
                            SubjectId = mark.Subject.Id,
                            ExamId = mark.Exam.Id,
                            CreatedAt = mark.CreatedAt,
                            ModifiedAt = mark.ModifiedAt,
                        });
                    var jsonObj = new HazelcastJsonValue(jsonString);
                    marksDictionary.Add(mark.Id, jsonObj);
                }

                // Create a sorted index on the __key attribute
                await map.AddIndexAsync(IndexType.Sorted, "__key").ConfigureAwait(false);

                // Create a sorted index on the MarkValue attribute
                await map.AddIndexAsync(IndexType.Sorted, "MarkValue").ConfigureAwait(false);

                #region bitmap indexes are not supported by Hazelcast SQL
                //// Create a bitmap index on the StudentId attribute
                //await map.AddIndexAsync(IndexType.Bitmap, "StudentId").ConfigureAwait(false);

                //// Create a bitmap index on the SubjectId attribute
                //await map.AddIndexAsync(IndexType.Bitmap, "SubjectId").ConfigureAwait(false);

                //// Create a bitmap index on the ExamId attribute
                //await map.AddIndexAsync(IndexType.Bitmap, "ExamId").ConfigureAwait(false);
                #endregion

                // Create a hash index on the StudentId attribute
                await map.AddIndexAsync(IndexType.Hashed, "StudentId").ConfigureAwait(false);

                // Create a hash index on the SubjectId attribute
                await map.AddIndexAsync(IndexType.Hashed, "SubjectId").ConfigureAwait(false);

                // Create a hash index on the ExamId attribute
                await map.AddIndexAsync(IndexType.Hashed, "ExamId").ConfigureAwait(false);

                await client.Sql.ExecuteCommandAsync($@"
CREATE OR REPLACE MAPPING 
{map.Name} (
__key BIGINT,
Id BIGINT,
MarkValue DOUBLE,
StudentId BIGINT,
SubjectId BIGINT,
ExamId BIGINT,
CreatedAt TIMESTAMP WITH TIME ZONE,
ModifiedAt TIMESTAMP WITH TIME ZONE)
TYPE IMap OPTIONS ('keyFormat'='bigint', 'valueFormat'='json-flat')", cancellationToken: token).ConfigureAwait(false);

                await map.SetAllAsync(marksDictionary).ConfigureAwait(false);
                int count = await map.GetSizeAsync().ConfigureAwait(false);

                await map.DisposeAsync().ConfigureAwait(false);
                await client.DisposeAsync().ConfigureAwait(false);

                return count;
            }
            catch (Exception)
            {

                throw;
            }
        }

SQL join query:

public async Task<IEnumerable<StudentExamMarksDto>> LoadStudentsWithHighestMarksAsync(int numberOfStudents = 1, CancellationToken token = default)
        {
            try
            {
                var studentExamMarksDtoList = new List<StudentExamMarksDto>();

                await using var client = await HazelcastClientFactory.StartNewClientAsync(_options, cancellationToken: token).ConfigureAwait(false);

                await using var result = await client.Sql.ExecuteQueryAsync($@"
SELECT 
    s.Name
    ,s.RollNumber
    ,CAST(m.ExamCount AS INT) AS ExamCount
    ,ROUND(m.TotalMarks, 2) AS TotalMarks
FROM (
  SELECT 
    StudentId
    ,COUNT(DISTINCT ExamId) AS ExamCount
    ,SUM(MarkValue) AS TotalMarks
  FROM marks
  GROUP BY StudentId
) m
JOIN students s ON s.__key = m.StudentId
JOIN (
  SELECT 
    MIN(ExamCount) AS MinExamCount
  FROM (
    SELECT 
        COUNT(DISTINCT ExamId) AS ExamCount
    FROM marks
    GROUP BY StudentId
  ) subq
) mec ON m.ExamCount = mec.MinExamCount
ORDER BY m.TotalMarks DESC
LIMIT ?", cancellationToken: token, parameters: numberOfStudents);

                studentExamMarksDtoList = await result.Select(row =>
                    new StudentExamMarksDto
                    {
                        Name = row.GetColumn<string>("Name"),
                        RollNumber = row.GetColumn<string>("RollNumber"),
                        ExamCount = row.GetColumn<int>("ExamCount"),
                        TotalMarks = row.GetColumn<double>("TotalMarks"),
                    }).ToListAsync(token).ConfigureAwait(false);

                return studentExamMarksDtoList;
            }
            catch (Exception)
            {

                throw;
            }
        }

Is it in-memory or do I have to do something else to achieve it? Every bit of help would be appreciated.

I did not try out your code but the approach seems correct. Here are the steps .1 Create a mapping to an existing IMap 2. Execute an SQL statement. The SQL is Hazelcast SQL dialect. The results are returned as rows. Then iterate over the rows and fetch the result. Everything is executed in memory. I hope this helps — Orçun Çolak
– Orçun Çolak, Commented Aug 22, 2023 at 6:10
@OrçunÇolak I followed the same steps and tested with the same data, memory, and GPUs against the MSSQL server yielding slower results. — Ebnul Mahmood ShovOn
– Ebnul Mahmood ShovOn, Commented Aug 22, 2023 at 9:35

Collectives™ on Stack Overflow

how to implement hazelcast in-memory distributed cache using dotnet core 6

0

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

0

Know someone who can answer? Share a link to this question via email, Twitter, or Facebook.

Your Answer

Sign up or log in

Post as a guest