Attach File to Document Set via REST

Below Code will Create DocumentSet "Docset1" inside the folder "Folder1" and then it will upload the document to docset.

$( document ).ready(function() {

var webUrl = _spPageContextInfo.webAbsoluteUrl
CreateDocumentSet(webUrl, "TestLibrary", "Folder1", "Docset1", "0x0120D520").then(function(res){
    var docSetitemId = res.ID;
    var serverRelativeUrlTodocset = TestLibrary + "/" + Folder1 + "/" + Docset1;
    var filename = "<FilePath>"
    UploadDocumentToDocSet(webUrl, serverRelativeUrlTodocset, filename)
});

});

function UploadDocumentToDocSet(webUrl, serverRelativeUrlTodocset, filename){

getFileBuffer(file).then(function (buffer) {
    var bytes = new Uint8Array(buffer);
    var content = new SP.Base64EncodedByteArray(); 

    var fileCollectionEndpoint = String.format(
            "{0}/_api/web/getfolderbyserverrelativeurl('{1}')/files" +
            "/add(overwrite=true, url='{2}')",
            _spPageContextInfo.webAbsoluteUrl, serverRelativeUrlTodocset, fileName);

    $.ajax({
        url: fileCollectionEndpoint,
        type: "POST",
        processData: false,
        contentType: "application/json;odata=verbose",
        data: buffer,
        headers: {
            "accept": "application/json;odata=verbose",
            "X-RequestDigest": FormDigest,
            "content-length": buffer.byteLength
        },
        success: function (data) {
            alert('Doccument has been uploaded to Doc Set');
        },
        error: function (jqXHR, textStatus, errorThrown) {
            console.log(JSON.stringify(jqXHR));
        }
    });
});

}

function getFileBuffer(file) {

var deferred = $.Deferred();
var reader = new FileReader();
reader.onload = function (e) {
    deferred.resolve(e.target.result);
}

reader.onerror = function (e) {
    deferred.reject(e.target.error);
}

reader.readAsArrayBuffer(file);

return deferred.promise();
}

function CreateDocumentSet(webUrl, listTitle, folderName, docSetName, docSetContentTypeID) {

var ctx = new SP.ClientContext(webUrl);
var web = ctx.get_web();
var list = web.get_lists().getByTitle(listTitle);
ctx.load(list);

var parentFolder = list.get_rootFolder();
ctx.load(parentFolder);

var deff = $q.defer();
ctx.executeQueryAsync(function () {
    var folderURL = parentFolder.get_serverRelativeUrl() + '/' + folderName;

    var itemCreateInfo = new SP.ListItemCreationInformation();
    itemCreateInfo.set_underlyingObjectType(SP.FileSystemObjectType.folder);
    itemCreateInfo.set_folderUrl(folderURL);
    itemCreateInfo.set_leafName(docSetName);

    var item = list.addItem(itemCreateInfo);
    item.set_item("ContentTypeId", docSetContentTypeID);
    item.set_item('Title', docSetName);

    item.update();
    ctx.load(item);
    ctx.executeQueryAsync(
        Function.createDelegate(ctx,
            function () {
                deff.resolve(item.get_fieldValues());
            }),
        Function.createDelegate(ctx,
            function (sender, args) {
                deff.reject(args);
            }));
});
return deff.promise;
}

I have added working code in Github to Upload document inside a Folder. In the above code i have just added code to create the documentset inside the folder.