Creating a SharePoint Wiki Page via JavaScript and REST or CSOM

You did not mention what kind of error occured, but most probably it is related with filename parameter. AddTemplateFile endpoint accepts relative file Url for urlOfFile parameter.

How to create a Wiki page via SharePoint REST API

The following example demonstrates how to create Wiki page via SharePoint REST API. Since AddTemplateFile endpoint accepts relative file Url for urlOfFile parameter, the helper function getListUrl is used to resolve List Url by its title

function createWikiPage(webUrl,listTitle,fileName,success, failure)
{

  getListUrl(webUrl,listTitle,
    function(listUrl){  

     var fileUrl = listUrl + '/' + fileName
     var url = webUrl + "/_api/web/GetFolderByServerRelativeUrl('" + listUrl + "')/Files" +
               "/AddTemplateFile(urlOfFile='" + fileUrl + "',templateFileType=1)";
     $.ajax({
        url: url,
        method: "POST",
        headers: {
               "accept": "application/json;odata=verbose",
               "content-type": "application/json;odata=verbose",
               "X-RequestDigest" : $("#__REQUESTDIGEST").val()
        },
        success: function (data) {
            success(data.d);
        },
        error: function (data) {
            failure(data);
        }
     });

    },
    failure
  );


}


function getListUrl(webUrl,listTitle,success, failure)
{
    var url = webUrl + "/_api/web/lists/GetByTitle('" + listTitle +  "')/RootFolder"; 
    $.ajax({
        url: url,
        method: "GET",
        headers: {
               "accept": "application/json;odata=verbose",
               "content-type": "application/json;odata=verbose"
        },
        success: function (data) {
            success(data.d.ServerRelativeUrl);
        },
        error: function (data) {
            failure(data);
        }
    });
}


//Usage
createWikiPage(_spPageContextInfo.webAbsoluteUrl,'Pages','WikiTestPage.aspx',
  function(page){  
    console.log(JSON.stringify(page));
  },
  function(error){
    console.log(JSON.stringify(error));
  }
);

Note: The request executor is not utilized in this example since it is assumed that site URL is in the same domain as the app,but if needed it could be transformed using request executor.

Try to encode every Strings value you are sending to the WebApp using encodeURIComponent() function, as SharePoint is very sensitive on the special characters passed in URL.

Basically, this function replace special chars like ':', '/' or ' ' by it's corresponding hexadecimal value. For instance, the whitespace char will be replaced by "%20" value.