/* global module require process */ /* eslint no-undef: "error" */ /* eslint no-console: "off" */ 'use strict'; const requiredNodeVersion = 11; if (parseInt((/^v(\d+)\./).exec(process.version)[1]) < requiredNodeVersion) { throw Error(`requires at least node: ${requiredNodeVersion}`); } module.exports = function(settings) { // wrapper in case we're in module_context mode const hackyProcessSelectFiles = settings.filenames !== undefined; const cache = new (require('inmemfilecache'))(); const Feed = require('feed').Feed; const fs = require('fs'); const glob = require('glob'); const Handlebars = require('handlebars'); const hanson = require('hanson'); const marked = require('marked'); const path = require('path'); const Promise = require('promise'); const sitemap = require('sitemap'); const utils = require('./utils'); const moment = require('moment'); const url = require('url'); //process.title = 'build'; let numErrors = 0; function error(...args) { ++numErrors; console.error(...args); } const executeP = Promise.denodeify(utils.execute); marked.setOptions({ rawHtml: true, //pedantic: true, }); function applyObject(src, dst) { Object.keys(src).forEach(function(key) { dst[key] = src[key]; }); return dst; } function mergeObjects() { const merged = {}; Array.prototype.slice.call(arguments).forEach(function(src) { applyObject(src, merged); }); return merged; } function readFile(fileName) { return cache.readFileSync(fileName, 'utf-8'); } function writeFileIfChanged(fileName, content) { if (fs.existsSync(fileName)) { const old = readFile(fileName); if (content === old) { return; } } fs.writeFileSync(fileName, content); console.log('Wrote: ' + fileName); // eslint-disable-line } function copyFile(src, dst) { writeFileIfChanged(dst, readFile(src)); } function replaceParams(str, params) { const template = Handlebars.compile(str); if (Array.isArray(params)) { params = mergeObjects.apply(null, params.slice().reverse()); } return template(params); } function encodeParams(params) { const values = Object.values(params).filter(v => v); if (!values.length) { return ''; } return '&' + Object.entries(params).map((kv) => { return `${encodeURIComponent(kv[0])}=${encodeURIComponent(kv[1])}`; }).join('&'); } function encodeQuery(query) { if (!query) { return ''; } return '?' + query.split('&').map(function(pair) { return pair.split('=').map(function(kv) { return encodeURIComponent(decodeURIComponent(kv)); }).join('='); }).join('&'); } function encodeUrl(src) { const u = url.parse(src); u.search = encodeQuery(u.query); return url.format(u); } function TemplateManager() { const templates = {}; this.apply = function(filename, params) { let template = templates[filename]; if (!template) { template = Handlebars.compile(readFile(filename)); templates[filename] = template; } if (Array.isArray(params)) { params = mergeObjects.apply(null, params.slice().reverse()); } return template(params); }; } const templateManager = new TemplateManager(); Handlebars.registerHelper('include', function(filename, options) { let context; if (options && options.hash && options.hash.filename) { const varName = options.hash.filename; filename = options.data.root[varName]; context = options.hash; } else { context = options.data.root; } return templateManager.apply(filename, context); }); Handlebars.registerHelper('example', function(options) { options.hash.width = options.hash.width ? 'width: ' + options.hash.width + 'px;' : ''; options.hash.height = options.hash.height ? 'height: ' + options.hash.height + 'px;' : ''; options.hash.caption = options.hash.caption || options.data.root.defaultExampleCaption; options.hash.examplePath = options.data.root.examplePath; options.hash.encodedUrl = encodeURIComponent(encodeUrl(options.hash.url)); options.hash.url = encodeUrl(options.hash.url); options.hash.params = encodeParams({ startPane: options.hash.startPane, }); return templateManager.apply('build/templates/example.template', options.hash); }); Handlebars.registerHelper('diagram', function(options) { options.hash.width = options.hash.width || '400'; options.hash.height = options.hash.height || '300'; options.hash.examplePath = options.data.root.examplePath; options.hash.className = options.hash.className || ''; options.hash.url = encodeUrl(options.hash.url); return templateManager.apply('build/templates/diagram.template', options.hash); }); Handlebars.registerHelper('image', function(options) { options.hash.examplePath = options.data.root.examplePath; options.hash.className = options.hash.className || ''; options.hash.caption = options.hash.caption || undefined; if (options.hash.url.substring(0, 4) === 'http') { options.hash.examplePath = ''; } return templateManager.apply('build/templates/image.template', options.hash); }); Handlebars.registerHelper('selected', function(options) { const key = options.hash.key; const value = options.hash.value; const re = options.hash.re; const sub = options.hash.sub; const a = this[key]; let b = options.data.root[value]; if (re) { const r = new RegExp(re); b = b.replace(r, sub); } return a === b ? 'selected' : ''; }); function slashify(s) { return s.replace(/\\/g, '/'); } function articleFilter(f) { if (hackyProcessSelectFiles) { if (!settings.filenames.has(f)) { return false; } } return !process.env['ARTICLE_FILTER'] || f.indexOf(process.env['ARTICLE_FILTER']) >= 0; } const Builder = function(outBaseDir, options) { const g_articlesByLang = {}; let g_articles = []; let g_langInfo; const g_langDB = {}; const g_outBaseDir = outBaseDir; const g_origPath = options.origPath; // This are the english articles. const g_origArticles = glob. sync(path.join(g_origPath, '*.md')) .map(a => path.basename(a)) .filter(a => a !== 'index.md') .filter(articleFilter); const extractHeader = (function() { const headerRE = /([A-Z0-9_-]+): (.*?)$/i; return function(content) { const metaData = { }; const lines = content.split('\n'); for (;;) { const line = lines[0].trim(); const m = headerRE.exec(line); if (!m) { break; } metaData[m[1].toLowerCase()] = m[2]; lines.shift(); } return { content: lines.join('\n'), headers: metaData, }; }; }()); const parseMD = function(content) { return extractHeader(content); }; const loadMD = function(contentFileName) { const content = cache.readFileSync(contentFileName, 'utf-8'); return parseMD(content); }; function extractHandlebars(content) { const tripleRE = /\{\{\{.*?\}\}\}/g; const doubleRE = /\{\{\{.*?\}\}\}/g; let numExtractions = 0; const extractions = { }; function saveHandlebar(match) { const id = '==HANDLEBARS_ID_' + (++numExtractions) + '=='; extractions[id] = match; return id; } content = content.replace(tripleRE, saveHandlebar); content = content.replace(doubleRE, saveHandlebar); return { content: content, extractions: extractions, }; } function insertHandlebars(info, content) { const handlebarRE = /==HANDLEBARS_ID_\d+==/g; function restoreHandlebar(match) { const value = info.extractions[match]; if (value === undefined) { throw new Error('no match restoring handlebar for: ' + match); } return value; } content = content.replace(handlebarRE, restoreHandlebar); return content; } function isSameDomain(url, pageUrl) { const fdq1 = new URL(pageUrl); const fdq2 = new URL(url, pageUrl); return fdq1.origin === fdq2.origin; } function getUrlPath(url) { // yes, this is a hack const q = url.indexOf('?'); return q >= 0 ? url.substring(0, q) : url; } // Try top fix relative links. This *should* only // happen in translations const iframeLinkRE = /( pageUrl:', pageUrl); function fixRelLink(m, prefix, url, suffix) { if (isSameDomain(url, pageUrl)) { // a link that starts with "../" should be "../../" if it's in a translation // a link that starts with "resources" should be "../resources" if it's in a translation if (url.startsWith('../') || url.startsWith('resources')) { // console.log(' url:', url); return `${prefix}../${url}${suffix}`; } } return m; } return content .replace(imgLinkRE, fixRelLink) .replace(aLinkRE, fixRelLink) .replace(iframeLinkRE, fixRelLink); } /** * Get all the local urls based on a regex that has */ function getUrls(regex, str) { const links = new Set(); let m; do { m = regex.exec(str); if (m && m[2][0] !== '#' && isSameDomain(m[2], 'http://example.com/a/b/c/d')) { links.add(getUrlPath(m[2])); } } while (m); return links; } /** * Get all the local links in content */ function getLinks(content) { return new Set(linkREs.map(re => [...getUrls(re, content)]).flat()); } function fixUrls(regex, content, origLinks) { return content.replace(regex, (m, prefix, url, suffix) => { const q = url.indexOf('?'); const urlPath = q >= 0 ? url.substring(0, q) : url; const urlQuery = q >= 0 ? url.substring(q) : ''; if (!origLinks.has(urlPath) && isSameDomain(urlPath, 'https://foo.com/a/b/c/d.html') && !(/\/..\/^/.test(urlPath)) && // hacky test for link to main page. Example /webgl/lessons/ja/ urlPath[0] !== '#') { // test for same page anchor -- bad test :( for (const origLink of origLinks) { if (urlPath.endsWith(origLink)) { const newUrl = `${origLink}${urlQuery}`; console.log(' fixing:', url, 'to', newUrl); return `${prefix}${newUrl}${suffix}`; } } error('could not fix:', url); } return m; }); } const applyTemplateToContent = function(templatePath, contentFileName, outFileName, opt_extra, data) { // Call prep's Content which parses the HTML. This helps us find missing tags // should probably call something else. //Convert(md_content) const relativeOutName = slashify(outFileName).substring(g_outBaseDir.length); const pageUrl = `${settings.baseUrl}${relativeOutName}`; const metaData = data.headers; const content = data.content; //console.log(JSON.stringify(metaData, undefined, ' ')); const info = extractHandlebars(content); let html = marked(info.content); // HACK! :-( // There's probably a way to do this in marked html = html.replace(/
 1) {
      html = hackRelLinks(html, pageUrl);
    }
    html = insertHandlebars(info, html);
    html = replaceParams(html, [opt_extra, g_langInfo]);
    const pathRE = new RegExp(`^\\/${settings.rootFolder}\\/lessons\\/$`);
    const langs = Object.keys(g_langDB).map((name) => {
      const lang = g_langDB[name];
      const url = slashify(path.join(lang.basePath, path.basename(outFileName)))
         .replace('index.html', '')
         .replace(pathRE, '/');
      return {
        lang: lang.lang,
        language: lang.language,
        url: url,
      };
    });
    metaData['content'] = html;
    metaData['langs'] = langs;
    metaData['src_file_name'] = slashify(contentFileName);
    metaData['dst_file_name'] = relativeOutName;
    metaData['basedir'] = '';
    metaData['toc'] = opt_extra.toc;
    metaData['templateOptions'] = opt_extra.templateOptions;
    metaData['langInfo'] = g_langInfo;
    metaData['url'] = pageUrl;
    metaData['relUrl'] = relativeOutName;
    metaData['screenshot'] = `${settings.baseUrl}/${settings.rootFolder}/lessons/resources/${settings.siteThumbnail}`;
    const basename = path.basename(contentFileName, '.md');
    ['.jpg', '.png'].forEach(function(ext) {
      const filename = path.join(settings.rootFolder, 'lessons', 'screenshots', basename + ext);
      if (fs.existsSync(filename)) {
        metaData['screenshot'] = `${settings.baseUrl}/${settings.rootFolder}/lessons/screenshots/${basename}${ext}`;
      }
    });
    const output = templateManager.apply(templatePath, metaData);
    writeFileIfChanged(outFileName, output);

    return metaData;
  };

  const applyTemplateToFile = function(templatePath, contentFileName, outFileName, opt_extra) {
    console.log('processing: ', contentFileName);  // eslint-disable-line
    opt_extra = opt_extra || {};
    const data = loadMD(contentFileName);
    const metaData = applyTemplateToContent(templatePath, contentFileName, outFileName, opt_extra, data);
    g_articles.push(metaData);
  };

  const applyTemplateToFiles = function(templatePath, filesSpec, extra) {
    const files = glob
        .sync(filesSpec)
        .sort()
        .filter(articleFilter);
    files.forEach(function(fileName) {
      const ext = path.extname(fileName);
      const baseName = fileName.substr(0, fileName.length - ext.length);
      const outFileName = path.join(outBaseDir, baseName + '.html');
      applyTemplateToFile(templatePath, fileName, outFileName, extra);
    });

  };

  const addArticleByLang = function(article, lang) {
    const filename = path.basename(article.dst_file_name);
    let articleInfo = g_articlesByLang[filename];
    const url = `${settings.baseUrl}${article.dst_file_name}`;
    if (!articleInfo) {
      articleInfo = {
        url: url,
        changefreq: 'monthly',
        links: [],
      };
      g_articlesByLang[filename] = articleInfo;
    }
    articleInfo.links.push({
      url: url,
      lang: lang,
    });
  };

  const getLanguageSelection = function(lang) {
    const lessons = lang.lessons;
    const langInfo = hanson.parse(fs.readFileSync(path.join(lessons, 'langinfo.hanson'), {encoding: 'utf8'}));
    langInfo.langCode = langInfo.langCode || lang.lang;
    langInfo.home = lang.home;
    g_langDB[lang.lang] = {
      lang: lang.lang,
      language: langInfo.language,
      basePath: '/' + lessons,
      langInfo: langInfo,
    };
  };

  this.preProcess = function(langs) {
     langs.forEach(getLanguageSelection);
  };

  this.process = function(options) {
    console.log('Processing Lang: ' + options.lang);  // eslint-disable-line
    g_articles = [];
    g_langInfo = g_langDB[options.lang].langInfo;

    applyTemplateToFiles(options.template, path.join(options.lessons, settings.lessonGrep), options);

    const articlesFilenames = g_articles.map(a => path.basename(a.src_file_name));

    // should do this first was easier to add here
    if (options.lang !== 'en') {
      const existing = g_origArticles.filter(name => articlesFilenames.indexOf(name) >= 0);
      existing.forEach((name) => {
        const origMdFilename = path.join(g_origPath, name);
        const transMdFilename = path.join(g_origPath, options.lang, name);
        const origLinks = getLinks(loadMD(origMdFilename).content);
        const transLinks = getLinks(loadMD(transMdFilename).content);

        if (process.env['ARTICLE_VERBOSE']) {
          console.log('---[', transMdFilename, ']---');
          console.log('origLinks: ---\n   ', [...origLinks].join('\n    '));
          console.log('transLinks: ---\n   ', [...transLinks].join('\n    '));
        }

        let show = true;
        transLinks.forEach((link) => {
          if (!origLinks.has(link)) {
            if (show) {
              show = false;
              error('---[', transMdFilename, ']---');
            }
            error('   link:[', link, '] not found in English file');
          }
        });

        if (!show && process.env['ARTICLE_FIX']) {
          // there was an error, try to auto-fix
          let fixedMd = fs.readFileSync(transMdFilename, {encoding: 'utf8'});
          linkREs.forEach((re) => {
            fixedMd = fixUrls(re, fixedMd, origLinks);
          });
          fs.writeFileSync(transMdFilename, fixedMd);
        }
      });
    }

    if (hackyProcessSelectFiles) {
      return Promise.resolve();
    }

    // generate place holders for non-translated files
    const missing = g_origArticles.filter(name => articlesFilenames.indexOf(name) < 0);
    missing.forEach(name => {
      const ext = path.extname(name);
      const baseName = name.substr(0, name.length - ext.length);
      const outFileName = path.join(outBaseDir, options.lessons, baseName + '.html');
      const data = Object.assign({}, loadMD(path.join(g_origPath, name)));
      data.content = g_langInfo.missing;
      const extra = {
        origLink: '/' + slashify(path.join(g_origPath, baseName + '.html')),
        toc: options.toc,
      };
      console.log('  generating missing:', outFileName);  // eslint-disable-line
      applyTemplateToContent(
          'build/templates/missing.template',
          path.join(options.lessons, 'langinfo.hanson'),
          outFileName,
          extra,
          data);
    });

    function utcMomentFromGitLog(result, filename, timeType) {
      const dateStr = result.stdout.split('\n')[0].trim();
      const utcDateStr = dateStr
        .replace(/"/g, '')   // WTF to these quotes come from!??!
        .replace(' ', 'T')
        .replace(' ', '')
        .replace(/(\d\d)$/, ':$1');
      const m = moment.utc(utcDateStr);
      if (m.isValid()) {
        return m;
      }
      const stat = fs.statSync(filename);
      return moment(stat[timeType]);
    }

    const tasks = g_articles.map((article) => {
      return function() {
        return executeP('git', [
          'log',
          '--format="%ci"',
          '--name-only',
          '--diff-filter=A',
          article.src_file_name,
        ]).then((result) => {
          article.dateAdded = utcMomentFromGitLog(result, article.src_file_name, 'ctime');
        });
      };
    }).concat(g_articles.map((article) => {
       return function() {
         return executeP('git', [
           'log',
           '--format="%ci"',
           '--name-only',
           '--max-count=1',
           article.src_file_name,
         ]).then((result) => {
           article.dateModified = utcMomentFromGitLog(result, article.src_file_name, 'mtime');
         });
       };
    }));

    return tasks.reduce(function(cur, next){
        return cur.then(next);
    }, Promise.resolve()).then(function() {
      let articles = g_articles.filter(function(article) {
        return article.dateAdded !== undefined;
      });
      articles = articles.sort(function(a, b) {
        return b.dateAdded - a.dateAdded;
      });

      if (articles.length) {
        const feed = new Feed({
          title:          g_langInfo.title,
          description:    g_langInfo.description,
          link:           g_langInfo.link,
          image:          `${settings.baseUrl}/${settings.rootFolder}/lessons/resources/${settings.siteThumbnail}`,
          date:           articles[0].dateModified.toDate(),
          published:      articles[0].dateModified.toDate(),
          updated:        articles[0].dateModified.toDate(),
          author: {
            name:       `${settings.siteName} Contributors`,
            link:       `${settings.baseUrl}/contributors.html`,
          },
        });

        articles.forEach(function(article) {
          feed.addItem({
            title:          article.title,
            link:           `${settings.baseUrl}${article.dst_file_name}`,
            description:    '',
            author: [
              {
                name:       `${settings.siteName} Contributors`,
                link:       `${settings.baseUrl}/contributors.html`,
              },
            ],
            // contributor: [
            // ],
            date:           article.dateModified.toDate(),
            published:      article.dateAdded.toDate(),
            // image:          posts[key].image
          });

          addArticleByLang(article, options.lang);
        });

        try {
          const outPath = path.join(g_outBaseDir, options.lessons, 'atom.xml');
          console.log('write:', outPath);  // eslint-disable-line
          writeFileIfChanged(outPath, feed.atom1());
        } catch (err) {
          return Promise.reject(err);
        }
      } else {
        console.log('no articles!');  // eslint-disable-line
      }

      return Promise.resolve();
    }).then(function() {
      // this used to insert a table of contents
      // but it was useless being auto-generated
      applyTemplateToFile('build/templates/index.template', path.join(options.lessons, 'index.md'), path.join(g_outBaseDir, options.lessons, 'index.html'), {
        table_of_contents: '',
        templateOptions: g_langInfo,
      });
      return Promise.resolve();
    }, function(err) {
      error('ERROR!:');
      error(err);
      if (err.stack) {
        error(err.stack);  // eslint-disable-line
      }
      throw new Error(err.toString());
    });
  };

  this.writeGlobalFiles = function() {
    const sm = sitemap.createSitemap({
      hostname: settings.baseUrl,
      cacheTime: 600000,
    });
    const articleLangs = { };
    Object.keys(g_articlesByLang).forEach(function(filename) {
      const article = g_articlesByLang[filename];
      const langs = {};
      article.links.forEach(function(link) {
        langs[link.lang] = true;
      });
      articleLangs[filename] = langs;
      sm.add(article);
    });
    // var langInfo = {
    //   articles: articleLangs,
    //   langs: g_langDB,
    // };
    // var langJS = 'window.langDB = ' + JSON.stringify(langInfo, null, 2);
    // writeFileIfChanged(path.join(g_outBaseDir, 'langdb.js'), langJS);
    writeFileIfChanged(path.join(g_outBaseDir, 'sitemap.xml'), sm.toString());
    copyFile(path.join(g_outBaseDir, `${settings.rootFolder}/lessons/atom.xml`), path.join(g_outBaseDir, 'atom.xml'));
    copyFile(path.join(g_outBaseDir, `${settings.rootFolder}/lessons/index.html`), path.join(g_outBaseDir, 'index.html'));

    applyTemplateToFile('build/templates/index.template', 'contributors.md', path.join(g_outBaseDir, 'contributors.html'), {
      table_of_contents: '',
      templateOptions: '',
    });

    {
      const filename = path.join(settings.outDir, 'link-check.html');
      const html = `
      
      
      ${langs.map(lang => `${lang.lang}`).join('\n')}
      
      
      `;
      writeFileIfChanged(filename, html);
    }
  };


};

const b = new Builder(settings.outDir, {
  origPath: `${settings.rootFolder}/lessons`,  // english articles
});

const readdirs = function(dirpath) {
  const dirsOnly = function(filename) {
    const stat = fs.statSync(filename);
    return stat.isDirectory();
  };

  const addPath = function(filename) {
    return path.join(dirpath, filename);
  };

  return fs.readdirSync(`${settings.rootFolder}/lessons`)
      .map(addPath)
      .filter(dirsOnly);
};

const isLangFolder = function(dirname) {
  const filename = path.join(dirname, 'langinfo.hanson');
  return fs.existsSync(filename);
};


const pathToLang = function(filename) {
  const lang = path.basename(filename);
  const lessonBase = `${settings.rootFolder}/lessons`;
  const lessons = `${lessonBase}/${lang}`;
  return {
    lang,
    toc: `${settings.rootFolder}/lessons/${lang}/toc.html`,
    lessons: `${lessonBase}/${lang}`,
    template: 'build/templates/lesson.template',
    examplePath: `/${lessonBase}/`,
    home: `/${lessons}/`,
  };
};

let langs = [
  // English is special (sorry it's where I started)
  {
    template: 'build/templates/lesson.template',
    lessons: `${settings.rootFolder}/lessons`,
    lang: 'en',
    toc: `${settings.rootFolder}/lessons/toc.html`,
    examplePath: `/${settings.rootFolder}/lessons/`,
    home: '/',
  },
];

langs = langs.concat(readdirs(`${settings.rootFolder}/lessons`)
    .filter(isLangFolder)
    .map(pathToLang));

b.preProcess(langs);

if (hackyProcessSelectFiles) {
  const langsInFilenames = new Set();
  [...settings.filenames].forEach((filename) => {
    const m = /lessons\/(\w{2}|\w{5})\//.exec(filename);
    const lang = m ? m[1] : 'en';
    langsInFilenames.add(lang);
  });
  langs = langs.filter(lang => langsInFilenames.has(lang.lang));
}

const tasks = langs.map(function(lang) {
  return function() {
    return b.process(lang);
  };
});

return tasks.reduce(function(cur, next) {
  return cur.then(next);
}, Promise.resolve()).then(function() {
  if (!hackyProcessSelectFiles) {
    b.writeGlobalFiles(langs);
  }
  return numErrors ? Promise.reject(new Error(`${numErrors} errors`)) : Promise.resolve();
}).finally(() => {
  cache.clear();
});

};