JavaScript 查找文章中出现频率最高的单词的多种方法

function findMostFrequentWord(text) {
  // 1. 将文本转换为小写并分割成单词数组
  const words = text.toLowerCase().match(/\b\w+\b/g) || [];
  
  // 2. 创建单词频率统计对象
  const frequency = {};
  
  // 3. 统计每个单词出现的次数
  words.forEach(word => {
    frequency[word] = (frequency[word] || 0) + 1;
  });
  
  // 4. 找出出现频率最高的单词
  let maxCount = 0;
  let mostFrequentWord = '';
  
  for (const word in frequency) {
    if (frequency[word] > maxCount) {
      maxCount = frequency[word];
      mostFrequentWord = word;
    }
  }
  
  return {
    word: mostFrequentWord,
    count: maxCount,
    frequency: frequency // 可选：返回完整的频率统计对象
  };
}

// 测试用例
const article = `JavaScript is a programming language that conforms to the ECMAScript specification. 
JavaScript is high-level, often just-in-time compiled, and multi-paradigm. It has curly-bracket syntax, 
dynamic typing, prototype-based object-orientation, and first-class functions. JavaScript is one of 
the core technologies of the World Wide Web. Over 97% of websites use it client-side for web page 
behavior, often incorporating third-party libraries. All major web browsers have a dedicated 
JavaScript engine to execute the code on the user's device.`;

const result = findMostFrequentWord(article);
console.log(`最常见的单词是 "${result.word}", 出现了 ${result.count} 次`);

function findMostFrequentWordAdvanced(text, customStopWords = []) {
  // 常见英文停用词列表
  const defaultStopWords = ['a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'of', 'to', 'in', 'it', 'that', 'on', 'for', 'as', 'with', 'by', 'at'];
  const stopWords = [...defaultStopWords, ...customStopWords];
  
  const words = text.toLowerCase().match(/\b\w+\b/g) || [];
  
  const frequency = {};
  
  words.forEach(word => {
    // 过滤停用词
    if (!stopWords.includes(word)) {
      frequency[word] = (frequency[word] || 0) + 1;
    }
  });
  
  let maxCount = 0;
  let mostFrequentWord = '';
  
  for (const word in frequency) {
    if (frequency[word] > maxCount) {
      maxCount = frequency[word];
      mostFrequentWord = word;
    }
  }
  
  return {
    word: mostFrequentWord,
    count: maxCount,
    frequency: frequency
  };
}

// 测试
const resultAdvanced = findMostFrequentWordAdvanced(article);
console.log(`过滤停用词后最常见的单词是 "${resultAdvanced.word}", 出现了 ${resultAdvanced.count} 次`);

function findMostFrequentWords(text, topN = 1, customStopWords = []) {
  const defaultStopWords = ['a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'of', 'to', 'in', 'it', 'that', 'on', 'for', 'as', 'with', 'by', 'at'];
  const stopWords = [...defaultStopWords, ...customStopWords];
  
  const words = text.toLowerCase().match(/\b\w+\b/g) || [];
  
  const frequency = {};
  
  words.forEach(word => {
    if (!stopWords.includes(word)) {
      frequency[word] = (frequency[word] || 0) + 1;
    }
  });
  
  // 将频率对象转换为数组并排序
  const sortedWords = Object.entries(frequency)
    .sort((a, b) => b[1] - a[1]);
  
  // 获取前N个高频单词
  const topWords = sortedWords.slice(0, topN);
  
  // 检查是否有并列情况
  const maxCount = topWords[0][1];
  const allTopWords = sortedWords.filter(word => word[1] === maxCount);
  
  return {
    topWords: topWords.map(([word, count]) => ({ word, count })),
    allTopWords: allTopWords.map(([word, count]) => ({ word, count })),
    frequency: frequency
  };
}

// 测试
const resultMulti = findMostFrequentWords(article, 5);
console.log("前5个高频单词:", resultMulti.topWords);
console.log("所有并列最高频单词:", resultMulti.allTopWords);

function findMostFrequentWordOptimized(text) {
  const words = text.toLowerCase().match(/\b\w+\b/g) || [];
  
  // 使用Map存储频率
  const frequency = new Map();
  
  words.forEach(word => {
    frequency.set(word, (frequency.get(word) || 0) + 1);
  });
  
  let maxCount = 0;
  let mostFrequentWord = '';
  
  // 遍历Map找出最高频单词
  for (const [word, count] of frequency) {
    if (count > maxCount) {
      maxCount = count;
      mostFrequentWord = word;
    }
  }
  
  return {
    word: mostFrequentWord,
    count: maxCount,
    frequency: Object.fromEntries(frequency) // 转换为普通对象方便查看
  };
}

// 测试大数据量
const largeText = new Array(10000).fill(article).join(' ');
console.time('优化版本');
const resultOptimized = findMostFrequentWordOptimized(largeText);
console.timeEnd('优化版本');
console.log(resultOptimized);

function findMostFrequentWordWithReduce(text) {
  const words = text.toLowerCase().match(/\b\w+\b/g) || [];
  
  const frequency = words.reduce((acc, word) => {
    acc[word] = (acc[word] || 0) + 1;
    return acc;
  }, {});
  
  const [mostFrequentWord, maxCount] = Object.entries(frequency)
    .reduce((max, current) => current[1] > max[1] ? current : max, ['', 0]);
  
  return {
    word: mostFrequentWord,
    count: maxCount
  };
}

function findMostFrequentWordUnicode(text) {
  // 使用Unicode属性转义匹配单词
  const words = text.toLowerCase().match(/\p{L}+/gu) || [];
  
  const frequency = {};
  
  words.forEach(word => {
    frequency[word] = (frequency[word] || 0) + 1;
  });
  
  const [mostFrequentWord, maxCount] = Object.entries(frequency)
    .reduce((max, current) => current[1] > max[1] ? current : max, ['', 0]);
  
  return {
    word: mostFrequentWord,
    count: maxCount
  };
}

// 测试多语言文本
const multiLanguageText = "JavaScript是一种编程语言，JavaScript很流行。编程语言有很多种。";
const resultUnicode = findMostFrequentWordUnicode(multiLanguageText);
console.log(resultUnicode); // { word: "javascript", count: 2 }

// 简单的词干提取函数（实际应用中使用专业库如natural或stemmer更好）
function simpleStemmer(word) {
  // 基本规则：去除常见的复数形式和-ing/-ed结尾
  return word
    .replace(/(ies)$/, 'y')
    .replace(/(es)$/, '')
    .replace(/(s)$/, '')
    .replace(/(ing)$/, '')
    .replace(/(ed)$/, '');
}

function findMostFrequentWordWithStemming(text) {
  const words = text.toLowerCase().match(/\b\w+\b/g) || [];
  
  const frequency = {};
  
  words.forEach(word => {
    const stemmedWord = simpleStemmer(word);
    frequency[stemmedWord] = (frequency[stemmedWord] || 0) + 1;
  });
  
  const [mostFrequentWord, maxCount] = Object.entries(frequency)
    .reduce((max, current) => current[1] > max[1] ? current : max, ['', 0]);
  
  return {
    word: mostFrequentWord,
    count: maxCount,
    originalWord: Object.entries(frequency)
      .find(([w]) => simpleStemmer(w) === mostFrequentWord)[0]
  };
}

// 测试
const textWithDifferentForms = "I love running. He loves to run. They loved the runner.";
const resultStemmed = findMostFrequentWordWithStemming(textWithDifferentForms);
console.log(resultStemmed); // { word: "love", count: 3, originalWord: "love" }

class WordFrequencyAnalyzer {
  constructor(options = {}) {
    // 默认停用词列表
    this.defaultStopWords = [
      'a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
      'to', 'of', 'in', 'on', 'at', 'for', 'with', 'by', 'as', 'from', 'that', 'this', 'these',
      'those', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'should', 'can', 'could',
      'about', 'above', 'after', 'before', 'between', 'into', 'through', 'during', 'over', 'under'
    ];
    
    // 合并自定义停用词
    this.stopWords = [...this.defaultStopWords, ...(options.stopWords || [])];
    
    // 是否启用词干提取
    this.enableStemming = options.enableStemming || false;
    
    // 是否区分大小写
    this.caseSensitive = options.caseSensitive || false;
  }
  
  // 简单的词干提取函数
  stemWord(word) {
    if (!this.enableStemming) return word;
    
    return word
      .replace(/(ies)$/, 'y')
      .replace(/(es)$/, '')
      .replace(/(s)$/, '')
      .replace(/(ing)$/, '')
      .replace(/(ed)$/, '');
  }
  
  // 分析文本并返回单词频率
  analyze(text, topN = 10) {
    // 预处理文本
    const processedText = this.caseSensitive ? text : text.toLowerCase();
    
    // 匹配单词（支持Unicode）
    const words = processedText.match(/[\p{L}']+/gu) || [];
    
    const frequency = new Map();
    
    // 统计频率
    words.forEach(word => {
      // 处理撇号（如 don't → dont）
      const cleanedWord = word.replace(/'/g, '');
      
      // 词干提取
      const stemmedWord = this.stemWord(cleanedWord);
      
      // 过滤停用词
      if (!this.stopWords.includes(cleanedWord) && 
          !this.stopWords.includes(stemmedWord)) {
        frequency.set(stemmedWord, (frequency.get(stemmedWord) || 0) + 1);
      }
    });
    
    // 转换为数组并排序
    const sortedWords = Array.from(frequency.entries())
      .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
    
    // 获取前N个单词
    const topWords = sortedWords.slice(0, topN);
    
    // 获取最高频单词及其计数
    const maxCount = topWords[0]?.[1] || 0;
    const allTopWords = sortedWords.filter(([, count]) => count === maxCount);
    
    return {
      topWords: topWords.map(([word, count]) => ({ word, count })),
      allTopWords: allTopWords.map(([word, count]) => ({ word, count })),
      frequency: Object.fromEntries(frequency)
    };
  }
}

// 使用示例
const analyzer = new WordFrequencyAnalyzer({
  stopWords: ['javascript', 'language'], // 添加自定义停用词
  enableStemming: true
});

const analysisResult = analyzer.analyze(article, 5);
console.log("分析结果:", analysisResult.topWords);